fluent-plugin-map 0.1.3 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +7 -4
- data/Appraisals +0 -8
- data/README.markdown +8 -0
- data/fluent-plugin-map.gemspec +2 -2
- data/lib/fluent/plugin/filter_map.rb +5 -5
- data/lib/fluent/plugin/map_config_param.rb +12 -10
- data/lib/fluent/plugin/map_support.rb +71 -69
- data/lib/fluent/plugin/out_map.rb +9 -17
- data/test/helper.rb +1 -1
- data/test/plugin/test_filter_map.rb +46 -50
- data/test/plugin/test_out_map.rb +79 -79
- metadata +9 -5
- data/gemfiles/fluentd_0_10.gemfile +0 -7
- data/gemfiles/fluentd_0_12.gemfile +0 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 246617171836e1abe17ac43a1ded20f5b1aad8bc
|
4
|
+
data.tar.gz: '05190f0f3356ed013af121d2e93c9511556b3684'
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: bc9b78138641d44a4a4d45ba6c101b64163a2749cd8e0328cec4e78117e1038e515dbfa87734a689e769c9ab769c5a4da67847916547354a23d459450a8ddb23
|
7
|
+
data.tar.gz: b7ecca39e249641bfc25cfaee8b106ee5087773087d0d5f984b9419a7aa98941bdf134c5c5e8a2226432e545a61c4dc0cbfa8d50ab359a1e61f73e89115a272f
|
data/.travis.yml
CHANGED
@@ -1,9 +1,12 @@
|
|
1
1
|
language: ruby
|
2
|
+
before_install:
|
3
|
+
- gem update bundler
|
4
|
+
|
2
5
|
rvm:
|
3
6
|
- 2.1
|
4
7
|
- 2.2
|
5
|
-
|
6
|
-
-
|
8
|
+
- 2.3.3
|
9
|
+
- 2.4.0
|
7
10
|
gemfile:
|
8
|
-
- gemfiles/
|
9
|
-
-
|
11
|
+
- gemfiles/fluentd_0_14.gemfile
|
12
|
+
- Gemfile
|
data/Appraisals
CHANGED
data/README.markdown
CHANGED
@@ -4,6 +4,14 @@
|
|
4
4
|
|
5
5
|
fluent-plugin-map(out\_map) is the non-buffered plugin that can convert an event log to different event log(s)
|
6
6
|
|
7
|
+
|
8
|
+
## Requirements
|
9
|
+
|
10
|
+
| fluent-plugin-map | fluentd | ruby |
|
11
|
+
|-------------------|---------|------|
|
12
|
+
| >= 0.2.0 | >= v0.14.0 | >= 2.1 |
|
13
|
+
| < 0.2.0 | >= v0.12.0 | >= 1.9 |
|
14
|
+
|
7
15
|
## MapFilter
|
8
16
|
|
9
17
|
### Example
|
data/fluent-plugin-map.gemspec
CHANGED
@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = "fluent-plugin-map"
|
6
|
-
s.version = "0.
|
6
|
+
s.version = "0.2.0"
|
7
7
|
s.authors = ["Kohei Tomita", "Hiroshi Hatake", "Kenji Okomoto"]
|
8
8
|
s.email = ["tommy.fmale@gmail.com", "cosmo0920.oucc@gmail.com", "okkez000@gmail.com"]
|
9
9
|
s.homepage = "https://github.com/fluent-plugins-nursery/fluent-plugin-map"
|
@@ -19,7 +19,7 @@ Gem::Specification.new do |s|
|
|
19
19
|
s.license = "Apache-2.0"
|
20
20
|
|
21
21
|
s.add_development_dependency "rake"
|
22
|
-
s.add_development_dependency "fluentd", ["
|
22
|
+
s.add_development_dependency "fluentd", [">= 0.14.0", "< 2"]
|
23
23
|
s.add_development_dependency "test-unit", "~> 3.1"
|
24
24
|
s.add_development_dependency "appraisal"
|
25
25
|
end
|
@@ -14,16 +14,16 @@
|
|
14
14
|
# limitations under the License.
|
15
15
|
#
|
16
16
|
|
17
|
-
require 'fluent/filter'
|
17
|
+
require 'fluent/plugin/filter'
|
18
18
|
require 'fluent/plugin/map_support'
|
19
19
|
require 'fluent/plugin/map_config_param'
|
20
20
|
require 'fluent/plugin/parse_map_mixin'
|
21
21
|
|
22
|
-
module Fluent
|
23
|
-
class MapFilter < Fluent::Filter
|
22
|
+
module Fluent::Plugin
|
23
|
+
class MapFilter < Fluent::Plugin::Filter
|
24
24
|
Fluent::Plugin.register_filter('map', self)
|
25
25
|
|
26
|
-
include Fluent::MapConfigParam
|
26
|
+
include Fluent::Plugin::MapConfigParam
|
27
27
|
include Fluent::ParseMap::Mixin
|
28
28
|
|
29
29
|
def configure(conf)
|
@@ -31,7 +31,7 @@ module Fluent
|
|
31
31
|
@format = determine_format()
|
32
32
|
configure_format()
|
33
33
|
@map = create_map(conf)
|
34
|
-
@map_support = Fluent::MapSupport.new(@map, self)
|
34
|
+
@map_support = Fluent::Plugin::MapSupport.new(@map, self)
|
35
35
|
end
|
36
36
|
|
37
37
|
def determine_format()
|
@@ -15,16 +15,18 @@
|
|
15
15
|
#
|
16
16
|
|
17
17
|
module Fluent
|
18
|
-
module
|
19
|
-
|
20
|
-
klass
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
18
|
+
module Plugin
|
19
|
+
module MapConfigParam
|
20
|
+
def self.included(klass)
|
21
|
+
klass.instance_eval {
|
22
|
+
config_param :map, :string, :default => nil
|
23
|
+
config_param :time, :string, :default => nil
|
24
|
+
config_param :record, :string, :default => nil
|
25
|
+
config_param :multi, :bool, :default => false
|
26
|
+
config_param :timeout, :time, :default => 1
|
27
|
+
config_param :format, :string, :default => nil
|
28
|
+
}
|
29
|
+
end
|
28
30
|
end
|
29
31
|
end
|
30
32
|
end
|
@@ -15,98 +15,100 @@
|
|
15
15
|
#
|
16
16
|
|
17
17
|
module Fluent
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
18
|
+
module Plugin
|
19
|
+
class MapSupport
|
20
|
+
def initialize(map, plugin)
|
21
|
+
@map = map
|
22
|
+
@plugin = plugin
|
23
|
+
if plugin.is_a?(Fluent::Plugin::Filter)
|
24
|
+
singleton_class.module_eval(<<-CODE)
|
24
25
|
def map_func(time, record)
|
25
26
|
#{@map}
|
26
27
|
end
|
27
28
|
CODE
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
29
|
+
class << self
|
30
|
+
alias_method :generate_tuples, :generate_tuples_filter
|
31
|
+
alias_method :do_map, :do_map_filter
|
32
|
+
end
|
33
|
+
elsif plugin.is_a?(Fluent::Plugin::Output)
|
34
|
+
singleton_class.module_eval(<<-CODE)
|
34
35
|
def map_func(tag, time, record)
|
35
36
|
#{@map}
|
36
37
|
end
|
37
38
|
CODE
|
38
|
-
|
39
|
-
|
40
|
-
|
39
|
+
class << self
|
40
|
+
alias_method :generate_tuples, :generate_tuples_output
|
41
|
+
alias_method :do_map, :do_map_output
|
42
|
+
end
|
41
43
|
end
|
42
44
|
end
|
43
|
-
end
|
44
45
|
|
45
|
-
|
46
|
-
|
47
|
-
|
46
|
+
def do_map(tag, es)
|
47
|
+
# This method will be overwritten in #initailize.
|
48
|
+
end
|
48
49
|
|
49
|
-
|
50
|
-
|
50
|
+
def do_map_filter(tag, es)
|
51
|
+
tuples = generate_tuples(tag, es)
|
51
52
|
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
53
|
+
tag_output_es = Hash.new{|h, key| h[key] = Fluent::MultiEventStream.new}
|
54
|
+
tuples.each do |time, record|
|
55
|
+
if time == nil || record == nil
|
56
|
+
raise SyntaxError.new
|
57
|
+
end
|
58
|
+
tag_output_es[tag].add(time, record)
|
59
|
+
@plugin.log.trace { [tag, time, record].inspect }
|
56
60
|
end
|
57
|
-
tag_output_es
|
58
|
-
@plugin.log.trace { [tag, time, record].inspect }
|
61
|
+
tag_output_es
|
59
62
|
end
|
60
|
-
tag_output_es
|
61
|
-
end
|
62
63
|
|
63
|
-
|
64
|
-
|
64
|
+
def do_map_output(tag, es)
|
65
|
+
tuples = generate_tuples(tag, es)
|
65
66
|
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
67
|
+
tag_output_es = Hash.new{|h, key| h[key] = Fluent::MultiEventStream.new}
|
68
|
+
tuples.each do |tag, time, record|
|
69
|
+
if time == nil || record == nil
|
70
|
+
raise SyntaxError.new
|
71
|
+
end
|
72
|
+
tag_output_es[tag].add(time, record)
|
73
|
+
@plugin.log.trace { [tag, time, record].inspect }
|
70
74
|
end
|
71
|
-
tag_output_es
|
72
|
-
@plugin.log.trace { [tag, time, record].inspect }
|
75
|
+
tag_output_es
|
73
76
|
end
|
74
|
-
tag_output_es
|
75
|
-
end
|
76
|
-
|
77
|
-
def generate_tuples
|
78
|
-
# This method will be overwritten in #initailize.
|
79
|
-
end
|
80
77
|
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
timeout_block do
|
85
|
-
new_tuple = map_func(time, record)
|
86
|
-
tuples.concat new_tuple
|
87
|
-
end
|
88
|
-
}
|
89
|
-
tuples
|
90
|
-
end
|
78
|
+
def generate_tuples
|
79
|
+
# This method will be overwritten in #initailize.
|
80
|
+
end
|
91
81
|
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
82
|
+
def generate_tuples_filter(tag, es)
|
83
|
+
tuples = []
|
84
|
+
es.each {|time, record|
|
85
|
+
timeout_block do
|
86
|
+
new_tuple = map_func(time, record)
|
87
|
+
tuples.concat new_tuple
|
88
|
+
end
|
89
|
+
}
|
90
|
+
tuples
|
91
|
+
end
|
102
92
|
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
93
|
+
def generate_tuples_output(tag, es)
|
94
|
+
tuples = []
|
95
|
+
es.each {|time, record|
|
96
|
+
timeout_block do
|
97
|
+
new_tuple = map_func(tag, time, record)
|
98
|
+
tuples.concat new_tuple
|
99
|
+
end
|
107
100
|
}
|
108
|
-
|
109
|
-
|
101
|
+
tuples
|
102
|
+
end
|
103
|
+
|
104
|
+
def timeout_block
|
105
|
+
begin
|
106
|
+
Timeout.timeout(@plugin.timeout){
|
107
|
+
yield
|
108
|
+
}
|
109
|
+
rescue Timeout::Error
|
110
|
+
@plugin.log.error {"Timeout: #{Time.at(time)} #{tag} #{record.inspect}"}
|
111
|
+
end
|
110
112
|
end
|
111
113
|
end
|
112
114
|
end
|
@@ -14,36 +14,30 @@
|
|
14
14
|
# limitations under the License.
|
15
15
|
#
|
16
16
|
|
17
|
-
require 'fluent/output'
|
17
|
+
require 'fluent/plugin/output'
|
18
18
|
require 'fluent/plugin/map_support'
|
19
19
|
require 'fluent/plugin/map_config_param'
|
20
20
|
require 'fluent/plugin/parse_map_mixin'
|
21
21
|
|
22
|
-
module Fluent
|
23
|
-
class MapOutput < Fluent::Output
|
22
|
+
module Fluent::Plugin
|
23
|
+
class MapOutput < Fluent::Plugin::Output
|
24
24
|
Fluent::Plugin.register_output('map', self)
|
25
25
|
|
26
|
-
|
27
|
-
include Fluent::ParseMap::Mixin
|
28
|
-
|
29
|
-
# Define `router` method of v0.12 to support v0.10 or earlier
|
30
|
-
unless method_defined?(:router)
|
31
|
-
define_method("router") { Fluent::Engine }
|
32
|
-
end
|
26
|
+
helpers :event_emitter
|
33
27
|
|
34
|
-
|
35
|
-
|
36
|
-
end
|
28
|
+
include Fluent::Plugin::MapConfigParam
|
29
|
+
include Fluent::ParseMap::Mixin
|
37
30
|
|
38
31
|
config_param :key, :string, :default => nil #deprecated
|
39
32
|
config_param :tag, :string, :default => nil
|
40
33
|
|
41
34
|
def configure(conf)
|
35
|
+
log.warn "out_map is now deprecated. It will be removed in a future release. Please consider to use filter_map."
|
42
36
|
super
|
43
37
|
@format = determine_format()
|
44
38
|
configure_format()
|
45
39
|
@map = create_map(conf)
|
46
|
-
@map_support = Fluent::MapSupport.new(@map, self)
|
40
|
+
@map_support = Fluent::Plugin::MapSupport.new(@map, self)
|
47
41
|
end
|
48
42
|
|
49
43
|
def determine_format()
|
@@ -84,16 +78,14 @@ module Fluent
|
|
84
78
|
end
|
85
79
|
end
|
86
80
|
|
87
|
-
def
|
81
|
+
def process(tag, es)
|
88
82
|
begin
|
89
83
|
tag_output_es = @map_support.do_map(tag, es)
|
90
84
|
tag_output_es.each_pair do |tag, output_es|
|
91
85
|
router.emit_stream(tag, output_es)
|
92
86
|
end
|
93
|
-
chain.next
|
94
87
|
tag_output_es
|
95
88
|
rescue SyntaxError => e
|
96
|
-
chain.next
|
97
89
|
log.error "map command is syntax error: #{@map}"
|
98
90
|
e #for test
|
99
91
|
end
|
data/test/helper.rb
CHANGED
@@ -1,23 +1,25 @@
|
|
1
1
|
require 'helper'
|
2
|
+
require 'fluent/test/driver/filter'
|
3
|
+
require 'fluent/test/helpers'
|
4
|
+
include Fluent::Test::Helpers
|
2
5
|
|
3
6
|
class MapFilterTest < Test::Unit::TestCase
|
4
7
|
def setup
|
5
|
-
omit "Use Fluentd v0.12 or later" unless defined?(Fluent::Filter)
|
6
8
|
Fluent::Test.setup
|
7
9
|
end
|
8
10
|
|
9
11
|
CONFIG = %[
|
10
|
-
map [time, record]
|
12
|
+
map ([time, record])
|
11
13
|
multi false
|
12
14
|
]
|
13
15
|
|
14
|
-
def create_driver(conf = CONFIG
|
15
|
-
Fluent::Test::
|
16
|
+
def create_driver(conf = CONFIG)
|
17
|
+
Fluent::Test::Driver::Filter.new(Fluent::Plugin::MapFilter).configure(conf)
|
16
18
|
end
|
17
19
|
|
18
20
|
def test_syntax_error
|
19
21
|
tag = "tag"
|
20
|
-
time =
|
22
|
+
time = event_time('2012-10-10 10:10:10')
|
21
23
|
record = {'code' => '300'}
|
22
24
|
|
23
25
|
#map is syntax error
|
@@ -25,20 +27,20 @@ class MapFilterTest < Test::Unit::TestCase
|
|
25
27
|
map time.
|
26
28
|
]
|
27
29
|
assert_raise SyntaxError do
|
28
|
-
create_driver(syntax_error_config
|
30
|
+
create_driver(syntax_error_config)
|
29
31
|
end
|
30
32
|
end
|
31
33
|
|
32
34
|
def test_syntax_error2
|
33
35
|
tag = "tag"
|
34
|
-
time =
|
36
|
+
time = event_time('2012-10-10 10:10:10')
|
35
37
|
record = {'code' => '300'}
|
36
38
|
|
37
39
|
#map output lligal format
|
38
40
|
syntax_error_config = %[
|
39
41
|
map time
|
40
42
|
]
|
41
|
-
d1 = create_driver(syntax_error_config
|
43
|
+
d1 = create_driver(syntax_error_config)
|
42
44
|
es = Fluent::OneEventStream.new(time, record)
|
43
45
|
e = d1.instance.filter_stream(tag, es)
|
44
46
|
assert e.kind_of?(SyntaxError)
|
@@ -46,19 +48,19 @@ class MapFilterTest < Test::Unit::TestCase
|
|
46
48
|
|
47
49
|
def test_tag_convert_using_time_record
|
48
50
|
tag = 'tag.raw'
|
49
|
-
time =
|
51
|
+
time = event_time('2012-10-10 10:10:10')
|
50
52
|
record = {'code' => '300'}
|
51
53
|
|
52
54
|
d1 = create_driver %[
|
53
55
|
time time
|
54
56
|
record record
|
55
|
-
]
|
56
|
-
d1.run do
|
57
|
-
d1.
|
57
|
+
]
|
58
|
+
d1.run(default_tag: tag) do
|
59
|
+
d1.feed(time, record)
|
58
60
|
end
|
59
|
-
filtered = d1.
|
61
|
+
filtered = d1.filtered
|
60
62
|
assert_equal 1, filtered.length
|
61
|
-
assert_equal [
|
63
|
+
assert_equal [time, record], filtered[0]
|
62
64
|
end
|
63
65
|
|
64
66
|
#deprected specification test
|
@@ -70,25 +72,23 @@ class MapFilterTest < Test::Unit::TestCase
|
|
70
72
|
assert_raise(Fluent::ConfigError){
|
71
73
|
create_driver %[
|
72
74
|
record record
|
73
|
-
]
|
75
|
+
]
|
74
76
|
}
|
75
77
|
end
|
76
78
|
|
77
79
|
def test_config_error_record
|
78
|
-
|
79
|
-
time = Time.local(2012, 10, 10, 10, 10, 0).to_i
|
80
|
+
time = event_time('2012-10-10 10:10:10')
|
80
81
|
|
81
82
|
#require record
|
82
83
|
assert_raise(Fluent::ConfigError){
|
83
84
|
create_driver %[
|
84
85
|
time time
|
85
|
-
]
|
86
|
+
]
|
86
87
|
}
|
87
88
|
end
|
88
89
|
|
89
90
|
def test_config_error_multi
|
90
|
-
|
91
|
-
time = Time.local(2012, 10, 10, 10, 10, 0).to_i
|
91
|
+
time = event_time('2012-10-10 10:10:10')
|
92
92
|
record = {'code' => '300'}
|
93
93
|
|
94
94
|
#require time
|
@@ -97,13 +97,12 @@ class MapFilterTest < Test::Unit::TestCase
|
|
97
97
|
time time
|
98
98
|
record record
|
99
99
|
multi true
|
100
|
-
]
|
100
|
+
]
|
101
101
|
}
|
102
102
|
end
|
103
103
|
|
104
104
|
def test_config_error_sleep
|
105
|
-
|
106
|
-
time = Time.local(2012, 10, 10, 10, 10, 10).to_i
|
105
|
+
time = event_time('2012-10-10 10:10:10')
|
107
106
|
record = {'code' => '300'}
|
108
107
|
|
109
108
|
assert_raise(SyntaxError) {
|
@@ -112,7 +111,7 @@ class MapFilterTest < Test::Unit::TestCase
|
|
112
111
|
time sleep 10
|
113
112
|
record record
|
114
113
|
timeout 1s
|
115
|
-
]
|
114
|
+
]
|
116
115
|
}
|
117
116
|
end
|
118
117
|
|
@@ -123,60 +122,57 @@ class MapFilterTest < Test::Unit::TestCase
|
|
123
122
|
|
124
123
|
def test_convert_format_map
|
125
124
|
tag = 'tag.raw'
|
126
|
-
time =
|
125
|
+
time = event_time('2012-10-10 10:10:10')
|
127
126
|
record = {'code1' => '300', 'code2' => '400'}
|
128
127
|
|
129
128
|
d1 = create_driver %[
|
130
129
|
format map
|
131
|
-
map [[time, record["code1"]], [time, record["code2"]]]
|
130
|
+
map ([[time, record["code1"]], [time, record["code2"]]])
|
132
131
|
multi true
|
133
|
-
]
|
134
|
-
d1.run do
|
135
|
-
d1.
|
132
|
+
]
|
133
|
+
d1.run(default_tag: tag) do
|
134
|
+
d1.feed(time, record)
|
136
135
|
end
|
137
|
-
filtered = d1.
|
136
|
+
filtered = d1.filtered
|
138
137
|
assert_equal 2, filtered.length
|
139
|
-
assert_equal
|
140
|
-
assert_equal [
|
141
|
-
assert_equal ["tag.raw", time, record["code2"]], filtered[1]
|
138
|
+
assert_equal [time, record["code1"]], filtered[0]
|
139
|
+
assert_equal [time, record["code2"]], filtered[1]
|
142
140
|
end
|
143
141
|
|
144
142
|
def test_tag_convert_format_record
|
145
143
|
tag = 'tag.raw'
|
146
|
-
time =
|
144
|
+
time = event_time('2012-10-10 10:10:10')
|
147
145
|
record = {'code' => '300'}
|
148
146
|
|
149
147
|
d1 = create_driver %[
|
150
148
|
format record
|
151
149
|
time time
|
152
150
|
record record
|
153
|
-
]
|
154
|
-
d1.run do
|
155
|
-
d1.
|
151
|
+
]
|
152
|
+
d1.run(default_tag: tag) do
|
153
|
+
d1.feed(time, record)
|
156
154
|
end
|
157
|
-
filtered = d1.
|
155
|
+
filtered = d1.filtered
|
158
156
|
assert_equal 1, filtered.length
|
159
|
-
assert_equal
|
160
|
-
assert_equal ["tag.raw", time, record], filtered[0]
|
157
|
+
assert_equal [time, record], filtered[0]
|
161
158
|
end
|
162
159
|
|
163
160
|
def test_convert_format_multimap
|
164
161
|
tag = 'tag.raw'
|
165
|
-
time =
|
162
|
+
time = event_time('2012-10-10 10:10:10')
|
166
163
|
record = {'code1' => '300', 'code2' => '400'}
|
167
164
|
|
168
165
|
d1 = create_driver %[
|
169
166
|
format multimap
|
170
|
-
mmap1 [time, record["code1"]]
|
171
|
-
mmap2 [time, record["code2"]]
|
172
|
-
]
|
173
|
-
d1.run do
|
174
|
-
d1.
|
167
|
+
mmap1 ([time, record["code1"]])
|
168
|
+
mmap2 ([time, record["code2"]])
|
169
|
+
]
|
170
|
+
d1.run(default_tag: tag) do
|
171
|
+
d1.feed(time, record)
|
175
172
|
end
|
176
|
-
filtered = d1.
|
173
|
+
filtered = d1.filtered
|
177
174
|
assert_equal 2, filtered.length
|
178
|
-
assert_equal
|
179
|
-
assert_equal [
|
180
|
-
assert_equal ["tag.raw", time, record["code2"]], filtered[1]
|
175
|
+
assert_equal [time, record["code1"]], filtered[0]
|
176
|
+
assert_equal [time, record["code2"]], filtered[1]
|
181
177
|
end
|
182
178
|
end
|
data/test/plugin/test_out_map.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
require 'helper'
|
2
|
+
require 'fluent/test/driver/output'
|
2
3
|
|
3
4
|
class MapOutputTest < Test::Unit::TestCase
|
4
5
|
def setup
|
@@ -10,47 +11,47 @@ class MapOutputTest < Test::Unit::TestCase
|
|
10
11
|
multi false
|
11
12
|
]
|
12
13
|
|
13
|
-
def create_driver(conf = CONFIG
|
14
|
-
Fluent::Test::
|
14
|
+
def create_driver(conf = CONFIG)
|
15
|
+
Fluent::Test::Driver::Output.new(Fluent::Plugin::MapOutput).configure(conf)
|
15
16
|
end
|
16
17
|
|
17
18
|
def test_tag_convert
|
18
19
|
tag = 'tag'
|
19
|
-
time =
|
20
|
+
time = event_time('2012-10-10 10:10:10')
|
20
21
|
record = {'code' => '300'}
|
21
22
|
|
22
23
|
d1 = create_driver %[
|
23
|
-
map ["newtag", time, record]
|
24
|
-
]
|
25
|
-
d1.run do
|
26
|
-
d1.
|
24
|
+
map (["newtag", time, record])
|
25
|
+
]
|
26
|
+
d1.run(default_tag: tag) do
|
27
|
+
d1.feed(time, record)
|
27
28
|
end
|
28
|
-
|
29
|
-
assert_equal 1,
|
30
|
-
assert_equal ["newtag", time, record],
|
29
|
+
events = d1.events
|
30
|
+
assert_equal 1, events.length
|
31
|
+
assert_equal ["newtag", time, record], events[0]
|
31
32
|
end
|
32
33
|
|
33
34
|
def test_convert_multi_tag
|
34
35
|
tag = 'tag'
|
35
|
-
time =
|
36
|
+
time = event_time('2012-10-10 10:10:10')
|
36
37
|
record = {'code' => '300'}
|
37
38
|
|
38
39
|
d1 = create_driver %[
|
39
|
-
map [["tag1", time, record], ["tag2", time, record]]
|
40
|
+
map ([["tag1", time, record], ["tag2", time, record]])
|
40
41
|
multi true
|
41
|
-
]
|
42
|
-
d1.run do
|
43
|
-
d1.
|
42
|
+
]
|
43
|
+
d1.run(default_tag: tag) do
|
44
|
+
d1.feed(time, record)
|
44
45
|
end
|
45
|
-
|
46
|
-
assert_equal 2,
|
47
|
-
assert_equal ["tag1", time, record],
|
48
|
-
assert_equal ["tag2", time, record],
|
46
|
+
events = d1.events
|
47
|
+
assert_equal 2, events.length
|
48
|
+
assert_equal ["tag1", time, record], events[0]
|
49
|
+
assert_equal ["tag2", time, record], events[1]
|
49
50
|
end
|
50
51
|
|
51
52
|
def test_syntax_error
|
52
53
|
tag = "tag"
|
53
|
-
time =
|
54
|
+
time = event_time('2012-10-10 10:10:10')
|
54
55
|
record = {'code' => '300'}
|
55
56
|
|
56
57
|
#map is syntax error
|
@@ -58,66 +59,65 @@ class MapOutputTest < Test::Unit::TestCase
|
|
58
59
|
map tag.
|
59
60
|
]
|
60
61
|
assert_raise SyntaxError do
|
61
|
-
|
62
|
+
create_driver(syntax_error_config)
|
62
63
|
end
|
63
64
|
end
|
64
65
|
|
65
66
|
def test_syntax_error2
|
66
67
|
tag = "tag"
|
67
|
-
time =
|
68
|
+
time = event_time('2012-10-10 10:10:10')
|
68
69
|
record = {'code' => '300'}
|
69
70
|
|
70
71
|
#map output lligal format
|
71
72
|
syntax_error_config = %[
|
72
73
|
map tag
|
73
74
|
]
|
74
|
-
d1 = create_driver(syntax_error_config
|
75
|
+
d1 = create_driver(syntax_error_config)
|
75
76
|
es = Fluent::OneEventStream.new(time, record)
|
76
|
-
|
77
|
-
e = d1.instance.emit(tag, es, chain)
|
77
|
+
e = d1.instance.process(tag, es)
|
78
78
|
assert e.kind_of?(SyntaxError)
|
79
79
|
end
|
80
80
|
|
81
81
|
def test_tag_convert_using_tag_time_record
|
82
82
|
tag = 'tag'
|
83
|
-
time =
|
83
|
+
time = event_time('2012-10-10 10:10:10')
|
84
84
|
record = {'code' => '300'}
|
85
85
|
|
86
86
|
d1 = create_driver %[
|
87
|
-
tag "newtag"
|
87
|
+
tag ("newtag")
|
88
88
|
time time
|
89
89
|
record record
|
90
|
-
]
|
91
|
-
d1.run do
|
92
|
-
d1.
|
90
|
+
]
|
91
|
+
d1.run(default_tag: tag) do
|
92
|
+
d1.feed(time, record)
|
93
93
|
end
|
94
|
-
|
95
|
-
assert_equal 1,
|
96
|
-
assert_equal ["newtag", time, record],
|
94
|
+
events = d1.events
|
95
|
+
assert_equal 1, events.length
|
96
|
+
assert_equal ["newtag", time, record], events[0]
|
97
97
|
end
|
98
98
|
|
99
99
|
#deprected specification test
|
100
100
|
def test_tag_convert_using_key_time_record
|
101
101
|
tag = 'tag'
|
102
|
-
time =
|
102
|
+
time = event_time('2012-10-10 10:10:10')
|
103
103
|
record = {'code' => '300'}
|
104
104
|
|
105
105
|
d1 = create_driver %[
|
106
|
-
key "newtag"
|
106
|
+
key ("newtag")
|
107
107
|
time time
|
108
108
|
record record
|
109
|
-
]
|
110
|
-
d1.run do
|
111
|
-
d1.
|
109
|
+
]
|
110
|
+
d1.run(default_tag: tag) do
|
111
|
+
d1.feed(time, record)
|
112
112
|
end
|
113
|
-
|
114
|
-
assert_equal 1,
|
115
|
-
assert_equal ["newtag", time, record],
|
113
|
+
events = d1.events
|
114
|
+
assert_equal 1, events.length
|
115
|
+
assert_equal ["newtag", time, record], events[0]
|
116
116
|
end
|
117
117
|
|
118
118
|
def test_config_error_tag
|
119
119
|
tag = "tag"
|
120
|
-
time =
|
120
|
+
time = event_time('2012-10-10 10:10:10')
|
121
121
|
record = {'code' => '300'}
|
122
122
|
|
123
123
|
#require time
|
@@ -125,7 +125,7 @@ class MapOutputTest < Test::Unit::TestCase
|
|
125
125
|
create_driver %[
|
126
126
|
time time
|
127
127
|
record record
|
128
|
-
]
|
128
|
+
]
|
129
129
|
}
|
130
130
|
end
|
131
131
|
|
@@ -138,7 +138,7 @@ class MapOutputTest < Test::Unit::TestCase
|
|
138
138
|
create_driver %[
|
139
139
|
tag "newtag"
|
140
140
|
record record
|
141
|
-
]
|
141
|
+
]
|
142
142
|
}
|
143
143
|
end
|
144
144
|
|
@@ -149,40 +149,40 @@ class MapOutputTest < Test::Unit::TestCase
|
|
149
149
|
#require record
|
150
150
|
assert_raise(Fluent::ConfigError){
|
151
151
|
create_driver %[
|
152
|
-
tag "newtag"
|
152
|
+
tag ("newtag")
|
153
153
|
time time
|
154
|
-
]
|
154
|
+
]
|
155
155
|
}
|
156
156
|
end
|
157
157
|
|
158
158
|
def test_config_error_multi
|
159
159
|
tag = "tag"
|
160
|
-
time =
|
160
|
+
time = event_time('2012-10-10 10:10:10')
|
161
161
|
record = {'code' => '300'}
|
162
162
|
|
163
163
|
#require time
|
164
164
|
assert_raise(Fluent::ConfigError){
|
165
165
|
create_driver %[
|
166
|
-
tag "newtag"
|
166
|
+
tag ("newtag")
|
167
167
|
time time
|
168
168
|
record record
|
169
169
|
multi true
|
170
|
-
]
|
170
|
+
]
|
171
171
|
}
|
172
172
|
end
|
173
173
|
|
174
174
|
def test_config_error_sleep
|
175
175
|
tag = 'tag'
|
176
|
-
time =
|
176
|
+
time = event_time('2012-10-10 10:10:10')
|
177
177
|
record = {'code' => '300'}
|
178
178
|
|
179
179
|
assert_raise(SyntaxError) {
|
180
180
|
create_driver %[
|
181
|
-
key "newtag"
|
181
|
+
key ("newtag")
|
182
182
|
time sleep 10
|
183
183
|
record record
|
184
184
|
timeout 1s
|
185
|
-
]
|
185
|
+
]
|
186
186
|
}
|
187
187
|
end
|
188
188
|
|
@@ -193,58 +193,58 @@ class MapOutputTest < Test::Unit::TestCase
|
|
193
193
|
|
194
194
|
def test_convert_format_map
|
195
195
|
tag = 'tag'
|
196
|
-
time =
|
196
|
+
time = event_time('2012-10-10 10:10:10')
|
197
197
|
record = {'code1' => '300', 'code2' => '400'}
|
198
198
|
|
199
199
|
d1 = create_driver %[
|
200
200
|
format map
|
201
|
-
map [["tag1", time, record["code1"]], ["tag2", time, record["code2"]]]
|
201
|
+
map ([["tag1", time, record["code1"]], ["tag2", time, record["code2"]]])
|
202
202
|
multi true
|
203
|
-
]
|
204
|
-
d1.run do
|
205
|
-
d1.
|
203
|
+
]
|
204
|
+
d1.run(default_tag: tag) do
|
205
|
+
d1.feed(time, record)
|
206
206
|
end
|
207
|
-
|
208
|
-
assert_equal 2,
|
209
|
-
assert_equal ["tag1", time, record["code1"]],
|
210
|
-
assert_equal ["tag2", time, record["code2"]],
|
207
|
+
events = d1.events
|
208
|
+
assert_equal 2, events.length
|
209
|
+
assert_equal ["tag1", time, record["code1"]], events[0]
|
210
|
+
assert_equal ["tag2", time, record["code2"]], events[1]
|
211
211
|
end
|
212
212
|
|
213
213
|
def test_tag_convert_format_record
|
214
214
|
tag = 'tag'
|
215
|
-
time =
|
215
|
+
time = event_time('2012-10-10 10:10:10')
|
216
216
|
record = {'code' => '300'}
|
217
217
|
|
218
218
|
d1 = create_driver %[
|
219
219
|
format record
|
220
|
-
tag "newtag"
|
220
|
+
tag ("newtag")
|
221
221
|
time time
|
222
222
|
record record
|
223
|
-
]
|
224
|
-
d1.run do
|
225
|
-
d1.
|
223
|
+
]
|
224
|
+
d1.run(default_tag: tag) do
|
225
|
+
d1.feed(time, record)
|
226
226
|
end
|
227
|
-
|
228
|
-
assert_equal 1,
|
229
|
-
assert_equal ["newtag", time, record],
|
227
|
+
events = d1.events
|
228
|
+
assert_equal 1, events.length
|
229
|
+
assert_equal ["newtag", time, record], events[0]
|
230
230
|
end
|
231
231
|
|
232
232
|
def test_convert_format_multimap
|
233
233
|
tag = 'tag'
|
234
|
-
time =
|
234
|
+
time = event_time('2012-10-10 10:10:10')
|
235
235
|
record = {'code1' => '300', 'code2' => '400'}
|
236
236
|
|
237
237
|
d1 = create_driver %[
|
238
238
|
format multimap
|
239
|
-
mmap1 ["tag1", time, record["code1"]]
|
240
|
-
mmap2 ["tag2", time, record["code2"]]
|
241
|
-
]
|
242
|
-
d1.run do
|
243
|
-
d1.
|
239
|
+
mmap1 (["tag1", time, record["code1"]])
|
240
|
+
mmap2 (["tag2", time, record["code2"]])
|
241
|
+
]
|
242
|
+
d1.run(default_tag: tag) do
|
243
|
+
d1.feed(time, record)
|
244
244
|
end
|
245
|
-
|
246
|
-
assert_equal 2,
|
247
|
-
assert_equal ["tag1", time, record["code1"]],
|
248
|
-
assert_equal ["tag2", time, record["code2"]],
|
245
|
+
events = d1.events
|
246
|
+
assert_equal 2, events.length
|
247
|
+
assert_equal ["tag1", time, record["code1"]], events[0]
|
248
|
+
assert_equal ["tag2", time, record["code2"]], events[1]
|
249
249
|
end
|
250
250
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-map
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Kohei Tomita
|
@@ -30,16 +30,22 @@ dependencies:
|
|
30
30
|
name: fluentd
|
31
31
|
requirement: !ruby/object:Gem::Requirement
|
32
32
|
requirements:
|
33
|
-
- - "
|
33
|
+
- - ">="
|
34
34
|
- !ruby/object:Gem::Version
|
35
35
|
version: 0.14.0
|
36
|
+
- - "<"
|
37
|
+
- !ruby/object:Gem::Version
|
38
|
+
version: '2'
|
36
39
|
type: :development
|
37
40
|
prerelease: false
|
38
41
|
version_requirements: !ruby/object:Gem::Requirement
|
39
42
|
requirements:
|
40
|
-
- - "
|
43
|
+
- - ">="
|
41
44
|
- !ruby/object:Gem::Version
|
42
45
|
version: 0.14.0
|
46
|
+
- - "<"
|
47
|
+
- !ruby/object:Gem::Version
|
48
|
+
version: '2'
|
43
49
|
- !ruby/object:Gem::Dependency
|
44
50
|
name: test-unit
|
45
51
|
requirement: !ruby/object:Gem::Requirement
|
@@ -86,8 +92,6 @@ files:
|
|
86
92
|
- README.markdown
|
87
93
|
- Rakefile
|
88
94
|
- fluent-plugin-map.gemspec
|
89
|
-
- gemfiles/fluentd_0_10.gemfile
|
90
|
-
- gemfiles/fluentd_0_12.gemfile
|
91
95
|
- gemfiles/fluentd_0_14.gemfile
|
92
96
|
- lib/fluent/plugin/filter_map.rb
|
93
97
|
- lib/fluent/plugin/map_config_param.rb
|