fluent-plugin-map 0.1.0 → 0.1.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.markdown +77 -8
- data/fluent-plugin-map.gemspec +1 -1
- data/lib/fluent/plugin/filter_map.rb +89 -0
- data/lib/fluent/plugin/map_config_param.rb +30 -0
- data/lib/fluent/plugin/map_support.rb +113 -0
- data/lib/fluent/plugin/out_map.rb +13 -87
- data/lib/fluent/plugin/parse_map_mixin.rb +59 -0
- data/test/helper.rb +1 -0
- data/test/plugin/test_filter_map.rb +182 -0
- metadata +8 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4ef5c9646ede101aef6159afc29fe2c23f71ae13
|
4
|
+
data.tar.gz: 4b85b98b5bc707817699449ae65b7a9fef64f670
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d022df78ec9b3e3b9eaf60b7c24f48f2b21e455f4e551ad147807e76bd01a78a7f7de89ea7ab1ed6e5edf5d0232789c6bdf7b9b47f0462539c4c46e910e14afa
|
7
|
+
data.tar.gz: 5927afac4efd90c7b3a5a85901db2e93328f4b2144ba6e87d30312f9988e4612108411cd726656c3120eaa6a9a21d489603e18ed9749cd28ceed3ae14279033c
|
data/README.markdown
CHANGED
@@ -4,27 +4,93 @@
|
|
4
4
|
|
5
5
|
fluent-plugin-map(out\_map) is the non-buffered plugin that can convert an event log to different event log(s)
|
6
6
|
|
7
|
-
##
|
7
|
+
## MapFilter
|
8
|
+
|
9
|
+
### Example
|
10
|
+
|
11
|
+
This sample config filter code file and time file.
|
12
|
+
|
13
|
+
<source>
|
14
|
+
@type tail
|
15
|
+
format apache
|
16
|
+
path /var/log/httpd-access.log
|
17
|
+
tag tag
|
18
|
+
@label @raw
|
19
|
+
</source>
|
20
|
+
<label @raw>
|
21
|
+
<match **>
|
22
|
+
@type copy
|
23
|
+
<store>
|
24
|
+
@type relabel
|
25
|
+
@label @code
|
26
|
+
</store>
|
27
|
+
<store>
|
28
|
+
@type relabel
|
29
|
+
@label @time
|
30
|
+
</store>
|
31
|
+
</match>
|
32
|
+
</label>
|
33
|
+
<label @code>
|
34
|
+
<filter **>
|
35
|
+
@type map
|
36
|
+
map ([time, {"code" => record["code"].to_i}])
|
37
|
+
</filter>
|
38
|
+
<match **>
|
39
|
+
@type file
|
40
|
+
path code.log
|
41
|
+
</match>
|
42
|
+
</label>
|
43
|
+
<label @time>
|
44
|
+
<filter **>
|
45
|
+
@type map
|
46
|
+
map ([time, {"time" => record["time"].to_i}])
|
47
|
+
</filter>
|
48
|
+
<match **>
|
49
|
+
@type file
|
50
|
+
path time.log
|
51
|
+
</match>
|
52
|
+
</label>
|
53
|
+
|
54
|
+
|
55
|
+
The parameter "map" can use 2 variables in event log; time, record. The format of time is an integer number of seconds since the Epoch. The format of record is hash.
|
56
|
+
The config file parses # as the begin of comment. So the "map" value cannot use #{tag} operation.
|
57
|
+
This plugin can output multi logs by seting multi to true.
|
58
|
+
|
59
|
+
If you don't use multi option, you can use time, record parameter. The 2 following filter directive is same:
|
60
|
+
|
61
|
+
<filter tag>
|
62
|
+
@type map
|
63
|
+
map ([time, {"code" => record["code"].to_i}])
|
64
|
+
</filter>
|
65
|
+
<filter tag>
|
66
|
+
@type map
|
67
|
+
time time
|
68
|
+
record ({"code" => record["code"].to_i})
|
69
|
+
</filter>
|
70
|
+
|
71
|
+
## MapOutput
|
72
|
+
|
73
|
+
### Example
|
8
74
|
|
9
75
|
This sample config output code file and time file.
|
10
76
|
|
11
77
|
<source>
|
12
|
-
type tail
|
78
|
+
@type tail
|
13
79
|
format apache
|
14
80
|
path /var/log/httpd-access.log
|
15
81
|
tag tag
|
16
82
|
</source>
|
17
83
|
<match tag>
|
18
|
-
type map
|
84
|
+
@type map
|
19
85
|
map ([["code." + tag, time, {"code" => record["code"].to_i}], ["time." + tag, time, {"time" => record["time"].to_i}]])
|
20
86
|
multi true
|
21
87
|
</match>
|
22
88
|
<match code.tag>
|
23
|
-
type file
|
89
|
+
@type file
|
24
90
|
path code.log
|
25
91
|
</match>
|
26
92
|
<match time.tag>
|
27
|
-
type file
|
93
|
+
@type file
|
28
94
|
path time.log
|
29
95
|
</match>
|
30
96
|
|
@@ -36,17 +102,20 @@ This plugin can output multi logs by seting multi to true.
|
|
36
102
|
If you don't use multi option, you can use key, time, record parameter. The 2 following match directive is same:
|
37
103
|
|
38
104
|
<match tag>
|
39
|
-
type map
|
105
|
+
@type map
|
40
106
|
map (["code." + tag, time, {"code" => record["code"].to_i}])
|
41
107
|
</match>
|
42
108
|
<match tag>
|
43
|
-
type map
|
109
|
+
@type map
|
44
110
|
tag ("code." + tag)
|
45
111
|
time time
|
46
112
|
record ({"code" => record["code"].to_i})
|
47
113
|
</match>
|
48
114
|
|
49
|
-
|
115
|
+
|
116
|
+
## Note
|
117
|
+
|
118
|
+
you have to wrap some configuration values with parenthesis like `("code." + tag)`, to avoid parsing by Fluentd itself.
|
50
119
|
See also: [Fluentd | Configuration File | Format tips](http://docs.fluentd.org/articles/config-file#format-tips)
|
51
120
|
|
52
121
|
## Copyright
|
data/fluent-plugin-map.gemspec
CHANGED
@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = "fluent-plugin-map"
|
6
|
-
s.version = "0.1.
|
6
|
+
s.version = "0.1.1"
|
7
7
|
s.authors = ["Kohei Tomita", "Hiroshi Hatake", "Kenji Okomoto"]
|
8
8
|
s.email = ["tommy.fmale@gmail.com", "cosmo0920.oucc@gmail.com", "okkez000@gmail.com"]
|
9
9
|
s.homepage = "https://github.com/fluent-plugins-nursery/fluent-plugin-map"
|
@@ -0,0 +1,89 @@
|
|
1
|
+
#
|
2
|
+
# fluent-plugin-map
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'fluent/plugin/map_support'
|
18
|
+
require 'fluent/plugin/map_config_param'
|
19
|
+
require 'fluent/plugin/parse_map_mixin'
|
20
|
+
|
21
|
+
module Fluent
|
22
|
+
class MapFilter < Fluent::Filter
|
23
|
+
Fluent::Plugin.register_filter('map', self)
|
24
|
+
|
25
|
+
include Fluent::MapConfigParam
|
26
|
+
include Fluent::ParseMap::Mixin
|
27
|
+
|
28
|
+
def configure(conf)
|
29
|
+
super
|
30
|
+
@format = determine_format()
|
31
|
+
configure_format()
|
32
|
+
@map = create_map(conf)
|
33
|
+
@map_support = Fluent::MapSupport.new(@map, self)
|
34
|
+
end
|
35
|
+
|
36
|
+
def determine_format()
|
37
|
+
if @format
|
38
|
+
@format
|
39
|
+
elsif @map
|
40
|
+
"map"
|
41
|
+
elsif @time && @record
|
42
|
+
"record"
|
43
|
+
else
|
44
|
+
raise Fluent::ConfigError, "Any of map, 2 parameters(time, and record) or format is required "
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def configure_format()
|
49
|
+
case @format
|
50
|
+
when "map"
|
51
|
+
# pass
|
52
|
+
when "record"
|
53
|
+
raise Fluent::ConfigError, "multi and 2 parameters(time, and record) are not compatible" if @multi
|
54
|
+
when "multimap"
|
55
|
+
# pass.
|
56
|
+
else
|
57
|
+
raise Fluent::ConfigError, "format #{@format} is invalid."
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
def create_map(conf)
|
62
|
+
# return string like double array.
|
63
|
+
case @format
|
64
|
+
when "map"
|
65
|
+
parse_map()
|
66
|
+
when "record"
|
67
|
+
"[[#{@time}, #{@record}]]"
|
68
|
+
when "multimap"
|
69
|
+
parse_multimap(conf)
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
def filter_stream(tag, es)
|
74
|
+
begin
|
75
|
+
new_es = Fluent::MultiEventStream.new
|
76
|
+
tag_output_es = @map_support.do_map(tag, es)
|
77
|
+
tag_output_es.each_pair do |tag, output_es|
|
78
|
+
output_es.each{|time, record|
|
79
|
+
new_es.add(time, record)
|
80
|
+
}
|
81
|
+
end
|
82
|
+
new_es
|
83
|
+
rescue SyntaxError => e
|
84
|
+
log.error "map command is syntax error: #{@map}"
|
85
|
+
e #for test
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
#
|
2
|
+
# fluent-plugin-map
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
module Fluent
|
18
|
+
module MapConfigParam
|
19
|
+
def self.included(klass)
|
20
|
+
klass.instance_eval {
|
21
|
+
config_param :map, :string, :default => nil
|
22
|
+
config_param :time, :string, :default => nil
|
23
|
+
config_param :record, :string, :default => nil
|
24
|
+
config_param :multi, :bool, :default => false
|
25
|
+
config_param :timeout, :time, :default => 1
|
26
|
+
config_param :format, :string, :default => nil
|
27
|
+
}
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,113 @@
|
|
1
|
+
#
|
2
|
+
# fluent-plugin-map
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
module Fluent
|
18
|
+
class MapSupport
|
19
|
+
def initialize(map, plugin)
|
20
|
+
@map = map
|
21
|
+
@plugin = plugin
|
22
|
+
if defined?(Fluent::Filter) and plugin.is_a?(Fluent::Filter)
|
23
|
+
singleton_class.module_eval(<<-CODE)
|
24
|
+
def map_func(time, record)
|
25
|
+
#{@map}
|
26
|
+
end
|
27
|
+
CODE
|
28
|
+
class << self
|
29
|
+
alias_method :generate_tuples, :generate_tuples_filter
|
30
|
+
alias_method :do_map, :do_map_filter
|
31
|
+
end
|
32
|
+
elsif plugin.is_a?(Fluent::Output)
|
33
|
+
singleton_class.module_eval(<<-CODE)
|
34
|
+
def map_func(tag, time, record)
|
35
|
+
#{@map}
|
36
|
+
end
|
37
|
+
CODE
|
38
|
+
class << self
|
39
|
+
alias_method :generate_tuples, :generate_tuples_output
|
40
|
+
alias_method :do_map, :do_map_output
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def do_map(tag, es)
|
46
|
+
# This method will be overwritten in #initailize.
|
47
|
+
end
|
48
|
+
|
49
|
+
def do_map_filter(tag, es)
|
50
|
+
tuples = generate_tuples(tag, es)
|
51
|
+
|
52
|
+
tag_output_es = Hash.new{|h, key| h[key] = Fluent::MultiEventStream.new}
|
53
|
+
tuples.each do |time, record|
|
54
|
+
if time == nil || record == nil
|
55
|
+
raise SyntaxError.new
|
56
|
+
end
|
57
|
+
tag_output_es[tag].add(time, record)
|
58
|
+
@plugin.log.trace { [tag, time, record].inspect }
|
59
|
+
end
|
60
|
+
tag_output_es
|
61
|
+
end
|
62
|
+
|
63
|
+
def do_map_output(tag, es)
|
64
|
+
tuples = generate_tuples(tag, es)
|
65
|
+
|
66
|
+
tag_output_es = Hash.new{|h, key| h[key] = Fluent::MultiEventStream.new}
|
67
|
+
tuples.each do |tag, time, record|
|
68
|
+
if time == nil || record == nil
|
69
|
+
raise SyntaxError.new
|
70
|
+
end
|
71
|
+
tag_output_es[tag].add(time, record)
|
72
|
+
@plugin.log.trace { [tag, time, record].inspect }
|
73
|
+
end
|
74
|
+
tag_output_es
|
75
|
+
end
|
76
|
+
|
77
|
+
def generate_tuples
|
78
|
+
# This method will be overwritten in #initailize.
|
79
|
+
end
|
80
|
+
|
81
|
+
def generate_tuples_filter(tag, es)
|
82
|
+
tuples = []
|
83
|
+
es.each {|time, record|
|
84
|
+
timeout_block do
|
85
|
+
new_tuple = map_func(time, record)
|
86
|
+
tuples.concat new_tuple
|
87
|
+
end
|
88
|
+
}
|
89
|
+
tuples
|
90
|
+
end
|
91
|
+
|
92
|
+
def generate_tuples_output(tag, es)
|
93
|
+
tuples = []
|
94
|
+
es.each {|time, record|
|
95
|
+
timeout_block do
|
96
|
+
new_tuple = map_func(tag, time, record)
|
97
|
+
tuples.concat new_tuple
|
98
|
+
end
|
99
|
+
}
|
100
|
+
tuples
|
101
|
+
end
|
102
|
+
|
103
|
+
def timeout_block
|
104
|
+
begin
|
105
|
+
Timeout.timeout(@plugin.timeout){
|
106
|
+
yield
|
107
|
+
}
|
108
|
+
rescue Timeout::Error
|
109
|
+
@plugin.log.error {"Timeout: #{Time.at(time)} #{tag} #{record.inspect}"}
|
110
|
+
end
|
111
|
+
end
|
112
|
+
end
|
113
|
+
end
|
@@ -14,10 +14,17 @@
|
|
14
14
|
# limitations under the License.
|
15
15
|
#
|
16
16
|
|
17
|
+
require 'fluent/plugin/map_support'
|
18
|
+
require 'fluent/plugin/map_config_param'
|
19
|
+
require 'fluent/plugin/parse_map_mixin'
|
20
|
+
|
17
21
|
module Fluent
|
18
22
|
class MapOutput < Fluent::Output
|
19
23
|
Fluent::Plugin.register_output('map', self)
|
20
24
|
|
25
|
+
include Fluent::MapConfigParam
|
26
|
+
include Fluent::ParseMap::Mixin
|
27
|
+
|
21
28
|
# Define `router` method of v0.12 to support v0.10 or earlier
|
22
29
|
unless method_defined?(:router)
|
23
30
|
define_method("router") { Fluent::Engine }
|
@@ -27,27 +34,15 @@ module Fluent
|
|
27
34
|
define_method("log") { $log }
|
28
35
|
end
|
29
36
|
|
30
|
-
config_param :map, :string, :default => nil
|
31
|
-
config_param :tag, :string, :default => nil
|
32
37
|
config_param :key, :string, :default => nil #deprecated
|
33
|
-
config_param :
|
34
|
-
config_param :record, :string, :default => nil
|
35
|
-
config_param :multi, :bool, :default => false
|
36
|
-
config_param :timeout, :time, :default => 1
|
37
|
-
config_param :format, :string, :default => nil
|
38
|
-
|
39
|
-
MMAP_MAX_NUM = 50
|
38
|
+
config_param :tag, :string, :default => nil
|
40
39
|
|
41
40
|
def configure(conf)
|
42
41
|
super
|
43
42
|
@format = determine_format()
|
44
43
|
configure_format()
|
45
44
|
@map = create_map(conf)
|
46
|
-
|
47
|
-
def map_func(tag, time, record)
|
48
|
-
#{@map}
|
49
|
-
end
|
50
|
-
CODE
|
45
|
+
@map_support = Fluent::MapSupport.new(@map, self)
|
51
46
|
end
|
52
47
|
|
53
48
|
def determine_format()
|
@@ -58,7 +53,7 @@ module Fluent
|
|
58
53
|
elsif (@tag || @key) && @time && @record
|
59
54
|
"record"
|
60
55
|
else
|
61
|
-
raise ConfigError, "Any of map, 3 parameters(tag, time, and record) or format is required "
|
56
|
+
raise Fluent::ConfigError, "Any of map, 3 parameters(tag, time, and record) or format is required "
|
62
57
|
end
|
63
58
|
end
|
64
59
|
|
@@ -68,11 +63,11 @@ module Fluent
|
|
68
63
|
# pass
|
69
64
|
when "record"
|
70
65
|
@tag ||= @key
|
71
|
-
raise ConfigError, "multi and 3 parameters(tag, time, and record) are not compatible" if @multi
|
66
|
+
raise Fluent::ConfigError, "multi and 3 parameters(tag, time, and record) are not compatible" if @multi
|
72
67
|
when "multimap"
|
73
68
|
# pass.
|
74
69
|
else
|
75
|
-
raise ConfigError, "format #{@format} is invalid."
|
70
|
+
raise Fluent::ConfigError, "format #{@format} is invalid."
|
76
71
|
end
|
77
72
|
end
|
78
73
|
|
@@ -88,45 +83,9 @@ module Fluent
|
|
88
83
|
end
|
89
84
|
end
|
90
85
|
|
91
|
-
def parse_map()
|
92
|
-
if @multi
|
93
|
-
@map
|
94
|
-
else
|
95
|
-
"[#{@map}]"
|
96
|
-
end
|
97
|
-
end
|
98
|
-
|
99
|
-
def parse_multimap(conf)
|
100
|
-
check_mmap_range(conf)
|
101
|
-
|
102
|
-
prev_mmap = nil
|
103
|
-
result_mmaps = (1..MMAP_MAX_NUM).map { |i|
|
104
|
-
mmap = conf["mmap#{i}"]
|
105
|
-
if (i > 1) && prev_mmap.nil? && !mmap.nil?
|
106
|
-
raise ConfigError, "Jump of mmap index found. mmap#{i - 1} is missing."
|
107
|
-
end
|
108
|
-
prev_mmap = mmap
|
109
|
-
next if mmap.nil?
|
110
|
-
|
111
|
-
mmap
|
112
|
-
}.compact.join(',')
|
113
|
-
"[#{result_mmaps}]"
|
114
|
-
end
|
115
|
-
|
116
|
-
def check_mmap_range(conf)
|
117
|
-
invalid_mmap = conf.keys.select { |k|
|
118
|
-
m = k.match(/^mmap(\d+)$/)
|
119
|
-
m ? !((1..MMAP_MAX_NUM).include?(m[1].to_i)) : false
|
120
|
-
}
|
121
|
-
unless invalid_mmap.empty?
|
122
|
-
raise ConfigError, "Invalid mmapN found. N should be 1 - #{MMAP_MAX_NUM}: " + invalid_mmap.join(",")
|
123
|
-
end
|
124
|
-
end
|
125
|
-
|
126
|
-
|
127
86
|
def emit(tag, es, chain)
|
128
87
|
begin
|
129
|
-
tag_output_es = do_map(tag, es)
|
88
|
+
tag_output_es = @map_support.do_map(tag, es)
|
130
89
|
tag_output_es.each_pair do |tag, output_es|
|
131
90
|
router.emit_stream(tag, output_es)
|
132
91
|
end
|
@@ -138,38 +97,5 @@ module Fluent
|
|
138
97
|
e #for test
|
139
98
|
end
|
140
99
|
end
|
141
|
-
|
142
|
-
def do_map(tag, es)
|
143
|
-
tuples = generate_tuples(tag, es)
|
144
|
-
|
145
|
-
tag_output_es = Hash.new{|h, key| h[key] = MultiEventStream::new}
|
146
|
-
tuples.each do |tag, time, record|
|
147
|
-
if time == nil || record == nil
|
148
|
-
raise SyntaxError.new
|
149
|
-
end
|
150
|
-
tag_output_es[tag].add(time, record)
|
151
|
-
log.trace { [tag, time, record].inspect }
|
152
|
-
end
|
153
|
-
tag_output_es
|
154
|
-
end
|
155
|
-
|
156
|
-
def generate_tuples(tag, es)
|
157
|
-
tuples = []
|
158
|
-
es.each {|time, record|
|
159
|
-
new_tuple = map_func(tag, time, record)
|
160
|
-
tuples.concat new_tuple
|
161
|
-
}
|
162
|
-
tuples
|
163
|
-
end
|
164
|
-
|
165
|
-
def timeout_block(tag, time, record)
|
166
|
-
begin
|
167
|
-
Timeout.timeout(@timeout){
|
168
|
-
yield
|
169
|
-
}
|
170
|
-
rescue Timeout::Error
|
171
|
-
log.error {"Timeout: #{Time.at(time)} #{tag} #{record.inspect}"}
|
172
|
-
end
|
173
|
-
end
|
174
100
|
end
|
175
101
|
end
|
@@ -0,0 +1,59 @@
|
|
1
|
+
#
|
2
|
+
# fluent-plugin-map
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
module Fluent
|
18
|
+
module ParseMap
|
19
|
+
module Mixin
|
20
|
+
|
21
|
+
MMAP_MAX_NUM = 50
|
22
|
+
|
23
|
+
def parse_map()
|
24
|
+
if @multi
|
25
|
+
@map
|
26
|
+
else
|
27
|
+
"[#{@map}]"
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def parse_multimap(conf)
|
32
|
+
check_mmap_range(conf)
|
33
|
+
|
34
|
+
prev_mmap = nil
|
35
|
+
result_mmaps = (1..MMAP_MAX_NUM).map { |i|
|
36
|
+
mmap = conf["mmap#{i}"]
|
37
|
+
if (i > 1) && prev_mmap.nil? && !mmap.nil?
|
38
|
+
raise Fluent::ConfigError, "Jump of mmap index found. mmap#{i - 1} is missing."
|
39
|
+
end
|
40
|
+
prev_mmap = mmap
|
41
|
+
next if mmap.nil?
|
42
|
+
|
43
|
+
mmap
|
44
|
+
}.compact.join(',')
|
45
|
+
"[#{result_mmaps}]"
|
46
|
+
end
|
47
|
+
|
48
|
+
def check_mmap_range(conf)
|
49
|
+
invalid_mmap = conf.keys.select { |k|
|
50
|
+
m = k.match(/^mmap(\d+)$/)
|
51
|
+
m ? !((1..MMAP_MAX_NUM).include?(m[1].to_i)) : false
|
52
|
+
}
|
53
|
+
unless invalid_mmap.empty?
|
54
|
+
raise Fluent::ConfigError, "Invalid mmapN found. N should be 1 - #{MMAP_MAX_NUM}: " + invalid_mmap.join(",")
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
data/test/helper.rb
CHANGED
@@ -0,0 +1,182 @@
|
|
1
|
+
require 'helper'
|
2
|
+
|
3
|
+
class MapFilterTest < Test::Unit::TestCase
|
4
|
+
def setup
|
5
|
+
omit "Use Fluentd v0.12 or later" unless defined?(Fluent::Filter)
|
6
|
+
Fluent::Test.setup
|
7
|
+
end
|
8
|
+
|
9
|
+
CONFIG = %[
|
10
|
+
map [time, record]
|
11
|
+
multi false
|
12
|
+
]
|
13
|
+
|
14
|
+
def create_driver(conf = CONFIG, tag='test.input')
|
15
|
+
Fluent::Test::FilterTestDriver.new(Fluent::MapFilter, tag).configure(conf)
|
16
|
+
end
|
17
|
+
|
18
|
+
def test_syntax_error
|
19
|
+
tag = "tag"
|
20
|
+
time = Time.local(2012, 10, 10, 10, 10, 0).to_i
|
21
|
+
record = {'code' => '300'}
|
22
|
+
|
23
|
+
#map is syntax error
|
24
|
+
syntax_error_config = %[
|
25
|
+
map time.
|
26
|
+
]
|
27
|
+
assert_raise SyntaxError do
|
28
|
+
create_driver(syntax_error_config, tag)
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def test_syntax_error2
|
33
|
+
tag = "tag"
|
34
|
+
time = Time.local(2012, 10, 10, 10, 10, 0).to_i
|
35
|
+
record = {'code' => '300'}
|
36
|
+
|
37
|
+
#map output lligal format
|
38
|
+
syntax_error_config = %[
|
39
|
+
map time
|
40
|
+
]
|
41
|
+
d1 = create_driver(syntax_error_config, tag)
|
42
|
+
es = Fluent::OneEventStream.new(time, record)
|
43
|
+
e = d1.instance.filter_stream(tag, es)
|
44
|
+
assert e.kind_of?(SyntaxError)
|
45
|
+
end
|
46
|
+
|
47
|
+
def test_tag_convert_using_time_record
|
48
|
+
tag = 'tag.raw'
|
49
|
+
time = Time.local(2012, 10, 10, 10, 10, 10).to_i
|
50
|
+
record = {'code' => '300'}
|
51
|
+
|
52
|
+
d1 = create_driver %[
|
53
|
+
time time
|
54
|
+
record record
|
55
|
+
], tag
|
56
|
+
d1.run do
|
57
|
+
d1.emit(record, time)
|
58
|
+
end
|
59
|
+
filtered = d1.filtered_as_array
|
60
|
+
assert_equal 1, filtered.length
|
61
|
+
assert_equal ["tag.raw", time, record], filtered[0]
|
62
|
+
end
|
63
|
+
|
64
|
+
#deprected specification test
|
65
|
+
def test_config_error_time
|
66
|
+
tag = "tag"
|
67
|
+
record = {'code' => '300'}
|
68
|
+
|
69
|
+
#require time
|
70
|
+
assert_raise(Fluent::ConfigError){
|
71
|
+
create_driver %[
|
72
|
+
record record
|
73
|
+
], tag
|
74
|
+
}
|
75
|
+
end
|
76
|
+
|
77
|
+
def test_config_error_record
|
78
|
+
tag = "tag"
|
79
|
+
time = Time.local(2012, 10, 10, 10, 10, 0).to_i
|
80
|
+
|
81
|
+
#require record
|
82
|
+
assert_raise(Fluent::ConfigError){
|
83
|
+
create_driver %[
|
84
|
+
time time
|
85
|
+
], tag
|
86
|
+
}
|
87
|
+
end
|
88
|
+
|
89
|
+
def test_config_error_multi
|
90
|
+
tag = "tag"
|
91
|
+
time = Time.local(2012, 10, 10, 10, 10, 0).to_i
|
92
|
+
record = {'code' => '300'}
|
93
|
+
|
94
|
+
#require time
|
95
|
+
assert_raise(Fluent::ConfigError){
|
96
|
+
create_driver %[
|
97
|
+
time time
|
98
|
+
record record
|
99
|
+
multi true
|
100
|
+
], tag
|
101
|
+
}
|
102
|
+
end
|
103
|
+
|
104
|
+
def test_config_error_sleep
|
105
|
+
tag = 'tag'
|
106
|
+
time = Time.local(2012, 10, 10, 10, 10, 10).to_i
|
107
|
+
record = {'code' => '300'}
|
108
|
+
|
109
|
+
assert_raise(SyntaxError) {
|
110
|
+
create_driver %[
|
111
|
+
tag "newtag"
|
112
|
+
time sleep 10
|
113
|
+
record record
|
114
|
+
timeout 1s
|
115
|
+
], tag
|
116
|
+
}
|
117
|
+
end
|
118
|
+
|
119
|
+
# Add format test
|
120
|
+
## test format type (map, record, maps)
|
121
|
+
## test Backward compatibility without format
|
122
|
+
##
|
123
|
+
|
124
|
+
def test_convert_format_map
|
125
|
+
tag = 'tag.raw'
|
126
|
+
time = Time.local(2012, 10, 10, 10, 10, 10).to_i
|
127
|
+
record = {'code1' => '300', 'code2' => '400'}
|
128
|
+
|
129
|
+
d1 = create_driver %[
|
130
|
+
format map
|
131
|
+
map [[time, record["code1"]], [time, record["code2"]]]
|
132
|
+
multi true
|
133
|
+
], tag
|
134
|
+
d1.run do
|
135
|
+
d1.emit(record, time)
|
136
|
+
end
|
137
|
+
filtered = d1.filtered_as_array
|
138
|
+
assert_equal 2, filtered.length
|
139
|
+
assert_equal tag, d1.tag # tag shouldn't be changed
|
140
|
+
assert_equal ["tag.raw", time, record["code1"]], filtered[0]
|
141
|
+
assert_equal ["tag.raw", time, record["code2"]], filtered[1]
|
142
|
+
end
|
143
|
+
|
144
|
+
def test_tag_convert_format_record
|
145
|
+
tag = 'tag.raw'
|
146
|
+
time = Time.local(2012, 10, 10, 10, 10, 10).to_i
|
147
|
+
record = {'code' => '300'}
|
148
|
+
|
149
|
+
d1 = create_driver %[
|
150
|
+
format record
|
151
|
+
time time
|
152
|
+
record record
|
153
|
+
], tag
|
154
|
+
d1.run do
|
155
|
+
d1.emit(record, time)
|
156
|
+
end
|
157
|
+
filtered = d1.filtered_as_array
|
158
|
+
assert_equal 1, filtered.length
|
159
|
+
assert_equal tag, d1.tag # tag shouldn't be changed
|
160
|
+
assert_equal ["tag.raw", time, record], filtered[0]
|
161
|
+
end
|
162
|
+
|
163
|
+
def test_convert_format_multimap
|
164
|
+
tag = 'tag.raw'
|
165
|
+
time = Time.local(2012, 10, 10, 10, 10, 10).to_i
|
166
|
+
record = {'code1' => '300', 'code2' => '400'}
|
167
|
+
|
168
|
+
d1 = create_driver %[
|
169
|
+
format multimap
|
170
|
+
mmap1 [time, record["code1"]]
|
171
|
+
mmap2 [time, record["code2"]]
|
172
|
+
], tag
|
173
|
+
d1.run do
|
174
|
+
d1.emit(record, time)
|
175
|
+
end
|
176
|
+
filtered = d1.filtered_as_array
|
177
|
+
assert_equal 2, filtered.length
|
178
|
+
assert_equal tag, d1.tag # tag shouldn't be changed
|
179
|
+
assert_equal ["tag.raw", time, record["code1"]], filtered[0]
|
180
|
+
assert_equal ["tag.raw", time, record["code2"]], filtered[1]
|
181
|
+
end
|
182
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-map
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Kohei Tomita
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2017-03-
|
13
|
+
date: 2017-03-14 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: rake
|
@@ -95,8 +95,13 @@ files:
|
|
95
95
|
- gemfiles/fluentd_0_10.gemfile
|
96
96
|
- gemfiles/fluentd_0_12.gemfile
|
97
97
|
- gemfiles/fluentd_0_14.gemfile
|
98
|
+
- lib/fluent/plugin/filter_map.rb
|
99
|
+
- lib/fluent/plugin/map_config_param.rb
|
100
|
+
- lib/fluent/plugin/map_support.rb
|
98
101
|
- lib/fluent/plugin/out_map.rb
|
102
|
+
- lib/fluent/plugin/parse_map_mixin.rb
|
99
103
|
- test/helper.rb
|
104
|
+
- test/plugin/test_filter_map.rb
|
100
105
|
- test/plugin/test_out_map.rb
|
101
106
|
homepage: https://github.com/fluent-plugins-nursery/fluent-plugin-map
|
102
107
|
licenses:
|
@@ -125,4 +130,5 @@ summary: fluent-plugin-map is the non-buffered plugin that can convert an event
|
|
125
130
|
to different event log(s).
|
126
131
|
test_files:
|
127
132
|
- test/helper.rb
|
133
|
+
- test/plugin/test_filter_map.rb
|
128
134
|
- test/plugin/test_out_map.rb
|