fluent-plugin-map 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.markdown +77 -8
- data/fluent-plugin-map.gemspec +1 -1
- data/lib/fluent/plugin/filter_map.rb +89 -0
- data/lib/fluent/plugin/map_config_param.rb +30 -0
- data/lib/fluent/plugin/map_support.rb +113 -0
- data/lib/fluent/plugin/out_map.rb +13 -87
- data/lib/fluent/plugin/parse_map_mixin.rb +59 -0
- data/test/helper.rb +1 -0
- data/test/plugin/test_filter_map.rb +182 -0
- metadata +8 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4ef5c9646ede101aef6159afc29fe2c23f71ae13
|
4
|
+
data.tar.gz: 4b85b98b5bc707817699449ae65b7a9fef64f670
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d022df78ec9b3e3b9eaf60b7c24f48f2b21e455f4e551ad147807e76bd01a78a7f7de89ea7ab1ed6e5edf5d0232789c6bdf7b9b47f0462539c4c46e910e14afa
|
7
|
+
data.tar.gz: 5927afac4efd90c7b3a5a85901db2e93328f4b2144ba6e87d30312f9988e4612108411cd726656c3120eaa6a9a21d489603e18ed9749cd28ceed3ae14279033c
|
data/README.markdown
CHANGED
@@ -4,27 +4,93 @@
|
|
4
4
|
|
5
5
|
fluent-plugin-map(out\_map) is the non-buffered plugin that can convert an event log to different event log(s)
|
6
6
|
|
7
|
-
##
|
7
|
+
## MapFilter
|
8
|
+
|
9
|
+
### Example
|
10
|
+
|
11
|
+
This sample config filter code file and time file.
|
12
|
+
|
13
|
+
<source>
|
14
|
+
@type tail
|
15
|
+
format apache
|
16
|
+
path /var/log/httpd-access.log
|
17
|
+
tag tag
|
18
|
+
@label @raw
|
19
|
+
</source>
|
20
|
+
<label @raw>
|
21
|
+
<match **>
|
22
|
+
@type copy
|
23
|
+
<store>
|
24
|
+
@type relabel
|
25
|
+
@label @code
|
26
|
+
</store>
|
27
|
+
<store>
|
28
|
+
@type relabel
|
29
|
+
@label @time
|
30
|
+
</store>
|
31
|
+
</match>
|
32
|
+
</label>
|
33
|
+
<label @code>
|
34
|
+
<filter **>
|
35
|
+
@type map
|
36
|
+
map ([time, {"code" => record["code"].to_i}])
|
37
|
+
</filter>
|
38
|
+
<match **>
|
39
|
+
@type file
|
40
|
+
path code.log
|
41
|
+
</match>
|
42
|
+
</label>
|
43
|
+
<label @time>
|
44
|
+
<filter **>
|
45
|
+
@type map
|
46
|
+
map ([time, {"time" => record["time"].to_i}])
|
47
|
+
</filter>
|
48
|
+
<match **>
|
49
|
+
@type file
|
50
|
+
path time.log
|
51
|
+
</match>
|
52
|
+
</label>
|
53
|
+
|
54
|
+
|
55
|
+
The parameter "map" can use 2 variables in event log; time, record. The format of time is an integer number of seconds since the Epoch. The format of record is hash.
|
56
|
+
The config file parses # as the begin of comment. So the "map" value cannot use #{tag} operation.
|
57
|
+
This plugin can output multi logs by seting multi to true.
|
58
|
+
|
59
|
+
If you don't use multi option, you can use time, record parameter. The 2 following filter directive is same:
|
60
|
+
|
61
|
+
<filter tag>
|
62
|
+
@type map
|
63
|
+
map ([time, {"code" => record["code"].to_i}])
|
64
|
+
</filter>
|
65
|
+
<filter tag>
|
66
|
+
@type map
|
67
|
+
time time
|
68
|
+
record ({"code" => record["code"].to_i})
|
69
|
+
</filter>
|
70
|
+
|
71
|
+
## MapOutput
|
72
|
+
|
73
|
+
### Example
|
8
74
|
|
9
75
|
This sample config output code file and time file.
|
10
76
|
|
11
77
|
<source>
|
12
|
-
type tail
|
78
|
+
@type tail
|
13
79
|
format apache
|
14
80
|
path /var/log/httpd-access.log
|
15
81
|
tag tag
|
16
82
|
</source>
|
17
83
|
<match tag>
|
18
|
-
type map
|
84
|
+
@type map
|
19
85
|
map ([["code." + tag, time, {"code" => record["code"].to_i}], ["time." + tag, time, {"time" => record["time"].to_i}]])
|
20
86
|
multi true
|
21
87
|
</match>
|
22
88
|
<match code.tag>
|
23
|
-
type file
|
89
|
+
@type file
|
24
90
|
path code.log
|
25
91
|
</match>
|
26
92
|
<match time.tag>
|
27
|
-
type file
|
93
|
+
@type file
|
28
94
|
path time.log
|
29
95
|
</match>
|
30
96
|
|
@@ -36,17 +102,20 @@ This plugin can output multi logs by seting multi to true.
|
|
36
102
|
If you don't use multi option, you can use key, time, record parameter. The 2 following match directive is same:
|
37
103
|
|
38
104
|
<match tag>
|
39
|
-
type map
|
105
|
+
@type map
|
40
106
|
map (["code." + tag, time, {"code" => record["code"].to_i}])
|
41
107
|
</match>
|
42
108
|
<match tag>
|
43
|
-
type map
|
109
|
+
@type map
|
44
110
|
tag ("code." + tag)
|
45
111
|
time time
|
46
112
|
record ({"code" => record["code"].to_i})
|
47
113
|
</match>
|
48
114
|
|
49
|
-
|
115
|
+
|
116
|
+
## Note
|
117
|
+
|
118
|
+
you have to wrap some configuration values with parenthesis like `("code." + tag)`, to avoid parsing by Fluentd itself.
|
50
119
|
See also: [Fluentd | Configuration File | Format tips](http://docs.fluentd.org/articles/config-file#format-tips)
|
51
120
|
|
52
121
|
## Copyright
|
data/fluent-plugin-map.gemspec
CHANGED
@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = "fluent-plugin-map"
|
6
|
-
s.version = "0.1.
|
6
|
+
s.version = "0.1.1"
|
7
7
|
s.authors = ["Kohei Tomita", "Hiroshi Hatake", "Kenji Okomoto"]
|
8
8
|
s.email = ["tommy.fmale@gmail.com", "cosmo0920.oucc@gmail.com", "okkez000@gmail.com"]
|
9
9
|
s.homepage = "https://github.com/fluent-plugins-nursery/fluent-plugin-map"
|
@@ -0,0 +1,89 @@
|
|
1
|
+
#
|
2
|
+
# fluent-plugin-map
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'fluent/plugin/map_support'
|
18
|
+
require 'fluent/plugin/map_config_param'
|
19
|
+
require 'fluent/plugin/parse_map_mixin'
|
20
|
+
|
21
|
+
module Fluent
|
22
|
+
class MapFilter < Fluent::Filter
|
23
|
+
Fluent::Plugin.register_filter('map', self)
|
24
|
+
|
25
|
+
include Fluent::MapConfigParam
|
26
|
+
include Fluent::ParseMap::Mixin
|
27
|
+
|
28
|
+
def configure(conf)
|
29
|
+
super
|
30
|
+
@format = determine_format()
|
31
|
+
configure_format()
|
32
|
+
@map = create_map(conf)
|
33
|
+
@map_support = Fluent::MapSupport.new(@map, self)
|
34
|
+
end
|
35
|
+
|
36
|
+
def determine_format()
|
37
|
+
if @format
|
38
|
+
@format
|
39
|
+
elsif @map
|
40
|
+
"map"
|
41
|
+
elsif @time && @record
|
42
|
+
"record"
|
43
|
+
else
|
44
|
+
raise Fluent::ConfigError, "Any of map, 2 parameters(time, and record) or format is required "
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def configure_format()
|
49
|
+
case @format
|
50
|
+
when "map"
|
51
|
+
# pass
|
52
|
+
when "record"
|
53
|
+
raise Fluent::ConfigError, "multi and 2 parameters(time, and record) are not compatible" if @multi
|
54
|
+
when "multimap"
|
55
|
+
# pass.
|
56
|
+
else
|
57
|
+
raise Fluent::ConfigError, "format #{@format} is invalid."
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
def create_map(conf)
|
62
|
+
# return string like double array.
|
63
|
+
case @format
|
64
|
+
when "map"
|
65
|
+
parse_map()
|
66
|
+
when "record"
|
67
|
+
"[[#{@time}, #{@record}]]"
|
68
|
+
when "multimap"
|
69
|
+
parse_multimap(conf)
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
def filter_stream(tag, es)
|
74
|
+
begin
|
75
|
+
new_es = Fluent::MultiEventStream.new
|
76
|
+
tag_output_es = @map_support.do_map(tag, es)
|
77
|
+
tag_output_es.each_pair do |tag, output_es|
|
78
|
+
output_es.each{|time, record|
|
79
|
+
new_es.add(time, record)
|
80
|
+
}
|
81
|
+
end
|
82
|
+
new_es
|
83
|
+
rescue SyntaxError => e
|
84
|
+
log.error "map command is syntax error: #{@map}"
|
85
|
+
e #for test
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
#
|
2
|
+
# fluent-plugin-map
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
module Fluent
|
18
|
+
module MapConfigParam
|
19
|
+
def self.included(klass)
|
20
|
+
klass.instance_eval {
|
21
|
+
config_param :map, :string, :default => nil
|
22
|
+
config_param :time, :string, :default => nil
|
23
|
+
config_param :record, :string, :default => nil
|
24
|
+
config_param :multi, :bool, :default => false
|
25
|
+
config_param :timeout, :time, :default => 1
|
26
|
+
config_param :format, :string, :default => nil
|
27
|
+
}
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,113 @@
|
|
1
|
+
#
|
2
|
+
# fluent-plugin-map
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
module Fluent
|
18
|
+
class MapSupport
|
19
|
+
def initialize(map, plugin)
|
20
|
+
@map = map
|
21
|
+
@plugin = plugin
|
22
|
+
if defined?(Fluent::Filter) and plugin.is_a?(Fluent::Filter)
|
23
|
+
singleton_class.module_eval(<<-CODE)
|
24
|
+
def map_func(time, record)
|
25
|
+
#{@map}
|
26
|
+
end
|
27
|
+
CODE
|
28
|
+
class << self
|
29
|
+
alias_method :generate_tuples, :generate_tuples_filter
|
30
|
+
alias_method :do_map, :do_map_filter
|
31
|
+
end
|
32
|
+
elsif plugin.is_a?(Fluent::Output)
|
33
|
+
singleton_class.module_eval(<<-CODE)
|
34
|
+
def map_func(tag, time, record)
|
35
|
+
#{@map}
|
36
|
+
end
|
37
|
+
CODE
|
38
|
+
class << self
|
39
|
+
alias_method :generate_tuples, :generate_tuples_output
|
40
|
+
alias_method :do_map, :do_map_output
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def do_map(tag, es)
|
46
|
+
# This method will be overwritten in #initailize.
|
47
|
+
end
|
48
|
+
|
49
|
+
def do_map_filter(tag, es)
|
50
|
+
tuples = generate_tuples(tag, es)
|
51
|
+
|
52
|
+
tag_output_es = Hash.new{|h, key| h[key] = Fluent::MultiEventStream.new}
|
53
|
+
tuples.each do |time, record|
|
54
|
+
if time == nil || record == nil
|
55
|
+
raise SyntaxError.new
|
56
|
+
end
|
57
|
+
tag_output_es[tag].add(time, record)
|
58
|
+
@plugin.log.trace { [tag, time, record].inspect }
|
59
|
+
end
|
60
|
+
tag_output_es
|
61
|
+
end
|
62
|
+
|
63
|
+
def do_map_output(tag, es)
|
64
|
+
tuples = generate_tuples(tag, es)
|
65
|
+
|
66
|
+
tag_output_es = Hash.new{|h, key| h[key] = Fluent::MultiEventStream.new}
|
67
|
+
tuples.each do |tag, time, record|
|
68
|
+
if time == nil || record == nil
|
69
|
+
raise SyntaxError.new
|
70
|
+
end
|
71
|
+
tag_output_es[tag].add(time, record)
|
72
|
+
@plugin.log.trace { [tag, time, record].inspect }
|
73
|
+
end
|
74
|
+
tag_output_es
|
75
|
+
end
|
76
|
+
|
77
|
+
def generate_tuples
|
78
|
+
# This method will be overwritten in #initailize.
|
79
|
+
end
|
80
|
+
|
81
|
+
def generate_tuples_filter(tag, es)
|
82
|
+
tuples = []
|
83
|
+
es.each {|time, record|
|
84
|
+
timeout_block do
|
85
|
+
new_tuple = map_func(time, record)
|
86
|
+
tuples.concat new_tuple
|
87
|
+
end
|
88
|
+
}
|
89
|
+
tuples
|
90
|
+
end
|
91
|
+
|
92
|
+
def generate_tuples_output(tag, es)
|
93
|
+
tuples = []
|
94
|
+
es.each {|time, record|
|
95
|
+
timeout_block do
|
96
|
+
new_tuple = map_func(tag, time, record)
|
97
|
+
tuples.concat new_tuple
|
98
|
+
end
|
99
|
+
}
|
100
|
+
tuples
|
101
|
+
end
|
102
|
+
|
103
|
+
def timeout_block
|
104
|
+
begin
|
105
|
+
Timeout.timeout(@plugin.timeout){
|
106
|
+
yield
|
107
|
+
}
|
108
|
+
rescue Timeout::Error
|
109
|
+
@plugin.log.error {"Timeout: #{Time.at(time)} #{tag} #{record.inspect}"}
|
110
|
+
end
|
111
|
+
end
|
112
|
+
end
|
113
|
+
end
|
@@ -14,10 +14,17 @@
|
|
14
14
|
# limitations under the License.
|
15
15
|
#
|
16
16
|
|
17
|
+
require 'fluent/plugin/map_support'
|
18
|
+
require 'fluent/plugin/map_config_param'
|
19
|
+
require 'fluent/plugin/parse_map_mixin'
|
20
|
+
|
17
21
|
module Fluent
|
18
22
|
class MapOutput < Fluent::Output
|
19
23
|
Fluent::Plugin.register_output('map', self)
|
20
24
|
|
25
|
+
include Fluent::MapConfigParam
|
26
|
+
include Fluent::ParseMap::Mixin
|
27
|
+
|
21
28
|
# Define `router` method of v0.12 to support v0.10 or earlier
|
22
29
|
unless method_defined?(:router)
|
23
30
|
define_method("router") { Fluent::Engine }
|
@@ -27,27 +34,15 @@ module Fluent
|
|
27
34
|
define_method("log") { $log }
|
28
35
|
end
|
29
36
|
|
30
|
-
config_param :map, :string, :default => nil
|
31
|
-
config_param :tag, :string, :default => nil
|
32
37
|
config_param :key, :string, :default => nil #deprecated
|
33
|
-
config_param :
|
34
|
-
config_param :record, :string, :default => nil
|
35
|
-
config_param :multi, :bool, :default => false
|
36
|
-
config_param :timeout, :time, :default => 1
|
37
|
-
config_param :format, :string, :default => nil
|
38
|
-
|
39
|
-
MMAP_MAX_NUM = 50
|
38
|
+
config_param :tag, :string, :default => nil
|
40
39
|
|
41
40
|
def configure(conf)
|
42
41
|
super
|
43
42
|
@format = determine_format()
|
44
43
|
configure_format()
|
45
44
|
@map = create_map(conf)
|
46
|
-
|
47
|
-
def map_func(tag, time, record)
|
48
|
-
#{@map}
|
49
|
-
end
|
50
|
-
CODE
|
45
|
+
@map_support = Fluent::MapSupport.new(@map, self)
|
51
46
|
end
|
52
47
|
|
53
48
|
def determine_format()
|
@@ -58,7 +53,7 @@ module Fluent
|
|
58
53
|
elsif (@tag || @key) && @time && @record
|
59
54
|
"record"
|
60
55
|
else
|
61
|
-
raise ConfigError, "Any of map, 3 parameters(tag, time, and record) or format is required "
|
56
|
+
raise Fluent::ConfigError, "Any of map, 3 parameters(tag, time, and record) or format is required "
|
62
57
|
end
|
63
58
|
end
|
64
59
|
|
@@ -68,11 +63,11 @@ module Fluent
|
|
68
63
|
# pass
|
69
64
|
when "record"
|
70
65
|
@tag ||= @key
|
71
|
-
raise ConfigError, "multi and 3 parameters(tag, time, and record) are not compatible" if @multi
|
66
|
+
raise Fluent::ConfigError, "multi and 3 parameters(tag, time, and record) are not compatible" if @multi
|
72
67
|
when "multimap"
|
73
68
|
# pass.
|
74
69
|
else
|
75
|
-
raise ConfigError, "format #{@format} is invalid."
|
70
|
+
raise Fluent::ConfigError, "format #{@format} is invalid."
|
76
71
|
end
|
77
72
|
end
|
78
73
|
|
@@ -88,45 +83,9 @@ module Fluent
|
|
88
83
|
end
|
89
84
|
end
|
90
85
|
|
91
|
-
def parse_map()
|
92
|
-
if @multi
|
93
|
-
@map
|
94
|
-
else
|
95
|
-
"[#{@map}]"
|
96
|
-
end
|
97
|
-
end
|
98
|
-
|
99
|
-
def parse_multimap(conf)
|
100
|
-
check_mmap_range(conf)
|
101
|
-
|
102
|
-
prev_mmap = nil
|
103
|
-
result_mmaps = (1..MMAP_MAX_NUM).map { |i|
|
104
|
-
mmap = conf["mmap#{i}"]
|
105
|
-
if (i > 1) && prev_mmap.nil? && !mmap.nil?
|
106
|
-
raise ConfigError, "Jump of mmap index found. mmap#{i - 1} is missing."
|
107
|
-
end
|
108
|
-
prev_mmap = mmap
|
109
|
-
next if mmap.nil?
|
110
|
-
|
111
|
-
mmap
|
112
|
-
}.compact.join(',')
|
113
|
-
"[#{result_mmaps}]"
|
114
|
-
end
|
115
|
-
|
116
|
-
def check_mmap_range(conf)
|
117
|
-
invalid_mmap = conf.keys.select { |k|
|
118
|
-
m = k.match(/^mmap(\d+)$/)
|
119
|
-
m ? !((1..MMAP_MAX_NUM).include?(m[1].to_i)) : false
|
120
|
-
}
|
121
|
-
unless invalid_mmap.empty?
|
122
|
-
raise ConfigError, "Invalid mmapN found. N should be 1 - #{MMAP_MAX_NUM}: " + invalid_mmap.join(",")
|
123
|
-
end
|
124
|
-
end
|
125
|
-
|
126
|
-
|
127
86
|
def emit(tag, es, chain)
|
128
87
|
begin
|
129
|
-
tag_output_es = do_map(tag, es)
|
88
|
+
tag_output_es = @map_support.do_map(tag, es)
|
130
89
|
tag_output_es.each_pair do |tag, output_es|
|
131
90
|
router.emit_stream(tag, output_es)
|
132
91
|
end
|
@@ -138,38 +97,5 @@ module Fluent
|
|
138
97
|
e #for test
|
139
98
|
end
|
140
99
|
end
|
141
|
-
|
142
|
-
def do_map(tag, es)
|
143
|
-
tuples = generate_tuples(tag, es)
|
144
|
-
|
145
|
-
tag_output_es = Hash.new{|h, key| h[key] = MultiEventStream::new}
|
146
|
-
tuples.each do |tag, time, record|
|
147
|
-
if time == nil || record == nil
|
148
|
-
raise SyntaxError.new
|
149
|
-
end
|
150
|
-
tag_output_es[tag].add(time, record)
|
151
|
-
log.trace { [tag, time, record].inspect }
|
152
|
-
end
|
153
|
-
tag_output_es
|
154
|
-
end
|
155
|
-
|
156
|
-
def generate_tuples(tag, es)
|
157
|
-
tuples = []
|
158
|
-
es.each {|time, record|
|
159
|
-
new_tuple = map_func(tag, time, record)
|
160
|
-
tuples.concat new_tuple
|
161
|
-
}
|
162
|
-
tuples
|
163
|
-
end
|
164
|
-
|
165
|
-
def timeout_block(tag, time, record)
|
166
|
-
begin
|
167
|
-
Timeout.timeout(@timeout){
|
168
|
-
yield
|
169
|
-
}
|
170
|
-
rescue Timeout::Error
|
171
|
-
log.error {"Timeout: #{Time.at(time)} #{tag} #{record.inspect}"}
|
172
|
-
end
|
173
|
-
end
|
174
100
|
end
|
175
101
|
end
|
@@ -0,0 +1,59 @@
|
|
1
|
+
#
|
2
|
+
# fluent-plugin-map
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
module Fluent
|
18
|
+
module ParseMap
|
19
|
+
module Mixin
|
20
|
+
|
21
|
+
MMAP_MAX_NUM = 50
|
22
|
+
|
23
|
+
def parse_map()
|
24
|
+
if @multi
|
25
|
+
@map
|
26
|
+
else
|
27
|
+
"[#{@map}]"
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def parse_multimap(conf)
|
32
|
+
check_mmap_range(conf)
|
33
|
+
|
34
|
+
prev_mmap = nil
|
35
|
+
result_mmaps = (1..MMAP_MAX_NUM).map { |i|
|
36
|
+
mmap = conf["mmap#{i}"]
|
37
|
+
if (i > 1) && prev_mmap.nil? && !mmap.nil?
|
38
|
+
raise Fluent::ConfigError, "Jump of mmap index found. mmap#{i - 1} is missing."
|
39
|
+
end
|
40
|
+
prev_mmap = mmap
|
41
|
+
next if mmap.nil?
|
42
|
+
|
43
|
+
mmap
|
44
|
+
}.compact.join(',')
|
45
|
+
"[#{result_mmaps}]"
|
46
|
+
end
|
47
|
+
|
48
|
+
def check_mmap_range(conf)
|
49
|
+
invalid_mmap = conf.keys.select { |k|
|
50
|
+
m = k.match(/^mmap(\d+)$/)
|
51
|
+
m ? !((1..MMAP_MAX_NUM).include?(m[1].to_i)) : false
|
52
|
+
}
|
53
|
+
unless invalid_mmap.empty?
|
54
|
+
raise Fluent::ConfigError, "Invalid mmapN found. N should be 1 - #{MMAP_MAX_NUM}: " + invalid_mmap.join(",")
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
data/test/helper.rb
CHANGED
@@ -0,0 +1,182 @@
|
|
1
|
+
require 'helper'
|
2
|
+
|
3
|
+
class MapFilterTest < Test::Unit::TestCase
|
4
|
+
def setup
|
5
|
+
omit "Use Fluentd v0.12 or later" unless defined?(Fluent::Filter)
|
6
|
+
Fluent::Test.setup
|
7
|
+
end
|
8
|
+
|
9
|
+
CONFIG = %[
|
10
|
+
map [time, record]
|
11
|
+
multi false
|
12
|
+
]
|
13
|
+
|
14
|
+
def create_driver(conf = CONFIG, tag='test.input')
|
15
|
+
Fluent::Test::FilterTestDriver.new(Fluent::MapFilter, tag).configure(conf)
|
16
|
+
end
|
17
|
+
|
18
|
+
def test_syntax_error
|
19
|
+
tag = "tag"
|
20
|
+
time = Time.local(2012, 10, 10, 10, 10, 0).to_i
|
21
|
+
record = {'code' => '300'}
|
22
|
+
|
23
|
+
#map is syntax error
|
24
|
+
syntax_error_config = %[
|
25
|
+
map time.
|
26
|
+
]
|
27
|
+
assert_raise SyntaxError do
|
28
|
+
create_driver(syntax_error_config, tag)
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def test_syntax_error2
|
33
|
+
tag = "tag"
|
34
|
+
time = Time.local(2012, 10, 10, 10, 10, 0).to_i
|
35
|
+
record = {'code' => '300'}
|
36
|
+
|
37
|
+
#map output lligal format
|
38
|
+
syntax_error_config = %[
|
39
|
+
map time
|
40
|
+
]
|
41
|
+
d1 = create_driver(syntax_error_config, tag)
|
42
|
+
es = Fluent::OneEventStream.new(time, record)
|
43
|
+
e = d1.instance.filter_stream(tag, es)
|
44
|
+
assert e.kind_of?(SyntaxError)
|
45
|
+
end
|
46
|
+
|
47
|
+
def test_tag_convert_using_time_record
|
48
|
+
tag = 'tag.raw'
|
49
|
+
time = Time.local(2012, 10, 10, 10, 10, 10).to_i
|
50
|
+
record = {'code' => '300'}
|
51
|
+
|
52
|
+
d1 = create_driver %[
|
53
|
+
time time
|
54
|
+
record record
|
55
|
+
], tag
|
56
|
+
d1.run do
|
57
|
+
d1.emit(record, time)
|
58
|
+
end
|
59
|
+
filtered = d1.filtered_as_array
|
60
|
+
assert_equal 1, filtered.length
|
61
|
+
assert_equal ["tag.raw", time, record], filtered[0]
|
62
|
+
end
|
63
|
+
|
64
|
+
#deprected specification test
|
65
|
+
def test_config_error_time
|
66
|
+
tag = "tag"
|
67
|
+
record = {'code' => '300'}
|
68
|
+
|
69
|
+
#require time
|
70
|
+
assert_raise(Fluent::ConfigError){
|
71
|
+
create_driver %[
|
72
|
+
record record
|
73
|
+
], tag
|
74
|
+
}
|
75
|
+
end
|
76
|
+
|
77
|
+
def test_config_error_record
|
78
|
+
tag = "tag"
|
79
|
+
time = Time.local(2012, 10, 10, 10, 10, 0).to_i
|
80
|
+
|
81
|
+
#require record
|
82
|
+
assert_raise(Fluent::ConfigError){
|
83
|
+
create_driver %[
|
84
|
+
time time
|
85
|
+
], tag
|
86
|
+
}
|
87
|
+
end
|
88
|
+
|
89
|
+
def test_config_error_multi
|
90
|
+
tag = "tag"
|
91
|
+
time = Time.local(2012, 10, 10, 10, 10, 0).to_i
|
92
|
+
record = {'code' => '300'}
|
93
|
+
|
94
|
+
#require time
|
95
|
+
assert_raise(Fluent::ConfigError){
|
96
|
+
create_driver %[
|
97
|
+
time time
|
98
|
+
record record
|
99
|
+
multi true
|
100
|
+
], tag
|
101
|
+
}
|
102
|
+
end
|
103
|
+
|
104
|
+
def test_config_error_sleep
|
105
|
+
tag = 'tag'
|
106
|
+
time = Time.local(2012, 10, 10, 10, 10, 10).to_i
|
107
|
+
record = {'code' => '300'}
|
108
|
+
|
109
|
+
assert_raise(SyntaxError) {
|
110
|
+
create_driver %[
|
111
|
+
tag "newtag"
|
112
|
+
time sleep 10
|
113
|
+
record record
|
114
|
+
timeout 1s
|
115
|
+
], tag
|
116
|
+
}
|
117
|
+
end
|
118
|
+
|
119
|
+
# Add format test
|
120
|
+
## test format type (map, record, maps)
|
121
|
+
## test Backward compatibility without format
|
122
|
+
##
|
123
|
+
|
124
|
+
def test_convert_format_map
|
125
|
+
tag = 'tag.raw'
|
126
|
+
time = Time.local(2012, 10, 10, 10, 10, 10).to_i
|
127
|
+
record = {'code1' => '300', 'code2' => '400'}
|
128
|
+
|
129
|
+
d1 = create_driver %[
|
130
|
+
format map
|
131
|
+
map [[time, record["code1"]], [time, record["code2"]]]
|
132
|
+
multi true
|
133
|
+
], tag
|
134
|
+
d1.run do
|
135
|
+
d1.emit(record, time)
|
136
|
+
end
|
137
|
+
filtered = d1.filtered_as_array
|
138
|
+
assert_equal 2, filtered.length
|
139
|
+
assert_equal tag, d1.tag # tag shouldn't be changed
|
140
|
+
assert_equal ["tag.raw", time, record["code1"]], filtered[0]
|
141
|
+
assert_equal ["tag.raw", time, record["code2"]], filtered[1]
|
142
|
+
end
|
143
|
+
|
144
|
+
def test_tag_convert_format_record
|
145
|
+
tag = 'tag.raw'
|
146
|
+
time = Time.local(2012, 10, 10, 10, 10, 10).to_i
|
147
|
+
record = {'code' => '300'}
|
148
|
+
|
149
|
+
d1 = create_driver %[
|
150
|
+
format record
|
151
|
+
time time
|
152
|
+
record record
|
153
|
+
], tag
|
154
|
+
d1.run do
|
155
|
+
d1.emit(record, time)
|
156
|
+
end
|
157
|
+
filtered = d1.filtered_as_array
|
158
|
+
assert_equal 1, filtered.length
|
159
|
+
assert_equal tag, d1.tag # tag shouldn't be changed
|
160
|
+
assert_equal ["tag.raw", time, record], filtered[0]
|
161
|
+
end
|
162
|
+
|
163
|
+
def test_convert_format_multimap
|
164
|
+
tag = 'tag.raw'
|
165
|
+
time = Time.local(2012, 10, 10, 10, 10, 10).to_i
|
166
|
+
record = {'code1' => '300', 'code2' => '400'}
|
167
|
+
|
168
|
+
d1 = create_driver %[
|
169
|
+
format multimap
|
170
|
+
mmap1 [time, record["code1"]]
|
171
|
+
mmap2 [time, record["code2"]]
|
172
|
+
], tag
|
173
|
+
d1.run do
|
174
|
+
d1.emit(record, time)
|
175
|
+
end
|
176
|
+
filtered = d1.filtered_as_array
|
177
|
+
assert_equal 2, filtered.length
|
178
|
+
assert_equal tag, d1.tag # tag shouldn't be changed
|
179
|
+
assert_equal ["tag.raw", time, record["code1"]], filtered[0]
|
180
|
+
assert_equal ["tag.raw", time, record["code2"]], filtered[1]
|
181
|
+
end
|
182
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-map
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Kohei Tomita
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2017-03-
|
13
|
+
date: 2017-03-14 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: rake
|
@@ -95,8 +95,13 @@ files:
|
|
95
95
|
- gemfiles/fluentd_0_10.gemfile
|
96
96
|
- gemfiles/fluentd_0_12.gemfile
|
97
97
|
- gemfiles/fluentd_0_14.gemfile
|
98
|
+
- lib/fluent/plugin/filter_map.rb
|
99
|
+
- lib/fluent/plugin/map_config_param.rb
|
100
|
+
- lib/fluent/plugin/map_support.rb
|
98
101
|
- lib/fluent/plugin/out_map.rb
|
102
|
+
- lib/fluent/plugin/parse_map_mixin.rb
|
99
103
|
- test/helper.rb
|
104
|
+
- test/plugin/test_filter_map.rb
|
100
105
|
- test/plugin/test_out_map.rb
|
101
106
|
homepage: https://github.com/fluent-plugins-nursery/fluent-plugin-map
|
102
107
|
licenses:
|
@@ -125,4 +130,5 @@ summary: fluent-plugin-map is the non-buffered plugin that can convert an event
|
|
125
130
|
to different event log(s).
|
126
131
|
test_files:
|
127
132
|
- test/helper.rb
|
133
|
+
- test/plugin/test_filter_map.rb
|
128
134
|
- test/plugin/test_out_map.rb
|