fluent-plugin-eval-filter 0.0.2 → 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/.travis.yml +8 -0
- data/README.md +15 -1
- data/Rakefile +10 -1
- data/fluent-plugin-eval-filter.gemspec +1 -1
- data/lib/fluent/plugin/out_eval_filter.rb +77 -50
- data/test/helper.rb +28 -0
- data/test/plugin/test_out_eval_filter.rb +187 -0
- metadata +9 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6765bfbe982ee8ab3f79203bc16c41905c367373
|
4
|
+
data.tar.gz: cc5f7b09aae48c79710766f835fe5cf419800209
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 05b3f8c2ea08ec5b9aa6865ce8fcee0bb1449d1ffbcbbfbc75d63127396e6b673cda42e53009a804ca47bc50681d61fdc61ca240debbf78df520bac6b3cabbdc
|
7
|
+
data.tar.gz: 70701cb426b287d3494faf4803a0b50306387f8a2a768de03b17e4cea34d28c5d2bc4d44273f695387305487d8469e49bc737eb5d79a288f52dde1b45ec1d7fc
|
data/.gitignore
CHANGED
data/.travis.yml
ADDED
data/README.md
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
# fluent-plugin-eval-filter
|
1
|
+
# fluent-plugin-eval-filter, a plugin for [Fluentd](http://fluentd.org) [](https://travis-ci.org/ephemeralsnow/fluent-plugin-eval-filter) [](https://codeclimate.com/github/ephemeralsnow/fluent-plugin-eval-filter)
|
2
2
|
|
3
3
|
## Installation
|
4
4
|
|
@@ -28,6 +28,20 @@ Or install it yourself as:
|
|
28
28
|
</match>
|
29
29
|
```
|
30
30
|
|
31
|
+
## Limitation
|
32
|
+
|
33
|
+
Can not be used expression substitution.
|
34
|
+
```
|
35
|
+
<match raw.apache.access>
|
36
|
+
filter1 "#{tag}"
|
37
|
+
</match>
|
38
|
+
```
|
39
|
+
|
40
|
+
'#' Is interpreted as the beginning of a comment.
|
41
|
+
```
|
42
|
+
filter1 #=> "\""
|
43
|
+
```
|
44
|
+
|
31
45
|
## Contributing
|
32
46
|
|
33
47
|
1. Fork it
|
data/Rakefile
CHANGED
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
4
|
|
5
5
|
Gem::Specification.new do |spec|
|
6
6
|
spec.name = "fluent-plugin-eval-filter"
|
7
|
-
spec.version = "0.0.
|
7
|
+
spec.version = "0.0.3"
|
8
8
|
spec.authors = ["Yuzuki Masaru"]
|
9
9
|
spec.email = ["ephemeralsnow@gmail.com"]
|
10
10
|
spec.description = %q{Fluentd Output eval filter plugin.}
|
@@ -1,50 +1,77 @@
|
|
1
|
-
class Fluent::EvalFilterOutput < Fluent::Output
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
end
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
1
|
+
class Fluent::EvalFilterOutput < Fluent::Output
|
2
|
+
|
3
|
+
Fluent::Plugin.register_output('eval_filter', self)
|
4
|
+
|
5
|
+
def configure(conf)
|
6
|
+
super
|
7
|
+
|
8
|
+
if remove_tag_prefix = conf['remove_tag_prefix']
|
9
|
+
@remove_tag_prefix = /^#{Regexp.escape(remove_tag_prefix)}\.*/
|
10
|
+
end
|
11
|
+
if remove_tag_suffix = conf['remove_tag_suffix']
|
12
|
+
@remove_tag_suffix = /\.*#{Regexp.escape(remove_tag_suffix)}$/
|
13
|
+
end
|
14
|
+
@add_tag_prefix = conf['add_tag_prefix']
|
15
|
+
@add_tag_suffix = conf['add_tag_suffix']
|
16
|
+
|
17
|
+
conf.keys.select { |key| key =~ /^config\d+$/ }.sort_by { |key| key.sub('config', '').to_i }.each do |key|
|
18
|
+
begin
|
19
|
+
instance_eval("#{conf[key]}")
|
20
|
+
rescue Exception => e
|
21
|
+
raise Fluent::ConfigError, "#{key} #{conf[key]}\n" + e.to_s
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
@filters = []
|
26
|
+
conf.keys.select { |key| key =~ /^filter\d+$/ }.sort_by { |key| key.sub('filter', '').to_i }.each do |key|
|
27
|
+
begin
|
28
|
+
@filters << instance_eval("lambda do |tag, time, record| #{conf[key]} end")
|
29
|
+
rescue Exception => e
|
30
|
+
raise Fluent::ConfigError, "#{key} #{conf[key]}\n" + e.to_s
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
if @filters.empty?
|
35
|
+
raise Fluent::ConfigError, "missing filters"
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def emit(tag, es, chain)
|
40
|
+
tag = handle_tag(tag)
|
41
|
+
es.each do |time, record|
|
42
|
+
result = filter_record(tag, time, record)
|
43
|
+
Fluent::Engine.emit(*result) if result
|
44
|
+
end
|
45
|
+
chain.next
|
46
|
+
end
|
47
|
+
|
48
|
+
def handle_tag(tag)
|
49
|
+
tag = tag.sub(@remove_tag_prefix, '') if @remove_tag_prefix
|
50
|
+
tag = tag.sub(@remove_tag_suffix, '') if @remove_tag_suffix
|
51
|
+
tag = tag.sub(/^\.*/, "#{@add_tag_prefix}.") if @add_tag_prefix
|
52
|
+
tag = tag.sub(/\.*$/, ".#{@add_tag_suffix}") if @add_tag_suffix
|
53
|
+
tag
|
54
|
+
end
|
55
|
+
|
56
|
+
def filter_record(tag, time, record)
|
57
|
+
@filters.each do |filter|
|
58
|
+
filter_result = filter.call(tag, time, record)
|
59
|
+
result = create_result(tag, time, record, filter_result) if filter_result
|
60
|
+
return result if result
|
61
|
+
end
|
62
|
+
nil
|
63
|
+
end
|
64
|
+
|
65
|
+
def create_result(tag, time, record, result)
|
66
|
+
result = [result] unless result.is_a?(Array)
|
67
|
+
|
68
|
+
result.each do |value|
|
69
|
+
tag = value if value.is_a?(String)
|
70
|
+
time = value if value.is_a?(Integer)
|
71
|
+
record = value if value.is_a?(Hash)
|
72
|
+
end
|
73
|
+
|
74
|
+
[tag, time, record]
|
75
|
+
end
|
76
|
+
|
77
|
+
end
|
data/test/helper.rb
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'bundler'
|
3
|
+
begin
|
4
|
+
Bundler.setup(:default, :development)
|
5
|
+
rescue Bundler::BundlerError => e
|
6
|
+
$stderr.puts e.message
|
7
|
+
$stderr.puts "Run `bundle install` to install missing gems"
|
8
|
+
exit e.status_code
|
9
|
+
end
|
10
|
+
require 'test/unit'
|
11
|
+
|
12
|
+
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
13
|
+
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
14
|
+
require 'fluent/test'
|
15
|
+
unless ENV.has_key?('VERBOSE')
|
16
|
+
nulllogger = Object.new
|
17
|
+
nulllogger.instance_eval {|obj|
|
18
|
+
def method_missing(method, *args)
|
19
|
+
# pass
|
20
|
+
end
|
21
|
+
}
|
22
|
+
$log = nulllogger
|
23
|
+
end
|
24
|
+
|
25
|
+
require 'fluent/plugin/out_eval_filter'
|
26
|
+
|
27
|
+
class Test::Unit::TestCase
|
28
|
+
end
|
@@ -0,0 +1,187 @@
|
|
1
|
+
require 'helper'
|
2
|
+
|
3
|
+
class EvalFilterTest < Test::Unit::TestCase
|
4
|
+
|
5
|
+
def setup
|
6
|
+
Fluent::Test.setup
|
7
|
+
end
|
8
|
+
|
9
|
+
def create_driver(conf, tag = 'test')
|
10
|
+
Fluent::Test::OutputTestDriver.new(Fluent::EvalFilterOutput, tag).configure(conf)
|
11
|
+
end
|
12
|
+
|
13
|
+
def test_configure
|
14
|
+
assert_raise(Fluent::ConfigError) do
|
15
|
+
create_driver('')
|
16
|
+
end
|
17
|
+
assert_raise(Fluent::ConfigError) do
|
18
|
+
create_driver(%[config1 @test = "\#{self.to_s}"])
|
19
|
+
end
|
20
|
+
assert_raise(Fluent::ConfigError) do
|
21
|
+
create_driver(%[filter1 "\#{tag}"])
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def test_remove_tag_prefix
|
26
|
+
d = create_driver(%[
|
27
|
+
remove_tag_prefix t1
|
28
|
+
filter1 tag
|
29
|
+
], 't1.t2.t3')
|
30
|
+
|
31
|
+
d.run { d.emit({}) }
|
32
|
+
|
33
|
+
emits = d.emits
|
34
|
+
assert_equal 1, emits.size
|
35
|
+
p emits[0]
|
36
|
+
assert_equal 't2.t3', emits[0][0]
|
37
|
+
end
|
38
|
+
|
39
|
+
def test_remove_tag_suffix
|
40
|
+
d = create_driver(%[
|
41
|
+
remove_tag_suffix t3
|
42
|
+
filter1 tag
|
43
|
+
], 't1.t2.t3')
|
44
|
+
|
45
|
+
d.run { d.emit({}) }
|
46
|
+
|
47
|
+
emits = d.emits
|
48
|
+
assert_equal 1, emits.size
|
49
|
+
p emits[0]
|
50
|
+
assert_equal 't1.t2', emits[0][0]
|
51
|
+
end
|
52
|
+
|
53
|
+
def test_add_tag_prefix
|
54
|
+
d = create_driver(%[
|
55
|
+
add_tag_prefix t0
|
56
|
+
filter1 tag
|
57
|
+
], 't1.t2.t3')
|
58
|
+
|
59
|
+
d.run { d.emit({}) }
|
60
|
+
|
61
|
+
emits = d.emits
|
62
|
+
assert_equal 1, emits.size
|
63
|
+
p emits[0]
|
64
|
+
assert_equal 't0.t1.t2.t3', emits[0][0]
|
65
|
+
end
|
66
|
+
|
67
|
+
def test_add_tag_suffix
|
68
|
+
d = create_driver(%[
|
69
|
+
add_tag_suffix t4
|
70
|
+
filter1 tag
|
71
|
+
], 't1.t2.t3')
|
72
|
+
|
73
|
+
d.run { d.emit({}) }
|
74
|
+
|
75
|
+
emits = d.emits
|
76
|
+
assert_equal 1, emits.size
|
77
|
+
p emits[0]
|
78
|
+
assert_equal 't1.t2.t3.t4', emits[0][0]
|
79
|
+
end
|
80
|
+
|
81
|
+
def test_handle_tag_all
|
82
|
+
d = create_driver(%[
|
83
|
+
remove_tag_prefix t1
|
84
|
+
remove_tag_suffix t3
|
85
|
+
add_tag_prefix t4
|
86
|
+
add_tag_suffix t5
|
87
|
+
filter1 tag
|
88
|
+
], 't1.t2.t3')
|
89
|
+
|
90
|
+
d.run { d.emit({}) }
|
91
|
+
|
92
|
+
emits = d.emits
|
93
|
+
assert_equal 1, emits.size
|
94
|
+
p emits[0]
|
95
|
+
assert_equal 't4.t2.t5', emits[0][0]
|
96
|
+
end
|
97
|
+
|
98
|
+
def test_drop_all_filter
|
99
|
+
d = create_driver(%[
|
100
|
+
filter1 nil
|
101
|
+
])
|
102
|
+
|
103
|
+
d.run { d.emit({}) }
|
104
|
+
|
105
|
+
emits = d.emits
|
106
|
+
p emits
|
107
|
+
assert_equal 0, emits.size
|
108
|
+
end
|
109
|
+
|
110
|
+
def test_modify_record_filter
|
111
|
+
d = create_driver(%[
|
112
|
+
filter1 record.merge!({'key' => 'value'})
|
113
|
+
])
|
114
|
+
|
115
|
+
d.run { d.emit({}) }
|
116
|
+
|
117
|
+
emits = d.emits
|
118
|
+
assert_equal 1, emits.size
|
119
|
+
p emits[0]
|
120
|
+
assert_equal 'test', emits[0][0]
|
121
|
+
assert_equal 1, emits[0][2].size
|
122
|
+
assert_equal true, emits[0][2].key?('key')
|
123
|
+
assert_equal 'value', emits[0][2]['key']
|
124
|
+
end
|
125
|
+
|
126
|
+
def test_replace_all_filter
|
127
|
+
d = create_driver(%[
|
128
|
+
filter1 nil
|
129
|
+
filter2 ['tag', 0, {'key' => 'value'}]
|
130
|
+
])
|
131
|
+
|
132
|
+
d.run { d.emit({}) }
|
133
|
+
|
134
|
+
emits = d.emits
|
135
|
+
assert_equal 1, emits.size
|
136
|
+
p emits[0]
|
137
|
+
assert_equal 'tag', emits[0][0]
|
138
|
+
assert_equal 0, emits[0][1]
|
139
|
+
assert_equal 1, emits[0][2].size
|
140
|
+
assert_equal true, emits[0][2].key?('key')
|
141
|
+
assert_equal 'value', emits[0][2]['key']
|
142
|
+
end
|
143
|
+
|
144
|
+
def test_conditional_filter
|
145
|
+
d = create_driver(%[
|
146
|
+
filter1 [['http', tag].join('.'), record] if /^http:/.match(record['url'])
|
147
|
+
filter2 (record['secure'] = true; [['https', tag].join('.'), record]) if /^https:/.match(record['url'])
|
148
|
+
])
|
149
|
+
|
150
|
+
d.run do
|
151
|
+
d.emit({'url' => 'http://example.com/'})
|
152
|
+
d.emit({'url' => 'https://example.com/'})
|
153
|
+
d.emit({'url' => 'ftp://example.com/'})
|
154
|
+
end
|
155
|
+
|
156
|
+
emits = d.emits
|
157
|
+
assert_equal 2, emits.size
|
158
|
+
p emits[0]
|
159
|
+
assert_equal 'http.test', emits[0][0]
|
160
|
+
assert_equal 1, emits[0][2].size
|
161
|
+
assert_equal true, emits[0][2].key?('url')
|
162
|
+
assert_equal 'http://example.com/', emits[0][2]['url']
|
163
|
+
p emits[1]
|
164
|
+
assert_equal 'https.test', emits[1][0]
|
165
|
+
assert_equal 2, emits[1][2].size
|
166
|
+
assert_equal true, emits[1][2].key?('url')
|
167
|
+
assert_equal 'https://example.com/', emits[1][2]['url']
|
168
|
+
assert_equal true, emits[1][2].key?('secure')
|
169
|
+
assert_equal true, emits[1][2]['secure']
|
170
|
+
end
|
171
|
+
|
172
|
+
def test_reference_to_an_instance_variable_filter
|
173
|
+
hostname = `hostname -s`.chomp
|
174
|
+
d = create_driver(%[
|
175
|
+
config1 @hostname = `hostname -s`.chomp
|
176
|
+
filter1 [tag, @hostname].join('.')
|
177
|
+
])
|
178
|
+
|
179
|
+
d.run { d.emit({}) }
|
180
|
+
|
181
|
+
emits = d.emits
|
182
|
+
assert_equal 1, emits.size
|
183
|
+
p emits[0]
|
184
|
+
assert_equal "test.#{hostname}", emits[0][0]
|
185
|
+
end
|
186
|
+
|
187
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-eval-filter
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yuzuki Masaru
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2014-05-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -46,12 +46,15 @@ extensions: []
|
|
46
46
|
extra_rdoc_files: []
|
47
47
|
files:
|
48
48
|
- ".gitignore"
|
49
|
+
- ".travis.yml"
|
49
50
|
- Gemfile
|
50
51
|
- LICENSE.txt
|
51
52
|
- README.md
|
52
53
|
- Rakefile
|
53
54
|
- fluent-plugin-eval-filter.gemspec
|
54
55
|
- lib/fluent/plugin/out_eval_filter.rb
|
56
|
+
- test/helper.rb
|
57
|
+
- test/plugin/test_out_eval_filter.rb
|
55
58
|
homepage: https://github.com/ephemeralsnow/fluent-plugin-eval-filter
|
56
59
|
licenses:
|
57
60
|
- Apache 2.0
|
@@ -72,8 +75,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
72
75
|
version: '0'
|
73
76
|
requirements: []
|
74
77
|
rubyforge_project:
|
75
|
-
rubygems_version: 2.2.
|
78
|
+
rubygems_version: 2.2.2
|
76
79
|
signing_key:
|
77
80
|
specification_version: 4
|
78
81
|
summary: Fluentd Output eval filter plugin.
|
79
|
-
test_files:
|
82
|
+
test_files:
|
83
|
+
- test/helper.rb
|
84
|
+
- test/plugin/test_out_eval_filter.rb
|