fluent-plugin-eval-filter 0.0.5 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +2 -1
- data/README.md +61 -0
- data/fluent-plugin-eval-filter.gemspec +2 -1
- data/lib/fluent/plugin/filter_eval.rb +68 -0
- data/lib/fluent/plugin/out_eval_filter.rb +18 -1
- data/test/helper.rb +1 -0
- data/test/plugin/test_filter_eval.rb +128 -0
- data/test/plugin/test_out_eval_filter.rb +20 -1
- metadata +20 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: e111ef217c550c8411a41a3d3f8955d0b101211c
|
|
4
|
+
data.tar.gz: 030da12335bab1ba7fe71c6287891c03127a90e1
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 420ab83d2f9be13c25f172210c98ebdc830bad7b2550e47ba7a68225011340d1247f4404a580f484cdeaa54572d656a9762e3fd30cd47da17fda334f59d7820f
|
|
7
|
+
data.tar.gz: e42b7f259334fc99085d6cc36a1d06a3e9f13773af31cbadf8b67efd090aaa0cd01b3671f2d27f238f988c8b66c0d634536e06db635a791c30195efd28b5e36e
|
data/.travis.yml
CHANGED
data/README.md
CHANGED
|
@@ -28,6 +28,67 @@ Or install it yourself as:
|
|
|
28
28
|
</match>
|
|
29
29
|
```
|
|
30
30
|
|
|
31
|
+
### require libraries
|
|
32
|
+
```
|
|
33
|
+
<match raw.apache.access>
|
|
34
|
+
type eval_filter
|
|
35
|
+
remove_tag_prefix raw
|
|
36
|
+
add_tag_prefix filtered
|
|
37
|
+
requires yaml # comma separated values
|
|
38
|
+
|
|
39
|
+
config1 @hostname = YAML.load({'hostname' => 'web01'})['hostname']
|
|
40
|
+
|
|
41
|
+
filter1 [[tag, @hostname].join('.'), time, record] if record['method'] == 'GET'
|
|
42
|
+
</match>
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
## Filter Plugin
|
|
46
|
+
|
|
47
|
+
Note that this filter version does not have rewrite tag functionality.
|
|
48
|
+
Should return [time, record].
|
|
49
|
+
|
|
50
|
+
## Configuration
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
### filter:
|
|
54
|
+
|
|
55
|
+
<filter **>
|
|
56
|
+
type eval
|
|
57
|
+
filter1 "[time, record] if record['status'] == '404'"
|
|
58
|
+
filter2 "[time, record] if record['status'] == 'POST'"
|
|
59
|
+
</filter>
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
### typecast(string to integer):
|
|
63
|
+
|
|
64
|
+
<filter **>
|
|
65
|
+
type eval
|
|
66
|
+
filter1 "record['status'] = record['status'].to_i; [time, record]"
|
|
67
|
+
</filter>
|
|
68
|
+
|
|
69
|
+
### modify record(add value):
|
|
70
|
+
|
|
71
|
+
<filter **>
|
|
72
|
+
type eval
|
|
73
|
+
filter1 "record['user_id'] = record['message'].split(':').last.to_i; [time, record]"
|
|
74
|
+
</filter>
|
|
75
|
+
|
|
76
|
+
#### input
|
|
77
|
+
{'status' => '301', 'message' => 'user_id:1'}
|
|
78
|
+
{'status' => '302', 'message' => 'user_id:2'}
|
|
79
|
+
{'status' => '404', 'message' => 'user_id:3'}
|
|
80
|
+
{'status' => '503', 'message' => 'user_id:4'}
|
|
81
|
+
{'status' => '401', 'message' => 'user_id:5'}
|
|
82
|
+
|
|
83
|
+
#### output
|
|
84
|
+
{'status' => '301', 'message' => 'user_id:1', 'user_id' => 1}
|
|
85
|
+
{'status' => '302', 'message' => 'user_id:2', 'user_id' => 2}
|
|
86
|
+
{'status' => '404', 'message' => 'user_id:3', 'user_id' => 3}
|
|
87
|
+
{'status' => '503', 'message' => 'user_id:4', 'user_id' => 4}
|
|
88
|
+
{'status' => '401', 'message' => 'user_id:5', 'user_id' => 5}
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
|
|
31
92
|
## Limitation
|
|
32
93
|
|
|
33
94
|
Can not be used expression substitution.
|
|
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
|
4
4
|
|
|
5
5
|
Gem::Specification.new do |spec|
|
|
6
6
|
spec.name = "fluent-plugin-eval-filter"
|
|
7
|
-
spec.version = "0.0
|
|
7
|
+
spec.version = "0.1.0"
|
|
8
8
|
spec.authors = ["Yuzuki Masaru"]
|
|
9
9
|
spec.email = ["ephemeralsnow@gmail.com"]
|
|
10
10
|
spec.description = %q{Fluentd Output eval filter plugin.}
|
|
@@ -19,4 +19,5 @@ Gem::Specification.new do |spec|
|
|
|
19
19
|
|
|
20
20
|
spec.add_dependency "fluentd", "~> 0"
|
|
21
21
|
spec.add_development_dependency "rake", "~> 0"
|
|
22
|
+
spec.add_development_dependency "test-unit", "~> 3.1.0"
|
|
22
23
|
end
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
module Fluent
|
|
2
|
+
class EvalFilter < Filter
|
|
3
|
+
Fluent::Plugin.register_filter('eval', self)
|
|
4
|
+
|
|
5
|
+
config_param :requires, :string, default: nil, :desc => "require libraries."
|
|
6
|
+
|
|
7
|
+
def initialize
|
|
8
|
+
super
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
def configure(conf)
|
|
12
|
+
super
|
|
13
|
+
|
|
14
|
+
if @requires
|
|
15
|
+
@requires.split(',').each do |lib|
|
|
16
|
+
begin
|
|
17
|
+
require lib
|
|
18
|
+
rescue Exception => e
|
|
19
|
+
raise Fluent::ConfigError, "\n#{e.message}\n#{e.backtrace.join("\n")}"
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
conf.keys.select { |key| key =~ /^config\d+$/ }.sort_by { |key| key.sub('config', '').to_i }.each do |key|
|
|
25
|
+
begin
|
|
26
|
+
instance_eval("#{conf[key]}")
|
|
27
|
+
rescue Exception => e
|
|
28
|
+
raise Fluent::ConfigError, "#{key} #{conf[key]}\n" + e.to_s
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
@filters = []
|
|
33
|
+
conf.keys.select { |key| key =~ /^filter\d+$/ }.sort_by { |key| key.sub('filter', '').to_i }.each do |key|
|
|
34
|
+
begin
|
|
35
|
+
@filters << instance_eval("lambda do |tag, time, record| #{conf[key]} end")
|
|
36
|
+
rescue Exception => e
|
|
37
|
+
raise Fluent::ConfigError, "#{key} #{conf[key]}\n" + e.to_s
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
if @filters.empty?
|
|
42
|
+
raise Fluent::ConfigError, "missing filters"
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def filter_stream(tag, es)
|
|
47
|
+
new_es = MultiEventStream.new
|
|
48
|
+
es.each { |time, record|
|
|
49
|
+
begin
|
|
50
|
+
filtered_record = filter_record(tag, time, record)
|
|
51
|
+
new_es.add(*filtered_record) if filtered_record
|
|
52
|
+
rescue => e
|
|
53
|
+
router.emit_error_event(tag, time, record, e)
|
|
54
|
+
end
|
|
55
|
+
}
|
|
56
|
+
new_es
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
private
|
|
60
|
+
def filter_record(tag, time, record)
|
|
61
|
+
@filters.each do |filter|
|
|
62
|
+
filter_results = filter.call(tag, time, record)
|
|
63
|
+
return filter_results if filter_results
|
|
64
|
+
end
|
|
65
|
+
nil
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
end
|
|
@@ -2,9 +2,26 @@ class Fluent::EvalFilterOutput < Fluent::Output
|
|
|
2
2
|
|
|
3
3
|
Fluent::Plugin.register_output('eval_filter', self)
|
|
4
4
|
|
|
5
|
+
config_param :requires, :string, default: nil, :desc => "require libraries."
|
|
6
|
+
|
|
7
|
+
# Define `router` method of v0.12 to support v0.10 or earlier
|
|
8
|
+
unless method_defined?(:router)
|
|
9
|
+
define_method("router") { Fluent::Engine }
|
|
10
|
+
end
|
|
11
|
+
|
|
5
12
|
def configure(conf)
|
|
6
13
|
super
|
|
7
14
|
|
|
15
|
+
if @requires
|
|
16
|
+
@requires.split(',').each do |lib|
|
|
17
|
+
begin
|
|
18
|
+
require lib
|
|
19
|
+
rescue Exception => e
|
|
20
|
+
raise Fluent::ConfigError, "\n#{e.message}\n#{e.backtrace.join("\n")}"
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
|
|
8
25
|
if remove_tag_prefix = conf['remove_tag_prefix']
|
|
9
26
|
@remove_tag_prefix = /^#{Regexp.escape(remove_tag_prefix)}\.*/
|
|
10
27
|
end
|
|
@@ -42,7 +59,7 @@ class Fluent::EvalFilterOutput < Fluent::Output
|
|
|
42
59
|
results = filter_record(tag, time, record)
|
|
43
60
|
if results
|
|
44
61
|
results.each do |result|
|
|
45
|
-
|
|
62
|
+
router.emit(*result)
|
|
46
63
|
end
|
|
47
64
|
end
|
|
48
65
|
end
|
data/test/helper.rb
CHANGED
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
require 'helper'
|
|
2
|
+
|
|
3
|
+
class EvalFilterTest < Test::Unit::TestCase
|
|
4
|
+
include Fluent
|
|
5
|
+
|
|
6
|
+
setup do
|
|
7
|
+
Fluent::Test.setup
|
|
8
|
+
@time = Fluent::Engine.now
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
def create_driver(conf = '')
|
|
12
|
+
Test::FilterTestDriver.new(EvalFilter).configure(conf)
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def filter(config, msgs)
|
|
16
|
+
d = create_driver(config)
|
|
17
|
+
d.run {
|
|
18
|
+
msgs.each {|msg|
|
|
19
|
+
d.filter(msg, @time) # Filterプラグインにメッセージを通す
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
filtered = d.filtered_as_array # 結果を受け取る. [tag, time, record]の配列
|
|
23
|
+
filtered
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
sub_test_case 'configure' do
|
|
27
|
+
test 'check default' do
|
|
28
|
+
config = %[
|
|
29
|
+
filter1 [time, record] if record['status'] == '404'
|
|
30
|
+
]
|
|
31
|
+
assert_nothing_raised {
|
|
32
|
+
create_driver(config)
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
assert_raise(Fluent::ConfigError) do
|
|
36
|
+
create_driver('')
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
sub_test_case 'filter stream' do
|
|
42
|
+
test 'filter' do
|
|
43
|
+
msgs = [
|
|
44
|
+
{'status' => '301', 'message' => 'message'},
|
|
45
|
+
{'status' => '302', 'message' => 'message'},
|
|
46
|
+
{'status' => '404', 'message' => 'message'},
|
|
47
|
+
{'status' => '503', 'message' => 'message'},
|
|
48
|
+
{'status' => '401', 'message' => 'message'}
|
|
49
|
+
]
|
|
50
|
+
config = %[
|
|
51
|
+
filter1 [time, record] if record['status'] == '404'
|
|
52
|
+
filter2 [time, record] if record['status'] == '503'
|
|
53
|
+
]
|
|
54
|
+
es = filter(config, msgs)
|
|
55
|
+
assert_equal(es.size, 2)
|
|
56
|
+
assert_equal(es[0][2]['status'], '404')
|
|
57
|
+
assert_equal(es[1][2]['status'], '503')
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
sub_test_case 'convert type' do
|
|
62
|
+
test 'to_i' do
|
|
63
|
+
msgs = [
|
|
64
|
+
{'status' => '301', 'message' => 'message'},
|
|
65
|
+
{'status' => '302', 'message' => 'message'},
|
|
66
|
+
{'status' => '404', 'message' => 'message'},
|
|
67
|
+
{'status' => '503', 'message' => 'message'},
|
|
68
|
+
{'status' => '401', 'message' => 'message'}
|
|
69
|
+
]
|
|
70
|
+
config = %[
|
|
71
|
+
filter1 record['status'] = record['status'].to_i; [time, record]
|
|
72
|
+
]
|
|
73
|
+
es = filter(config, msgs)
|
|
74
|
+
assert_equal(es.size, 5)
|
|
75
|
+
assert_equal(es[0][2]['status'], 301)
|
|
76
|
+
assert_equal(es[1][2]['status'], 302)
|
|
77
|
+
assert_equal(es[2][2]['status'], 404)
|
|
78
|
+
assert_equal(es[3][2]['status'], 503)
|
|
79
|
+
assert_equal(es[4][2]['status'], 401)
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
sub_test_case 'add value' do
|
|
84
|
+
test 'add user_id' do
|
|
85
|
+
msgs = [
|
|
86
|
+
{'status' => '301', 'message' => 'user_id:1'},
|
|
87
|
+
{'status' => '302', 'message' => 'user_id:2'},
|
|
88
|
+
{'status' => '404', 'message' => 'user_id:3'},
|
|
89
|
+
{'status' => '503', 'message' => 'user_id:4'},
|
|
90
|
+
{'status' => '401', 'message' => 'user_id:5'}
|
|
91
|
+
]
|
|
92
|
+
config = %[
|
|
93
|
+
filter1 record['user_id'] = record['message'].split(':').last.to_i; [time, record]
|
|
94
|
+
]
|
|
95
|
+
es = filter(config, msgs)
|
|
96
|
+
assert_equal(es.size, 5)
|
|
97
|
+
assert_equal(es[0][2]['user_id'], 1)
|
|
98
|
+
assert_equal(es[1][2]['user_id'], 2)
|
|
99
|
+
assert_equal(es[2][2]['user_id'], 3)
|
|
100
|
+
assert_equal(es[3][2]['user_id'], 4)
|
|
101
|
+
assert_equal(es[4][2]['user_id'], 5)
|
|
102
|
+
end
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
sub_test_case 'require libraries' do
|
|
106
|
+
test 'require yaml' do
|
|
107
|
+
config = %[
|
|
108
|
+
requires yaml
|
|
109
|
+
filter1 record.to_yaml; [time, record]
|
|
110
|
+
]
|
|
111
|
+
assert_nothing_raised {
|
|
112
|
+
create_driver(config)
|
|
113
|
+
es = filter(config, [{'key' => 'value'}])
|
|
114
|
+
assert_equal(es.size, 1)
|
|
115
|
+
}
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
test 'require error' do
|
|
119
|
+
config = %[
|
|
120
|
+
requires hoge
|
|
121
|
+
filter1 record.to_yaml; [time, record]
|
|
122
|
+
]
|
|
123
|
+
assert_raise(Fluent::ConfigError) do
|
|
124
|
+
create_driver(config)
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
end
|
|
128
|
+
end
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
require 'helper'
|
|
2
2
|
|
|
3
|
-
class
|
|
3
|
+
class EvalFilterOutputTest < Test::Unit::TestCase
|
|
4
4
|
|
|
5
5
|
def setup
|
|
6
6
|
Fluent::Test.setup
|
|
@@ -296,4 +296,23 @@ class EvalFilterTest < Test::Unit::TestCase
|
|
|
296
296
|
assert_equal 'test3', emits[2][2]['value']
|
|
297
297
|
end
|
|
298
298
|
|
|
299
|
+
def test_require_libraries
|
|
300
|
+
d = create_driver(%[
|
|
301
|
+
requires yaml
|
|
302
|
+
filter1 record.to_yaml; ['tag', 0, record]
|
|
303
|
+
])
|
|
304
|
+
assert_nothing_raised {
|
|
305
|
+
d.run { d.emit({'key' => 'value'}) }
|
|
306
|
+
}
|
|
307
|
+
end
|
|
308
|
+
|
|
309
|
+
def test_require_error
|
|
310
|
+
assert_raise(Fluent::ConfigError) do
|
|
311
|
+
d = create_driver(%[
|
|
312
|
+
requires hoge
|
|
313
|
+
filter1 record.to_yaml; ['tag', 0, record]
|
|
314
|
+
])
|
|
315
|
+
end
|
|
316
|
+
end
|
|
317
|
+
|
|
299
318
|
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: fluent-plugin-eval-filter
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.0
|
|
4
|
+
version: 0.1.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Yuzuki Masaru
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2016-03-26 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: fluentd
|
|
@@ -38,6 +38,20 @@ dependencies:
|
|
|
38
38
|
- - "~>"
|
|
39
39
|
- !ruby/object:Gem::Version
|
|
40
40
|
version: '0'
|
|
41
|
+
- !ruby/object:Gem::Dependency
|
|
42
|
+
name: test-unit
|
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
|
44
|
+
requirements:
|
|
45
|
+
- - "~>"
|
|
46
|
+
- !ruby/object:Gem::Version
|
|
47
|
+
version: 3.1.0
|
|
48
|
+
type: :development
|
|
49
|
+
prerelease: false
|
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
51
|
+
requirements:
|
|
52
|
+
- - "~>"
|
|
53
|
+
- !ruby/object:Gem::Version
|
|
54
|
+
version: 3.1.0
|
|
41
55
|
description: Fluentd Output eval filter plugin.
|
|
42
56
|
email:
|
|
43
57
|
- ephemeralsnow@gmail.com
|
|
@@ -52,8 +66,10 @@ files:
|
|
|
52
66
|
- README.md
|
|
53
67
|
- Rakefile
|
|
54
68
|
- fluent-plugin-eval-filter.gemspec
|
|
69
|
+
- lib/fluent/plugin/filter_eval.rb
|
|
55
70
|
- lib/fluent/plugin/out_eval_filter.rb
|
|
56
71
|
- test/helper.rb
|
|
72
|
+
- test/plugin/test_filter_eval.rb
|
|
57
73
|
- test/plugin/test_out_eval_filter.rb
|
|
58
74
|
homepage: https://github.com/ephemeralsnow/fluent-plugin-eval-filter
|
|
59
75
|
licenses:
|
|
@@ -75,10 +91,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
75
91
|
version: '0'
|
|
76
92
|
requirements: []
|
|
77
93
|
rubyforge_project:
|
|
78
|
-
rubygems_version: 2.
|
|
94
|
+
rubygems_version: 2.5.1
|
|
79
95
|
signing_key:
|
|
80
96
|
specification_version: 4
|
|
81
97
|
summary: Fluentd Output eval filter plugin.
|
|
82
98
|
test_files:
|
|
83
99
|
- test/helper.rb
|
|
100
|
+
- test/plugin/test_filter_eval.rb
|
|
84
101
|
- test/plugin/test_out_eval_filter.rb
|