fluent-plugin-eval-filter 0.0.2 → 0.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: a257fc65a23332bc41401a9778f14a8bf68a0d18
4
- data.tar.gz: 22336f20a60b02d204341b61012c9efb5f3d08b5
3
+ metadata.gz: 6765bfbe982ee8ab3f79203bc16c41905c367373
4
+ data.tar.gz: cc5f7b09aae48c79710766f835fe5cf419800209
5
5
  SHA512:
6
- metadata.gz: 70ce05adece616793f764b662ccd87a2799f4762defed8fbc5ffbf604ca8d481214e698b2c2c491c4350fd20032b3435aee36c4106e9c818a00827cd44f57b45
7
- data.tar.gz: 75f31b4b744bff8f512e96d026f0fea6cf94514822741574fbcc0315cc2fa1e1cee245ca277dcb6011451aac8ef4d914dfdd019501898d7ed3af4b3ceb9031d4
6
+ metadata.gz: 05b3f8c2ea08ec5b9aa6865ce8fcee0bb1449d1ffbcbbfbc75d63127396e6b673cda42e53009a804ca47bc50681d61fdc61ca240debbf78df520bac6b3cabbdc
7
+ data.tar.gz: 70701cb426b287d3494faf4803a0b50306387f8a2a768de03b17e4cea34d28c5d2bc4d44273f695387305487d8469e49bc737eb5d79a288f52dde1b45ec1d7fc
data/.gitignore CHANGED
@@ -15,3 +15,4 @@ spec/reports
15
15
  test/tmp
16
16
  test/version_tmp
17
17
  tmp
18
+ vendor
data/.travis.yml ADDED
@@ -0,0 +1,8 @@
1
+ language: ruby
2
+ rvm:
3
+ - 2.1.0
4
+ - 2.0.0
5
+ - 1.9.3
6
+ - 1.9.2
7
+ before_install:
8
+ - gem update --remote bundler
data/README.md CHANGED
@@ -1,4 +1,4 @@
1
- # fluent-plugin-eval-filter
1
+ # fluent-plugin-eval-filter, a plugin for [Fluentd](http://fluentd.org) [![Build Status](https://travis-ci.org/ephemeralsnow/fluent-plugin-eval-filter.png?branch=master)](https://travis-ci.org/ephemeralsnow/fluent-plugin-eval-filter) [![Code Climate](https://codeclimate.com/github/ephemeralsnow/fluent-plugin-eval-filter.png)](https://codeclimate.com/github/ephemeralsnow/fluent-plugin-eval-filter)
2
2
 
3
3
  ## Installation
4
4
 
@@ -28,6 +28,20 @@ Or install it yourself as:
28
28
  </match>
29
29
  ```
30
30
 
31
+ ## Limitation
32
+
33
+ Can not be used expression substitution.
34
+ ```
35
+ <match raw.apache.access>
36
+ filter1 "#{tag}"
37
+ </match>
38
+ ```
39
+
40
+ '#' Is interpreted as the beginning of a comment.
41
+ ```
42
+ filter1 #=> "\""
43
+ ```
44
+
31
45
  ## Contributing
32
46
 
33
47
  1. Fork it
data/Rakefile CHANGED
@@ -1 +1,10 @@
1
- require "bundler/gem_tasks"
1
+ require 'bundler/gem_tasks'
2
+ require 'rake/testtask'
3
+
4
+ Rake::TestTask.new(:test) do |test|
5
+ test.libs << 'lib' << 'test'
6
+ test.pattern = 'test/**/test_*.rb'
7
+ test.verbose = true
8
+ end
9
+
10
+ task default: :test
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
4
 
5
5
  Gem::Specification.new do |spec|
6
6
  spec.name = "fluent-plugin-eval-filter"
7
- spec.version = "0.0.2"
7
+ spec.version = "0.0.3"
8
8
  spec.authors = ["Yuzuki Masaru"]
9
9
  spec.email = ["ephemeralsnow@gmail.com"]
10
10
  spec.description = %q{Fluentd Output eval filter plugin.}
@@ -1,50 +1,77 @@
1
- class Fluent::EvalFilterOutput < Fluent::Output
2
- include Fluent::HandleTagNameMixin
3
-
4
- Fluent::Plugin.register_output('eval_filter', self)
5
-
6
- def configure(conf)
7
- super
8
-
9
- conf.keys.select { |key| key =~ /^config\d+$/ }.sort_by { |key| key.sub('config', '').to_i }.each do |key|
10
- instance_eval("#{conf[key]}")
11
- end
12
-
13
- @filters = []
14
- conf.keys.select { |key| key =~ /^filter\d+$/ }.sort_by { |key| key.sub('filter', '').to_i }.each do |key|
15
- @filters << instance_eval("lambda do |tag, time, record| #{conf[key]} end")
16
- end
17
- end
18
-
19
- def emit(tag, es, chain)
20
- es.each do |time, record|
21
- result = filter_record(tag.clone, time, record)
22
- Fluent::Engine.emit(*result) if result
23
- end
24
- chain.next
25
- end
26
-
27
- def filter_record(tag, time, record)
28
- super
29
-
30
- @filters.each do |filter|
31
- filter_result = filter.call(tag, time, record)
32
- result = create_result(tag, time, record, filter_result) if filter_result
33
- return result if result
34
- end
35
- nil
36
- end
37
-
38
- def create_result(tag, time, record, result)
39
- result = [result] unless result.is_a?(Array)
40
-
41
- result.each do |value|
42
- tag = value if value.is_a?(String)
43
- time = value if value.is_a?(Integer)
44
- record = value if value.is_a?(Hash)
45
- end
46
-
47
- [tag, time, record]
48
- end
49
-
50
- end
1
+ class Fluent::EvalFilterOutput < Fluent::Output
2
+
3
+ Fluent::Plugin.register_output('eval_filter', self)
4
+
5
+ def configure(conf)
6
+ super
7
+
8
+ if remove_tag_prefix = conf['remove_tag_prefix']
9
+ @remove_tag_prefix = /^#{Regexp.escape(remove_tag_prefix)}\.*/
10
+ end
11
+ if remove_tag_suffix = conf['remove_tag_suffix']
12
+ @remove_tag_suffix = /\.*#{Regexp.escape(remove_tag_suffix)}$/
13
+ end
14
+ @add_tag_prefix = conf['add_tag_prefix']
15
+ @add_tag_suffix = conf['add_tag_suffix']
16
+
17
+ conf.keys.select { |key| key =~ /^config\d+$/ }.sort_by { |key| key.sub('config', '').to_i }.each do |key|
18
+ begin
19
+ instance_eval("#{conf[key]}")
20
+ rescue Exception => e
21
+ raise Fluent::ConfigError, "#{key} #{conf[key]}\n" + e.to_s
22
+ end
23
+ end
24
+
25
+ @filters = []
26
+ conf.keys.select { |key| key =~ /^filter\d+$/ }.sort_by { |key| key.sub('filter', '').to_i }.each do |key|
27
+ begin
28
+ @filters << instance_eval("lambda do |tag, time, record| #{conf[key]} end")
29
+ rescue Exception => e
30
+ raise Fluent::ConfigError, "#{key} #{conf[key]}\n" + e.to_s
31
+ end
32
+ end
33
+
34
+ if @filters.empty?
35
+ raise Fluent::ConfigError, "missing filters"
36
+ end
37
+ end
38
+
39
+ def emit(tag, es, chain)
40
+ tag = handle_tag(tag)
41
+ es.each do |time, record|
42
+ result = filter_record(tag, time, record)
43
+ Fluent::Engine.emit(*result) if result
44
+ end
45
+ chain.next
46
+ end
47
+
48
+ def handle_tag(tag)
49
+ tag = tag.sub(@remove_tag_prefix, '') if @remove_tag_prefix
50
+ tag = tag.sub(@remove_tag_suffix, '') if @remove_tag_suffix
51
+ tag = tag.sub(/^\.*/, "#{@add_tag_prefix}.") if @add_tag_prefix
52
+ tag = tag.sub(/\.*$/, ".#{@add_tag_suffix}") if @add_tag_suffix
53
+ tag
54
+ end
55
+
56
+ def filter_record(tag, time, record)
57
+ @filters.each do |filter|
58
+ filter_result = filter.call(tag, time, record)
59
+ result = create_result(tag, time, record, filter_result) if filter_result
60
+ return result if result
61
+ end
62
+ nil
63
+ end
64
+
65
+ def create_result(tag, time, record, result)
66
+ result = [result] unless result.is_a?(Array)
67
+
68
+ result.each do |value|
69
+ tag = value if value.is_a?(String)
70
+ time = value if value.is_a?(Integer)
71
+ record = value if value.is_a?(Hash)
72
+ end
73
+
74
+ [tag, time, record]
75
+ end
76
+
77
+ end
data/test/helper.rb ADDED
@@ -0,0 +1,28 @@
1
+ require 'rubygems'
2
+ require 'bundler'
3
+ begin
4
+ Bundler.setup(:default, :development)
5
+ rescue Bundler::BundlerError => e
6
+ $stderr.puts e.message
7
+ $stderr.puts "Run `bundle install` to install missing gems"
8
+ exit e.status_code
9
+ end
10
+ require 'test/unit'
11
+
12
+ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
13
+ $LOAD_PATH.unshift(File.dirname(__FILE__))
14
+ require 'fluent/test'
15
+ unless ENV.has_key?('VERBOSE')
16
+ nulllogger = Object.new
17
+ nulllogger.instance_eval {|obj|
18
+ def method_missing(method, *args)
19
+ # pass
20
+ end
21
+ }
22
+ $log = nulllogger
23
+ end
24
+
25
+ require 'fluent/plugin/out_eval_filter'
26
+
27
+ class Test::Unit::TestCase
28
+ end
@@ -0,0 +1,187 @@
1
+ require 'helper'
2
+
3
+ class EvalFilterTest < Test::Unit::TestCase
4
+
5
+ def setup
6
+ Fluent::Test.setup
7
+ end
8
+
9
+ def create_driver(conf, tag = 'test')
10
+ Fluent::Test::OutputTestDriver.new(Fluent::EvalFilterOutput, tag).configure(conf)
11
+ end
12
+
13
+ def test_configure
14
+ assert_raise(Fluent::ConfigError) do
15
+ create_driver('')
16
+ end
17
+ assert_raise(Fluent::ConfigError) do
18
+ create_driver(%[config1 @test = "\#{self.to_s}"])
19
+ end
20
+ assert_raise(Fluent::ConfigError) do
21
+ create_driver(%[filter1 "\#{tag}"])
22
+ end
23
+ end
24
+
25
+ def test_remove_tag_prefix
26
+ d = create_driver(%[
27
+ remove_tag_prefix t1
28
+ filter1 tag
29
+ ], 't1.t2.t3')
30
+
31
+ d.run { d.emit({}) }
32
+
33
+ emits = d.emits
34
+ assert_equal 1, emits.size
35
+ p emits[0]
36
+ assert_equal 't2.t3', emits[0][0]
37
+ end
38
+
39
+ def test_remove_tag_suffix
40
+ d = create_driver(%[
41
+ remove_tag_suffix t3
42
+ filter1 tag
43
+ ], 't1.t2.t3')
44
+
45
+ d.run { d.emit({}) }
46
+
47
+ emits = d.emits
48
+ assert_equal 1, emits.size
49
+ p emits[0]
50
+ assert_equal 't1.t2', emits[0][0]
51
+ end
52
+
53
+ def test_add_tag_prefix
54
+ d = create_driver(%[
55
+ add_tag_prefix t0
56
+ filter1 tag
57
+ ], 't1.t2.t3')
58
+
59
+ d.run { d.emit({}) }
60
+
61
+ emits = d.emits
62
+ assert_equal 1, emits.size
63
+ p emits[0]
64
+ assert_equal 't0.t1.t2.t3', emits[0][0]
65
+ end
66
+
67
+ def test_add_tag_suffix
68
+ d = create_driver(%[
69
+ add_tag_suffix t4
70
+ filter1 tag
71
+ ], 't1.t2.t3')
72
+
73
+ d.run { d.emit({}) }
74
+
75
+ emits = d.emits
76
+ assert_equal 1, emits.size
77
+ p emits[0]
78
+ assert_equal 't1.t2.t3.t4', emits[0][0]
79
+ end
80
+
81
+ def test_handle_tag_all
82
+ d = create_driver(%[
83
+ remove_tag_prefix t1
84
+ remove_tag_suffix t3
85
+ add_tag_prefix t4
86
+ add_tag_suffix t5
87
+ filter1 tag
88
+ ], 't1.t2.t3')
89
+
90
+ d.run { d.emit({}) }
91
+
92
+ emits = d.emits
93
+ assert_equal 1, emits.size
94
+ p emits[0]
95
+ assert_equal 't4.t2.t5', emits[0][0]
96
+ end
97
+
98
+ def test_drop_all_filter
99
+ d = create_driver(%[
100
+ filter1 nil
101
+ ])
102
+
103
+ d.run { d.emit({}) }
104
+
105
+ emits = d.emits
106
+ p emits
107
+ assert_equal 0, emits.size
108
+ end
109
+
110
+ def test_modify_record_filter
111
+ d = create_driver(%[
112
+ filter1 record.merge!({'key' => 'value'})
113
+ ])
114
+
115
+ d.run { d.emit({}) }
116
+
117
+ emits = d.emits
118
+ assert_equal 1, emits.size
119
+ p emits[0]
120
+ assert_equal 'test', emits[0][0]
121
+ assert_equal 1, emits[0][2].size
122
+ assert_equal true, emits[0][2].key?('key')
123
+ assert_equal 'value', emits[0][2]['key']
124
+ end
125
+
126
+ def test_replace_all_filter
127
+ d = create_driver(%[
128
+ filter1 nil
129
+ filter2 ['tag', 0, {'key' => 'value'}]
130
+ ])
131
+
132
+ d.run { d.emit({}) }
133
+
134
+ emits = d.emits
135
+ assert_equal 1, emits.size
136
+ p emits[0]
137
+ assert_equal 'tag', emits[0][0]
138
+ assert_equal 0, emits[0][1]
139
+ assert_equal 1, emits[0][2].size
140
+ assert_equal true, emits[0][2].key?('key')
141
+ assert_equal 'value', emits[0][2]['key']
142
+ end
143
+
144
+ def test_conditional_filter
145
+ d = create_driver(%[
146
+ filter1 [['http', tag].join('.'), record] if /^http:/.match(record['url'])
147
+ filter2 (record['secure'] = true; [['https', tag].join('.'), record]) if /^https:/.match(record['url'])
148
+ ])
149
+
150
+ d.run do
151
+ d.emit({'url' => 'http://example.com/'})
152
+ d.emit({'url' => 'https://example.com/'})
153
+ d.emit({'url' => 'ftp://example.com/'})
154
+ end
155
+
156
+ emits = d.emits
157
+ assert_equal 2, emits.size
158
+ p emits[0]
159
+ assert_equal 'http.test', emits[0][0]
160
+ assert_equal 1, emits[0][2].size
161
+ assert_equal true, emits[0][2].key?('url')
162
+ assert_equal 'http://example.com/', emits[0][2]['url']
163
+ p emits[1]
164
+ assert_equal 'https.test', emits[1][0]
165
+ assert_equal 2, emits[1][2].size
166
+ assert_equal true, emits[1][2].key?('url')
167
+ assert_equal 'https://example.com/', emits[1][2]['url']
168
+ assert_equal true, emits[1][2].key?('secure')
169
+ assert_equal true, emits[1][2]['secure']
170
+ end
171
+
172
+ def test_reference_to_an_instance_variable_filter
173
+ hostname = `hostname -s`.chomp
174
+ d = create_driver(%[
175
+ config1 @hostname = `hostname -s`.chomp
176
+ filter1 [tag, @hostname].join('.')
177
+ ])
178
+
179
+ d.run { d.emit({}) }
180
+
181
+ emits = d.emits
182
+ assert_equal 1, emits.size
183
+ p emits[0]
184
+ assert_equal "test.#{hostname}", emits[0][0]
185
+ end
186
+
187
+ end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-eval-filter
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.2
4
+ version: 0.0.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Yuzuki Masaru
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2013-12-27 00:00:00.000000000 Z
11
+ date: 2014-05-23 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
@@ -46,12 +46,15 @@ extensions: []
46
46
  extra_rdoc_files: []
47
47
  files:
48
48
  - ".gitignore"
49
+ - ".travis.yml"
49
50
  - Gemfile
50
51
  - LICENSE.txt
51
52
  - README.md
52
53
  - Rakefile
53
54
  - fluent-plugin-eval-filter.gemspec
54
55
  - lib/fluent/plugin/out_eval_filter.rb
56
+ - test/helper.rb
57
+ - test/plugin/test_out_eval_filter.rb
55
58
  homepage: https://github.com/ephemeralsnow/fluent-plugin-eval-filter
56
59
  licenses:
57
60
  - Apache 2.0
@@ -72,8 +75,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
72
75
  version: '0'
73
76
  requirements: []
74
77
  rubyforge_project:
75
- rubygems_version: 2.2.0
78
+ rubygems_version: 2.2.2
76
79
  signing_key:
77
80
  specification_version: 4
78
81
  summary: Fluentd Output eval filter plugin.
79
- test_files: []
82
+ test_files:
83
+ - test/helper.rb
84
+ - test/plugin/test_out_eval_filter.rb