fluent-plugin-fields-parser 0.1.1 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: a4469d964d78e15cc78e179a61f24e0ada3e49a2
4
+ data.tar.gz: 8f557a228d09be7eea53329b262094694a4274ca
5
+ SHA512:
6
+ metadata.gz: c1da9beeebfa4a08ba0be782b15423cd1f0857d245c98698b1f3907206c23113110054deab81ece237d22944bd7d2a3272b9370b9de39e9513afc9828a64032d
7
+ data.tar.gz: 6f27be00797bf5d196006a95ed7503246de335fe564a8f51250fd17d5bd88b8225064ef86e22ec115a9d3a187cec7c5608281069e8e96121fcebb4421f5b11c4
@@ -0,0 +1,6 @@
1
+ language: ruby
2
+ rvm:
3
+ - 2.4.2
4
+ - 2.3.5
5
+ - 2.2
6
+ - 2.1
data/Gemfile ADDED
@@ -0,0 +1,2 @@
1
+ source 'https://rubygems.org'
2
+ gemspec
data/README.md CHANGED
@@ -1,5 +1,4 @@
1
- fluent-plugin-fields-parser
2
- ===========================
1
+ # fluent-plugin-fields-parser [![Build Status](https://travis-ci.org/tomas-zemres/fluent-plugin-fields-parser.png)](https://travis-ci.org/tomas-zemres/fluent-plugin-fields-parser)
3
2
 
4
3
  Fluent output filter plugin for parsing key/value fields in records
5
4
  based on <key>=<value> pattern.
data/Rakefile CHANGED
@@ -11,4 +11,4 @@ Rake::TestTask.new(:test) do |test|
11
11
  test.verbose = true
12
12
  end
13
13
 
14
- task :default => [:build]
14
+ task :default => :test
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.1.1
1
+ 0.1.2
@@ -17,8 +17,9 @@ Gem::Specification.new do |gem|
17
17
  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
18
18
  gem.require_paths = ['lib']
19
19
 
20
- gem.add_dependency "fluentd"
20
+ gem.add_dependency "fluentd", '>= 0.14.0'
21
21
  gem.add_dependency "logfmt"
22
22
  gem.add_development_dependency "rake"
23
+ gem.add_development_dependency 'test-unit'
23
24
  end
24
25
 
@@ -1,9 +1,12 @@
1
1
  require "logfmt"
2
+ require "fluent/plugin/output"
2
3
 
3
- module Fluent
4
- class OutputFieldsParser < Fluent::Output
4
+ module Fluent::Plugin
5
+ class OutputFieldsParser < Output
5
6
  Fluent::Plugin.register_output('fields_parser', self)
6
7
 
8
+ helpers :event_emitter
9
+
7
10
  config_param :remove_tag_prefix, :string, :default => nil
8
11
  config_param :add_tag_prefix, :string, :default => nil
9
12
  config_param :parse_key, :string, :default => 'message'
@@ -16,12 +19,11 @@ module Fluent
16
19
  @compiled_pattern ||= Regexp.new(pattern)
17
20
  end
18
21
 
19
- def emit(tag, es, chain)
22
+ def process(tag, es)
20
23
  tag = update_tag(tag)
21
24
  es.each { |time, record|
22
- Engine.emit(tag, time, parse_fields(record))
25
+ router.emit(tag, time, parse_fields(record))
23
26
  }
24
- chain.next
25
27
  end
26
28
 
27
29
  def update_tag(tag)
@@ -1,4 +1,5 @@
1
1
  require 'fluent/test'
2
+ require 'fluent/test/driver/output'
2
3
  require 'fluent/plugin/out_fields_parser'
3
4
 
4
5
  class FieldsParserOutputTest < Test::Unit::TestCase
@@ -6,24 +7,24 @@ class FieldsParserOutputTest < Test::Unit::TestCase
6
7
  Fluent::Test.setup
7
8
  end
8
9
 
9
- def create_driver(conf='', tag='orig.test.tag')
10
- Fluent::Test::OutputTestDriver.new(Fluent::OutputFieldsParser, tag).configure(conf)
10
+ def create_driver(conf='')
11
+ Fluent::Test::Driver::Output.new(Fluent::Plugin::OutputFieldsParser).configure(conf)
11
12
  end
12
13
 
13
14
  def test_config_defaults
14
15
  d = create_driver()
15
16
 
16
17
  orig_message = %{parse this num=-56.7 tok=abc%25 null=}
17
- d.run do
18
- d.emit({
18
+ d.run(default_tag: 'orig.test.tag') do
19
+ d.feed({
19
20
  'message' => orig_message,
20
21
  'other_key' => %{ test2 a=b },
21
22
  })
22
23
  end
23
24
 
24
- emits = d.emits
25
- assert_equal 1, emits.size
26
- assert_equal "orig.test.tag", emits[0][0]
25
+ events = d.events
26
+ assert_equal 1, events.size
27
+ assert_equal "orig.test.tag", events[0][0]
27
28
  assert_equal(
28
29
  {
29
30
  'message' => orig_message,
@@ -32,7 +33,7 @@ class FieldsParserOutputTest < Test::Unit::TestCase
32
33
  'tok' => 'abc%25',
33
34
  'null' => '',
34
35
  },
35
- emits[0][2]
36
+ events[0][2]
36
37
  )
37
38
  end
38
39
 
@@ -40,15 +41,15 @@ class FieldsParserOutputTest < Test::Unit::TestCase
40
41
  d = create_driver()
41
42
 
42
43
  orig_message = %{blax dq="asd ' asd +3" sq='as " s " 4' s=yu 6}
43
- d.run do
44
- d.emit({
44
+ d.run(default_tag: 'orig.test.tag') do
45
+ d.feed({
45
46
  'message' => orig_message,
46
47
  })
47
48
  end
48
49
 
49
- emits = d.emits
50
- assert_equal 1, emits.size
51
- assert_equal "orig.test.tag", emits[0][0]
50
+ events = d.events
51
+ assert_equal 1, events.size
52
+ assert_equal "orig.test.tag", events[0][0]
52
53
  assert_equal(
53
54
  {
54
55
  'message' => orig_message,
@@ -56,23 +57,23 @@ class FieldsParserOutputTest < Test::Unit::TestCase
56
57
  'sq' => 'as " s " 4',
57
58
  's' => 'yu'
58
59
  },
59
- emits[0][2]
60
+ events[0][2]
60
61
  )
61
62
  end
62
63
 
63
64
  def test_parsed_key_is_missing
64
65
  d = create_driver()
65
66
 
66
- d.run do
67
- d.emit({})
67
+ d.run(default_tag: 'orig.test.tag') do
68
+ d.feed({})
68
69
  end
69
70
 
70
- emits = d.emits
71
- assert_equal 1, emits.size
72
- assert_equal "orig.test.tag", emits[0][0]
71
+ events = d.events
72
+ assert_equal 1, events.size
73
+ assert_equal "orig.test.tag", events[0][0]
73
74
  assert_equal(
74
75
  {},
75
- emits[0][2]
76
+ events[0][2]
76
77
  )
77
78
  end
78
79
 
@@ -80,20 +81,20 @@ class FieldsParserOutputTest < Test::Unit::TestCase
80
81
  d = create_driver()
81
82
 
82
83
  orig_message = %{mock a=77 message=blax a=999 e=5}
83
- d.run do
84
- d.emit({'message' => orig_message, 'e' => nil })
84
+ d.run(default_tag: 'orig.test.tag') do
85
+ d.feed({'message' => orig_message, 'e' => nil })
85
86
  end
86
87
 
87
- emits = d.emits
88
- assert_equal 1, emits.size
89
- assert_equal "orig.test.tag", emits[0][0]
88
+ events = d.events
89
+ assert_equal 1, events.size
90
+ assert_equal "orig.test.tag", events[0][0]
90
91
  assert_equal(
91
92
  {
92
93
  'message' => orig_message,
93
94
  'a' => '77',
94
95
  'e' => nil,
95
96
  },
96
- emits[0][2]
97
+ events[0][2]
97
98
  )
98
99
  end
99
100
 
@@ -103,80 +104,80 @@ class FieldsParserOutputTest < Test::Unit::TestCase
103
104
  add_tag_prefix new
104
105
  })
105
106
 
106
- d.run do
107
- d.emit({ message => 'abc' })
107
+ d.run(default_tag: 'orig.test.tag') do
108
+ d.feed({ 'message' => 'abc' })
108
109
  end
109
110
 
110
- emits = d.emits
111
- assert_equal 1, emits.size
112
- assert_equal "new.test.tag", emits[0][0]
111
+ events = d.events
112
+ assert_equal 1, events.size
113
+ assert_equal "new.test.tag", events[0][0]
113
114
 
114
115
  d = create_driver(%{
115
116
  remove_tag_prefix orig
116
117
  add_tag_prefix new
117
- }, tag=nil)
118
+ })
118
119
 
119
- d.run do
120
- d.emit({ message => 'abc' })
120
+ d.run(default_tag: '') do
121
+ d.feed({ 'message' => 'abc' })
121
122
  end
122
123
 
123
- emits = d.emits
124
- assert_equal 1, emits.size
125
- assert_equal "new", emits[0][0]
124
+ events = d.events
125
+ assert_equal 1, events.size
126
+ assert_equal "new", events[0][0]
126
127
 
127
128
  d = create_driver(%{
128
129
  remove_tag_prefix orig
129
130
  add_tag_prefix new
130
- }, tag='original')
131
+ })
131
132
 
132
- d.run do
133
- d.emit({ message => 'abc' })
133
+ d.run(default_tag: 'original') do
134
+ d.feed({ 'message' => 'abc' })
134
135
  end
135
136
 
136
- emits = d.emits
137
- assert_equal 1, emits.size
138
- assert_equal "new.original", emits[0][0]
137
+ events = d.events
138
+ assert_equal 1, events.size
139
+ assert_equal "new.original", events[0][0]
139
140
 
140
141
  d = create_driver(%{
141
142
  remove_tag_prefix orig
142
143
  add_tag_prefix new
143
- }, tag='orig')
144
+ })
144
145
 
145
- d.run do
146
- d.emit({ message => 'abc' })
146
+ d.run(default_tag: 'orig') do
147
+ d.feed({ 'message' => 'abc' })
147
148
  end
148
149
 
149
- emits = d.emits
150
- assert_equal 1, emits.size
151
- assert_equal "new", emits[0][0]
150
+ events = d.events
151
+ assert_equal 1, events.size
152
+ assert_equal "new", events[0][0]
152
153
  end
153
154
 
154
155
  def test_parse_key
155
156
  d = create_driver('parse_key custom_key')
156
157
 
157
- d.run do
158
- d.emit({
158
+ d.run(default_tag: 'orig.test.tag') do
159
+ d.feed({
159
160
  'message' => %{ test2 c=d },
160
161
  'custom_key' => %{ test2 a=b },
161
162
  })
162
- d.emit({})
163
+ d.feed({})
163
164
  end
164
165
 
165
- emits = d.emits
166
- assert_equal 2, emits.size
167
- assert_equal "orig.test.tag", emits[0][0]
166
+ events = d.events
167
+ assert_equal 2, events.size
168
+ assert_equal "orig.test.tag", events[0][0]
168
169
  assert_equal(
169
170
  {
170
171
  'message' => %{ test2 c=d },
171
172
  'custom_key' => %{ test2 a=b },
172
173
  'a' => 'b'
173
174
  },
174
- emits[0][2]
175
+ events[0][2]
175
176
  )
176
177
  assert_equal(
177
178
  {
178
179
  },
179
- emits[1][2]
180
+ events[1][2]
180
181
  )
181
182
  end
182
183
 
@@ -184,13 +185,13 @@ class FieldsParserOutputTest < Test::Unit::TestCase
184
185
  d = create_driver("fields_key output-key")
185
186
 
186
187
  orig_message = %{parse this num=-56.7 tok=abc%25 message=a+b}
187
- d.run do
188
- d.emit({'message' => orig_message})
188
+ d.run(default_tag: 'orig.test.tag') do
189
+ d.feed({'message' => orig_message})
189
190
  end
190
191
 
191
- emits = d.emits
192
- assert_equal 1, emits.size
193
- assert_equal "orig.test.tag", emits[0][0]
192
+ events = d.events
193
+ assert_equal 1, events.size
194
+ assert_equal "orig.test.tag", events[0][0]
194
195
  assert_equal(
195
196
  {
196
197
  'message' => orig_message,
@@ -200,7 +201,7 @@ class FieldsParserOutputTest < Test::Unit::TestCase
200
201
  'message' => 'a+b',
201
202
  }
202
203
  },
203
- emits[0][2]
204
+ events[0][2]
204
205
  )
205
206
  end
206
207
 
@@ -208,63 +209,62 @@ class FieldsParserOutputTest < Test::Unit::TestCase
208
209
  d = create_driver("pattern (\\w+):(\\d+)")
209
210
 
210
211
  orig_message = %{parse this a:44 b:ignore-this h=7 bbb:999}
211
- d.run do
212
- d.emit({'message' => orig_message})
213
- d.emit({'message' => 'a'})
212
+ d.run(default_tag: 'orig.test.tag') do
213
+ d.feed({'message' => orig_message})
214
+ d.feed({'message' => 'a'})
214
215
  end
215
216
 
216
- emits = d.emits
217
- assert_equal 2, emits.size
218
- assert_equal "orig.test.tag", emits[0][0]
217
+ events = d.events
218
+ assert_equal 2, events.size
219
+ assert_equal "orig.test.tag", events[0][0]
219
220
  assert_equal(
220
221
  {
221
222
  'message' => orig_message,
222
223
  'a' => '44',
223
224
  'bbb' => '999',
224
225
  },
225
- emits[0][2]
226
+ events[0][2]
226
227
  )
227
228
  assert_equal(
228
229
  {
229
230
  'message' => 'a',
230
231
  },
231
- emits[1][2]
232
+ events[1][2]
232
233
  )
233
234
  end
234
235
 
235
- def test_strict_key_value
236
- d = create_driver("strict_key_value true")
237
-
238
- orig_message = %{msg="Audit log" user=Johnny action="add-user" dontignore=don't-ignore-this result=success iVal=23 fVal=1.02 bVal=true}
239
- d.run do
240
- d.emit({'message' => orig_message})
241
- d.emit({'message' => 'a'})
242
- end
236
+ def test_strict_key_value
237
+ d = create_driver("strict_key_value true")
243
238
 
244
- emits = d.emits
245
- assert_equal 2, emits.size
246
- assert_equal "orig.test.tag", emits[0][0]
247
- assert_equal(
248
- {
249
- 'message' => orig_message,
250
- "msg"=>"Audit log",
251
- 'user' => "Johnny",
252
- 'action' => 'add-user',
253
- 'dontignore' => "don't-ignore-this",
254
- 'result' => 'success',
255
- 'iVal' => 23,
256
- 'fVal' => 1.02,
257
- 'bVal' => "true"
258
- },
259
- emits[0][2]
260
- )
261
- assert_equal(
262
- {
263
- 'message' => 'a',
264
- },
265
- emits[1][2]
266
- )
267
- end
239
+ orig_message = %{msg="Audit log" user=Johnny action="add-user" dontignore=don't-ignore-this result=success iVal=23 fVal=1.02 bVal=true}
240
+ d.run(default_tag: 'orig.test.tag') do
241
+ d.feed({'message' => orig_message})
242
+ d.feed({'message' => 'a'})
243
+ end
268
244
 
245
+ events = d.events
246
+ assert_equal 2, events.size
247
+ assert_equal "orig.test.tag", events[0][0]
248
+ assert_equal(
249
+ {
250
+ 'message' => orig_message,
251
+ "msg"=>"Audit log",
252
+ 'user' => "Johnny",
253
+ 'action' => 'add-user',
254
+ 'dontignore' => "don't-ignore-this",
255
+ 'result' => 'success',
256
+ 'iVal' => 23,
257
+ 'fVal' => 1.02,
258
+ 'bVal' => "true"
259
+ },
260
+ events[0][2]
261
+ )
262
+ assert_equal(
263
+ {
264
+ 'message' => 'a',
265
+ },
266
+ events[1][2]
267
+ )
268
+ end
269
269
 
270
270
  end
metadata CHANGED
@@ -1,62 +1,69 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-fields-parser
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.1
5
- prerelease:
4
+ version: 0.1.2
6
5
  platform: ruby
7
6
  authors:
8
7
  - Tomas Pokorny
9
8
  autorequire:
10
9
  bindir: bin
11
10
  cert_chain: []
12
- date: 2014-08-19 00:00:00.000000000 Z
11
+ date: 2017-12-21 00:00:00.000000000 Z
13
12
  dependencies:
14
13
  - !ruby/object:Gem::Dependency
15
14
  name: fluentd
16
15
  requirement: !ruby/object:Gem::Requirement
17
- none: false
18
16
  requirements:
19
- - - ! '>='
17
+ - - ">="
20
18
  - !ruby/object:Gem::Version
21
- version: '0'
19
+ version: 0.14.0
22
20
  type: :runtime
23
21
  prerelease: false
24
22
  version_requirements: !ruby/object:Gem::Requirement
25
- none: false
26
23
  requirements:
27
- - - ! '>='
24
+ - - ">="
28
25
  - !ruby/object:Gem::Version
29
- version: '0'
26
+ version: 0.14.0
30
27
  - !ruby/object:Gem::Dependency
31
28
  name: logfmt
32
29
  requirement: !ruby/object:Gem::Requirement
33
- none: false
34
30
  requirements:
35
- - - ! '>='
31
+ - - ">="
36
32
  - !ruby/object:Gem::Version
37
33
  version: '0'
38
34
  type: :runtime
39
35
  prerelease: false
40
36
  version_requirements: !ruby/object:Gem::Requirement
41
- none: false
42
37
  requirements:
43
- - - ! '>='
38
+ - - ">="
44
39
  - !ruby/object:Gem::Version
45
40
  version: '0'
46
41
  - !ruby/object:Gem::Dependency
47
42
  name: rake
48
43
  requirement: !ruby/object:Gem::Requirement
49
- none: false
50
44
  requirements:
51
- - - ! '>='
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: '0'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: test-unit
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ">="
52
60
  - !ruby/object:Gem::Version
53
61
  version: '0'
54
62
  type: :development
55
63
  prerelease: false
56
64
  version_requirements: !ruby/object:Gem::Requirement
57
- none: false
58
65
  requirements:
59
- - - ! '>='
66
+ - - ">="
60
67
  - !ruby/object:Gem::Version
61
68
  version: '0'
62
69
  description: Fluent output filter plugin for parsing key/value fields in records
@@ -66,7 +73,9 @@ executables: []
66
73
  extensions: []
67
74
  extra_rdoc_files: []
68
75
  files:
69
- - .gitignore
76
+ - ".gitignore"
77
+ - ".travis.yml"
78
+ - Gemfile
70
79
  - README.md
71
80
  - Rakefile
72
81
  - VERSION
@@ -76,26 +85,26 @@ files:
76
85
  homepage: https://github.com/tomas-zemres/fluent-plugin-fields-parser
77
86
  licenses:
78
87
  - MIT
88
+ metadata: {}
79
89
  post_install_message:
80
90
  rdoc_options: []
81
91
  require_paths:
82
92
  - lib
83
93
  required_ruby_version: !ruby/object:Gem::Requirement
84
- none: false
85
94
  requirements:
86
- - - ! '>='
95
+ - - ">="
87
96
  - !ruby/object:Gem::Version
88
97
  version: '0'
89
98
  required_rubygems_version: !ruby/object:Gem::Requirement
90
- none: false
91
99
  requirements:
92
- - - ! '>='
100
+ - - ">="
93
101
  - !ruby/object:Gem::Version
94
102
  version: '0'
95
103
  requirements: []
96
104
  rubyforge_project:
97
- rubygems_version: 1.8.23
105
+ rubygems_version: 2.5.2.1
98
106
  signing_key:
99
- specification_version: 3
107
+ specification_version: 4
100
108
  summary: Fluent output filter plugin for parsing key/value fields in records
101
- test_files: []
109
+ test_files:
110
+ - test/out_fields_parser.rb