fluent-plugin-parser 0.6.0 → 0.6.1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: e389a96d9ae681aa25c0de95d9d1abb9b1c4189b
4
- data.tar.gz: 3808413fd2cdd85483eeca9d01cdaed62e4fe1de
3
+ metadata.gz: 53fe6dc45845fe49522ee90562816fb82500e5a1
4
+ data.tar.gz: 3d4308519a8ba566ee7fe4f04667e077eeca6235
5
5
  SHA512:
6
- metadata.gz: 7174e622a987623dced94a830681e5d6f31742e2e2205964c709f9cccebca4a8a0b89b1ad082ded4cc9a8aedc8b71fe74e38af360bf33522476845639817d7c5
7
- data.tar.gz: d6ea265808450c244ae34cc05efc62ef2e64a48b3b0c92327db373c49383d4d852b3ceb485dcf426d8200efc993fbf7ce3691ae8691f810296e85b19767781ef
6
+ metadata.gz: 3a82d60fef713d0d405bf529a3aae05e30eb7a2cc5824a24839cbbe125022443c692c56785de50c65e845ad73ef6842f81838d164db2f0007c450c1e3109e45d
7
+ data.tar.gz: 88ee689f9124a3abde8bb887231aa26eb95d352a12d62e689c4a636c93e76d62016789f3db9b7a03dff95b6538efc3bef7ce5c35689d97908947eb19221f7cb3
@@ -1,4 +1,6 @@
1
1
  language: ruby
2
+ before_install:
3
+ - gem update bundler
2
4
  rvm:
3
5
  - 1.9.3
4
6
  - 2.0.0
data/README.md CHANGED
@@ -7,10 +7,18 @@
7
7
  This is a [Fluentd](http://fluentd.org) plugin to parse strings in log messages
8
8
  and re-emit them.
9
9
 
10
+ ### ParserFilter
11
+
12
+ Filter version of ParserOutput. In fluentd v0.12 or later, ParserFilter is recommended for simple configuartion and better performance.
13
+
10
14
  ### DeparserOutput
11
15
 
12
16
  Generate string log value from log message, with specified format and fields, and re-emit.
13
17
 
18
+ ### DeparserFilter
19
+
20
+ Filter version of DeparserOutput. In fluentd v0.12 or later, DeparserFilter is recommended for simple configuartion and better performance.
21
+
14
22
  ## Configuration
15
23
 
16
24
  ### ParserOutput
@@ -18,7 +26,7 @@ Generate string log value from log message, with specified format and fields, an
18
26
  ParserOutput has just same with 'in_tail' about 'format' and 'time\_format':
19
27
 
20
28
  <match raw.apache.common.*>
21
- type parser
29
+ @type parser
22
30
  remove_prefix raw
23
31
  format /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$/
24
32
  time_format %d/%b/%Y:%H:%M:%S %z
@@ -28,7 +36,7 @@ ParserOutput has just same with 'in_tail' about 'format' and 'time\_format':
28
36
  Of course, you can use predefined format 'apache' and 'syslog':
29
37
 
30
38
  <match raw.apache.combined.*>
31
- type parser
39
+ @type parser
32
40
  remove_prefix raw
33
41
  format apache
34
42
  key_name message
@@ -40,7 +48,7 @@ See document page for more details: http://docs.fluentd.org/articles/parser-plug
40
48
  If you want original attribute-data pair in re-emitted message, specify 'reserve_data':
41
49
 
42
50
  <match raw.apache.*>
43
- type parser
51
+ @type parser
44
52
  tag apache
45
53
  format apache
46
54
  key_name message
@@ -51,7 +59,7 @@ If you want to suppress 'pattern not match' log, specify 'suppress\_parse\_error
51
59
  default value is false.
52
60
 
53
61
  <match in.hogelog>
54
- type parser
62
+ @type parser
55
63
  tag hogelog
56
64
  format /^col1=(?<col1>.+) col2=(?<col2>.+)$/
57
65
  key_name message
@@ -61,7 +69,7 @@ default value is false.
61
69
  To store parsed values with specified key name prefix, use `inject_key_prefix` option:
62
70
 
63
71
  <match raw.sales.*>
64
- type parser
72
+ @type parser
65
73
  tag sales
66
74
  format json
67
75
  key_name sales
@@ -74,7 +82,7 @@ To store parsed values with specified key name prefix, use `inject_key_prefix` o
74
82
  To store parsed values as a hash value in a field, use `hash_value_field` option:
75
83
 
76
84
  <match raw.sales.*>
77
- type parser
85
+ @type parser
78
86
  tag sales
79
87
  format json
80
88
  key_name sales
@@ -105,7 +113,7 @@ Not to parse times (reserve that field like 'time' in record), specify `time_par
105
113
  To build CSV from field 'store','item','num', as field 'csv', without raw data:
106
114
 
107
115
  <match in.marketlog.**>
108
- type deparser
116
+ @type deparser
109
117
  remove_prefix in
110
118
  format %s,%s,%s
111
119
  format_key_names store,item,num
@@ -115,7 +123,7 @@ To build CSV from field 'store','item','num', as field 'csv', without raw data:
115
123
  To build same CSV, as additional field 'csv', with reserved raw fields:
116
124
 
117
125
  <match in.marketlog>
118
- type deparser
126
+ @type deparser
119
127
  tag marketlog
120
128
  format %s,%s,%s
121
129
  format_key_names store,item,num
@@ -123,6 +131,114 @@ To build same CSV, as additional field 'csv', with reserved raw fields:
123
131
  reserve_data yes
124
132
  </match>
125
133
 
134
+ ### ParserFilter
135
+
136
+ This is the filter version of ParserOutput.
137
+
138
+ Note that this filter version of parser plugin does not have modifing tag functionality.
139
+
140
+ ParserFilter has just same with 'in_tail' about 'format' and 'time\_format':
141
+
142
+ <filter raw.apache.common.*>
143
+ @type parser
144
+ format /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$/
145
+ time_format %d/%b/%Y:%H:%M:%S %z
146
+ key_name message
147
+ </filter>
148
+
149
+ Of course, you can use predefined format 'apache' and 'syslog':
150
+
151
+ <filter raw.apache.combined.*>
152
+ @type parser
153
+ format apache
154
+ key_name message
155
+ </filter>
156
+
157
+ `fluent-plugin-parser` uses parser plugins of Fluentd (and your own customized parser plugin).
158
+ See document page for more details: http://docs.fluentd.org/articles/parser-plugin-overview
159
+
160
+ If you want original attribute-data pair in re-emitted message, specify 'reserve_data':
161
+
162
+ <filter raw.apache.*>
163
+ @type parser
164
+ format apache
165
+ key_name message
166
+ reserve_data yes
167
+ </filter>
168
+
169
+ If you want to suppress 'pattern not match' log, specify 'suppress\_parse\_error\_log true' to configuration.
170
+ default value is false.
171
+
172
+ <filter in.hogelog>
173
+ @type parser
174
+ format /^col1=(?<col1>.+) col2=(?<col2>.+)$/
175
+ key_name message
176
+ suppress_parse_error_log true
177
+ </filter>
178
+
179
+ To store parsed values with specified key name prefix, use `inject_key_prefix` option:
180
+
181
+ <filter raw.sales.*>
182
+ @type parser
183
+ format json
184
+ key_name sales
185
+ reserve_data yes
186
+ inject_key_prefix sales.
187
+ </filter>
188
+ # input string of 'sales': {"user":1,"num":2}
189
+ # output data: {"sales":"{\"user\":1,\"num\":2}","sales.user":1, "sales.num":2}
190
+
191
+ To store parsed values as a hash value in a field, use `hash_value_field` option:
192
+
193
+ <filter raw.sales.*>
194
+ @type parser
195
+ tag sales
196
+ format json
197
+ key_name sales
198
+ hash_value_field parsed
199
+ </filter>
200
+ # input string of 'sales': {"user":1,"num":2}
201
+ # output data: {"parsed":{"user":1, "num":2}}
202
+
203
+ Other options (ex: `reserve_data`, `inject_key_prefix`) are available with `hash_value_field`.
204
+
205
+ # output data: {"sales":"{\"user\":1,\"num\":2}", "parsed":{"sales.user":1, "sales.num":2}}
206
+
207
+ Not to parse times (reserve that field like 'time' in record), specify `time_parse no`:
208
+
209
+ <filter raw.sales.*>
210
+ @type parser
211
+ format json
212
+ key_name sales
213
+ hash_value_field parsed
214
+ time_parse no
215
+ </filter>
216
+ # input string of 'sales': {"user":1,"num":2,"time":"2013-10-31 12:48:33"}
217
+ # output data: {"parsed":{"user":1, "num":2,"time":"2013-10-31 12:48:33"}}
218
+
219
+ ### DeparserFilter
220
+
221
+ Note that this filter version of deparser plugin does not have modifing tag functionality.
222
+
223
+ To build CSV from field 'store','item','num', as field 'csv', without raw data:
224
+
225
+ <filter in.marketlog.**>
226
+ @type deparser
227
+ format %s,%s,%s
228
+ format_key_names store,item,num
229
+ key_name csv
230
+ </filter>
231
+
232
+ To build same CSV, as additional field 'csv', with reserved raw fields:
233
+
234
+ <filter in.marketlog>
235
+ @type deparser
236
+ format %s,%s,%s
237
+ format_key_names store,item,num
238
+ key_name csv
239
+ reserve_data yes
240
+ </filter>
241
+
126
242
  ## TODO
127
243
 
128
244
  * consider what to do next
@@ -1,13 +1,13 @@
1
1
  # -*- encoding: utf-8 -*-
2
2
  Gem::Specification.new do |gem|
3
3
  gem.name = "fluent-plugin-parser"
4
- gem.version = "0.6.0"
4
+ gem.version = "0.6.1"
5
5
  gem.authors = ["TAGOMORI Satoshi"]
6
6
  gem.email = ["tagomoris@gmail.com"]
7
7
  gem.description = %q{fluentd plugin to parse single field, or to combine log structure into single field}
8
8
  gem.summary = %q{plugin to parse/combine fluentd log messages}
9
9
  gem.homepage = "https://github.com/tagomoris/fluent-plugin-parser"
10
- gem.license = "APLv2"
10
+ gem.license = "Apache-2.0"
11
11
 
12
12
  gem.files = `git ls-files`.split($\)
13
13
  gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
@@ -16,5 +16,6 @@ Gem::Specification.new do |gem|
16
16
 
17
17
  gem.add_development_dependency "test-unit"
18
18
  gem.add_development_dependency "rake"
19
+ gem.add_development_dependency "oj"
19
20
  gem.add_runtime_dependency "fluentd", "~> 0.12.0"
20
21
  end
@@ -1,4 +1,4 @@
1
- class Fluent::DeparserFilter < Fluent::Output
1
+ class Fluent::DeparserFilter < Fluent::Filter
2
2
  Fluent::Plugin.register_filter('deparser', self)
3
3
 
4
4
  config_param :format, :string
@@ -10,6 +10,7 @@ class Fluent::ParserFilter < Fluent::Filter
10
10
  config_param :hash_value_field, :string, default: nil
11
11
  config_param :suppress_parse_error_log, :bool, default: false
12
12
  config_param :time_parse, :bool, default: true
13
+ config_param :ignore_key_not_exist, :bool, default: false
13
14
 
14
15
  attr_reader :parser
15
16
 
@@ -36,6 +37,11 @@ class Fluent::ParserFilter < Fluent::Filter
36
37
  new_es = Fluent::MultiEventStream.new
37
38
  es.each do |time,record|
38
39
  raw_value = record[@key_name]
40
+ if raw_value.nil?
41
+ log.warn "#{@key_name} does not exist" unless @ignore_key_not_exist
42
+ new_es.add(time, handle_parsed(tag, record, time, {})) if @reserve_data
43
+ next
44
+ end
39
45
  begin
40
46
  @parser.parse(raw_value) do |t,values|
41
47
  if values
@@ -13,6 +13,7 @@ class Fluent::ParserOutput < Fluent::Output
13
13
  config_param :hash_value_field, :string, :default => nil
14
14
  config_param :suppress_parse_error_log, :bool, :default => false
15
15
  config_param :time_parse, :bool, :default => true
16
+ config_param :ignore_key_not_exist, :bool, default: false
16
17
 
17
18
  attr_reader :parser
18
19
 
@@ -73,6 +74,11 @@ class Fluent::ParserOutput < Fluent::Output
73
74
  end
74
75
  es.each do |time,record|
75
76
  raw_value = record[@key_name]
77
+ if raw_value.nil?
78
+ log.warn "#{@key_name} does not exist" unless @ignore_key_not_exist
79
+ handle_parsed(tag, record, time, {}) if @reserve_data
80
+ next
81
+ end
76
82
  begin
77
83
  @parser.parse(raw_value) do |t,values|
78
84
  if values
@@ -541,6 +541,47 @@ class ParserFilterTest < Test::Unit::TestCase
541
541
  assert_equal '?'.force_encoding('US-ASCII'), filtered[0][2]['message']
542
542
  end
543
543
 
544
+ CONFIG_NOT_IGNORE = %[
545
+ remove_prefix test
546
+ key_name data
547
+ format json
548
+ hash_value_field parsed
549
+ ]
550
+ CONFIG_IGNORE = CONFIG_NOT_IGNORE + %[
551
+ ignore_key_not_exist true
552
+ ]
553
+ CONFIG_PASS_SAME_RECORD = CONFIG_IGNORE + %[
554
+ reserve_data true
555
+ ]
556
+ def test_filter_key_not_exist
557
+ d = create_driver(CONFIG_NOT_IGNORE, 'test.no.ignore')
558
+ assert_nothing_raised {
559
+ d.run do
560
+ d.filter({'foo' => 'bar'}, Time.now.to_i)
561
+ end
562
+ }
563
+ assert_match /data does not exist/, d.instance.log.out.logs.first
564
+
565
+ d = create_driver(CONFIG_IGNORE, 'test.ignore')
566
+ assert_nothing_raised {
567
+ d.run do
568
+ d.filter({'foo' => 'bar'}, Time.now.to_i)
569
+ end
570
+ }
571
+ assert_not_match /data does not exist/, d.instance.log.out.logs.first
572
+
573
+ d = create_driver(CONFIG_PASS_SAME_RECORD, 'test.pass_same_record')
574
+ assert_nothing_raised {
575
+ d.run do
576
+ d.filter({'foo' => 'bar'}, Time.now.to_i)
577
+ end
578
+ }
579
+ filtered = d.filtered_as_array
580
+ assert_equal 1, filtered.length
581
+ assert_nil filtered[0][2]['data']
582
+ assert_equal 'bar', filtered[0][2]['foo']
583
+ end
584
+
544
585
  # suppress_parse_error_log test
545
586
  CONFIG_DISABELED_SUPPRESS_PARSE_ERROR_LOG = %[
546
587
  tag hogelog
@@ -598,6 +598,47 @@ class ParserOutputTest < Test::Unit::TestCase
598
598
  assert_equal '?'.force_encoding('US-ASCII'), emits[0][2]['message']
599
599
  end
600
600
 
601
+ CONFIG_NOT_IGNORE = %[
602
+ remove_prefix test
603
+ key_name data
604
+ format json
605
+ hash_value_field parsed
606
+ ]
607
+ CONFIG_IGNORE = CONFIG_NOT_IGNORE + %[
608
+ ignore_key_not_exist true
609
+ ]
610
+ CONFIG_EMIT_SAME_RECORD = CONFIG_IGNORE + %[
611
+ reserve_data true
612
+ ]
613
+ def test_emit_key_not_exist
614
+ d = create_driver(CONFIG_NOT_IGNORE, 'test.no.ignore')
615
+ assert_nothing_raised {
616
+ d.run do
617
+ d.emit({'foo' => 'bar'}, Time.now.to_i)
618
+ end
619
+ }
620
+ assert_match /data does not exist/, d.instance.log.out.logs.first
621
+
622
+ d = create_driver(CONFIG_IGNORE, 'test.ignore')
623
+ assert_nothing_raised {
624
+ d.run do
625
+ d.emit({'foo' => 'bar'}, Time.now.to_i)
626
+ end
627
+ }
628
+ assert_not_match /data does not exist/, d.instance.log.out.logs.first
629
+
630
+ d = create_driver(CONFIG_EMIT_SAME_RECORD, 'test.emit_same_record')
631
+ assert_nothing_raised {
632
+ d.run do
633
+ d.emit({'foo' => 'bar'}, Time.now.to_i)
634
+ end
635
+ }
636
+ emits = d.emits
637
+ assert_equal 1, emits.length
638
+ assert_nil emits[0][2]['data']
639
+ assert_equal 'bar', emits[0][2]['foo']
640
+ end
641
+
601
642
  # suppress_parse_error_log test
602
643
  CONFIG_DISABELED_SUPPRESS_PARSE_ERROR_LOG = %[
603
644
  tag hogelog
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-parser
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.6.0
4
+ version: 0.6.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - TAGOMORI Satoshi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-09-04 00:00:00.000000000 Z
11
+ date: 2016-04-25 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: test-unit
@@ -38,6 +38,20 @@ dependencies:
38
38
  - - ">="
39
39
  - !ruby/object:Gem::Version
40
40
  version: '0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: oj
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: '0'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
41
55
  - !ruby/object:Gem::Dependency
42
56
  name: fluentd
43
57
  requirement: !ruby/object:Gem::Requirement
@@ -80,7 +94,7 @@ files:
80
94
  - test/plugin/test_out_parser_for_parsers.rb
81
95
  homepage: https://github.com/tagomoris/fluent-plugin-parser
82
96
  licenses:
83
- - APLv2
97
+ - Apache-2.0
84
98
  metadata: {}
85
99
  post_install_message:
86
100
  rdoc_options: []
@@ -98,7 +112,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
98
112
  version: '0'
99
113
  requirements: []
100
114
  rubyforge_project:
101
- rubygems_version: 2.4.5
115
+ rubygems_version: 2.5.1
102
116
  signing_key:
103
117
  specification_version: 4
104
118
  summary: plugin to parse/combine fluentd log messages