fluent-plugin-parser 0.2.3 → 0.2.4

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 2e0393ad371a16008916ab9e6bf614d56be0a6a7
4
+ data.tar.gz: 98efdde5b556caa2452769a66582beba39bce9de
5
+ SHA512:
6
+ metadata.gz: 3cb147341062d8f32d9e3eeaee6971bbf1d539994148b9eb63260c2aafc77c0a684ceb042f1a01c6c8abea3ecd1f18bf366108f92b7d2e796d98c5728df4dcc0
7
+ data.tar.gz: d4c5308d819343ca895f62949c38d6e2fd35bf4b63ae78c42aa1aba4da9513ac293e14242e57abc4722b14475c8bb4d93f8b4a302ebc8d046db9e4e98d820d62
data/README.md CHANGED
@@ -67,7 +67,7 @@ Format 'ltsv'(Labeled-TSV (Tab separated values)) is also supported:
67
67
 
68
68
  About LTSV, see: http://ltsv.org/
69
69
 
70
- If you want to suppress 'pattern not match' log, specify 'suppress_parse_error_log true' to configuration.
70
+ If you want to suppress 'pattern not match' log, specify 'suppress\_parse\_error\_log true' to configuration.
71
71
  default value is false.
72
72
 
73
73
  <match in.hogelog>
@@ -78,6 +78,18 @@ default value is false.
78
78
  suppress_parse_error_log true
79
79
  </match>
80
80
 
81
+ To store parsed values with specified key name prefix, use `inject_key_prefix` option:
82
+
83
+ <match raw.sales.*>
84
+ type parser
85
+ tag sales
86
+ format json
87
+ key_name sales
88
+ reserve_data yes
89
+ inject_key_prefix sales.
90
+ </match>
91
+ # input string of 'sales': {"user":1,"num":2}
92
+ # output data: {"sales":"{\"user\":1,\"num\":2}","sales.user":1, "sales.num":2}
81
93
 
82
94
  ### DeparserOutput
83
95
 
@@ -1,7 +1,7 @@
1
1
  # -*- encoding: utf-8 -*-
2
2
  Gem::Specification.new do |gem|
3
3
  gem.name = "fluent-plugin-parser"
4
- gem.version = "0.2.3"
4
+ gem.version = "0.2.4"
5
5
  gem.authors = ["TAGOMORI Satoshi"]
6
6
  gem.email = ["tagomoris@gmail.com"]
7
7
  gem.description = %q{fluentd plugin to parse single field, or to combine log structure into single field}
@@ -141,25 +141,25 @@ class FluentExt::TextParser
141
141
 
142
142
  user = m['user']
143
143
  user = (user == '-') ? nil : user
144
-
144
+
145
145
  time = m['time']
146
146
  time = Time.strptime(time, "%d/%b/%Y:%H:%M:%S %z").to_i
147
-
147
+
148
148
  method = m['method']
149
149
  path = m['path']
150
-
151
- code = m['code'].to_i
150
+
151
+ code = m['code'].to_i
152
152
  code = nil if code == 0
153
153
 
154
154
  size = m['size']
155
155
  size = (size == '-') ? nil : size.to_i
156
-
156
+
157
157
  referer = m['referer']
158
158
  referer = (referer == '-') ? nil : referer
159
-
159
+
160
160
  agent = m['agent']
161
161
  agent = (agent == '-') ? nil : agent
162
-
162
+
163
163
  record = {
164
164
  "host" => host,
165
165
  "user" => user,
@@ -169,7 +169,7 @@ class FluentExt::TextParser
169
169
  "size" => size,
170
170
  "referer" => referer,
171
171
  "agent" => agent,
172
- }
172
+ }
173
173
 
174
174
  return time, record
175
175
  end
@@ -187,7 +187,7 @@ class FluentExt::TextParser
187
187
  }
188
188
 
189
189
  def self.register_template(name, regexp_or_proc, time_format=nil)
190
-
190
+
191
191
  factory = if regexp_or_proc.is_a?(Regexp)
192
192
  regexp = regexp_or_proc
193
193
  Proc.new { RegexpParser.new(regexp, {'time_format'=>time_format}) }
@@ -8,6 +8,7 @@ class Fluent::ParserOutput < Fluent::Output
8
8
  config_param :add_prefix, :string, :default => nil
9
9
  config_param :key_name, :string
10
10
  config_param :reserve_data, :bool, :default => false
11
+ config_param :inject_key_prefix, :string, :default => nil
11
12
  config_param :replace_invalid_sequence, :bool, :default => false
12
13
 
13
14
  def initialize
@@ -43,8 +44,8 @@ class Fluent::ParserOutput < Fluent::Output
43
44
  if @remove_prefix and
44
45
  ( (tag.start_with?(@removed_prefix_string) and tag.length > @removed_length) or tag == @remove_prefix)
45
46
  tag = tag[@removed_length..-1]
46
- end
47
- if @add_prefix
47
+ end
48
+ if @add_prefix
48
49
  tag = if tag and tag.length > 0
49
50
  @added_prefix_string + tag
50
51
  else
@@ -63,6 +64,9 @@ class Fluent::ParserOutput < Fluent::Output
63
64
  end
64
65
  t ||= time
65
66
  r = if values
67
+ if @inject_key_prefix
68
+ values = Hash[values.map{|k,v| [ @inject_key_prefix + k, v ]}]
69
+ end
66
70
  record.merge(values)
67
71
  else
68
72
  record
@@ -73,7 +77,11 @@ class Fluent::ParserOutput < Fluent::Output
73
77
  es.each {|time,record|
74
78
  value = record[@key_name]
75
79
  t,values = if value
76
- parse(value)
80
+ parsed = parse(value)
81
+ if @inject_key_prefix
82
+ parsed = Hash[parsed.map{|k,v| [ @inject_key_prefix + k, v ]}]
83
+ end
84
+ parsed
77
85
  else
78
86
  [nil, nil]
79
87
  end
@@ -4,7 +4,7 @@ class ParserOutputTest < Test::Unit::TestCase
4
4
  def setup
5
5
  Fluent::Test.setup
6
6
  end
7
-
7
+
8
8
  CONFIG = %[
9
9
  remove_prefix test
10
10
  add_prefix parsed
@@ -292,7 +292,7 @@ class ParserOutputTest < Test::Unit::TestCase
292
292
  key_name data
293
293
  keys key1,key2,key3
294
294
  ]
295
- def test_emit_ltsv
295
+ def test_emit_tsv
296
296
  d = create_driver(CONFIG_TSV, 'foo.baz.test')
297
297
  time = Time.parse("2012-04-02 18:20:59").to_i
298
298
  d.run do
@@ -317,7 +317,7 @@ class ParserOutputTest < Test::Unit::TestCase
317
317
  key_name data
318
318
  keys key1,key2,key3
319
319
  ]
320
- def test_emit_ltsv
320
+ def test_emit_csv
321
321
  d = create_driver(CONFIG_CSV, 'foo.baz.test')
322
322
  time = Time.parse("2012-04-02 18:20:59").to_i
323
323
  d.run do
@@ -335,6 +335,35 @@ class ParserOutputTest < Test::Unit::TestCase
335
335
  assert_equal 'value"ThreeYes!', first[2]['key3']
336
336
  end
337
337
 
338
+ CONFIG_KEY_PREFIX = %[
339
+ remove_prefix foo.baz
340
+ add_prefix foo.bar
341
+ format json
342
+ key_name data
343
+ reserve_data yes
344
+ inject_key_prefix data.
345
+ ]
346
+ def test_inject_key_prefix
347
+ d = create_driver(CONFIG_KEY_PREFIX, 'foo.baz.test')
348
+ time = Time.parse("2012-04-02 18:20:59").to_i
349
+ d.run do
350
+ d.emit({'data' => '{"xxx":"first","yyy":"second"}', 'xxx' => 'x', 'yyy' => 'y'}, time)
351
+ end
352
+ emits = d.emits
353
+ assert_equal 1, emits.length
354
+
355
+ first = emits[0]
356
+ assert_equal 'foo.bar.test', first[0]
357
+ assert_equal time, first[1]
358
+
359
+ assert_equal '{"xxx":"first","yyy":"second"}', first[2]['data']
360
+ assert_equal 'x', first[2]['xxx']
361
+ assert_equal 'y', first[2]['yyy']
362
+ assert_equal 'first', first[2]['data.xxx']
363
+ assert_equal 'second', first[2]['data.yyy']
364
+ assert_equal 5, first[2].keys.size
365
+ end
366
+
338
367
  #TODO: apache2
339
368
  # REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/
340
369
 
@@ -393,18 +422,18 @@ class ParserOutputTest < Test::Unit::TestCase
393
422
  assert_equal '?'.force_encoding('US-ASCII'), emits[0][2]['message']
394
423
  end
395
424
 
396
- # suppress_parse_error_log test
425
+ # suppress_parse_error_log test
397
426
  CONFIG_DISABELED_SUPPRESS_PARSE_ERROR_LOG = %[
398
427
  tag hogelog
399
428
  format /^col1=(?<col1>.+) col2=(?<col2>.+)$/
400
429
  key_name message
401
- suppress_parse_error_log false
430
+ suppress_parse_error_log false
402
431
  ]
403
432
  CONFIG_ENABELED_SUPPRESS_PARSE_ERROR_LOG = %[
404
433
  tag hogelog
405
434
  format /^col1=(?<col1>.+) col2=(?<col2>.+)$/
406
435
  key_name message
407
- suppress_parse_error_log true
436
+ suppress_parse_error_log true
408
437
  ]
409
438
  CONFIG_DEFAULT_SUPPRESS_PARSE_ERROR_LOG = %[
410
439
  tag hogelog
metadata CHANGED
@@ -1,46 +1,41 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-parser
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.3
5
- prerelease:
4
+ version: 0.2.4
6
5
  platform: ruby
7
6
  authors:
8
7
  - TAGOMORI Satoshi
9
8
  autorequire:
10
9
  bindir: bin
11
10
  cert_chain: []
12
- date: 2013-04-08 00:00:00.000000000 Z
11
+ date: 2013-08-01 00:00:00.000000000 Z
13
12
  dependencies:
14
13
  - !ruby/object:Gem::Dependency
15
14
  name: rake
16
15
  requirement: !ruby/object:Gem::Requirement
17
- none: false
18
16
  requirements:
19
- - - ! '>='
17
+ - - '>='
20
18
  - !ruby/object:Gem::Version
21
19
  version: '0'
22
20
  type: :development
23
21
  prerelease: false
24
22
  version_requirements: !ruby/object:Gem::Requirement
25
- none: false
26
23
  requirements:
27
- - - ! '>='
24
+ - - '>='
28
25
  - !ruby/object:Gem::Version
29
26
  version: '0'
30
27
  - !ruby/object:Gem::Dependency
31
28
  name: fluentd
32
29
  requirement: !ruby/object:Gem::Requirement
33
- none: false
34
30
  requirements:
35
- - - ! '>='
31
+ - - '>='
36
32
  - !ruby/object:Gem::Version
37
33
  version: '0'
38
34
  type: :runtime
39
35
  prerelease: false
40
36
  version_requirements: !ruby/object:Gem::Requirement
41
- none: false
42
37
  requirements:
43
- - - ! '>='
38
+ - - '>='
44
39
  - !ruby/object:Gem::Version
45
40
  version: '0'
46
41
  description: fluentd plugin to parse single field, or to combine log structure into
@@ -65,29 +60,29 @@ files:
65
60
  - test/plugin/test_out_parser.rb
66
61
  homepage: https://github.com/tagomoris/fluent-plugin-parser
67
62
  licenses: []
63
+ metadata: {}
68
64
  post_install_message:
69
65
  rdoc_options: []
70
66
  require_paths:
71
67
  - lib
72
68
  required_ruby_version: !ruby/object:Gem::Requirement
73
- none: false
74
69
  requirements:
75
- - - ! '>='
70
+ - - '>='
76
71
  - !ruby/object:Gem::Version
77
72
  version: '0'
78
73
  required_rubygems_version: !ruby/object:Gem::Requirement
79
- none: false
80
74
  requirements:
81
- - - ! '>='
75
+ - - '>='
82
76
  - !ruby/object:Gem::Version
83
77
  version: '0'
84
78
  requirements: []
85
79
  rubyforge_project:
86
- rubygems_version: 1.8.23
80
+ rubygems_version: 2.0.2
87
81
  signing_key:
88
- specification_version: 3
82
+ specification_version: 4
89
83
  summary: plugin to parse/combine fluentd log messages
90
84
  test_files:
91
85
  - test/helper.rb
92
86
  - test/plugin/test_deparser.rb
93
87
  - test/plugin/test_out_parser.rb
88
+ has_rdoc: