fluent-plugin-parser 0.6.0 → 0.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e389a96d9ae681aa25c0de95d9d1abb9b1c4189b
-  data.tar.gz: 3808413fd2cdd85483eeca9d01cdaed62e4fe1de
+  metadata.gz: 53fe6dc45845fe49522ee90562816fb82500e5a1
+  data.tar.gz: 3d4308519a8ba566ee7fe4f04667e077eeca6235
 SHA512:
-  metadata.gz: 7174e622a987623dced94a830681e5d6f31742e2e2205964c709f9cccebca4a8a0b89b1ad082ded4cc9a8aedc8b71fe74e38af360bf33522476845639817d7c5
-  data.tar.gz: d6ea265808450c244ae34cc05efc62ef2e64a48b3b0c92327db373c49383d4d852b3ceb485dcf426d8200efc993fbf7ce3691ae8691f810296e85b19767781ef
+  metadata.gz: 3a82d60fef713d0d405bf529a3aae05e30eb7a2cc5824a24839cbbe125022443c692c56785de50c65e845ad73ef6842f81838d164db2f0007c450c1e3109e45d
+  data.tar.gz: 88ee689f9124a3abde8bb887231aa26eb95d352a12d62e689c4a636c93e76d62016789f3db9b7a03dff95b6538efc3bef7ce5c35689d97908947eb19221f7cb3

data/.travis.yml

@@ -1,4 +1,6 @@
 language: ruby
+before_install:
+  - gem update bundler
 rvm:
   - 1.9.3
   - 2.0.0

data/README.md

@@ -7,10 +7,18 @@
 This is a [Fluentd](http://fluentd.org) plugin to parse strings in log messages
 and re-emit them.
 
+### ParserFilter
+
+Filter version of ParserOutput. In fluentd v0.12 or later, ParserFilter is recommended for simple configuartion and better performance.
+
 ### DeparserOutput
 
 Generate string log value from log message, with specified format and fields, and re-emit.
 
+### DeparserFilter
+
+Filter version of DeparserOutput. In fluentd v0.12 or later, DeparserFilter is recommended for simple configuartion and better performance.
+
 ## Configuration
 
 ### ParserOutput
@@ -18,7 +26,7 @@ Generate string log value from log message, with specified format and fields, an
 ParserOutput has just same with 'in_tail' about 'format' and 'time\_format':
 
     <match raw.apache.common.*>
-      type parser
+      @type parser
       remove_prefix raw
       format /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$/
       time_format %d/%b/%Y:%H:%M:%S %z
@@ -28,7 +36,7 @@ ParserOutput has just same with 'in_tail' about 'format' and 'time\_format':
 Of course, you can use predefined format 'apache' and 'syslog':
 
     <match raw.apache.combined.*>
-      type parser
+      @type parser
       remove_prefix raw
       format apache
       key_name message
@@ -40,7 +48,7 @@ See document page for more details: http://docs.fluentd.org/articles/parser-plug
 If you want original attribute-data pair in re-emitted message, specify 'reserve_data':
 
     <match raw.apache.*>
-      type parser
+      @type parser
       tag apache
       format apache
       key_name message
@@ -51,7 +59,7 @@ If you want to suppress 'pattern not match' log, specify 'suppress\_parse\_error
 default value is false.
 
     <match in.hogelog>
-      type parser
+      @type parser
       tag hogelog
       format /^col1=(?<col1>.+) col2=(?<col2>.+)$/
       key_name message
@@ -61,7 +69,7 @@ default value is false.
 To store parsed values with specified key name prefix, use `inject_key_prefix` option:
 
     <match raw.sales.*>
-      type parser
+      @type parser
       tag sales
       format json
       key_name sales
@@ -74,7 +82,7 @@ To store parsed values with specified key name prefix, use `inject_key_prefix` o
 To store parsed values as a hash value in a field, use `hash_value_field` option:
 
     <match raw.sales.*>
-      type parser
+      @type parser
       tag sales
       format json
       key_name sales
@@ -105,7 +113,7 @@ Not to parse times (reserve that field like 'time' in record), specify `time_par
 To build CSV from field 'store','item','num', as field 'csv', without raw data:
 
     <match in.marketlog.**>
-      type deparser
+      @type deparser
       remove_prefix in
       format %s,%s,%s
       format_key_names store,item,num
@@ -115,7 +123,7 @@ To build CSV from field 'store','item','num', as field 'csv', without raw data:
 To build same CSV, as additional field 'csv', with reserved raw fields:
 
     <match in.marketlog>
-      type deparser
+      @type deparser
       tag marketlog
       format %s,%s,%s
       format_key_names store,item,num
@@ -123,6 +131,114 @@ To build same CSV, as additional field 'csv', with reserved raw fields:
       reserve_data yes
     </match>
 
+### ParserFilter
+
+This is the filter version of ParserOutput.
+
+Note that this filter version of parser plugin does not have modifing tag functionality.
+
+ParserFilter has just same with 'in_tail' about 'format' and 'time\_format':
+
+    <filter raw.apache.common.*>
+      @type parser
+      format /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$/
+      time_format %d/%b/%Y:%H:%M:%S %z
+      key_name message
+    </filter>
+
+Of course, you can use predefined format 'apache' and 'syslog':
+
+    <filter raw.apache.combined.*>
+      @type parser
+      format apache
+      key_name message
+    </filter>
+
+`fluent-plugin-parser` uses parser plugins of Fluentd (and your own customized parser plugin).
+See document page for more details: http://docs.fluentd.org/articles/parser-plugin-overview
+
+If you want original attribute-data pair in re-emitted message, specify 'reserve_data':
+
+    <filter raw.apache.*>
+      @type parser
+      format apache
+      key_name message
+      reserve_data yes
+    </filter>
+
+If you want to suppress 'pattern not match' log, specify 'suppress\_parse\_error\_log true' to configuration.
+default value is false.
+
+    <filter in.hogelog>
+      @type parser
+      format /^col1=(?<col1>.+) col2=(?<col2>.+)$/
+      key_name message
+      suppress_parse_error_log true
+    </filter>
+
+To store parsed values with specified key name prefix, use `inject_key_prefix` option:
+
+    <filter raw.sales.*>
+      @type parser
+      format json
+      key_name sales
+      reserve_data      yes
+      inject_key_prefix sales.
+    </filter>
+    # input string of 'sales': {"user":1,"num":2}
+    # output data: {"sales":"{\"user\":1,\"num\":2}","sales.user":1, "sales.num":2}
+
+To store parsed values as a hash value in a field, use `hash_value_field` option:
+
+    <filter raw.sales.*>
+      @type parser
+      tag sales
+      format json
+      key_name sales
+      hash_value_field parsed
+    </filter>
+    # input string of 'sales': {"user":1,"num":2}
+    # output data: {"parsed":{"user":1, "num":2}}
+
+Other options (ex: `reserve_data`, `inject_key_prefix`) are available with `hash_value_field`.
+
+    # output data: {"sales":"{\"user\":1,\"num\":2}", "parsed":{"sales.user":1, "sales.num":2}}
+
+Not to parse times (reserve that field like 'time' in record), specify `time_parse no`:
+
+    <filter raw.sales.*>
+      @type parser
+      format json
+      key_name sales
+      hash_value_field parsed
+      time_parse no
+    </filter>
+    # input string of 'sales': {"user":1,"num":2,"time":"2013-10-31 12:48:33"}
+    # output data: {"parsed":{"user":1, "num":2,"time":"2013-10-31 12:48:33"}}
+
+### DeparserFilter
+
+Note that this filter version of deparser plugin does not have modifing tag functionality.
+
+To build CSV from field 'store','item','num', as field 'csv', without raw data:
+
+    <filter in.marketlog.**>
+      @type deparser
+      format %s,%s,%s
+      format_key_names store,item,num
+      key_name csv
+    </filter>
+
+To build same CSV, as additional field 'csv', with reserved raw fields:
+
+    <filter in.marketlog>
+      @type deparser
+      format %s,%s,%s
+      format_key_names store,item,num
+      key_name csv
+      reserve_data yes
+    </filter>
+
 ## TODO
 
 * consider what to do next

data/fluent-plugin-parser.gemspec

@@ -1,13 +1,13 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-parser"
-  gem.version       = "0.6.0"
+  gem.version       = "0.6.1"
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
   gem.description   = %q{fluentd plugin to parse single field, or to combine log structure into single field}
   gem.summary       = %q{plugin to parse/combine fluentd log messages}
   gem.homepage      = "https://github.com/tagomoris/fluent-plugin-parser"
-  gem.license       = "APLv2"
+  gem.license       = "Apache-2.0"
 
   gem.files         = `git ls-files`.split($\)
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
@@ -16,5 +16,6 @@ Gem::Specification.new do |gem|
 
   gem.add_development_dependency "test-unit"
   gem.add_development_dependency "rake"
+  gem.add_development_dependency "oj"
   gem.add_runtime_dependency "fluentd", "~> 0.12.0"
 end

data/lib/fluent/plugin/filter_deparser.rb

@@ -1,4 +1,4 @@
-class Fluent::DeparserFilter < Fluent::Output
+class Fluent::DeparserFilter < Fluent::Filter
   Fluent::Plugin.register_filter('deparser', self)
 
   config_param :format, :string

data/lib/fluent/plugin/filter_parser.rb

@@ -10,6 +10,7 @@ class Fluent::ParserFilter < Fluent::Filter
   config_param :hash_value_field, :string, default: nil
   config_param :suppress_parse_error_log, :bool, default: false
   config_param :time_parse, :bool, default: true
+  config_param :ignore_key_not_exist, :bool, default: false
 
   attr_reader :parser
 
@@ -36,6 +37,11 @@ class Fluent::ParserFilter < Fluent::Filter
     new_es = Fluent::MultiEventStream.new
     es.each do |time,record|
       raw_value = record[@key_name]
+      if raw_value.nil?
+        log.warn "#{@key_name} does not exist" unless @ignore_key_not_exist
+        new_es.add(time, handle_parsed(tag, record, time, {})) if @reserve_data
+        next
+      end
       begin
         @parser.parse(raw_value) do |t,values|
           if values

data/lib/fluent/plugin/out_parser.rb

@@ -13,6 +13,7 @@ class Fluent::ParserOutput < Fluent::Output
   config_param :hash_value_field, :string, :default => nil
   config_param :suppress_parse_error_log, :bool, :default => false
   config_param :time_parse, :bool, :default => true
+  config_param :ignore_key_not_exist, :bool, default: false
 
   attr_reader :parser
 
@@ -73,6 +74,11 @@ class Fluent::ParserOutput < Fluent::Output
           end
     es.each do |time,record|
       raw_value = record[@key_name]
+      if raw_value.nil?
+        log.warn "#{@key_name} does not exist" unless @ignore_key_not_exist
+        handle_parsed(tag, record, time, {}) if @reserve_data
+        next
+      end
       begin
         @parser.parse(raw_value) do |t,values|
           if values

data/test/plugin/test_filter_parser.rb

@@ -541,6 +541,47 @@ class ParserFilterTest < Test::Unit::TestCase
     assert_equal '?'.force_encoding('US-ASCII'), filtered[0][2]['message']
   end
 
+  CONFIG_NOT_IGNORE = %[
+    remove_prefix    test
+    key_name         data
+    format           json
+    hash_value_field parsed
+  ]
+  CONFIG_IGNORE = CONFIG_NOT_IGNORE + %[
+    ignore_key_not_exist true
+  ]
+  CONFIG_PASS_SAME_RECORD = CONFIG_IGNORE + %[
+    reserve_data true
+  ]
+  def test_filter_key_not_exist
+    d = create_driver(CONFIG_NOT_IGNORE, 'test.no.ignore')
+    assert_nothing_raised {
+      d.run do
+        d.filter({'foo' => 'bar'}, Time.now.to_i)
+      end
+    }
+    assert_match /data does not exist/, d.instance.log.out.logs.first
+
+    d = create_driver(CONFIG_IGNORE, 'test.ignore')
+    assert_nothing_raised {
+      d.run do
+        d.filter({'foo' => 'bar'}, Time.now.to_i)
+      end
+    }
+    assert_not_match /data does not exist/, d.instance.log.out.logs.first
+
+    d = create_driver(CONFIG_PASS_SAME_RECORD, 'test.pass_same_record')
+    assert_nothing_raised {
+      d.run do
+        d.filter({'foo' => 'bar'}, Time.now.to_i)
+      end
+    }
+    filtered = d.filtered_as_array
+    assert_equal 1, filtered.length
+    assert_nil filtered[0][2]['data']
+    assert_equal 'bar', filtered[0][2]['foo']
+  end
+
   # suppress_parse_error_log test
   CONFIG_DISABELED_SUPPRESS_PARSE_ERROR_LOG = %[
     tag hogelog

data/test/plugin/test_out_parser.rb

@@ -598,6 +598,47 @@ class ParserOutputTest < Test::Unit::TestCase
     assert_equal '?'.force_encoding('US-ASCII'), emits[0][2]['message']
   end
 
+  CONFIG_NOT_IGNORE = %[
+    remove_prefix    test
+    key_name         data
+    format           json
+    hash_value_field parsed
+  ]
+  CONFIG_IGNORE = CONFIG_NOT_IGNORE + %[
+    ignore_key_not_exist true
+  ]
+  CONFIG_EMIT_SAME_RECORD = CONFIG_IGNORE + %[
+    reserve_data true
+  ]
+  def test_emit_key_not_exist
+    d = create_driver(CONFIG_NOT_IGNORE, 'test.no.ignore')
+    assert_nothing_raised {
+      d.run do
+        d.emit({'foo' => 'bar'}, Time.now.to_i)
+      end
+    }
+    assert_match /data does not exist/, d.instance.log.out.logs.first
+
+    d = create_driver(CONFIG_IGNORE, 'test.ignore')
+    assert_nothing_raised {
+      d.run do
+        d.emit({'foo' => 'bar'}, Time.now.to_i)
+      end
+    }
+    assert_not_match /data does not exist/, d.instance.log.out.logs.first
+
+    d = create_driver(CONFIG_EMIT_SAME_RECORD, 'test.emit_same_record')
+    assert_nothing_raised {
+      d.run do
+        d.emit({'foo' => 'bar'}, Time.now.to_i)
+      end
+    }
+    emits = d.emits
+    assert_equal 1, emits.length
+    assert_nil emits[0][2]['data']
+    assert_equal 'bar', emits[0][2]['foo']
+  end
+
   # suppress_parse_error_log test
   CONFIG_DISABELED_SUPPRESS_PARSE_ERROR_LOG = %[
     tag hogelog

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-parser
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.6.1
 platform: ruby
 authors:
 - TAGOMORI Satoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-09-04 00:00:00.000000000 Z
+date: 2016-04-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: test-unit
@@ -38,6 +38,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: oj
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
@@ -80,7 +94,7 @@ files:
 - test/plugin/test_out_parser_for_parsers.rb
 homepage: https://github.com/tagomoris/fluent-plugin-parser
 licenses:
-- APLv2
+- Apache-2.0
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -98,7 +112,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: plugin to parse/combine fluentd log messages