fluent-plugin-parser 0.2.4 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2e0393ad371a16008916ab9e6bf614d56be0a6a7
-  data.tar.gz: 98efdde5b556caa2452769a66582beba39bce9de
+  metadata.gz: d94615d4d0da1036e495a635ee91d1165281053d
+  data.tar.gz: e076017291eaa10cec2dbe7676e1d45b7e953bcb
 SHA512:
-  metadata.gz: 3cb147341062d8f32d9e3eeaee6971bbf1d539994148b9eb63260c2aafc77c0a684ceb042f1a01c6c8abea3ecd1f18bf366108f92b7d2e796d98c5728df4dcc0
-  data.tar.gz: d4c5308d819343ca895f62949c38d6e2fd35bf4b63ae78c42aa1aba4da9513ac293e14242e57abc4722b14475c8bb4d93f8b4a302ebc8d046db9e4e98d820d62
+  metadata.gz: 61820d91940e4322f7715ba8949dc6ca3e9cf558c4d2f21fecf14da57cc1f0211e35550c51ae6b81a97f9c80e2116d8bff881fa0c55133514d6c1e274f828f91
+  data.tar.gz: 514f354cd2d3674815fd213135bb0b843f2f79d23e69bee0f4826aae1f9528df3df31b5296721b5472bffd2a9086ac4be70420a58e55b0b1f7c04ffd1ce85369

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+rvm:
+  - 1.9.3
+  - 2.0.0

data/README.md

@@ -91,6 +91,22 @@ To store parsed values with specified key name prefix, use `inject_key_prefix` o
     # input string of 'sales': {"user":1,"num":2}
     # output data: {"sales":"{\"user\":1,\"num\":2}","sales.user":1, "sales.num":2}
 
+To store parsed values as a hash value in a field, use `hash_value_field` option:
+
+    <match raw.sales.*>
+      type parser
+      tag sales
+      format json
+      key_name sales
+      hash_value_field parsed
+    </match>
+    # input string of 'sales': {"user":1,"num":2}
+    # output data: {"parsed":{"user":1, "num":2}}
+
+Other options (ex: `reserve_data`, `inject_key_prefix`) are available with `hash_value_field`.
+
+    # output data: {"sales":"{\"user\":1,\"num\":2}", "parsed":{"sales.user":1, "sales.num":2}}
+
 ### DeparserOutput
 
 To build CSV from field 'store','item','num', as field 'csv', without raw data:

data/fluent-plugin-parser.gemspec

@@ -1,12 +1,13 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-parser"
-  gem.version       = "0.2.4"
+  gem.version       = "0.3.0"
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
   gem.description   = %q{fluentd plugin to parse single field, or to combine log structure into single field}
   gem.summary       = %q{plugin to parse/combine fluentd log messages}
   gem.homepage      = "https://github.com/tagomoris/fluent-plugin-parser"
+  gem.license       = "APLv2"
 
   gem.files         = `git ls-files`.split($\)
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }

data/lib/fluent/plugin/out_parser.rb

@@ -10,6 +10,7 @@ class Fluent::ParserOutput < Fluent::Output
   config_param :reserve_data, :bool, :default => false
   config_param :inject_key_prefix, :string, :default => nil
   config_param :replace_invalid_sequence, :bool, :default => false
+  config_param :hash_value_field, :string, :default => nil
 
   def initialize
     super
@@ -54,43 +55,23 @@ class Fluent::ParserOutput < Fluent::Output
             end
             tag
           end
-    if @reserve_data
-      es.each {|time,record|
-        value = record[@key_name]
-        t,values = if value
-                     parse(value)
-                   else
-                     [nil, nil]
-                   end
-        t ||= time
-        r = if values
-              if @inject_key_prefix
-                values = Hash[values.map{|k,v| [ @inject_key_prefix + k, v ]}]
-              end
-              record.merge(values)
-            else
-              record
-            end
+    es.each do |time,record|
+      raw_value = record[@key_name]
+      t,values = raw_value ? parse(raw_value) : [nil, nil]
+      t ||= time
+
+      if values && @inject_key_prefix
+        values = Hash[values.map{|k,v| [ @inject_key_prefix + k, v ]}]
+      end
+      r = @hash_value_field ? {@hash_value_field => values} : values
+      if @reserve_data
+        r = r ? record.merge(r) : record
+      end
+      if r
         Fluent::Engine.emit(tag, t, r)
-      }
-    else
-      es.each {|time,record|
-        value = record[@key_name]
-        t,values = if value
-                     parsed = parse(value)
-                     if @inject_key_prefix
-                       parsed = Hash[parsed.map{|k,v| [ @inject_key_prefix + k, v ]}]
-                     end
-                     parsed
-                   else
-                     [nil, nil]
-                   end
-        t ||= time
-        if values
-          Fluent::Engine.emit(tag, t, values)
-        end
-      }
+      end
     end
+
     chain.next
   end
 

data/test/plugin/test_out_parser.rb

@@ -364,6 +364,84 @@ class ParserOutputTest < Test::Unit::TestCase
     assert_equal 5, first[2].keys.size
   end
 
+  CONFIG_HASH_VALUE_FIELD = %[
+    remove_prefix foo.baz
+    add_prefix foo.bar
+    format       json
+    key_name     data
+    hash_value_field parsed
+  ]
+  CONFIG_HASH_VALUE_FIELD_RESERVE_DATA = %[
+    remove_prefix foo.baz
+    add_prefix foo.bar
+    format       json
+    key_name     data
+    reserve_data yes
+    hash_value_field parsed
+  ]
+  CONFIG_HASH_VALUE_FIELD_WITH_INJECT_KEY_PREFIX = %[
+    remove_prefix foo.baz
+    add_prefix foo.bar
+    format       json
+    key_name     data
+    hash_value_field parsed
+    inject_key_prefix data.
+  ]
+  def test_inject_hash_value_field
+    original = {'data' => '{"xxx":"first","yyy":"second"}', 'xxx' => 'x', 'yyy' => 'y'}
+
+    d = create_driver(CONFIG_HASH_VALUE_FIELD, 'foo.baz.test')
+    time = Time.parse("2012-04-02 18:20:59").to_i
+    d.run do
+      d.emit(original, time)
+    end
+    emits = d.emits
+    assert_equal 1, emits.length
+
+    first = emits[0]
+    assert_equal 'foo.bar.test', first[0]
+    assert_equal time, first[1]
+
+    record = first[2]
+    assert_equal 1, record.keys.size
+    assert_equal({"xxx"=>"first","yyy"=>"second"}, record['parsed'])
+
+    d = create_driver(CONFIG_HASH_VALUE_FIELD_RESERVE_DATA, 'foo.baz.test')
+    time = Time.parse("2012-04-02 18:20:59").to_i
+    d.run do
+      d.emit(original, time)
+    end
+    emits = d.emits
+    assert_equal 1, emits.length
+
+    first = emits[0]
+    assert_equal 'foo.bar.test', first[0]
+    assert_equal time, first[1]
+
+    record = first[2]
+    assert_equal 4, record.keys.size
+    assert_equal original['data'], record['data']
+    assert_equal original['xxx'], record['xxx']
+    assert_equal original['yyy'], record['yyy']
+    assert_equal({"xxx"=>"first","yyy"=>"second"}, record['parsed'])
+
+    d = create_driver(CONFIG_HASH_VALUE_FIELD_WITH_INJECT_KEY_PREFIX, 'foo.baz.test')
+    time = Time.parse("2012-04-02 18:20:59").to_i
+    d.run do
+      d.emit(original, time)
+    end
+    emits = d.emits
+    assert_equal 1, emits.length
+
+    first = emits[0]
+    assert_equal 'foo.bar.test', first[0]
+    assert_equal time, first[1]
+
+    record = first[2]
+    assert_equal 1, record.keys.size
+    assert_equal({"data.xxx"=>"first","data.yyy"=>"second"}, record['parsed'])
+  end
+
   #TODO: apache2
   # REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-parser
 version: !ruby/object:Gem::Version
-  version: 0.2.4
+  version: 0.3.0
 platform: ruby
 authors:
 - TAGOMORI Satoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-08-01 00:00:00.000000000 Z
+date: 2013-10-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -47,6 +47,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .travis.yml
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -59,7 +60,8 @@ files:
 - test/plugin/test_deparser.rb
 - test/plugin/test_out_parser.rb
 homepage: https://github.com/tagomoris/fluent-plugin-parser
-licenses: []
+licenses:
+- APLv2
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -77,7 +79,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.2
+rubygems_version: 2.0.3
 signing_key: 
 specification_version: 4
 summary: plugin to parse/combine fluentd log messages