fluent-plugin-parser 0.2.3 → 0.2.4

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 2e0393ad371a16008916ab9e6bf614d56be0a6a7
+  data.tar.gz: 98efdde5b556caa2452769a66582beba39bce9de
+SHA512:
+  metadata.gz: 3cb147341062d8f32d9e3eeaee6971bbf1d539994148b9eb63260c2aafc77c0a684ceb042f1a01c6c8abea3ecd1f18bf366108f92b7d2e796d98c5728df4dcc0
+  data.tar.gz: d4c5308d819343ca895f62949c38d6e2fd35bf4b63ae78c42aa1aba4da9513ac293e14242e57abc4722b14475c8bb4d93f8b4a302ebc8d046db9e4e98d820d62

data/README.md

@@ -67,7 +67,7 @@ Format 'ltsv'(Labeled-TSV (Tab separated values)) is also supported:
 
 About LTSV, see: http://ltsv.org/
 
-If you want to suppress 'pattern not match' log, specify 'suppress_parse_error_log true' to configuration.
+If you want to suppress 'pattern not match' log, specify 'suppress\_parse\_error\_log true' to configuration.
 default value is false.
 
     <match in.hogelog>
@@ -78,6 +78,18 @@ default value is false.
       suppress_parse_error_log true
     </match>
 
+To store parsed values with specified key name prefix, use `inject_key_prefix` option:
+
+    <match raw.sales.*>
+      type parser
+      tag sales
+      format json
+      key_name sales
+      reserve_data      yes
+      inject_key_prefix sales.
+    </match>
+    # input string of 'sales': {"user":1,"num":2}
+    # output data: {"sales":"{\"user\":1,\"num\":2}","sales.user":1, "sales.num":2}
 
 ### DeparserOutput
 

data/fluent-plugin-parser.gemspec

@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-parser"
-  gem.version       = "0.2.3"
+  gem.version       = "0.2.4"
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
   gem.description   = %q{fluentd plugin to parse single field, or to combine log structure into single field}

data/lib/fluent/plugin/fixed_parser.rb

@@ -141,25 +141,25 @@ class FluentExt::TextParser
 
       user = m['user']
       user = (user == '-') ? nil : user
-      
+
       time = m['time']
       time = Time.strptime(time, "%d/%b/%Y:%H:%M:%S %z").to_i
-      
+
       method = m['method']
       path = m['path']
-      
-      code = m['code'].to_i 
+
+      code = m['code'].to_i
       code = nil if code == 0
 
       size = m['size']
       size = (size == '-') ? nil : size.to_i
-      
+
       referer = m['referer']
       referer = (referer == '-') ? nil : referer
-      
+
       agent = m['agent']
       agent = (agent == '-') ? nil : agent
-      
+
       record = {
         "host" => host,
         "user" => user,
@@ -169,7 +169,7 @@ class FluentExt::TextParser
         "size" => size,
         "referer" => referer,
         "agent" => agent,
-      } 
+      }
 
       return time, record
     end
@@ -187,7 +187,7 @@ class FluentExt::TextParser
   }
 
   def self.register_template(name, regexp_or_proc, time_format=nil)
-    
+
     factory = if regexp_or_proc.is_a?(Regexp)
                 regexp = regexp_or_proc
                 Proc.new { RegexpParser.new(regexp, {'time_format'=>time_format}) }

data/lib/fluent/plugin/out_parser.rb

@@ -8,6 +8,7 @@ class Fluent::ParserOutput < Fluent::Output
   config_param :add_prefix, :string, :default => nil
   config_param :key_name, :string
   config_param :reserve_data, :bool, :default => false
+  config_param :inject_key_prefix, :string, :default => nil
   config_param :replace_invalid_sequence, :bool, :default => false
 
   def initialize
@@ -43,8 +44,8 @@ class Fluent::ParserOutput < Fluent::Output
             if @remove_prefix and
                 ( (tag.start_with?(@removed_prefix_string) and tag.length > @removed_length) or tag == @remove_prefix)
               tag = tag[@removed_length..-1]
-            end 
-            if @add_prefix 
+            end
+            if @add_prefix
               tag = if tag and tag.length > 0
                       @added_prefix_string + tag
                     else
@@ -63,6 +64,9 @@ class Fluent::ParserOutput < Fluent::Output
                    end
         t ||= time
         r = if values
+              if @inject_key_prefix
+                values = Hash[values.map{|k,v| [ @inject_key_prefix + k, v ]}]
+              end
               record.merge(values)
             else
               record
@@ -73,7 +77,11 @@ class Fluent::ParserOutput < Fluent::Output
       es.each {|time,record|
         value = record[@key_name]
         t,values = if value
-                     parse(value)
+                     parsed = parse(value)
+                     if @inject_key_prefix
+                       parsed = Hash[parsed.map{|k,v| [ @inject_key_prefix + k, v ]}]
+                     end
+                     parsed
                    else
                      [nil, nil]
                    end

data/test/plugin/test_out_parser.rb

@@ -4,7 +4,7 @@ class ParserOutputTest < Test::Unit::TestCase
   def setup
     Fluent::Test.setup
   end
-  
+
   CONFIG = %[
     remove_prefix test
     add_prefix    parsed
@@ -292,7 +292,7 @@ class ParserOutputTest < Test::Unit::TestCase
     key_name data
     keys key1,key2,key3
   ]
-  def test_emit_ltsv
+  def test_emit_tsv
     d = create_driver(CONFIG_TSV, 'foo.baz.test')
     time = Time.parse("2012-04-02 18:20:59").to_i
     d.run do
@@ -317,7 +317,7 @@ class ParserOutputTest < Test::Unit::TestCase
     key_name data
     keys key1,key2,key3
   ]
-  def test_emit_ltsv
+  def test_emit_csv
     d = create_driver(CONFIG_CSV, 'foo.baz.test')
     time = Time.parse("2012-04-02 18:20:59").to_i
     d.run do
@@ -335,6 +335,35 @@ class ParserOutputTest < Test::Unit::TestCase
     assert_equal 'value"ThreeYes!', first[2]['key3']
   end
 
+  CONFIG_KEY_PREFIX = %[
+    remove_prefix foo.baz
+    add_prefix foo.bar
+    format       json
+    key_name     data
+    reserve_data yes
+    inject_key_prefix data.
+  ]
+  def test_inject_key_prefix
+    d = create_driver(CONFIG_KEY_PREFIX, 'foo.baz.test')
+    time = Time.parse("2012-04-02 18:20:59").to_i
+    d.run do
+      d.emit({'data' => '{"xxx":"first","yyy":"second"}', 'xxx' => 'x', 'yyy' => 'y'}, time)
+    end
+    emits = d.emits
+    assert_equal 1, emits.length
+
+    first = emits[0]
+    assert_equal 'foo.bar.test', first[0]
+    assert_equal time, first[1]
+
+    assert_equal '{"xxx":"first","yyy":"second"}', first[2]['data']
+    assert_equal 'x', first[2]['xxx']
+    assert_equal 'y', first[2]['yyy']
+    assert_equal 'first', first[2]['data.xxx']
+    assert_equal 'second', first[2]['data.yyy']
+    assert_equal 5, first[2].keys.size
+  end
+
   #TODO: apache2
   # REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/
 
@@ -393,18 +422,18 @@ class ParserOutputTest < Test::Unit::TestCase
     assert_equal '?'.force_encoding('US-ASCII'), emits[0][2]['message']
   end
 
-  # suppress_parse_error_log test 
+  # suppress_parse_error_log test
   CONFIG_DISABELED_SUPPRESS_PARSE_ERROR_LOG = %[
     tag hogelog
     format /^col1=(?<col1>.+) col2=(?<col2>.+)$/
     key_name message
-    suppress_parse_error_log false 
+    suppress_parse_error_log false
   ]
   CONFIG_ENABELED_SUPPRESS_PARSE_ERROR_LOG = %[
     tag hogelog
     format /^col1=(?<col1>.+) col2=(?<col2>.+)$/
     key_name message
-    suppress_parse_error_log true 
+    suppress_parse_error_log true
   ]
   CONFIG_DEFAULT_SUPPRESS_PARSE_ERROR_LOG = %[
     tag hogelog

metadata

@@ -1,46 +1,41 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-parser
 version: !ruby/object:Gem::Version
-  version: 0.2.3
-  prerelease: 
+  version: 0.2.4
 platform: ruby
 authors:
 - TAGOMORI Satoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-04-08 00:00:00.000000000 Z
+date: 2013-08-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 description: fluentd plugin to parse single field, or to combine log structure into
@@ -65,29 +60,29 @@ files:
 - test/plugin/test_out_parser.rb
 homepage: https://github.com/tagomoris/fluent-plugin-parser
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 2.0.2
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: plugin to parse/combine fluentd log messages
 test_files:
 - test/helper.rb
 - test/plugin/test_deparser.rb
 - test/plugin/test_out_parser.rb
+has_rdoc: