fluent-plugin-parser 0.3.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d94615d4d0da1036e495a635ee91d1165281053d
-  data.tar.gz: e076017291eaa10cec2dbe7676e1d45b7e953bcb
+  metadata.gz: 981df9c360ea929797303eedebec95b3f884a574
+  data.tar.gz: ea2f3528e33cc43af7627b81ddc9b6be9ae8137a
 SHA512:
-  metadata.gz: 61820d91940e4322f7715ba8949dc6ca3e9cf558c4d2f21fecf14da57cc1f0211e35550c51ae6b81a97f9c80e2116d8bff881fa0c55133514d6c1e274f828f91
-  data.tar.gz: 514f354cd2d3674815fd213135bb0b843f2f79d23e69bee0f4826aae1f9528df3df31b5296721b5472bffd2a9086ac4be70420a58e55b0b1f7c04ffd1ce85369
+  metadata.gz: 61c26d1823b7e64f01495e6aae818d84e1e8cbc06d3545342f80d235474fed197e8965dbdfd33e2b746c8e54e24c35516bea7e295c4c75a793094d5f8ddffdfc
+  data.tar.gz: 82fa801ee3ecd797a43628f5d0c1019898b815d7197339efc11625374570687a6364a66920c97f56fc0e4d3ac58d18c90a72926690cdfe33bb50a76f902fb9ed

data/README.md

@@ -107,6 +107,19 @@ Other options (ex: `reserve_data`, `inject_key_prefix`) are available with `hash
 
     # output data: {"sales":"{\"user\":1,\"num\":2}", "parsed":{"sales.user":1, "sales.num":2}}
 
+Not to parse times (reserve that field like 'time' in record), specify `time_parse no`:
+
+    <match raw.sales.*>
+      type parser
+      tag sales
+      format json
+      key_name sales
+      hash_value_field parsed
+      time_parse no
+    </match>
+    # input string of 'sales': {"user":1,"num":2,"time":"2013-10-31 12:48:33"}
+    # output data: {"parsed":{"user":1, "num":2,"time":"2013-10-31 12:48:33"}}
+
 ### DeparserOutput
 
 To build CSV from field 'store','item','num', as field 'csv', without raw data:
@@ -137,5 +150,7 @@ To build same CSV, as additional field 'csv', with reserved raw fields:
 
 ## Copyright
 
-Copyright:: Copyright (c) 2012- TAGOMORI Satoshi (tagomoris)
-License::   Apache License, Version 2.0
+* Copyright
+  * Copyright (c) 2012- TAGOMORI Satoshi (tagomoris)
+* License
+  * Apache License, Version 2.0

data/fluent-plugin-parser.gemspec

@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-parser"
-  gem.version       = "0.3.0"
+  gem.version       = "0.3.1"
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
   gem.description   = %q{fluentd plugin to parse single field, or to combine log structure into single field}

data/lib/fluent/plugin/fixed_parser.rb

@@ -4,10 +4,40 @@
 module FluentExt; end
 
 class FluentExt::TextParser
-  class RegexpParser
+  class GenericParser
     include Fluent::Configurable
 
+    config_param :time_key, :string, :default => 'time'
     config_param :time_format, :string, :default => nil
+    config_param :time_parse, :bool, :default => true
+
+    def parse_time(record)
+      time = nil
+
+      unless @time_parse
+        return time, record
+      end
+
+      if value = record.delete(@time_key)
+        begin
+          time = if @time_format
+                   Time.strptime(value, @time_format).to_i
+                 else
+                   Time.parse(value).to_i
+                 end
+        rescue TypeError, ArgumentError => e
+          $log.warn "Failed to parse time", :key => @time_key, :value => value
+          record[@time_key] = value
+        end
+      end
+
+      return time, record
+    end
+  end
+
+  class RegexpParser < GenericParser
+    include Fluent::Configurable
+
     config_param :suppress_parse_error_log, :bool, :default => false
 
     def initialize(regexp, conf={})
@@ -28,44 +58,11 @@ class FluentExt::TextParser
         return nil, nil
       end
 
-      time = nil
       record = {}
-
       m.names.each {|name|
-        if value = m[name]
-          case name
-          when "time"
-            if @time_format
-              time = Time.strptime(value, @time_format).to_i
-            else
-              time = Time.parse(value).to_i
-            end
-          else
-            record[name] = value
-          end
-        end
+        record[name] = m[name] if m[name]
       }
-
-      return time, record
-    end
-  end
-
-  class GenericParser
-    include Fluent::Configurable
-
-    config_param :time_key, :string, :default => 'time'
-    config_param :time_format, :string, :default => nil
-
-    def parse_time(record)
-      time = nil
-      if value = record.delete(@time_key)
-        time = if @time_format
-                 Time.strptime(value, @time_format).to_i
-               else
-                 Time.parse(value).to_i
-               end
-      end
-      return time, record
+      parse_time(record)
     end
   end
 

data/test/plugin/test_out_parser.rb

@@ -442,6 +442,72 @@ class ParserOutputTest < Test::Unit::TestCase
     assert_equal({"data.xxx"=>"first","data.yyy"=>"second"}, record['parsed'])
   end
 
+  CONFIG_DONT_PARSE_TIME = %[
+    remove_prefix test
+    key_name data
+    format json
+    time_parse no
+  ]
+  def test_time_should_be_reserved
+    t = Time.now.to_i
+    d = create_driver(CONFIG_DONT_PARSE_TIME, 'test.in')
+
+    assert_equal false, d.instance.instance_eval{ @parser }.instance_eval{ @parser }.time_parse
+
+    d.run do
+      d.emit({'data' => '{"time":1383190430, "f1":"v1"}'}, t)
+      d.emit({'data' => '{"time":"1383190430", "f1":"v1"}'}, t)
+      d.emit({'data' => '{"time":"2013-10-31 12:34:03 +0900", "f1":"v1"}'}, t)
+    end
+    emits = d.emits
+    assert_equal 3, emits.length
+
+    assert_equal 'in', emits[0][0]
+    assert_equal 'v1', emits[0][2]['f1']
+    assert_equal 1383190430, emits[0][2]['time']
+    assert_equal t, emits[0][1]
+
+    assert_equal 'in', emits[1][0]
+    assert_equal 'v1', emits[1][2]['f1']
+    assert_equal "1383190430", emits[1][2]['time']
+    assert_equal t, emits[1][1]
+
+    assert_equal 'in', emits[2][0]
+    assert_equal 'v1', emits[2][2]['f1']
+    assert_equal '2013-10-31 12:34:03 +0900', emits[2][2]['time']
+    assert_equal t, emits[2][1]
+  end
+
+  CONFIG_INVALID_TIME_VALUE = %[
+    remove_prefix test
+    key_name data
+    format json
+  ] # 'time' is implicit @time_key
+  def test_invalid_time_data
+    # should not raise errors
+    t = Time.now.to_i
+    d = create_driver(CONFIG_INVALID_TIME_VALUE, 'test.in')
+    assert_nothing_raised {
+      d.run do
+        d.emit({'data' => '{"time":[], "f1":"v1"}'}, t)
+        d.emit({'data' => '{"time":"thisisnottime", "f1":"v1"}'}, t)
+      end
+    }
+    emits = d.emits
+    assert_equal 2, emits.length
+
+    assert_equal 'in', emits[0][0]
+    assert_equal t, emits[0][1]
+    assert_equal 'v1', emits[0][2]['f1']
+    assert_equal [], emits[0][2]['time']
+
+    assert_equal 'in', emits[1][0]
+    assert_equal t, emits[1][1]
+    assert_equal 'v1', emits[1][2]['f1']
+    assert_equal 'thisisnottime', emits[1][2]['time']
+  end
+
+
   #TODO: apache2
   # REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-parser
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors:
 - TAGOMORI Satoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-18 00:00:00.000000000 Z
+date: 2013-10-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake