fluent-plugin-parser 0.4.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1cfb579022231c0bf26eb16a1156a7e9e5aa24b8
-  data.tar.gz: 6b096b93cce0ddf6995e92058e44a46c0ac59b6f
+  metadata.gz: 7a7890c46c538f109454bf0442db0dcd2a4f75b0
+  data.tar.gz: f50a1814587e4702f3ae47eef54f6a4208ecd4ad
 SHA512:
-  metadata.gz: 9f59e117075d751a7a734b8e4ad15e9ef1e1a0bce142285c4f6e7b534d235009ee207d0918219ec5a3fbfdcb928280abc893957587ff5d8059d519cf0f016469
-  data.tar.gz: e171adea68763ffd821fc34d7ef27298122c2155e8238d461aa563e3cf3395a97c8b18ae5a125e9368222d3c9fd08ac9f1e63bea67063ad6d49f194f00065e9e
+  metadata.gz: 37956f2b75dc8aed2f471ea321afbea843f97ca568a09e3cf169bb159f38c8d481e431c30141611cb909d8a7dfb712856aec4fd24d5937edb9610c1576fc85af
+  data.tar.gz: 5e8af5d4544c7cd879345c06a10d28210974f2de34cd1a75ad1e63232739269b32e6e0eedd5ffc13ee740ec2fadd10fefe385bdfa1186b76be266f0399027b59

data/fluent-plugin-parser.gemspec

@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-parser"
-  gem.version       = "0.4.0"
+  gem.version       = "0.4.1"
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
   gem.description   = %q{fluentd plugin to parse single field, or to combine log structure into single field}
@@ -14,6 +14,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
 
+  gem.add_development_dependency "test-unit"
   gem.add_development_dependency "rake"
   gem.add_runtime_dependency "fluentd", ">= 0.10.54"
 end

data/lib/fluent/plugin/out_parser.rb

@@ -21,6 +21,11 @@ class Fluent::ParserOutput < Fluent::Output
     require 'time'
   end
 
+  # Define `log` method for v0.10.42 or earlier
+  unless method_defined?(:log)
+    define_method("log") { $log }
+  end
+
   def configure(conf)
     super
 
@@ -70,8 +75,16 @@ class Fluent::ParserOutput < Fluent::Output
       raw_value = record[@key_name]
       begin
         @parser.parse(raw_value) do |t,values|
-          t ||= time
-          handle_parsed(tag, record, t, values)
+          if values
+            t ||= time
+            handle_parsed(tag, record, t, values)
+          else
+            log.warn "pattern not match with data '#{raw_value}'" unless @suppress_parse_error_log
+            if @reserve_data
+              t = time
+              handle_parsed(tag, record, time, {})
+            end
+          end
         end
       rescue Fluent::TextParser::ParserError => e
         log.warn e.message unless @suppress_parse_error_log
@@ -82,8 +95,16 @@ class Fluent::ParserOutput < Fluent::Output
           end
           replaced_string = replace_invalid_byte(raw_value)
           @parser.parse(replaced_string) do |t,values|
-            t ||= time
-            handle_parsed(tag, record, t, values)
+            if values
+              t ||= time
+              handle_parsed(tag, record, t, values)
+            else
+              log.warn "pattern not match with data '#{raw_value}'" unless @suppress_parse_error_log
+              if @reserve_data
+                t = time
+                handle_parsed(tag, record, time, {})
+              end
+            end
           end
         else
           raise
@@ -106,11 +127,7 @@ class Fluent::ParserOutput < Fluent::Output
     if @reserve_data
       r = r ? record.merge(r) : record
     end
-    if r
-      Fluent::Engine.emit(tag, t, r)
-    else
-      log.warn "pattern not match #{raw_value}" unless @suppress_parse_error_log
-    end
+    Fluent::Engine.emit(tag, t, r)
   end
 
   def replace_invalid_byte(string)

data/test/custom_parser.rb

@@ -1,6 +1,8 @@
 module Fluent
   class TextParser
-    class KVPairParser
+    class KVPairParser < Parser
+      Plugin.register_parser('kv_pair', self)
+
       # key<delim1>value is pair and <pair><delim2><pair> ...
       # newline splits records
       include Configurable
@@ -16,7 +18,7 @@ module Fluent
         @time_parser = TimeParser.new(@time_format)
       end
 
-      def call(text)
+      def parse(text)
         text.split("\n").each do |line|
           pairs = text.split(@delim2)
           record = {}
@@ -33,7 +35,5 @@ module Fluent
         end
       end
     end
-
-    register_template("kv_pair", Proc.new { KVPairParser.new })
   end
 end

data/test/plugin/test_out_parser.rb

@@ -196,7 +196,7 @@ class ParserOutputTest < Test::Unit::TestCase
     d3x = create_driver(%[
       tag parsed
       key_name      data
-      format        /^(?<x>\d)(?<y>\d) (?<t>.+)$/
+      format        /^(?<x>\\d)(?<y>\\d) (?<t>.+)$/
       reserve_data  yes
     ], 'test.in')
     time = Time.parse("2012-04-02 18:20:59").to_i

data/test/plugin/test_out_parser_for_parsers.rb

@@ -153,7 +153,7 @@ add_prefix    parsed
 key_name      message
 format        syslog
 EOF
-    time = Time.parse("2014-11-05 15:59:30").to_i
+    time = Time.parse("11/05 15:59:30").to_i # time is assumed as current year
     d.run do
       d.emit({"message" => logs[0]}, time)
       d.emit({"message" => logs[1]}, time)
@@ -165,7 +165,7 @@ EOF
 
     e = emits[0]
     assert_equal 'parsed.in', e[0]
-    assert_equal Time.parse("2014-11-05 16:19:48").to_i, e[1]
+    assert_equal Time.parse("11/05 16:19:48").to_i, e[1]
     r = e[2]
     assert_equal 'myhost.local', r['host']
     assert_equal 'netbiosd', r['ident']
@@ -174,7 +174,7 @@ EOF
 
     e = emits[1]
     assert_equal 'parsed.in', e[0]
-    assert_equal Time.parse("2014-11-05 16:21:20").to_i, e[1]
+    assert_equal Time.parse("11/05 16:21:20").to_i, e[1]
     r = e[2]
     assert_equal 'myhost.local', r['host']
     assert_equal 'coreaudiod', r['ident']
@@ -183,7 +183,7 @@ EOF
 
     e = emits[2]
     assert_equal 'parsed.in', e[0]
-    assert_equal Time.parse("2014-11-05 16:21:20").to_i, e[1]
+    assert_equal Time.parse("11/05 16:21:20").to_i, e[1]
     r = e[2]
     assert_equal 'myhost.local', r['host']
     assert_equal 'coreaudiod', r['ident']

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-parser
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.1
 platform: ruby
 authors:
 - TAGOMORI Satoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-11-05 00:00:00.000000000 Z
+date: 2015-02-20 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -80,7 +94,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: plugin to parse/combine fluentd log messages