fluent-plugin-parser 0.3.3 → 0.3.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2326d0682f48ee2aa993c41114c4b6135cf610e5
-  data.tar.gz: aa40d1f259c0781492004e21a79c64118d5ae2cf
+  metadata.gz: 11c8d0bf5065c2ea089887da434f380c6477797c
+  data.tar.gz: 05dd0a148aed3f8020eb839fd5042298464296ed
 SHA512:
-  metadata.gz: 10d523349d0c40468ce83b886e74bcbb2d89d3138cf36d09858c3302d398d67b0114e8b1693407b8fcc774ea9b2114d3b94cc8d560fcd4045a885f62f15d598b
-  data.tar.gz: cc0a2694b738d268faff7bdf3d7da786da0ffd785662ff130da252aa18b3c1cf0e6931b5417dae4047cae5ef5ee34b60163fbf6da26f505f60a6acbb2543aa3f
+  metadata.gz: 4bffef34b921fe845581381aa759fdeea68bf40ea7b95031f7e8909bd5516828f9b31964b550048bd1507e2dd337394e3c2f51ca38afd814c37c17c610353d40
+  data.tar.gz: 3471c464f75070f30e4ce2bd97e6bb5a10662e779d7cce00b4d0f41f2efdf6d5ba60c77f36a73ad46a81aa48f17ea10ff0e3f6527f9e1f82b7872ebb47fb1fbf

data/fluent-plugin-parser.gemspec

@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-parser"
-  gem.version       = "0.3.3"
+  gem.version       = "0.3.4"
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
   gem.description   = %q{fluentd plugin to parse single field, or to combine log structure into single field}

data/lib/fluent/plugin/fixed_parser.rb

@@ -11,10 +11,18 @@ class FluentExt::TextParser
     config_param :time_format, :string, :default => nil
     config_param :time_parse, :bool, :default => true
 
-    @cache1_key = nil
-    @cache1_time = nil
-    @cache2_key = nil
-    @cache2_time = nil
+    attr_accessor :log
+
+    def initialize
+      super
+
+      @cache1_key = nil
+      @cache1_time = nil
+      @cache2_key = nil
+      @cache2_time = nil
+
+      @log = nil
+    end
 
     def parse_time(record)
       time = nil
@@ -40,7 +48,7 @@ class FluentExt::TextParser
             @cache2_key = value
             @cache2_time = time
           rescue TypeError, ArgumentError => e
-            $log.warn "Failed to parse time", :key => @time_key, :value => value
+            @log.warn "Failed to parse time", :key => @time_key, :value => value
             record[@time_key] = value
           end
         end
@@ -67,7 +75,7 @@ class FluentExt::TextParser
       m = @regexp.match(text)
       unless m
         unless @suppress_parse_error_log
-          $log.warn "pattern not match: #{text}"
+          @log.warn "pattern not match: #{text}"
         end
 
         return nil, nil
@@ -87,7 +95,7 @@ class FluentExt::TextParser
       return parse_time(record)
     rescue Yajl::ParseError
       unless @suppress_parse_error_log
-        $log.warn "pattern not match(json): #{text.inspect}: #{$!}"
+        @log.warn "pattern not match(json): #{text.inspect}: #{$!}"
       end
 
       return nil, nil
@@ -133,16 +141,23 @@ class FluentExt::TextParser
     end
   end
 
-  class ApacheParser
+  class ApacheParser < GenericParser
     include Fluent::Configurable
 
     REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/
 
+    def initialize
+      super
+
+      @time_key = "time"
+      @time_format = "%d/%b/%Y:%H:%M:%S %z"
+   end
+
     def call(text)
       m = REGEXP.match(text)
       unless m
         unless @suppress_parse_error_log
-          $log.warn "pattern not match: #{text.inspect}"
+          @log.warn "pattern not match: #{text.inspect}"
         end
 
         return nil, nil
@@ -155,7 +170,6 @@ class FluentExt::TextParser
       user = (user == '-') ? nil : user
 
       time = m['time']
-      time = Time.strptime(time, "%d/%b/%Y:%H:%M:%S %z").to_i
 
       method = m['method']
       path = m['path']
@@ -173,6 +187,7 @@ class FluentExt::TextParser
       agent = (agent == '-') ? nil : agent
 
       record = {
+        "time" => time,
         "host" => host,
         "user" => user,
         "method" => method,
@@ -183,7 +198,7 @@ class FluentExt::TextParser
         "agent" => agent,
       }
 
-      return time, record
+      parse_time(record)
     end
   end
 
@@ -209,7 +224,11 @@ class FluentExt::TextParser
     TEMPLATE_FACTORIES[name] = factory
   end
 
-  def initialize
+  attr_accessor :log
+  attr_reader :parser
+
+  def initialize(logger)
+    @log = logger
     @parser = nil
   end
 
@@ -246,6 +265,8 @@ class FluentExt::TextParser
 
     end
 
+    @parser.log = @log
+
     if @parser.respond_to?(:configure)
       @parser.configure(conf)
     end

data/lib/fluent/plugin/out_parser.rb

@@ -12,6 +12,8 @@ class Fluent::ParserOutput < Fluent::Output
   config_param :replace_invalid_sequence, :bool, :default => false
   config_param :hash_value_field, :string, :default => nil
 
+  attr_reader :parser
+
   def initialize
     super
     require 'time'
@@ -39,7 +41,7 @@ class Fluent::ParserOutput < Fluent::Output
       @added_prefix_string = @add_prefix + '.'
     end
 
-    @parser = FluentExt::TextParser.new
+    @parser = FluentExt::TextParser.new(log())
     @parser.configure(conf)
   end
 

data/test/plugin/test_out_parser.rb

@@ -617,51 +617,74 @@ class ParserOutputTest < Test::Unit::TestCase
     end
   end
 
+  def swap_logger(instance)
+    raise "use with block" unless block_given?
+    parser_logger = instance.parser.log
+    dummy = DummyLogger.new
+    instance.parser.log = dummy
+    instance.parser.parser.log = dummy
+
+    restore = if instance.respond_to?("log=".to_sym)
+                saved_logger = instance.log
+                instance.log = dummy
+                lambda{ instance.log = saved_logger; instance.parser.log = instance.parser.parser.log = parser_logger }
+              else
+                saved_logger = $log
+                $log = dummy
+                lambda{ $log = saved_logger; instance.parser.log = instance.parser.parser.log = parser_logger }
+              end
+
+    yield
+
+    restore.call
+  end
+
   def test_suppress_parse_error_log
     # default(disabled) 'suppress_parse_error_log' is not specify
     d = create_driver(CONFIG_DEFAULT_SUPPRESS_PARSE_ERROR_LOG, 'test.in')
 
-    saved_logger = $log
-    $log = DummyLogger.new
-
-    assert_raise(DummyLoggerWarnedException) {
-      d.run do
-        d.emit({'message' => INVALID_MESSAGE}, Time.now.to_i)
-      end
-    }
-
-    assert_nothing_raised {
-      d.run do
-        d.emit({'message' => VALID_MESSAGE}, Time.now.to_i)
-      end
-    }
+    swap_logger(d.instance) do
+      assert_raise(DummyLoggerWarnedException) {
+        d.run do
+          d.emit({'message' => INVALID_MESSAGE}, Time.now.to_i)
+        end
+      }
+
+      assert_nothing_raised {
+        d.run do
+          d.emit({'message' => VALID_MESSAGE}, Time.now.to_i)
+        end
+      }
+    end
 
     # disabled 'suppress_parse_error_log'
     d = create_driver(CONFIG_DISABELED_SUPPRESS_PARSE_ERROR_LOG, 'test.in')
 
-    assert_raise(DummyLoggerWarnedException) {
-      d.run do
-        d.emit({'message' => INVALID_MESSAGE}, Time.now.to_i)
-      end
-    }
-
-    assert_nothing_raised {
-      d.run do
-        d.emit({'message' => VALID_MESSAGE}, Time.now.to_i)
-      end
-    }
+    swap_logger(d.instance) do
+      assert_raise(DummyLoggerWarnedException) {
+        d.run do
+          d.emit({'message' => INVALID_MESSAGE}, Time.now.to_i)
+        end
+      }
+
+      assert_nothing_raised {
+        d.run do
+          d.emit({'message' => VALID_MESSAGE}, Time.now.to_i)
+        end
+      }
+    end
 
     # enabled 'suppress_parse_error_log'
     d = create_driver(CONFIG_ENABELED_SUPPRESS_PARSE_ERROR_LOG, 'test.in')
 
-    assert_nothing_raised {
-      d.run do
-        d.emit({'message' => INVALID_MESSAGE}, Time.now.to_i)
-        d.emit({'message' => VALID_MESSAGE},   Time.now.to_i)
-      end
-    }
-
-    $log = saved_logger
+    swap_logger(d.instance) do
+      assert_nothing_raised {
+        d.run do
+          d.emit({'message' => INVALID_MESSAGE}, Time.now.to_i)
+          d.emit({'message' => VALID_MESSAGE},   Time.now.to_i)
+        end
+      }
+    end
   end
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-parser
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.3.4
 platform: ruby
 authors:
 - TAGOMORI Satoshi