fluentd 0.10.57 → 0.10.58

data/lib/fluent/input.rb

@@ -21,12 +21,16 @@ module Fluent
     include PluginId
     include PluginLoggerMixin
 
+    attr_accessor :router
+
     def initialize
       super
     end
 
     def configure(conf)
       super
+
+      @router = Engine
     end
 
     def start

data/lib/fluent/output.rb

@@ -61,12 +61,16 @@ module Fluent
     include PluginId
     include PluginLoggerMixin
 
+    attr_accessor :router
+
     def initialize
       super
     end
 
     def configure(conf)
       super
+
+      @router = Engine
     end
 
     def start
@@ -202,7 +206,7 @@ module Fluent
       }
 
       if sconf = conf.elements.select {|e| e.name == 'secondary' }.first
-        type = sconf['type'] || conf['type']
+        type = sconf['@type'] || conf['@type'] || sconf['type'] || conf['type']
         @secondary = Plugin.new_output(type)
         @secondary.configure(sconf)
 
@@ -395,7 +399,7 @@ module Fluent
                # secondary retry
                @retry_wait * (2 ** (@num_errors - 2 - @retry_limit))
              end
-      retry_wait = wait + (rand * (wait / 4.0) - (wait / 8.0))
+      retry_wait = wait.finite? ? wait + (rand * (wait / 4.0) - (wait / 8.0)) : wait
       @max_retry_wait ? [retry_wait, @max_retry_wait].min : retry_wait
     end
 
@@ -469,6 +473,7 @@ module Fluent
     config_set_default :flush_interval, nil
 
     attr_accessor :localtime
+    attr_reader :time_slicer # for test
 
     def configure(conf)
       super

data/lib/fluent/parser.rb

@@ -18,9 +18,38 @@
 module Fluent
   require 'fluent/registry'
 
-  class TextParser
-    class ParserError < StandardError
+  class ParserError < StandardError
+  end
+
+  class Parser
+    include Configurable
+
+    # SET false BEFORE CONFIGURE, to return nil when time not parsed
+    # 'configure()' may raise errors for unexpected configurations
+    attr_accessor :estimate_current_event
+
+    def initialize
+      super
+      @estimate_current_event = true
+    end
+
+    def configure(conf)
+      super
+    end
+
+    def parse(text)
+      raise NotImplementedError, "Implement this method in child class"
+    end
+
+    # Keep backward compatibility for existing plugins
+    def call(*a, &b)
+      parse(*a, &b)
     end
+  end
+
+  class TextParser
+    # Keep backward compatibility for existing plugins
+    ParserError = ::Fluent::ParserError
 
     class TimeParser
       def initialize(time_format)
@@ -129,16 +158,11 @@ module Fluent
       end
     end
 
-    class RegexpParser
-      include Configurable
+    class RegexpParser < Parser
       include TypeConverter
 
       config_param :time_format, :string, :default => nil
 
-      # SET false BEFORE CONFIGURE, to return nil when time not parsed
-      # 'configure()' may raise errors for unexpected configurations
-      attr_accessor :estimate_current_event
-
       def initialize(regexp, conf={})
         super()
         @regexp = regexp
@@ -147,7 +171,6 @@ module Fluent
         end
 
         @time_parser = TimeParser.new(@time_format)
-        @estimate_current_event = true
         @mutex = Mutex.new
       end
 
@@ -160,7 +183,7 @@ module Fluent
         {'format' => @regexp, 'time_format' => @time_format}
       end
 
-      def call(text)
+      def parse(text)
         m = @regexp.match(text)
         unless m
           if block_given?
@@ -201,21 +224,10 @@ module Fluent
       end
     end
 
-    class JSONParser
-      include Configurable
-
+    class JSONParser < Parser
       config_param :time_key, :string, :default => 'time'
       config_param :time_format, :string, :default => nil
 
-      # SET false BEFORE CONFIGURE, to return nil when time not parsed
-      # 'configure()' may raise errors for unexpected configurations
-      attr_accessor :estimate_current_event
-
-      def initialize
-        super
-        @estimate_current_event = true
-      end
-
       def configure(conf)
         super
 
@@ -225,7 +237,7 @@ module Fluent
         end
       end
 
-      def call(text)
+      def parse(text)
         record = Yajl.load(text)
 
         if value = record.delete(@time_key)
@@ -260,21 +272,13 @@ module Fluent
       end
     end
 
-    class ValuesParser
-      include Configurable
+    class ValuesParser < Parser
       include TypeConverter
 
       config_param :keys, :string
       config_param :time_key, :string, :default => nil
       config_param :time_format, :string, :default => nil
 
-      attr_accessor :estimate_current_event
-
-      def initialize
-        super
-        @estimate_current_event = true
-      end
-
       def configure(conf)
         super
 
@@ -331,11 +335,16 @@ module Fluent
     class TSVParser < ValuesParser
       config_param :delimiter, :string, :default => "\t"
 
-      def call(text)
+      def configure(conf)
+        super
+        @key_num = @keys.length
+      end
+
+      def parse(text)
         if block_given?
-          yield values_map(text.split(@delimiter))
+          yield values_map(text.split(@delimiter, @key_num))
         else
-          return values_map(text.split(@delimiter))
+          return values_map(text.split(@delimiter, @key_num))
         end
       end
     end
@@ -350,7 +359,7 @@ module Fluent
         super(conf)
       end
 
-      def call(text)
+      def parse(text)
         @keys  = []
         values = []
 
@@ -374,7 +383,7 @@ module Fluent
         require 'csv'
       end
 
-      def call(text)
+      def parse(text)
         if block_given?
           yield values_map(CSV.parse_line(text))
         else
@@ -383,19 +392,10 @@ module Fluent
       end
     end
 
-    class NoneParser
-      include Configurable
-
+    class NoneParser < Parser
       config_param :message_key, :string, :default => 'message'
 
-      attr_accessor :estimate_current_event
-
-      def initialize
-        super
-        @estimate_current_event = true
-      end
-
-      def call(text)
+      def parse(text)
         record = {}
         record[@message_key] = text
         time = @estimate_current_event ? Engine.now : nil
@@ -407,13 +407,12 @@ module Fluent
       end
     end
 
-    class ApacheParser
-      include Configurable
-
+    class ApacheParser < Parser
       REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/
       TIME_FORMAT = "%d/%b/%Y:%H:%M:%S %z"
 
       def initialize
+        super
         @time_parser = TimeParser.new(TIME_FORMAT)
         @mutex = Mutex.new
       end
@@ -422,7 +421,7 @@ module Fluent
         {'format' => REGEXP, 'time_format' => TIME_FORMAT}
       end
 
-      def call(text)
+      def parse(text)
         m = REGEXP.match(text)
         unless m
           if block_given?
@@ -476,9 +475,7 @@ module Fluent
       end
     end
 
-    class SyslogParser
-      include Configurable
-
+    class SyslogParser < Parser
       # From existence TextParser pattern
       REGEXP = /^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
       # From in_syslog default pattern
@@ -487,11 +484,8 @@ module Fluent
       config_param :time_format, :string, :default => "%b %d %H:%M:%S"
       config_param :with_priority, :bool, :default => false
 
-      attr_accessor :estimate_current_event
-
       def initialize
         super
-        @estimate_current_event = true
         @mutex = Mutex.new
       end
 
@@ -506,7 +500,7 @@ module Fluent
         {'format' => @regexp, 'time_format' => @time_format}
       end
 
-      def call(text)
+      def parse(text)
         m = @regexp.match(text)
         unless m
           if block_given?
@@ -545,9 +539,7 @@ module Fluent
       end
     end
 
-    class MultilineParser
-      include Configurable
-
+    class MultilineParser < Parser
       config_param :format_firstline, :string, :default => nil
 
       FORMAT_MAX_NUM = 20
@@ -572,7 +564,7 @@ module Fluent
         end
       end
 
-      def call(text, &block)
+      def parse(text, &block)
         if block
           @parser.call(text, &block)
         else
@@ -648,7 +640,9 @@ module Fluent
     }
 
     def self.register_template(name, regexp_or_proc, time_format=nil)
-      if regexp_or_proc.is_a?(Regexp)
+      if regexp_or_proc.is_a?(Class)
+        factory = Proc.new { regexp_or_proc.new }
+      elsif regexp_or_proc.is_a?(Regexp)
         regexp = regexp_or_proc
         factory = Proc.new { RegexpParser.new(regexp, {'time_format'=>time_format}) }
       else
@@ -658,26 +652,9 @@ module Fluent
       TEMPLATE_REGISTRY.register(name, factory)
     end
 
-    def initialize
-      @parser = nil
-      @estimate_current_event = nil
-    end
-
-    attr_reader :parser
-
-    # SET false BEFORE CONFIGURE, to return nil when time not parsed
-    # 'configure()' may raise errors for unexpected configurations
-    attr_accessor :estimate_current_event
-
-    def configure(conf, required=true)
-      format = conf['format']
-
-      if format == nil
-        if required
-          raise ConfigError, "'format' parameter is required"
-        else
-          return nil
-        end
+    def self.lookup(format)
+      if format.nil?
+        raise ConfigError, "'format' parameter is required"
       end
 
       if format[0] == ?/ && format[format.length-1] == ?/
@@ -691,7 +668,7 @@ module Fluent
           raise ConfigError, "Invalid regexp '#{format[1..-2]}': #{$!}"
         end
 
-        @parser = RegexpParser.new(regexp, conf)
+        RegexpParser.new(regexp)
       else
         # built-in template
         begin
@@ -699,9 +676,26 @@ module Fluent
         rescue ConfigError => e # keep same error message
           raise ConfigError, "Unknown format template '#{format}'"
         end
-        @parser = factory.call
+
+        factory.call
       end
+    end
+
+    def initialize
+      @parser = nil
+      @estimate_current_event = nil
+    end
+
+    attr_reader :parser
+
+    # SET false BEFORE CONFIGURE, to return nil when time not parsed
+    # 'configure()' may raise errors for unexpected configurations
+    attr_accessor :estimate_current_event
+
+    def configure(conf, required=true)
+      format = conf['format']
 
+      @parser = TextParser.lookup(format)
       if ! @estimate_current_event.nil? && @parser.respond_to?(:'estimate_current_event=')
         @parser.estimate_current_event = @estimate_current_event
       end
@@ -715,9 +709,9 @@ module Fluent
 
     def parse(text, &block)
       if block
-        @parser.call(text, &block)
+        @parser.parse(text, &block)
       else # keep backward compatibility. Will be removed at v1
-        return @parser.call(text)
+        return @parser.parse(text)
       end
     end
   end

data/lib/fluent/plugin.rb

@@ -17,6 +17,8 @@
 #
 module Fluent
   class PluginClass
+    # This class is refactored using Fluent::Registry at v0.14
+
     def initialize
       @input = {}
       @output = {}
@@ -35,6 +37,14 @@ module Fluent
       register_impl('buffer', @buffer, type, klass)
     end
 
+    def register_parser(type, klass)
+      TextParser.register_template(type, klass)
+    end
+
+    def register_formatter(type, klass)
+      TextFormatter.register_template(type, klass)
+    end
+
     def new_input(type)
       new_impl('input', @input, type)
     end
@@ -47,6 +57,14 @@ module Fluent
       new_impl('buffer', @buffer, type)
     end
 
+    def new_parser(type)
+      TextParser.lookup(type)
+    end
+
+    def new_formatter(type)
+      TextFormatter.lookup(type)
+    end
+
     def load_plugins
       dir = File.join(File.dirname(__FILE__), "plugin")
       load_plugin_dir(dir)

data/lib/fluent/plugin/buf_file.rb

@@ -91,9 +91,9 @@ module Fluent
       super
 
       if @@buffer_paths.has_key?(@buffer_path)
-        raise ConfigError, "Other '#{@@buffer_paths[@buffer_path]}' plugin already use same buffer_path: type = #{conf['type']}, buffer_path = #{@buffer_path}"
+        raise ConfigError, "Other '#{@@buffer_paths[@buffer_path]}' plugin already use same buffer_path: type = #{conf['@type'] || conf['type']}, buffer_path = #{@buffer_path}"
       else
-        @@buffer_paths[@buffer_path] = conf['type']
+        @@buffer_paths[@buffer_path] = conf['@type'] || conf['type']
       end
 
       if pos = @buffer_path.index('*')
@@ -112,7 +112,7 @@ module Fluent
     end
 
     def start
-      FileUtils.mkdir_p File.dirname(@buffer_path_prefix+"path")
+      FileUtils.mkdir_p File.dirname(@buffer_path_prefix + "path"), :mode => DEFAULT_DIR_PERMISSION
       super
     end