fluentd 0.10.45 → 0.10.46

data/lib/fluent/configurable.rb

@@ -0,0 +1,108 @@
+module Fluent
+  require 'fluent/config/configure_proxy'
+  require 'fluent/config/section'
+  require 'fluent/config/error'
+  require 'fluent/registry'
+
+  module Configurable
+    attr_reader :config
+
+    def self.included(mod)
+      mod.extend(ClassMethods)
+    end
+
+    def initialize
+      # to simulate implicit 'attr_accessor' by config_param / config_section and its value by config_set_default
+      proxy = self.class.merged_configure_proxy
+      proxy.params.keys.each do |name|
+        if proxy.defaults.has_key?(name)
+          instance_variable_set("@#{name}".to_sym, proxy.defaults[name])
+        end
+      end
+      proxy.sections.keys.each do |name|
+        subproxy = proxy.sections[name]
+        if subproxy.multi?
+          instance_variable_set("@#{subproxy.param_name}".to_sym, [])
+        else
+          instance_variable_set("@#{subproxy.param_name}".to_sym, nil)
+        end
+      end
+    end
+
+    def configure(conf)
+      @config = conf
+
+      logger = self.respond_to?(:log) ? log : $log
+      proxy = self.class.merged_configure_proxy
+
+      root = Fluent::Config::SectionGenerator.generate(proxy, conf, logger)
+      @config_root_section = root
+
+      root.instance_eval{ @params.keys }.each do |param_name|
+        varname = "@#{param_name}".to_sym
+        if (! root[param_name].nil?) || instance_variable_get(varname).nil?
+          instance_variable_set(varname, root[param_name])
+        end
+      end
+
+      self
+    end
+
+    CONFIG_TYPE_REGISTRY = Registry.new(:config_type, 'fluent/plugin/type_')
+
+    def self.register_type(type, callable = nil, &block)
+      callable ||= block
+      CONFIG_TYPE_REGISTRY.register(type, callable)
+    end
+
+    def self.lookup_type(type)
+      CONFIG_TYPE_REGISTRY.lookup(type)
+    end
+
+    module ClassMethods
+      def configure_proxy_map
+        map = {}
+        self.define_singleton_method(:configure_proxy_map){ map }
+        map
+      end
+
+      def configure_proxy(mod_name)
+        map = configure_proxy_map
+        unless map[mod_name]
+          proxy = Fluent::Config::ConfigureProxy.new(mod_name, required: true, multi: false)
+          map[mod_name] = proxy
+        end
+        map[mod_name]
+      end
+
+      def config_param(name, *args, &block)
+        configure_proxy(self.name).config_param(name, *args, &block)
+        attr_accessor name
+      end
+
+      def config_set_default(name, defval)
+        configure_proxy(self.name).config_set_default(name, defval)
+      end
+
+      def config_section(name, *args, &block)
+        configure_proxy(self.name).config_section(name, *args, &block)
+        attr_accessor configure_proxy(self.name).sections[name].param_name
+      end
+
+      def merged_configure_proxy
+        configurables = ancestors.reverse.select{ |a| a.respond_to?(:configure_proxy) }
+
+        # 'a.object_id.to_s' is to support anonymous class
+        #   which created in tests to overwrite original behavior temporally
+        #
+        # p Module.new.name   #=> nil
+        # p Class.new.name    #=> nil
+        # p AnyGreatClass.dup.name #=> nil
+        configurables.map{ |a| a.configure_proxy(a.name || a.object_id.to_s) }.reduce(:merge)
+      end
+    end
+  end
+
+  # load default types
+  require 'fluent/config/types'
+end

data/lib/fluent/engine.rb

@@ -68,11 +68,12 @@ module Fluent
       }
     end
 
-    def parse_config(io, fname, basepath=Dir.pwd)
+    def parse_config(io, fname, basepath = Dir.pwd, v1_config = false)
       conf = if fname =~ /\.rb$/
+               require 'fluent/config/dsl'
                Config::DSL::Parser.parse(io, File.join(basepath, fname))
              else
-               Config.parse(io, fname, basepath)
+               Config.parse(io, fname, basepath, v1_config)
              end
       configure(conf)
       conf.check_not_fetched {|key,e|
@@ -144,7 +145,7 @@ module Fluent
         # this is not thread-safe but inconsistency doesn't
         # cause serious problems while locking causes.
         if @match_cache_keys.size >= MATCH_CACHE_SIZE
-          @match_cache_keys.delete @match_cache_keys.shift
+          @match_cache.delete @match_cache_keys.shift
         end
         @match_cache[tag] = target
         @match_cache_keys << tag

data/lib/fluent/load.rb

@@ -23,7 +23,6 @@ require 'fluent/version'
 require 'fluent/log'
 require 'fluent/status'
 require 'fluent/config'
-require 'fluent/config_dsl'
 require 'fluent/engine'
 require 'fluent/mixin'
 require 'fluent/process'

data/lib/fluent/parser.rb

@@ -16,6 +16,8 @@
 #    limitations under the License.
 #
 module Fluent
+  require 'fluent/registry'
+
   class TextParser
     class TimeParser
       def initialize(time_format)
@@ -236,7 +238,11 @@ module Fluent
 
         if @time_key
           value = record.delete(@time_key)
-          time = @mutex.synchronize { @time_parser.parse(value) }
+          time = if value.nil?
+                   Engine.now
+                 else
+                   @mutex.synchronize { @time_parser.parse(value) }
+                 end
         else
           time = Engine.now
         end
@@ -444,7 +450,8 @@ module Fluent
       end
     end
 
-    TEMPLATE_FACTORIES = {
+    TEMPLATE_REGISTRY = Registry.new(:config_type, 'fluent/plugin/parser_')
+    {
       'apache' => Proc.new { RegexpParser.new(/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/, {'time_format'=>"%d/%b/%Y:%H:%M:%S %z"}) },
       'apache2' => Proc.new { ApacheParser.new },
       'syslog' => Proc.new { RegexpParser.new(/^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?[^\:]*\: *(?<message>.*)$/, {'time_format'=>"%b %d %H:%M:%S"}) },
@@ -455,6 +462,8 @@ module Fluent
       'nginx' => Proc.new { RegexpParser.new(/^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/,  {'time_format'=>"%d/%b/%Y:%H:%M:%S %z"}) },
       'none' => Proc.new { NoneParser.new },
       'multiline' => Proc.new { MultilineParser.new },
+    }.each { |name, factory|
+      TEMPLATE_REGISTRY.register(name, factory)
     }
 
     def self.register_template(name, regexp_or_proc, time_format=nil)
@@ -465,7 +474,7 @@ module Fluent
         factory = regexp_or_proc
       end
 
-      TEMPLATE_FACTORIES[name] = factory
+      TEMPLATE_REGISTRY.register(name, factory)
     end
 
     def initialize
@@ -500,8 +509,9 @@ module Fluent
 
       else
         # built-in template
-        factory = TEMPLATE_FACTORIES[format]
-        unless factory
+        begin
+          factory = TEMPLATE_REGISTRY.lookup(format)
+        rescue ConfigError => e # keep same error message
           raise ConfigError, "Unknown format template '#{format}'"
         end
         @parser = factory.call

data/lib/fluent/plugin/buf_memory.rb

@@ -71,20 +71,28 @@ module Fluent
       super
     end
 
+    config_param :flush_at_shutdown, :bool, :default => true
     # Overwrite default BasicBuffer#buffer_queue_limit
     # to limit total memory usage upto 512MB.
     config_set_default :buffer_queue_limit, 64
 
     def configure(conf)
       super
+
+      unless @flush_at_shutdown
+        $log.warn "When flush_at_shutdown is false, buf_memory discards buffered chunks at shutdown."
+        $log.warn "Please confirm 'flush_at_shutdown false' configuration is correct or not."
+      end
     end
 
     def before_shutdown(out)
-      synchronize do
-        @map.each_key {|key|
-          push(key)
-        }
-        while pop(out)
+      if @flush_at_shutdown
+        synchronize do
+          @map.each_key {|key|
+            push(key)
+          }
+          while pop(out)
+          end
         end
       end
     end

data/lib/fluent/plugin/in_forward.rb

@@ -31,6 +31,8 @@ module Fluent
     config_param :backlog, :integer, :default => nil
     # SO_LINGER 0 to send RST rather than FIN to avoid lots of connections sitting in TIME_WAIT at src
     config_param :linger_timeout, :integer, :default => 0
+    # This option is for Cool.io's loop wait timeout to avoid loop stuck at shutdown. Almost users don't need to change this value.
+    config_param :blocking_timeout, :time, :default => 0.5
 
     def configure(conf)
       super
@@ -56,10 +58,12 @@ module Fluent
       @loop.watchers.each {|w| w.detach }
       @loop.stop
       @usock.close
-      listen_address = (@bind == '0.0.0.0' ? '127.0.0.1' : @bind)
-      # This line is for connecting listen socket to stop the event loop.
-      # We should use more better approach, e.g. using pipe, fixing cool.io with timeout, etc.
-      TCPSocket.open(listen_address, @port) {|sock| } # FIXME @thread.join blocks without this line
+      unless support_blocking_timeout?
+        listen_address = (@bind == '0.0.0.0' ? '127.0.0.1' : @bind)
+        # This line is for connecting listen socket to stop the event loop.
+        # We should use more better approach, e.g. using pipe, fixing cool.io with timeout, etc.
+        TCPSocket.open(listen_address, @port) {|sock| } # FIXME @thread.join blocks without this line
+      end
       @thread.join
       @lsock.close
     end
@@ -82,13 +86,22 @@ module Fluent
     #end
 
     def run
-      @loop.run
+      if support_blocking_timeout?
+        @loop.run(@blocking_timeout)
+      else
+        @loop.run
+      end
     rescue => e
       log.error "unexpected error", :error => e, :error_class => e.class
       log.error_backtrace
     end
 
     protected
+
+    def support_blocking_timeout?
+      @loop.method(:run).arity.nonzero?
+    end
+
     # message Entry {
     #   1: long time
     #   2: object record

data/lib/fluent/plugin/in_http.rb

@@ -25,6 +25,7 @@ module Fluent
 
     def initialize
       require 'webrick/httputils'
+      require 'uri'
       super
     end
 
@@ -246,7 +247,8 @@ module Fluent
 
         @env['REMOTE_ADDR'] = @remote_addr if @remote_addr
 
-        params = WEBrick::HTTPUtils.parse_query(@parser.query_string)
+        uri = URI.parse(@parser.request_url)
+        params = WEBrick::HTTPUtils.parse_query(uri.query)
 
         if @content_type =~ /^application\/x-www-form-urlencoded/
           params.update WEBrick::HTTPUtils.parse_query(@body)
@@ -256,7 +258,7 @@ module Fluent
         elsif @content_type =~ /^application\/json/
           params['json'] = @body
         end
-        path_info = @parser.request_path
+        path_info = uri.path
 
         params.merge!(@env)
         @env.clear

data/lib/fluent/plugin/in_tail.rb

@@ -237,7 +237,7 @@ module Fluent
         end
       rescue => e
         log.warn line.dump, :error => e.to_s
-        log.debug_backtrace(e)
+        log.debug_backtrace(e.backtrace)
       end
     end
 

data/lib/fluent/plugin/out_forward.rb

@@ -43,6 +43,7 @@ module Fluent
     config_param :hard_timeout, :time, :default => 60
     config_param :expire_dns_cache, :time, :default => nil  # 0 means disable cache
     config_param :phi_threshold, :integer, :default => 16
+    config_param :phi_failure_detector, :bool, :default => true
     attr_reader :nodes
 
     # backward compatibility
@@ -82,8 +83,10 @@ module Fluent
         end
 
         failure = FailureDetector.new(@heartbeat_interval, @hard_timeout, Time.now.to_i.to_f)
-        @nodes << Node.new(name, host, port, weight, standby, failure,
-          @phi_threshold, recover_sample_size, @expire_dns_cache, log)
+
+        node_conf = NodeConfig.new(name, host, port, weight, standby, failure,
+          @phi_threshold, recover_sample_size, @expire_dns_cache, @phi_failure_detector)
+        @nodes << Node.new(log, node_conf)
         log.info "adding forwarding server '#{name}'", :host=>host, :port=>port, :weight=>weight
       }
     end
@@ -324,40 +327,40 @@ module Fluent
       end
     end
 
+    NodeConfig = Struct.new("NodeConfig", :name, :host, :port, :weight, :standby, :failure,
+      :phi_threshold, :recover_sample_size, :expire_dns_cache, :phi_failure_detector)
+
     class Node
-      def initialize(name, host, port, weight, standby, failure,
-          phi_threshold, recover_sample_size, expire_dns_cache, log)
-        @name = name
-        @host = host
-        @port = port
-        @weight = weight
-        @standby = standby
-        @failure = failure
-        @phi_threshold = phi_threshold
-        @recover_sample_size = recover_sample_size
-        @expire_dns_cache = expire_dns_cache
-        @available = true
+      def initialize(log, conf)
         @log = log
+        @conf = conf
+        @name = @conf.name
+        @host = @conf.host
+        @port = @conf.port
+        @weight = @conf.weight
+        @failure = @conf.failure
+        @available = true
 
         @resolved_host = nil
         @resolved_time = 0
         resolved_host  # check dns
       end
 
+      attr_reader :conf
       attr_reader :name, :host, :port, :weight
-      attr_writer :weight, :standby, :available
       attr_reader :sockaddr  # used by on_heartbeat
+      attr_reader :failure, :available # for test
 
       def available?
         @available
       end
 
       def standby?
-        @standby
+        @conf.standby
       end
 
       def resolved_host
-        case @expire_dns_cache
+        case @conf.expire_dns_cache
         when 0
           # cache is disabled
           return resolve_dns!
@@ -369,7 +372,7 @@ module Fluent
         else
           now = Engine.now
           rh = @resolved_host
-          if !rh || now - @resolved_time >= @expire_dns_cache
+          if !rh || now - @resolved_time >= @conf.expire_dns_cache
             rh = @resolved_host = resolve_dns!
             @resolved_time = now
           end
@@ -401,24 +404,25 @@ module Fluent
           return true
         end
 
-        phi = @failure.phi(now)
-        #$log.trace "phi '#{@name}'", :host=>@host, :port=>@port, :phi=>phi
-        if phi > @phi_threshold
-          @log.warn "detached forwarding server '#{@name}'", :host=>@host, :port=>@port, :phi=>phi
-          @available = false
-          @resolved_host = nil  # expire cached host
-          @failure.clear
-          return true
-        else
-          return false
+        if @conf.phi_failure_detector
+          phi = @failure.phi(now)
+          #$log.trace "phi '#{@name}'", :host=>@host, :port=>@port, :phi=>phi
+          if phi > @conf.phi_threshold
+            @log.warn "detached forwarding server '#{@name}'", :host=>@host, :port=>@port, :phi=>phi
+            @available = false
+            @resolved_host = nil  # expire cached host
+            @failure.clear
+            return true
+          end
         end
+        return false
       end
 
       def heartbeat(detect=true)
         now = Time.now.to_f
         @failure.add(now)
         #@log.trace "heartbeat from '#{@name}'", :host=>@host, :port=>@port, :available=>@available, :sample_size=>@failure.sample_size
-        if detect && !@available && @failure.sample_size > @recover_sample_size
+        if detect && !@available && @failure.sample_size > @conf.recover_sample_size
           @available = true
           @log.warn "recovered forwarding server '#{@name}'", :host=>@host, :port=>@port
           return true

data/lib/fluent/registry.rb

@@ -0,0 +1,76 @@
+module Fluent
+  require 'fluent/config/error'
+
+  class Registry
+    def initialize(kind, search_prefix)
+      @kind = kind
+      @search_prefix = search_prefix
+      @map = {}
+    end
+
+    attr_reader :kind
+
+    def register(type, value)
+      type = type.to_sym
+      @map[type] = value
+    end
+
+    def lookup(type)
+      type = type.to_sym
+      if value = @map[type]
+        return value
+      end
+      search(type)
+      if value = @map[type]
+        return value
+      end
+      raise ConfigError, "Unknown #{@kind} plugin '#{type}'. Run 'gem search -rd fluentd-plugin' to find plugins"  # TODO error class
+    end
+
+    def search(type)
+      path = "#{@search_prefix}#{type}"
+
+      # prefer LOAD_PATH than gems
+      files = $LOAD_PATH.map { |lp|
+        lpath = File.expand_path(File.join(lp, "#{path}.rb"))
+        File.exist?(lpath) ? lpath : nil
+      }.compact
+      unless files.empty?
+        # prefer newer version
+        require files.sort.last
+        return
+      end
+
+      # search gems
+      if defined?(::Gem::Specification) && ::Gem::Specification.respond_to?(:find_all)
+        specs = Gem::Specification.find_all { |spec|
+          spec.contains_requirable_file? path
+        }
+
+        # prefer newer version
+        specs = specs.sort_by { |spec| spec.version }
+        if spec = specs.last
+          spec.require_paths.each { |lib|
+            file = "#{spec.full_gem_path}/#{lib}/#{path}"
+            require file
+          }
+        end
+
+        # backward compatibility for rubygems < 1.8
+      elsif defined?(::Gem) && ::Gem.respond_to?(:searcher)
+        #files = Gem.find_files(path).sort
+        specs = Gem.searcher.find_all(path)
+
+        # prefer newer version
+        specs = specs.sort_by { |spec| spec.version }
+        specs.reverse_each { |spec|
+          files = Gem.searcher.matching_files(spec, path)
+          unless files.empty?
+            require files.first
+            break
+          end
+        }
+      end
+    end
+  end
+end