fluentd 1.7.4-x64-mingw32 → 1.8.0-x64-mingw32

data/lib/fluent/config/types.rb

@@ -20,7 +20,9 @@ require 'fluent/config/error'
 
 module Fluent
   module Config
-    def self.size_value(str)
+    def self.size_value(str, opts = {}, name = nil)
+      return nil if str.nil?
+
       case str.to_s
       when /([0-9]+)k/i
         $~[1].to_i * 1024
@@ -31,11 +33,13 @@ module Fluent
       when /([0-9]+)t/i
         $~[1].to_i * (1024 ** 4)
       else
-        str.to_i
+        INTEGER_TYPE.call(str, opts, name)
       end
     end
 
-    def self.time_value(str)
+    def self.time_value(str, opts = {}, name = nil)
+      return nil if str.nil?
+
       case str.to_s
       when /([0-9]+)s/
         $~[1].to_i
@@ -46,12 +50,13 @@ module Fluent
       when /([0-9]+)d/
         $~[1].to_i * 24 * 60 * 60
       else
-        str.to_f
+        FLOAT_TYPE.call(str, opts, name)
       end
     end
 
-    def self.bool_value(str)
+    def self.bool_value(str, opts = {}, name = nil)
       return nil if str.nil?
+
       case str.to_s
       when 'true', 'yes'
         true
@@ -64,14 +69,17 @@ module Fluent
         # parser should pass empty string in this case but changing behaviour may break existing environment so keep parser behaviour. Just ignore comment value in boolean handling for now.
         if str.respond_to?('start_with?') && str.start_with?('#')
           true
+        elsif opts[:strict]
+          raise Fluent::ConfigError, "#{name}: invalid bool value: #{str}"
         else
           nil
         end
       end
     end
 
-    def self.regexp_value(str)
+    def self.regexp_value(str, opts = {}, name = nil)
       return nil unless str
+
       return Regexp.compile(str) unless str.start_with?("/")
       right_slash_position = str.rindex("/")
       if right_slash_position < str.size - 3
@@ -84,12 +92,21 @@ module Fluent
       Regexp.compile(str[1...right_slash_position], option)
     end
 
-    STRING_TYPE = Proc.new { |val, opts|
+    def self.string_value(val, opts = {}, name = nil)
+      return nil if val.nil?
+
       v = val.to_s
       v = v.frozen? ? v.dup : v # config_param can't assume incoming string is mutable
       v.force_encoding(Encoding::UTF_8)
+    end
+
+    STRING_TYPE = Proc.new { |val, opts = {}, name = nil|
+      Config.string_value(val, opts, name)
     }
-    ENUM_TYPE = Proc.new { |val, opts|
+
+    def self.enum_value(val, opts = {}, name = nil)
+      return nil if val.nil?
+
       s = val.to_sym
       list = opts[:list]
       raise "Plugin BUG: config type 'enum' requires :list of symbols" unless list.is_a?(Array) && list.all?{|v| v.is_a? Symbol }
@@ -97,33 +114,77 @@ module Fluent
         raise ConfigError, "valid options are #{list.join(',')} but got #{val}"
       end
       s
+    end
+
+    ENUM_TYPE = Proc.new { |val, opts = {}, name = nil|
+      Config.enum_value(val, opts, name)
+    }
+
+    INTEGER_TYPE = Proc.new { |val, opts = {}, name = nil|
+      if val.nil?
+        nil
+      elsif opts[:strict]
+        begin
+          Integer(val)
+        rescue ArgumentError, TypeError => e
+          raise ConfigError, "#{name}: #{e.message}"
+        end
+      else
+        val.to_i
+      end
     }
-    INTEGER_TYPE = Proc.new { |val, opts| val.to_i }
-    FLOAT_TYPE = Proc.new { |val, opts| val.to_f }
-    SIZE_TYPE = Proc.new { |val, opts| Config.size_value(val) }
-    BOOL_TYPE = Proc.new { |val, opts| Config.bool_value(val) }
-    TIME_TYPE = Proc.new { |val, opts| Config.time_value(val) }
-    REGEXP_TYPE = Proc.new { |val, opts| Config.regexp_value(val) }
-
-    REFORMAT_VALUE = ->(type, value) {
+
+    FLOAT_TYPE = Proc.new { |val, opts = {}, name = nil|
+      if val.nil?
+        nil
+      elsif opts[:strict]
+        begin
+          Float(val)
+        rescue ArgumentError, TypeError => e
+          raise ConfigError, "#{name}: #{e.message}"
+        end
+      else
+        val.to_f
+      end
+    }
+
+    SIZE_TYPE = Proc.new { |val, opts = {}, name = nil|
+      Config.size_value(val, opts, name)
+    }
+
+    BOOL_TYPE = Proc.new { |val, opts = {}, name = nil|
+      Config.bool_value(val, opts, name)
+    }
+
+    TIME_TYPE = Proc.new { |val, opts = {}, name = nil|
+      Config.time_value(val, opts, name)
+    }
+
+    REGEXP_TYPE = Proc.new { |val, opts = {}, name = nil|
+      Config.regexp_value(val, opts, name)
+    }
+
+    REFORMAT_VALUE = ->(type, value, opts = {}, name = nil) {
       if value.nil?
         value
       else
         case type
         when :string  then value.to_s.force_encoding(Encoding::UTF_8)
-        when :integer then value.to_i
-        when :float   then value.to_f
-        when :size then Config.size_value(value)
-        when :bool then Config.bool_value(value)
-        when :time then Config.time_value(value)
-        when :regexp then Config.regexp_value(value)
+        when :integer then INTEGER_TYPE.call(value, opts, name)
+        when :float   then FLOAT_TYPE.call(value, opts, name)
+        when :size then Config.size_value(value, opts, name)
+        when :bool then Config.bool_value(value, opts, name)
+        when :time then Config.time_value(value, opts, name)
+        when :regexp then Config.regexp_value(value, opts, name)
         else
           raise "unknown type in REFORMAT: #{type}"
         end
       end
     }
 
-    HASH_TYPE = Proc.new { |val, opts|
+    def self.hash_value(val, opts = {}, name = nil)
+      return nil if val.nil?
+
       param = if val.is_a?(String)
                 val.start_with?('{') ? JSON.load(val) : Hash[val.strip.split(/\s*,\s*/).map{|v| v.split(':', 2)}]
               else
@@ -138,12 +199,19 @@ module Fluent
         newparam = {}
         param.each_pair do |key, value|
           new_key = opts[:symbolize_keys] ? key.to_sym : key
-          newparam[new_key] = opts[:value_type] ? REFORMAT_VALUE.call(opts[:value_type], value) : value
+          newparam[new_key] = opts[:value_type] ? REFORMAT_VALUE.call(opts[:value_type], value, opts, new_key) : value
         end
         newparam
       end
+    end
+
+    HASH_TYPE = Proc.new { |val, opts = {}, name = nil|
+      Config.hash_value(val, opts, name)
     }
-    ARRAY_TYPE = Proc.new { |val, opts|
+
+    def self.array_value(val, opts = {}, name = nil)
+      return nil if val.nil?
+
       param = if val.is_a?(String)
                 val.start_with?('[') ? JSON.load(val) : val.strip.split(/\s*,\s*/)
               else
@@ -153,10 +221,14 @@ module Fluent
         raise ConfigError, "array required but got #{val.inspect}"
       end
       if opts[:value_type]
-        param.map{|v| REFORMAT_VALUE.call(opts[:value_type], v) }
+        param.map{|v| REFORMAT_VALUE.call(opts[:value_type], v, opts, nil) }
       else
         param
       end
+    end
+
+    ARRAY_TYPE = Proc.new { |val, opts = {}, name = nil|
+      Config.array_value(val, opts, name)
     }
   end
 end

data/lib/fluent/configurable.rb

@@ -74,7 +74,7 @@ module Fluent
       Fluent::Config::SectionGenerator.generate(proxy, conf, nil, nil)
     end
 
-    def configure(conf)
+    def configure(conf, strict_config_value=false)
       @config = conf
 
       logger = if self.respond_to?(:log)
@@ -91,7 +91,7 @@ module Fluent
 
       # In the nested section, can't get plugin class through proxies so get plugin class here
       plugin_class = Fluent::Plugin.lookup_type_from_class(proxy.name.to_s)
-      root = Fluent::Config::SectionGenerator.generate(proxy, conf, logger, plugin_class)
+      root = Fluent::Config::SectionGenerator.generate(proxy, conf, logger, plugin_class, [], strict_config_value)
       @config_root_section = root
 
       root.instance_eval{ @params.keys }.each do |param_name|

data/lib/fluent/counter/base_socket.rb

@@ -20,14 +20,12 @@ require 'fluent/msgpack_factory'
 module Fluent
   module Counter
     class BaseSocket < Coolio::TCPSocket
-      include Fluent::MessagePackFactory::Mixin
-
       def packed_write(data)
         write pack(data)
       end
 
       def on_read(data)
-        msgpack_unpacker.feed_each(data) do |d|
+        Fluent::MessagePackFactory.msgpack_unpacker.feed_each(data) do |d|
           on_message d
         end
       end
@@ -39,7 +37,7 @@ module Fluent
       private
 
       def pack(data)
-        msgpack_packer.pack(data)
+        Fluent::MessagePackFactory.msgpack_packer.pack(data)
       end
     end
   end

data/lib/fluent/engine.rb

@@ -22,46 +22,36 @@ require 'fluent/root_agent'
 require 'fluent/time'
 require 'fluent/system_config'
 require 'fluent/plugin'
+require 'fluent/fluent_log_event_router'
 
 module Fluent
   class EngineClass
+    # For compat. remove it in fluentd v2
     include Fluent::MessagePackFactory::Mixin
 
     def initialize
       @root_agent = nil
-      @default_loop = nil
       @engine_stopped = false
       @_worker_id = nil
 
-      @log_event_router = nil
-      @log_emit_thread = nil
-      @log_event_loop_stop = false
-      @log_event_loop_graceful_stop = false
-      @log_event_queue = []
       @log_event_verbose = false
-
       @suppress_config_dump = false
+      @without_source = false
 
+      @fluent_log_event_router = nil
       @system_config = SystemConfig.new
 
-      @dry_run_mode = false
+      @supervisor_mode = false
     end
 
     MAINLOOP_SLEEP_INTERVAL = 0.3
 
-    MATCH_CACHE_SIZE = 1024
-    LOG_EMIT_INTERVAL = 0.1
-
-    attr_reader :root_agent
-    attr_reader :matches, :sources
-    attr_reader :system_config
+    attr_reader :root_agent, :system_config, :supervisor_mode
 
-    attr_accessor :dry_run_mode
-
-    def init(system_config)
+    def init(system_config, supervisor_mode: false)
       @system_config = system_config
+      @supervisor_mode = supervisor_mode
 
-      suppress_interval(system_config.emit_error_log_interval) unless system_config.emit_error_log_interval.nil?
       @suppress_config_dump = system_config.suppress_config_dump unless system_config.suppress_config_dump.nil?
       @without_source = system_config.without_source unless system_config.without_source.nil?
 
@@ -69,8 +59,6 @@ module Fluent
 
       @root_agent = RootAgent.new(log: log, system_config: @system_config)
 
-      MessagePackFactory.init
-
       self
     end
 
@@ -78,11 +66,6 @@ module Fluent
       $log
     end
 
-    def suppress_interval(interval_time)
-      @suppress_emit_error_log_interval = interval_time
-      @next_emit_error_log_time = Time.now.to_i
-    end
-
     def parse_config(io, fname, basepath = Dir.pwd, v1_config = false)
       if fname =~ /\.rb$/
         require 'fluent/config/dsl'
@@ -92,80 +75,47 @@ module Fluent
       end
     end
 
-    def run_configure(conf)
+    def run_configure(conf, dry_run: false)
       configure(conf)
-      conf.check_not_fetched { |key, e|
+      conf.check_not_fetched do |key, e|
         parent_name, plugin_name = e.unused_in
-        if parent_name
-          message = if plugin_name
-                      "section <#{e.name}> is not used in <#{parent_name}> of #{plugin_name} plugin"
-                    else
-                      "section <#{e.name}> is not used in <#{parent_name}>"
-                    end
-          if e.for_every_workers?
-            $log.warn :worker0, message
-          elsif e.for_this_worker?
-            $log.warn message
-          end
-          next
-        end
-        unless e.name == 'system'
-          unless @without_source && e.name == 'source'
-            message = "parameter '#{key}' in #{e.to_s.strip} is not used."
-            if e.for_every_workers?
-              $log.warn :worker0, message
-            elsif e.for_this_worker?
-              $log.warn message
-            end
-          end
+        message = if parent_name && plugin_name
+                    "section <#{e.name}> is not used in <#{parent_name}> of #{plugin_name} plugin"
+                  elsif parent_name
+                    "section <#{e.name}> is not used in <#{parent_name}>"
+                  elsif e.name != 'system' && !(@without_source && e.name == 'source')
+                    "parameter '#{key}' in #{e.to_s.strip} is not used."
+                  else
+                    nil
+                  end
+        next if message.nil?
+
+        if dry_run && @supervisor_mode
+          $log.warn :supervisor, message
+        elsif e.for_every_workers?
+          $log.warn :worker0, message
+        elsif e.for_this_worker?
+          $log.warn message
         end
-      }
+      end
     end
 
     def configure(conf)
-      # plugins / configuration dumps
-      Gem::Specification.find_all.select{|x| x.name =~ /^fluent(d|-(plugin|mixin)-.*)$/}.each do |spec|
-        $log.info :worker0, "gem '#{spec.name}' version '#{spec.version}'"
-      end
-
       @root_agent.configure(conf)
 
-      begin
-        log_event_agent = @root_agent.find_label(Fluent::Log::LOG_EVENT_LABEL)
-        log_event_router = log_event_agent.event_router
+      @fluent_log_event_router = FluentLogEventRouter.build(@root_agent)
 
-        # suppress mismatched tags only for <label @FLUENT_LOG> label.
-        # it's not suppressed in default event router for non-log-event events
-        log_event_router.suppress_missing_match!
-
-        @log_event_router = log_event_router
-
-        unmatched_tags = Fluent::Log.event_tags.select{|t| !@log_event_router.match?(t) }
-        unless unmatched_tags.empty?
-          $log.warn "match for some tags of log events are not defined (to be ignored)", tags: unmatched_tags
-        end
-      rescue ArgumentError # ArgumentError "#{label_name} label not found"
-        # use default event router if <label @FLUENT_LOG> is missing in configuration
-        log_event_router = @root_agent.event_router
-
-        if Fluent::Log.event_tags.any?{|t| log_event_router.match?(t) }
-          @log_event_router = log_event_router
-
-          unmatched_tags = Fluent::Log.event_tags.select{|t| !@log_event_router.match?(t) }
-          unless unmatched_tags.empty?
-            $log.warn "match for some tags of log events are not defined (to be ignored)", tags: unmatched_tags
-          end
-        end
+      if @fluent_log_event_router.emittable?
+        $log.enable_event(true)
       end
 
-      $log.enable_event(true) if @log_event_router
-
       unless @suppress_config_dump
         $log.info :supervisor, "using configuration file: #{conf.to_s.rstrip}"
       end
     end
 
     def add_plugin_dir(dir)
+      $log.warn('Deprecated method: this method is going to be deleted. Use Fluent::Plugin.add_plugin_dir')
       Plugin.add_plugin_dir(dir)
     end
 
@@ -190,39 +140,12 @@ module Fluent
       Fluent::EventTime.now
     end
 
-    def log_event_loop
-      $log.disable_events(Thread.current)
-
-      while sleep(LOG_EMIT_INTERVAL)
-        break if @log_event_loop_stop
-        break if @log_event_loop_graceful_stop && @log_event_queue.empty?
-        next if @log_event_queue.empty?
-
-        # NOTE: thead-safe of slice! depends on GVL
-        events = @log_event_queue.slice!(0..-1)
-        next if events.empty?
-
-        events.each {|tag,time,record|
-          begin
-            @log_event_router.emit(tag, time, record)
-          rescue => e
-            # This $log.error doesn't emit log events, because of `$log.disable_events(Thread.current)` above
-            $log.error "failed to emit fluentd's log event", tag: tag, event: record, error: e
-          end
-        }
-      end
-    end
-
     def run
       begin
         $log.info "starting fluentd worker", pid: Process.pid, ppid: Process.ppid, worker: worker_id
         start
 
-        if @log_event_router
-          $log.enable_event(true)
-          @log_emit_thread = Thread.new(&method(:log_event_loop))
-          @log_emit_thread.abort_on_exception = true
-        end
+        @fluent_log_event_router.start
 
         $log.info "fluentd worker is now running", worker: worker_id
         sleep MAINLOOP_SLEEP_INTERVAL until @engine_stopped
@@ -236,19 +159,12 @@ module Fluent
 
       unless @log_event_verbose
         $log.enable_event(false)
-        if @log_emit_thread
-          # to make sure to emit all log events into router, before shutting down
-          @log_event_loop_graceful_stop = true
-          @log_emit_thread.join
-          @log_emit_thread = nil
-        end
+        @fluent_log_event_router.graceful_stop
       end
       $log.info "shutting down fluentd worker", worker: worker_id
       shutdown
-      if @log_emit_thread
-        @log_event_loop_stop = true
-        @log_emit_thread.join
-      end
+
+      @fluent_log_event_router.stop
     end
 
     def stop
@@ -257,11 +173,14 @@ module Fluent
     end
 
     def push_log_event(tag, time, record)
-      return if @log_emit_thread.nil?
-      @log_event_queue.push([tag, time, record])
+      @fluent_log_event_router.emit_event([tag, time, record])
     end
 
     def worker_id
+      if @supervisor_mode
+        return -1
+      end
+
       return @_worker_id if @_worker_id
       # if ENV doesn't have SERVERENGINE_WORKER_ID, it is a worker under --no-supervisor or in tests
       # so it's (almost) a single worker, worker_id=0