fluentd 0.14.8 → 0.14.9

data/lib/fluent/plugin_helper/compat_parameters.rb

@@ -48,10 +48,19 @@ module Fluent
       }
 
       PARSER_PARAMS = {
-        "format" => "@type",
+        "format" => nil,
+        "types" => nil,
+        "types_delimiter" => nil,
+        "types_label_delimiter" => nil,
+        "keys" => "keys", # CSVParser, TSVParser (old ValuesParser)
         "time_key"    => "time_key",
         "time_format" => "time_format",
+        "localtim" => nil,
+        "utc" => nil,
         "delimiter"   => "delimiter",
+        "keep_time_key" => "keep_time_key",
+        "null_empty_string" => "null_empty_string",
+        "null_value_pattern" => "null_value_pattern",
         "json_parser"      => "json_parser", # JSONParser
         "label_delimiter"  => "label_delimiter", # LabeledTSVParser
         "format_firstline" => "format_firstline", # MultilineParser
@@ -62,19 +71,29 @@ module Fluent
 
       INJECT_PARAMS = {
         "include_time_key" => nil,
-        "time_key"      => "time_key",
-        "time_format"   => "time_format",
-        "timezone"      => "timezone",
+        "time_key"    => "time_key",
+        "time_format" => "time_format",
+        "timezone"    => "timezone",
         "include_tag_key" => nil,
         "tag_key" => "tag_key",
         "localtime" => nil,
         "utc" => nil,
       }
 
+      EXTRACT_PARAMS = {
+        "time_key"    => "time_key",
+        "time_format" => "time_format",
+        "timezone"    => "timezone",
+        "tag_key" => "tag_key",
+        "localtime" => nil,
+        "utc" => nil,
+      }
+
       FORMATTER_PARAMS = {
         "format" => "@type",
         "delimiter" => "delimiter",
         "force_quotes" => "force_quotes", # CsvFormatter
+        "keys" => "keys", # TSVFormatter
         "fields" => "fields", # CsvFormatter
         "json_parser" => "json_parser", # JSONFormatter
         "label_delimiter" => "label_delimiter", # LabeledTSVFormatter
@@ -95,6 +114,8 @@ module Fluent
             compat_parameters_buffer(conf, **kwargs)
           when :inject
             compat_parameters_inject(conf)
+          when :extract
+            compat_parameters_extract(conf)
           when :parser
             compat_parameters_parser(conf)
           when :formatter
@@ -167,6 +188,7 @@ module Fluent
           hash['time_type'] ||= 'string'
         end
         if conf.has_key?('time_as_epoch') && Fluent::Config.bool_value(conf['time_as_epoch'])
+          hash['time_key'] ||= 'time'
           hash['time_type'] = 'unixtime'
         end
         if conf.has_key?('localtime') || conf.has_key?('utc')
@@ -193,6 +215,40 @@ module Fluent
         conf
       end
 
+      def compat_parameters_extract(conf)
+        return unless conf.elements('extract').empty?
+        return if EXTRACT_PARAMS.keys.all?{|k| !conf.has_key?(k) } && !conf.has_key?('format')
+
+        # TODO: warn obsolete parameters if these are deprecated
+        hash = compat_parameters_copy_to_subsection_attributes(conf, EXTRACT_PARAMS)
+
+        if conf.has_key?('time_as_epoch') && Fluent::Config.bool_value(conf['time_as_epoch'])
+          hash['time_key'] ||= 'time'
+          hash['time_type'] = 'unixtime'
+        elsif conf.has_key?('format') && conf["format"].start_with?("/") && conf["format"].end_with?("/") # old-style regexp parser
+          hash['time_key'] ||= 'time'
+          hash['time_type'] ||= 'string'
+        end
+        if conf.has_key?('localtime') || conf.has_key?('utc')
+          if conf.has_key?('localtime') && conf.has_key?('utc')
+            raise Fluent::ConfigError, "both of utc and localtime are specified, use only one of them"
+          elsif conf.has_key?('localtime')
+            hash['localtime'] = Fluent::Config.bool_value(conf['localtime'])
+          elsif conf.has_key?('utc')
+            hash['localtime'] = !(Fluent::Config.bool_value(conf['utc']))
+            # Specifying "localtime false" means using UTC in TimeFormatter
+            # And specifying "utc" is different from specifying "timezone +0000"(it's not always UTC).
+            # There are difference between "Z" and "+0000" in timezone formatting.
+            # TODO: add kwargs to TimeFormatter to specify "using localtime", "using UTC" or "using specified timezone" in more explicit way
+          end
+        end
+
+        e = Fluent::Config::Element.new('extract', '', hash, [])
+        conf.elements << e
+
+        conf
+      end
+
       def compat_parameters_parser(conf)
         return unless conf.elements('parse').empty?
         return if PARSER_PARAMS.keys.all?{|k| !conf.has_key?(k) }
@@ -200,6 +256,39 @@ module Fluent
         # TODO: warn obsolete parameters if these are deprecated
         hash = compat_parameters_copy_to_subsection_attributes(conf, PARSER_PARAMS)
 
+        if conf["format"]
+          if conf["format"].start_with?("/") && conf["format"].end_with?("/")
+            hash["@type"] = "regexp"
+            hash["expression"] = conf["format"][1..-2]
+          else
+            hash["@type"] = conf["format"]
+          end
+        end
+
+        if conf["types"]
+          delimiter = conf["types_delimiter"] || ','
+          label_delimiter = conf["types_label_delimiter"] || ':'
+          types = {}
+          conf['types'].split(delimiter).each do |pair|
+            key, value = pair.split(label_delimiter, 2)
+            types[key] = value
+          end
+          hash["types"] = JSON.dump(types)
+        end
+        if conf.has_key?('localtime') || conf.has_key?('utc')
+          if conf.has_key?('localtime') && conf.has_key?('utc')
+            raise Fluent::ConfigError, "both of utc and localtime are specified, use only one of them"
+          elsif conf.has_key?('localtime')
+            hash['localtime'] = Fluent::Config.bool_value(conf['localtime'])
+          elsif conf.has_key?('utc')
+            hash['localtime'] = !(Fluent::Config.bool_value(conf['utc']))
+            # Specifying "localtime false" means using UTC in TimeFormatter
+            # And specifying "utc" is different from specifying "timezone +0000"(it's not always UTC).
+            # There are difference between "Z" and "+0000" in timezone formatting.
+            # TODO: add kwargs to TimeFormatter to specify "using localtime", "using UTC" or "using specified timezone" in more explicit way
+          end
+        end
+
         e = Fluent::Config::Element.new('parse', '', hash, [])
         conf.elements << e
 

data/lib/fluent/plugin_helper/event_emitter.rb

@@ -26,10 +26,14 @@ module Fluent
 
       def router
         @_event_emitter_used_actually = true
+        if @_event_emitter_lazy_init
+          @router = @primary_instance.router
+        end
         @router
       end
 
       def router=(r)
+        # not recommended now...
         @router = r
       end
 
@@ -44,14 +48,23 @@ module Fluent
       def event_emitter_router(label_name)
         if label_name
           Engine.root_agent.find_label(label_name).event_router
+        elsif self.respond_to?(:as_secondary) && self.as_secondary
+          if @primary_instance.has_router?
+            @_event_emitter_lazy_init = true
+            nil # primary plugin's event router is not initialized yet, here.
+          else
+            @primary_instance.context_router
+          end
         else
-          Engine.root_agent.event_router
+          # `Engine.root_agent.event_router` is for testing
+          self.context_router || Engine.root_agent.event_router
         end
       end
 
       def initialize
         super
         @_event_emitter_used_actually = false
+        @_event_emitter_lazy_init = false
         @router = nil
       end
 

data/lib/fluent/plugin_helper/extract.rb

@@ -25,7 +25,8 @@ module Fluent
         return nil unless @_extract_enabled
 
         if @_extract_tag_key && record.has_key?(@_extract_tag_key)
-          return record[@_extract_tag_key].to_s
+          v = @_extract_keep_tag_key ? record[@_extract_tag_key] : record.delete(@_extract_tag_key)
+          return v.to_s
         end
 
         nil
@@ -35,7 +36,8 @@ module Fluent
         return nil unless @_extract_enabled
 
         if @_extract_time_key && record.has_key?(@_extract_time_key)
-          return @_extract_time_parser.call(record[@_extract_time_key])
+          v = @_extract_keep_time_key ? record[@_extract_time_key] : record.delete(@_extract_time_key)
+          return @_extract_time_parser.call(v)
         end
 
         nil
@@ -45,7 +47,9 @@ module Fluent
         include Fluent::Configurable
         config_section :extract, required: false, multi: false, param_name: :extract_config do
           config_param :tag_key, :string, default: nil
+          config_param :keep_tag_key, :bool, default: false
           config_param :time_key, :string, default: nil
+          config_param :keep_time_key, :bool, default: false
 
           # To avoid defining :time_type twice
           config_param :time_type, :enum, list: [:float, :unixtime, :string], default: :float
@@ -64,7 +68,9 @@ module Fluent
         super
         @_extract_enabled = false
         @_extract_tag_key = nil
+        @_extract_keep_tag_key = nil
         @_extract_time_key = nil
+        @_extract_keep_time_key = nil
         @_extract_time_parser = nil
       end
 
@@ -73,15 +79,21 @@ module Fluent
 
         if @extract_config
           @_extract_tag_key = @extract_config.tag_key
+          @_extract_keep_tag_key = @extract_config.keep_tag_key
           @_extract_time_key = @extract_config.time_key
           if @_extract_time_key
+            @_extract_keep_time_key = @extract_config.keep_time_key
             @_extract_time_parser = case @extract_config.time_type
-                                    when :float then ->(v){ Fluent::EventTime.new(v.to_i, ((v.to_f - v.to_i) * 1_000_000_000).to_i) }
-                                    when :unixtime then ->(v){ Fluent::EventTime.new(v.to_i, 0) }
+                                    when :float then Fluent::NumericTimeParser.new(:float)
+                                    when :unixtime then Fluent::NumericTimeParser.new(:unixtime)
                                     else
                                       localtime = @extract_config.localtime && !@extract_config.utc
                                       Fluent::TimeParser.new(@extract_config.time_format, localtime, @extract_config.timezone)
                                     end
+          else
+            if @extract_config.time_format
+              log.warn "'time_format' specified without 'time_key', will be ignored"
+            end
           end
 
           @_extract_enabled = @_extract_tag_key || @_extract_time_key

data/lib/fluent/plugin_helper/formatter.rb

@@ -24,7 +24,7 @@ module Fluent
     module Formatter
       def formatter_create(usage: '', type: nil, conf: nil, default_type: nil)
         formatter = @_formatters[usage]
-        return formatter if formatter
+        return formatter if formatter && !type && !conf
 
         type = if type
                  type
@@ -61,9 +61,9 @@ module Fluent
       module FormatterParams
         include Fluent::Configurable
         # minimum section definition to instantiate formatter plugin instances
-        config_section :format, required: false, multi: true, param_name: :formatter_configs do
+        config_section :format, required: false, multi: true, init: true, param_name: :formatter_configs do
           config_argument :usage, :string, default: ''
-          config_param    :@type, :string
+          config_param    :@type, :string # config_set_default required for :@type
         end
       end
 
@@ -82,15 +82,13 @@ module Fluent
       def configure(conf)
         super
 
-        if @formatter_configs
-          @formatter_configs.each do |section|
-            if @_formatters[section.usage]
-              raise Fluent::ConfigError, "duplicated formatter configured: #{section.usage}"
-            end
-            formatter = Plugin.new_formatter(section[:@type], parent: self)
-            formatter.configure(section.corresponding_config_element)
-            @_formatters[section.usage] = formatter
+        @formatter_configs.each do |section|
+          if @_formatters[section.usage]
+            raise Fluent::ConfigError, "duplicated formatter configured: #{section.usage}"
           end
+          formatter = Plugin.new_formatter(section[:@type], parent: self)
+          formatter.configure(section.corresponding_config_element)
+          @_formatters[section.usage] = formatter
         end
       end
 

data/lib/fluent/plugin_helper/inject.rb

@@ -113,6 +113,10 @@ module Fluent
                                         localtime = @inject_config.localtime && !@inject_config.utc
                                         Fluent::TimeFormatter.new(@inject_config.time_format, localtime, @inject_config.timezone)
                                       end
+          else
+            if @inject_config.time_format
+              log.warn "'time_format' specified without 'time_key', will be ignored"
+            end
           end
 
           @_inject_enabled = @_inject_hostname_key || @_inject_tag_key || @_inject_time_key

data/lib/fluent/plugin_helper/parser.rb

@@ -24,7 +24,7 @@ module Fluent
     module Parser
       def parser_create(usage: '', type: nil, conf: nil, default_type: nil)
         parser = @_parsers[usage]
-        return parser if parser
+        return parser if parser && !type && !conf
 
         type = if type
                  type
@@ -61,9 +61,9 @@ module Fluent
       module ParserParams
         include Fluent::Configurable
         # minimum section definition to instantiate parser plugin instances
-        config_section :parse, required: false, multi: true, param_name: :parser_configs do
+        config_section :parse, required: false, multi: true, init: true, param_name: :parser_configs do
           config_argument :usage, :string, default: ''
-          config_param    :@type, :string
+          config_param    :@type, :string # config_set_default required for :@type
         end
       end
 

data/lib/fluent/root_agent.rb

@@ -233,7 +233,7 @@ module Fluent
       # <source> emits events to the top-level event router (RootAgent#event_router).
       # Input#configure overwrites event_router to a label's event_router if it has `@label` parameter.
       # See also 'fluentd/plugin/input.rb'
-      input.router = @event_router
+      input.context_router = @event_router
       input.configure(conf)
       @inputs << input
 

data/lib/fluent/test/driver/base.rb

@@ -23,9 +23,13 @@ require 'timeout'
 module Fluent
   module Test
     module Driver
+      class TestTimedOut < RuntimeError; end
+
       class Base
         attr_reader :instance, :logs
 
+        DEFAULT_TIMEOUT = 300
+
         def initialize(klass, opts: {}, &block)
           if klass.is_a?(Class)
             if block
@@ -44,6 +48,8 @@ module Fluent
 
           @logs = []
 
+          @test_clock_id = Process::CLOCK_MONOTONIC_RAW rescue Process::CLOCK_MONOTONIC
+
           @run_post_conditions = []
           @run_breaking_conditions = []
           @broken = false
@@ -70,15 +76,25 @@ module Fluent
         def run(timeout: nil, start: true, shutdown: true, &block)
           instance_start if start
 
-          if @instance.respond_to?(:thread_wait_until_start)
-            @instance.thread_wait_until_start
-          end
-          if @instance.respond_to?(:event_loop_wait_until_start)
-            @instance.event_loop_wait_until_start
+          timeout ||= DEFAULT_TIMEOUT
+          stop_at = Process.clock_gettime(@test_clock_id) + timeout
+          @run_breaking_conditions << ->(){ Process.clock_gettime(@test_clock_id) >= stop_at }
+
+          if !block_given? && @run_post_conditions.empty? && @run_breaking_conditions.empty?
+            raise ArgumentError, "no stop conditions nor block specified"
           end
 
+          sleep_with_watching_threads = ->(){
+            if @instance.respond_to?(:_threads)
+              @instance._threads.values.each{|t| t.join(0) }
+            end
+            sleep 0.1
+          }
+
           begin
-            run_actual(timeout: timeout, &block)
+            retval = run_actual(timeout: timeout, &block)
+            sleep_with_watching_threads.call until stop?
+            retval
           ensure
             instance_shutdown if shutdown
           end
@@ -87,18 +103,33 @@ module Fluent
         def instance_start
           unless @instance.started?
             @instance.start
-            instance_hook_after_started
           end
           unless @instance.after_started?
             @instance.after_start
           end
+
+          if @instance.respond_to?(:thread_wait_until_start)
+            @instance.thread_wait_until_start
+          end
+
+          if @instance.respond_to?(:event_loop_wait_until_start)
+            @instance.event_loop_wait_until_start
+          end
+
+          instance_hook_after_started
         end
 
         def instance_hook_after_started
           # insert hooks for tests available after instance.start
         end
 
+        def instance_hook_before_stopped
+          # same with above
+        end
+
         def instance_shutdown
+          instance_hook_before_stopped
+
           @instance.stop            unless @instance.stopped?
           @instance.before_shutdown unless @instance.before_shutdown?
           @instance.shutdown        unless @instance.shutdown?
@@ -117,50 +148,33 @@ module Fluent
           @instance.terminate unless @instance.terminated?
         end
 
-        def run_actual(timeout: nil, &block)
+        def run_actual(timeout: DEFAULT_TIMEOUT, &block)
           if @instance.respond_to?(:_threads)
-            until @instance._threads.values.all?(&:alive?)
-              sleep 0.01
-            end
+            sleep 0.1 until @instance._threads.values.all?(&:alive?)
           end
 
           if @instance.respond_to?(:event_loop_running?)
-            until @instance.event_loop_running?
-              sleep 0.01
-            end
-          end
-
-          if timeout
-            stop_at = Time.now + timeout
-            @run_breaking_conditions << ->(){ Time.now >= stop_at }
+            sleep 0.1 until @instance.event_loop_running?
           end
 
-          if !block_given? && @run_post_conditions.empty? && @run_breaking_conditions.empty?
-            raise ArgumentError, "no stop conditions nor block specified"
+          if @instance.respond_to?(:_child_process_processes)
+            sleep 0.1 until @instance._child_process_processes.values.all?{|pinfo| pinfo.alive }
           end
 
-          proc = if block_given?
-                   ->(){ block.call; sleep(0.1) until stop? }
-                 else
-                   ->(){ sleep(0.1) until stop? }
-                 end
-
-          if timeout
-            begin
-              Timeout.timeout(timeout * 1.1) do |sec|
-                proc.call
-              end
-            rescue Timeout::Error
-              @broken = true
+          return_value = nil
+          begin
+            Timeout.timeout(timeout * 1.1) do |sec|
+              return_value = block.call if block_given?
             end
-          else
-            proc.call
+          rescue Timeout::Error
+            raise TestTimedOut, "Test case timed out with hard limit."
           end
+          return_value
         end
 
         def stop?
           # Should stop running if post conditions are not registered.
-          return true unless @run_post_conditions
+          return true unless @run_post_conditions || @run_post_conditions.empty?
 
           # Should stop running if all of the post conditions are true.
           return true if @run_post_conditions.all? {|proc| proc.call }