fluentd 0.14.1 → 0.14.2

data/lib/fluent/plugin_helper/compat_parameters.rb

@@ -14,6 +14,7 @@
 #    limitations under the License.
 #
 
+require 'fluent/config/types'
 require 'fluent/config/element'
 
 module Fluent
@@ -24,7 +25,7 @@ module Fluent
       # This helper is mainly to convert plugins from v0.12 API
       # to v0.14 API safely, without breaking user deployment.
 
-      PARAMS_MAP = {
+      BUFFER_PARAMS = {
         "buffer_type" => "@type",
         "buffer_path" => "path",
         "num_threads"                 => "flush_thread_count",
@@ -40,59 +41,183 @@ module Fluent
         "flush_at_shutdown" => "flush_at_shutdown",
       }
 
-      TIME_SLICED_PARAMS = {
+      BUFFER_TIME_SLICED_PARAMS = {
         "time_slice_format" => nil,
         "time_slice_wait" => "timekey_wait",
       }
 
-      def compat_parameters_default_chunk_key
-        # '', 'time' or 'tag'
-        raise NotImplementedError, "return one of '', 'time' or 'tag'"
+      PARSER_PARAMS = {
+        "format" => "@type",
+        "time_key"    => "time_key",
+        "time_format" => "time_format",
+        "delimiter"   => "delimiter",
+        "json_parser"      => "json_parser", # JSONParser
+        "label_delimiter"  => "label_delimiter", # LabeledTSVParser
+        "format_firstline" => "format_firstline", # MultilineParser
+        "message_key"      => "message_key", # NoneParser
+        "with_priority"    => "with_priority", # SyslogParser
+      }
+
+      INJECT_PARAMS = {
+        "include_time_key" => nil,
+        "time_key"      => "time_key",
+        "time_format"   => "time_format",
+        "timezone"      => "timezone",
+        "include_tag_key" => nil,
+        "tag_key" => "tag_key",
+        "localtime" => nil,
+        "utc" => nil,
+      }
+
+      FORMATTER_PARAMS = {
+        "format" => "@type",
+        "delimiter" => "delimiter",
+        "force_quotes" => "force_quotes", # CsvFormatter
+        "fields" => "fields", # CsvFormatter
+        "json_parser" => "json_parser", # JSONFormatter
+        "label_delimiter" => "label_delimiter", # LabeledTSVFormatter
+        "output_time" => "output_time", # OutFileFormatter
+        "output_tag"  => "output_tag", # OutFileFormatter
+        "message_key" => "message_key", # SingleValueFormatter
+        "add_newline" => "add_newline", # SingleValueFormatter
+        "output_type" => "output_type", # StdoutFormatter
+      }
+
+      def compat_parameters_convert(conf, *types, **kwargs)
+        types.each do |type|
+          case type
+          when :buffer
+            compat_parameters_buffer(conf, **kwargs)
+          when :inject
+            compat_parameters_inject(conf)
+          when :parser
+            compat_parameters_parser(conf)
+          when :formatter
+            compat_parameters_formatter(conf)
+          else
+            raise "BUG: unknown compat_parameters type: #{type}"
+          end
+        end
+
+        conf
       end
 
-      def configure(conf)
-        if conf.elements('buffer').empty? && PARAMS_MAP.keys.any?{|k| conf.has_key?(k) } || TIME_SLICED_PARAMS.keys.any?{|k| conf.has_key?(k) }
-          # TODO: warn obsolete parameters if these are deprecated
-          attr = {}
-          PARAMS_MAP.each do |compat, current|
-            next unless current
-            if conf.has_key?(compat)
-              if compat == 'buffer_queue_full_action' && conf[compat] == 'exception'
-                attr[current] = 'throw_exception'
-              else
-                attr[current] = conf[compat]
-              end
-            end
+      def compat_parameters_buffer(conf, default_chunk_key: '')
+        # return immediately if <buffer> section exists, or any buffer-related parameters don't exist
+        return unless conf.elements('buffer').empty?
+        return if (BUFFER_PARAMS.keys + BUFFER_TIME_SLICED_PARAMS.keys).all?{|k| !conf.has_key?(k) }
+
+        # TODO: warn obsolete parameters if these are deprecated
+        buffer_params = BUFFER_PARAMS.merge(BUFFER_TIME_SLICED_PARAMS)
+        attr = compat_parameters_copy_to_subsection_attributes(conf, buffer_params) do |compat_key, value|
+          if compat_key == 'buffer_queue_full_action' && value == 'exception'
+            'throw_exception'
+          else
+            value
           end
-          TIME_SLICED_PARAMS.each do |compat, current|
-            next unless current
-            attr[current] = conf[compat] if conf.has_key?(compat)
+        end
+
+        chunk_key = default_chunk_key
+
+        if conf.has_key?('time_slice_format')
+          chunk_key = 'time'
+          attr['timekey'] = case conf['time_slice_format']
+                            when /\%S/ then 1
+                            when /\%M/ then 60
+                            when /\%H/ then 3600
+                            when /\%d/ then 86400
+                            else
+                              raise Fluent::ConfigError, "time_slice_format only with %Y or %m is too long"
+                            end
+        else
+          if chunk_key == 'time'
+            attr['timekey'] = 86400 # TimeSliceOutput.time_slice_format default value is '%Y%m%d'
           end
+        end
 
-          chunk_key = nil
-
-          if conf.has_key?('time_slice_format')
-            chunk_key = 'time'
-            attr['timekey'] = case conf['time_slice_format']
-                              when /\%S/ then 1
-                              when /\%M/ then 60
-                              when /\%H/ then 3600
-                              when /\%d/ then 86400
-                              else
-                                raise Fluent::ConfigError, "time_slice_format only with %Y or %m is too long"
-                              end
-          else
-            chunk_key = compat_parameters_default_chunk_key
-            if chunk_key == 'time'
-              attr['timekey'] = 86400 # TimeSliceOutput.time_slice_format default value is '%Y%m%d'
-            end
+        e = Fluent::Config::Element.new('buffer', chunk_key, attr, [])
+        conf.elements << e
+
+        conf
+      end
+
+      def compat_parameters_inject(conf)
+        return unless conf.elements('inject').empty?
+        return if INJECT_PARAMS.keys.all?{|k| !conf.has_key?(k) }
+
+        # TODO: warn obsolete parameters if these are deprecated
+        attr = compat_parameters_copy_to_subsection_attributes(conf, INJECT_PARAMS)
+
+        if conf.has_key?('include_time_key') && Fluent::Config.bool_value(conf['include_time_key'])
+          attr['time_key'] ||= 'time'
+          attr['time_type'] ||= 'string'
+        end
+        if conf.has_key?('time_as_epoch') && Fluent::Config.bool_value(conf['time_as_epoch'])
+          attr['time_type'] = 'unixtime'
+        end
+        if conf.has_key?('localtime') || conf.has_key?('utc')
+          if conf.has_key?('localtime') && conf.has_key?('utc')
+            raise Fluent::ConfigError, "both of utc and localtime are specified, use only one of them"
+          elsif conf.has_key?('localtime')
+            attr['localtime'] = Fluent::Config.bool_value(conf['localtime'])
+          elsif conf.has_key?('utc')
+            attr['localtime'] = !(Fluent::Config.bool_value(conf['utc']))
+            # Specifying "localtime false" means using UTC in TimeFormatter
+            # And specifying "utc" is different from specifying "timezone +0000"(it's not always UTC).
+            # There are difference between "Z" and "+0000" in timezone formatting.
+            # TODO: add kwargs to TimeFormatter to specify "using localtime", "using UTC" or "using specified timezone" in more explicit way
           end
+        end
 
-          e = Fluent::Config::Element.new('buffer', chunk_key, attr, [])
-          conf.elements << e
+        if conf.has_key?('include_tag_key') && Fluent::Config.bool_value(conf['include_tag_key'])
+          attr['tag_key'] ||= 'tag'
         end
 
-        super
+        e = Fluent::Config::Element.new('inject', '', attr, [])
+        conf.elements << e
+
+        conf
+      end
+
+      def compat_parameters_parser(conf)
+        return unless conf.elements('parse').empty?
+        return if PARSER_PARAMS.keys.all?{|k| !conf.has_key?(k) }
+
+        # TODO: warn obsolete parameters if these are deprecated
+        attr = compat_parameters_copy_to_subsection_attributes(conf, PARSER_PARAMS)
+
+        e = Fluent::Config::Element.new('parse', '', attr, [])
+        conf.elements << e
+
+        conf
+      end
+
+      def compat_parameters_formatter(conf)
+        return unless conf.elements('format').empty?
+        return if FORMATTER_PARAMS.keys.all?{|k| !conf.has_key?(k) }
+
+        # TODO: warn obsolete parameters if these are deprecated
+        attr = compat_parameters_copy_to_subsection_attributes(conf, FORMATTER_PARAMS)
+
+        e = Fluent::Config::Element.new('format', '', attr, [])
+        conf.elements << e
+
+        conf
+      end
+
+      def compat_parameters_copy_to_subsection_attributes(conf, params, &block)
+        attr = {}
+        params.each do |compat, current|
+          next unless current
+          if conf.has_key?(compat)
+            if block_given?
+              attr[current] = block.call(compat, conf[compat])
+            else
+              attr[current] = conf[compat]
+            end
+          end
+        end
+        attr
       end
     end
   end

data/lib/fluent/plugin_helper/formatter.rb

@@ -17,19 +17,25 @@
 require 'fluent/plugin'
 require 'fluent/plugin/formatter'
 require 'fluent/config/element'
+require 'fluent/configurable'
 
 module Fluent
   module PluginHelper
     module Formatter
-      def formatter_create(usage: '', type: nil, conf: nil)
+      def formatter_create(usage: '', type: nil, conf: nil, default_type: nil)
         formatter = @_formatters[usage]
         return formatter if formatter
 
-        if !type
-          raise ArgumentError, "BUG: both type and conf are not specified" unless conf
-          raise Fluent::ConfigError, "@type is required in <format>" unless conf['@type']
-          type = conf['@type']
-        end
+        type = if type
+                 type
+               elsif conf && conf.respond_to?(:[])
+                 raise Fluent::ConfigError, "@type is required in <format>" unless conf['@type']
+                 conf['@type']
+               elsif default_type
+                 default_type
+               else
+                 raise ArgumentError, "BUG: both type and conf are not specified"
+               end
         formatter = Fluent::Plugin.new_formatter(type, parent: self)
         config = case conf
                  when Fluent::Config::Element
@@ -52,16 +58,19 @@ module Fluent
         formatter
       end
 
-      def self.included(mod)
-        mod.instance_eval do
-          # minimum section definition to instantiate formatter plugin instances
-          config_section :format, required: false, multi: true, param_name: :formatter_configs do
-            config_argument :usage, :string, default: ''
-            config_param    :@type, :string
-          end
+      module FormatterParams
+        include Fluent::Configurable
+        # minimum section definition to instantiate formatter plugin instances
+        config_section :format, required: false, multi: true, param_name: :formatter_configs do
+          config_argument :usage, :string, default: ''
+          config_param    :@type, :string
         end
       end
 
+      def self.included(mod)
+        mod.include FormatterParams
+      end
+
       attr_reader :_formatters # for tests
 
       def initialize
@@ -73,13 +82,15 @@ module Fluent
       def configure(conf)
         super
 
-        @formatter_configs.each do |section|
-          if @_formatters[section.usage]
-            raise Fluent::ConfigError, "duplicated formatter configured: #{section.usage}"
+        if @formatter_configs
+          @formatter_configs.each do |section|
+            if @_formatters[section.usage]
+              raise Fluent::ConfigError, "duplicated formatter configured: #{section.usage}"
+            end
+            formatter = Plugin.new_formatter(section[:@type], parent: self)
+            formatter.configure(section.corresponding_config_element)
+            @_formatters[section.usage] = formatter
           end
-          formatter = Plugin.new_formatter(section[:@type], parent: self)
-          formatter.configure(section.corresponding_config_element)
-          @_formatters[section.usage] = formatter
         end
       end
 

data/lib/fluent/plugin_helper/inject.rb

@@ -16,6 +16,7 @@
 
 require 'fluent/event'
 require 'time'
+require 'fluent/configurable'
 
 module Fluent
   module PluginHelper
@@ -58,22 +59,28 @@ module Fluent
         new_es
       end
 
-      def self.included(mod)
-        mod.instance_eval do
-          config_section :inject, required: false, multi: false, param_name: :inject_config do
-            config_param :hostname_key, :string, default: nil
-            config_param :hostname, :string, default: nil
-            config_param :tag_key, :string, default: nil
-            config_param :time_key, :string, default: nil
-            config_param :time_type, :enum, list: [:float, :unixtime, :string], default: :float
-            config_param :time_format, :string, default: nil
-            config_param :timezone, :string, default: "#{Time.now.strftime('%z')}" # localtime
-          end
+      module InjectParams
+        include Fluent::Configurable
+        config_section :inject, required: false, multi: false, param_name: :inject_config do
+          config_param :hostname_key, :string, default: nil
+          config_param :hostname, :string, default: nil
+          config_param :tag_key, :string, default: nil
+          config_param :time_key, :string, default: nil
+          config_param :time_type, :enum, list: [:float, :unixtime, :string], default: :float
+          config_param :time_format, :string, default: nil
+          config_param :localtime, :bool, default: true # if localtime is false and timezone is nil, then utc
+          config_param :utc, :bool, default: false # placeholder to turn localtime to false
+          config_param :timezone, :string, default: nil
         end
       end
 
+      def self.included(mod)
+        mod.include InjectParams
+      end
+
       def initialize
         super
+        @_inject_enabled = false
         @_inject_hostname_key = nil
         @_inject_hostname = nil
         @_inject_tag_key = nil
@@ -82,6 +89,12 @@ module Fluent
       end
 
       def configure(conf)
+        conf.elements('inject').each do |e|
+          if e.has_key?('utc') && Fluent::Config.bool_value(e['utc'])
+            e['localtime'] = 'false'
+          end
+        end
+
         super
 
         if @inject_config
@@ -100,7 +113,7 @@ module Fluent
                                       when :float then ->(time){ time.to_r.truncate(+6).to_f } # microsecond floating point value
                                       when :unixtime then ->(time){ time.to_i }
                                       else
-                                        Fluent::TimeFormatter.new(@inject_config.time_format, false, @inject_config.timezone)
+                                        Fluent::TimeFormatter.new(@inject_config.time_format, @inject_config.localtime, @inject_config.timezone)
                                       end
           end
 

data/lib/fluent/plugin_helper/parser.rb

@@ -17,19 +17,25 @@
 require 'fluent/plugin'
 require 'fluent/plugin/parser'
 require 'fluent/config/element'
+require 'fluent/configurable'
 
 module Fluent
   module PluginHelper
     module Parser
-      def parser_create(usage: '', type: nil, conf: nil)
+      def parser_create(usage: '', type: nil, conf: nil, default_type: nil)
         parser = @_parsers[usage]
         return parser if parser
 
-        if !type
-          raise ArgumentError, "BUG: both type and conf are not specified" unless conf
-          raise Fluent::ConfigError, "@type is required in <parse>" unless conf['@type']
-          type = conf['@type']
-        end
+        type = if type
+                 type
+               elsif conf && conf.respond_to?(:[])
+                 raise Fluent::ConfigError, "@type is required in <parse>" unless conf['@type']
+                 conf['@type']
+               elsif default_type
+                 default_type
+               else
+                 raise ArgumentError, "BUG: both type and conf are not specified"
+               end
         parser = Fluent::Plugin.new_parser(type, parent: self)
         config = case conf
                  when Fluent::Config::Element
@@ -52,16 +58,19 @@ module Fluent
         parser
       end
 
-      def self.included(mod)
-        mod.instance_eval do
-          # minimum section definition to instantiate parser plugin instances
-          config_section :parse, required: false, multi: true, param_name: :parser_configs do
-            config_argument :usage, :string, default: ''
-            config_param    :@type, :string
-          end
+      module ParserParams
+        include Fluent::Configurable
+        # minimum section definition to instantiate parser plugin instances
+        config_section :parse, required: false, multi: true, param_name: :parser_configs do
+          config_argument :usage, :string, default: ''
+          config_param    :@type, :string
         end
       end
 
+      def self.included(mod)
+        mod.include ParserParams
+      end
+
       attr_reader :_parsers # for tests
 
       def initialize

data/lib/fluent/plugin_helper/storage.rb

@@ -20,6 +20,7 @@ require 'fluent/plugin'
 require 'fluent/plugin/storage'
 require 'fluent/plugin_helper/timer'
 require 'fluent/config/element'
+require 'fluent/configurable'
 
 module Fluent
   module PluginHelper
@@ -28,18 +29,23 @@ module Fluent
 
       StorageState = Struct.new(:storage, :running)
 
-      def storage_create(usage: '', type: nil, conf: nil)
+      def storage_create(usage: '', type: nil, conf: nil, default_type: nil)
         s = @_storages[usage]
         if s && s.running
           return s.storage
         elsif s
           # storage is already created, but not loaded / started
         else # !s
-          if !type
-            raise ArgumentError, "BUG: both type and conf are not specified" unless conf
-            raise Fluent::ConfigError, "@type is not specified for <storage>" unless conf['@type']
-            type = conf['@type']
-          end
+          type = if type
+                   type
+                 elsif conf && conf.respond_to?(:[])
+                   raise Fluent::ConfigError, "@type is required in <storage>" unless conf['@type']
+                   conf['@type']
+                 elsif default_type
+                   default_type
+                 else
+                   raise ArgumentError, "BUG: both type and conf are not specified"
+                 end
           storage = Plugin.new_storage(type, parent: self)
           config = case conf
                    when Fluent::Config::Element
@@ -63,16 +69,19 @@ module Fluent
         s.storage
       end
 
-      def self.included(mod)
-        mod.instance_eval do
-          # minimum section definition to instantiate storage plugin instances
-          config_section :storage, required: false, multi: true, param_name: :storage_configs do
-            config_argument :usage, :string, default: ''
-            config_param    :@type, :string, default: Fluent::Plugin::Storage::DEFAULT_TYPE
-          end
+      module StorageParams
+        include Fluent::Configurable
+        # minimum section definition to instantiate storage plugin instances
+        config_section :storage, required: false, multi: true, param_name: :storage_configs do
+          config_argument :usage, :string, default: ''
+          config_param    :@type, :string, default: Fluent::Plugin::Storage::DEFAULT_TYPE
         end
       end
 
+      def self.included(mod)
+        mod.include StorageParams
+      end
+
       attr_reader :_storages # for tests
 
       def initialize