fluentd 0.14.6 → 0.14.7

data/lib/fluent/plugin/parser_apache2.rb

@@ -26,7 +26,7 @@ module Fluent
 
       def initialize
         super
-        @time_parser = TimeParser.new(TIME_FORMAT)
+        @time_parser = time_parser_create(format: TIME_FORMAT)
         @mutex = Mutex.new
       end
 

data/lib/fluent/plugin/parser_json.rb

@@ -15,6 +15,7 @@
 #
 
 require 'fluent/plugin/parser'
+require 'fluent/env'
 require 'fluent/time'
 
 require 'yajl'
@@ -25,21 +26,20 @@ module Fluent
       Plugin.register_parser('json', self)
 
       config_param :time_key, :string, default: 'time'
-      config_param :time_format, :string, default: nil
       config_param :json_parser, :string, default: 'oj'
 
       def configure(conf)
         super
 
-        unless @time_format.nil?
-          @time_parser = TimeParser.new(@time_format)
+        if @time_format
+          @time_parser = time_parser_create
           @mutex = Mutex.new
         end
 
         begin
           raise LoadError unless @json_parser == 'oj'
           require 'oj'
-          Oj.default_options = {bigdecimal_load: :float, mode: :strict}
+          Oj.default_options = Fluent::DEFAULT_OJ_OPTIONS
           @load_proc = Oj.method(:load)
           @error_class = Oj::ParseError
         rescue LoadError

data/lib/fluent/plugin/parser_multiline.rb

@@ -38,7 +38,7 @@ module Fluent
           @parser = Fluent::Plugin::RegexpParser.new
           @parser.configure(conf + regexp_conf)
         rescue => e
-          raise ConfigError, "Invalid regexp '#{formats}': #{e}"
+          raise Fluent::ConfigError, "Invalid regexp '#{formats}': #{e}"
         end
 
         if @format_firstline
@@ -68,7 +68,7 @@ module Fluent
         (1..FORMAT_MAX_NUM).map { |i|
           format = conf["format#{i}"]
           if (i > 1) && prev_format.nil? && !format.nil?
-            raise ConfigError, "Jump of format index found. format#{i - 1} is missing."
+            raise Fluent::ConfigError, "Jump of format index found. format#{i - 1} is missing."
           end
           prev_format = format
           next if format.nil?
@@ -84,7 +84,7 @@ module Fluent
           m ? !((1..FORMAT_MAX_NUM).include?(m[1].to_i)) : false
         }
         unless invalid_formats.empty?
-          raise ConfigError, "Invalid formatN found. N should be 1 - #{FORMAT_MAX_NUM}: " + invalid_formats.join(",")
+          raise Fluent::ConfigError, "Invalid formatN found. N should be 1 - #{FORMAT_MAX_NUM}: " + invalid_formats.join(",")
         end
       end
 
@@ -93,10 +93,10 @@ module Fluent
           begin
             Regexp.new(format[1..-2], Regexp::MULTILINE)
           rescue => e
-            raise ConfigError, "Invalid regexp in #{key}: #{e}"
+            raise Fluent::ConfigError, "Invalid regexp in #{key}: #{e}"
           end
         else
-          raise ConfigError, "format should be Regexp, need //, in #{key}: '#{format}'"
+          raise Fluent::ConfigError, "format should be Regexp, need //, in #{key}: '#{format}'"
         end
       end
     end

data/lib/fluent/plugin/parser_regexp.rb

@@ -9,7 +9,6 @@ module Fluent
       config_param :ignorecase, :bool, default: false
       config_param :multiline, :bool, default: false
       config_param :time_key, :string, default: 'time'
-      config_param :time_format, :string, default: nil
 
       def initialize
         super
@@ -18,7 +17,7 @@ module Fluent
 
       def configure(conf)
         super
-        @time_parser = TimeParser.new(@time_format)
+        @time_parser = time_parser_create
         unless @expression.empty?
           if @expression[0] == "/" && @expression[-1] == "/"
             regexp_option = 0

data/lib/fluent/plugin/parser_syslog.rb

@@ -28,7 +28,7 @@ module Fluent
       # From in_syslog default pattern
       REGEXP_WITH_PRI = /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
 
-      config_param :time_format, :string, default: "%b %d %H:%M:%S"
+      config_set_default :time_format, "%b %d %H:%M:%S"
       config_param :with_priority, :bool, default: false
 
       def initialize
@@ -40,7 +40,7 @@ module Fluent
         super
 
         @regexp = @with_priority ? REGEXP_WITH_PRI : REGEXP
-        @time_parser = TimeParser.new(@time_format)
+        @time_parser = time_parser_create
       end
 
       def patterns

data/lib/fluent/plugin/storage_local.rb

@@ -56,7 +56,8 @@ module Fluent
         elsif @path
           # ok
         else # @_plugin_id_configured is true
-          raise NotImplementedError, "implement this feature later with system_config"
+          log.warn "path for <storage> is not specified. Using on-memory store temporarily, but will use file store after support global storage path"
+          @on_memory = true
           ## TODO: get process-wide directory for plugin storage, and generate path for this plugin storage instance
           # path = 
         end

data/lib/fluent/plugin_helper.rb

@@ -23,6 +23,7 @@ require 'fluent/plugin_helper/storage'
 require 'fluent/plugin_helper/parser'
 require 'fluent/plugin_helper/formatter'
 require 'fluent/plugin_helper/inject'
+require 'fluent/plugin_helper/extract'
 require 'fluent/plugin_helper/retry_state'
 require 'fluent/plugin_helper/compat_parameters'
 

data/lib/fluent/plugin_helper/compat_parameters.rb

@@ -44,6 +44,7 @@ module Fluent
       BUFFER_TIME_SLICED_PARAMS = {
         "time_slice_format" => nil,
         "time_slice_wait" => "timekey_wait",
+        "timezone" => "timekey_zone",
       }
 
       PARSER_PARAMS = {
@@ -56,6 +57,7 @@ module Fluent
         "format_firstline" => "format_firstline", # MultilineParser
         "message_key"      => "message_key", # NoneParser
         "with_priority"    => "with_priority", # SyslogParser
+        # There has been no parsers which can handle timezone in v0.12
       }
 
       INJECT_PARAMS = {
@@ -77,7 +79,10 @@ module Fluent
         "json_parser" => "json_parser", # JSONFormatter
         "label_delimiter" => "label_delimiter", # LabeledTSVFormatter
         "output_time" => "output_time", # OutFileFormatter
-        "output_tag"  => "output_tag", # OutFileFormatter
+        "output_tag"  => "output_tag",  # OutFileFormatter
+        "localtime"   => "localtime",   # OutFileFormatter
+        "utc"         => "utc",         # OutFileFormatter
+        "timezone"    => "timezone",    # OutFileFormatter
         "message_key" => "message_key", # SingleValueFormatter
         "add_newline" => "add_newline", # SingleValueFormatter
         "output_type" => "output_type", # StdoutFormatter
@@ -109,7 +114,7 @@ module Fluent
 
         # TODO: warn obsolete parameters if these are deprecated
         buffer_params = BUFFER_PARAMS.merge(BUFFER_TIME_SLICED_PARAMS)
-        attr = compat_parameters_copy_to_subsection_attributes(conf, buffer_params) do |compat_key, value|
+        hash = compat_parameters_copy_to_subsection_attributes(conf, buffer_params) do |compat_key, value|
           if compat_key == 'buffer_queue_full_action' && value == 'exception'
             'throw_exception'
           else
@@ -121,7 +126,7 @@ module Fluent
 
         if conf.has_key?('time_slice_format')
           chunk_key = 'time'
-          attr['timekey'] = case conf['time_slice_format']
+          hash['timekey'] = case conf['time_slice_format']
                             when /\%S/ then 1
                             when /\%M/ then 60
                             when /\%H/ then 3600
@@ -129,13 +134,22 @@ module Fluent
                             else
                               raise Fluent::ConfigError, "time_slice_format only with %Y or %m is too long"
                             end
+          if conf.has_key?('localtime') || conf.has_key?('utc')
+            if conf.has_key?('localtime') && conf.has_key?('utc')
+              raise Fluent::ConfigError, "both of utc and localtime are specified, use only one of them"
+            elsif conf.has_key?('localtime')
+              hash['timekey_use_utc'] = !(Fluent::Config.bool_value(conf['localtime']))
+            elsif conf.has_key?('utc')
+              hash['timekey_use_utc'] = Fluent::Config.bool_value(conf['utc'])
+            end
+          end
         else
           if chunk_key == 'time'
-            attr['timekey'] = 86400 # TimeSliceOutput.time_slice_format default value is '%Y%m%d'
+            hash['timekey'] = 86400 # TimeSliceOutput.time_slice_format default value is '%Y%m%d'
           end
         end
 
-        e = Fluent::Config::Element.new('buffer', chunk_key, attr, [])
+        e = Fluent::Config::Element.new('buffer', chunk_key, hash, [])
         conf.elements << e
 
         conf
@@ -146,22 +160,22 @@ module Fluent
         return if INJECT_PARAMS.keys.all?{|k| !conf.has_key?(k) }
 
         # TODO: warn obsolete parameters if these are deprecated
-        attr = compat_parameters_copy_to_subsection_attributes(conf, INJECT_PARAMS)
+        hash = compat_parameters_copy_to_subsection_attributes(conf, INJECT_PARAMS)
 
         if conf.has_key?('include_time_key') && Fluent::Config.bool_value(conf['include_time_key'])
-          attr['time_key'] ||= 'time'
-          attr['time_type'] ||= 'string'
+          hash['time_key'] ||= 'time'
+          hash['time_type'] ||= 'string'
         end
         if conf.has_key?('time_as_epoch') && Fluent::Config.bool_value(conf['time_as_epoch'])
-          attr['time_type'] = 'unixtime'
+          hash['time_type'] = 'unixtime'
         end
         if conf.has_key?('localtime') || conf.has_key?('utc')
           if conf.has_key?('localtime') && conf.has_key?('utc')
             raise Fluent::ConfigError, "both of utc and localtime are specified, use only one of them"
           elsif conf.has_key?('localtime')
-            attr['localtime'] = Fluent::Config.bool_value(conf['localtime'])
+            hash['localtime'] = Fluent::Config.bool_value(conf['localtime'])
           elsif conf.has_key?('utc')
-            attr['localtime'] = !(Fluent::Config.bool_value(conf['utc']))
+            hash['localtime'] = !(Fluent::Config.bool_value(conf['utc']))
             # Specifying "localtime false" means using UTC in TimeFormatter
             # And specifying "utc" is different from specifying "timezone +0000"(it's not always UTC).
             # There are difference between "Z" and "+0000" in timezone formatting.
@@ -170,10 +184,10 @@ module Fluent
         end
 
         if conf.has_key?('include_tag_key') && Fluent::Config.bool_value(conf['include_tag_key'])
-          attr['tag_key'] ||= 'tag'
+          hash['tag_key'] ||= 'tag'
         end
 
-        e = Fluent::Config::Element.new('inject', '', attr, [])
+        e = Fluent::Config::Element.new('inject', '', hash, [])
         conf.elements << e
 
         conf
@@ -184,9 +198,9 @@ module Fluent
         return if PARSER_PARAMS.keys.all?{|k| !conf.has_key?(k) }
 
         # TODO: warn obsolete parameters if these are deprecated
-        attr = compat_parameters_copy_to_subsection_attributes(conf, PARSER_PARAMS)
+        hash = compat_parameters_copy_to_subsection_attributes(conf, PARSER_PARAMS)
 
-        e = Fluent::Config::Element.new('parse', '', attr, [])
+        e = Fluent::Config::Element.new('parse', '', hash, [])
         conf.elements << e
 
         conf
@@ -197,27 +211,31 @@ module Fluent
         return if FORMATTER_PARAMS.keys.all?{|k| !conf.has_key?(k) }
 
         # TODO: warn obsolete parameters if these are deprecated
-        attr = compat_parameters_copy_to_subsection_attributes(conf, FORMATTER_PARAMS)
+        hash = compat_parameters_copy_to_subsection_attributes(conf, FORMATTER_PARAMS)
+
+        if conf.has_key?('time_as_epoch') && Fluent::Config.bool_value(conf['time_as_epoch'])
+          hash['time_type'] = 'unixtime'
+        end
 
-        e = Fluent::Config::Element.new('format', '', attr, [])
+        e = Fluent::Config::Element.new('format', '', hash, [])
         conf.elements << e
 
         conf
       end
 
       def compat_parameters_copy_to_subsection_attributes(conf, params, &block)
-        attr = {}
+        hash = {}
         params.each do |compat, current|
           next unless current
           if conf.has_key?(compat)
             if block_given?
-              attr[current] = block.call(compat, conf[compat])
+              hash[current] = block.call(compat, conf[compat])
             else
-              attr[current] = conf[compat]
+              hash[current] = conf[compat]
             end
           end
         end
-        attr
+        hash
       end
     end
   end

data/lib/fluent/plugin_helper/extract.rb

@@ -0,0 +1,92 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/event'
+require 'fluent/time'
+require 'fluent/configurable'
+
+module Fluent
+  module PluginHelper
+    module Extract
+      def extract_tag_from_record(record)
+        return nil unless @_extract_enabled
+
+        if @_extract_tag_key && record.has_key?(@_extract_tag_key)
+          return record[@_extract_tag_key].to_s
+        end
+
+        nil
+      end
+
+      def extract_time_from_record(record)
+        return nil unless @_extract_enabled
+
+        if @_extract_time_key && record.has_key?(@_extract_time_key)
+          return @_extract_time_parser.call(record[@_extract_time_key])
+        end
+
+        nil
+      end
+
+      module ExtractParams
+        include Fluent::Configurable
+        config_section :extract, required: false, multi: false, param_name: :extract_config do
+          config_param :tag_key, :string, default: nil
+          config_param :time_key, :string, default: nil
+
+          # To avoid defining :time_type twice
+          config_param :time_type, :enum, list: [:float, :unixtime, :string], default: :float
+
+          Fluent::TimeMixin::TIME_PARAMETERS.each do |name, type, opts|
+            config_param name, type, opts
+          end
+        end
+      end
+
+      def self.included(mod)
+        mod.include ExtractParams
+      end
+
+      def initialize
+        super
+        @_extract_enabled = false
+        @_extract_tag_key = nil
+        @_extract_time_key = nil
+        @_extract_time_parser = nil
+      end
+
+      def configure(conf)
+        super
+
+        if @extract_config
+          @_extract_tag_key = @extract_config.tag_key
+          @_extract_time_key = @extract_config.time_key
+          if @_extract_time_key
+            @_extract_time_parser = case @extract_config.time_type
+                                    when :float then ->(v){ Fluent::EventTime.new(v.to_i, ((v.to_f - v.to_i) * 1_000_000_000).to_i) }
+                                    when :unixtime then ->(v){ Fluent::EventTime.new(v.to_i, 0) }
+                                    else
+                                      localtime = @extract_config.localtime && !@extract_config.utc
+                                      Fluent::TimeParser.new(@extract_config.time_format, localtime, @extract_config.timezone)
+                                    end
+          end
+
+          @_extract_enabled = @_extract_tag_key || @_extract_time_key
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/inject.rb

@@ -15,8 +15,9 @@
 #
 
 require 'fluent/event'
-require 'time'
+require 'fluent/time'
 require 'fluent/configurable'
+require 'socket'
 
 module Fluent
   module PluginHelper
@@ -66,11 +67,13 @@ module Fluent
           config_param :hostname, :string, default: nil
           config_param :tag_key, :string, default: nil
           config_param :time_key, :string, default: nil
+
+          # To avoid defining :time_type twice
           config_param :time_type, :enum, list: [:float, :unixtime, :string], default: :float
-          config_param :time_format, :string, default: nil
-          config_param :localtime, :bool, default: true # if localtime is false and timezone is nil, then utc
-          config_param :utc, :bool, default: false # placeholder to turn localtime to false
-          config_param :timezone, :string, default: nil
+
+          Fluent::TimeMixin::TIME_PARAMETERS.each do |name, type, opts|
+            config_param name, type, opts
+          end
         end
       end
 
@@ -89,12 +92,6 @@ module Fluent
       end
 
       def configure(conf)
-        conf.elements('inject').each do |e|
-          if e.has_key?('utc') && Fluent::Config.bool_value(e['utc'])
-            e['localtime'] = 'false'
-          end
-        end
-
         super
 
         if @inject_config
@@ -113,7 +110,8 @@ module Fluent
                                       when :float then ->(time){ time.to_r.truncate(+6).to_f } # microsecond floating point value
                                       when :unixtime then ->(time){ time.to_i }
                                       else
-                                        Fluent::TimeFormatter.new(@inject_config.time_format, @inject_config.localtime, @inject_config.timezone)
+                                        localtime = @inject_config.localtime && !@inject_config.utc
+                                        Fluent::TimeFormatter.new(@inject_config.time_format, localtime, @inject_config.timezone)
                                       end
           end
 

data/lib/fluent/plugin_helper/thread.rb

@@ -70,7 +70,7 @@ module Fluent
             thread_exit = true
             raise
           ensure
-            unless thread_exit
+            if ::Thread.current.alive? && !thread_exit
               log.warn "thread doesn't exit correctly (killed or other reason)", plugin: self.class, title: title, thread: ::Thread.current, error: $!
             end
             @_threads_mutex.synchronize do
@@ -110,11 +110,31 @@ module Fluent
 
       def stop
         super
+        wakeup_threads = []
         @_threads_mutex.synchronize do
-          @_threads.each_pair do |obj_id, thread|
+          @_threads.each_value do |thread|
             thread[:_fluentd_plugin_helper_thread_running] = false
+            wakeup_threads << thread if thread.alive? && thread.status == "sleep"
           end
         end
+        wakeup_threads.each do |thread|
+          if thread.alive?
+            thread.wakeup
+          end
+        end
+      end
+
+      def after_shutdown
+        super
+        wakeup_threads = []
+        @_threads_mutex.synchronize do
+          @_threads.each_value do |thread|
+            wakeup_threads << thread if thread.alive? && thread.status == "sleep"
+          end
+        end
+        wakeup_threads.each do |thread|
+          thread.wakeup if thread.alive?
+        end
       end
 
       def close
@@ -132,7 +152,7 @@ module Fluent
         super
         @_threads_mutex.synchronize{ @_threads.keys }.each do |obj_id|
           thread = @_threads[obj_id]
-          log.warn "killing existing thead", thread: thread
+          log.warn "killing existing thread", thread: thread
           thread.kill if thread
         end
         @_threads_mutex.synchronize{ @_threads.keys }.each do |obj_id|