fluentd 0.14.7-x86-mingw32 → 0.14.10-x86-mingw32

data/lib/fluent/plugin/out_stdout.rb

@@ -22,6 +22,7 @@ module Fluent::Plugin
 
     helpers :inject, :formatter, :compat_parameters
 
+    DEFAULT_LINE_FORMAT_TYPE = 'stdout'
     DEFAULT_FORMAT_TYPE = 'json'
     TIME_FORMAT = '%Y-%m-%d %H:%M:%S.%9N %z'
 
@@ -32,33 +33,22 @@ module Fluent::Plugin
     end
 
     config_section :format do
-      config_set_default :@type, DEFAULT_FORMAT_TYPE
+      config_set_default :@type, DEFAULT_LINE_FORMAT_TYPE
+      config_set_default :output_type, DEFAULT_FORMAT_TYPE
     end
 
     def prefer_buffered_processing
       false
     end
 
-    def prefer_delayed_commit
-      @delayed
-    end
-
-    attr_accessor :delayed
-
-    def initialize
-      super
-      @delayed = false
-    end
+    attr_accessor :formatter
 
     def configure(conf)
-      if conf['output_type'] && !conf['format']
-        conf['format'] = conf['output_type']
-      end
       compat_parameters_convert(conf, :inject, :formatter)
 
       super
 
-      @formatter = formatter_create(conf: conf.elements('format').first, default_type: DEFAULT_FORMAT_TYPE)
+      @formatter = formatter_create
     end
 
     def process(tag, es)
@@ -70,16 +60,11 @@ module Fluent::Plugin
 
     def format(tag, time, record)
       record = inject_values_to_record(tag, time, record)
-      "#{Time.at(time).localtime.strftime(TIME_FORMAT)} #{tag}: #{@formatter.format(tag, time, record).chomp}\n"
+      @formatter.format(tag, time, record).chomp + "\n"
     end
 
     def write(chunk)
       chunk.write_to($log)
     end
-
-    def try_write(chunk)
-      chunk.write_to($log)
-      commit_write(chunk.unique_id)
-    end
   end
 end

data/lib/fluent/plugin/output.rb

@@ -40,6 +40,8 @@ module Fluent
 
       CHUNKING_FIELD_WARN_NUM = 4
 
+      PROCESS_CLOCK_ID = Process::CLOCK_MONOTONIC_RAW rescue Process::CLOCK_MONOTONIC
+
       config_param :time_as_integer, :bool, default: false
 
       # `<buffer>` and `<secondary>` sections are available only when '#format' and '#write' are implemented
@@ -138,7 +140,7 @@ module Fluent
       end
 
       # Internal states
-      FlushThreadState = Struct.new(:thread, :next_time)
+      FlushThreadState = Struct.new(:thread, :next_clock)
       DequeuedChunkInfo = Struct.new(:chunk_id, :time, :timeout) do
         def expired?
           time + timeout < Time.now
@@ -150,10 +152,9 @@ module Fluent
 
       # for tests
       attr_reader :buffer, :retry, :secondary, :chunk_keys, :chunk_key_time, :chunk_key_tag
-      attr_accessor :output_enqueue_thread_waiting, :in_tests
-
+      attr_accessor :output_enqueue_thread_waiting, :dequeued_chunks, :dequeued_chunks_mutex
       # output_enqueue_thread_waiting: for test of output.rb itself
-      # in_tests: for tests of plugins with test drivers
+      attr_accessor :retry_for_error_chunk # if true, error flush will be retried even if under_plugin_development is true
 
       def initialize
         super
@@ -161,7 +162,6 @@ module Fluent
         @buffering = false
         @delayed_commit = false
         @as_secondary = false
-        @in_tests = false
         @primary_instance = nil
 
         # TODO: well organized counters
@@ -188,6 +188,7 @@ module Fluent
         @secondary = nil
         @retry = nil
         @dequeued_chunks = nil
+        @dequeued_chunks_mutex = nil
         @output_enqueue_thread = nil
         @output_flush_threads = nil
 
@@ -195,6 +196,8 @@ module Fluent
         @chunk_keys = @chunk_key_time = @chunk_key_tag = nil
         @flush_mode = nil
         @timekey_zone = nil
+
+        @retry_for_error_chunk = false
       end
 
       def acts_as_secondary(primary)
@@ -207,6 +210,7 @@ module Fluent
           @timekey_zone = @primary_instance.timekey_zone
           @output_time_formatter_cache = {}
         end
+        self.context_router = primary.context_router
 
         (class << self; self; end).module_eval do
           define_method(:commit_write){ |chunk_id| @primary_instance.commit_write(chunk_id, delayed: delayed_commit, secondary: true) }
@@ -317,7 +321,6 @@ module Fluent
           @secondary = Plugin.new_output(secondary_type)
           @secondary.acts_as_secondary(self)
           @secondary.configure(secondary_conf)
-          @secondary.router = router if @secondary.has_router?
           if (self.class != @secondary.class) && (@custom_format || @secondary.implement?(:custom_format))
             log.warn "secondary type should be same with primary one", primary: self.class.to_s, secondary: @secondary.class.to_s
           end
@@ -399,10 +402,8 @@ module Fluent
           end
           @output_flush_thread_current_position = 0
 
-          unless @in_tests
-            if @flush_mode == :interval || @chunk_key_time
-              @output_enqueue_thread = thread_create(:enqueue_thread, &method(:enqueue_thread_run))
-            end
+          if !@under_plugin_development && (@flush_mode == :interval || @chunk_key_time)
+            @output_enqueue_thread = thread_create(:enqueue_thread, &method(:enqueue_thread_run))
           end
         end
         @secondary.start if @secondary
@@ -899,9 +900,9 @@ module Fluent
         @retry_mutex.synchronize do
           if @retry # success to flush chunks in retries
             if secondary
-              log.warn "retry succeeded by secondary.", plugin_id: plugin_id, chunk_id: dump_unique_id_hex(chunk_id)
+              log.warn "retry succeeded by secondary.", chunk_id: dump_unique_id_hex(chunk_id)
             else
-              log.warn "retry succeeded.", plugin_id: plugin_id, chunk_id: dump_unique_id_hex(chunk_id)
+              log.warn "retry succeeded.", chunk_id: dump_unique_id_hex(chunk_id)
             end
             @retry = nil
           end
@@ -919,6 +920,8 @@ module Fluent
         # in many cases, false can be just ignored
         if @buffer.takeback_chunk(chunk_id)
           @counters_monitor.synchronize{ @rollback_count += 1 }
+          primary = @as_secondary ? @primary_instance : self
+          primary.update_retry_state(chunk_id, @as_secondary)
           true
         else
           false
@@ -931,7 +934,9 @@ module Fluent
             info = @dequeued_chunks.shift
             if @buffer.takeback_chunk(info.chunk_id)
               @counters_monitor.synchronize{ @rollback_count += 1 }
-              log.warn "failed to flush the buffer chunk, timeout to commit.", plugin_id: plugin_id, chunk_id: dump_unique_id_hex(info.chunk_id), flushed_at: info.time
+              log.warn "failed to flush the buffer chunk, timeout to commit.", chunk_id: dump_unique_id_hex(info.chunk_id), flushed_at: info.time
+              primary = @as_secondary ? @primary_instance : self
+              primary.update_retry_state(info.chunk_id, @as_secondary)
             end
           end
         end
@@ -944,7 +949,9 @@ module Fluent
             info = @dequeued_chunks.shift
             if @buffer.takeback_chunk(info.chunk_id)
               @counters_monitor.synchronize{ @rollback_count += 1 }
-              log.info "delayed commit for buffer chunks was cancelled in shutdown", plugin_id: plugin_id, chunk_id: dump_unique_id_hex(info.chunk_id)
+              log.info "delayed commit for buffer chunks was cancelled in shutdown", chunk_id: dump_unique_id_hex(info.chunk_id)
+              primary = @as_secondary ? @primary_instance : self
+              primary.update_retry_state(info.chunk_id, @as_secondary)
             end
           end
         end
@@ -981,11 +988,11 @@ module Fluent
           if output.delayed_commit
             log.trace "executing delayed write and commit", chunk: dump_unique_id_hex(chunk.unique_id)
             @counters_monitor.synchronize{ @write_count += 1 }
-            output.try_write(chunk)
             @dequeued_chunks_mutex.synchronize do
               # delayed_commit_timeout for secondary is configured in <buffer> of primary (<secondary> don't get <buffer>)
               @dequeued_chunks << DequeuedChunkInfo.new(chunk.unique_id, Time.now, self.delayed_commit_timeout)
             end
+            output.try_write(chunk)
           else # output plugin without delayed purge
             chunk_id = chunk.unique_id
             dump_chunk_id = dump_unique_id_hex(chunk_id)
@@ -994,44 +1001,64 @@ module Fluent
             log.trace "executing sync write", chunk: dump_chunk_id
             output.write(chunk)
             log.trace "write operation done, committing", chunk: dump_chunk_id
-            commit_write(chunk_id, secondary: using_secondary)
+            commit_write(chunk_id, delayed: false, secondary: using_secondary)
             log.trace "done to commit a chunk", chunk: dump_chunk_id
           end
         rescue => e
-          log.debug "taking back chunk for errors.", plugin_id: plugin_id, chunk: dump_unique_id_hex(chunk.unique_id)
+          log.debug "taking back chunk for errors.", chunk: dump_unique_id_hex(chunk.unique_id)
+          if output.delayed_commit
+            @dequeued_chunks_mutex.synchronize do
+              @dequeued_chunks.delete_if{|d| d.chunk_id == chunk.unique_id }
+            end
+          end
           @buffer.takeback_chunk(chunk.unique_id)
 
-          if @under_plugin_development
-            raise
+          update_retry_state(chunk.unique_id, using_secondary, e)
+
+          raise if @under_plugin_development && !@retry_for_error_chunk
+        end
+      end
+
+      def update_retry_state(chunk_id, using_secondary, error = nil)
+        @retry_mutex.synchronize do
+          @counters_monitor.synchronize{ @num_errors += 1 }
+          chunk_id_hex = dump_unique_id_hex(chunk_id)
+
+          unless @retry
+            @retry = retry_state(@buffer_config.retry_randomize)
+            if error
+              log.warn "failed to flush the buffer.", retry_time: @retry.steps, next_retry_seconds: @retry.next_time, chunk: chunk_id_hex, error: error
+              log.warn_backtrace error.backtrace
+            end
+            return
           end
 
-          @retry_mutex.synchronize do
-            if @retry
-              @counters_monitor.synchronize{ @num_errors += 1 }
-              if @retry.limit?
-                records = @buffer.queued_records
-                log.error "failed to flush the buffer, and hit limit for retries. dropping all chunks in the buffer queue.", plugin_id: plugin_id, retry_times: @retry.steps, records: records, error: e
-                log.error_backtrace e.backtrace
-                @buffer.clear_queue!
-                log.debug "buffer queue cleared", plugin_id: plugin_id
-                @retry = nil
-              else
-                @retry.step
-                msg = if using_secondary
-                        "failed to flush the buffer with secondary output."
-                      else
-                        "failed to flush the buffer."
-                      end
-                log.warn msg, plugin_id: plugin_id, retry_time: @retry.steps, next_retry: @retry.next_time, chunk: dump_unique_id_hex(chunk.unique_id), error: e
-                log.warn_backtrace e.backtrace
-              end
+          # @retry exists
+
+          if error
+            if @retry.limit?
+              records = @buffer.queued_records
+              msg = "failed to flush the buffer, and hit limit for retries. dropping all chunks in the buffer queue."
+              log.error msg, retry_times: @retry.steps, records: records, error: error
+              log.error_backtrace error.backtrace
+            elsif using_secondary
+              msg = "failed to flush the buffer with secondary output."
+              log.warn msg, retry_time: @retry.steps, next_retry_seconds: @retry.next_time, chunk: chunk_id_hex, error: error
+              log.warn_backtrace error.backtrace
             else
-              @retry = retry_state(@buffer_config.retry_randomize)
-              @counters_monitor.synchronize{ @num_errors += 1 }
-              log.warn "failed to flush the buffer.", plugin_id: plugin_id, retry_time: @retry.steps, next_retry: @retry.next_time, chunk: dump_unique_id_hex(chunk.unique_id), error: e
-              log.warn_backtrace e.backtrace
+              msg = "failed to flush the buffer."
+              log.warn msg, retry_time: @retry.steps, next_retry_seconds: @retry.next_time, chunk: chunk_id_hex, error: error
+              log.warn_backtrace error.backtrace
             end
           end
+
+          if @retry.limit?
+            @buffer.clear_queue!
+            log.debug "buffer queue cleared"
+            @retry = nil
+          else
+            @retry.step
+          end
         end
       end
 
@@ -1058,8 +1085,12 @@ module Fluent
         # Without locks: it is rough but enough to select "next" writer selection
         @output_flush_thread_current_position = (@output_flush_thread_current_position + 1) % @buffer_config.flush_thread_count
         state = @output_flush_threads[@output_flush_thread_current_position]
-        state.next_time = 0
-        state.thread.run
+        state.next_clock = 0
+        if state.thread && state.thread.status # "run"/"sleep"/"aborting" or false(successfully stop) or nil(killed by exception)
+          state.thread.run
+        else
+          log.warn "thread is already dead"
+        end
       end
 
       def force_flush
@@ -1096,7 +1127,7 @@ module Fluent
       # only for tests of output plugin
       def flush_thread_wakeup
         @output_flush_threads.each do |state|
-          state.next_time = 0
+          state.next_clock = 0
           state.thread.run
         end
       end
@@ -1150,7 +1181,7 @@ module Fluent
               end
             rescue => e
               raise if @under_plugin_development
-              log.error "unexpected error while checking flushed chunks. ignored.", plugin_id: plugin_id, error: e
+              log.error "unexpected error while checking flushed chunks. ignored.", error: e
               log.error_backtrace
             ensure
               @output_enqueue_thread_waiting = false
@@ -1160,7 +1191,7 @@ module Fluent
           end
         rescue => e
           # normal errors are rescued by inner begin-rescue clause.
-          log.error "error on enqueue thread", plugin_id: plugin_id, error: e
+          log.error "error on enqueue thread", error: e
           log.error_backtrace
           raise
         end
@@ -1169,9 +1200,7 @@ module Fluent
       def flush_thread_run(state)
         flush_thread_interval = @buffer_config.flush_thread_interval
 
-        # If the given clock_id is not supported, Errno::EINVAL is raised.
-        clock_id = Process::CLOCK_MONOTONIC rescue Process::CLOCK_MONOTONIC_RAW
-        state.next_time = Process.clock_gettime(clock_id) + flush_thread_interval
+        state.next_clock = Process.clock_gettime(PROCESS_CLOCK_ID) + flush_thread_interval
 
         while !self.after_started? && !self.stopped?
           sleep 0.5
@@ -1181,16 +1210,18 @@ module Fluent
         begin
           # This thread don't use `thread_current_running?` because this thread should run in `before_shutdown` phase
           while @output_flush_threads_running
-            time = Process.clock_gettime(clock_id)
-            interval = state.next_time - time
+            current_clock = Process.clock_gettime(PROCESS_CLOCK_ID)
+            interval = state.next_clock - current_clock
 
-            if state.next_time <= time
+            if state.next_clock <= current_clock && (!@retry || @retry_mutex.synchronize{ @retry.next_time } <= Time.now)
               try_flush
-              # next_flush_interval uses flush_thread_interval or flush_thread_burst_interval (or retrying)
+
+              # next_flush_time uses flush_thread_interval or flush_thread_burst_interval (or retrying)
               interval = next_flush_time.to_f - Time.now.to_f
-              # TODO: if secondary && delayed-commit, next_flush_time will be much longer than expected (because @retry still exists)
-              #   @retry should be cleared if delayed commit is enabled? Or any other solution?
-              state.next_time = Process.clock_gettime(clock_id) + interval
+              # TODO: if secondary && delayed-commit, next_flush_time will be much longer than expected
+              #       because @retry still exists (#commit_write is not called yet in #try_flush)
+              #       @retry should be cleared if delayed commit is enabled? Or any other solution?
+              state.next_clock = Process.clock_gettime(PROCESS_CLOCK_ID) + interval
             end
 
             if @dequeued_chunks_mutex.synchronize{ !@dequeued_chunks.empty? && @dequeued_chunks.first.expired? }
@@ -1204,7 +1235,7 @@ module Fluent
         rescue => e
           # normal errors are rescued by output plugins in #try_flush
           # so this rescue section is for critical & unrecoverable errors
-          log.error "error on output thread", plugin_id: plugin_id, error_class: e.class, error: e
+          log.error "error on output thread", error: e
           log.error_backtrace
           raise
         end

data/lib/fluent/plugin/parser.rb

@@ -31,17 +31,43 @@ module Fluent
 
       configured_in :parse
 
-      # SET false BEFORE CONFIGURE, to return nil when time not parsed
-      attr_accessor :estimate_current_event
+      ### types can be specified as string-based hash style
+      # field1:type, field2:type, field3:type:option, field4:type:option
+      ### or, JSON format
+      # {"field1":"type", "field2":"type", "field3":"type:option", "field4":"type:option"}
+      config_param :types, :hash, value_type: :string, default: nil
 
+      # available options are:
+      # array: (1st) delimiter
+      # time : type[, format, timezone] -> type should be a valid "time_type"(string/unixtime/float)
+      #      : format[, timezone]
+
+      config_param :time_key, :string, default: nil
+      config_param :null_value_pattern, :string, default: nil
+      config_param :null_empty_string, :bool, default: false
+      config_param :estimate_current_event, :bool, default: true
       config_param :keep_time_key, :bool, default: false
 
-      def initialize
+      AVAILABLE_PARSER_VALUE_TYPES = ['string', 'integer', 'float', 'bool', 'time', 'array']
+
+      # for tests
+      attr_reader :type_converters
+
+      PARSER_TYPES = [:text_per_line, :text, :binary]
+      def parser_type
+        :text_per_line
+      end
+
+      def configure(conf)
         super
-        @estimate_current_event = true
+
+        @time_parser = time_parser_create
+        @null_value_regexp = @null_value_pattern && Regexp.new(@null_value_pattern)
+        @type_converters = build_type_converters(@types)
+        @execute_convert_values = @type_converters || @null_value_regexp || @null_empty_string
       end
 
-      def parse(text)
+      def parse(text, &block)
         raise NotImplementedError, "Implement this method in child class"
       end
 
@@ -51,80 +77,114 @@ module Fluent
         parse(*a, &b)
       end
 
-      TimeParser = Fluent::TimeParser
-    end
-
-    class ValuesParser < Parser
-      include Fluent::TypeConverter
+      def implement?(feature)
+        methods_of_plugin = self.class.instance_methods(false)
+        case feature
+        when :parse_io then methods_of_plugin.include?(:parse_io)
+        when :parse_partial_data then methods_of_plugin.include?(:parse_partial_data)
+        else
+          raise ArgumentError, "Unknown feature for parser plugin: #{feature}"
+        end
+      end
 
-      config_param :keys, :array, default: []
-      config_param :time_key, :string, default: nil
-      config_param :null_value_pattern, :string, default: nil
-      config_param :null_empty_string, :bool, default: false
+      def parse_io(io, &block)
+        raise NotImplementedError, "Optional API #parse_io is not implemented"
+      end
 
-      def configure(conf)
-        super
+      def parse_partial_data(data, &block)
+        raise NotImplementedError, "Optional API #parse_partial_data is not implemented"
+      end
 
-        if @time_key && !@keys.include?(@time_key) && @estimate_current_event
-          raise Fluent::ConfigError, "time_key (#{@time_key.inspect}) is not included in keys (#{@keys.inspect})"
+      def parse_time(record)
+        if @time_key && record.respond_to?(:has_key?) && record.has_key?(@time_key)
+          src = if @keep_time_key
+                  record[@time_key]
+                else
+                  record.delete(@time_key)
+                end
+          @time_parser.parse(src)
+        elsif @estimate_current_event
+          Fluent::EventTime.now
+        else
+          nil
         end
+      rescue Fluent::TimeParser::TimeParseError => e
+        raise ParserError, e.message
+      end
 
-        if @time_format && !@time_key
-          raise Fluent::ConfigError, "time_format parameter is ignored because time_key parameter is not set. at #{conf.inspect}"
-        end
+      # def parse(text, &block)
+      #   time, record = convert_values(time, record)
+      #   yield time, record
+      # end
+      def convert_values(time, record)
+        return time, record unless @execute_convert_values
 
-        @time_parser = time_parser_create
+        record.each_key do |key|
+          value = record[key]
+          next unless value # nil/null value is always left as-is.
 
-        if @null_value_pattern
-          @null_value_pattern = Regexp.new(@null_value_pattern)
+          if value.is_a?(String) && string_like_null(value)
+            record[key] = nil
+            next
+          end
+
+          if @type_converters && @type_converters.has_key?(key)
+            record[key] = @type_converters[key].call(value)
+          end
         end
 
-        @mutex = Mutex.new
+        return time, record
       end
 
-      def values_map(values)
-        record = Hash[keys.zip(values.map { |value| convert_value_to_nil(value) })]
-
-        if @time_key
-          value = @keep_time_key ? record[@time_key] : record.delete(@time_key)
-          time = if value.nil?
-                   if @estimate_current_event
-                     Fluent::EventTime.now
-                   else
-                     nil
-                   end
-                 else
-                   @mutex.synchronize { @time_parser.parse(value) }
-                 end
-        elsif @estimate_current_event
-          time = Fluent::EventTime.now
-        else
-          time = nil
-        end
+      def string_like_null(value, null_empty_string = @null_empty_string, null_value_regexp = @null_value_regexp)
+        null_empty_string && value.empty? || null_value_regexp && string_safe_encoding(value){|s| null_value_regexp.match(s) }
+      end
 
-        convert_field_type!(record) if @type_converters
+      TRUTHY_VALUES = ['true', 'yes', '1']
 
-        return time, record
-      end
+      def build_type_converters(types)
+        return nil unless types
 
-      private
+        converters = {}
 
-      def convert_field_type!(record)
-        @type_converters.each_key { |key|
-          if value = record[key]
-            record[key] = convert_type(key, value)
+        types.each_pair do |field_name, type_definition|
+          type, option = type_definition.split(":", 2)
+          unless AVAILABLE_PARSER_VALUE_TYPES.include?(type)
+            raise Fluent::ConfigError, "unknown value conversion for key:'#{field_name}', type:'#{type}'"
           end
-        }
-      end
 
-      def convert_value_to_nil(value)
-        if value and @null_empty_string
-          value = (value == '') ? nil : value
-        end
-        if value and @null_value_pattern
-          value = ::Fluent::StringUtil.match_regexp(@null_value_pattern, value) ? nil : value
+          conv = case type
+                 when 'string' then ->(v){ v.to_s }
+                 when 'integer' then ->(v){ v.to_i rescue v.to_s.to_i }
+                 when 'float' then ->(v){ v.to_f rescue v.to_s.to_f }
+                 when 'bool' then ->(v){ TRUTHY_VALUES.include?(v.to_s.downcase) }
+                 when 'time'
+                   # comma-separated: time:[timezone:]time_format
+                   # time_format is unixtime/float/string-time-format
+                   timep = if option
+                             time_type = 'string' # estimate
+                             timezone, time_format = option.split(':', 2)
+                             unless Fluent::Timezone.validate(timezone)
+                               timezone, time_format = nil, option
+                             end
+                             if Fluent::TimeMixin::TIME_TYPES.include?(time_format)
+                               time_type, time_format = time_format, nil # unixtime/float
+                             end
+                             time_parser_create(type: time_type.to_sym, format: time_format, timezone: timezone)
+                           else
+                             time_parser_create(type: :string, format: nil, timezone: nil)
+                           end
+                   ->(v){ timep.parse(v) rescue nil }
+                 when 'array'
+                   delimiter = option ? option.to_s : ','
+                   ->(v){ string_safe_encoding(v.to_s){|s| s.split(delimiter) } }
+                 else
+                   raise "BUG: unknown type even after check: #{type}"
+                 end
+          converters[field_name] = conv
         end
-        value
+
+        converters
       end
     end
   end