fluentd 0.14.8 → 0.14.9

data/lib/fluent/plugin/out_stdout.rb

@@ -22,6 +22,7 @@ module Fluent::Plugin
 
     helpers :inject, :formatter, :compat_parameters
 
+    DEFAULT_LINE_FORMAT_TYPE = 'stdout'
     DEFAULT_FORMAT_TYPE = 'json'
     TIME_FORMAT = '%Y-%m-%d %H:%M:%S.%9N %z'
 
@@ -32,33 +33,22 @@ module Fluent::Plugin
     end
 
     config_section :format do
-      config_set_default :@type, DEFAULT_FORMAT_TYPE
+      config_set_default :@type, DEFAULT_LINE_FORMAT_TYPE
+      config_set_default :output_type, DEFAULT_FORMAT_TYPE
     end
 
     def prefer_buffered_processing
       false
     end
 
-    def prefer_delayed_commit
-      @delayed
-    end
-
-    attr_accessor :delayed
-
-    def initialize
-      super
-      @delayed = false
-    end
+    attr_accessor :formatter
 
     def configure(conf)
-      if conf['output_type'] && !conf['format']
-        conf['format'] = conf['output_type']
-      end
       compat_parameters_convert(conf, :inject, :formatter)
 
       super
 
-      @formatter = formatter_create(conf: conf.elements('format').first, default_type: DEFAULT_FORMAT_TYPE)
+      @formatter = formatter_create
     end
 
     def process(tag, es)
@@ -70,16 +60,11 @@ module Fluent::Plugin
 
     def format(tag, time, record)
       record = inject_values_to_record(tag, time, record)
-      "#{Time.at(time).localtime.strftime(TIME_FORMAT)} #{tag}: #{@formatter.format(tag, time, record).chomp}\n"
+      @formatter.format(tag, time, record).chomp + "\n"
     end
 
     def write(chunk)
       chunk.write_to($log)
     end
-
-    def try_write(chunk)
-      chunk.write_to($log)
-      commit_write(chunk.unique_id)
-    end
   end
 end

data/lib/fluent/plugin/output.rb

@@ -150,10 +150,9 @@ module Fluent
 
       # for tests
       attr_reader :buffer, :retry, :secondary, :chunk_keys, :chunk_key_time, :chunk_key_tag
-      attr_accessor :output_enqueue_thread_waiting, :in_tests
-
+      attr_accessor :output_enqueue_thread_waiting, :dequeued_chunks, :dequeued_chunks_mutex
       # output_enqueue_thread_waiting: for test of output.rb itself
-      # in_tests: for tests of plugins with test drivers
+      attr_accessor :retry_for_error_chunk # if true, error flush will be retried even if under_plugin_development is true
 
       def initialize
         super
@@ -161,7 +160,6 @@ module Fluent
         @buffering = false
         @delayed_commit = false
         @as_secondary = false
-        @in_tests = false
         @primary_instance = nil
 
         # TODO: well organized counters
@@ -188,6 +186,7 @@ module Fluent
         @secondary = nil
         @retry = nil
         @dequeued_chunks = nil
+        @dequeued_chunks_mutex = nil
         @output_enqueue_thread = nil
         @output_flush_threads = nil
 
@@ -195,6 +194,8 @@ module Fluent
         @chunk_keys = @chunk_key_time = @chunk_key_tag = nil
         @flush_mode = nil
         @timekey_zone = nil
+
+        @retry_for_error_chunk = false
       end
 
       def acts_as_secondary(primary)
@@ -207,6 +208,7 @@ module Fluent
           @timekey_zone = @primary_instance.timekey_zone
           @output_time_formatter_cache = {}
         end
+        self.context_router = primary.context_router
 
         (class << self; self; end).module_eval do
           define_method(:commit_write){ |chunk_id| @primary_instance.commit_write(chunk_id, delayed: delayed_commit, secondary: true) }
@@ -317,7 +319,6 @@ module Fluent
           @secondary = Plugin.new_output(secondary_type)
           @secondary.acts_as_secondary(self)
           @secondary.configure(secondary_conf)
-          @secondary.router = router if @secondary.has_router?
           if (self.class != @secondary.class) && (@custom_format || @secondary.implement?(:custom_format))
             log.warn "secondary type should be same with primary one", primary: self.class.to_s, secondary: @secondary.class.to_s
           end
@@ -399,10 +400,8 @@ module Fluent
           end
           @output_flush_thread_current_position = 0
 
-          unless @in_tests
-            if @flush_mode == :interval || @chunk_key_time
-              @output_enqueue_thread = thread_create(:enqueue_thread, &method(:enqueue_thread_run))
-            end
+          if !@under_plugin_development && (@flush_mode == :interval || @chunk_key_time)
+            @output_enqueue_thread = thread_create(:enqueue_thread, &method(:enqueue_thread_run))
           end
         end
         @secondary.start if @secondary
@@ -981,11 +980,11 @@ module Fluent
           if output.delayed_commit
             log.trace "executing delayed write and commit", chunk: dump_unique_id_hex(chunk.unique_id)
             @counters_monitor.synchronize{ @write_count += 1 }
-            output.try_write(chunk)
             @dequeued_chunks_mutex.synchronize do
               # delayed_commit_timeout for secondary is configured in <buffer> of primary (<secondary> don't get <buffer>)
               @dequeued_chunks << DequeuedChunkInfo.new(chunk.unique_id, Time.now, self.delayed_commit_timeout)
             end
+            output.try_write(chunk)
           else # output plugin without delayed purge
             chunk_id = chunk.unique_id
             dump_chunk_id = dump_unique_id_hex(chunk_id)
@@ -994,16 +993,17 @@ module Fluent
             log.trace "executing sync write", chunk: dump_chunk_id
             output.write(chunk)
             log.trace "write operation done, committing", chunk: dump_chunk_id
-            commit_write(chunk_id, secondary: using_secondary)
+            commit_write(chunk_id, delayed: false, secondary: using_secondary)
             log.trace "done to commit a chunk", chunk: dump_chunk_id
           end
         rescue => e
           log.debug "taking back chunk for errors.", plugin_id: plugin_id, chunk: dump_unique_id_hex(chunk.unique_id)
-          @buffer.takeback_chunk(chunk.unique_id)
-
-          if @under_plugin_development
-            raise
+          if output.delayed_commit
+            @dequeued_chunks_mutex.synchronize do
+              @dequeued_chunks.delete_if{|d| d.chunk_id == chunk.unique_id }
+            end
           end
+          @buffer.takeback_chunk(chunk.unique_id)
 
           @retry_mutex.synchronize do
             if @retry
@@ -1032,6 +1032,8 @@ module Fluent
               log.warn_backtrace e.backtrace
             end
           end
+
+          raise if @under_plugin_development && !@retry_for_error_chunk
         end
       end
 
@@ -1059,7 +1061,11 @@ module Fluent
         @output_flush_thread_current_position = (@output_flush_thread_current_position + 1) % @buffer_config.flush_thread_count
         state = @output_flush_threads[@output_flush_thread_current_position]
         state.next_time = 0
-        state.thread.run
+        if state.thread && state.thread.status # "run"/"sleep"/"aborting" or false(successfully stop) or nil(killed by exception)
+          state.thread.run
+        else
+          log.warn "thread is already dead"
+        end
       end
 
       def force_flush
@@ -1204,7 +1210,7 @@ module Fluent
         rescue => e
           # normal errors are rescued by output plugins in #try_flush
           # so this rescue section is for critical & unrecoverable errors
-          log.error "error on output thread", plugin_id: plugin_id, error_class: e.class, error: e
+          log.error "error on output thread", plugin_id: plugin_id, error: e
           log.error_backtrace
           raise
         end

data/lib/fluent/plugin/parser.rb

@@ -31,17 +31,43 @@ module Fluent
 
       configured_in :parse
 
-      # SET false BEFORE CONFIGURE, to return nil when time not parsed
-      attr_accessor :estimate_current_event
+      ### types can be specified as string-based hash style
+      # field1:type, field2:type, field3:type:option, field4:type:option
+      ### or, JSON format
+      # {"field1":"type", "field2":"type", "field3":"type:option", "field4":"type:option"}
+      config_param :types, :hash, value_type: :string, default: nil
 
+      # available options are:
+      # array: (1st) delimiter
+      # time : type[, format, timezone] -> type should be a valid "time_type"(string/unixtime/float)
+      #      : format[, timezone]
+
+      config_param :time_key, :string, default: nil
+      config_param :null_value_pattern, :string, default: nil
+      config_param :null_empty_string, :bool, default: false
+      config_param :estimate_current_event, :bool, default: true
       config_param :keep_time_key, :bool, default: false
 
-      def initialize
+      AVAILABLE_PARSER_VALUE_TYPES = ['string', 'integer', 'float', 'bool', 'time', 'array']
+
+      # for tests
+      attr_reader :type_converters
+
+      PARSER_TYPES = [:text_per_line, :text, :binary]
+      def parser_type
+        :text_per_line
+      end
+
+      def configure(conf)
         super
-        @estimate_current_event = true
+
+        @time_parser = time_parser_create
+        @null_value_regexp = @null_value_pattern && Regexp.new(@null_value_pattern)
+        @type_converters = build_type_converters(@types)
+        @execute_convert_values = @type_converters || @null_value_regexp || @null_empty_string
       end
 
-      def parse(text)
+      def parse(text, &block)
         raise NotImplementedError, "Implement this method in child class"
       end
 
@@ -51,80 +77,114 @@ module Fluent
         parse(*a, &b)
       end
 
-      TimeParser = Fluent::TimeParser
-    end
-
-    class ValuesParser < Parser
-      include Fluent::TypeConverter
+      def implement?(feature)
+        methods_of_plugin = self.class.instance_methods(false)
+        case feature
+        when :parse_io then methods_of_plugin.include?(:parse_io)
+        when :parse_partial_data then methods_of_plugin.include?(:parse_partial_data)
+        else
+          raise ArgumentError, "Unknown feature for parser plugin: #{feature}"
+        end
+      end
 
-      config_param :keys, :array, default: []
-      config_param :time_key, :string, default: nil
-      config_param :null_value_pattern, :string, default: nil
-      config_param :null_empty_string, :bool, default: false
+      def parse_io(io, &block)
+        raise NotImplementedError, "Optional API #parse_io is not implemented"
+      end
 
-      def configure(conf)
-        super
+      def parse_partial_data(data, &block)
+        raise NotImplementedError, "Optional API #parse_partial_data is not implemented"
+      end
 
-        if @time_key && !@keys.include?(@time_key) && @estimate_current_event
-          raise Fluent::ConfigError, "time_key (#{@time_key.inspect}) is not included in keys (#{@keys.inspect})"
+      def parse_time(record)
+        if @time_key && record.respond_to?(:has_key?) && record.has_key?(@time_key)
+          src = if @keep_time_key
+                  record[@time_key]
+                else
+                  record.delete(@time_key)
+                end
+          @time_parser.parse(src)
+        elsif @estimate_current_event
+          Fluent::EventTime.now
+        else
+          nil
         end
+      rescue Fluent::TimeParser::TimeParseError => e
+        raise ParserError, e.message
+      end
 
-        if @time_format && !@time_key
-          raise Fluent::ConfigError, "time_format parameter is ignored because time_key parameter is not set. at #{conf.inspect}"
-        end
+      # def parse(text, &block)
+      #   time, record = convert_values(time, record)
+      #   yield time, record
+      # end
+      def convert_values(time, record)
+        return time, record unless @execute_convert_values
 
-        @time_parser = time_parser_create
+        record.each_key do |key|
+          value = record[key]
+          next unless value # nil/null value is always left as-is.
 
-        if @null_value_pattern
-          @null_value_pattern = Regexp.new(@null_value_pattern)
+          if value.is_a?(String) && string_like_null(value)
+            record[key] = nil
+            next
+          end
+
+          if @type_converters && @type_converters.has_key?(key)
+            record[key] = @type_converters[key].call(value)
+          end
         end
 
-        @mutex = Mutex.new
+        return time, record
       end
 
-      def values_map(values)
-        record = Hash[keys.zip(values.map { |value| convert_value_to_nil(value) })]
-
-        if @time_key
-          value = @keep_time_key ? record[@time_key] : record.delete(@time_key)
-          time = if value.nil?
-                   if @estimate_current_event
-                     Fluent::EventTime.now
-                   else
-                     nil
-                   end
-                 else
-                   @mutex.synchronize { @time_parser.parse(value) }
-                 end
-        elsif @estimate_current_event
-          time = Fluent::EventTime.now
-        else
-          time = nil
-        end
+      def string_like_null(value, null_empty_string = @null_empty_string, null_value_regexp = @null_value_regexp)
+        null_empty_string && value.empty? || null_value_regexp && string_safe_encoding(value){|s| null_value_regexp.match(s) }
+      end
 
-        convert_field_type!(record) if @type_converters
+      TRUTHY_VALUES = ['true', 'yes', '1']
 
-        return time, record
-      end
+      def build_type_converters(types)
+        return nil unless types
 
-      private
+        converters = {}
 
-      def convert_field_type!(record)
-        @type_converters.each_key { |key|
-          if value = record[key]
-            record[key] = convert_type(key, value)
+        types.each_pair do |field_name, type_definition|
+          type, option = type_definition.split(":", 2)
+          unless AVAILABLE_PARSER_VALUE_TYPES.include?(type)
+            raise Fluent::ConfigError, "unknown value conversion for key:'#{field_name}', type:'#{type}'"
           end
-        }
-      end
 
-      def convert_value_to_nil(value)
-        if value and @null_empty_string
-          value = (value == '') ? nil : value
-        end
-        if value and @null_value_pattern
-          value = ::Fluent::StringUtil.match_regexp(@null_value_pattern, value) ? nil : value
+          conv = case type
+                 when 'string' then ->(v){ v.to_s }
+                 when 'integer' then ->(v){ v.to_i rescue v.to_s.to_i }
+                 when 'float' then ->(v){ v.to_f rescue v.to_s.to_f }
+                 when 'bool' then ->(v){ TRUTHY_VALUES.include?(v.to_s.downcase) }
+                 when 'time'
+                   # comma-separated: time:[timezone:]time_format
+                   # time_format is unixtime/float/string-time-format
+                   timep = if option
+                             time_type = 'string' # estimate
+                             timezone, time_format = option.split(':', 2)
+                             unless Fluent::Timezone.validate(timezone)
+                               timezone, time_format = nil, option
+                             end
+                             if Fluent::TimeMixin::TIME_TYPES.include?(time_format)
+                               time_type, time_format = time_format, nil # unixtime/float
+                             end
+                             time_parser_create(type: time_type.to_sym, format: time_format, timezone: timezone)
+                           else
+                             time_parser_create(type: :string, format: nil, timezone: nil)
+                           end
+                   ->(v){ timep.parse(v) rescue nil }
+                 when 'array'
+                   delimiter = option ? option.to_s : ','
+                   ->(v){ string_safe_encoding(v.to_s){|s| s.split(delimiter) } }
+                 else
+                   raise "BUG: unknown type even after check: #{type}"
+                 end
+          converters[field_name] = conv
         end
-        value
+
+        converters
       end
     end
   end

data/lib/fluent/plugin/parser_csv.rb

@@ -20,13 +20,19 @@ require 'csv'
 
 module Fluent
   module Plugin
-    class CSVParser < ValuesParser
+    class CSVParser < Parser
       Plugin.register_parser('csv', self)
 
+      desc 'Names of fields included in each lines'
+      config_param :keys, :array, value_type: :string
+      desc 'The delimiter character (or string) of CSV values'
       config_param :delimiter, :string, default: ','
 
-      def parse(text)
-        yield values_map(CSV.parse_line(text, col_sep: @delimiter))
+      def parse(text, &block)
+        values = CSV.parse_line(text, col_sep: @delimiter)
+        r = Hash[@keys.zip(values)]
+        time, record = convert_values(parse_time(r), r)
+        yield time, record
       end
     end
   end

data/lib/fluent/plugin/parser_json.rb

@@ -19,61 +19,63 @@ require 'fluent/env'
 require 'fluent/time'
 
 require 'yajl'
+require 'json'
 
 module Fluent
   module Plugin
     class JSONParser < Parser
       Plugin.register_parser('json', self)
 
-      config_param :time_key, :string, default: 'time'
-      config_param :json_parser, :string, default: 'oj'
+      config_set_default :time_key, 'time'
+      config_param :json_parser, :enum, list: [:oj, :yajl, :json], default: :oj
 
-      def configure(conf)
-        super
+      config_set_default :time_type, :float
 
-        if @time_format
-          @time_parser = time_parser_create
-          @mutex = Mutex.new
+      def configure(conf)
+        if conf.has_key?('time_format')
+          conf['time_type'] ||= 'string'
         end
 
-        begin
-          raise LoadError unless @json_parser == 'oj'
+        super
+        @load_proc, @error_class = configure_json_parser(@json_parser)
+      end
+
+      def configure_json_parser(name)
+        case name
+        when :oj
           require 'oj'
           Oj.default_options = Fluent::DEFAULT_OJ_OPTIONS
-          @load_proc = Oj.method(:load)
-          @error_class = Oj::ParseError
-        rescue LoadError
-          @load_proc = Yajl.method(:load)
-          @error_class = Yajl::ParseError
+          [Oj.method(:load), Oj::ParseError]
+        when :json then [JSON.method(:load), JSON::ParserError]
+        when :yajl then [Yajl.method(:load), Yajl::ParseError]
+        else
+          raise "BUG: unknown json parser specified: #{name}"
         end
+      rescue LoadError
+        name = :yajl
+        log.info "Oj is not installed, and failing back to Yajl for json parser"
+        retry
       end
 
       def parse(text)
-        record = @load_proc.call(text)
-
-        value = @keep_time_key ? record[@time_key] : record.delete(@time_key)
-        if value
-          if @time_format
-            time = @mutex.synchronize { @time_parser.parse(value) }
-          else
-            begin
-              time = Fluent::EventTime.from_time(Time.at(value.to_f))
-            rescue => e
-              raise ParserError, "invalid time value: value = #{value}, error_class = #{e.class.name}, error = #{e.message}"
-            end
-          end
-        else
-          if @estimate_current_event
-            time = Fluent::EventTime.now
-          else
-            time = nil
-          end
-        end
-
+        r = @load_proc.call(text)
+        time, record = convert_values(parse_time(r), r)
         yield time, record
       rescue @error_class
         yield nil, nil
       end
+
+      def parser_type
+        :text
+      end
+
+      def parse_io(io, &block)
+        y = Yajl::Parser.new
+        y.on_parse_complete = ->(record){
+          block.call(parse_time(record), record)
+        }
+        y.parse(io)
+      end
     end
   end
 end