fluentd 1.6.3 → 1.7.0.rc1

data/lib/fluent/plugin/output.rb

@@ -18,6 +18,7 @@ require 'fluent/error'
 require 'fluent/plugin/base'
 require 'fluent/plugin/buffer'
 require 'fluent/plugin_helper/record_accessor'
+require 'fluent/msgpack_factory'
 require 'fluent/log'
 require 'fluent/plugin_id'
 require 'fluent/plugin_helper'
@@ -173,7 +174,7 @@ module Fluent
 
       def initialize
         super
-        @counters_monitor = Monitor.new
+        @counter_mutex = Mutex.new
         @buffering = false
         @delayed_commit = false
         @as_secondary = false
@@ -780,26 +781,26 @@ module Fluent
       end
 
       def emit_sync(tag, es)
-        @counters_monitor.synchronize{ @emit_count += 1 }
+        @counter_mutex.synchronize{ @emit_count += 1 }
         begin
           process(tag, es)
-          @counters_monitor.synchronize{ @emit_records += es.size }
+          @counter_mutex.synchronize{ @emit_records += es.size }
         rescue
-          @counters_monitor.synchronize{ @num_errors += 1 }
+          @counter_mutex.synchronize{ @num_errors += 1 }
           raise
         end
       end
 
       def emit_buffered(tag, es)
-        @counters_monitor.synchronize{ @emit_count += 1 }
+        @counter_mutex.synchronize{ @emit_count += 1 }
         begin
           execute_chunking(tag, es, enqueue: (@flush_mode == :immediate))
-          if !@retry && @buffer.queued?
+          if !@retry && @buffer.queued?(nil, optimistic: true)
             submit_flush_once
           end
         rescue
           # TODO: separate number of errors into emit errors and write/flush errors
-          @counters_monitor.synchronize{ @num_errors += 1 }
+          @counter_mutex.synchronize{ @num_errors += 1 }
           raise
         end
       end
@@ -857,15 +858,8 @@ module Fluent
       def chunk_for_test(tag, time, record)
         require 'fluent/plugin/buffer/memory_chunk'
 
-        m = metadata_for_test(tag, time, record)
-        Fluent::Plugin::Buffer::MemoryChunk.new(m)
-      end
-
-      def metadata_for_test(tag, time, record)
-        raise "BUG: #metadata_for_test is available only when no actual metadata exists" unless @buffer.metadata_list.empty?
         m = metadata(tag, time, record)
-        @buffer.metadata_list_clear!
-        m
+        Fluent::Plugin::Buffer::MemoryChunk.new(m)
       end
 
       def execute_chunking(tag, es, enqueue: false)
@@ -919,10 +913,10 @@ module Fluent
         end
       end
 
-      FORMAT_MSGPACK_STREAM = ->(e){ e.to_msgpack_stream }
-      FORMAT_COMPRESSED_MSGPACK_STREAM = ->(e){ e.to_compressed_msgpack_stream }
-      FORMAT_MSGPACK_STREAM_TIME_INT = ->(e){ e.to_msgpack_stream(time_int: true) }
-      FORMAT_COMPRESSED_MSGPACK_STREAM_TIME_INT = ->(e){ e.to_compressed_msgpack_stream(time_int: true) }
+      FORMAT_MSGPACK_STREAM = ->(e){ e.to_msgpack_stream(packer: Fluent::MessagePackFactory.thread_local_msgpack_packer) }
+      FORMAT_COMPRESSED_MSGPACK_STREAM = ->(e){ e.to_compressed_msgpack_stream(packer: Fluent::MessagePackFactory.thread_local_msgpack_packer) }
+      FORMAT_MSGPACK_STREAM_TIME_INT = ->(e){ e.to_msgpack_stream(time_int: true, packer: Fluent::MessagePackFactory.thread_local_msgpack_packer) }
+      FORMAT_COMPRESSED_MSGPACK_STREAM_TIME_INT = ->(e){ e.to_compressed_msgpack_stream(time_int: true, packer: Fluent::MessagePackFactory.thread_local_msgpack_packer) }
 
       def generate_format_proc
         if @buffer && @buffer.compress == :gzip
@@ -944,7 +938,7 @@ module Fluent
       def handle_stream_with_custom_format(tag, es, enqueue: false)
         meta_and_data = {}
         records = 0
-        es.each do |time, record|
+        es.each(unpacker: Fluent::MessagePackFactory.thread_local_msgpack_unpacker) do |time, record|
           meta = metadata(tag, time, record)
           meta_and_data[meta] ||= []
           res = format(tag, time, record)
@@ -956,7 +950,7 @@ module Fluent
         write_guard do
           @buffer.write(meta_and_data, enqueue: enqueue)
         end
-        @counters_monitor.synchronize{ @emit_records += records }
+        @counter_mutex.synchronize{ @emit_records += records }
         true
       end
 
@@ -964,7 +958,7 @@ module Fluent
         format_proc = generate_format_proc
         meta_and_data = {}
         records = 0
-        es.each do |time, record|
+        es.each(unpacker: Fluent::MessagePackFactory.thread_local_msgpack_unpacker) do |time, record|
           meta = metadata(tag, time, record)
           meta_and_data[meta] ||= MultiEventStream.new
           meta_and_data[meta].add(time, record)
@@ -973,7 +967,7 @@ module Fluent
         write_guard do
           @buffer.write(meta_and_data, format: format_proc, enqueue: enqueue)
         end
-        @counters_monitor.synchronize{ @emit_records += records }
+        @counter_mutex.synchronize{ @emit_records += records }
         true
       end
 
@@ -984,7 +978,7 @@ module Fluent
         if @custom_format
           records = 0
           data = []
-          es.each do |time, record|
+          es.each(unpacker: Fluent::MessagePackFactory.thread_local_msgpack_unpacker) do |time, record|
             res = format(tag, time, record)
             if res
               data << res
@@ -998,7 +992,7 @@ module Fluent
         write_guard do
           @buffer.write({meta => data}, format: format_proc, enqueue: enqueue)
         end
-        @counters_monitor.synchronize{ @emit_records += records }
+        @counter_mutex.synchronize{ @emit_records += records }
         true
       end
 
@@ -1036,7 +1030,7 @@ module Fluent
         #         false if chunk was already flushed and couldn't be rollbacked unexpectedly
         # in many cases, false can be just ignored
         if @buffer.takeback_chunk(chunk_id)
-          @counters_monitor.synchronize{ @rollback_count += 1 }
+          @counter_mutex.synchronize{ @rollback_count += 1 }
           if update_retry
             primary = @as_secondary ? @primary_instance : self
             primary.update_retry_state(chunk_id, @as_secondary)
@@ -1052,7 +1046,7 @@ module Fluent
           while @dequeued_chunks.first && @dequeued_chunks.first.expired?
             info = @dequeued_chunks.shift
             if @buffer.takeback_chunk(info.chunk_id)
-              @counters_monitor.synchronize{ @rollback_count += 1 }
+              @counter_mutex.synchronize{ @rollback_count += 1 }
               log.warn "failed to flush the buffer chunk, timeout to commit.", chunk_id: dump_unique_id_hex(info.chunk_id), flushed_at: info.time
               primary = @as_secondary ? @primary_instance : self
               primary.update_retry_state(info.chunk_id, @as_secondary)
@@ -1067,7 +1061,7 @@ module Fluent
           until @dequeued_chunks.empty?
             info = @dequeued_chunks.shift
             if @buffer.takeback_chunk(info.chunk_id)
-              @counters_monitor.synchronize{ @rollback_count += 1 }
+              @counter_mutex.synchronize{ @rollback_count += 1 }
               log.info "delayed commit for buffer chunks was cancelled in shutdown", chunk_id: dump_unique_id_hex(info.chunk_id)
               primary = @as_secondary ? @primary_instance : self
               primary.update_retry_state(info.chunk_id, @as_secondary)
@@ -1110,7 +1104,7 @@ module Fluent
 
           if output.delayed_commit
             log.trace "executing delayed write and commit", chunk: dump_unique_id_hex(chunk.unique_id)
-            @counters_monitor.synchronize{ @write_count += 1 }
+            @counter_mutex.synchronize{ @write_count += 1 }
             @dequeued_chunks_mutex.synchronize do
               # delayed_commit_timeout for secondary is configured in <buffer> of primary (<secondary> don't get <buffer>)
               @dequeued_chunks << DequeuedChunkInfo.new(chunk.unique_id, Time.now, self.delayed_commit_timeout)
@@ -1122,7 +1116,7 @@ module Fluent
             chunk_id = chunk.unique_id
             dump_chunk_id = dump_unique_id_hex(chunk_id)
             log.trace "adding write count", instance: self.object_id
-            @counters_monitor.synchronize{ @write_count += 1 }
+            @counter_mutex.synchronize{ @write_count += 1 }
             log.trace "executing sync write", chunk: dump_chunk_id
 
             output.write(chunk)
@@ -1178,7 +1172,7 @@ module Fluent
           end
 
           if @buffer.takeback_chunk(chunk.unique_id)
-            @counters_monitor.synchronize { @rollback_count += 1 }
+            @counter_mutex.synchronize { @rollback_count += 1 }
           end
 
           update_retry_state(chunk.unique_id, using_secondary, e)
@@ -1209,9 +1203,9 @@ module Fluent
       def check_slow_flush(start)
         elapsed_time = Fluent::Clock.now - start
         elapsed_millsec = (elapsed_time * 1000).to_i
-        @counters_monitor.synchronize { @flush_time_count += elapsed_millsec }
+        @counter_mutex.synchronize { @flush_time_count += elapsed_millsec }
         if elapsed_time > @slow_flush_log_threshold
-          @counters_monitor.synchronize { @slow_flush_count += 1 }
+          @counter_mutex.synchronize { @slow_flush_count += 1 }
           log.warn "buffer flush took longer time than slow_flush_log_threshold:",
                    elapsed_time: elapsed_time, slow_flush_log_threshold: @slow_flush_log_threshold, plugin_id: self.plugin_id
         end
@@ -1219,7 +1213,7 @@ module Fluent
 
       def update_retry_state(chunk_id, using_secondary, error = nil)
         @retry_mutex.synchronize do
-          @counters_monitor.synchronize{ @num_errors += 1 }
+          @counter_mutex.synchronize{ @num_errors += 1 }
           chunk_id_hex = dump_unique_id_hex(chunk_id)
 
           unless @retry
@@ -1376,7 +1370,7 @@ module Fluent
                 # This block should be done by integer values.
                 # If both of flush_interval & flush_thread_interval are 1s, expected actual flush timing is 1.5s.
                 # If we use integered values for this comparison, expected actual flush timing is 1.0s.
-                @buffer.enqueue_all{ |metadata, chunk| chunk.created_at.to_i + flush_interval <= now_int }
+                @buffer.enqueue_all{ |metadata, chunk| chunk.raw_create_at + flush_interval <= now_int }
               end
 
               if @chunk_key_time

data/lib/fluent/plugin/parser.rb

@@ -17,13 +17,58 @@
 require 'fluent/plugin/base'
 require 'fluent/plugin/owned_by_mixin'
 
+require 'fluent/error'
 require 'fluent/mixin' # for TypeConverter
 require 'fluent/time'
 require 'fluent/plugin/string_util'
 
+require 'serverengine/blocking_flag'
+
 module Fluent
   module Plugin
     class Parser < Base
+      class TimeoutChecker
+        # This implementation now uses mutex because parser is typically used in input.
+        # If this has a performance issue under high concurreny, use concurrent-ruby's map instead.
+        def initialize(timeout)
+          @map = {}
+          @flag = ServerEngine::BlockingFlag.new
+          @mutex = Mutex.new
+          @timeout = timeout
+        end
+
+        def start
+          @thread = ::Thread.new {
+            until @flag.wait_for_set(0.5)
+              now = Time.now
+              @mutex.synchronize {
+                @map.keys.each { |th|
+                  time = @map[th]
+                  if now - time > @timeout
+                    th.raise UncatchableError, "parsing timed out"
+                    @map.delete(th)
+                  end
+                }
+              }
+            end
+          }
+        end
+
+        def stop
+          @flag.set!
+          @thread.join
+        end
+
+        def execute
+          th = Thread.current
+          @mutex.synchronize { @map[th] = Time.now }
+          yield
+        ensure
+          # Need clean up here because if next event is delayed, incorrect exception will be raised in normal flow.
+          @mutex.synchronize { @map.delete(th) }
+        end
+      end
+
       include OwnedByMixin
       include TimeMixin::Parser
 
@@ -47,6 +92,7 @@ module Fluent
       config_param :null_empty_string, :bool, default: false
       config_param :estimate_current_event, :bool, default: true
       config_param :keep_time_key, :bool, default: false
+      config_param :timeout, :time, default: nil
 
       AVAILABLE_PARSER_VALUE_TYPES = ['string', 'integer', 'float', 'bool', 'time', 'array']
 
@@ -65,12 +111,43 @@ module Fluent
         @null_value_regexp = @null_value_pattern && Regexp.new(@null_value_pattern)
         @type_converters = build_type_converters(@types)
         @execute_convert_values = @type_converters || @null_value_regexp || @null_empty_string
+        @timeout_checker = if @timeout
+                             class << self
+                               alias_method :parse_orig, :parse
+                               alias_method :parse, :parse_with_timeout
+                             end
+                             TimeoutChecker.new(@timeout)
+                           else
+                             nil
+                           end
+      end
+
+      def start
+        super
+
+        @timeout_checker.start if @timeout_checker
+      end
+
+      def stop
+        super
+
+        @timeout_checker.stop if @timeout_checker
       end
 
       def parse(text, &block)
         raise NotImplementedError, "Implement this method in child class"
       end
 
+      def parse_with_timeout(text, &block)
+        @timeout_checker.execute {
+          parse_orig(text, &block)
+        }
+      rescue UncatchableError
+        log.warn "parsing timed out with #{self.class}: text = #{text}"
+        # Return nil instead of raising error. in_tail or other plugin can emit broken line.
+        yield nil, nil
+      end
+
       def call(*a, &b)
         # Keep backward compatibility for existing plugins
         # TODO: warn when deprecated

data/lib/fluent/plugin/parser_csv.rb

@@ -27,6 +27,23 @@ module Fluent
       config_param :keys, :array, value_type: :string
       desc 'The delimiter character (or string) of CSV values'
       config_param :delimiter, :string, default: ','
+      desc 'The parser type used to parse CSV line'
+      config_param :parser_type, :enum, list: [:normal, :fast], default: :normal
+
+      def configure(conf)
+        super
+
+
+        if @parser_type == :fast
+          @quote_char = '"'
+          @escape_pattern = Regexp.compile(@quote_char * 2)
+
+          m = method(:parse_fast)
+          self.singleton_class.module_eval do
+            define_method(:parse, m)
+          end
+        end
+      end
 
       def parse(text, &block)
         values = CSV.parse_line(text, col_sep: @delimiter)
@@ -34,6 +51,64 @@ module Fluent
         time, record = convert_values(parse_time(r), r)
         yield time, record
       end
+
+      def parse_fast(text, &block)
+        r = parse_fast_internal(text)
+        time, record = convert_values(parse_time(r), r)
+        yield time, record
+      end
+
+      # CSV.parse_line is too slow due to initialize lots of object and
+      # CSV module doesn't provide the efficient method for parsing single line.
+      # This method avoids the overhead of CSV.parse_line for typical patterns
+      def parse_fast_internal(text)
+        record = {}
+        text.chomp!
+
+        return record if text.empty?
+
+        # use while because while is now faster than each_with_index
+        columns = text.split(@delimiter, -1)
+        num_columns = columns.size
+        i = 0
+        j = 0
+        while j < num_columns
+          column = columns[j]
+
+          case column.count(@quote_char)
+          when 0
+            if column.empty?
+              column = nil
+            end
+          when 1
+            if column.start_with?(@quote_char)
+              to_merge = [column]
+              j += 1
+              while j < num_columns
+                merged_col = columns[j]
+                to_merge << merged_col
+                break if merged_col.end_with?(@quote_char)
+                j += 1
+              end
+              column = to_merge.join(@delimiter)[1..-2]
+            end
+          when 2
+            if column.start_with?(@quote_char) && column.end_with?(@quote_char)
+              column = column[1..-2]
+            end
+          else
+            if column.start_with?(@quote_char) && column.end_with?(@quote_char)
+              column = column[1..-2]
+            end
+            column.gsub!(@escape_pattern, @quote_char)
+          end
+
+          record[@keys[i]] = column
+          j += 1
+          i += 1
+        end
+        record
+      end
     end
   end
 end

data/lib/fluent/plugin_helper/server.rb

@@ -377,7 +377,7 @@ module Fluent
       end
 
       # Use string "?" for port, not integer or nil. "?" is clear than -1 or nil in the log.
-      PEERADDR_FAILED = ["?", "?", "name resolusion failed", "?"]
+      PEERADDR_FAILED = ["?", "?", "name resolution failed", "?"]
 
       class CallbackSocket
         def initialize(server_type, sock, enabled_events = [], close_socket: true)

data/lib/fluent/plugin_helper/thread.rb

@@ -92,6 +92,7 @@ module Fluent
           end
         end
         thread.abort_on_exception = true
+        thread.name = title.to_s if thread.respond_to?(:name)
         @_threads_mutex.synchronize do
           @_threads[thread.object_id] = thread
         end

data/lib/fluent/root_agent.rb

@@ -201,7 +201,7 @@ module Fluent
     def start
       lifecycle(desc: true) do |i| # instance
         i.start unless i.started?
-        # Input#start sometimes emits lots of evetns with in_tail/`read_from_head true` case
+        # Input#start sometimes emits lots of events with in_tail/`read_from_head true` case
         # and it causes deadlock for small buffer/queue output. To avoid such problem,
         # buffer related output threads should be run before `Input#start`.
         # This is why after_start should be called immediately after start call.

data/lib/fluent/time.rb

@@ -78,7 +78,7 @@ module Fluent
       end
     rescue
       def to_time
-        Time.at(@sec, @nsec / 1000.0)
+        Time.at(Rational(@sec * 1_000_000_000 + @nsec, 1_000_000_000))
       end
     end
 
@@ -111,7 +111,9 @@ module Fluent
     end
 
     def self.now
-      from_time(Time.now)
+      # This method is called many time. so call Process.clock_gettime directly instead of Fluent::Clock.real_now
+      now = Process.clock_gettime(Process::CLOCK_REALTIME, :nanosecond)
+      Fluent::EventTime.new(now / 1_000_000_000, now % 1_000_000_000)
     end
 
     def self.parse(*args)