fluentd 0.14.1 → 0.14.2

data/lib/fluent/compat/socket_util.rb

@@ -0,0 +1,165 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'ipaddr'
+
+require 'cool.io'
+
+require 'fluent/plugin'
+require 'fluent/input'
+
+module Fluent
+  module Compat
+    module SocketUtil
+      def create_udp_socket(host)
+        if IPAddr.new(IPSocket.getaddress(host)).ipv4?
+          UDPSocket.new
+        else
+          UDPSocket.new(Socket::AF_INET6)
+        end
+      end
+      module_function :create_udp_socket
+
+      class UdpHandler < Coolio::IO
+        def initialize(io, log, body_size_limit, callback)
+          super(io)
+          @io = io
+          @log = log
+          @body_size_limit = body_size_limit
+          @callback = callback
+        end
+
+        def on_readable
+          msg, addr = @io.recvfrom_nonblock(@body_size_limit)
+          msg.chomp!
+          @callback.call(msg, addr)
+        rescue => e
+          @log.error "unexpected error", error: e
+        end
+      end
+
+      class TcpHandler < Coolio::Socket
+        PEERADDR_FAILED = ["?", "?", "name resolusion failed", "?"]
+
+        def initialize(io, log, delimiter, callback)
+          super(io)
+          @timeout = 0
+          if io.is_a?(TCPSocket)
+            @addr = (io.peeraddr rescue PEERADDR_FAILED)
+
+            opt = [1, @timeout.to_i].pack('I!I!')  # { int l_onoff; int l_linger; }
+            io.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
+          end
+          @delimiter = delimiter
+          @callback = callback
+          @log = log
+          @log.trace { "accepted fluent socket object_id=#{self.object_id}" }
+          @buffer = "".force_encoding('ASCII-8BIT')
+        end
+
+        def on_connect
+        end
+
+        def on_read(data)
+          @buffer << data
+          pos = 0
+
+          while i = @buffer.index(@delimiter, pos)
+            msg = @buffer[pos...i]
+            @callback.call(msg, @addr)
+            pos = i + @delimiter.length
+          end
+          @buffer.slice!(0, pos) if pos > 0
+        rescue => e
+          @log.error "unexpected error", error: e
+          close
+        end
+
+        def on_close
+          @log.trace { "closed fluent socket object_id=#{self.object_id}" }
+        end
+      end
+
+      class BaseInput < Fluent::Input
+        def initialize
+          super
+          require 'fluent/parser'
+        end
+
+        desc 'Tag of output events.'
+        config_param :tag, :string
+        desc 'The format of the payload.'
+        config_param :format, :string
+        desc 'The port to listen to.'
+        config_param :port, :integer, default: 5150
+        desc 'The bind address to listen to.'
+        config_param :bind, :string, default: '0.0.0.0'
+        desc "The field name of the client's hostname."
+        config_param :source_host_key, :string, default: nil
+        config_param :blocking_timeout, :time, default: 0.5
+
+        def configure(conf)
+          super
+
+          @parser = Plugin.new_parser(@format)
+          @parser.configure(conf)
+        end
+
+        def start
+          super
+
+          @loop = Coolio::Loop.new
+          @handler = listen(method(:on_message))
+          @loop.attach(@handler)
+          @thread = Thread.new(&method(:run))
+        end
+
+        def shutdown
+          @loop.watchers.each { |w| w.detach }
+          @loop.stop if @loop.instance_variable_get("@running")
+          @handler.close
+          @thread.join
+
+          super
+        end
+
+        def run
+          @loop.run(@blocking_timeout)
+        rescue => e
+          log.error "unexpected error", error: e
+          log.error_backtrace
+        end
+
+        private
+
+        def on_message(msg, addr)
+          @parser.parse(msg) { |time, record|
+            unless time && record
+              log.warn "pattern not match: #{msg.inspect}"
+              return
+            end
+
+            record[@source_host_key] = addr[3] if @source_host_key
+            router.emit(@tag, time, record)
+          }
+        rescue => e
+          log.error msg.dump, error: e, host: addr[3]
+          log.error_backtrace
+        end
+      end
+    end
+  end
+end

data/lib/fluent/compat/string_util.rb

@@ -0,0 +1,34 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  module Compat
+    module StringUtil
+      def match_regexp(regexp, string)
+        begin
+          return regexp.match(string)
+        rescue ArgumentError => e
+          raise e unless e.message.index("invalid byte sequence in".freeze).zero?
+          $log.info "invalid byte sequence is replaced in `#{string}`"
+          string = string.scrub('?')
+          retry
+        end
+        return true
+      end
+      module_function :match_regexp
+    end
+  end
+end

data/lib/fluent/{test/driver/owner.rb → compat/structured_format_mixin.rb}

@@ -14,18 +14,12 @@
 #    limitations under the License.
 #
 
-require 'fluent/plugin/base'
-require 'fluent/plugin_id'
-require 'fluent/log'
-require 'fluent/plugin_helper'
-
 module Fluent
-  module Test
-    module Driver
-      class Owner < Fluent::Plugin::Base
-        include PluginId
-        include PluginLoggerMixin
-        include PluginHelper::Mixin
+  module Compat
+    module StructuredFormatMixin
+      def format(tag, time, record)
+        filter_record(tag, time, record)
+        format_record(record)
       end
     end
   end

data/lib/fluent/config/element.rb

@@ -140,7 +140,7 @@ module Fluent
           out << "#{indent}<#{@name} #{@arg}>\n"
         end
         each_pair { |k, v|
-          out << dump_value(k, v, indent, nindent)
+          out << dump_value(k, v, nindent)
         }
         @elements.each { |e|
           out << e.to_s(nest + 1)
@@ -186,7 +186,7 @@ module Fluent
         opts[:type]
       end
 
-      def dump_value(k, v, indent, nindent)
+      def dump_value(k, v, nindent)
         if secret_param?(k)
           "#{nindent}#{k} xxxxxx\n"
         else

data/lib/fluent/configurable.rb

@@ -144,9 +144,10 @@ module Fluent
       end
 
       def config_section(name, **kwargs, &block)
+        section_already_exists = !!merged_configure_proxy.sections[name]
         configure_proxy(self.name).config_section(name, **kwargs, &block)
         variable_name = configure_proxy(self.name).sections[name].variable_name
-        unless self.respond_to?(variable_name)
+        if !section_already_exists && !self.respond_to?(variable_name)
           attr_accessor variable_name
         end
       end

data/lib/fluent/event.rb

@@ -44,6 +44,10 @@ module Fluent
       false
     end
 
+    def slice(index, num)
+      raise NotImplementedError, "DO NOT USE THIS CLASS directly."
+    end
+
     def each(&block)
       raise NotImplementedError, "DO NOT USE THIS CLASS directly."
     end
@@ -124,6 +128,10 @@ module Fluent
       @entries.empty?
     end
 
+    def slice(index, num)
+      ArrayEventStream.new(@entries.slice(index, num))
+    end
+
     def each(&block)
       @entries.each(&block)
       nil
@@ -167,6 +175,10 @@ module Fluent
       @time_array.empty?
     end
 
+    def slice(index, num)
+      MultiEventStream.new(@time_array.slice(index, num), @record_array.slice(index, num))
+    end
+
     def each(&block)
       time_array = @time_array
       record_array = @record_array
@@ -178,23 +190,32 @@ module Fluent
   end
 
   class MessagePackEventStream < EventStream
-    # Keep cached_unpacker argument for existence plugins
-    def initialize(data, cached_unpacker = nil, size = 0)
+    # https://github.com/msgpack/msgpack-ruby/issues/119
+
+    # Keep cached_unpacker argument for existing plugins
+    def initialize(data, cached_unpacker = nil, size = 0, unpacked_times: nil, unpacked_records: nil)
       @data = data
       @size = size
+      @unpacked_times = unpacked_times
+      @unpacked_records = unpacked_records
     end
 
     def empty?
-      # This is not correct, but actual number of records will be shown after iteration, and
-      # "size" argument is always 0 currently (because forward protocol doesn't tell it to destination)
-      false
+      @data.empty?
     end
 
     def dup
-      MessagePackEventStream.new(@data.dup, @size)
+      if @unpacked_times
+        MessagePackEventStream.new(@data.dup, nil, @size, unpacked_times: @unpacked_times, unpacked_records: @unpacked_records.map(&:dup))
+      else
+        MessagePackEventStream.new(@data.dup, nil, @size)
+      end
     end
 
     def size
+      # @size is unbelievable always when @size == 0
+      # If the number of events is really zero, unpacking events takes very short time.
+      ensure_unpacked! if @size == 0
       @size
     end
 
@@ -202,8 +223,41 @@ module Fluent
       true
     end
 
+    def ensure_unpacked!
+      return if @unpacked_times && @unpacked_records
+      @unpacked_times = []
+      @unpacked_records = []
+      msgpack_unpacker.feed_each(@data) do |time, record|
+        @unpacked_times << time
+        @unpacked_records << record
+      end
+      # @size should be updated always right after unpack.
+      # The real size of unpacked objects are correct, rather than given size.
+      @size = @unpacked_times.size
+    end
+
+    # This method returns MultiEventStream, because there are no reason
+    # to surve binary serialized by msgpack.
+    def slice(index, num)
+      ensure_unpacked!
+      MultiEventStream.new(@unpacked_times.slice(index, num), @unpacked_records.slice(index, num))
+    end
+
     def each(&block)
-      msgpack_unpacker.feed_each(@data, &block)
+      if @unpacked_times
+        @unpacked_times.each_with_index do |time, i|
+          block.call(time, @unpacked_records[i])
+        end
+      else
+        @unpacked_times = []
+        @unpacked_records = []
+        msgpack_unpacker.feed_each(@data) do |time, record|
+          @unpacked_times << time
+          @unpacked_records << record
+          block.call(time, record)
+        end
+        @size = @unpacked_times.size
+      end
       nil
     end
 

data/lib/fluent/event_router.rb

@@ -159,7 +159,7 @@ module Fluent
       pipeline = nil
       @match_rules.each_with_index { |rule, i|
         if rule.match?(tag)
-          if rule.collector.is_a?(Filter)
+          if rule.collector.is_a?(Plugin::Filter)
             pipeline ||= Pipeline.new
             pipeline.add_filter(rule.collector)
           else

data/lib/fluent/plugin.rb

@@ -26,15 +26,15 @@ module Fluent
     #   ex: storage, buffer chunk, ...
 
     # first class plugins (instantiated by Engine)
-    INPUT_REGISTRY     = Registry.new(:input,     'fluent/plugin/in_')
-    OUTPUT_REGISTRY    = Registry.new(:output,    'fluent/plugin/out_')
-    FILTER_REGISTRY    = Registry.new(:filter,    'fluent/plugin/filter_')
+    INPUT_REGISTRY     = Registry.new(:input,     'fluent/plugin/in_',         dir_search_prefix: 'in_')
+    OUTPUT_REGISTRY    = Registry.new(:output,    'fluent/plugin/out_',        dir_search_prefix: 'out_')
+    FILTER_REGISTRY    = Registry.new(:filter,    'fluent/plugin/filter_',     dir_search_prefix: 'filter_')
 
     # feature plugin: second class plugins (instanciated by Plugins or Helpers)
-    BUFFER_REGISTRY    = Registry.new(:buffer,    'fluent/plugin/buf_')
-    PARSER_REGISTRY    = Registry.new(:parser,    'fluent/plugin/parser_')
-    FORMATTER_REGISTRY = Registry.new(:formatter, 'fluent/plugin/formatter_')
-    STORAGE_REGISTRY   = Registry.new(:storage,   'fluent/plugin/storage_')
+    BUFFER_REGISTRY    = Registry.new(:buffer,    'fluent/plugin/buf_',        dir_search_prefix: 'buf_')
+    PARSER_REGISTRY    = Registry.new(:parser,    'fluent/plugin/parser_',     dir_search_prefix: 'parser_')
+    FORMATTER_REGISTRY = Registry.new(:formatter, 'fluent/plugin/formatter_',  dir_search_prefix: 'formatter_')
+    STORAGE_REGISTRY   = Registry.new(:storage,   'fluent/plugin/storage_',    dir_search_prefix: 'storage_')
 
     REGISTRIES = [INPUT_REGISTRY, OUTPUT_REGISTRY, FILTER_REGISTRY, BUFFER_REGISTRY, PARSER_REGISTRY, FORMATTER_REGISTRY, STORAGE_REGISTRY]
 

data/lib/fluent/plugin/buf_file.rb

@@ -122,14 +122,17 @@ module Fluent
           m = new_metadata() # this metadata will be overwritten by resuming .meta file content
                              # so it should not added into @metadata_list for now
           mode = Fluent::Plugin::Buffer::FileChunk.assume_chunk_state(path)
+          if mode == :unknown
+            log.debug "uknown state chunk found", path: path
+            next
+          end
+
           chunk = Fluent::Plugin::Buffer::FileChunk.new(m, path, mode) # file chunk resumes contents of metadata
           case chunk.state
           when :staged
             stage[chunk.metadata] = chunk
           when :queued
             queue << chunk
-          else
-            raise "BUG: unexpected chunk state '#{chunk.state}' for path '#{path}'"
           end
         end
 

data/lib/fluent/plugin/buffer.rb

@@ -104,11 +104,13 @@ module Fluent
           @queued_num[chunk.metadata] += 1
           @queue_size += chunk.bytesize
         end
+        log.debug "buffer started", instance: self.object_id, stage_size: @stage_size, queue_size: @queue_size
       end
 
       def close
         super
         synchronize do
+          log.debug "closing buffer", instance: self.object_id
           @dequeued.each_pair do |chunk_id, chunk|
             chunk.close
           end
@@ -156,6 +158,7 @@ module Fluent
       end
 
       def add_metadata(metadata)
+        log.trace "adding metadata", instance: self.object_id, metadata: metadata
         synchronize do
           if i = @metadata_list.index(metadata)
             @metadata_list[i]
@@ -172,26 +175,38 @@ module Fluent
       end
 
       # metadata MUST have consistent object_id for each variation
-      # data MUST be Array of serialized events
+      # data MUST be Array of serialized events, or EventStream
       # metadata_and_data MUST be a hash of { metadata => data }
-      def write(metadata_and_data, bulk: false, enqueue: false)
+      def write(metadata_and_data, format: nil, size: nil, enqueue: false)
         return if metadata_and_data.size < 1
         raise BufferOverflowError, "buffer space has too many data" unless storable?
 
+        log.trace "writing events into buffer", instance: self.object_id, metadata_size: metadata_and_data.size
+
         staged_bytesize = 0
         operated_chunks = []
+        unstaged_chunks = {} # metadata => [chunk, chunk, ...]
+        chunks_to_enqueue = []
 
         begin
           metadata_and_data.each do |metadata, data|
-            write_once(metadata, data, bulk: bulk) do |chunk, adding_bytesize|
+            write_once(metadata, data, format: format, size: size) do |chunk, adding_bytesize|
               chunk.mon_enter # add lock to prevent to be committed/rollbacked from other threads
               operated_chunks << chunk
-              staged_bytesize += adding_bytesize
+              if chunk.staged?
+                staged_bytesize += adding_bytesize
+              elsif chunk.unstaged?
+                unstaged_chunks[metadata] ||= []
+                unstaged_chunks[metadata] << chunk
+              end
             end
           end
 
           return if operated_chunks.empty?
 
+          # Now, this thread acquires many locks of chunks... getting buffer-global lock causes dead lock.
+          # Any operations needs buffer-global lock (including enqueueing) should be done after releasing locks.
+
           first_chunk = operated_chunks.shift
           # Following commits for other chunks also can finish successfully if the first commit operation
           # finishes without any exceptions.
@@ -199,7 +214,9 @@ module Fluent
           # permission errors, disk failures and other permanent(fatal) errors.
           begin
             first_chunk.commit
-            enqueue_chunk(first_chunk.metadata) if enqueue || chunk_size_full?(first_chunk)
+            if enqueue || first_chunk.unstaged? || chunk_size_full?(first_chunk)
+              chunks_to_enqueue << first_chunk
+            end
             first_chunk.mon_exit
           rescue
             operated_chunks.unshift(first_chunk)
@@ -211,7 +228,9 @@ module Fluent
           operated_chunks.each do |chunk|
             begin
               chunk.commit
-              enqueue_chunk(chunk.metadata) if enqueue || chunk_size_full?(chunk)
+              if enqueue || chunk.unstaged? || chunk_size_full?(chunk)
+                chunks_to_enqueue << chunk
+              end
               chunk.mon_exit
             rescue => e
               chunk.rollback
@@ -219,9 +238,34 @@ module Fluent
               errors << e
             end
           end
-          operated_chunks.clear if errors.empty?
 
-          @stage_size += staged_bytesize
+          # All locks about chunks are released.
+
+          synchronize do
+            # At here, staged chunks may be enqueued by other threads.
+            @stage_size += staged_bytesize
+
+            chunks_to_enqueue.each do |c|
+              if c.staged? && (enqueue || chunk_size_full?(c))
+                m = c.metadata
+                enqueue_chunk(m)
+                if unstaged_chunks[m]
+                  u = unstaged_chunks[m].pop
+                  if u.unstaged? && !chunk_size_full?(u)
+                    @stage[m] = u.staged!
+                    @stage_size += u.bytesize
+                  end
+                end
+              elsif c.unstaged?
+                enqueue_unstaged_chunk(c)
+              else
+                # previously staged chunk is already enqueued, closed or purged.
+                # no problem.
+              end
+            end
+          end
+
+          operated_chunks.clear if errors.empty?
 
           if errors.size > 0
             log.warn "error occurs in committing chunks: only first one raised", errors: errors.map(&:class)
@@ -230,6 +274,9 @@ module Fluent
         ensure
           operated_chunks.each do |chunk|
             chunk.rollback rescue nil # nothing possible to do for #rollback failure
+            if chunk.unstaged?
+              chunk.purge rescue nil # to prevent leakage of unstaged chunks
+            end
             chunk.mon_exit rescue nil # this may raise ThreadError for chunks already committed
           end
         end
@@ -251,6 +298,7 @@ module Fluent
       end
 
       def enqueue_chunk(metadata)
+        log.debug "enqueueing chunk", instance: self.object_id, metadata: metadata
         synchronize do
           chunk = @stage.delete(metadata)
           return nil unless chunk
@@ -271,7 +319,21 @@ module Fluent
         nil
       end
 
+      def enqueue_unstaged_chunk(chunk)
+        log.debug "enqueueing unstaged chunk", instance: self.object_id, metadata: chunk.metadata
+        synchronize do
+          chunk.synchronize do
+            metadata = chunk.metadata
+            @queue << chunk
+            @queued_num[metadata] = @queued_num.fetch(metadata, 0) + 1
+            chunk.enqueued! if chunk.respond_to?(:enqueued!)
+          end
+          @queue_size += chunk.bytesize
+        end
+      end
+
       def enqueue_all
+        log.debug "enqueueing all chunks in buffer", instance: self.object_id
         synchronize do
           if block_given?
             @stage.keys.each do |metadata|
@@ -289,6 +351,7 @@ module Fluent
 
       def dequeue_chunk
         return nil if @queue.empty?
+        log.debug "dequeueing a chunk", instance: self.object_id
         synchronize do
           chunk = @queue.shift
 
@@ -297,15 +360,18 @@ module Fluent
 
           @dequeued[chunk.unique_id] = chunk
           @queued_num[chunk.metadata] -= 1 # BUG if nil, 0 or subzero
+          log.debug "chunk dequeued", instance: self.object_id, metadata: chunk.metadata
           chunk
         end
       end
 
       def takeback_chunk(chunk_id)
+        log.debug "taking back a chunk", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id)
         synchronize do
           chunk = @dequeued.delete(chunk_id)
           return false unless chunk # already purged by other thread
           @queue.unshift(chunk)
+          log.debug "chunk taken back", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id), metadata: chunk.metadata
           @queued_num[chunk.metadata] += 1 # BUG if nil
         end
         true
@@ -317,22 +383,26 @@ module Fluent
           return nil unless chunk # purged by other threads
 
           metadata = chunk.metadata
+          log.debug "purging a chunk", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id), metadata: metadata
           begin
             bytesize = chunk.bytesize
             chunk.purge
             @queue_size -= bytesize
           rescue => e
             log.error "failed to purge buffer chunk", chunk_id: dump_unique_id_hex(chunk_id), error_class: e.class, error: e
+            log.error_backtrace
           end
 
           if metadata && !@stage[metadata] && (!@queued_num[metadata] || @queued_num[metadata] < 1)
             @metadata_list.delete(metadata)
           end
+          log.debug "chunk purged", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id), metadata: metadata
         end
         nil
       end
 
       def clear_queue!
+        log.debug "clearing queue", instance: self.object_id
         synchronize do
           until @queue.empty?
             begin
@@ -341,6 +411,7 @@ module Fluent
               q.purge
             rescue => e
               log.error "unexpected error while clearing buffer queue", error_class: e.class, error: e
+              log.error_backtrace
             end
           end
           @queue_size = 0
@@ -357,16 +428,20 @@ module Fluent
 
       class ShouldRetry < StandardError; end
 
-      def write_once(metadata, data, bulk: false, &block)
-        return if !bulk && (data.nil? || data.empty?)
-        return if bulk && (data.empty? || data.first.nil? || data.first.empty?)
+      # write once into a chunk
+      # 1. append whole data into existing chunk
+      # 2. commit it & return unless chunk_size_over?
+      # 3. enqueue existing chunk & retry whole method if chunk was not empty
+      # 4. go to step_by_step writing
+
+      def write_once(metadata, data, format: nil, size: nil, &block)
+        return if data.empty?
 
         stored = false
         adding_bytesize = nil
 
-        chunk = synchronize { @stage[metadata] ||= generate_chunk(metadata) }
-        enqueue_list = []
-
+        chunk = synchronize { @stage[metadata] ||= generate_chunk(metadata).staged! }
+        enqueue_chunk_before_retry = false
         chunk.synchronize do
           # retry this method if chunk is already queued (between getting chunk and entering critical section)
           raise ShouldRetry unless chunk.staged?
@@ -375,20 +450,27 @@ module Fluent
 
           original_bytesize = chunk.bytesize
           begin
-            if bulk
-              content, size = data
-              chunk.concat(content, size)
+            if format
+              serialized = format.call(data)
+              chunk.concat(serialized, size ? size.call : data.size)
             else
               chunk.append(data)
             end
             adding_bytesize = chunk.bytesize - original_bytesize
 
             if chunk_size_over?(chunk)
-              if empty_chunk && bulk
-                log.warn "chunk bytes limit exceeds for a bulk event stream: #{bulk.bytesize}bytes"
-                stored = true
-              else
-                chunk.rollback
+              if format && empty_chunk
+                log.warn "chunk bytes limit exceeds for an emitted event stream: #{adding_bytesize}bytes"
+              end
+              chunk.rollback
+
+              if format && !empty_chunk
+                # Event streams should be appended into a chunk at once
+                # as far as possible, to improve performance of formatting.
+                # Event stream may be a MessagePackEventStream. We don't want to split it into
+                # 2 or more chunks (except for a case that the event stream is larger than chunk limit).
+                enqueue_chunk_before_retry = true
+                raise ShouldRetry
               end
             else
               stored = true
@@ -400,74 +482,122 @@ module Fluent
 
           if stored
             block.call(chunk, adding_bytesize)
-          elsif bulk
-            # this metadata might be enqueued already by other threads
-            # but #enqueue_chunk does nothing in such case
-            enqueue_list << metadata
-            raise ShouldRetry
           end
         end
 
         unless stored
           # try step-by-step appending if data can't be stored into existing a chunk in non-bulk mode
-          write_step_by_step(metadata, data, data.size / 3, &block)
+          #
+          # 1/10 size of original event stream (splits_count == 10) seems enough small
+          # to try emitting events into existing chunk.
+          # it does not matter to split event stream into very small splits, because chunks have less
+          # overhead to write data many times (even about file buffer chunks).
+          write_step_by_step(metadata, data, format, 10, &block)
         end
       rescue ShouldRetry
-        enqueue_list.each do |m|
-          enqueue_chunk(m)
-        end
+        enqueue_chunk(metadata) if enqueue_chunk_before_retry
         retry
       end
 
-      def write_step_by_step(metadata, data, attempt_records, &block)
-        while data.size > 0
-          if attempt_records < MINIMUM_APPEND_ATTEMPT_RECORDS
-            attempt_records = MINIMUM_APPEND_ATTEMPT_RECORDS
-          end
+      # EventStream can be split into many streams
+      # because (es1 + es2).to_msgpack_stream == es1.to_msgpack_stream + es2.to_msgpack_stream
+
+      # 1. split event streams into many (10 -> 100 -> 1000 -> ...) chunks
+      # 2. append splits into the staged chunks as much as possible
+      # 3. create unstaged chunk and append rest splits -> repeat it for all splits
+
+      def write_step_by_step(metadata, data, format, splits_count, &block)
+        splits = []
+        if splits_count > data.size
+          splits_count = data.size
+        end
+        slice_size = if data.size % splits_count == 0
+                       data.size / splits_count
+                     else
+                       data.size / (splits_count - 1)
+                     end
+        slice_origin = 0
+        while slice_origin < data.size
+          splits << data.slice(slice_origin, slice_size)
+          slice_origin += slice_size
+        end
+
+        # This method will append events into the staged chunk at first.
+        # Then, will generate chunks not staged (not queued) to append rest data.
+        staged_chunk_used = false
+        modified_chunks = []
+        get_next_chunk = ->(){
+          c = if staged_chunk_used
+                # Staging new chunk here is bad idea:
+                # Recovering whole state including newly staged chunks is much harder than current implementation.
+                generate_chunk(metadata)
+              else
+                synchronize{ @stage[metadata] ||= generate_chunk(metadata).staged! }
+              end
+          modified_chunks << c
+          c
+        }
+
+        writing_splits_index = 0
+        enqueue_chunk_before_retry = false
+
+        while writing_splits_index < splits.size
+          chunk = get_next_chunk.call
+          chunk.synchronize do
+            raise ShouldRetry unless chunk.writable?
+            staged_chunk_used = true if chunk.staged?
 
-          chunk = synchronize{ @stage[metadata] ||= generate_chunk(metadata) }
-          chunk.synchronize do # critical section for chunk (chunk append/commit/rollback)
-            raise ShouldRetry unless chunk.staged?
+            original_bytesize = chunk.bytesize
             begin
-              empty_chunk = chunk.empty?
-              original_bytesize = chunk.bytesize
+              while writing_splits_index < splits.size
+                split = splits[writing_splits_index]
+                if format
+                  chunk.concat(format.call(split), split.size)
+                else
+                  chunk.append(split)
+                end
 
-              attempt = data.slice(0, attempt_records)
-              chunk.append(attempt)
-              adding_bytesize = (chunk.bytesize - original_bytesize)
+                if chunk_size_over?(chunk) # split size is larger than difference between size_full? and size_over?
+                  chunk.rollback
 
-              if chunk_size_over?(chunk)
-                chunk.rollback
+                  if split.size == 1 && original_bytesize == 0
+                    big_record_size = format ? format.call(split).bytesize : split.first.bytesize
+                    raise BufferChunkOverflowError, "a #{big_record_size}bytes record is larger than buffer chunk limit size"
+                  end
 
-                if attempt_records <= MINIMUM_APPEND_ATTEMPT_RECORDS
-                  if empty_chunk # record is too large even for empty chunk
-                    raise BufferChunkOverflowError, "minimum append butch exceeds chunk bytes limit"
+                  if chunk_size_full?(chunk) || split.size == 1
+                    enqueue_chunk_before_retry = true
+                  else
+                    splits_count *= 10
                   end
-                  # no more records for this chunk -> enqueue -> to be flushed
-                  enqueue_chunk(metadata) # `chunk` will be removed from stage
-                  attempt_records = data.size # fresh chunk may have enough space
-                else
-                  # whole data can be processed by twice operation
-                  #  ( by using apttempt /= 2, 3 operations required for odd numbers of data)
-                  attempt_records = (attempt_records / 2) + 1
+
+                  raise ShouldRetry
                 end
 
-                next
-              end
+                writing_splits_index += 1
 
-              block.call(chunk, adding_bytesize)
-              data.slice!(0, attempt_records)
-              # same attempt size
-              nil # discard return value of data.slice!() immediately
+                if chunk_size_full?(chunk)
+                  break
+                end
+              end
             rescue
-              chunk.rollback
+              chunk.purge if chunk.unstaged? # unstaged chunk will leak unless purge it
               raise
             end
+
+            block.call(chunk, chunk.bytesize - original_bytesize)
           end
         end
       rescue ShouldRetry
+        modified_chunks.each do |mc|
+          mc.rollback rescue nil
+          if mc.unstaged?
+            mc.purge rescue nil
+          end
+        end
+        enqueue_chunk(metadata) if enqueue_chunk_before_retry
         retry
-      end # write_step_by_step
+      end
     end
   end
 end