fluent-plugin-buffer-event_limited 0.1.4 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d9837aeb9ac272dabafad51da6d4e94209894cb2
-  data.tar.gz: ccc9d6a9d68ae52e67679c8d0a23ad2907bb47c4
+  metadata.gz: 9ab9aa64fad78b7f58fb453867cee20b4c87b7fa
+  data.tar.gz: 5443046e647187e4bf5ee215bcb925ddc6a50092
 SHA512:
-  metadata.gz: 22ddeaedff64fd9c1bea85e4f54e2c3ad44f24f1b4d844e9d2413103abb506322420d326b8efb20c260b8935f567d70d5cd53be0796cc75b82ea2c4a8a315317
-  data.tar.gz: 2e7ff811aeb6f0397bcd2d62d8dd373fa034939ee7c8513c2a3e181165cce725fc6a3ab3d98f16727b46022c3ae835717452196b4a7eea68db9b7cef6c6340df
+  metadata.gz: 88483e3ca720d2518657c686b5fa5a0a3c3cacb8689e302725d628d08d557ea0efcec28b85c6ad1c70430c154cc702fc9766c05bfcd662249877c9d9c1c5c218
+  data.tar.gz: 653ab305d9e8c33fd4898494c5f8bb1c96ee14b66f2df9d463101cf8ac4183a7caa7f188e213f57af462266e9ecbe7e3b0b02e0578a24fd1c5b14adb05de5d50

data/README.md

@@ -2,10 +2,9 @@
 
 This gem is a mutation of the [fluent-plugin-buffer-lightening](https://github.com/tagomoris/fluent-plugin-buffer-lightening) buffer plugin by [tagomoris](https://github.com/tagomoris).
 
-[Fluentd](http://fluentd.org) buffer plugin on memory to flush with many types of chunk limit methods:
-  * events count limit in chunk
-
-These options are to decrease latency from emit to write, and to control chunk sizes and flush sizes.
+* The buffer is able to limit the number of events that are buffered in a buffer chunk.
+* The buffer only supports output plugins that return msgpack from their `#format` methods.
+* The buffer doesn't check the bytesize of the buffers just the number of messages
 
 ## Installation
 
@@ -33,9 +32,6 @@ Options of `buffer_type file` are also available:
   buffer_type event_limited
   buffer_chunk_limit 10M
   buffer_chunk_records_limit 100
-  buffer_chunk_message_separator newline
-  # buffer_chunk_message_separator tab
-  # buffer_chunk_message_separator msgpack
   # other options...
 </match>
 ```

data/fluent-plugin-buffer-event_limited.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-buffer-event_limited"
-  spec.version       = "0.1.4"
+  spec.version       = "0.1.5"
   spec.authors       = ["TAGOMORI Satoshi", 'Gergo Sulymosi']
   spec.email         = ["tagomoris@gmail.com", 'gergo.sulymosi@gmail.com']
   spec.description   = %q{Fluentd memory buffer plugin with many types of chunk limits}
@@ -18,5 +18,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", "~> 1.3"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "test-unit"
+  spec.add_development_dependency "pry"
   spec.add_runtime_dependency "fluentd", ">= 0.10.42"
 end

data/lib/fluent/plugin/buf_event_limited.rb

@@ -1,39 +1,57 @@
 require 'fluent/plugin/buf_file'
+require 'stringio'
 
 module Fluent
-  class EventLimitedBufferChunk < FileBufferChunk
-    attr_reader :record_counter
+  class MessagePackFormattedBufferData
+    attr_reader :data
+
+    def initialize(data)
+      @data = data.to_str.freeze
+    end
 
-    def initialize(key, path, unique_id, separator, mode = "a+", symlink_path = nil)
-      super(key, path, unique_id, mode = "a+", symlink_path = nil)
-      init_counter(separator)
+    def records
+      @records ||= (data.empty? ? [] : unpack(data)).freeze
     end
 
-    def <<(data)
-      result = super
-      @record_counter += 1
+    def as_events
+      records.dup
+    end
 
-      return result
+    def size
+      @size ||= records.size
     end
 
+    alias_method :to_str, :data
+    alias_method :as_msg_pack, :data
+
     private
 
-    def init_counter(separator)
-      old_pos = @file.pos
-      @file.rewind
-
-      @record_counter = (
-        case separator
-        when 'msgpack' then MessagePack::Unpacker.new(@file).each
-        when 'newline' then @file.each($/)
-        when 'tab'     then @file.each("\t")
-        else
-          raise ArgumentError, "Separator #{separator.inspect} is not supported"
-        end
-      ).inject(0) { |c, _| c + 1 }
+    def unpack(data)
+      MessagePack::Unpacker.new(StringIO.new(data)).each.to_a
+    end
+  end
+
+  class EventLimitedBufferChunk < FileBufferChunk
+    attr_reader :record_count, :limit
+
+    def initialize(key, path, unique_id, limit, mode = "a+")
+      super(key, path, unique_id, mode = "a+")
+      @limit = limit
+      @record_count = MessagePackFormattedBufferData.new(read).size
+    end
+
+    def <<(data, record_count)
+      super(data)
+      @record_count += record_count
+    end
+    alias_method :write, :<<
+
+    def remaining_capacity
+      @limit - record_count
+    end
 
-      @file.pos = old_pos
-      $log.trace("#init_counter(#{[path, separator].join(', ')}) => #{@record_counter}")
+    def full?
+      record_count >= limit
     end
   end
 
@@ -41,11 +59,35 @@ module Fluent
     Fluent::Plugin.register_buffer('event_limited', self)
 
     config_param :buffer_chunk_records_limit, :integer, :default => Float::INFINITY
-    config_param :buffer_chunk_message_separator, :string, :default => 'msgpack'
 
-    def storable?(chunk, data)
-      (chunk.record_counter < @buffer_chunk_records_limit) &&
-        ((chunk.size + data.bytesize) <= @buffer_chunk_limit)
+    def emit(key, data, chain)
+      data = MessagePackFormattedBufferData.new(data)
+      key = key.to_s
+      flush_trigger = false
+
+      synchronize do
+        # Get the active chunk if it exists
+        chunk = (@map[key] ||= new_chunk(key))
+
+        # Partition the data into chunks that can be written into new chunks
+        events = data.as_events
+        [
+          events.shift(chunk.remaining_capacity),
+          *events.each_slice(@buffer_chunk_records_limit)
+        ].each do |event_group|
+          chunk, queue_size = rotate_chunk!(chunk, key)
+          # Trigger flush only when we put the first chunk into it
+          flush_trigger ||= (queue_size == 0)
+
+          chain.next
+          chunk.write(
+            event_group.map { |d| MessagePack.pack(d) }.join(''),
+            event_group.size
+          )
+        end
+
+        return flush_trigger
+      end
     end
 
     def new_chunk(key)
@@ -53,7 +95,7 @@ module Fluent
       path, tsuffix = make_path(encoded_key, 'b')
       unique_id = tsuffix_to_unique_id(tsuffix)
 
-      EventLimitedBufferChunk.new(key, path, unique_id, @buffer_chunk_message_separator, 'a+', @symlink_path)
+      chunk_factory(key, path, unique_id, 'a+')
     end
 
     # Copied here from
@@ -74,11 +116,9 @@ module Fluent
 
         case bq
         when 'b'
-          chunk = EventLimitedBufferChunk.new(key, path, unique_id, @buffer_chunk_message_separator, "a+")
-          maps << [timestamp, chunk]
+          maps << [timestamp, chunk_factory(key, path, unique_id, 'a+')]
         when 'q'
-          chunk = EventLimitedBufferChunk.new(key, path, unique_id, @buffer_chunk_message_separator, "r")
-          queues << [timestamp, chunk]
+          queues << [timestamp, chunk_factory(key, path, unique_id, 'r')]
         end
       end
 
@@ -93,5 +133,29 @@ module Fluent
 
       return queue, map
     end
+
+    private
+
+    def rotate_chunk!(chunk, key)
+      queue_size = nil
+      return chunk unless chunk.full?
+
+      @queue.synchronize do
+        queue_size = @queue.size
+        enqueue(chunk) # this is buffer enqueue *hook*
+        @queue << chunk
+        chunk = (@map[key] = new_chunk(key))
+      end
+
+      return chunk, queue_size
+    end
+
+    def storable?(chunk, data)
+      (chunk.record_count + data.size) <= @buffer_chunk_records_limit
+    end
+
+    def chunk_factory(key, path, uniq_id, mode)
+      EventLimitedBufferChunk.new(key, path, uniq_id, @buffer_chunk_records_limit, mode)
+    end
   end
 end

data/test/plugin/test_buf_event_limited.rb

@@ -29,7 +29,6 @@ class EventLimitedFileBufferTest < Test::Unit::TestCase
       flush_interval 0.1
       try_flush_interval 0.03
       buffer_chunk_records_limit 10
-      buffer_chunk_message_separator newline
       buffer_path #{@buffer_path}
     ]
   end
@@ -43,7 +42,6 @@ class EventLimitedFileBufferTest < Test::Unit::TestCase
   def create_buffer_with_attributes(config = {})
     config = {
       'buffer_path' => @buffer_path,
-      'buffer_chunk_message_separator' => 'newline'
     }.merge(config)
     buf = Fluent::EventLimitedFileBuffer.new
     Fluent::EventLimitedFileBuffer.send(:class_variable_set, :'@@buffer_paths', {})
@@ -62,124 +60,123 @@ class EventLimitedFileBufferTest < Test::Unit::TestCase
     assert_equal 0.1,  output.flush_interval
     assert_equal 0.03, output.try_flush_interval
     assert_equal 10,   buffer.buffer_chunk_records_limit
-    assert_equal 'newline', buffer.buffer_chunk_message_separator
   end
 
   def test_emit
     d = create_driver
-
     buffer = d.instance.instance_variable_get(:@buffer)
-    assert buffer
-    buffer.start
+    count_buffer_events = -> { buffer.instance_variable_get(:@map)[''].record_count }
 
-    assert_nil buffer.instance_variable_get(:@map)['']
+    buffer.start
+    assert_nil buffer.instance_variable_get(:@map)[''], "No chunks on start"
 
     d.emit({"a" => 1})
-    assert_equal 1, buffer.instance_variable_get(:@map)[''].record_counter
+    assert_equal 1, count_buffer_events.call
 
-    d.emit({"a" => 2}); d.emit({"a" => 3}); d.emit({"a" => 4})
-    d.emit({"a" => 5}); d.emit({"a" => 6}); d.emit({"a" => 7});
-    d.emit({"a" => 8});
-    assert_equal 8, buffer.instance_variable_get(:@map)[''].record_counter
+    (2..9).each { |i| d.emit({"a" => i}) }
+    assert_equal 9, count_buffer_events.call
 
     chain = DummyChain.new
     tag = d.instance.instance_variable_get(:@tag)
     time = Time.now.to_i
 
     # flush_trigger false
-    assert !buffer.emit(tag, d.instance.format(tag, time, {"a" => 9}), chain)
-    assert_equal 9, buffer.instance_variable_get(:@map)[''].record_counter
-
-    # flush_trigger false
-    assert !buffer.emit(tag, d.instance.format(tag, time, {"a" => 10}), chain)
-    assert_equal 10, buffer.instance_variable_get(:@map)[''].record_counter
+    assert !buffer.emit(tag, d.instance.format(tag, time, {"a" => 10}), chain), "Shouldn't trigger flush"
+    assert_equal 10, count_buffer_events.call
 
     # flush_trigger true
-    assert buffer.emit(tag, d.instance.format(tag, time, {"a" => 11}), chain)
-    assert_equal 1, buffer.instance_variable_get(:@map)[''].record_counter # new chunk
+    assert buffer.emit(tag, d.instance.format(tag, time, {"a" => 11}), chain), "Should trigger flush"
+    assert_equal 1, count_buffer_events.call # new chunk
 
     # flush_trigger false
-    assert !buffer.emit(tag, d.instance.format(tag, time, {"a" => 12}), chain)
-    assert_equal 2, buffer.instance_variable_get(:@map)[''].record_counter
+    assert !buffer.emit(tag, d.instance.format(tag, time, {"a" => 12}), chain), "Shouldn't trigger flush"
+    assert_equal 2, count_buffer_events.call
   end
 
-  def test_resume_from_plain_text_chunk
-    # Setup buffer to test chunks
-    buf1, prefix, suffix = create_buffer_with_attributes
-    buf1.start
+  def test_emit_with_oversized_streams
+    d = create_driver
+    buffer = d.instance.instance_variable_get(:@buffer)
+    chain = DummyChain.new
+    tag = d.instance.instance_variable_get(:@tag)
+    time = Time.now.to_i
+    count_buffer_events = -> { buffer.instance_variable_get(:@map)[''].record_count }
+    count_queued_buffers = -> { buffer.instance_variable_get(:@queue).size }
 
-    # Create chunks to test
-    chunk1 = buf1.new_chunk('key1')
-    chunk2 = buf1.new_chunk('key2')
-    assert_equal 0, chunk1.record_counter
-    assert_equal 0, chunk2.record_counter
+    buffer.start
 
-    # Write data into chunks
-    chunk1 << "data1\ndata2\n"
-    chunk2 << "data3\ndata4\n"
+    events = 21.times.map { |i| [time, {a: i}] }
+    event_stream = d.instance.format_stream(tag, events)
+    assert buffer.emit(tag, event_stream, chain), "Should trigger flush"
+    assert_equal 2, count_queued_buffers.call, "Data should fill up two buffers"
+    assert_equal 1, count_buffer_events.call, "Data should overflow into a new buffer"
+    assert buffer.instance_variable_get(:@queue).all? { |b| b.record_count == 10 }
+  end
 
-    # Enqueue chunk1 and leave chunk2 open
-    buf1.enqueue(chunk1)
-    assert \
-      chunk1.path =~ /\A#{prefix}[-_.a-zA-Z0-9\%]+\.q[0-9a-f]+#{suffix}\Z/,
-      "chunk1 must be enqueued"
-    assert \
-      chunk2.path =~ /\A#{prefix}[-_.a-zA-Z0-9\%]+\.b[0-9a-f]+#{suffix}\Z/,
-      "chunk2 is not enqueued yet"
-    buf1.shutdown
+  def test_emit_with_oversized_streams_and_ongoing_buffer_chunks
+    d = create_driver
+    buffer = d.instance.instance_variable_get(:@buffer)
+    chain = DummyChain.new
+    tag = d.instance.instance_variable_get(:@tag)
+    time = Time.now.to_i
+    count_buffer_events = -> { buffer.instance_variable_get(:@map)[''].record_count }
+    count_queued_buffers = -> { buffer.instance_variable_get(:@queue).size }
 
-    # Setup a new buffer to test resume
-    buf2, *_ = create_buffer_with_attributes
-    queue, map = buf2.resume
+    buffer.start
 
-    # Returns with the open and the closed buffers
-    assert_equal 1, queue.size # closed buffer
-    assert_equal 1, map.values.size # open buffer
+    data_streams = [2, 21].map do |stream_size|
+      d.instance.format_stream(
+        tag,
+        stream_size.times.map { |i| [time, {a: i}] }
+      )
+    end
 
-    # The paths of the resumed chunks are the same but they themselfs are not
-    resumed_chunk1 = queue.first
-    resumed_chunk2 = map.values.first
-    assert_equal chunk1.path, resumed_chunk1.path
-    assert_equal chunk2.path, resumed_chunk2.path
-    assert chunk1 != resumed_chunk1
-    assert chunk2 != resumed_chunk2
+    assert !buffer.emit(tag, data_streams[0], chain), "Should not trigger flush"
+    assert buffer.emit(tag, data_streams[1], chain), "Should trigger flush"
 
-    # Resume with the proper type of buffer chunk
-    assert_equal Fluent::EventLimitedBufferChunk, resumed_chunk1.class
-    assert_equal Fluent::EventLimitedBufferChunk, resumed_chunk2.class
+    assert_equal 2, count_queued_buffers.call, "Data should fill up two buffers"
+    assert_equal 3, count_buffer_events.call, "Data should overflow into a new buffer"
+    assert buffer.instance_variable_get(:@queue).all? { |b| b.record_count == 10 }
+  end
+
+  def test_new_chunk
+    d = create_driver
+    buffer = d.instance.instance_variable_get(:@buffer)
 
-    assert_equal "data1\ndata2\n", resumed_chunk1.read
-    assert_equal "data3\ndata4\n", resumed_chunk2.read
+    chunk1 = buffer.new_chunk('')
+    chunk2 = buffer.new_chunk('')
 
-    assert_equal 2, resumed_chunk1.record_counter
-    assert_equal 2, resumed_chunk2.record_counter
+    assert chunk1 != chunk2
+    assert chunk1.path != chunk2.path
   end
 
   def test_resume_from_msgpack_chunks
+    d = create_driver
+    events = 2.times.map { |i| [Time.now.to_i, {a: i}] }
+    event_stream = d.instance.format_stream('test', events)
     # Setup buffer to test chunks
-    buf1, prefix, suffix = create_buffer_with_attributes({'buffer_chunk_message_separator' => 'msgpack'})
+    buf1, prefix, suffix = create_buffer_with_attributes
     buf1.start
 
     # Create chunks to test
     chunk1 = buf1.new_chunk('key1')
     chunk2 = buf1.new_chunk('key2')
-    assert_equal 0, chunk1.record_counter
-    assert_equal 0, chunk2.record_counter
+    assert_equal 0, chunk1.record_count
+    assert_equal 0, chunk2.record_count
 
     # Write data into chunks
-    chunk1 << MessagePack.pack('data1')
-    chunk1 << MessagePack.pack('data2')
-    chunk2 << MessagePack.pack('data3')
-    chunk2 << MessagePack.pack('data4')
+    chunk1.write(event_stream, 2)
+    chunk2.write(event_stream, 2)
 
     # Enqueue chunk1 and leave chunk2 open
     buf1.enqueue(chunk1)
-    assert \
+    assert(
       chunk1.path =~ /\A#{prefix}[-_.a-zA-Z0-9\%]+\.q[0-9a-f]+#{suffix}\Z/,
       "chunk1 must be enqueued"
-    assert \
+    )
+    assert(
       chunk2.path =~ /\A#{prefix}[-_.a-zA-Z0-9\%]+\.b[0-9a-f]+#{suffix}\Z/,
       "chunk2 is not enqueued yet"
+    )
     buf1.shutdown
 
     # Setup a new buffer to test resume
@@ -202,14 +199,10 @@ class EventLimitedFileBufferTest < Test::Unit::TestCase
     assert_equal Fluent::EventLimitedBufferChunk, resumed_chunk1.class
     assert_equal Fluent::EventLimitedBufferChunk, resumed_chunk2.class
 
-    assert_equal \
-      MessagePack.pack('data1') + MessagePack.pack('data2'),
-      resumed_chunk1.read
-    assert_equal \
-      MessagePack.pack('data3') + MessagePack.pack('data4'),
-      resumed_chunk2.read
+    assert_equal event_stream, resumed_chunk1.read
+    assert_equal event_stream, resumed_chunk2.read
 
-    assert_equal 2, resumed_chunk1.record_counter
-    assert_equal 2, resumed_chunk2.record_counter
+    assert_equal 2, resumed_chunk1.record_count
+    assert_equal 2, resumed_chunk2.record_count
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-buffer-event_limited
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.5
 platform: ruby
 authors:
 - TAGOMORI Satoshi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-25 00:00:00.000000000 Z
+date: 2015-11-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -53,6 +53,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement