fluent-plugin-buffer-event_limited 0.1.5 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9ab9aa64fad78b7f58fb453867cee20b4c87b7fa
-  data.tar.gz: 5443046e647187e4bf5ee215bcb925ddc6a50092
+  metadata.gz: 6dac5d961204427f1720db8a996c381430953103
+  data.tar.gz: f17424fa7ae74a0bb472f2f5b4bc0141d4a82670
 SHA512:
-  metadata.gz: 88483e3ca720d2518657c686b5fa5a0a3c3cacb8689e302725d628d08d557ea0efcec28b85c6ad1c70430c154cc702fc9766c05bfcd662249877c9d9c1c5c218
-  data.tar.gz: 653ab305d9e8c33fd4898494c5f8bb1c96ee14b66f2df9d463101cf8ac4183a7caa7f188e213f57af462266e9ecbe7e3b0b02e0578a24fd1c5b14adb05de5d50
+  metadata.gz: 78739c15f9b891f7131fa018f07bbecda52010522810434941b5c90d557afa98ad2bcf710dc5e060c823199b8fd69289ca78e9598002e0b9c22af12f9a2e14b3
+  data.tar.gz: 7437256f6949ee8c5ebcc040cac28ac6a54316794c7f8dbeec0a6af1af3b23231718cfa04e0a4d3fe9261313240d56be3a57d2f2f2f7d8f39adbeabd870c749d

data/fluent-plugin-buffer-event_limited.gemspec

@@ -2,11 +2,11 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-buffer-event_limited"
-  spec.version       = "0.1.5"
-  spec.authors       = ["TAGOMORI Satoshi", 'Gergo Sulymosi']
-  spec.email         = ["tagomoris@gmail.com", 'gergo.sulymosi@gmail.com']
-  spec.description   = %q{Fluentd memory buffer plugin with many types of chunk limits}
-  spec.summary       = %q{Alternative file buffer plugin for Fluentd to realize less delay}
+  spec.version       = "0.1.6"
+  spec.authors       = ['Gergo Sulymosi']
+  spec.email         = ['gergo.sulymosi@gmail.com']
+  spec.description   = %{Fluentd memory buffer plugin with many types of chunk limits}
+  spec.summary       = %{Alternative file buffer plugin for Fluentd to limit events in a buffer not it's size}
   spec.homepage      = "https://github.com/trekdemo/fluent-plugin-buffer-event_limited"
   spec.license       = "APLv2"
 

data/lib/fluent/plugin/buf_event_limited.rb

@@ -9,16 +9,30 @@ module Fluent
       @data = data.to_str.freeze
     end
 
-    def records
-      @records ||= (data.empty? ? [] : unpack(data)).freeze
-    end
+    # Partition the data into required sizes
+    def each_slice(target_sizes)
+      target_size = target_sizes.next
+      slice_size = 0
+      slice_data = ''
+
+      reader.each do |event|
+        if slice_size == target_size
+          yield(slice_data, slice_size)
+
+          target_size = target_sizes.next
+          slice_size = 0
+          slice_data = ''
+        end
+
+        slice_data << pack(event)
+        slice_size += 1
+      end
 
-    def as_events
-      records.dup
+      yield(slice_data, slice_size)
     end
 
     def size
-      @size ||= records.size
+      @size ||= reader.each.reduce(0) { |c, _| c + 1 }
     end
 
     alias_method :to_str, :data
@@ -26,8 +40,12 @@ module Fluent
 
     private
 
-    def unpack(data)
-      MessagePack::Unpacker.new(StringIO.new(data)).each.to_a
+    def pack(event)
+      MessagePack.pack(event)
+    end
+
+    def reader
+      @reader ||= MessagePack::Unpacker.new(StringIO.new(data))
     end
   end
 
@@ -60,44 +78,6 @@ module Fluent
 
     config_param :buffer_chunk_records_limit, :integer, :default => Float::INFINITY
 
-    def emit(key, data, chain)
-      data = MessagePackFormattedBufferData.new(data)
-      key = key.to_s
-      flush_trigger = false
-
-      synchronize do
-        # Get the active chunk if it exists
-        chunk = (@map[key] ||= new_chunk(key))
-
-        # Partition the data into chunks that can be written into new chunks
-        events = data.as_events
-        [
-          events.shift(chunk.remaining_capacity),
-          *events.each_slice(@buffer_chunk_records_limit)
-        ].each do |event_group|
-          chunk, queue_size = rotate_chunk!(chunk, key)
-          # Trigger flush only when we put the first chunk into it
-          flush_trigger ||= (queue_size == 0)
-
-          chain.next
-          chunk.write(
-            event_group.map { |d| MessagePack.pack(d) }.join(''),
-            event_group.size
-          )
-        end
-
-        return flush_trigger
-      end
-    end
-
-    def new_chunk(key)
-      encoded_key = encode_key(key)
-      path, tsuffix = make_path(encoded_key, 'b')
-      unique_id = tsuffix_to_unique_id(tsuffix)
-
-      chunk_factory(key, path, unique_id, 'a+')
-    end
-
     # Copied here from
     # https://github.com/fluent/fluentd/blob/d3ae305b6e7521fafac6ad30c6b0a8763c363b65/lib/fluent/plugin/buf_file.rb#L128-L165
     def resume
@@ -134,8 +114,40 @@ module Fluent
       return queue, map
     end
 
+    def emit(key, data, chain)
+      data = MessagePackFormattedBufferData.new(data)
+      key = key.to_s
+      flush_trigger = false
+
+      synchronize do
+        # Get the current open chunk
+        chunk = (@map[key] ||= new_chunk(key))
+
+        data.each_slice(chunk_sizes(chunk.remaining_capacity)) do |data, size|
+          chain.next
+          chunk.write(data, size)
+          chunk, queue_size = rotate_chunk!(chunk, key)
+          flush_trigger ||= (queue_size == 0)
+        end
+      end
+
+      return flush_trigger
+    end
+
+    def new_chunk(key)
+      encoded_key = encode_key(key)
+      path, tsuffix = make_path(encoded_key, 'b')
+      unique_id = tsuffix_to_unique_id(tsuffix)
+
+      chunk_factory(key, path, unique_id, 'a+')
+    end
+
     private
 
+    def chunk_factory(key, path, uniq_id, mode)
+      EventLimitedBufferChunk.new(key, path, uniq_id, @buffer_chunk_records_limit, mode)
+    end
+
     def rotate_chunk!(chunk, key)
       queue_size = nil
       return chunk unless chunk.full?
@@ -150,12 +162,15 @@ module Fluent
       return chunk, queue_size
     end
 
-    def storable?(chunk, data)
-      (chunk.record_count + data.size) <= @buffer_chunk_records_limit
-    end
-
-    def chunk_factory(key, path, uniq_id, mode)
-      EventLimitedBufferChunk.new(key, path, uniq_id, @buffer_chunk_records_limit, mode)
+    # Generates infinite sequence with and initial value followed by the chunk
+    # limit
+    #
+    # Eg.: [2, 5, 5, 5, 5, 5, ...]
+    def chunk_sizes(initial_size)
+      Enumerator.new do |y|
+        y << initial_size
+        y << @buffer_chunk_records_limit while true
+      end
     end
   end
 end

data/test/plugin/test_buf_event_limited.rb

@@ -73,24 +73,24 @@ class EventLimitedFileBufferTest < Test::Unit::TestCase
     d.emit({"a" => 1})
     assert_equal 1, count_buffer_events.call
 
-    (2..9).each { |i| d.emit({"a" => i}) }
-    assert_equal 9, count_buffer_events.call
+    (2..8).each { |i| d.emit({"a" => i}) }
+    assert_equal 8, count_buffer_events.call
 
     chain = DummyChain.new
     tag = d.instance.instance_variable_get(:@tag)
     time = Time.now.to_i
 
     # flush_trigger false
-    assert !buffer.emit(tag, d.instance.format(tag, time, {"a" => 10}), chain), "Shouldn't trigger flush"
-    assert_equal 10, count_buffer_events.call
+    assert !buffer.emit(tag, d.instance.format(tag, time, {"a" => 9}), chain), "Shouldn't trigger flush"
+    assert_equal 9, count_buffer_events.call
 
     # flush_trigger true
-    assert buffer.emit(tag, d.instance.format(tag, time, {"a" => 11}), chain), "Should trigger flush"
-    assert_equal 1, count_buffer_events.call # new chunk
+    assert buffer.emit(tag, d.instance.format(tag, time, {"a" => 10}), chain), "Should trigger flush"
+    assert_equal 0, count_buffer_events.call # new chunk
 
     # flush_trigger false
-    assert !buffer.emit(tag, d.instance.format(tag, time, {"a" => 12}), chain), "Shouldn't trigger flush"
-    assert_equal 2, count_buffer_events.call
+    assert !buffer.emit(tag, d.instance.format(tag, time, {"a" => 11}), chain), "Shouldn't trigger flush"
+    assert_equal 1, count_buffer_events.call
   end
 
   def test_emit_with_oversized_streams

metadata

@@ -1,10 +1,9 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-buffer-event_limited
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.1.6
 platform: ruby
 authors:
-- TAGOMORI Satoshi
 - Gergo Sulymosi
 autorequire: 
 bindir: bin
@@ -83,7 +82,6 @@ dependencies:
         version: 0.10.42
 description: Fluentd memory buffer plugin with many types of chunk limits
 email:
-- tagomoris@gmail.com
 - gergo.sulymosi@gmail.com
 executables: []
 extensions: []
@@ -124,7 +122,8 @@ rubyforge_project:
 rubygems_version: 2.4.5.1
 signing_key: 
 specification_version: 4
-summary: Alternative file buffer plugin for Fluentd to realize less delay
+summary: Alternative file buffer plugin for Fluentd to limit events in a buffer not
+  it's size
 test_files:
 - test/plugin/dummy_chain.rb
 - test/plugin/test_buf_event_limited.rb