fluent-plugin-buffer-event_limited 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: da62097bd8e3f3ef5005a70271cf63a51eb7ffd8
-  data.tar.gz: 1ebc0568345ce566851ace6206e49b303b0a4b98
+  metadata.gz: de3aaaafca6b7e849146aa11137d8925c79621ad
+  data.tar.gz: 3089551096e1a8b3f203492d37e275c6167d52bb
 SHA512:
-  metadata.gz: f5498467b6ee27c0589e40af603f2e6079d1bf063f6dee59864598b6ec46a09d19955b2667d8fc96a6b403e85ef46c318778e57db4aafb1e214e08897c9b3da6
-  data.tar.gz: 548e98a7238a41b3d752bb9985a0ca814689fd35d6c3e0846ab29e4afb87bfedd285420846217c6f2c0609cc58c7555cfc16bd6cbc7edfed42c5e45d73408b5d
+  metadata.gz: e9db61d4d7cf99dfce50fd88edf289e8ce5d617c744099e3ba72c8e91a47b679869726c087205bb695836b1651b6546b2439116df02872224e42160f61b1f5a1
+  data.tar.gz: db5259ef0fedd5d6a1d77fc4d2f35364c210ce398443d441098b54baa14f805bedef1148ded0c94e6ce655ec6b68f2370c65d49b78c97dc46606789ccd428a4b

data/README.md

@@ -33,6 +33,9 @@ Options of `buffer_type file` are also available:
   buffer_type event_limited
   buffer_chunk_limit 10M
   buffer_chunk_records_limit 100
+  buffer_chunk_message_separator newline
+  # buffer_chunk_message_separator tab
+  # buffer_chunk_message_separator msgpack
   # other options...
 </match>
 ```

data/fluent-plugin-buffer-event_limited.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-buffer-event_limited"
-  spec.version       = "0.1.2"
+  spec.version       = "0.1.3"
   spec.authors       = ["TAGOMORI Satoshi", 'Gergo Sulymosi']
   spec.email         = ["tagomoris@gmail.com", 'gergo.sulymosi@gmail.com']
   spec.description   = %q{Fluentd memory buffer plugin with many types of chunk limits}

data/lib/fluent/plugin/buf_event_limited.rb

@@ -4,9 +4,9 @@ module Fluent
   class EventLimitedBufferChunk < FileBufferChunk
     attr_reader :record_counter
 
-    def initialize(key, path, unique_id, mode = "a+", symlink_path = nil)
-      super
-      @record_counter = File.foreach(path).inject(0) { |c, _| c + 1 }
+    def initialize(key, path, unique_id, separator, mode = "a+", symlink_path = nil)
+      super(key, path, unique_id, mode = "a+", symlink_path = nil)
+      init_counter(path, separator)
     end
 
     def <<(data)
@@ -15,19 +15,36 @@ module Fluent
 
       return result
     end
+
+    private
+
+    def init_counter(path, separator)
+      @record_counter = \
+        case separator
+        when 'msgpack'
+          MessagePack::Unpacker.new(File.open(path)).each.inject(0) { |c, _| c + 1 }
+        when 'newline'
+          File.foreach(path, $/).inject(0) { |c, _| c + 1 }
+        when 'tab'
+          File.foreach(path, "\t").inject(0) { |c, _| c + 1 }
+        else
+          raise ArgumentError, "Separator #{separator.inspect} is not supported"
+        end
+    end
   end
 
   class EventLimitedFileBuffer < FileBuffer
     Fluent::Plugin.register_buffer('event_limited', self)
 
     config_param :buffer_chunk_records_limit, :integer, :default => Float::INFINITY
+    config_param :buffer_chunk_message_separator, :string, :default => 'msgpack'
 
     def new_chunk(key)
       encoded_key = encode_key(key)
       path, tsuffix = make_path(encoded_key, 'b')
       unique_id = tsuffix_to_unique_id(tsuffix)
 
-      EventLimitedBufferChunk.new(key, path, unique_id, 'a+', @symlink_path)
+      EventLimitedBufferChunk.new(key, path, unique_id, @buffer_chunk_message_separator, 'a+', @symlink_path)
     end
 
     # Copied here from
@@ -46,27 +63,23 @@ module Fluent
           unique_id = tsuffix_to_unique_id(tsuffix)
 
           if bq == 'b'
-            chunk = EventLimitedBufferChunk.new(key, path, unique_id, "a+")
+            chunk = EventLimitedBufferChunk.new(key, path, unique_id, @buffer_chunk_message_separator, "a+")
             maps << [timestamp, chunk]
           elsif bq == 'q'
-            chunk = EventLimitedBufferChunk.new(key, path, unique_id, "r")
+            chunk = EventLimitedBufferChunk.new(key, path, unique_id, @buffer_chunk_message_separator, "r")
             queues << [timestamp, chunk]
           end
         end
       }
 
       map = {}
-      maps.sort_by {|(timestamp,chunk)|
-        timestamp
-      }.each {|(timestamp,chunk)|
-        map[chunk.key] = chunk
-      }
+      maps
+        .sort_by { |(timestamp, chunk)| timestamp }
+        .each    { |(timestamp, chunk)| map[chunk.key] = chunk }
 
-      queue = queues.sort_by {|(timestamp,chunk)|
-        timestamp
-      }.map {|(timestamp,chunk)|
-        chunk
-      }
+      queue = queues
+        .sort_by { |(timestamp, _chunk)| timestamp }
+        .map     { |(_timestamp, chunk)| chunk }
 
       return queue, map
     end

data/test/plugin/test_buf_event_limited.rb

@@ -2,6 +2,7 @@ require_relative '../test_helper'
 require 'fluent/plugin/buf_event_limited'
 require_relative 'test_event_recorder_buffered_output'
 require_relative 'dummy_chain'
+require 'msgpack'
 
 class Hash
   def corresponding_proxies
@@ -19,7 +20,7 @@ class EventLimitedFileBufferTest < Test::Unit::TestCase
   end
 
   def teardown
-    FileUtils.remove_entry_secure @buffer_path
+    FileUtils.rmdir @buffer_path
   end
 
   def default_config
@@ -28,6 +29,7 @@ class EventLimitedFileBufferTest < Test::Unit::TestCase
       flush_interval 0.1
       try_flush_interval 0.03
       buffer_chunk_records_limit 10
+      buffer_chunk_message_separator newline
       buffer_path #{@buffer_path}
     ]
   end
@@ -38,6 +40,20 @@ class EventLimitedFileBufferTest < Test::Unit::TestCase
       .configure(conf)
   end
 
+  def create_buffer_with_attributes(config = {})
+    config = {
+      'buffer_path' => @buffer_path,
+      'buffer_chunk_message_separator' => 'newline'
+    }.merge(config)
+    buf = Fluent::EventLimitedFileBuffer.new
+    Fluent::EventLimitedFileBuffer.send(:class_variable_set, :'@@buffer_paths', {})
+    buf.configure(config)
+    prefix = buf.instance_eval{ @buffer_path_prefix }
+    suffix = buf.instance_eval{ @buffer_path_suffix }
+
+    [buf, prefix, suffix]
+  end
+
   def test_plugin_configuration
     output = create_driver.instance
     buffer = output.instance_variable_get(:@buffer)
@@ -46,6 +62,7 @@ class EventLimitedFileBufferTest < Test::Unit::TestCase
     assert_equal 0.1,  output.flush_interval
     assert_equal 0.03, output.try_flush_interval
     assert_equal 10,   buffer.buffer_chunk_records_limit
+    assert_equal 'newline', buffer.buffer_chunk_message_separator
   end
 
   def test_emit
@@ -86,57 +103,113 @@ class EventLimitedFileBufferTest < Test::Unit::TestCase
     assert_equal 2, buffer.instance_variable_get(:@map)[''].record_counter
   end
 
-  def test_resume
-    buf1 = Fluent::EventLimitedFileBuffer.new
-    buf1.configure({'buffer_path' => @buffer_path})
-    prefix = buf1.instance_eval{ @buffer_path_prefix }
-    suffix = buf1.instance_eval{ @buffer_path_suffix }
-
+  def test_resume_from_plain_text_chunk
+    # Setup buffer to test chunks
+    buf1, prefix, suffix = create_buffer_with_attributes
     buf1.start
 
+    # Create chunks to test
     chunk1 = buf1.new_chunk('key1')
+    chunk2 = buf1.new_chunk('key2')
     assert_equal 0, chunk1.record_counter
-    chunk1 << "data1\ndata2\n"
+    assert_equal 0, chunk2.record_counter
 
-    chunk2 = buf1.new_chunk('key2')
+    # Write data into chunks
+    chunk1 << "data1\ndata2\n"
     chunk2 << "data3\ndata4\n"
 
-    assert chunk1
-    assert chunk1.path =~ /\A#{prefix}[-_.a-zA-Z0-9\%]+\.b[0-9a-f]+#{suffix}\Z/, "path from new_chunk must be a 'b' buffer chunk"
-
+    # Enqueue chunk1 and leave chunk2 open
     buf1.enqueue(chunk1)
+    assert \
+      chunk1.path =~ /\A#{prefix}[-_.a-zA-Z0-9\%]+\.q[0-9a-f]+#{suffix}\Z/,
+      "chunk1 must be enqueued"
+    assert \
+      chunk2.path =~ /\A#{prefix}[-_.a-zA-Z0-9\%]+\.b[0-9a-f]+#{suffix}\Z/,
+      "chunk2 is not enqueued yet"
+    buf1.shutdown
 
-    assert chunk1
-    assert chunk1.path =~ /\A#{prefix}[-_.a-zA-Z0-9\%]+\.q[0-9a-f]+#{suffix}\Z/, "chunk1 must be enqueued"
-    assert chunk2
-    assert chunk2.path =~ /\A#{prefix}[-_.a-zA-Z0-9\%]+\.b[0-9a-f]+#{suffix}\Z/, "chunk2 is not enqueued yet"
+    # Setup a new buffer to test resume
+    buf2, *_ = create_buffer_with_attributes
+    queue, map = buf2.resume
 
-    buf1.shutdown
+    # Returns with the open and the closed buffers
+    assert_equal 1, queue.size # closed buffer
+    assert_equal 1, map.values.size # open buffer
 
-    buf2 = Fluent::EventLimitedFileBuffer.new
-    Fluent::EventLimitedFileBuffer.send(:class_variable_set, :'@@buffer_paths', {})
-    buf2.configure({'buffer_path' => @buffer_path})
-    prefix = buf2.instance_eval{ @buffer_path_prefix }
-    suffix = buf2.instance_eval{ @buffer_path_suffix }
+    # The paths of the resumed chunks are the same but they themselfs are not
+    resumed_chunk1 = queue.first
+    resumed_chunk2 = map.values.first
+    assert_equal chunk1.path, resumed_chunk1.path
+    assert_equal chunk2.path, resumed_chunk2.path
+    assert chunk1 != resumed_chunk1
+    assert chunk2 != resumed_chunk2
 
-    # buf1.start -> resume is normal operation, but now, we cannot it.
+    # Resume with the proper type of buffer chunk
+    assert_equal Fluent::EventLimitedBufferChunk, resumed_chunk1.class
+    assert_equal Fluent::EventLimitedBufferChunk, resumed_chunk2.class
+
+    assert_equal "data1\ndata2\n", resumed_chunk1.read
+    assert_equal "data3\ndata4\n", resumed_chunk2.read
+
+    assert_equal 2, resumed_chunk1.record_counter
+    assert_equal 2, resumed_chunk2.record_counter
+  end
+
+  def test_resume_from_msgpack_chunks
+    # Setup buffer to test chunks
+    buf1, prefix, suffix = create_buffer_with_attributes({'buffer_chunk_message_separator' => 'msgpack'})
+    buf1.start
+
+    # Create chunks to test
+    chunk1 = buf1.new_chunk('key1')
+    chunk2 = buf1.new_chunk('key2')
+    assert_equal 0, chunk1.record_counter
+    assert_equal 0, chunk2.record_counter
+
+    # Write data into chunks
+    chunk1 << MessagePack.pack('data1')
+    chunk1 << MessagePack.pack('data2')
+    chunk2 << MessagePack.pack('data3')
+    chunk2 << MessagePack.pack('data4')
+
+    # Enqueue chunk1 and leave chunk2 open
+    buf1.enqueue(chunk1)
+    assert \
+      chunk1.path =~ /\A#{prefix}[-_.a-zA-Z0-9\%]+\.q[0-9a-f]+#{suffix}\Z/,
+      "chunk1 must be enqueued"
+    assert \
+      chunk2.path =~ /\A#{prefix}[-_.a-zA-Z0-9\%]+\.b[0-9a-f]+#{suffix}\Z/,
+      "chunk2 is not enqueued yet"
+    buf1.shutdown
+
+    # Setup a new buffer to test resume
+    buf2, *_ = create_buffer_with_attributes({'buffer_chunk_message_separator' => 'msgpack'})
     queue, map = buf2.resume
 
-    assert_equal 1, queue.size
-    assert_equal 1, map.size
+    # Returns with the open and the closed buffers
+    assert_equal 1, queue.size # closed buffer
+    assert_equal 1, map.values.size # open buffer
 
+    # The paths of the resumed chunks are the same but they themselfs are not
     resumed_chunk1 = queue.first
+    resumed_chunk2 = map.values.first
     assert_equal chunk1.path, resumed_chunk1.path
-    resumed_chunk2 = map['key2']
     assert_equal chunk2.path, resumed_chunk2.path
+    assert chunk1 != resumed_chunk1
+    assert chunk2 != resumed_chunk2
 
+    # Resume with the proper type of buffer chunk
     assert_equal Fluent::EventLimitedBufferChunk, resumed_chunk1.class
     assert_equal Fluent::EventLimitedBufferChunk, resumed_chunk2.class
 
+    assert_equal \
+      MessagePack.pack('data1') + MessagePack.pack('data2'),
+      resumed_chunk1.read
+    assert_equal \
+      MessagePack.pack('data3') + MessagePack.pack('data4'),
+      resumed_chunk2.read
+
     assert_equal 2, resumed_chunk1.record_counter
     assert_equal 2, resumed_chunk2.record_counter
-
-    assert_equal "data1\ndata2\n", resumed_chunk1.read
-    assert_equal "data3\ndata4\n", resumed_chunk2.read
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-buffer-event_limited
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - TAGOMORI Satoshi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-09 00:00:00.000000000 Z
+date: 2015-11-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler