fluent-plugin-buffer-event_limited 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bde16a6ec9ee256c8eb1d8eff67a03f15857a67a
-  data.tar.gz: 93341c7a6f0b72d0003e907e31195a0e4703eddc
+  metadata.gz: 1ac06f3326a37f00dbce173463746fac8a77a693
+  data.tar.gz: 35b7b2257f3f435b25dd6cd743ad2634b98e5d86
 SHA512:
-  metadata.gz: d83a6398da1b8e118cffbfb221802396d6da93b530176f526c59b64ce565d69c1975de6b52ba389d7d82707bdc060745c83a14a5f59a762d7907b7c486ad4dce
-  data.tar.gz: 71ee96f41c83236bac796a0184d4dae7a6a9d532ffaf42599a4f2378091ffef9cbf25751033523b0e220c1a9acdb211622670222c555c359c5d0b501cbaea3ad
+  metadata.gz: 7eb922abcbee4971fdaf89999d508883d4120c4298f080a7c832c2e0905c1286c154f04d64486adcd83dc23ec42e8cada739d689e0f842a94a1d33affe6109a4
+  data.tar.gz: 73b1866f83553e3ca96cd7546c5b3c57d4dd4f5bf571eda5b82096f5b6d528f92825248852e0c5cc62cb6535152a6928d015ab40658e9c1edceac1c49df46ee3

data/README.md

@@ -1,5 +1,7 @@
 # fluent-plugin-buffer-event_limited
 
+This gem is a mutation of the [fluent-plugin-buffer-lightening](https://github.com/tagomoris/fluent-plugin-buffer-lightening) buffer plugin by [tagomoris](https://github.com/tagomoris).
+
 [Fluentd](http://fluentd.org) buffer plugin on memory to flush with many types of chunk limit methods:
   * events count limit in chunk
 
@@ -49,13 +51,3 @@ For more frequently flushing, use `flush_interval` and `try_flush_interval` with
 </match>
 ```
 
-## TODO
-
-* more limit patterns
-* patches welcome!
-
-## Copyright
-
-* Copyright (c) 2013- TAGOMORI Satoshi (tagomoris)
-* License
-  * Apache License, Version 2.0

data/fluent-plugin-buffer-event_limited.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-buffer-event_limited"
-  spec.version       = "0.1.0"
+  spec.version       = "0.1.1"
   spec.authors       = ["TAGOMORI Satoshi", 'Gergo Sulymosi']
   spec.email         = ["tagomoris@gmail.com", 'gergo.sulymosi@gmail.com']
   spec.description   = %q{Fluentd memory buffer plugin with many types of chunk limits}

data/lib/fluent/plugin/buf_event_limited.rb

@@ -6,12 +6,14 @@ module Fluent
 
     def initialize(key, path, unique_id, mode = "a+", symlink_path = nil)
       super
-      @record_counter = 0
+      @record_counter = %x{wc -l < "#{path}"}.to_i
     end
 
     def <<(data)
-      super
+      result = super
       @record_counter += 1
+
+      return result
     end
   end
 
@@ -28,6 +30,47 @@ module Fluent
       EventLimitedBufferChunk.new(key, path, unique_id, 'a+', @symlink_path)
     end
 
+    # Copied here from
+    # https://github.com/fluent/fluentd/blob/d3ae305b6e7521fafac6ad30c6b0a8763c363b65/lib/fluent/plugin/buf_file.rb#L128-L165
+    def resume
+      maps = []
+      queues = []
+
+      Dir.glob("#{@buffer_path_prefix}*#{@buffer_path_suffix}") {|path|
+        identifier_part = chunk_identifier_in_path(path)
+        if m = PATH_MATCH.match(identifier_part)
+          key = decode_key(m[1])
+          bq = m[2]
+          tsuffix = m[3]
+          timestamp = m[3].to_i(16)
+          unique_id = tsuffix_to_unique_id(tsuffix)
+
+          if bq == 'b'
+            chunk = EventLimitedBufferChunk.new(key, path, unique_id, "a+")
+            maps << [timestamp, chunk]
+          elsif bq == 'q'
+            chunk = EventLimitedBufferChunk.new(key, path, unique_id, "r")
+            queues << [timestamp, chunk]
+          end
+        end
+      }
+
+      map = {}
+      maps.sort_by {|(timestamp,chunk)|
+        timestamp
+      }.each {|(timestamp,chunk)|
+        map[chunk.key] = chunk
+      }
+
+      queue = queues.sort_by {|(timestamp,chunk)|
+        timestamp
+      }.map {|(timestamp,chunk)|
+        chunk
+      }
+
+      return queue, map
+    end
+
     def storable?(chunk, data)
       chunk.record_counter < @buffer_chunk_records_limit &&
         (chunk.size + data.bytesize) <= @buffer_chunk_limit

data/test/plugin/test_buf_event_limited.rb

@@ -1,15 +1,25 @@
 require_relative '../test_helper'
+require 'fluent/plugin/buf_event_limited'
 require_relative 'test_event_recorder_buffered_output'
 require_relative 'dummy_chain'
 
+class Hash
+  def corresponding_proxies
+    @corresponding_proxies ||= []
+  end
+
+  def to_masked_element
+    self
+  end
+end
+
 class EventLimitedFileBufferTest < Test::Unit::TestCase
   def setup
-    @buffer = Tempfile.new('event-limited-file-buffer')
+    @buffer_path = Dir.mktmpdir('event-limited-file-buffer')
   end
 
   def teardown
-    @buffer.close
-    @buffer.unlink
+    FileUtils.remove_entry_secure @buffer_path
   end
 
   def default_config
@@ -18,7 +28,7 @@ class EventLimitedFileBufferTest < Test::Unit::TestCase
       flush_interval 0.1
       try_flush_interval 0.03
       buffer_chunk_records_limit 10
-      buffer_path #{@buffer.path}
+      buffer_path #{@buffer_path}
     ]
   end
 
@@ -75,4 +85,58 @@ class EventLimitedFileBufferTest < Test::Unit::TestCase
     assert !buffer.emit(tag, d.instance.format(tag, time, {"a" => 12}), chain)
     assert_equal 2, buffer.instance_variable_get(:@map)[''].record_counter
   end
+
+  def test_resume
+    buf1 = Fluent::EventLimitedFileBuffer.new
+    buf1.configure({'buffer_path' => @buffer_path})
+    prefix = buf1.instance_eval{ @buffer_path_prefix }
+    suffix = buf1.instance_eval{ @buffer_path_suffix }
+
+    buf1.start
+
+    chunk1 = buf1.new_chunk('key1')
+    assert_equal 0, chunk1.record_counter
+    chunk1 << "data1\ndata2\n"
+
+    chunk2 = buf1.new_chunk('key2')
+    chunk2 << "data3\ndata4\n"
+
+    assert chunk1
+    assert chunk1.path =~ /\A#{prefix}[-_.a-zA-Z0-9\%]+\.b[0-9a-f]+#{suffix}\Z/, "path from new_chunk must be a 'b' buffer chunk"
+
+    buf1.enqueue(chunk1)
+
+    assert chunk1
+    assert chunk1.path =~ /\A#{prefix}[-_.a-zA-Z0-9\%]+\.q[0-9a-f]+#{suffix}\Z/, "chunk1 must be enqueued"
+    assert chunk2
+    assert chunk2.path =~ /\A#{prefix}[-_.a-zA-Z0-9\%]+\.b[0-9a-f]+#{suffix}\Z/, "chunk2 is not enqueued yet"
+
+    buf1.shutdown
+
+    buf2 = Fluent::EventLimitedFileBuffer.new
+    Fluent::EventLimitedFileBuffer.send(:class_variable_set, :'@@buffer_paths', {})
+    buf2.configure({'buffer_path' => @buffer_path})
+    prefix = buf2.instance_eval{ @buffer_path_prefix }
+    suffix = buf2.instance_eval{ @buffer_path_suffix }
+
+    # buf1.start -> resume is normal operation, but now, we cannot it.
+    queue, map = buf2.resume
+
+    assert_equal 1, queue.size
+    assert_equal 1, map.size
+
+    resumed_chunk1 = queue.first
+    assert_equal chunk1.path, resumed_chunk1.path
+    resumed_chunk2 = map['key2']
+    assert_equal chunk2.path, resumed_chunk2.path
+
+    assert_equal Fluent::EventLimitedBufferChunk, resumed_chunk1.class
+    assert_equal Fluent::EventLimitedBufferChunk, resumed_chunk2.class
+
+    assert_equal 2, resumed_chunk1.record_counter
+    assert_equal 2, resumed_chunk2.record_counter
+
+    assert_equal "data1\ndata2\n", resumed_chunk1.read
+    assert_equal "data3\ndata4\n", resumed_chunk2.read
+  end
 end

data/test/test_helper.rb

@@ -10,5 +10,4 @@ unless ENV.has_key?('VERBOSE')
 end
 
 require 'fluent/test'
-require 'fluent/plugin/buf_event_limited'
 Fluent::Test.setup

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-buffer-event_limited
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - TAGOMORI Satoshi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-23 00:00:00.000000000 Z
+date: 2015-11-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler