fluent-plugin-buffer-lightening 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 10e80754b85bcc5fa9bc83d47e75ee7bf7637b68
-  data.tar.gz: 558822e85a83088e88268869ad13cbc121a7c39f
+  metadata.gz: 7d229e5d137896e042eb07b559b77b820d2986a3
+  data.tar.gz: 2d63ad73873268d9c126e9d8f98f50dda7c512bb
 SHA512:
-  metadata.gz: ef5d463b78bf73975c8c5ddad3e01a09960bc6b115ea578308d4dbf6d0afd5e9c0022b8164ea6f5e6b08573cf342819cfe8245304b6ff6c885019b6f93e8b03b
-  data.tar.gz: f6fdf47d488ecc61ce7efa12e392409686f2b1fbf94c2de8b2807f3e3eb5aa5eb1ca4f3303516ce1144f3e8740269a3c6b8e34994d89f12ac4ec395e2bc08bfa
+  metadata.gz: a56592eb36adf6ada002b03e8684bf39edb6b886c709f2d02e97078f2eecd0a610f5e88fdf85ee82df140c9bce3bc0f1afb98e99966896bc05ef975a7213203e
+  data.tar.gz: a98a9e1d136feed27f6422ebdf2b3c040428aafe30da29922769ab004d966116c15e2098daff266a88b3fc09cce4e7ea5ec30209e0a06184a99377b4f9e79f9d

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+rvm:
+  - 1.9.3
+  - 2.0.0

data/README.md

@@ -1,14 +1,12 @@
 # fluent-plugin-buffer-lightening
 
-Fluentd buffer plugin on memory to flush with many types of chunk limit methods:
+[Fluentd](http://fluentd.org) buffer plugin on memory to flush with many types of chunk limit methods:
   * events count limit in chunk
 
 These options are to decrease latency from emit to write, and to control chunk sizes and flush sizes.
 
 **NOTICE:** Lightening buffer plugin stores data on memory, so these data will be lost when process/server crashes.
 
-And current version of this plugin adds `try_flush_interval` option to BufferedOutput plugins, to flush buffer chunk with high frequency. For this option, run fluentd with `-r fluent/plugin/output_try_flush_interval_patch`.
-
 ## Installation
 
 Do `gem install fluent-plugin-buffer-lightening` or `fluent-gem ...`.
@@ -41,7 +39,7 @@ Options of `buffer_type memory` are also available:
 
 ### For less delay
 
-For more frequently flushing, use `flush_interval` and `try_flush_interval` with floating point values:
+For more frequently flushing, use `flush_interval` and `try_flush_interval` with floating point values on Fluentd v0.10.42 or later:
 ```
 <match data.**>
   type any_buffered_output_plugin
@@ -53,11 +51,8 @@ For more frequently flushing, use `flush_interval` and `try_flush_interval` with
 </match>
 ```
 
-And, execute fluentd as `fluentd -r fluent/plugin/output_try_flush_interval_patch -c fluentd.conf`.
-
 ## TODO
 
-* remove `output_try_flush_interval_patch` with incoming fluentd dependency
 * more limit patterns
 * patches welcome!
 

data/fluent-plugin-buffer-lightening.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-buffer-lightening"
-  spec.version       = "0.0.1"
+  spec.version       = "0.0.2"
   spec.authors       = ["TAGOMORI Satoshi"]
   spec.email         = ["tagomoris@gmail.com"]
   spec.description   = %q{Fluentd memory buffer plugin with many types of chunk limits}
@@ -17,5 +17,5 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "bundler", "~> 1.3"
   spec.add_development_dependency "rake"
-  spec.add_runtime_dependency "fluentd"
+  spec.add_runtime_dependency "fluentd", ">= 0.10.42"
 end

data/lib/fluent/plugin/buf_lightening.rb

@@ -1,5 +1,4 @@
 require 'fluent/plugin/buf_memory'
-require_relative 'output_try_flush_interval_patch'
 
 module Fluent
   class LighteningBufferChunk < MemoryBufferChunk
@@ -34,51 +33,5 @@ module Fluent
       return false if @buffer_chunk_records_limit && chunk.record_counter >= @buffer_chunk_records_limit
       true
     end
-
-    # TODO: remove w/ fluentd v0.10.42 (or td-agent including fluentd v0.10.42)
-    def emit(key, data, chain) # copy&paste from BasicBuffer, and fix to add hook point
-      key = key.to_s
-
-      synchronize do
-        top = (@map[key] ||= new_chunk(key))
-
-        if storable?(top, data) # hook point (FIXED THIS LINE ONLY)
-          chain.next
-          top << data
-          return false
-        elsif @queue.size >= @buffer_queue_limit
-          raise BufferQueueLimitError, "queue size exceeds limit"
-        end
-
-        if data.bytesize > @buffer_chunk_limit
-          $log.warn "Size of the emitted data exceeds buffer_chunk_limit."
-          $log.warn "This may occur problems in the output plugins ``at this server.``"
-          $log.warn "To avoid problems, set a smaller number to the buffer_chunk_limit"
-          $log.warn "in the forward output ``at the log forwarding server.``"
-        end
-
-        nc = new_chunk(key)
-        ok = false
-
-        begin
-          nc << data
-          chain.next
-
-          flush_trigger = false
-          @queue.synchronize {
-            enqueue(top)
-            flush_trigger = @queue.empty?
-            @queue << top
-            @map[key] = nc
-          }
-
-          ok = true
-          return flush_trigger
-        ensure
-          nc.purge unless ok
-        end
-
-      end  # synchronize
-    end
   end
 end

data/test/helper.rb

@@ -22,7 +22,6 @@ unless ENV.has_key?('VERBOSE')
   $log = nulllogger
 end
 
-require 'fluent/plugin/output_try_flush_interval_patch'
 require 'fluent/plugin/buf_lightening'
 
 class Test::Unit::TestCase

metadata

@@ -1,57 +1,57 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-buffer-lightening
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - TAGOMORI Satoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-22 00:00:00.000000000 Z
+date: 2014-05-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.10.42
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.10.42
 description: Fluentd memory buffer plugin with many types of chunk limits
 email:
 - tagomoris@gmail.com
@@ -59,14 +59,14 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
+- ".gitignore"
+- ".travis.yml"
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
 - fluent-plugin-buffer-lightening.gemspec
 - lib/fluent/plugin/buf_lightening.rb
-- lib/fluent/plugin/output_try_flush_interval_patch.rb
 - test/helper.rb
 - test/plugin/dummy_output.rb
 - test/plugin/test_buf_lightening.rb
@@ -80,17 +80,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.3
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Alternative memory buffer plugin for Fluentd to realize less delay
@@ -98,4 +98,3 @@ test_files:
 - test/helper.rb
 - test/plugin/dummy_output.rb
 - test/plugin/test_buf_lightening.rb
-has_rdoc: 

data/lib/fluent/plugin/output_try_flush_interval_patch.rb

@@ -1,113 +0,0 @@
-# MEMO: execute fluentd with "-r fluent/plugin/output_try_flush_interval_patch" switch
-
-# TODO: remove w/ fluentd v0.10.42 (or td-agent including fluentd v0.10.42)
-module Fluent
-  class BufferedOutput < Output
-    config_param :try_flush_interval, :float, :default => 1
-
-    # override with @try_flush_interval
-    def try_flush
-      time = Engine.now
-
-      empty = @buffer.queue_size == 0
-      if empty && @next_flush_time < (now = Engine.now)
-        @buffer.synchronize do
-          if @next_flush_time < now
-            enqueue_buffer
-            @next_flush_time = now + @flush_interval
-            empty = @buffer.queue_size == 0
-          end
-        end
-      end
-      if empty
-        return time + @try_flush_interval
-      end
-
-      begin
-        retrying = !@error_history.empty?
-
-        if retrying
-          @error_history.synchronize do
-            if retrying = !@error_history.empty?  # re-check in synchronize
-              if @next_retry_time >= time
-                # allow retrying for only one thread
-                return time + @try_flush_interval
-              end
-              # assume next retry failes and
-              # clear them if when it succeeds
-              @last_retry_time = time
-              @error_history << time
-              @next_retry_time += calc_retry_wait
-            end
-          end
-        end
-
-        if @secondary && @error_history.size > @retry_limit
-          has_next = flush_secondary(@secondary)
-        else
-          has_next = @buffer.pop(self)
-        end
-
-        # success
-        if retrying
-          @error_history.clear
-          # Note: don't notify to other threads to prevent
-          #       burst to recovered server
-          $log.warn "retry succeeded.", :instance=>object_id
-        end
-
-        if has_next
-          return Engine.now + @queued_chunk_flush_interval
-        else
-          return time + @try_flush_interval
-        end
-
-      rescue => e
-        if retrying
-          error_count = @error_history.size
-        else
-          # first error
-          error_count = 0
-          @error_history.synchronize do
-            if @error_history.empty?
-              @last_retry_time = time
-              @error_history << time
-              @next_retry_time = time + calc_retry_wait
-            end
-          end
-        end
-
-        if error_count < @retry_limit
-          $log.warn "temporarily failed to flush the buffer.", :next_retry=>Time.at(@next_retry_time), :error_class=>e.class.to_s, :error=>e.to_s, :instance=>object_id
-          $log.warn_backtrace e.backtrace
-
-        elsif @secondary
-          if error_count == @retry_limit
-            $log.warn "failed to flush the buffer.", :error_class=>e.class.to_s, :error=>e.to_s, :instance=>object_id
-            $log.warn "retry count exceededs limit. falling back to secondary output."
-            $log.warn_backtrace e.backtrace
-            retry  # retry immediately
-          elsif error_count <= @retry_limit + @secondary_limit
-            $log.warn "failed to flush the buffer, next retry will be with secondary output.", :next_retry=>Time.at(@next_retry_time), :error_class=>e.class.to_s, :error=>e.to_s, :instance=>object_id
-            $log.warn_backtrace e.backtrace
-          else
-            $log.warn "failed to flush the buffer.", :error_class=>e.class, :error=>e.to_s, :instance=>object_id
-            $log.warn "secondary retry count exceededs limit."
-            $log.warn_backtrace e.backtrace
-            write_abort
-            @error_history.clear
-          end
-
-        else
-          $log.warn "failed to flush the buffer.", :error_class=>e.class.to_s, :error=>e.to_s, :instance=>object_id
-          $log.warn "retry count exceededs limit."
-          $log.warn_backtrace e.backtrace
-          write_abort
-          @error_history.clear
-        end
-
-        return @next_retry_time
-      end
-    end
-  end
-end