fluent-plugin-bufferize 0.0.1

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+vendor/

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-bufferize.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Masahiro Sano
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,84 @@
+# fluent-plugin-bufferize
+
+An adapter plugin which enables existing non-buffered plugins to resend messages easily in case of unexpected exceptions without creating duplicate messages.
+
+## Why
+
+Buffered plugin accumulates many messages in buffer and sends all the messages at a same time. There are many APIs that does not support such reqeusts like bulk-insert. In that case, you have to use non-buffered output and implement resend meschanism yourself because non-buffered output lacks exception handling functionality. 
+
+To use this plugin, you just have to create non-buffered plugin without caring exception handling. If an exception happens in your plugin, the request is issued again automatically. With file buffer, none of messages are lost even on sudden fluentd process down.
+
+## Configuration
+
+Just embrace existing configuration by <config> directive.
+
+If you use following configuration:
+
+```
+<match *>
+  type http
+  endpoint_url    http://foo.bar.com/
+  http_method     put
+</match>
+```
+
+Modify it like this:
+
+```
+<match *>
+  type bufferize
+  buffer_type file
+  buffer_path /var/log/fluent/myapp.*.buffer
+  <config>
+    type http
+    endpoint_url    http://foo.bar.com/
+    http_method     put
+  </config>
+</match>
+```
+
+This is a buffered output plugin. For more information about parameters, please refer [official document](http://docs.fluentd.org/articles/buffer-plugin-overview).
+
+## Example of application
+
+These plugins are good compatibility to fluent-plugin-bufferize.
+
+- [fluent-plugin-out-http](https://github.com/ento/fluent-plugin-out-http)
+- [fluent-plugin-jubatus](https://github.com/katsyoshi/fluent-plugin-jubatus)
+- [fluent-plugin-irc](https://github.com/choplin/fluent-plugin-irc)
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'fluent-plugin-bufferize'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install fluent-plugin-bufferize
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+## Copyright
+
+<table>
+  <tr>
+    <td>Author</td><td>Masahiro Sano <sabottenda@gmail.com></td>
+  </tr>
+  <tr>
+    <td>Copyright</td><td>Copyright (c) 2013- Masahiro Sano</td>
+  </tr>
+  <tr>
+    <td>License</td><td>MIT License</td>
+  </tr>
+</table>

data/Rakefile

@@ -0,0 +1,13 @@
+require "bundler"
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => [:test]
+

data/fluent-plugin-bufferize.gemspec

@@ -0,0 +1,23 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-bufferize"
+  spec.version       = "0.0.1"
+  spec.authors       = ["Masahiro Sano"]
+  spec.email         = ["sabottenda@gmail.com"]
+  spec.description   = %q{A fluentd plugin that enhances existing non-buffered output plugin as buffered plugin.}
+  spec.summary       = %q{A fluentd plugin that enhances existing non-buffered output plugin as buffered plugin.}
+  spec.homepage      = "https://github.com/sabottenda/fluent-plugin-bufferize"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "fluentd", "~> 0.10.0"
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+end

data/lib/fluent/plugin/out_bufferize.rb

@@ -0,0 +1,120 @@
+module Fluent
+  class BufferizeOutput < BufferedOutput
+    Plugin.register_output('bufferize', self)
+
+    class PosKeeper
+      @@instances = {}
+
+      def self.get(chunk)
+        @@instances[chunk.unique_id] ||= PosKeeper.new(chunk)
+        @@instances[chunk.unique_id]
+      end
+
+      def self.remove(chunk)
+        @@instances.delete(chunk.unique_id)
+      end
+
+      def initialize(chunk)
+        @id = chunk.unique_id
+        @count = 0
+        @chunk = chunk
+
+        if chunk.respond_to? :path
+          @path = chunk.path +  ".pos"
+          mode = File::CREAT | File::RDWR
+          perm = DEFAULT_FILE_PERMISSION
+          @io = File.open(@path, mode, perm)
+          @io.sync = true
+          line = @io.gets
+          @count = line ? line.to_i : 0
+          @type = :file
+        else
+          @type = :mem
+        end
+      end
+
+      def each(&block)
+        @chunk.open do |io|
+          u = MessagePack::Unpacker.new(io)
+          begin
+            if @count > 0
+              $log.debug "Bufferize: skip first #{@count} messages" 
+              @count.times do
+                u.skip
+              end
+            end
+
+            loop do
+              tag, time, record = u.read
+              yield(tag, time, record)
+              increment
+            end
+
+          rescue EOFError
+          end
+        end
+        remove
+      end
+
+      def increment
+        @count += 1
+        if @type == :file
+          @io.seek(0, IO::SEEK_SET)
+          @io.puts(@count)
+        end
+      end
+
+      def remove
+        if @type == :file
+          @io.close unless @io.closed?
+          File.unlink(@path)
+        end
+      end      
+    end
+
+
+    attr_reader :output
+
+    def initialize
+      super
+    end
+
+    def configure(conf)
+      super
+
+      configs = conf.elements.select{|e| e.name == 'config'}
+      if configs.size != 1
+        raise ConfigError, "Befferize: just one <config> directive is required"
+      end
+
+      type = configs.first['type']
+      unless type
+        raise ConfigError, "Befferize: 'type' parameter is required in <config> directive"
+      end
+
+      @output = Plugin.new_output(type)
+      @output.configure(configs.first)
+    end
+
+    def start
+      super
+      @output.start
+    end
+
+    def shutdown
+      super
+      @output.shutdown
+    end
+
+    def format(tag, time, record)
+      [tag, time, record].to_msgpack
+    end
+
+    def write(chunk)
+      PosKeeper.get(chunk).each { |tag, time, record |
+        @output.emit(tag, OneEventStream.new(time, record), NullOutputChain.instance)
+      }
+      PosKeeper.remove(chunk)
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,28 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'fluent/test'
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  }
+  $log = nulllogger
+end
+
+require 'fluent/plugin/out_bufferize'
+
+class Test::Unit::TestCase
+end

data/test/plugin/test_out_bufferize.rb

@@ -0,0 +1,145 @@
+require 'helper'
+
+class BufferizeOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+    FileUtils.rm_rf('tmp')
+    FileUtils.mkdir_p('tmp')
+  end
+
+  def teardown
+    FileUtils.mkdir_p('tmp')
+  end
+
+  BASE_CONFIG = %[
+type bufferize
+]
+  CONFIG_NO_CONFIG = BASE_CONFIG
+  CONFIG_NO_TYPE = BASE_CONFIG + %[
+<config>
+</config>
+]
+  CONFIG_WITH_TYPE = BASE_CONFIG + %[
+<config>
+  type test
+</config>
+]
+
+  def create_driver(conf = CONFIG_WITH_TYPE, tag='test')
+    Fluent::Test::BufferedOutputTestDriver.new(Fluent::BufferizeOutput, tag).configure(conf)
+  end
+
+  def test_configure
+    assert_raise(Fluent::ConfigError) {
+      create_driver(CONFIG_NO_CONFIG)
+    }
+    assert_raise(Fluent::ConfigError) {
+      create_driver(CONFIG_NO_TYPE)
+    }
+    assert_nothing_raised(Fluent::ConfigError) {
+      create_driver(CONFIG_WITH_TYPE)
+    }
+  end
+
+  def create_resend_test_driver(conf = CONFIG_WITH_TYPE, tag='test')
+    output = Fluent::Plugin.new_output('test')
+    output.configure('name' => 'output')
+    output.define_singleton_method(:start) {}
+    output.define_singleton_method(:shutdown) {}
+    output.define_singleton_method(:emit) do |tag, es, chain|
+      @count ||= 0
+      es.each do |time, record|
+        @count += 1
+        raise if (@count % 3) == 0
+        super(tag, [[time, record]], chain)
+      end
+    end
+
+    d = create_driver
+    d.instance.instance_eval { @output = output }
+    d
+  end  
+
+  def test_resend_with_memory_buffer
+    d = create_resend_test_driver
+
+    time = Time.parse("2013-11-02 12:12:12 UTC").to_i
+    entries = []
+    1.upto(5) { |i|
+      entries << [time, {"a"=>i}]
+    }
+
+    es = Fluent::ArrayEventStream.new(entries)
+    buffer = d.instance.format_stream('test', es)
+    chunk = Fluent::MemoryBufferChunk.new('', buffer)
+
+    assert_raise(RuntimeError) {
+      d.instance.write(chunk)
+    }
+    assert_equal [
+      {"a"=>1}, {"a"=>2},
+    ], d.instance.output.records
+
+    assert_raise(RuntimeError) {
+      d.instance.write(chunk)
+    }
+    assert_equal [
+      {"a"=>1}, {"a"=>2}, {"a"=>3}, {"a"=>4},
+    ], d.instance.output.records
+
+    assert_nothing_raised(RuntimeError) {
+      d.instance.write(chunk)
+    }
+    assert_equal [
+      {"a"=>1}, {"a"=>2}, {"a"=>3}, {"a"=>4}, {"a"=>5},
+    ], d.instance.output.records
+  end
+
+  def test_resend_with_file_buffer
+    d = create_resend_test_driver(CONFIG_WITH_TYPE + %[
+buffer_type file
+])
+
+    time = Time.parse("2013-11-02 12:12:12 UTC").to_i
+    entries = []
+    1.upto(5) { |i|
+      entries << [time, {"b"=>i}]
+    }
+
+    es = Fluent::ArrayEventStream.new(entries)
+    buffer = d.instance.format_stream('test', es)
+    chunk = Fluent::MemoryBufferChunk.new('', buffer)
+
+    es = Fluent::ArrayEventStream.new(entries)
+    chunk = Fluent::FileBufferChunk.new('', './tmp/test_buffer', 'xyz', "a+", nil)
+    chunk << d.instance.format_stream('test', es)
+    pos_file_path = "#{chunk.path}.pos"
+
+    assert_raise(RuntimeError) {
+      d.instance.write(chunk)
+    }
+    assert_equal [
+      {"b"=>1}, {"b"=>2},
+    ], d.instance.output.records
+    assert File.exists?(pos_file_path)
+    assert_equal `head #{pos_file_path}`.chomp.to_i, 2
+
+    assert_raise(RuntimeError) {
+      d.instance.write(chunk)
+    }
+    assert_equal [
+      {"b"=>1}, {"b"=>2}, {"b"=>3}, {"b"=>4},
+    ], d.instance.output.records
+    assert File.exists?(pos_file_path)
+    assert_equal `head #{pos_file_path}`.chomp.to_i, 4
+
+    assert_nothing_raised(RuntimeError) {
+      d.instance.write(chunk)
+    }
+    assert_equal [
+      {"b"=>1}, {"b"=>2}, {"b"=>3}, {"b"=>4}, {"b"=>5},
+    ], d.instance.output.records
+    assert !File.exists?(pos_file_path)
+  end
+
+end

metadata

@@ -0,0 +1,107 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-bufferize
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Masahiro Sano
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-11-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.10.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.10.0
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A fluentd plugin that enhances existing non-buffered output plugin as
+  buffered plugin.
+email:
+- sabottenda@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- fluent-plugin-bufferize.gemspec
+- lib/fluent/plugin/out_bufferize.rb
+- test/helper.rb
+- test/plugin/test_out_bufferize.rb
+homepage: https://github.com/sabottenda/fluent-plugin-bufferize
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: A fluentd plugin that enhances existing non-buffered output plugin as buffered
+  plugin.
+test_files:
+- test/helper.rb
+- test/plugin/test_out_bufferize.rb