fluent-plugin-fork 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 9a6046f60ea7ea3def073da545f25629badae3e2
+  data.tar.gz: 89c121b947e6e4ecb7c578eb00890f68cc36f8b1
+SHA512:
+  metadata.gz: 68a3e769a9fb09e3086026d3ce4ccefd19f488e5949285a8810c1268945b0695a8fd4009e6e10506bd6ae9821275baad6588101e142e6c332bc591ee82fbcb25
+  data.tar.gz: 9dee3e9e0ba50b0d03576044e44057780d97d3935d010013b4bb06a069c14a3d0347145f08dbd6d9775a1119c3532e1e121faeed17955c0dcdcd31a2b11e9534

data/.gitignore

@@ -0,0 +1,14 @@
+*.gem
+~*
+*~
+.bundle
+Gemfile.lock
+vendor/
+doc/
+tmp/
+log/
+pkg/
+coverage/
+.yardoc
+.rbenv-version
+.ruby-version

data/.travis.yml

@@ -0,0 +1,12 @@
+language: ruby
+
+rvm:
+  - 1.9.3
+  - 2.0.0
+  - 2.1.0
+
+script: "bundle exec rake spec"
+
+branches:
+  only:
+    - master

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+
+gemspec

data/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 Daisuke Taniwaki
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,88 @@
+[![Build Status](https://secure.travis-ci.org/dtaniwaki/fluent-plugin-fork.png?branch=master)](http://travis-ci.org/dtaniwaki/fluent-plugin-fork) [![Coverage Status](https://coveralls.io/repos/dtaniwaki/fluent-plugin-fork/badge.png?branch=master)](https://coveralls.io/r/dtaniwaki/fluent-plugin-fork?branch=master)
+
+# fluent-plugin-fork
+
+Fork output by separating values for fluentd
+
+# Installation
+
+### td-agent(Linux)
+
+```
+/usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-fork
+```
+
+### td-agent(Mac)
+
+```
+sudo /usr/local/Cellar/td-agent/1.1.XX/bin/fluent-gem install fluent-plugin-fork
+```
+
+### fluentd only
+
+```
+gem install fluent-plugin-fork
+```
+
+# Configuration
+
+```
+output_tag   tag_to_output
+output_key   key_to_output
+separator    ,
+fork_key     key_to_fork
+max_size     15
+max_fallback log
+no_unique    true
+```
+
+### output_tag
+
+Tag to output forked values
+
+### output_key
+
+Key name to output forked values
+
+### fork_key
+
+Key name to fork
+
+### separator (Optional)
+
+Separator to separate the values
+
+Default: `,`
+
+### max_size (Optional)
+
+Max size of forked values.
+
+Default: `nil`
+
+### max_fallback (Optional)
+
+Strategy when the size of values exceeds max_size. Only effective when you set max_size.
+
+`log` to log the record
+`drop` to drop exceeded values
+
+Default: `log`
+
+### no_unique (Optional)
+
+Flag to emit redundant values.
+
+Default: `false`
+
+# Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new [Pull Request](../../pull/new/master)
+
+# Copyright
+
+Copyright (c) 2014 Daisuke Taniwaki. See [LICENSE](LICENSE) for details.

data/Rakefile

@@ -0,0 +1,8 @@
+require "bundler/gem_tasks"
+
+require 'rspec/core'
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = FileList['spec/**/*_spec.rb']
+end
+task :default => :spec

data/fluent-plugin-fork.gemspec

@@ -0,0 +1,22 @@
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name        = "fluent-plugin-fork"
+  gem.version     = "0.1.0"
+  gem.authors     = ["Daisuke Taniwaki"]
+  gem.email       = "daisuketaniwaki@gmail.com"
+  gem.homepage    = "https://github.com/dtaniwaki/fluent-plugin-fork"
+  gem.description = "Fork output by separating values for fluentd"
+  gem.summary     = "Fork output by separating values for fluentd"
+  gem.licenses    = ["MIT"]
+
+  gem.files       = `git ls-files`.split("\n")
+  gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
+
+  gem.add_runtime_dependency "fluentd"
+  gem.add_development_dependency "rake"
+  gem.add_development_dependency "rspec"
+  gem.add_development_dependency "coveralls"
+end

data/lib/fluent/plugin/out_fork.rb

@@ -0,0 +1,64 @@
+module Fluent
+  class ForkOutput < Output
+    class MaxForkSizeError < StandardError; end
+
+    Fluent::Plugin.register_output('fork', self)
+
+    unless method_defined?(:log)
+      define_method(:log) { $log }
+    end
+
+    def initialize
+      super
+    end
+
+    config_param :output_tag,    :string
+    config_param :output_key,    :string
+    config_param :fork_key,      :string
+    config_param :separator,     :string,  default: ','
+    config_param :max_size,      :integer, default: nil
+    config_param :max_fallback,  :string,  default: 'log'
+    config_param :no_unique,     :bool,    default: false
+
+    def configure(conf)
+      super
+
+      fallbacks = %w(drop log)
+      raise Fluent::ConfigError, "max_fallback must be one of #{fallbacks.inspect}" unless fallbacks.include?(@max_fallback)
+    end
+
+    def emit(tag, es, chain)
+      es.each do |time, record|
+        org_value = record[@fork_key]
+        if org_value.nil?
+          log.trace "#{tag} - #{time}: skip to fork #{@fork_key}=#{org_value}"
+          next
+        end
+        log.trace "#{tag} - #{time}: try to fork #{@fork_key}=#{org_value}"
+
+        values = org_value.split(@separator)
+
+        values = values.uniq unless @no_unique
+
+        if @max_size && @max_size < values.size
+          case @max_fallback
+          when 'drop'
+            log.warn "#{tag} - #{time}: Drop too many forked values (max=#{@max_size}) : #{org_value}"
+            values = values.take(@max_size)
+          when 'log'
+            log.info "#{tag} - #{time}: Too many forked values (max=#{@max_size}) : #{org_value}"
+          end
+        end
+
+        values.reject{ |value| value.to_s == '' }.each do |value|
+          log.trace "#{tag} - #{time}: reemit #{@output_key}=#{value} for #{@output_tag}"
+          Engine.emit(@output_tag, time, record.reject{ |k, v| k == @fork_key }.merge(@output_key => value))
+        end
+      end
+
+      chain.next
+    rescue => e
+      log.error "#{e.message}: #{e.backtrace.join(', ')}"
+    end
+  end
+end

data/spec/plugin/out_fork_spec.rb

@@ -0,0 +1,117 @@
+require 'spec_helper'
+
+describe Fluent::ForkOutput do
+  let(:tag) { "test.fork" }
+  let(:required_params) { {output_tag: 'ot', output_key: 'ok', fork_key: 'sk'} }
+  let(:params) { required_params }
+  let(:config) { params.map{ |k, v| "#{k} #{v}" }.join("\n") }
+  subject { Fluent::Test::OutputTestDriver.new(Fluent::ForkOutput, tag).configure(config) }
+
+  describe "#configure" do
+    let(:params) { required_params.merge(separator: '-', max_size: 5, max_fallback: 'drop', no_unique: true) }
+    it "does not raise any error" do
+      expect{ subject }.not_to raise_error
+      expect(subject.instance.output_tag).to eq('ot')
+      expect(subject.instance.output_key).to eq('ok')
+      expect(subject.instance.fork_key).to eq('sk')
+      expect(subject.instance.separator).to eq('-')
+      expect(subject.instance.no_unique).to eq(true)
+      expect(subject.instance.max_size).to eq(5)
+      expect(subject.instance.max_fallback).to eq('drop')
+    end
+    context "without optional params" do
+      let(:params) { required_params }
+      it "use default values" do
+        expect{ subject }.not_to raise_error
+        expect(subject.instance.output_tag).to eq('ot')
+        expect(subject.instance.output_key).to eq('ok')
+        expect(subject.instance.fork_key).to eq('sk')
+        expect(subject.instance.separator).to eq(',')
+        expect(subject.instance.no_unique).to eq(false)
+        expect(subject.instance.max_size).to eq(nil)
+        expect(subject.instance.max_fallback).to eq('log')
+      end
+    end
+    context "with invalid fallback" do
+      let(:params) { required_params.merge(max_fallback: 'invalid') }
+      it "raise an error" do
+        expect{ subject }.to raise_error(Fluent::ConfigError)
+      end
+    end
+    [:output_tag, :output_key, :fork_key].each do |rk|
+      context "no #{rk}" do
+        let(:params) { required_params.reject{ |k, v| k == rk } }
+        it "raises Fluent::ConfigError" do
+          expect{ subject }.to raise_error(Fluent::ConfigError)
+        end
+      end
+    end
+  end
+
+  describe "#run" do
+    let(:time) { Time.now.to_i }
+    it "forks" do
+      subject.run { subject.emit({"sk" => "2,3,4,5"}, time) }
+      expect(subject.emits.size).to eq(4)
+      expect(subject.emits).to include(["ot", time, {"ok" => "2"}])
+      expect(subject.emits).to include(["ot", time, {"ok" => "3"}])
+      expect(subject.emits).to include(["ot", time, {"ok" => "4"}])
+      expect(subject.emits).to include(["ot", time, {"ok" => "5"}])
+    end
+    it "forks uniquely" do
+      subject.run { subject.emit({"sk" => "2,3,4,3"}, time) }
+      expect(subject.emits.size).to eq(3)
+      expect(subject.emits).to include(["ot", time, {"ok" => "2"}])
+      expect(subject.emits).to include(["ot", time, {"ok" => "3"}])
+      expect(subject.emits).to include(["ot", time, {"ok" => "4"}])
+    end
+    it "forks with other params" do
+      subject.run { subject.emit({"sk" => "2,3,4,3", "o1" => 1, "o2" => 2}, time) }
+      expect(subject.emits.size).to eq(3)
+      expect(subject.emits).to include(["ot", time, {"ok" => "2", "o1" => 1, "o2" => 2}])
+      expect(subject.emits).to include(["ot", time, {"ok" => "3", "o1" => 1, "o2" => 2}])
+      expect(subject.emits).to include(["ot", time, {"ok" => "4", "o1" => 1, "o2" => 2}])
+    end
+    context "with no_unique option" do
+      let(:params) { required_params.merge(no_unique: true) }
+      it "forks for redundant values" do
+        subject.run { subject.emit({"sk" => "2,3,4,3"}, time) }
+        expect(subject.emits.size).to eq(4)
+        expect(subject.emits).to include(["ot", time, {"ok" => "2"}])
+        expect(subject.emits).to include(["ot", time, {"ok" => "3"}])
+        expect(subject.emits).to include(["ot", time, {"ok" => "4"}])
+        expect(subject.emits).to include(["ot", time, {"ok" => "3"}])
+      end
+    end
+    context "with separator option" do
+      let(:params) { required_params.merge(separator: '-') }
+      it "forks by separating with '-'" do
+        subject.run { subject.emit({"sk" => "2-3-4-5"}, time) }
+        expect(subject.emits.size).to eq(4)
+        expect(subject.emits).to include(["ot", time, {"ok" => "2"}])
+        expect(subject.emits).to include(["ot", time, {"ok" => "3"}])
+        expect(subject.emits).to include(["ot", time, {"ok" => "4"}])
+        expect(subject.emits).to include(["ot", time, {"ok" => "5"}])
+      end
+    end
+    context "with max_size and max_fallback options" do
+      describe "log" do
+        let(:params) { required_params.merge(max_size: 3, max_fallback: 'log') }
+        it "writes a log" do
+          expect(subject.instance.log).to receive(:info).with(/Too many forked values/)
+          subject.run { subject.emit({"sk" => "2,3,4,5"}, time) }
+        end
+      end
+      describe "drop" do
+        let(:params) { required_params.merge(max_size: 3, max_fallback: 'drop') }
+        it "drops exceeded values" do
+          subject.run { subject.emit({"sk" => "2,3,4,5"}, time) }
+          expect(subject.emits.size).to eq(3)
+          expect(subject.emits).to include(["ot", time, {"ok" => "2"}])
+          expect(subject.emits).to include(["ot", time, {"ok" => "3"}])
+          expect(subject.emits).to include(["ot", time, {"ok" => "4"}])
+        end
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,10 @@
+require 'rubygems'
+require 'coveralls'
+Coveralls.wear!
+
+require 'fluent/test'
+require 'fluent/plugin/out_fork'
+
+RSpec.configure do |config|
+end
+

metadata

@@ -0,0 +1,112 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-fork
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Daisuke Taniwaki
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-05-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: coveralls
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Fork output by separating values for fluentd
+email: daisuketaniwaki@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- fluent-plugin-fork.gemspec
+- lib/fluent/plugin/out_fork.rb
+- spec/plugin/out_fork_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/dtaniwaki/fluent-plugin-fork
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.14
+signing_key: 
+specification_version: 4
+summary: Fork output by separating values for fluentd
+test_files:
+- spec/plugin/out_fork_spec.rb
+- spec/spec_helper.rb
+has_rdoc: