fluent-plugin-fork 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9a6046f60ea7ea3def073da545f25629badae3e2
-  data.tar.gz: 89c121b947e6e4ecb7c578eb00890f68cc36f8b1
+  metadata.gz: fc184ac71ede6fa2f8f6350c3f3a95b2eca661ec
+  data.tar.gz: 99a85b33ae597ef2e22ed94d4344970cb02b8118
 SHA512:
-  metadata.gz: 68a3e769a9fb09e3086026d3ce4ccefd19f488e5949285a8810c1268945b0695a8fd4009e6e10506bd6ae9821275baad6588101e142e6c332bc591ee82fbcb25
-  data.tar.gz: 9dee3e9e0ba50b0d03576044e44057780d97d3935d010013b4bb06a069c14a3d0347145f08dbd6d9775a1119c3532e1e121faeed17955c0dcdcd31a2b11e9534
+  metadata.gz: 880f9933eeac2a5b8e2ba7653ea06a933886b7ac433b4f61fe4a880ce0247003690b3c054846ac0a439c00188dde9ea9f8e285f7600fc3aa36ed6ec8708934aa
+  data.tar.gz: 2b1699aa72558bb271f44e7bbbdc31476259a777cd66fbadc6e7bed68189f7db5f8c9a385153fe5eb31937c64c3d8eac8cd48e70e6e44ca9bf4b08863c350e0e

data/.gitignore

@@ -3,6 +3,7 @@
 *~
 .bundle
 Gemfile.lock
+gemfiles/*.lock
 vendor/
 doc/
 tmp/

data/.travis.yml

@@ -4,6 +4,9 @@ rvm:
   - 1.9.3
   - 2.0.0
   - 2.1.0
+gemfile:
+  - gemfiles/gemfile
+  - gemfiles/fluentd.0.10.42.gemfile
 
 script: "bundle exec rake spec"
 

data/README.md

@@ -1,8 +1,24 @@
-[![Build Status](https://secure.travis-ci.org/dtaniwaki/fluent-plugin-fork.png?branch=master)](http://travis-ci.org/dtaniwaki/fluent-plugin-fork) [![Coverage Status](https://coveralls.io/repos/dtaniwaki/fluent-plugin-fork/badge.png?branch=master)](https://coveralls.io/r/dtaniwaki/fluent-plugin-fork?branch=master)
-
 # fluent-plugin-fork
 
-Fork output by separating values for fluentd
+[![Gem Version](https://badge.fury.io/rb/fluent-plugin-fork.png)](https://rubygems.org/gems/fluent-plugin-fork) [![Build Status](https://secure.travis-ci.org/dtaniwaki/fluent-plugin-fork.png?branch=master)](http://travis-ci.org/dtaniwaki/fluent-plugin-fork) [![Coverage Status](https://coveralls.io/repos/dtaniwaki/fluent-plugin-fork/badge.png?branch=master)](https://coveralls.io/r/dtaniwaki/fluent-plugin-fork?branch=master)
+
+Fork output by separating values for [fluentd](http://fluentd.org/).
+
+### Example
+
+You can fork the following data
+
+```
+{"id": "1", "key": "2", "segments": "1,2,3"}
+```
+
+to
+
+```
+{"id": "1", "key": "2", "segments": "1"}
+{"id": "1", "key": "2", "segments": "2"}
+{"id": "1", "key": "2", "segments": "3"}
+```
 
 # Installation
 

data/fluent-plugin-fork.gemspec

@@ -2,7 +2,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |gem|
   gem.name        = "fluent-plugin-fork"
-  gem.version     = "0.1.0"
+  gem.version     = "0.1.1"
   gem.authors     = ["Daisuke Taniwaki"]
   gem.email       = "daisuketaniwaki@gmail.com"
   gem.homepage    = "https://github.com/dtaniwaki/fluent-plugin-fork"

data/gemfiles/fluentd.0.10.42.gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+gem 'fluentd', '= 0.10.42'
+gemspec :path => '../'

data/gemfiles/gemfile

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+
+gemspec :path => '../'

data/lib/fluent/plugin/out_fork.rb

@@ -23,7 +23,7 @@ module Fluent
     def configure(conf)
       super
 
-      fallbacks = %w(drop log)
+      fallbacks = %w(skip drop log)
       raise Fluent::ConfigError, "max_fallback must be one of #{fallbacks.inspect}" unless fallbacks.include?(@max_fallback)
     end
 
@@ -42,6 +42,9 @@ module Fluent
 
         if @max_size && @max_size < values.size
           case @max_fallback
+          when 'skip'
+            log.warn "#{tag} - #{time}: Skip too many forked values (max=#{@max_size}) : #{org_value}"
+            next
           when 'drop'
             log.warn "#{tag} - #{time}: Drop too many forked values (max=#{@max_size}) : #{org_value}"
             values = values.take(@max_size)
@@ -55,10 +58,10 @@ module Fluent
           Engine.emit(@output_tag, time, record.reject{ |k, v| k == @fork_key }.merge(@output_key => value))
         end
       end
-
-      chain.next
     rescue => e
       log.error "#{e.message}: #{e.backtrace.join(', ')}"
+    ensure
+      chain.next
     end
   end
 end

data/spec/plugin/out_fork_spec.rb

@@ -66,11 +66,21 @@ describe Fluent::ForkOutput do
       expect(subject.emits).to include(["ot", time, {"ok" => "4"}])
     end
     it "forks with other params" do
-      subject.run { subject.emit({"sk" => "2,3,4,3", "o1" => 1, "o2" => 2}, time) }
-      expect(subject.emits.size).to eq(3)
+      subject.run { subject.emit({"sk" => "2,3,4,5", "o1" => 1, "o2" => 2}, time) }
+      expect(subject.emits.size).to eq(4)
       expect(subject.emits).to include(["ot", time, {"ok" => "2", "o1" => 1, "o2" => 2}])
       expect(subject.emits).to include(["ot", time, {"ok" => "3", "o1" => 1, "o2" => 2}])
       expect(subject.emits).to include(["ot", time, {"ok" => "4", "o1" => 1, "o2" => 2}])
+      expect(subject.emits).to include(["ot", time, {"ok" => "5", "o1" => 1, "o2" => 2}])
+    end
+    it "does nothing for empty value" do
+      subject.run { subject.emit({"o1" => 1, "o2" => 2}, time) }
+      expect(subject.emits.size).to eq(0)
+    end
+    it "ignores exceptions and writes down the log" do
+      expect(subject.instance.log).to receive(:error).with(/^The error/)
+      allow_any_instance_of(String).to receive(:split).and_raise("The error")
+      subject.emit({"sk" => "2,3,4,5", "o1" => 1, "o2" => 2}, time)
     end
     context "with no_unique option" do
       let(:params) { required_params.merge(no_unique: true) }
@@ -112,6 +122,13 @@ describe Fluent::ForkOutput do
           expect(subject.emits).to include(["ot", time, {"ok" => "4"}])
         end
       end
+      describe "skip" do
+        let(:params) { required_params.merge(max_size: 3, max_fallback: 'skip') }
+        it "skip the values" do
+          subject.run { subject.emit({"sk" => "2,3,4,5"}, time) }
+          expect(subject.emits.size).to eq(0)
+        end
+      end
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-fork
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Daisuke Taniwaki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-22 00:00:00.000000000 Z
+date: 2014-05-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -79,6 +79,8 @@ files:
 - README.md
 - Rakefile
 - fluent-plugin-fork.gemspec
+- gemfiles/fluentd.0.10.42.gemfile
+- gemfiles/gemfile
 - lib/fluent/plugin/out_fork.rb
 - spec/plugin/out_fork_spec.rb
 - spec/spec_helper.rb