fluent-plugin-unit-time-filter 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 180c6e4465e1da97a1646bf9d82b9df431d363cf
-  data.tar.gz: 74a04b0c568bef90caff5691948abee16ed147cd
+  metadata.gz: 6e0bf31be1cb31c459450da1fe3d767a6ed82f3d
+  data.tar.gz: 48f82a08b5bd76cbe50f5d930a297d5ba67f74c4
 SHA512:
-  metadata.gz: b512af5d84a5593ec90d250e3eb1d4e89a8937a9b909f59bd71e831419c2e2c0454bd32924016a28e6afbf7cb9e986d48576968423ab3e8f3a183802dd8e17da
-  data.tar.gz: 6de88e2b63dade6980d063724ad88de35e4700b70d773709febde8fc32db77d5d5df34ea3997e8867bf54974d22d779520f1569faa8bb15643d43e1224933271
+  metadata.gz: 7dd5dedbbcbfaf1425baddaef19c51856ddfcdc149afcbd44d5b2d144dfa1806745ef72b063a0d6f7acc41763d36579fa7217ddae1a739286f93a6c149476457
+  data.tar.gz: 24a6577e6917f9c79f92521030c483f5884e449c47df6ec114c5b76693164a7fd9ab8de49e20a25a9153b6b95316d0a85252862865dde4a33b1edd63aa3f6e77

data/README.md

@@ -5,6 +5,10 @@ Plug-in to aggregate by unit time
 [![Gem Version](https://badge.fury.io/rb/fluent-plugin-unit-time-filter.png)](http://badge.fury.io/rb/fluent-plugin-unit-time-filter)
 [![Build Status](https://drone.io/bitbucket.org/winebarrel/fluent-plugin-unit-time-filter/status.png)](https://drone.io/bitbucket.org/winebarrel/fluent-plugin-unit-time-filter/latest)
 
+## Installation
+
+    $ gem install fluent-plugin-unit-time-filter
+
 ## Configuration
 
 ```
@@ -63,11 +67,3 @@ fluentd outputs the following:
 2014-02-08 00:39:00 +0900 filtered.my.data: {"count":41}
 ...
 ```
-
-## Contributing
-
-1. Fork it
-2. Create your feature branch (`git checkout -b my-new-feature`)
-3. Commit your changes (`git commit -am 'Add some feature'`)
-4. Push to the branch (`git push origin my-new-feature`)
-5. Create new Pull Request

data/fluent-plugin-unit-time-filter.gemspec

@@ -1,7 +1,7 @@
 # coding: utf-8
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-unit-time-filter"
-  spec.version       = "0.1.1"
+  spec.version       = "0.1.2"
   spec.authors       = ["Genki Sugawara"]
   spec.email         = ["sgwr_dts@yahoo.co.jp"]
   spec.description   = %q{Plug-in to aggregate by unit time}

data/lib/fluent/plugin/out_unit_time_filter.rb

@@ -3,6 +3,10 @@ require 'fluent/plugin/unit_time_filter_buffer'
 class Fluent::UnitTimeFilterOutput < Fluent::Output
   Fluent::Plugin.register_output('unit_time_filter', self)
 
+  unless method_defined?(:log)
+    define_method("log") { $log }
+  end
+
   config_param :filter_path,       :type => :string
   config_param :unit_sec,          :type => :size,   :default => 1
   config_param :prefix,            :type => :string, :default => 'filtered'
@@ -48,6 +52,7 @@ class Fluent::UnitTimeFilterOutput < Fluent::Output
         :pass_hash_row     => @pass_hash_row,
         :hash_row_time_key => @hash_row_time_key,
         :hash_row_tag_key  => @hash_row_tag_key,
+        :log               => log,
       })
 
       Thread.current[BUFFER_KEY] = buf
@@ -57,6 +62,7 @@ class Fluent::UnitTimeFilterOutput < Fluent::Output
       yield(buf)
     rescue Exception => e
       Thread.current[BUFFER_KEY] = nil
+      log.error(e)
       raise e
     end
   end

data/lib/fluent/plugin/unit_time_filter_buffer.rb

@@ -4,6 +4,7 @@ class Fluent::UnitTimeFilterOutput < Fluent::Output
 
     def initialize(options)
       @init_queue = []
+      @log = options[:log]
 
       @fiber = Fiber.new do |tag, es|
         @init_queue << [tag, es]
@@ -43,7 +44,7 @@ class Fluent::UnitTimeFilterOutput < Fluent::Output
         prev_time = time
         result
       }.each {|records|
-        records = records.inject([]) {|r, i| r + i[1] }
+        records = records.map {|i| i[1] }.inject(:+)
         records = conv_to_hash_rows_if_needed(records, options)
 
         time = get_time(records.first, options)
@@ -64,7 +65,11 @@ class Fluent::UnitTimeFilterOutput < Fluent::Output
 
     def emit_records(prefix, tag, time, records)
       records.each do |record|
-        Fluent::Engine.emit("#{prefix}.#{tag}", time, record)
+        if record.kind_of?(Hash)
+          Fluent::Engine.emit("#{prefix}.#{tag}", time, record)
+        else
+          @log.warn("Record must be Hash: #{record.inspect} (#{record.class})")
+        end
       end
     end
 

data/spec/out_unit_time_filter_spec.rb

@@ -235,7 +235,7 @@ describe Fluent::UnitTimeFilterOutput do
   end
 
   describe 'when an error happened' do
-    it 'filter name should be to be included in the error' do
+    it 'filter name should be included in the error' do
       filter = <<-EOS
         proc {|rs|
           raise 'Any error message'
@@ -243,6 +243,8 @@ describe Fluent::UnitTimeFilterOutput do
       EOS
 
       run_driver(:filter => filter, :tempfile => 'any_filter.rb') do |d|
+        d.instance.log.should_receive(:error) {|e| expect(e.message).to eq('Any error message') }
+
         begin
           (0...10).each do |i|
             d.emit({"key#{i}" => "val#{i}"}, time + i)
@@ -255,4 +257,109 @@ describe Fluent::UnitTimeFilterOutput do
       end
     end
   end
+
+  describe 'when an invalid record was included' do
+    it 'should skip the bad records (Filter returns a Hash)' do
+      filter = <<-EOS
+        proc {|rs|
+          if rs.any? {|i| i[2].has_key?('return_nil')}
+            nil
+          else
+            {
+              'count'   => rs.count,
+              'inspect' => rs.inspect
+            }
+          end
+        }
+      EOS
+
+      run_driver(:filter => filter) do |d|
+        d.instance.log.should_receive(:warn) {|msg| expect(msg).to eq('Record must be Hash: nil (NilClass)') }
+
+        (0...10).each do |i|
+          d.emit({"key#{i}" => "val#{i}"}, time + i)
+          d.emit({"key#{i}_" => "val#{i}_"}, time + i)
+        end
+
+        d.emit({"return_nil" => 1}, time + 10)
+
+        (10...20).each do |i|
+          d.emit({"key#{i}" => "val#{i}"}, time + i)
+          d.emit({"key#{i}_" => "val#{i}_"}, time + i)
+        end
+
+        expect(d.emits).to eq(
+          [["filtered.test.default", 1391832855, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832855, {\"key0\"=>\"val0\"}], [\"test.default\", 1391832855, {\"key0_\"=>\"val0_\"}]]"}],
+           ["filtered.test.default", 1391832856, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832856, {\"key1\"=>\"val1\"}], [\"test.default\", 1391832856, {\"key1_\"=>\"val1_\"}]]"}],
+           ["filtered.test.default", 1391832857, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832857, {\"key2\"=>\"val2\"}], [\"test.default\", 1391832857, {\"key2_\"=>\"val2_\"}]]"}],
+           ["filtered.test.default", 1391832858, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832858, {\"key3\"=>\"val3\"}], [\"test.default\", 1391832858, {\"key3_\"=>\"val3_\"}]]"}],
+           ["filtered.test.default", 1391832859, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832859, {\"key4\"=>\"val4\"}], [\"test.default\", 1391832859, {\"key4_\"=>\"val4_\"}]]"}],
+           ["filtered.test.default", 1391832860, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832860, {\"key5\"=>\"val5\"}], [\"test.default\", 1391832860, {\"key5_\"=>\"val5_\"}]]"}],
+           ["filtered.test.default", 1391832861, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832861, {\"key6\"=>\"val6\"}], [\"test.default\", 1391832861, {\"key6_\"=>\"val6_\"}]]"}],
+           ["filtered.test.default", 1391832862, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832862, {\"key7\"=>\"val7\"}], [\"test.default\", 1391832862, {\"key7_\"=>\"val7_\"}]]"}],
+           ["filtered.test.default", 1391832863, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832863, {\"key8\"=>\"val8\"}], [\"test.default\", 1391832863, {\"key8_\"=>\"val8_\"}]]"}],
+           ["filtered.test.default", 1391832864, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832864, {\"key9\"=>\"val9\"}], [\"test.default\", 1391832864, {\"key9_\"=>\"val9_\"}]]"}],
+           ["filtered.test.default", 1391832866, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832866, {\"key11\"=>\"val11\"}], [\"test.default\", 1391832866, {\"key11_\"=>\"val11_\"}]]"}],
+           ["filtered.test.default", 1391832867, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832867, {\"key12\"=>\"val12\"}], [\"test.default\", 1391832867, {\"key12_\"=>\"val12_\"}]]"}],
+           ["filtered.test.default", 1391832868, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832868, {\"key13\"=>\"val13\"}], [\"test.default\", 1391832868, {\"key13_\"=>\"val13_\"}]]"}],
+           ["filtered.test.default", 1391832869, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832869, {\"key14\"=>\"val14\"}], [\"test.default\", 1391832869, {\"key14_\"=>\"val14_\"}]]"}],
+           ["filtered.test.default", 1391832870, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832870, {\"key15\"=>\"val15\"}], [\"test.default\", 1391832870, {\"key15_\"=>\"val15_\"}]]"}],
+           ["filtered.test.default", 1391832871, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832871, {\"key16\"=>\"val16\"}], [\"test.default\", 1391832871, {\"key16_\"=>\"val16_\"}]]"}],
+           ["filtered.test.default", 1391832872, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832872, {\"key17\"=>\"val17\"}], [\"test.default\", 1391832872, {\"key17_\"=>\"val17_\"}]]"}]]
+        )
+      end
+    end
+
+    it 'should skip the bad records (Filter returns an Array)' do
+      filter = <<-EOS
+        proc {|rs|
+          if rs.any? {|i| i[2].has_key?('return_nil')}
+            [nil, 1]
+          else
+            {
+              'count'   => rs.count,
+              'inspect' => rs.inspect
+            }
+          end
+        }
+      EOS
+
+      run_driver(:filter => filter) do |d|
+        d.instance.log.should_receive(:warn) {|msg| expect(msg).to eq('Record must be Hash: nil (NilClass)') }
+        d.instance.log.should_receive(:warn) {|msg| expect(msg).to eq('Record must be Hash: 1 (Fixnum)') }
+
+        (0...10).each do |i|
+          d.emit({"key#{i}" => "val#{i}"}, time + i)
+          d.emit({"key#{i}_" => "val#{i}_"}, time + i)
+        end
+
+        d.emit({"return_nil" => 1}, time + 10)
+
+        (10...20).each do |i|
+          d.emit({"key#{i}" => "val#{i}"}, time + i)
+          d.emit({"key#{i}_" => "val#{i}_"}, time + i)
+        end
+
+        expect(d.emits).to eq(
+          [["filtered.test.default", 1391832855, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832855, {\"key0\"=>\"val0\"}], [\"test.default\", 1391832855, {\"key0_\"=>\"val0_\"}]]"}],
+           ["filtered.test.default", 1391832856, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832856, {\"key1\"=>\"val1\"}], [\"test.default\", 1391832856, {\"key1_\"=>\"val1_\"}]]"}],
+           ["filtered.test.default", 1391832857, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832857, {\"key2\"=>\"val2\"}], [\"test.default\", 1391832857, {\"key2_\"=>\"val2_\"}]]"}],
+           ["filtered.test.default", 1391832858, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832858, {\"key3\"=>\"val3\"}], [\"test.default\", 1391832858, {\"key3_\"=>\"val3_\"}]]"}],
+           ["filtered.test.default", 1391832859, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832859, {\"key4\"=>\"val4\"}], [\"test.default\", 1391832859, {\"key4_\"=>\"val4_\"}]]"}],
+           ["filtered.test.default", 1391832860, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832860, {\"key5\"=>\"val5\"}], [\"test.default\", 1391832860, {\"key5_\"=>\"val5_\"}]]"}],
+           ["filtered.test.default", 1391832861, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832861, {\"key6\"=>\"val6\"}], [\"test.default\", 1391832861, {\"key6_\"=>\"val6_\"}]]"}],
+           ["filtered.test.default", 1391832862, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832862, {\"key7\"=>\"val7\"}], [\"test.default\", 1391832862, {\"key7_\"=>\"val7_\"}]]"}],
+           ["filtered.test.default", 1391832863, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832863, {\"key8\"=>\"val8\"}], [\"test.default\", 1391832863, {\"key8_\"=>\"val8_\"}]]"}],
+           ["filtered.test.default", 1391832864, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832864, {\"key9\"=>\"val9\"}], [\"test.default\", 1391832864, {\"key9_\"=>\"val9_\"}]]"}],
+           ["filtered.test.default", 1391832866, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832866, {\"key11\"=>\"val11\"}], [\"test.default\", 1391832866, {\"key11_\"=>\"val11_\"}]]"}],
+           ["filtered.test.default", 1391832867, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832867, {\"key12\"=>\"val12\"}], [\"test.default\", 1391832867, {\"key12_\"=>\"val12_\"}]]"}],
+           ["filtered.test.default", 1391832868, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832868, {\"key13\"=>\"val13\"}], [\"test.default\", 1391832868, {\"key13_\"=>\"val13_\"}]]"}],
+           ["filtered.test.default", 1391832869, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832869, {\"key14\"=>\"val14\"}], [\"test.default\", 1391832869, {\"key14_\"=>\"val14_\"}]]"}],
+           ["filtered.test.default", 1391832870, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832870, {\"key15\"=>\"val15\"}], [\"test.default\", 1391832870, {\"key15_\"=>\"val15_\"}]]"}],
+           ["filtered.test.default", 1391832871, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832871, {\"key16\"=>\"val16\"}], [\"test.default\", 1391832871, {\"key16_\"=>\"val16_\"}]]"}],
+           ["filtered.test.default", 1391832872, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832872, {\"key17\"=>\"val17\"}], [\"test.default\", 1391832872, {\"key17_\"=>\"val17_\"}]]"}]]
+        )
+      end
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-unit-time-filter
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Genki Sugawara
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-09 00:00:00.000000000 Z
+date: 2014-02-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd