fluent-plugin-sampling-filter 0.1.3 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3bc7e8e8abf7ba9ab745bd507bda6cba1d6b0eb3
-  data.tar.gz: 73e288aaf2416e2c120ee5009ca05497e4f4896d
+  metadata.gz: 5a5722d8c7acf064693b07707aec8ea21d9f2475
+  data.tar.gz: a486b0d0db3cf4911ecb91721c694fceee9c345f
 SHA512:
-  metadata.gz: f17bcc17d7d9b8593e7c6d85c8ec5fa567484625ae43c9fd9e8cb1de0f3c0234151f46643e3645cff1457ddea98763601a87f3bd98755869a073a0c7e53fbb7b
-  data.tar.gz: 477ed760d0a90510ab1bbc8694fb8da4fd2be91a58668826a79d261b3e312f29b25b77b628f6603e2c8425cad1b462ec854395f7e334bb2e2824af203760c57d
+  metadata.gz: 80d72b30ff51aa9bc12a397782d19acecaf611c3a7f059c9aa831f2d2d489c9eb4a91a1761a579e2d7c801bf13bcdf01ba74fc35a04be7e3c503492e39afdce9
+  data.tar.gz: 1c0320e23f30de131122a8ab249a7eedd9580361c246eca7a9010cf7d3d310ac843cd0dbe9acae971fc36f4828d188f34545807abdc62e02f83988649269f74f

data/fluent-plugin-sampling-filter.gemspec

@@ -2,13 +2,13 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-sampling-filter"
-  gem.version       = "0.1.3"
+  gem.version       = "0.2.0"
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
   gem.description   = %q{fluentd plugin to pickup sample data from matched massages}
   gem.summary       = %q{fluentd plugin to pickup sample data from matched massages}
   gem.homepage      = "https://github.com/tagomoris/fluent-plugin-sampling-filter"
-  gem.license       = "APLv2"
+  gem.license       = "Apache-2.0"
 
   gem.files         = `git ls-files`.split($\)
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
@@ -16,5 +16,6 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
 
   gem.add_development_dependency "rake"
-  gem.add_runtime_dependency "fluentd"
+  gem.add_runtime_dependency "test-unit", "~> 3.1.0"
+  gem.add_runtime_dependency "fluentd", [">= 0.12.0", "< 2"]
 end

data/lib/fluent/plugin/filter_sampling.rb

@@ -0,0 +1,62 @@
+class Fluent::SamplingFilter < Fluent::Filter
+  Fluent::Plugin.register_filter('sampling_filter', self)
+
+  config_param :interval, :integer
+  config_param :sample_unit, :string, default: 'tag'
+  config_param :minimum_rate_per_min, :integer, default: nil
+
+  def configure(conf)
+    super
+
+    @sample_unit = case @sample_unit
+                   when 'tag'
+                     :tag
+                   when 'all'
+                     :all
+                   else
+                     raise Fluent::ConfigError, "sample_unit allows only 'tag' or 'all'"
+                   end
+    @counts = {}
+    @resets = {} if @minimum_rate_per_min
+  end
+
+  def filter_stream(tag, es)
+    t = if @sample_unit == :all
+          'all'
+        else
+          tag
+        end
+
+    new_es = Fluent::MultiEventStream.new
+
+    # Access to @counts SHOULD be protected by mutex, with a heavy penalty.
+    # Code below is not thread safe, but @counts (counter for sampling rate) is not
+    # so serious value (and probably will not be broken...),
+    # then i let here as it is now.
+    if @minimum_rate_per_min
+      unless @resets[t]
+        @resets[t] = Fluent::Engine.now + (60 - rand(30))
+      end
+      if Fluent::Engine.now > @resets[t]
+        @resets[t] = Fluent::Engine.now + 60
+        @counts[t] = 0
+      end
+      es.each do |time,record|
+        c = (@counts[t] = @counts.fetch(t, 0) + 1)
+        if c < @minimum_rate_per_min or c % @interval == 0
+          new_es.add(time, record.dup)
+        end
+      end
+    else
+      es.each do |time,record|
+        c = (@counts[t] = @counts.fetch(t, 0) + 1)
+        if c % @interval == 0
+          new_es.add(time, record.dup)
+          # reset only just before @counts[t] is to be bignum from fixnum
+          @counts[t] = 0 if c > 0x6fffffff
+        end
+      end
+    end
+    new_es
+  end
+end

data/lib/fluent/plugin/out_sampling_filter.rb

@@ -47,7 +47,7 @@ class Fluent::SamplingFilterOutput < Fluent::Output
     end
 
     time_record_pairs.each {|t,r|
-      Fluent::Engine.emit(tag, t, r)
+      router.emit(tag, t, r)
     }
   end
 

data/test/helper.rb

@@ -13,6 +13,7 @@ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 $LOAD_PATH.unshift(File.dirname(__FILE__))
 require 'fluent/test'
 require 'fluent/plugin/out_sampling_filter'
+require 'fluent/plugin/filter_sampling'
 
 class Test::Unit::TestCase
 end

data/test/plugin/test_filter_sampling.rb

@@ -0,0 +1,129 @@
+require 'helper'
+
+class SamplingFilterTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+    interval 10
+    sample_unit tag
+  ]
+
+  def create_driver(conf=CONFIG,tag='test')
+    Fluent::Test::FilterTestDriver.new(Fluent::SamplingFilter, tag).configure(conf)
+  end
+
+  def test_configure
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver('')
+    }
+    d = create_driver %[
+      interval 5
+    ]
+
+    assert_equal 5, d.instance.interval
+    assert_equal :tag, d.instance.sample_unit
+
+    d = create_driver %[
+      interval 1000
+      sample_unit all
+    ]
+    assert_equal 1000, d.instance.interval
+    assert_equal :all, d.instance.sample_unit
+  end
+
+  def test_filter
+    d1 = create_driver(CONFIG, 'input.hoge1')
+    time = Time.parse("2012-01-02 13:14:15").to_i
+    d1.run do
+      d1.filter({'field1' => 'record1', 'field2' => 1})
+      d1.filter({'field1' => 'record2', 'field2' => 2})
+      d1.filter({'field1' => 'record3', 'field2' => 3})
+      d1.filter({'field1' => 'record4', 'field2' => 4})
+      d1.filter({'field1' => 'record5', 'field2' => 5})
+      d1.filter({'field1' => 'record6', 'field2' => 6})
+      d1.filter({'field1' => 'record7', 'field2' => 7})
+      d1.filter({'field1' => 'record8', 'field2' => 8})
+      d1.filter({'field1' => 'record9', 'field2' => 9})
+      d1.filter({'field1' => 'record10', 'field2' => 10})
+      d1.filter({'field1' => 'record11', 'field2' => 11})
+      d1.filter({'field1' => 'record12', 'field2' => 12})
+    end
+    filtered = d1.filtered_as_array
+    assert_equal 1, filtered.length
+    assert_equal 'input.hoge1', filtered[0][0] # tag
+    assert_equal 'record10', filtered[0][2]['field1']
+    assert_equal 10, filtered[0][2]['field2']
+
+    d2 = create_driver(%[
+      interval 3
+    ], 'input.hoge2')
+    time = Time.parse("2012-01-02 13:14:15").to_i
+    d2.run do
+      d2.filter({'field1' => 'record1', 'field2' => 1})
+      d2.filter({'field1' => 'record2', 'field2' => 2})
+      d2.filter({'field1' => 'record3', 'field2' => 3})
+      d2.filter({'field1' => 'record4', 'field2' => 4})
+      d2.filter({'field1' => 'record5', 'field2' => 5})
+      d2.filter({'field1' => 'record6', 'field2' => 6})
+      d2.filter({'field1' => 'record7', 'field2' => 7})
+      d2.filter({'field1' => 'record8', 'field2' => 8})
+      d2.filter({'field1' => 'record9', 'field2' => 9})
+      d2.filter({'field1' => 'record10', 'field2' => 10})
+      d2.filter({'field1' => 'record11', 'field2' => 11})
+      d2.filter({'field1' => 'record12', 'field2' => 12})
+    end
+    filtered = d2.filtered_as_array
+    assert_equal 4, filtered.length
+    assert_equal 'input.hoge2', filtered[0][0] # tag
+
+    assert_equal 'record3', filtered[0][2]['field1']
+    assert_equal 'record6', filtered[1][2]['field1']
+    assert_equal 'record9', filtered[2][2]['field1']
+    assert_equal 'record12', filtered[3][2]['field1']
+  end
+
+  def test_filter_minimum_rate
+    config = %[
+interval 10
+sample_unit tag
+minimum_rate_per_min 100
+]
+    d = create_driver(config, 'input.hoge3')
+    time = Time.parse("2012-01-02 13:14:15").to_i
+    d.run do
+      (1..100).each do |t|
+        d.filter({'times' => t, 'data' => 'x'})
+      end
+      (101..130).each do |t|
+        d.filter({'times' => t, 'data' => 'y'})
+      end
+    end
+    filtered = d.filtered_as_array
+    assert_equal 103, filtered.length
+    assert_equal 'input.hoge3', filtered[0][0]
+    assert_equal ((1..100).map(&:to_i) + [110, 120, 130]), filtered.map{|t,time,r| r['times']}
+    assert_equal (['x']*100 + ['y']*3), filtered.map{|t,time,r| r['data']}
+  end
+
+  def test_filter_minimum_rate_expire
+    config = %[
+interval 10
+sample_unit tag
+minimum_rate_per_min 10
+]
+    d = create_driver(config, 'input.hoge4')
+    time = Time.parse("2012-01-02 13:14:15").to_i
+    d.run do
+      (1..30).each do |t|
+        d.filter({'times' => t, 'data' => 'x'})
+      end
+    end
+    filtered = d.filtered_as_array
+    assert_equal 12, filtered.length
+    assert_equal 'input.hoge4', filtered[0][0]
+    assert_equal ((1..10).map(&:to_i)+[20,30]), filtered.map{|t,time,r| r['times']}
+    assert_equal (['x']*12), filtered.map{|t,time,r| r['data']}
+  end
+end

data/test/plugin/test_out_sampling_filter.rb

@@ -118,8 +118,7 @@ minimum_rate_per_min 100
   end
   def test_minimum_rate_expire
     # hey, this test needs 60 seconds....
-    assert_equal 1, 1
-    return
+    omit("this test needs 60 seconds....") unless ENV["EXECLONGTEST"]
 
     config = %[
 interval 10

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-sampling-filter
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.2.0
 platform: ruby
 authors:
 - TAGOMORI Satoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-03-07 00:00:00.000000000 Z
+date: 2015-10-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -24,20 +24,40 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.1.0
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.12.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.12.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
 description: fluentd plugin to pickup sample data from matched massages
 email:
 - tagomoris@gmail.com
@@ -54,12 +74,14 @@ files:
 - README.md
 - Rakefile
 - fluent-plugin-sampling-filter.gemspec
+- lib/fluent/plugin/filter_sampling.rb
 - lib/fluent/plugin/out_sampling_filter.rb
 - test/helper.rb
+- test/plugin/test_filter_sampling.rb
 - test/plugin/test_out_sampling_filter.rb
 homepage: https://github.com/tagomoris/fluent-plugin-sampling-filter
 licenses:
-- APLv2
+- Apache-2.0
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -77,10 +99,12 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: fluentd plugin to pickup sample data from matched massages
 test_files:
 - test/helper.rb
+- test/plugin/test_filter_sampling.rb
 - test/plugin/test_out_sampling_filter.rb
+has_rdoc: