fluent-plugin-sampling-filter 0.1.3 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 3bc7e8e8abf7ba9ab745bd507bda6cba1d6b0eb3
-  data.tar.gz: 73e288aaf2416e2c120ee5009ca05497e4f4896d
+SHA256:
+  metadata.gz: 2830d5959dab3375606dd4733b95b1ecc4e7cec666791937a708791b6e841bf6
+  data.tar.gz: 0767a563091fa6674f7f37158857decdaef378c92b181c14054ccd8a183c6586
 SHA512:
-  metadata.gz: f17bcc17d7d9b8593e7c6d85c8ec5fa567484625ae43c9fd9e8cb1de0f3c0234151f46643e3645cff1457ddea98763601a87f3bd98755869a073a0c7e53fbb7b
-  data.tar.gz: 477ed760d0a90510ab1bbc8694fb8da4fd2be91a58668826a79d261b3e312f29b25b77b628f6603e2c8425cad1b462ec854395f7e334bb2e2824af203760c57d
+  metadata.gz: a65c1919e2e774326a3ef25e3f9348a530566434c2aca03803b5b63804b5aada9fa40ef3b4a1e137a00e6204b7f09f5d85d746f45c7f0a2e4177dbbe05afdb8b
+  data.tar.gz: c5602c44dc0dfd4007082e2c9138745423a1bed774ff0d453eed9e00c3b9941840fe6a3245b5cdd4b656fcd6960e014602c2a716a6d08653858dfb339cc07410

data/.travis.yml

@@ -1,4 +1,6 @@
+sudo: false
 language: ruby
 rvm:
-  - 1.9.3
-  - 2.0.0
+  - 2.1
+  - 2.2
+  - 2.3.0

data/README.md

@@ -1,26 +1,96 @@
 # fluent-plugin-sampling-filter
 
-## Component
+This is a [Fluentd](http://fluentd.org) plugin to sample matching messages to analyse and report messages behavior and emit sampled messages with modified tag.
 
-### SamplingFilterOutput
+* sampling rate per tags, message field, or all
 
-This is a [Fluentd](http://fluentd.org) plugin to sample matching messages to analyse and report messages behavior and emit sampled messages with modified tag.
+## Requirements
 
-* sampling rate per tags, or for all
-* remove_prefix of tags for input messages, and add_prefix of tags for output(sampled) messages
+| fluent-plugin-sampling-filter | fluentd    | ruby   |
+|-------------------------------|------------|--------|
+| >= 1.0.0                      | >= v0.14.0 | >= 2.1 |
+| <  1.0.0                      | <  v0.14.0 | >= 1.9 |
 
 ## Configuration
 
+### SamplingFilter
+
+This filter passes a specified part of whole events to following filter/output plugins:
+
+    <source>
+      @type any_great_input
+      @label @mydata
+    </source>
+
+    <label @mydata>
+      <filter **>
+        @type sampling
+        sample_unit all
+        interval 10    # pass 1/10 events to following plugins
+      </filter>
+
+      <match **>
+        @type ...
+      </match>
+    </label>
+
+Sampling is done for all events, but we can do it per matched tags:
+
+    <source>
+      @type any_great_input
+      @label @mydata
+    </source>
+
+    <label @mydata>
+      <filter **>
+        @type sampling
+        interval 10
+        sample_unit tag # 1/10 events for each tags
+      </filter>
+
+      <match **>
+        @type ...
+      </match>
+    </label>
+
+
+We can also sample based on a value in the message
+
+    <source>
+      @type any_great_input
+      @label @mydata
+    </source>
+
+    <label @mydata>
+      <filter **>
+        @type sampling
+        interval 10
+        # pass 1/10 events per user given events like: { user: { name: "Bob" }, ... }
+        sample_unit $.user.name
+      </filter>
+
+      <match **>
+        @type ...
+      </match>
+    </label>
+
+`minimum_rate_per_min` option(integer) configures this plugin to pass events with the specified rate even how small is the total number of whole events.
+
+`sample_unit` option(string) configures this plugin to sample data based on tag(default), 'all', or by field value
+using the [record accessor syntax](https://docs.fluentd.org/plugin-helper-overview/api-plugin-helper-record_accessor).
+
 ### SamplingFilterOutput
 
+**NOTE: This plugin is deprecated. Use filter plugin instead.**
+
 Pickup 1/10 messages about each tags(default: `sample_unit tag`), and add tag prefix `sampled`.
 
     <match **>
-      type sampling_filter
+      @type sampling_filter
       interval 10
       add_prefix sampled
     </match>
-    
+
     <match sampled.**>
       # output configurations where to send sampled messages
     </match>
@@ -28,20 +98,19 @@ Pickup 1/10 messages about each tags(default: `sample_unit tag`), and add tag pr
 Pickup 1/100 messages of all matched messages, and modify tags from `input.**` to `output.**`
 
     <match input.**>
-      type sampling_filter
+      @type sampling_filter
       interval 100
       sample_unit all
       remove_prefix input
       add_prefix output
     </match>
-    
+
     <match sampled.**>
       # output configurations where to send sampled messages
     </match>
 
 ## TODO
 
-* consider what to do next
 * patches welcome!
 
 ## Copyright

data/fluent-plugin-sampling-filter.gemspec

@@ -2,13 +2,13 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-sampling-filter"
-  gem.version       = "0.1.3"
+  gem.version       = "1.2.0"
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
   gem.description   = %q{fluentd plugin to pickup sample data from matched massages}
   gem.summary       = %q{fluentd plugin to pickup sample data from matched massages}
   gem.homepage      = "https://github.com/tagomoris/fluent-plugin-sampling-filter"
-  gem.license       = "APLv2"
+  gem.license       = "Apache-2.0"
 
   gem.files         = `git ls-files`.split($\)
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
@@ -16,5 +16,6 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
 
   gem.add_development_dependency "rake"
-  gem.add_runtime_dependency "fluentd"
+  gem.add_runtime_dependency "test-unit", "~> 3.1.0"
+  gem.add_runtime_dependency "fluentd", [">= 0.14.12", "< 2"]
 end

data/lib/fluent/plugin/filter_sampling.rb

@@ -0,0 +1,71 @@
+require 'fluent/plugin/filter'
+require 'fluent/clock'
+
+class Fluent::Plugin::SamplingFilter < Fluent::Plugin::Filter
+  Fluent::Plugin.register_filter('sampling', self)
+  Fluent::Plugin.register_filter('sampling_filter', self)
+
+  config_param :interval, :integer
+  config_param :sample_unit, :string, default: 'tag'
+  config_param :minimum_rate_per_min, :integer, default: nil
+
+  helpers :record_accessor
+
+  def configure(conf)
+    super
+
+    @counts = {}
+    @resets = {} if @minimum_rate_per_min
+    @accessor = record_accessor_create(@sample_unit) unless %w(all tag).include?(@sample_unit)
+  end
+
+  # Access to @counts SHOULD be protected by mutex, with a heavy penalty.
+  # Code below is not thread safe, but @counts (counter for sampling rate) is not
+  # so serious value (and probably will not be broken...),
+  # then i let here as it is now.
+
+  def filter(tag, _time, record)
+    t = record_key(tag, record)
+    if @minimum_rate_per_min
+      filter_with_minimum_rate(t, record)
+    else
+      filter_simple(t, record)
+    end
+  end
+
+  def filter_simple(t, record)
+    c = (@counts[t] = @counts.fetch(t, 0) + 1)
+    # reset only just before @counts[t] is to be bignum from fixnum
+    @counts[t] = 0 if c > 0x6fffffff
+    if c % @interval == 0
+      record
+    else
+      nil
+    end
+  end
+
+  def filter_with_minimum_rate(t, record)
+    @resets[t] ||= Fluent::Clock.now + (60 - rand(30))
+    if Fluent::Clock.now > @resets[t]
+      @resets[t] = Fluent::Clock.now + 60
+      @counts[t] = 0
+    end
+    c = (@counts[t] = @counts.fetch(t, 0) + 1)
+    if c < @minimum_rate_per_min || c % @interval == 0
+      record.dup
+    else
+      nil
+    end
+  end
+
+  def record_key(tag, record)
+    case @sample_unit
+    when 'all'
+      'all'
+    when 'tag'
+      tag
+    else
+      @accessor.call(record)
+    end
+  end
+end

data/lib/fluent/plugin/out_sampling_filter.rb

@@ -1,36 +1,31 @@
-class Fluent::SamplingFilterOutput < Fluent::Output
+require 'fluent/plugin/output'
+require 'fluent/clock'
+
+class Fluent::Plugin::SamplingFilterOutput < Fluent::Plugin::Output
   Fluent::Plugin.register_output('sampling_filter', self)
 
-  config_param :interval, :integer
-  config_param :sample_unit, :string, :default => 'tag'
-  config_param :remove_prefix, :string, :default => nil
-  config_param :add_prefix, :string, :default => 'sampled'
-  config_param :minimum_rate_per_min, :integer, :default => nil
+  helpers :event_emitter
 
-  # Define `log` method for v0.10.42 or earlier
-  unless method_defined?(:log)
-    define_method("log") { $log }
-  end
+  config_param :interval, :integer
+  config_param :sample_unit, :enum, list: [:tag, :all], default: :tag
+  config_param :remove_prefix, :string, default: nil
+  config_param :add_prefix, :string, default: 'sampled'
+  config_param :minimum_rate_per_min, :integer, default: nil
 
   def configure(conf)
     super
 
+    log.warn "sampling_filter output plugin is deprecated. use sampling_filter filter plugin instead with <label> routing."
+
     if @remove_prefix
       @removed_prefix_string = @remove_prefix + '.'
       @removed_length = @removed_prefix_string.length
     elsif @add_prefix.empty?
       raise Fluent::ConfigError, "either of 'add_prefix' or 'remove_prefix' must be specified"
     end
+    @added_prefix_string = nil
     @added_prefix_string = @add_prefix + '.' unless @add_prefix.empty?
 
-    @sample_unit = case @sample_unit
-                   when 'tag'
-                     :tag
-                   when 'all'
-                     :all
-                   else
-                     raise Fluent::ConfigError, "sample_unit allows only 'tag' or 'all'"
-                   end
     @counts = {}
     @resets = {} if @minimum_rate_per_min
   end
@@ -47,11 +42,11 @@ class Fluent::SamplingFilterOutput < Fluent::Output
     end
 
     time_record_pairs.each {|t,r|
-      Fluent::Engine.emit(tag, t, r)
+      router.emit(tag, t, r)
     }
   end
 
-  def emit(tag, es, chain)
+  def process(tag, es)
     t = if @sample_unit == :all
           'all'
         else
@@ -65,11 +60,9 @@ class Fluent::SamplingFilterOutput < Fluent::Output
     # so serious value (and probably will not be broken...),
     # then i let here as it is now.
     if @minimum_rate_per_min
-      unless @resets[t]
-        @resets[t] = Fluent::Engine.now + (60 - rand(30))
-      end
-      if Fluent::Engine.now > @resets[t]
-        @resets[t] = Fluent::Engine.now + 60
+      @resets[t] ||= Fluent::Clock.now + (60 - rand(30))
+      if Fluent::Clock.now > @resets[t]
+        @resets[t] = Fluent::Clock.now + 60
         @counts[t] = 0
       end
       es.each do |time,record|
@@ -90,7 +83,5 @@ class Fluent::SamplingFilterOutput < Fluent::Output
     end
 
     emit_sampled(tag, pairs)
-
-    chain.next
   end
 end

data/test/helper.rb

@@ -13,6 +13,7 @@ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 $LOAD_PATH.unshift(File.dirname(__FILE__))
 require 'fluent/test'
 require 'fluent/plugin/out_sampling_filter'
+require 'fluent/plugin/filter_sampling'
 
 class Test::Unit::TestCase
 end

data/test/plugin/test_filter_sampling.rb

@@ -0,0 +1,186 @@
+require 'helper'
+require 'fluent/test/driver/filter'
+
+class SamplingFilterTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+    interval 10
+    sample_unit tag
+  ]
+
+  def create_driver(conf=CONFIG)
+    Fluent::Test::Driver::Filter.new(Fluent::Plugin::SamplingFilter).configure(conf)
+  end
+
+  def test_configure
+    assert_raise(Fluent::ConfigError) {
+      create_driver('')
+    }
+    d = create_driver %[
+      interval 5
+    ]
+
+    assert_equal 5, d.instance.interval
+    assert_equal 'tag', d.instance.sample_unit
+
+    d = create_driver %[
+      interval 1000
+      sample_unit all
+    ]
+    assert_equal 1000, d.instance.interval
+    assert_equal 'all', d.instance.sample_unit
+
+    d = create_driver %[
+      interval 1000
+      sample_unit $fake
+    ]
+    assert_equal 1000, d.instance.interval
+    assert_equal "$fake", d.instance.sample_unit
+  end
+
+  def test_filter
+    d1 = create_driver(CONFIG)
+    time = Time.parse("2012-01-02 13:14:15").to_i
+    d1.run(default_tag: 'input.hoge1') do
+      d1.feed({'field1' => 'record1', 'field2' => 1})
+      d1.feed({'field1' => 'record2', 'field2' => 2})
+      d1.feed({'field1' => 'record3', 'field2' => 3})
+      d1.feed({'field1' => 'record4', 'field2' => 4})
+      d1.feed({'field1' => 'record5', 'field2' => 5})
+      d1.feed({'field1' => 'record6', 'field2' => 6})
+      d1.feed({'field1' => 'record7', 'field2' => 7})
+      d1.feed({'field1' => 'record8', 'field2' => 8})
+      d1.feed({'field1' => 'record9', 'field2' => 9})
+      d1.feed({'field1' => 'record10', 'field2' => 10})
+      d1.feed({'field1' => 'record11', 'field2' => 11})
+      d1.feed({'field1' => 'record12', 'field2' => 12})
+    end
+    filtered = d1.filtered
+    assert_equal 1, filtered.length
+    assert_equal 'record10', filtered[0][1]['field1']
+    assert_equal 10, filtered[0][1]['field2']
+
+    d2 = create_driver(%[
+      interval 3
+    ])
+    time = Time.parse("2012-01-02 13:14:15").to_i
+    d2.run(default_tag: 'input.hoge2') do
+      d2.feed({'field1' => 'record1', 'field2' => 1})
+      d2.feed({'field1' => 'record2', 'field2' => 2})
+      d2.feed({'field1' => 'record3', 'field2' => 3})
+      d2.feed({'field1' => 'record4', 'field2' => 4})
+      d2.feed({'field1' => 'record5', 'field2' => 5})
+      d2.feed({'field1' => 'record6', 'field2' => 6})
+      d2.feed({'field1' => 'record7', 'field2' => 7})
+      d2.feed({'field1' => 'record8', 'field2' => 8})
+      d2.feed({'field1' => 'record9', 'field2' => 9})
+      d2.feed({'field1' => 'record10', 'field2' => 10})
+      d2.feed({'field1' => 'record11', 'field2' => 11})
+      d2.feed({'field1' => 'record12', 'field2' => 12})
+    end
+    filtered = d2.filtered
+    assert_equal 4, filtered.length
+
+    assert_equal 'record3', filtered[0][1]['field1']
+    assert_equal 'record6', filtered[1][1]['field1']
+    assert_equal 'record9', filtered[2][1]['field1']
+    assert_equal 'record12', filtered[3][1]['field1']
+  end
+
+  def test_filter_minimum_rate
+    config = %[
+interval 10
+sample_unit tag
+minimum_rate_per_min 100
+]
+    d = create_driver(config)
+    time = Time.parse("2012-01-02 13:14:15").to_i
+    d.run(default_tag: 'input.hoge3') do
+      (1..100).each do |t|
+        d.feed(time, {'times' => t, 'data' => 'x'})
+      end
+      (101..130).each do |t|
+        d.feed(time, {'times' => t, 'data' => 'y'})
+      end
+    end
+    filtered = d.filtered
+    assert_equal 103, filtered.length
+    assert_equal ((1..100).map(&:to_i) + [110, 120, 130]), filtered.map{|_time,r| r['times']}
+    assert_equal (['x']*100 + ['y']*3), filtered.map{|_time,r| r['data']}
+  end
+
+  def test_filter_minimum_rate_expire
+    config = %[
+interval 10
+sample_unit tag
+minimum_rate_per_min 10
+]
+    d = create_driver(config)
+    time = Time.parse("2012-01-02 13:14:15").to_i
+    d.run(default_tag: 'input.hoge4') do
+      (1..30).each do |t|
+        d.feed(time, {'times' => t, 'data' => 'x'})
+      end
+    end
+    filtered = d.filtered
+    assert_equal 12, filtered.length
+    assert_equal ((1..10).map(&:to_i)+[20,30]), filtered.map{|_time,r| r['times']}
+    assert_equal (['x']*12), filtered.map{|_time,r| r['data']}
+  end
+
+  def test_filer_with_record_accessor
+    d2 = create_driver(%[
+      interval 3
+      sample_unit field3
+    ])
+    time = Time.parse("2012-01-02 13:14:15").to_i
+    d2.run(default_tag: 'input.hoge2') do
+      (1..12).each do |i|
+        [1,2].each do |sample_vaule|
+          d2.feed({'field1' => "record#{i}", 'field2' => i, 'field3' => sample_vaule})
+        end
+      end
+    end
+    filtered = d2.filtered
+    assert_equal 8, filtered.length
+
+    assert_equal 'record3', filtered[0][1]['field1']
+    assert_equal 1, filtered[0][1]['field3']
+    assert_equal 'record3', filtered[1][1]['field1']
+    assert_equal 2, filtered[1][1]['field3']
+    assert_equal 'record6', filtered[2][1]['field1']
+    assert_equal 1, filtered[2][1]['field3']
+    assert_equal 'record6', filtered[3][1]['field1']
+    assert_equal 2, filtered[3][1]['field3']
+    assert_equal 'record9', filtered[4][1]['field1']
+    assert_equal 1, filtered[4][1]['field3']
+    assert_equal 'record9', filtered[5][1]['field1']
+    assert_equal 2, filtered[5][1]['field3']
+    assert_equal 'record12', filtered[6][1]['field1']
+    assert_equal 1, filtered[6][1]['field3']
+    assert_equal 'record12', filtered[7][1]['field1']
+    assert_equal 2, filtered[7][1]['field3']
+  end
+
+  def test_filter_all
+    config = %[
+      interval 10
+      sample_unit all
+    ]
+    d1 = create_driver(config)
+    time = Time.parse("2012-01-02 13:14:15").to_i
+    d1.run do
+      6.times do |i|
+        [0,1].each do |j|
+          d1.feed("input.hoge#{2*i+j}", time, {'field1' => "record#{2*i+j+1}"})
+        end
+      end
+    end
+    filtered = d1.filtered
+    assert_equal 1, filtered.length
+    assert_equal 'record10', filtered[0][1]['field1']
+  end
+end

data/test/plugin/test_out_sampling_filter.rb

@@ -1,10 +1,11 @@
 require 'helper'
+require 'fluent/test/driver/output'
 
 class SamplingFilterOutputTest < Test::Unit::TestCase
   def setup
     Fluent::Test.setup
   end
-  
+
   CONFIG = %[
     interval 10
     sample_unit tag
@@ -12,13 +13,13 @@ class SamplingFilterOutputTest < Test::Unit::TestCase
     add_prefix sampled
   ]
 
-  def create_driver(conf=CONFIG,tag='test')
-    Fluent::Test::OutputTestDriver.new(Fluent::SamplingFilterOutput, tag).configure(conf)
+  def create_driver(conf=CONFIG)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::SamplingFilterOutput).configure(conf)
   end
 
   def test_configure
     assert_raise(Fluent::ConfigError) {
-      d = create_driver('')
+      create_driver('')
     }
     d = create_driver %[
       interval 5
@@ -33,7 +34,7 @@ class SamplingFilterOutputTest < Test::Unit::TestCase
       interval 1000
       sample_unit all
       remove_prefix test
-      add_prefix output      
+      add_prefix output
     ]
     assert_equal 1000, d.instance.interval
     assert_equal :all, d.instance.sample_unit
@@ -42,54 +43,54 @@ class SamplingFilterOutputTest < Test::Unit::TestCase
   end
 
   def test_emit
-    d1 = create_driver(CONFIG, 'input.hoge1')
+    d1 = create_driver(CONFIG)
     time = Time.parse("2012-01-02 13:14:15").to_i
-    d1.run do
-      d1.emit({'field1' => 'record1', 'field2' => 1})
-      d1.emit({'field1' => 'record2', 'field2' => 2})
-      d1.emit({'field1' => 'record3', 'field2' => 3})
-      d1.emit({'field1' => 'record4', 'field2' => 4})
-      d1.emit({'field1' => 'record5', 'field2' => 5})
-      d1.emit({'field1' => 'record6', 'field2' => 6})
-      d1.emit({'field1' => 'record7', 'field2' => 7})
-      d1.emit({'field1' => 'record8', 'field2' => 8})
-      d1.emit({'field1' => 'record9', 'field2' => 9})
-      d1.emit({'field1' => 'record10', 'field2' => 10})
-      d1.emit({'field1' => 'record11', 'field2' => 11})
-      d1.emit({'field1' => 'record12', 'field2' => 12})
+    d1.run(default_tag: 'input.hoge1') do
+      d1.feed(time, {'field1' => 'record1', 'field2' => 1})
+      d1.feed(time, {'field1' => 'record2', 'field2' => 2})
+      d1.feed(time, {'field1' => 'record3', 'field2' => 3})
+      d1.feed(time, {'field1' => 'record4', 'field2' => 4})
+      d1.feed(time, {'field1' => 'record5', 'field2' => 5})
+      d1.feed(time, {'field1' => 'record6', 'field2' => 6})
+      d1.feed(time, {'field1' => 'record7', 'field2' => 7})
+      d1.feed(time, {'field1' => 'record8', 'field2' => 8})
+      d1.feed(time, {'field1' => 'record9', 'field2' => 9})
+      d1.feed(time, {'field1' => 'record10', 'field2' => 10})
+      d1.feed(time, {'field1' => 'record11', 'field2' => 11})
+      d1.feed(time, {'field1' => 'record12', 'field2' => 12})
     end
-    emits = d1.emits
-    assert_equal 1, emits.length
-    assert_equal 'sampled.hoge1', emits[0][0] # tag
-    assert_equal 'record10', emits[0][2]['field1']
-    assert_equal 10, emits[0][2]['field2']
+    events = d1.events
+    assert_equal 1, events.length
+    assert_equal 'sampled.hoge1', events[0][0] # tag
+    assert_equal 'record10', events[0][2]['field1']
+    assert_equal 10, events[0][2]['field2']
 
     d2 = create_driver(%[
       interval 3
-    ], 'input.hoge2')
+    ])
     time = Time.parse("2012-01-02 13:14:15").to_i
-    d2.run do
-      d2.emit({'field1' => 'record1', 'field2' => 1})
-      d2.emit({'field1' => 'record2', 'field2' => 2})
-      d2.emit({'field1' => 'record3', 'field2' => 3})
-      d2.emit({'field1' => 'record4', 'field2' => 4})
-      d2.emit({'field1' => 'record5', 'field2' => 5})
-      d2.emit({'field1' => 'record6', 'field2' => 6})
-      d2.emit({'field1' => 'record7', 'field2' => 7})
-      d2.emit({'field1' => 'record8', 'field2' => 8})
-      d2.emit({'field1' => 'record9', 'field2' => 9})
-      d2.emit({'field1' => 'record10', 'field2' => 10})
-      d2.emit({'field1' => 'record11', 'field2' => 11})
-      d2.emit({'field1' => 'record12', 'field2' => 12})
+    d2.run(default_tag: 'input.hoge2') do
+      d2.feed(time, {'field1' => 'record1', 'field2' => 1})
+      d2.feed(time, {'field1' => 'record2', 'field2' => 2})
+      d2.feed(time, {'field1' => 'record3', 'field2' => 3})
+      d2.feed(time, {'field1' => 'record4', 'field2' => 4})
+      d2.feed(time, {'field1' => 'record5', 'field2' => 5})
+      d2.feed(time, {'field1' => 'record6', 'field2' => 6})
+      d2.feed(time, {'field1' => 'record7', 'field2' => 7})
+      d2.feed(time, {'field1' => 'record8', 'field2' => 8})
+      d2.feed(time, {'field1' => 'record9', 'field2' => 9})
+      d2.feed(time, {'field1' => 'record10', 'field2' => 10})
+      d2.feed(time, {'field1' => 'record11', 'field2' => 11})
+      d2.feed(time, {'field1' => 'record12', 'field2' => 12})
     end
-    emits = d2.emits
-    assert_equal 4, emits.length
-    assert_equal 'sampled.input.hoge2', emits[0][0] # tag
+    events = d2.events
+    assert_equal 4, events.length
+    assert_equal 'sampled.input.hoge2', events[0][0] # tag
 
-    assert_equal 'record3', emits[0][2]['field1']
-    assert_equal 'record6', emits[1][2]['field1']
-    assert_equal 'record9', emits[2][2]['field1']
-    assert_equal 'record12', emits[3][2]['field1']
+    assert_equal 'record3', events[0][2]['field1']
+    assert_equal 'record6', events[1][2]['field1']
+    assert_equal 'record9', events[2][2]['field1']
+    assert_equal 'record12', events[3][2]['field1']
   end
 
   def test_minimum_rate
@@ -99,27 +100,26 @@ sample_unit tag
 remove_prefix input
 minimum_rate_per_min 100
 ]
-    d = create_driver(config, 'input.hoge3')
+    d = create_driver(config)
     time = Time.parse("2012-01-02 13:14:15").to_i
-    d.run do
+    d.run(default_tag: 'input.hoge3') do
       (1..100).each do |t|
-        d.emit({'times' => t, 'data' => 'x'})
+        d.feed(time, {'times' => t, 'data' => 'x'})
       end
       (101..130).each do |t|
-        d.emit({'times' => t, 'data' => 'y'})
+        d.feed(time, {'times' => t, 'data' => 'y'})
       end
     end
-    emits = d.emits
-    assert_equal 103, emits.length
-    assert_equal 'sampled.hoge3', emits[0][0]
-    assert_equal ((1..100).map(&:to_i) + [110, 120, 130]), emits.map{|t,time,r| r['times']}
-    assert_equal (['x']*100 + ['y']*3), emits.map{|t,time,r| r['data']}
+    events = d.events
+    assert_equal 103, events.length
+    assert_equal 'sampled.hoge3', events[0][0]
+    assert_equal ((1..100).map(&:to_i) + [110, 120, 130]), events.map{|_tag,_time,r| r['times']}
+    assert_equal (['x']*100 + ['y']*3), events.map{|_tag,_time,r| r['data']}
 
   end
   def test_minimum_rate_expire
     # hey, this test needs 60 seconds....
-    assert_equal 1, 1
-    return
+    omit("this test needs 60 seconds....") unless ENV["EXECLONGTEST"]
 
     config = %[
 interval 10
@@ -127,39 +127,39 @@ sample_unit tag
 remove_prefix input
 minimum_rate_per_min 10
 ]
-    d = create_driver(config, 'input.hoge4')
+    d = create_driver(config)
     time = Time.parse("2012-01-02 13:14:15").to_i
-    d.run do
+    d.run(default_tag: 'input.hoge4') do
       (1..100).each do |t|
-        d.emit({'times' => t, 'data' => 'x'})
+        d.feed(time, {'times' => t, 'data' => 'x'})
       end
       sleep 60
       (101..130).each do |t|
-        d.emit({'times' => t, 'data' => 'y'})
+        d.feed(time+60, {'times' => t, 'data' => 'y'})
       end
     end
-    emits = d.emits
-    # assert_equal (19 + 12), emits.length
-    assert_equal 'sampled.hoge4', emits[0][0]
-    assert_equal ((1..10).map(&:to_i)+[20,30,40,50,60,70,80,90,100]+(101..110).map(&:to_i)+[120,130]), emits.map{|t,time,r| r['times']}
-    assert_equal (['x']*19 + ['y']*12), emits.map{|t,time,r| r['data']}
+    events = d.events
+    # assert_equal (19 + 12), events.length
+    assert_equal 'sampled.hoge4', events[0][0]
+    assert_equal ((1..10).map(&:to_i)+[20,30,40,50,60,70,80,90,100]+(101..110).map(&:to_i)+[120,130]), events.map{|_tag,_time,r| r['times']}
+    assert_equal (['x']*19 + ['y']*12), events.map{|_tag,_time,r| r['data']}
   end
 
   def test_without_add_prefix_but_remove_prefix
     config = %[
 interval 10
-add_prefix  # empty
+add_prefix
 remove_prefix input
 ]
-    d = create_driver(config, 'input.hoge3')
+    d = create_driver(config)
     time = Time.parse("2012-01-02 13:14:15").to_i
-    d.run do
+    d.run(default_tag: 'input.hoge3') do
       (1..100).each do |t|
-        d.emit({'times' => t, 'data' => 'x'})
+        d.feed(time, {'times' => t, 'data' => 'x'})
       end
     end
-    emits = d.emits
-    assert_equal 10, emits.length
-    assert_equal 'hoge3', emits[0][0]
+    events = d.events
+    assert_equal 10, events.length
+    assert_equal 'hoge3', events[0][0]
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-sampling-filter
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 1.2.0
 platform: ruby
 authors:
 - TAGOMORI Satoshi
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2014-03-07 00:00:00.000000000 Z
+date: 2021-06-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -24,20 +24,40 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.1.0
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.14.12
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.14.12
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
 description: fluentd plugin to pickup sample data from matched massages
 email:
 - tagomoris@gmail.com
@@ -54,14 +74,16 @@ files:
 - README.md
 - Rakefile
 - fluent-plugin-sampling-filter.gemspec
+- lib/fluent/plugin/filter_sampling.rb
 - lib/fluent/plugin/out_sampling_filter.rb
 - test/helper.rb
+- test/plugin/test_filter_sampling.rb
 - test/plugin/test_out_sampling_filter.rb
 homepage: https://github.com/tagomoris/fluent-plugin-sampling-filter
 licenses:
-- APLv2
+- Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -76,11 +98,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.2.2
-signing_key: 
+rubygems_version: 3.2.3
+signing_key:
 specification_version: 4
 summary: fluentd plugin to pickup sample data from matched massages
 test_files:
 - test/helper.rb
+- test/plugin/test_filter_sampling.rb
 - test/plugin/test_out_sampling_filter.rb