fluent-plugin-sampling-filter 0.1.1 → 0.1.2

data/VERSION

@@ -1 +1 @@
-0.1.1
+0.1.2

data/fluent-plugin-sampling-filter.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "fluent-plugin-sampling-filter"
-  s.version = "0.1.1"
+  s.version = "0.1.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["TAGOMORI Satoshi"]
-  s.date = "2012-03-29"
+  s.date = "2012-08-08"
   s.description = "fluentd plugin to pickup sample data from matched massages"
   s.email = "tagomoris@gmail.com"
   s.extra_rdoc_files = [
@@ -31,7 +31,7 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://github.com/tagomoris/fluent-plugin-sampling-filter"
   s.require_paths = ["lib"]
-  s.rubygems_version = "1.8.19"
+  s.rubygems_version = "1.8.21"
   s.summary = "fluentd plugin to pickup sample data from matched massages"
 
   if s.respond_to? :specification_version then

data/lib/fluent/plugin/out_sampling_filter.rb

@@ -5,6 +5,7 @@ class Fluent::SamplingFilterOutput < Fluent::Output
   config_param :sample_unit, :string, :default => 'tag'
   config_param :remove_prefix, :string, :default => nil
   config_param :add_prefix, :string, :default => 'sampled'
+  config_param :minimum_rate_per_min, :integer, :default => nil
 
   def configure(conf)
     super
@@ -24,6 +25,7 @@ class Fluent::SamplingFilterOutput < Fluent::Output
                      raise Fluent::ConfigError, "sample_unit allows only 'tag' or 'all'"
                    end
     @counts = {}
+    @resets = {} if @minimum_rate_per_min
   end
 
   def emit_sampled(tag, time_record_pairs)
@@ -48,19 +50,38 @@ class Fluent::SamplingFilterOutput < Fluent::Output
         else
           tag
         end
+
+    pairs = []
+
     # Access to @counts SHOULD be protected by mutex, with a heavy penalty.
     # Code below is not thread safe, but @counts (counter for sampling rate) is not
-    # so serious value (and probably will not be broke...),
+    # so serious value (and probably will not be broken...),
     # then i let here as it is now.
-    pairs = []
-    es.each {|time,record|
-      c = (@counts[t] = @counts.fetch(t, 0) + 1)
-      if c % @interval == 0 
-        pairs.push [time, record]
-        # reset only just before @counts[t] is to be bignum from fixnum
-        @counts[t] = 0 if c > 0x6fffffff
+    if @minimum_rate_per_min
+      unless @resets[t]
+        @resets[t] = Fluent::Engine.now + (60 - rand(30))
       end
-    }
+      if Fluent::Engine.now > @resets[t]
+        @resets[t] = Fluent::Engine.now + 60
+        @counts[t] = 0
+      end
+      es.each do |time,record|
+        c = (@counts[t] = @counts.fetch(t, 0) + 1)
+        if c < @minimum_rate_per_min or c % @interval == 0
+          pairs.push [time, record]
+        end
+      end
+    else
+      es.each do |time,record|
+        c = (@counts[t] = @counts.fetch(t, 0) + 1)
+        if c % @interval == 0
+          pairs.push [time, record]
+          # reset only just before @counts[t] is to be bignum from fixnum
+          @counts[t] = 0 if c > 0x6fffffff
+        end
+      end
+    end
+
     emit_sampled(tag, pairs)
 
     chain.next

data/test/plugin/test_out_sampling_filter.rb

@@ -41,12 +41,6 @@ class SamplingFilterOutputTest < Test::Unit::TestCase
     assert_equal 'output', d.instance.add_prefix
   end
 
-  # CONFIG = %[
-  #   interval 10
-  #   sample_unit tag
-  #   remove_prefix input
-  #   add_prefix sampled
-  # ]
   def test_emit
     d1 = create_driver(CONFIG, 'input.hoge1')
     time = Time.parse("2012-01-02 13:14:15").to_i
@@ -97,4 +91,57 @@ class SamplingFilterOutputTest < Test::Unit::TestCase
     assert_equal 'record9', emits[2][2]['field1']
     assert_equal 'record12', emits[3][2]['field1']
   end
+
+  def test_minimum_rate
+    config = %[
+interval 10
+sample_unit tag
+remove_prefix input
+minimum_rate_per_min 100
+]
+    d = create_driver(config, 'input.hoge3')
+    time = Time.parse("2012-01-02 13:14:15").to_i
+    d.run do
+      (1..100).each do |t|
+        d.emit({'times' => t, 'data' => 'x'})
+      end
+      (101..130).each do |t|
+        d.emit({'times' => t, 'data' => 'y'})
+      end
+    end
+    emits = d.emits
+    assert_equal 103, emits.length
+    assert_equal 'sampled.hoge3', emits[0][0]
+    assert_equal ((1..100).map(&:to_i) + [110, 120, 130]), emits.map{|t,time,r| r['times']}
+    assert_equal (['x']*100 + ['y']*3), emits.map{|t,time,r| r['data']}
+
+  end
+  def test_minimum_rate_expire
+    # hey, this test needs 60 seconds....
+    assert_equal 1, 1
+    return
+
+    config = %[
+interval 10
+sample_unit tag
+remove_prefix input
+minimum_rate_per_min 10
+]
+    d = create_driver(config, 'input.hoge4')
+    time = Time.parse("2012-01-02 13:14:15").to_i
+    d.run do
+      (1..100).each do |t|
+        d.emit({'times' => t, 'data' => 'x'})
+      end
+      sleep 60
+      (101..130).each do |t|
+        d.emit({'times' => t, 'data' => 'y'})
+      end
+    end
+    emits = d.emits
+    # assert_equal (19 + 12), emits.length
+    assert_equal 'sampled.hoge4', emits[0][0]
+    assert_equal ((1..10).map(&:to_i)+[20,30,40,50,60,70,80,90,100]+(101..110).map(&:to_i)+[120,130]), emits.map{|t,time,r| r['times']}
+    assert_equal (['x']*19 + ['y']*12), emits.map{|t,time,r| r['data']}
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-sampling-filter
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-03-29 00:00:00.000000000 Z
+date: 2012-08-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -140,7 +140,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 2842791238956206034
+      hash: 2658451737493219281
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -149,7 +149,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.19
+rubygems_version: 1.8.21
 signing_key: 
 specification_version: 3
 summary: fluentd plugin to pickup sample data from matched massages