fluent-plugin-suppress 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8c0b6ee711914b962c5f5d766cce357ddd110a25
-  data.tar.gz: ce81797b67eb2fb9b7370bae9437c31c6169324d
+  metadata.gz: 932d867f3ee590af09d05ecbc3ffb5011db6d3b6
+  data.tar.gz: c075ee6fdb2bc95efd4609f82cd50300525010a4
 SHA512:
-  metadata.gz: de1ffa0ddd281010c45bc2e1681cb49a02991c64e84e2860b839bce1cb81b48f0f8f5223504590edb04aa56c49d1f77af20ced5076c0a597a8665608ca8fac19
-  data.tar.gz: 1613ba5ab4ea3591f7d24b8c5bb13f05d057c2c5c2c0dabc1159bf936209ccf5ce078e16597bfc0eda4e01befb34e3a2e3fa349ea601f845a302796fa5222034
+  metadata.gz: ce6988ea6023d58966295a038864b3e39973f14c456cda0dfde4c96048dd4fdfbce4dc229f44de34daf8eecc19616edd8a6e3464ceeaec654967a08491751e72
+  data.tar.gz: f0b2d1150f429d5a875d420ca3171bb60c312cf69861b83d117fd1e421c2cb104e10a0caf28ee9231564627958dd4d8d818b6ead62feb1acd5fe5a4cf0024585

data/.travis.yml

@@ -4,5 +4,6 @@ rvm:
   - 2.2
   - 2.3
   - 2.4
+  - 2.5
 gemfile:
   - Gemfile

data/README.md

@@ -15,12 +15,14 @@ fluentd.conf
   @type          suppress
   interval       10
   num            2
+  max_slot_num   100000
   attr_keys      host,message
   add_tag_prefix sp.
 </match>
 ```
 
 In `interval` sec, `num` messages which grouped by `attr_keys` value, add tag prefix "`sp.`" and pass to follow process. Other messages will be removed.
+`max_slot_num` put an upper limit on the length of the internal variable which contains message occurences.
 
 Input messages:
 

data/fluent-plugin-suppress.gemspec

@@ -13,7 +13,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-suppress"
   gem.require_paths = ["lib"]
-  gem.version       = "1.0.0"
+  gem.version       = "1.1.0"
 
   gem.add_runtime_dependency "fluentd", [">= 0.14.8", "< 2"]
   gem.add_development_dependency "rake", ">= 0.9.2"

data/lib/fluent/plugin/filter_suppress.rb

@@ -6,6 +6,7 @@ module Fluent::Plugin
 
     config_param :attr_keys,     :string,  default: nil
     config_param :num,           :integer, default: 3
+    config_param :max_slot_num,  :integer, default: 100000
     config_param :interval,      :integer, default: 300
 
     def configure(conf)
@@ -38,6 +39,13 @@ module Fluent::Plugin
           next
         end
 
+        if @slots.length > @max_slot_num
+          (evict_key, evict_slot) = @slots.shift
+          if evict_slot.last && (evict_slot.last > expired)
+            log.warn "@slots length exceeded @max_slot_num: #{@max_slot_num}. Evicted slot for the key: #{evict_key}"
+          end
+        end
+
         slot.push(time.to_f)
         new_es.add(time, record)
       end

data/lib/fluent/plugin/out_suppress.rb

@@ -10,6 +10,7 @@ module Fluent::Plugin
 
     config_param :attr_keys,     :string,  default: nil
     config_param :num,           :integer, default: 3
+    config_param :max_slot_num,  :integer, default: 100000
     config_param :interval,      :integer, default: 300
 
     def configure(conf)
@@ -52,6 +53,13 @@ module Fluent::Plugin
           next
         end
 
+        if @slots.length > @max_slot_num
+          (evict_key, evict_slot) = @slots.shift
+          if evict_slot.last && (evict_slot.last > expired)
+            log.warn "@slots length exceeded @max_slot_num: #{@max_slot_num}. Evicted slot for the key: #{evict_key}"
+          end
+        end
+
         slot.push(time.to_f)
         _tag = tag.clone
         filter_record(_tag, time, record)

data/test/plugin/test_filter_suppress.rb

@@ -26,6 +26,13 @@ class SuppressFilterTest < Test::Unit::TestCase
     num      2
   ]
 
+  CONFIG_MAX_SLOT_NUM = %[
+    interval       10
+    num            2
+    max_slot_num   3
+    attr_keys      host, message
+  ]
+
   def create_driver(conf = CONFIG)
     Fluent::Test::Driver::Filter.new(Fluent::Plugin::SuppressFilter).configure(conf)
   end
@@ -109,4 +116,30 @@ class SuppressFilterTest < Test::Unit::TestCase
     assert_equal({"id"=>6, "host"=>"web06", "message"=>"6 error!!"}, records[2])
     assert_equal({"id"=>7, "host"=>"web07", "message"=>"7 error!!"}, records[3])
   end
+
+  def test_emit_max_slot_num
+    d = create_driver(CONFIG_MAX_SLOT_NUM)
+    es = Fluent::MultiEventStream.new
+
+    time = event_time("2012-11-22 11:22:33 UTC")
+    es.add(time + 1,  {"id" => 1, "host" => "web01", "message" => "1 error!!"})
+    es.add(time + 2,  {"id" => 2, "host" => "web02", "message" => "2 error!!"})
+    es.add(time + 3,  {"id" => 3, "host" => "web03", "message" => "3 error!!"})
+    es.add(time + 4,  {"id" => 4, "host" => "web01", "message" => "1 error!!"})
+    es.add(time + 5,  {"id" => 5, "host" => "web04", "message" => "4 error!!"})
+    es.add(time + 6,  {"id" => 6, "host" => "web01", "message" => "1 error!!"})
+
+    d.run(default_tag: "test.info") do
+      d.feed(es)
+    end
+    records = d.filtered_records
+
+    assert_equal 6, records.length
+    assert_equal({"id"=>1, "host"=>"web01", "message"=>"1 error!!"}, records[0])
+    assert_equal({"id"=>2, "host"=>"web02", "message"=>"2 error!!"}, records[1])
+    assert_equal({"id"=>3, "host"=>"web03", "message"=>"3 error!!"}, records[2])
+    assert_equal({"id"=>4, "host"=>"web01", "message"=>"1 error!!"}, records[3])
+    assert_equal({"id"=>5, "host"=>"web04", "message"=>"4 error!!"}, records[4])
+    assert_equal({"id"=>6, "host"=>"web01", "message"=>"1 error!!"}, records[5])
+  end
 end

data/test/plugin/test_out_suppress.rb

@@ -27,6 +27,14 @@ class SuppressOutputTest < Test::Unit::TestCase
     add_tag_prefix sp.
   ]
 
+  CONFIG_MAX_SLOT_NUM = %[
+    interval       10
+    num            2
+    max_slot_num   3
+    attr_keys      host, message
+    add_tag_prefix sp.
+  ]
+
   def create_driver(conf = CONFIG)
     Fluent::Test::Driver::Output.new(Fluent::Plugin::SuppressOutput).configure(conf)
   end
@@ -103,4 +111,27 @@ class SuppressOutputTest < Test::Unit::TestCase
     assert_equal ["sp.test.info", time + 12, {"id"=>6, "host"=>"web06", "message"=>"6 error!!"}], events[2]
     assert_equal ["sp.test.info", time + 13, {"id"=>7, "host"=>"web07", "message"=>"7 error!!"}], events[3]
   end
+
+  def test_emit_max_slot_num
+    d = create_driver(CONFIG_MAX_SLOT_NUM)
+
+    time = event_time("2012-11-22 11:22:33 UTC")
+    d.run(default_tag: "test.info") do
+      d.feed(time + 1,  {"id" => 1, "host" => "web01", "message" => "1 error!!"})
+      d.feed(time + 2,  {"id" => 2, "host" => "web02", "message" => "2 error!!"})
+      d.feed(time + 3,  {"id" => 3, "host" => "web03", "message" => "3 error!!"})
+      d.feed(time + 4,  {"id" => 4, "host" => "web01", "message" => "1 error!!"})
+      d.feed(time + 5,  {"id" => 5, "host" => "web04", "message" => "4 error!!"})
+      d.feed(time + 6,  {"id" => 6, "host" => "web01", "message" => "1 error!!"})
+    end
+
+    events = d.events
+    assert_equal 6, events.length
+    assert_equal ["sp.test.info", time + 1,  {"id"=>1, "host"=>"web01", "message"=>"1 error!!"}], events[0]
+    assert_equal ["sp.test.info", time + 2,  {"id"=>2, "host"=>"web02", "message"=>"2 error!!"}], events[1]
+    assert_equal ["sp.test.info", time + 3,  {"id"=>3, "host"=>"web03", "message"=>"3 error!!"}], events[2]
+    assert_equal ["sp.test.info", time + 4,  {"id"=>4, "host"=>"web01", "message"=>"1 error!!"}], events[3]
+    assert_equal ["sp.test.info", time + 5,  {"id"=>5, "host"=>"web04", "message"=>"4 error!!"}], events[4]
+    assert_equal ["sp.test.info", time + 6,  {"id"=>6, "host"=>"web01", "message"=>"1 error!!"}], events[5]
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-suppress
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - FUJIWARA Shunichiro
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-02-15 00:00:00.000000000 Z
+date: 2018-09-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd