fluent-plugin-dedup 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 75f9ab2798dd39e111ecac0a0afd0c1ac964bc05
-  data.tar.gz: aa7545d7df05055174952cf8edfcb87e3985658f
+  metadata.gz: 504ed27a9800ed32acf1e4c2872520394d935ee4
+  data.tar.gz: 3c48385579ce83f21be03e5d6052f3a1bdf59b55
 SHA512:
-  metadata.gz: f688f6d8f185e583cf6327375b40fc4eff8ca665c5d5cfad31106fa7950240c24a57b0c4bc9836f84411cb2f5c7fd302dbbc692ca25c3a74097b1a8046afdf90
-  data.tar.gz: 037a80be99a3156f450aada80bf75da9714ea8d19723cd46bef6207b453bf5ac9d4681a72a7a31857bc0f1847bc558254781c5c1d14b228b49cd88478e16aeba
+  metadata.gz: 35f86e880c21e1b40015d95c1684e2d9117e324fda2781a49fc233dd0dbed6b02894792e049226053a5cabf814a589207ef31a7353ccd5aca34d3bde4def4b9e
+  data.tar.gz: 95726e9ef89719c7cc382748b03842e0eee6872ea6e8693486de26f3202bd173ae35bcac879a8fcb0e89ecc2a940cbcc7df7d45284ea9850fb94a77cb18f17a5

data/fluent-plugin-dedup.gemspec

@@ -1,7 +1,7 @@
 # coding: utf-8
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-dedup"
-  spec.version       = "0.1.0"
+  spec.version       = "0.2.0"
   spec.authors       = ["edvakf"]
   spec.email         = ["taka.atsushi@gmail.com"]
   spec.summary       = %q{fluentd plugin for removing duplicate logs}
@@ -19,4 +19,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "test-unit", "~> 3.0"
 
   spec.add_runtime_dependency "fluentd"
+  spec.add_runtime_dependency "lru_redux"
 end

data/lib/fluent/plugin/out_dedup.rb

@@ -1,10 +1,12 @@
 require 'json'
+require 'lru_redux'
 
 class Fluent::DedupOutput < Fluent::Output
   Fluent::Plugin.register_output('dedup', self)
 
   config_param :key, :string, :default => nil
   config_param :file, :string, :default => nil
+  config_param :cache_per_tag, :size, :default => 1
 
   # Define `log` method for v0.10.42 or earlier
   unless method_defined?(:log)
@@ -24,9 +26,7 @@ class Fluent::DedupOutput < Fluent::Output
   def start
     super
 
-    if not @file.nil? and File.file?(@file)
-      @states = JSON.parse(File.open(@file).read) rescue {}
-    end
+    restore_states
   end
 
   def shutdown
@@ -38,7 +38,6 @@ class Fluent::DedupOutput < Fluent::Output
   def emit(tag, es, chain)
     es.each do |time, record|
       next if dup?(tag, record)
-      update_states(tag, record)
       Fluent::Engine.emit("dedup.#{tag}", time, record)
     end
 
@@ -46,22 +45,45 @@ class Fluent::DedupOutput < Fluent::Output
   end
 
   private
+  def restore_states
+    if not @file.nil? and File.file?(@file)
+      dump = JSON.parse(File.open(@file).read) rescue {}
+      dump.each do |tag, ids|
+        lru = new_lru
+        ids.each {|id| lru[id] = true}
+        @states[tag] = lru
+      end
+    end
+  end
+
   def save_states
     unless @file.nil?
       File.open(@file, 'wb') do |f|
-        f.print(@states.to_json)
+        dump = {}
+        @states.each do |tag, lru|
+          dump[tag] = lru.to_a.map(&:first)
+        end
+        f.print(dump.to_json)
       end
     end
   end
 
   def dup?(tag, record)
-    unless record.include?(@key)
+    is_dup = false
+    if record.include?(@key)
+      @states[tag] = new_lru unless @states.include?(tag)
+      if @states[tag].fetch(record[@key])
+        is_dup = true
+      else
+        @states[tag][record[@key]] = true
+      end
+    else
       log.warn "record does not have key `#{@key}`, record: #{record.to_json}"
     end
-    @states[tag] == record[@key]
+    is_dup
   end
 
-  def update_states(tag, record)
-    @states[tag] = record[@key]
+  def new_lru
+    LruRedux::ThreadSafeCache.new(@cache_per_tag)
   end
 end

data/test/fluent/plugin/test_out_dedup.rb

@@ -89,4 +89,25 @@ class DedupOutputTest < Test::Unit::TestCase
       assert_equal 0, d2.emits.length
     end
   end
+
+  sub_test_case '`cache_per_tag` parameter is present' do
+    test "a record identical to most recent N records is suppressed" do
+      config = %[
+        key           unique_id
+        cache_per_tag 2
+      ]
+
+      d = create_driver(config)
+      d.run do
+        d.emit({'unique_id' => '1'}, Time.now)
+        d.emit({'unique_id' => '1'}, Time.now) # dup
+        d.emit({'unique_id' => '2'}, Time.now)
+        d.emit({'unique_id' => '1'}, Time.now) # dup
+      end
+
+      assert_equal 2, d.emits.length
+      assert_equal '1', d.emits[0][2]['unique_id']
+      assert_equal '2', d.emits[1][2]['unique_id']
+    end
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-dedup
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - edvakf
@@ -66,6 +66,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: lru_redux
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: fluent-plugin-dedup is a fluentd plugin to suppress emission of subsequent
   logs identical to the first one.
 email: