fluent-plugin-dedup 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 504ed27a9800ed32acf1e4c2872520394d935ee4
-  data.tar.gz: 3c48385579ce83f21be03e5d6052f3a1bdf59b55
+  metadata.gz: a92bdeb92dfea752942d7e900ccfc3f3533b8413
+  data.tar.gz: bbf4d1075267b6aea71581d81575a49d6f8a7a21
 SHA512:
-  metadata.gz: 35f86e880c21e1b40015d95c1684e2d9117e324fda2781a49fc233dd0dbed6b02894792e049226053a5cabf814a589207ef31a7353ccd5aca34d3bde4def4b9e
-  data.tar.gz: 95726e9ef89719c7cc382748b03842e0eee6872ea6e8693486de26f3202bd173ae35bcac879a8fcb0e89ecc2a940cbcc7df7d45284ea9850fb94a77cb18f17a5
+  metadata.gz: eb8019884f0c506f9bb7a2d10ef5ac1a91db01537c59aa863c219a5eccb65ddf7c80596bef349433c8e9278f3aeab42b139444e10d4d9447f78accd4a8048fa2
+  data.tar.gz: 13074c5f076328e3a3993655db17c8083140c98640825da49e6721362f20084bfead1389920071de924ebf910b39dfa702069c85bdfb39fd1029456ff020e62e

data/.travis.yml

@@ -1,3 +1,4 @@
 language: ruby
 rvm:
   - 2.0.0
+  - 2.1.2

data/README.md

@@ -21,6 +21,8 @@ It's useful when the output of a command executed by `in_exec` only returns the
       type dedup
       key  unique_id # required
       file /tmp/dedup_state.json # optional. If set, saves the state to the file.
+      cache_per_tag 10 # optional. If set, recent logs up to this number is cached.
+      cache_ttl 600 # optional. If set, cache entries is expired in this TTL.
     </match>
 
     <match dedup.some.thing>
@@ -31,6 +33,10 @@ All logs that are processed by this plugin will have tag prefix `dedup`.
 
 If the optional `file` parameter is set, it dumps the state during shutdown and loads on start, so that it can still dedup after reload.
 
+If the optional `cache_per_tag` is set, it caches N recently appeared logs (only caches `unique_id` in this example) and compared to new logs.
+
+If the optional `cache_ttl` is set, it evicts cache entries in a specific amount of time.
+
 ## Testing
 
     bundle install

data/fluent-plugin-dedup.gemspec

@@ -1,7 +1,7 @@
 # coding: utf-8
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-dedup"
-  spec.version       = "0.2.0"
+  spec.version       = "0.3.0"
   spec.authors       = ["edvakf"]
   spec.email         = ["taka.atsushi@gmail.com"]
   spec.summary       = %q{fluentd plugin for removing duplicate logs}
@@ -17,6 +17,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", "~> 1.6"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "test-unit", "~> 3.0"
+  spec.add_development_dependency "timecop"
 
   spec.add_runtime_dependency "fluentd"
   spec.add_runtime_dependency "lru_redux"

data/lib/fluent/plugin/out_dedup.rb

@@ -7,6 +7,7 @@ class Fluent::DedupOutput < Fluent::Output
   config_param :key, :string, :default => nil
   config_param :file, :string, :default => nil
   config_param :cache_per_tag, :size, :default => 1
+  config_param :cache_ttl, :integer, :default => 0
 
   # Define `log` method for v0.10.42 or earlier
   unless method_defined?(:log)
@@ -18,8 +19,6 @@ class Fluent::DedupOutput < Fluent::Output
     unless conf.include?('key')
       raise Fluent::ConfigError, "config parameter `key` is required"
     end
-    @key = conf['key']
-    @file = conf['file']
     @states = {}
   end
 
@@ -84,6 +83,10 @@ class Fluent::DedupOutput < Fluent::Output
   end
 
   def new_lru
-    LruRedux::ThreadSafeCache.new(@cache_per_tag)
+    if 0 < @cache_ttl
+      LruRedux::TTL::ThreadSafeCache.new(@cache_per_tag, @cache_ttl)
+    else
+      LruRedux::ThreadSafeCache.new(@cache_per_tag)
+    end
   end
 end

data/test/fluent/plugin/test_out_dedup.rb

@@ -1,4 +1,5 @@
 require 'helper'
+require 'timecop'
 
 class DedupOutputTest < Test::Unit::TestCase
   def setup
@@ -110,4 +111,38 @@ class DedupOutputTest < Test::Unit::TestCase
       assert_equal '2', d.emits[1][2]['unique_id']
     end
   end
+
+  sub_test_case '`cache_ttl` parameter is present' do
+    setup do
+      Timecop.freeze(Time.now)
+    end
+
+    teardown do
+      Timecop.return
+    end
+
+    test "a record identical to most recent N records is suppressed with TTL option" do
+      config = %[
+        key           unique_id
+        cache_per_tag 2
+        cache_ttl     60
+      ]
+
+      d = create_driver(config)
+      d.run do
+        d.emit({'unique_id' => '1'}, Time.now)
+        d.emit({'unique_id' => '1'}, Time.now) # dup
+        Timecop.freeze(Time.now + 59)
+        d.emit({'unique_id' => '1'}, Time.now) # dup
+        Timecop.freeze(Time.now + 1) # expires cache
+        d.emit({'unique_id' => '1'}, Time.now)
+        Timecop.freeze(Time.now + 1)
+        d.emit({'unique_id' => '1'}, Time.now) # dup
+      end
+
+      assert_equal 2, d.emits.length
+      assert_equal '1', d.emits[0][2]['unique_id']
+      assert_equal '1', d.emits[1][2]['unique_id']
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-dedup
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - edvakf
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-12-07 00:00:00.000000000 Z
+date: 2015-11-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -52,6 +52,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement