fluent-plugin-querycombiner 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1124703306d418f033d3ff5127b918532e8b3930
-  data.tar.gz: df04cbd16f7465ec26fe3acadffa687163da88b2
+  metadata.gz: a7abce3623ce7f91ee8b41140d8a44a2e0d86621
+  data.tar.gz: 4ff7b64144bab38c734abcc8c9562419f21730fa
 SHA512:
-  metadata.gz: 82d7aef42eacf95e1f3282205e872523d9f00f742c606820e51358d3710966a76e5a640223a707facea22d18d24bb7e384c7e4c2b9dc9b701d2931b412ab9af2
-  data.tar.gz: 26a39c63fb7b00e9e143485ed820739417b27fa99b7f424e6f40f95251d710622228cf2c0afe68c58dd465c1affa3115b4d0835c27484ebe0746484738b7a4dc
+  metadata.gz: 174b614a77bb2599908f609829474149b92b0cf9eef7fde24f85da6d4321e102dc9df476f5385d6c082592db98e8889f3965913d291d9c4af3b2afa39e2d7a19
+  data.tar.gz: 1ed7b35fdc71f39798fe51235c318ab9438d9e780f4dbb316ce448c93825f74f187aa02ff9bd8443f0528c86961dac6deb191148e3408def37b0a81deae6e2d5

data/README.md

@@ -235,8 +235,13 @@ The inactive expire time in seconds. By default it's **1800** (30 minutes).
 ### buffer_size
 The max queries to store in redis. By default it's **1000**.
 
+### continuous_dump
+If you set this variable **true**, your pre-combined queries will not remove even after combined by `<dump>` block. Your pre-combined queries will remove only after their expire times set by `query_ttl`. Also your pre-combined queries will be prolonged if dumped.
+
+By default it's **false**.
+
 ### remove_interval
-The interval time to delete expired or overflowed queries which configured by `query_ttl` and `buffer_size`. By default it's 10 [sec].
+The interval time to delete expired or overflowed queries which configured by `query_ttl` and `buffer_size`. By default it's `10` [sec].
 
 ### redis_key_prefix
 

data/fluent-plugin-querycombiner.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-querycombiner"
-  spec.version       = "0.0.2"
+  spec.version       = "0.0.3"
   spec.authors       = ["Takahiro Kamatani"]
   spec.email         = ["buhii314@gmail.com"]
   spec.description   = %q{Fluent plugin to combine multiple queries.}

data/lib/fluent/plugin/out_query_combiner.rb

@@ -13,6 +13,7 @@ module Fluent
     config_param :query_identify, :string, :default => 'session-id'
     config_param :query_ttl, :integer, :default => 1800
     config_param :buffer_size, :integer, :default => 1000
+    config_param :continuous_dump, :bool, :default => false
 
     config_param :time_format, :string, :default => '$time'
 
@@ -229,7 +230,14 @@ module Fluent
         Fluent::Engine.emit @tag, Fluent::Engine.now, combined_record
 
         # remove qid
-        do_release(qid)
+        if not @continuous_dump
+          do_release(qid)
+        else
+          # continuous_dump will prolong qid's TTL.
+          tryOnRedis 'zadd', @redis_key_prefix, time, qid
+          tryOnRedis 'expire', @redis_key_prefix + qid, @query_ttl
+        end
+
       end
     end
 

data/test/plugin/test_out_query_combiner.rb

@@ -31,6 +31,19 @@ class QueryCombinerOutputTest < Test::Unit::TestCase
     @redis.quit
   end
 
+  def tryOnRedis(method, *args)
+    tries = 0
+    begin
+      @redis.send(method, *args) if @redis.respond_to? method
+    rescue Redis::CommandError => e
+      tries += 1
+      # retry 3 times
+      retry if tries <= @redis_retry
+      $log.warn %Q[redis command retry failed : #{method}(#{args.join(', ')})]
+      raise e.message
+    end
+  end
+
   def create_driver(conf, tag='test')
     Fluent::Test::BufferedOutputTestDriver.new(Fluent::QueryCombinerOutput, tag).configure(conf)
   end
@@ -410,12 +423,52 @@ class QueryCombinerOutputTest < Test::Unit::TestCase
 
   end
 
+  def test_query_ttl
+
+  end
+
   def test_buffer_size
 
   end
 
-  def test_query_ttl
+  def test_continuous_dump
+    d = create_driver %[
+      redis_key_prefix test_query_combiner:
+
+      query_identify  event_id
+      query_ttl       5   # time to live[sec]
+      buffer_size     1000   # queries
+      continuous_dump true
+      remove_interval 5
+
+      <catch>
+        condition     status == 'event-start'
+      </catch>
+
+      <dump>
+        condition     status == 'event-finish'
+      </dump>
+
+      <release>
+        condition     status == 'event-error'
+      </release>
+    ]
+    assert_equal false, (tryOnRedis 'exists', "test_query_combiner:id000")
+
+    d.emit({"event_id"=>"id000", "status"=>"event-start",  "key_init"=>"init"}, Time.now.to_f)
+    d.emit({"event_id"=>"id000", "status"=>"event-finish"}, Time.now.to_f)
+    d.emit({"event_id"=>"id000", "status"=>"event-finish", "key_fin1"=>"fin1"}, Time.now.to_f)
+    d.emit({"event_id"=>"id000", "status"=>"event-finish", "key_fin2"=>"fin2"}, Time.now.to_f)
+    d.run
+
+    assert_equal d.emits.length, 3
+    assert_equal d.emits[0][2], {"event_id"=>"id000", "status"=>"event-finish", "key_init"=>"init"}
+    assert_equal d.emits[1][2], {"event_id"=>"id000", "status"=>"event-finish", "key_init"=>"init", "key_fin1"=>"fin1"}
+    assert_equal d.emits[2][2], {"event_id"=>"id000", "status"=>"event-finish", "key_init"=>"init", "key_fin2"=>"fin2"}
 
+    assert_equal true, (tryOnRedis 'exists', "test_query_combiner:id000")
+    sleep 6
+    assert_equal false, (tryOnRedis 'exists', "test_query_combiner:id000")
   end
 
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-querycombiner
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - Takahiro Kamatani
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-10-31 00:00:00.000000000 Z
+date: 2014-12-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake