fluent-plugin-querycombiner 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 65fedb30907434824303a1a024616f1bc3431d3e
-  data.tar.gz: 12a5292be841fc2c060892f0fc588a0172ce01df
+  metadata.gz: 1124703306d418f033d3ff5127b918532e8b3930
+  data.tar.gz: df04cbd16f7465ec26fe3acadffa687163da88b2
 SHA512:
-  metadata.gz: fee92e2a0ffbe5c606663d55c7de4773ebd5d44b976f3523cd467dde981347a87ed752f803c0e4206070ca025a6db478703a7e0b4662771289467ed831132c57
-  data.tar.gz: 546fa4062b3ab8e6b2820c29062f3ad615b3ab43e1acc5209570e614c68ae4479241a73fe3b090064e1b6b424acd5734d17b52d3b72caf200193468bc315980f
+  metadata.gz: 82d7aef42eacf95e1f3282205e872523d9f00f742c606820e51358d3710966a76e5a640223a707facea22d18d24bb7e384c7e4c2b9dc9b701d2931b412ab9af2
+  data.tar.gz: 26a39c63fb7b00e9e143485ed820739417b27fa99b7f424e6f40f95251d710622228cf2c0afe68c58dd465c1affa3115b4d0835c27484ebe0746484738b7a4dc

data/README.md

@@ -78,7 +78,7 @@ Combined results will be:
 
 ### Replace some field names
 
-If messages has the same fields, these are overwritten in the combination process. You can use `replace` sentence in `<catch>` and `<dump>` blocks to avoid overwrite such fields.
+If messages has the same fields, these are overwritten in the combination process. You can use `replace` sentence in `<catch>` and `<dump>` blocks to avoid overwriting such fields.
 
 For example, you have some event messages like:
 
@@ -113,7 +113,6 @@ You can keep `time` fields which defined both `event-start` and `event-finish` b
   <catch>
     condition     status == 'event-start'
     replace       time => time_start
-
   </catch>
 
   <dump>
@@ -135,6 +134,15 @@ Combined results will be:
 }
 ```
 
+You can also replace multiple fields joined by comma(`,`):
+
+```
+<catch>
+  condition     status == 'event-start'
+  replace       time => time_start, condition => condition_start
+</catch>
+```
+
 ### \<release\> block
 
 In previous examples, messages with `"status": "event-start"` will be watched by plugin immediately.
@@ -166,7 +174,7 @@ You cannot use `replace` sentence in the `<release>` block.
 
 ### \<prolong\> block
 
-Suppose your `query_ttl` is **600** (10 minutes) and almost events are finished within **10 minutes**. But occasionally very-long events occur which finish about **1 hours**. These very-long events send `status: 'event-continue'` messages every 5 minutes for keep-alive.
+Suppose your `query_ttl` is **600** (10 minutes) and almost events are finished within **10 minutes**. But occasionally very-long events occur which finish about **1 hour**. These very-long events send `status: 'event-continue'` messages every 5 minutes for keep-alive.
 
 In this case you can use `<prolong>` block to reset expired time.
 
@@ -180,6 +188,34 @@ You cannot use `replace` sentence in the `<prolong>` block.
 
 Also you cannot combine messages which defined `<prolong>` blocks.
 
+### Record time of the event
+
+If you combine events, time of the events will be lost except defined in `<dump>` block.
+
+If you want record time of the event, you can define `time` sentence in `<catch>` and `<dump>` blocks.
+
+For example, if you configure your fluentd configuration like below,
+
+```
+  <catch>
+    condition     status == 'event-start'
+    replace       time => time_start, condition => condition_start
+    time          time-catch
+  </catch>
+```
+
+you can record time in `time-catch` field in the result.
+
+```
+{
+  "event_id":     "01234567",
+  "status":       "event-finish",
+  "time-catch":   1414715801.112015,
+}
+```
+
+You can set time formats by `time_format` configuration.
+
 
 ## Configuration
 
@@ -193,7 +229,7 @@ The basic information for connecting to Redis. By default it's **redis://127.0.0
 How many times should the plugin retry when performing a redis operation before raising a error.
 By default it's 3.
 
-### querl_ttl
+### query_ttl
 The inactive expire time in seconds. By default it's **1800** (30 minutes).
 
 ### buffer_size
@@ -202,7 +238,6 @@ The max queries to store in redis. By default it's **1000**.
 ### remove_interval
 The interval time to delete expired or overflowed queries which configured by `query_ttl` and `buffer_size`. By default it's 10 [sec].
 
-
 ### redis_key_prefix
 
 The key prefix for data stored in Redis. By default it's `query_combiner:`.
@@ -212,11 +247,19 @@ The key prefix for data stored in Redis. By default it's `query_combiner:`.
 Indicates how to extract the query identity from event record.
 It can be set as a single field name or multiple field names join by comma (`,`).
 
+### time_format
+
+The time format for recording time of the events. Default is `$time` which holds event time. You can also use [Ruby's Time module](http://www.ruby-doc.org/core/Time.html).
+If you want write ISO8601 format (e.g. `2014-10-31T09:32:57+09:00`), you can configure like below.
+
+```
+time_format     Time.at($time).iso8601
+```
+
 
 ## TODO
 
 - Multi-query combination
-- Support hyphen `-` and dollar `$` contained field names
 
 
 ## Copyright

data/fluent-plugin-querycombiner.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-querycombiner"
-  spec.version       = "0.0.1"
+  spec.version       = "0.0.2"
   spec.authors       = ["Takahiro Kamatani"]
   spec.email         = ["buhii314@gmail.com"]
   spec.description   = %q{Fluent plugin to combine multiple queries.}

data/lib/fluent/plugin/out_query_combiner.rb

@@ -14,6 +14,8 @@ module Fluent
     config_param :query_ttl, :integer, :default => 1800
     config_param :buffer_size, :integer, :default => 1000
 
+    config_param :time_format, :string, :default => '$time'
+
     config_param :flush_interval, :integer, :default => 60
     config_param :remove_interval, :integer, :default => 10
     config_param :tag, :string, :default => "query_combiner"
@@ -24,6 +26,7 @@ module Fluent
       require 'msgpack'
       require 'json'
       require 'rubygems'
+      require 'time'
     end
 
     def configure(conf)
@@ -34,6 +37,19 @@ module Fluent
 
       @query_identify = @query_identify.split(',').map { |qid| qid.strip }
 
+      # functions for time format
+      def create_time_formatter(expr)
+        begin
+          f = eval('lambda {|__arg_time__| ' + expr.gsub("$time", "__arg_time__") + '}')
+          return f
+        rescue SyntaxError
+          raise Fluent::ConfigError, "SyntaxError at time_format `#{expr}`"
+        end
+      end
+      @_time_formatter = create_time_formatter(@time_format)
+
+      @_time_keys = {}
+
       # Create functions for each conditions
       @_cond_funcs = {}
       @_replace_keys = {
@@ -88,6 +104,16 @@ module Fluent
             else
               raise Fluent::ConfigError, "`replace` configuration in #{element.name}: only allowed in `catch` and `dump`"
             end
+
+          elsif var == 'time'
+            if %w{catch dump}.include? element.name
+              @_time_keys[element.name] = expr
+            else
+              raise Fluent::ConfigError, "`time` configuration in #{element.name}: only allowed in `catch` and `dump`"
+            end
+
+          else
+            raise Fluent::ConfigError, "Unknown configuration `#{var}` in #{element.name}"
           end
         }
       }
@@ -164,6 +190,11 @@ module Fluent
         record[after] = record[before]
         record.delete(before)
       }
+      # add time key if configured
+      if @_time_keys.has_key? 'catch'
+        record[@_time_keys['catch']] = @_time_formatter.call(time)
+      end
+
       # save record
       tryOnRedis 'set',    @redis_key_prefix + qid, JSON.dump(record)
       # update qid's timestamp
@@ -179,7 +210,7 @@ module Fluent
       end
     end
 
-    def do_dump(qid, record)
+    def do_dump(qid, record, time)
       if (tryOnRedis 'exists', @redis_key_prefix + qid)
         # replace record keys
         @_replace_keys['dump'].each_pair { |before, after|
@@ -187,6 +218,11 @@ module Fluent
           record.delete(before)
         }
 
+        # add time key if configured
+        if @_time_keys.has_key? 'dump'
+          record[@_time_keys['dump']] = @_time_formatter.call(time)
+        end
+
         # emit
         catched_record = JSON.load(tryOnRedis('get', @redis_key_prefix + qid))
         combined_record = catched_record.merge(record)
@@ -229,7 +265,7 @@ module Fluent
                 when "prolong"
                   do_prolong(qid, time)
                 when "dump"
-                  do_dump(qid, record)
+                  do_dump(qid, record, time)
                 when "release"
                   do_release(qid)
                 end

data/test/plugin/test_out_query_combiner.rb

@@ -81,6 +81,27 @@ class QueryCombinerOutputTest < Test::Unit::TestCase
       ]
     }
 
+    # `time` configuration only allowed in <catch> and <dump>
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver %[
+        query_identify   event_id
+        <catch>
+          condition   status == 'start'
+          replace     hoge => hoge_start
+          time        time_catch
+        </catch>
+        <release>
+          condition   status == 'error'
+          time        time_release
+        </release>
+        <dump>
+          condition   status == 'finish'
+          replace     hoge => hoge_finish
+          time        time_dump
+        </dump>
+      ]
+    }
+
   end
 
   def test_readme_sample_basic_example
@@ -137,6 +158,36 @@ class QueryCombinerOutputTest < Test::Unit::TestCase
                    "time_finish"=>"2001-02-03T04:15:11Z"}
   end
 
+  def test_readme_sample_replace_multiple_fields
+    d = create_driver %[
+      query_identify  event_id
+      query_ttl       3600   # time to live[sec]
+      buffer_size     1000   # queries
+
+      <catch>
+        condition     status == 'event-start'
+        replace       time => time_start, condition => condition_start
+      </catch>
+
+      <dump>
+        condition     status == 'event-finish'
+        replace       time => time_finish, condition => condition_end
+      </dump>
+    ]
+    time = Time.now.to_i
+    d.emit({"event_id"=>"01234567", "status"=>"event-start", "time"=>"2001-02-03T04:05:06Z", "condition"=>"bad"}, time)
+    d.emit({"event_id"=>"01234567", "status"=>"event-finish", "time"=>"2001-02-03T04:15:11Z", "condition"=>"excellent"}, time)
+    d.run
+    assert_equal d.emits.length, 1
+    assert_equal d.emits[0][2], {
+                   "event_id"=>"01234567",
+                   "status"=>"event-finish",
+                   "time_start"=>"2001-02-03T04:05:06Z",
+                   "condition_start"=>"bad",
+                   "time_finish"=>"2001-02-03T04:15:11Z",
+                   "condition_end"=>"excellent"}
+  end
+
   def test_readme_sample_release
     d = create_driver %[
       query_identify  event_id
@@ -317,6 +368,48 @@ class QueryCombinerOutputTest < Test::Unit::TestCase
     assert_equal d.emits.size, finish_list.size
   end
 
+  def test_time_format_and_configuration
+    d = create_driver %[
+      query_identify  event_id
+      query_ttl       3600   # time to live[sec]
+      buffer_size     1000   # queries
+      time_format     Time.at($time).iso8601(3)
+
+      <catch>
+        condition     status == 'event-start'
+        time          time-catch
+      </catch>
+
+      <dump>
+        condition     status == 'event-finish'
+        time          time-dump
+      </dump>
+
+      <prolong>
+        condition     status == 'event-continue'
+      </prolong>
+
+      <release>
+        condition     status == 'event-error'
+      </release>
+    ]
+    def emit(d, status)
+      d.emit({"event_id"=>"01234567", "status"=>status}, Time.now.to_f)
+    end
+
+    emit(d, "event-start")
+    emit(d, "event-continue")
+    emit(d, "event-continue")
+    emit(d, "event-continue")
+    emit(d, "event-finish")
+    d.run
+
+    assert_equal d.emits.length, 1
+    assert_not_nil d.emits[0][2]['time-catch']
+    assert_not_nil d.emits[0][2]['time-dump']
+
+  end
+
   def test_buffer_size
 
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-querycombiner
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Takahiro Kamatani
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-15 00:00:00.000000000 Z
+date: 2014-10-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake