fluent-plugin-datacalculator 0.0.5 → 0.0.6

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 2c362933d723f8d7de355e0d81e36b9c7116ec7b
+  data.tar.gz: 9be7571d4f6c1b809eda3cb5b69504b05b21c95e
+SHA512:
+  metadata.gz: b8ad5784dea42de7e1fadead741046882e7a6be8ffa02fa3315d70cd67c2ffcb67aa58a4fe1f5f5066c48146b2a83bc5f5075760e3f7bb60c7692af13ffb49d9
+  data.tar.gz: cd0e9b17f3c57900b98ec0b526b0806433657cbb300110ca8033e924e83c3a60dda2086b0154e212f1af37e65f33e68ec39dfe35101531e465a00a422c5be7b4

data/README.md

@@ -0,0 +1,87 @@
+# fluent-plugin-datacalculator
+-----
+
+Simple Calculate messages and summarize the calculated results.
+
+* Summarize calculated results per min/hour/day
+* Summarize calculated results per second (average every min/hour/day)
+* Use finalizer of summarized results (e.g. average)
+
+## Usage
+
+if fluentd set like that:
+
+```
+<match payment.shop>
+  type datacalculator
+  tag result.shop
+  count_interval 5s
+  aggregate all
+  formulas sum = amount * price, cnt = 1, total = amount
+  finalizer ave = cnt > 0 ? 1.00 * sum / cnt : 0
+</match>
+```
+
+recieves bellow messages in a minute:
+
+```
+{"area_id": 1, "mission_id":1, "amount": 3, "price": 100}
+{"area_id": 2, "mission_id":2, "amount": 2, "price": 200}
+{"area_id": 3, "mission_id":1, "amount": 3, "price": 100}
+{"area_id": 4, "mission_id":1, "amount": 4, "price": 300}
+{"area_id": 5, "mission_id":2, "amount": 5, "price": 200}
+{"area_id": 1, "mission_id":1, "amount": 1, "price": 400}
+{"area_id": 4, "mission_id":1, "amount": 2, "price": 200}
+{"area_id": 3, "mission_id":2, "amount": 1, "price": 300}
+```
+
+then output below:
+
+```
+2014-02-26 13:52:28 +0900 result.shop: {"sum":4300.0,"cnt":8,"total":21.0,"ave":537.5}
+```
+
+## Configuration
+
+### Example
+
+```
+<match accesslog.**>
+  type datacalculate
+  unit minute
+  aggregate all
+  fomulas sum = amount * price, amounts = amount
+</match>
+```
+
+If you use finalizer, like this
+
+```
+<match accesslog.**>
+  type datacalculate
+  unit minute
+  aggregate all
+  fomulas sum = amount * price, amounts = amount
+  finalizer average = amounts > 0 ? 1.0 * sum / amounts : 0
+</match>
+```
+
+Finalizer uses the summarized output, so argv in finalizer must exist in left-hand side in fomulas.
+
+### Options
+
+* `count_interval`: aggregate time interval e.g. `5s`, `15m`, `3h`
+* `aggregate`: if set `all` then all matched tags are aggregated. if set `tag` then each tags are aggregated separately (default `tag`).
+* `input_tag_remove_prefix`: option available if you want to remove tag prefix from output field names. This option available when aggregate is set `tag`.
+* `retain_key_combinations`: option available if you want to retain key combination created in previous to next interval (default `true`).
+* `formulas`: define value and function comma separated. values are set in messages.
+* `finalizer`: functions defined are executed aggregated phase. value are set in messages.
+
+## TODO
+
+* multiple finalizer
+
+## Copyright
+
+Copyright:: Copyright (c) 2012- Muddy Dixon
+License::   Apache License, Version 2.0

data/example.conf

@@ -7,6 +7,7 @@
   tag result.quest
   count_interval 5s
   aggregate keys area_id, mission_id
+  retain_key_combinations false
   formulas sum = amount * price, cnt = 1, total = amount
   finalizer ave = cnt > 0 ? 1.00 * sum / cnt : 0
 	<unmatched>

data/fluent-plugin-datacalculator.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-datacalculator"
-  s.version     = "0.0.5"
+  s.version     = "0.0.6"
   s.authors     = ["Muddy Dixon"]
   s.email       = ["muddydixon@gmail.com"]
   s.homepage    = "https://github.com/muddydixon/fluent-plugin-datacalculator"

data/lib/fluent/plugin/out_datacalculator.rb

@@ -10,6 +10,7 @@ class Fluent::DataCalculatorOutput < Fluent::Output
   config_param :input_tag_remove_prefix, :string, :default => nil
   config_param :formulas, :string
   config_param :finalizer, :string, :default => nil
+  config_param :retain_key_combinations, :bool, :default => true
 
   attr_accessor :tick
   attr_accessor :counts
@@ -242,7 +243,11 @@ class Fluent::DataCalculatorOutput < Fluent::Output
   end
 
   def flush(step)
-    flushed, @counts = @counts,count_initialized(@counts.keys.dup)
+    if @retain_key_combinations
+      flushed, @counts = @counts,count_initialized(@counts.keys.dup)
+    else
+      flushed, @counts = @counts,count_initialized
+    end
     generate_output(flushed, step)
   end
 

data/test/plugin/test_out_datacalculator.rb

@@ -240,4 +240,38 @@ class DataCalculatorOutputTest < Test::Unit::TestCase
       assert_equal counts[pat], r['count']
     end
   end
+
+  def test_flush
+    # retain_key_combinations is true
+    d1 = create_driver(%[
+      unit minute
+      aggregate keys area_id, mission_id
+      formulas sum = amount * price, count = 1
+      retain_key_combinations true
+    ], 'test.input')
+    d1.run do
+      60.times do
+        d1.emit({'area_id' => 1, 'mission_id' => 1, 'amount' => 3, 'price' => 100})
+        d1.emit({'area_id' => 2, 'mission_id' => 1, 'amount' => 3, 'price' => 100})
+      end
+    end
+    assert_equal d1.instance.flush(60).size, 2
+    assert_equal d1.instance.flush(60).size, 2
+
+    # retain_key_combinations is false
+    d2 = create_driver(%[
+      unit minute
+      aggregate keys area_id, mission_id
+      formulas sum = amount * price, count = 1
+      retain_key_combinations false
+    ], 'test.input')
+    d2.run do
+      60.times do
+        d2.emit({'area_id' => 1, 'mission_id' => 1, 'amount' => 3, 'price' => 100})
+        d2.emit({'area_id' => 2, 'mission_id' => 1, 'amount' => 3, 'price' => 100})
+      end
+    end
+    assert_equal d2.instance.flush(60).size, 2
+    assert_equal d2.instance.flush(60).size, 0
+  end
 end

metadata

@@ -1,38 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-datacalculator
 version: !ruby/object:Gem::Version
-  version: 0.0.5
-  prerelease: 
+  version: 0.0.6
 platform: ruby
 authors:
 - Muddy Dixon
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-17 00:00:00.000000000 Z
+date: 2014-02-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
-  requirement: &2164769940 !ruby/object:Gem::Requirement
-    none: false
+  requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2164769940
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: fluentd
-  requirement: &2164769420 !ruby/object:Gem::Requirement
-    none: false
+  requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2164769420
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Output filter plugin to calculate messages that matches specified conditions
 email:
 - muddydixon@gmail.com
@@ -43,7 +48,7 @@ files:
 - .gitignore
 - Gemfile
 - LICENSE.txt
-- README.rdoc
+- README.md
 - Rakefile
 - example.conf
 - example.json
@@ -53,27 +58,26 @@ files:
 - test/plugin/test_out_datacalculator.rb
 homepage: https://github.com/muddydixon/fluent-plugin-datacalculator
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: fluent-plugin-datacalculator
-rubygems_version: 1.8.11
+rubygems_version: 2.0.14
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Output filter plugin to calculate messages that matches specified conditions
 test_files:
 - test/helper.rb

data/README.rdoc

@@ -1,52 +0,0 @@
-= fluent-plugin-datacalculator
-
-== Component
-
-=== DataCalculateOutput
-
-Simple Calculate messages and summarize the calculated results.
-
-- Summarize calculated results per min/hour/day 
-- Summarize calculated results per second (average every min/hour/day)
-- Use finalizer of summarized results (e.g. average)
-
-'input_tag_remove_prefix' option available if you want to remove tag prefix from output field names.
-
-== Configuration
-
-=== DataCalculateOutput
-
-    <match accesslog.**>
-      type datacalculate
-      unit minute
-      aggregate all
-      fomulas sum = amount * price, amounts = amount
-    </match>
-
-If you use finalizer, like this
-
-    <match accesslog.**>
-      type datacalculate
-      unit minute
-      aggregate all
-      fomulas sum = amount * price, amounts = amount
-      finalizer average = amounts > 0 ? 1.0 * sum / amounts : 0
-    </match>
-
-Finalizer uses the summarized output, so argv in finalizer must exist in left-hand side in fomulas.
-
-
-
-== TODO
-
-- multiple finalizer
-- patches welcome!
-
-== Thanks
-
-tagomoris's fluent-plugin-datacounter is AWESOME plugin! That's nice!
-
-== Copyright
-
-Copyright:: Copyright (c) 2012- Muddy Dixon
-License::   Apache License, Version 2.0