fluent-plugin-calc 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 509139d04a8c636beb1a9da08e6ee48df90b3a35
-  data.tar.gz: 47bb1eb2987b1fe83c9bac58709eaffcd19fc28f
+  metadata.gz: 0f8d625101f66b98091f034d26d6bdb61332f507
+  data.tar.gz: 5e65dc06ab15e2ff39d19cdc287fd71532a4a6d4
 SHA512:
-  metadata.gz: 7d4c04775644c8fe7d383e1b85ed803313d9f7e6ef6df4ba2d1f2b8421f8a79287bac9098d370fb8b8a9e78302ab5ec601b9135e16f09ff6e979064ebc7cc709
-  data.tar.gz: 40e571e79841859b9ca6d09b128f58ddc15f148974b59f9b448278647620bf1dd8bc439675bf8faddd9daf6824cedd27b573bf45566cb6b84672666d4313e261
+  metadata.gz: c9f94407643311294a2d6014dfbbaae0682e787cfa057934025a3f23531a35d9fd2665ca279cd875bc9dca3e18c17f5ba617d5dc2ce43b2cdf45f28cb1679ad3
+  data.tar.gz: e3fd1ab151b4cee10922e6b553a88f931ec0d4ab70a052c4f92d111f7dbcb6cf97e37c5373150d79f91bb5e75aa4a5b78542aaf5dea5853f14d564e9cc668e44

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.1.2 (2013/10/10)
+
+Enhancement:
+
+- Add `zero_emit` option to emit 0 on the next interval like datacounter plugin
+
 ## 0.1.1 (2013/09/02)
 
 Fixes

data/README.md

@@ -92,6 +92,14 @@ then output bocomes as belows:
 
     Store internal data into a file of the given path on shutdown, and load on starting.
 
+- zero_emit
+
+    Emit 0 on the next interval. This is useful for some software which requires to reset data such as [GrowthForecast](http://kazeburo.github.io/GrowthForecast/) . 
+
+        calc.foo.bar: {"4xx_count":5,"5xx_count":4","reqtime_max":24831,"reqtime_min":10,"reqtime_avg":270.46}
+        # after @interval later
+        calc.foo.bar: {"4xx_count":0,"5xx_count":0","reqtime_max":0,"reqtime_min":0,"reqtime_avg":0}
+
 ## ChangeLog
 
 See [CHANGELOG.md](CHANGELOG.md) for details.

data/fluent-plugin-calc.gemspec

@@ -3,11 +3,11 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-calc"
-  s.version     = "0.1.1"
+  s.version     = "0.1.2"
   s.authors     = ["Naotoshi SEO"]
   s.email       = ["sonots@gmail.com"]
   s.homepage    = "https://github.com/sonots/fluent-plugin-calc"
-  s.summary     = "fluentd plugin to calc messages"
+  s.summary     = "fluentd plugin to calcucate numbers in messages"
   s.description = s.summary
 
   s.rubyforge_project = "fluent-plugin-calc"

data/lib/fluent/plugin/out_calc.rb

@@ -24,8 +24,8 @@ class Fluent::CalcOutput < Fluent::Output
   config_param :add_tag_prefix, :string, :default => 'calc'
   config_param :aggregate, :string, :default => 'tag'
   config_param :store_file, :string, :default => nil
+  config_param :zero_emit, :bool, :default => false
 
-  attr_accessor :counts
   attr_accessor :matches
   attr_accessor :saved_duration
   attr_accessor :saved_at
@@ -53,11 +53,28 @@ class Fluent::CalcOutput < Fluent::Output
       raise Fluent::ConfigError, "tag must be specified for aggregate all" if @tag.nil?
     end
 
-    @counts = {}
     @matches = {}
     @mutex = Mutex.new
   end
 
+  def initial_matches(prev_matches = nil)
+    if @zero_emit && prev_matches
+      matches = {}
+      prev_matches.keys.each do |tag|
+        next unless prev_matches[tag][:count] > 0 # Prohibit to emit anymore
+        matches[tag] = { :count => 0, :sum => {}, :max => {}, :min => {}, :avg => {} }
+        # ToDo: would want default configuration for :max, :min
+        prev_matches[tag][:sum].keys.each {|key| matches[tag][:sum][key] = 0 }
+        prev_matches[tag][:max].keys.each {|key| matches[tag][:max][key] = 0 }
+        prev_matches[tag][:min].keys.each {|key| matches[tag][:min][key] = 0 }
+        prev_matches[tag][:avg].keys.each {|key| matches[tag][:avg][key] = 0 }
+      end
+      matches
+    else
+      {}
+    end
+  end
+
   def start
     super
     load_status(@store_file, @interval) if @store_file
@@ -75,7 +92,7 @@ class Fluent::CalcOutput < Fluent::Output
   def emit(tag, es, chain)
     tag = 'all' if @aggregate == 'all'
     # calc
-    count = 0; matches = { :sum => {}, :max => {}, :min => {}, :avg => {} }
+    matches = { :count => 0, :sum => {}, :max => {}, :min => {}, :avg => {} }
     es.each do |time, record|
       @sum_keys.each do |key|
         next unless record[key] and value = record[key].to_f
@@ -108,12 +125,11 @@ class Fluent::CalcOutput < Fluent::Output
           matches[:avg][key] = sum(matches[:avg][key], value) # sum yet
         end
       end if @sum || @max || @min || @avg
-      count += 1
+      matches[:count] += 1
     end
 
     # thread safe merge
-    @counts[tag] ||= 0
-    @matches[tag] ||= { :sum => {}, :max => {}, :min => {}, :avg => {} }
+    @matches[tag] ||= { :count => 0, :sum => {}, :max => {}, :min => {}, :avg => {} }
     @mutex.synchronize do
       matches[:sum].keys.each do |key|
         @matches[tag][:sum][key] = sum(@matches[tag][:sum][key], matches[:sum][key])
@@ -127,7 +143,7 @@ class Fluent::CalcOutput < Fluent::Output
       matches[:avg].keys.each do |key|
         @matches[tag][:avg][key] = sum(@matches[tag][:avg][key], matches[:avg][key]) # sum yet
       end
-      @counts[tag] += count
+      @matches[tag][:count] += matches[:count]
     end
 
     chain.next
@@ -156,18 +172,17 @@ class Fluent::CalcOutput < Fluent::Output
   # This method is the real one to emit
   def flush_emit(step)
     time = Fluent::Engine.now
-    flushed_counts, flushed_matches, @counts, @matches = @counts, @matches, {}, {}
+    flushed_matches, @matches = @matches, initial_matches(@matches)
 
-    flushed_counts.keys.each do |tag|
-      count = flushed_counts[tag]
+    flushed_matches.keys.each do |tag|
       matches = flushed_matches[tag]
-      output = generate_output(count, matches)
+      output = generate_output(matches)
       tag = @tag ? @tag : "#{@add_tag_prefix}.#{tag}"
       Fluent::Engine.emit(tag, time, output) if output
     end
   end
 
-  def generate_output(count, matches)
+  def generate_output(matches)
     return nil if matches.empty?
     output = {}
     matches[:sum].keys.each do |key|
@@ -180,7 +195,8 @@ class Fluent::CalcOutput < Fluent::Output
       output[key + @min_suffix] = matches[:min][key]
     end
     matches[:avg].keys.each do |key|
-      output[key + @avg_suffix] = matches[:avg][key] / count.to_f # compute avg here
+      output[key + @avg_suffix] = matches[:avg][key]
+      output[key + @avg_suffix] /= matches[:count].to_f if matches[:count] > 0
     end
     output
   end
@@ -208,7 +224,6 @@ class Fluent::CalcOutput < Fluent::Output
         @saved_at = Fluent::Engine.now
         @saved_duration = @saved_at - @last_checked
         Marshal.dump({
-          :counts           => @counts,
           :matches          => @matches,
           :saved_at         => @saved_at,
           :saved_duration   => @saved_duration,
@@ -246,10 +261,13 @@ class Fluent::CalcOutput < Fluent::Output
           end
 
           if Fluent::Engine.now <= stored[:saved_at] + interval
-            @counts = stored[:counts]
             @matches = stored[:matches]
             @saved_at = stored[:saved_at]
             @saved_duration = stored[:saved_duration]
+            # for lower compatibility
+            if counts = stored[:counts]
+              @matches.keys.each {|tag| @matches[tag][:count] = counts[tag] }
+            end
 
             # skip the saved duration to continue counting
             @last_checked = Fluent::Engine.now - @saved_duration

data/spec/out_calc_spec.rb

@@ -77,6 +77,9 @@ describe Fluent::CalcOutput do
       driver.run { messages.each {|message| driver.emit(message, time) } }
       driver.instance.flush_emit(0)
     end
+    let(:empty_emit) do
+      driver.instance.flush_emit(0)
+    end
 
     context 'sum/max/min/avg' do
       let(:config) do
@@ -96,6 +99,28 @@ describe Fluent::CalcOutput do
       it { emit }
     end
 
+    context 'zero_emit' do
+      let(:config) do
+        CONFIG + %[
+          sum _count$
+          max _max$
+          min _min$
+          avg _avg$
+          zero_emit true
+        ]
+      end
+      before do
+        Fluent::Engine.stub(:now).and_return(time)
+        Fluent::Engine.should_receive(:emit).with("calc.#{tag}", time, {
+          "4xx_count"=>6,"5xx_count"=>6,"reqtime_max"=>6,"reqtime_min"=>1,"reqtime_avg"=>3.0
+        })
+        Fluent::Engine.should_receive(:emit).with("calc.#{tag}", time, {
+          "4xx_count"=>0,"5xx_count"=>0,"reqtime_max"=>0,"reqtime_min"=>0,"reqtime_avg"=>0.0
+        })
+      end
+      it { emit; empty_emit }
+    end
+
     context 'sum/max/min/avg_keys' do
       let(:config) do
         CONFIG + %[
@@ -244,22 +269,18 @@ describe Fluent::CalcOutput do
       it 'stored_data and loaded_data should equal' do
         driver.run { messages.each {|message| driver.emit(message, time) } }
         driver.instance.shutdown
-        stored_counts = driver.instance.counts
         stored_matches = driver.instance.matches
         stored_saved_at = driver.instance.saved_at
         stored_saved_duration = driver.instance.saved_duration
-        driver.instance.counts = {}
         driver.instance.matches = {}
         driver.instance.saved_at = nil
         driver.instance.saved_duration = nil
 
         driver.instance.start
-        loaded_counts = driver.instance.counts
         loaded_matches = driver.instance.matches
         loaded_saved_at = driver.instance.saved_at
         loaded_saved_duration = driver.instance.saved_duration
 
-        loaded_counts.should == stored_counts
         loaded_matches.should == stored_matches
         loaded_saved_at.should == stored_saved_at
         loaded_saved_duration.should == stored_saved_duration

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-calc
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Naotoshi SEO
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-09-03 00:00:00.000000000 Z
+date: 2013-10-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -80,7 +80,7 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-description: fluentd plugin to calc messages
+description: fluentd plugin to calcucate numbers in messages
 email:
 - sonots@gmail.com
 executables: []
@@ -123,7 +123,7 @@ rubyforge_project: fluent-plugin-calc
 rubygems_version: 2.0.2
 signing_key: 
 specification_version: 4
-summary: fluentd plugin to calc messages
+summary: fluentd plugin to calcucate numbers in messages
 test_files:
 - spec/out_calc_spec.rb
 - spec/spec_helper.rb