fluent-plugin-calc 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c1a0fa5a4f57cb4b44e75759fdf27f0db87fe498
-  data.tar.gz: 994beab4aca4623426effc9fbd6d1d0e8dfcaf91
+  metadata.gz: fb6d86f52bcefcd6d104c356cc6df3b707196ec8
+  data.tar.gz: 03b34ce3169a1f29c08823d1cfd8f2850822d6c9
 SHA512:
-  metadata.gz: e1a71b149904a8a9768d32ca86a1f632f2f31dad02a95bc421323db9807b014e10abe3f7c47bbec675982fcebc3db386131286f10b182af6406ad7baf4858e8a
-  data.tar.gz: b8b9562c574be42c7e984b4d7c67ad9064a8009984389890d4d336cfe95fa803882c2d461ea74bcebd5b25e9e35eb5eb39368e863cb2aca9d77a6192376bde7f
+  metadata.gz: 1960a83a0a998933536bf6ea48c6129692258849e474f2ad701865c081b6fc268bdfc01e4d3591cbedc13b414a08837ea68fb6417f3097e49f0ba682c1e9fa0b
+  data.tar.gz: 6b97b056a3654180cbb8530e7744f75b606380d3cb513813f8b98a5327cb92fecf58ee69c49a5424025d48656533ebf2ca95aeb4c0cb9a452f59e48c42149061

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.0.4 (2013/08/30)
+
+Enhancement:
+
+- add `sum_suffix`, `max_suffix`, `min_suffix`, `avg_suffix`.
+
 ## 0.0.2 (2013/05/06)
 
 Bugfixes:

data/Gemfile

@@ -1,4 +1,3 @@
 source "http://rubygems.org"
 
-gem 'simplecov', git: 'https://github.com/colszowka/simplecov.git'
 gemspec

data/README.md

@@ -1,10 +1,12 @@
-# fluent-plugin-calc [![Build Status](https://secure.travis-ci.org/sonots/fluent-plugin-calc.png?branch=master)](http://travis-ci.org/sonots/fluent-plugin-calc) [![Dependency Status](https://gemnasium.com/sonots/fluent-plugin-calc.png)](https://gemnasium.com/sonots/fluent-plugin-calc) [![Coverage Status](https://coveralls.io/repos/sonots/fluent-plugin-calc/badge.png?branch=master)](https://coveralls.io/r/sonots/fluent-plugin-calc)
+# fluent-plugin-calc [![Build Status](https://secure.travis-ci.org/sonots/fluent-plugin-calc.png?branch=master)](http://travis-ci.org/sonots/fluent-plugin-calc) [![Dependency Status](https://gemnasium.com/sonots/fluent-plugin-calc.png)](https://gemnasium.com/sonots/fluent-plugin-calc)
 
 Simple fluentd plugin to calculate messages.
 
 ## Configuration
 
-Example (sum for xxx_count, max for xxx_max, min for xxx_min, avg for xxx_avg): 
+Example1)
+
+sum for xxx_count, max for xxx_max, min for xxx_min, avg for xxx_avg
 
     <match foo.**>
       type calc
@@ -24,7 +26,35 @@ Assuming following inputs are coming:
 
 then output bocomes as belows: 
 
-    calc.foo.bar: {"4xx_count":5,"5xx_count":4","reqtime_max":24831,"reqtime_min":82,"reqtime_avg":270.46}
+    calc.foo.bar: {"4xx_count":5,"5xx_count":4","reqtime_max":24831,"reqtime_min":10,"reqtime_avg":270.46}
+
+Example2)
+
+sum, max, min, avg for the same key
+
+    <match foo.**>
+      type calc
+      interval 5s
+      add_tag_prefix calc
+
+      sum ^reqtime$
+      max ^reqtime$
+      min ^reqtime$
+      avg ^reqtime$
+      sum_suffix _sum
+      max_suffix _max
+      min_suffix _min
+      avg_suffix _avg
+    </match>
+
+Assuming following inputs are coming:
+
+    foo.bar: {"reqtime":1.000}
+    foo.bar: {"reqtime":2.000}
+
+then output bocomes as belows: 
+
+    calc.foo.bar: {"reqtime_sum":3.000,"reqtime_max":2.000,"reqtime_min":1.000,"reqtime_avg":1.500}
 
 ## Parameters
 
@@ -32,6 +62,10 @@ then output bocomes as belows:
 
     Calculation. Specify input keys by a regular expression
 
+- sum\_suffix, min\_suffix, max\_suffix, avg\_suffix
+
+    Add a suffix to keys of the output record
+
 - interval
 
     The interval to calculate in seconds. Default is 5s. 
@@ -48,6 +82,10 @@ then output bocomes as belows:
     
     Calculate by each `tag` or `all`. The default value is `tag`.
 
+- store_file
+
+    Store internal data into a file of the given path on shutdown, and load on starting.
+
 ## ChangeLog
 
 See [CHANGELOG.md](CHANGELOG.md) for details.

data/fluent-plugin-calc.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-calc"
-  s.version     = "0.0.3"
+  s.version     = "0.0.4"
   s.authors     = ["Naotoshi SEO"]
   s.email       = ["sonots@gmail.com"]
   s.homepage    = "https://github.com/sonots/fluent-plugin-calc"
@@ -21,5 +21,5 @@ Gem::Specification.new do |s|
   s.add_development_dependency "rake"
   s.add_development_dependency "rspec"
   s.add_development_dependency "pry"
-  s.add_development_dependency 'coveralls'
+  s.add_development_dependency "pry-nav"
 end

data/lib/fluent/plugin/out_calc.rb

@@ -2,16 +2,29 @@
 class Fluent::CalcOutput < Fluent::Output
   Fluent::Plugin.register_output('calc', self)
 
+  def initialize
+    super
+    require 'pathname'
+  end
+
   config_param :sum, :string, :default => nil
   config_param :max, :string, :default => nil
   config_param :min, :string, :default => nil
   config_param :avg, :string, :default => nil
+  config_param :sum_suffix, :string, :default => ""
+  config_param :max_suffix, :string, :default => ""
+  config_param :min_suffix, :string, :default => ""
+  config_param :avg_suffix, :string, :default => ""
   config_param :interval, :time, :default => 5
   config_param :tag, :string, :default => nil
   config_param :add_tag_prefix, :string, :default => 'calc'
   config_param :aggregate, :string, :default => 'tag'
+  config_param :store_file, :string, :default => nil
 
+  attr_accessor :counts
   attr_accessor :matches
+  attr_accessor :saved_duration
+  attr_accessor :saved_at
   attr_accessor :last_checked
 
   def configure(conf)
@@ -39,6 +52,7 @@ class Fluent::CalcOutput < Fluent::Output
 
   def start
     super
+    load_status(@store_file, @interval) if @store_file
     @watcher = Thread.new(&method(:watcher))
   end
 
@@ -46,6 +60,7 @@ class Fluent::CalcOutput < Fluent::Output
     super
     @watcher.terminate
     @watcher.join
+    save_status(@store_file) if @store_file
   end
 
   # Called when new line comes. This method actually does not emit
@@ -53,17 +68,20 @@ class Fluent::CalcOutput < Fluent::Output
     tag = 'all' if @aggregate == 'all'
       
     # calc
-    count = 0; matches = {}
-    es.each do |time,record|
+    count = 0; matches = { :sum => {}, :max => {}, :min => {}, :avg => {} }
+    es.each do |time, record|
       record.keys.each do |key|
         if @sum and @sum.match(key)
-          matches[key] = (matches[key] ? matches[key] + record[key] : record[key])
-        elsif @max and @max.match(key)
-          matches[key] = (matches[key] ? [matches[key], record[key]].max : record[key])
-        elsif @min and @min.match(key)
-          matches[key] = (matches[key] ? [matches[key], record[key]].min : record[key])
-        elsif @avg and @avg.match(key)
-          matches[key] = (matches[key] ? matches[key] + record[key] : record[key]) # sum yet
+          matches[:sum][key] = sum(matches[:sum][key], record[key])
+        end
+        if @max and @max.match(key)
+          matches[:max][key] = max(matches[:max][key], record[key])
+        end
+        if @min and @min.match(key)
+          matches[:min][key] = min(matches[:min][key], record[key])
+        end
+        if @avg and @avg.match(key)
+          matches[:avg][key] = sum(matches[:avg][key], record[key]) # sum yet
         end
       end
       count += 1
@@ -71,18 +89,19 @@ class Fluent::CalcOutput < Fluent::Output
 
     # thread safe merge
     @counts[tag] ||= 0
-    @matches[tag] ||= {}
+    @matches[tag] ||= { :sum => {}, :max => {}, :min => {}, :avg => {} }
     @mutex.synchronize do
-      matches.keys.each do |key|
-        if @sum and @sum.match(key)
-          @matches[tag][key] = (@matches[tag][key] ? @matches[tag][key] + matches[key] : matches[key])
-        elsif @max and @max.match(key)
-          @matches[tag][key] = (@matches[tag][key] ? [@matches[tag][key], matches[key]].max : matches[key])
-        elsif @min and @min.match(key)
-          @matches[tag][key] = (@matches[tag][key] ? [@matches[tag][key], matches[key]].min : matches[key])
-        elsif @avg and @avg.match(key)
-          @matches[tag][key] = (@matches[tag][key] ? @matches[tag][key] + matches[key] : matches[key]) # sum yet
-        end
+      matches[:sum].keys.each do |key|
+        @matches[tag][:sum][key] = sum(@matches[tag][:sum][key], matches[:sum][key])
+      end
+      matches[:max].keys.each do |key|
+        @matches[tag][:max][key] = max(@matches[tag][:max][key], matches[:max][key])
+      end
+      matches[:min].keys.each do |key|
+        @matches[tag][:min][key] = min(@matches[tag][:min][key], matches[:min][key])
+      end
+      matches[:avg].keys.each do |key|
+        @matches[tag][:avg][key] = sum(@matches[tag][:avg][key], matches[:avg][key]) # sum yet
       end
       @counts[tag] += count
     end
@@ -95,7 +114,7 @@ class Fluent::CalcOutput < Fluent::Output
   # thread callback
   def watcher
     # instance variable, and public accessable, for test
-    @last_checked = Fluent::Engine.now
+    @last_checked ||= Fluent::Engine.now
     while true
       sleep 0.5
       begin
@@ -126,13 +145,100 @@ class Fluent::CalcOutput < Fluent::Output
 
   def generate_output(count, matches)
     return nil if matches.empty?
-    output = matches.dup
-    output.keys.each do |key|
-      if @avg and @avg.match(key)
-        output[key] = matches[key] / count.to_f # compute avg
-      end
+    output = {}
+    matches[:sum].keys.each do |key|
+      output[key + @sum_suffix] = matches[:sum][key]
+    end
+    matches[:max].keys.each do |key|
+      output[key + @max_suffix] = matches[:max][key]
+    end
+    matches[:min].keys.each do |key|
+      output[key + @min_suffix] = matches[:min][key]
+    end
+    matches[:avg].keys.each do |key|
+      output[key + @avg_suffix] = matches[:avg][key] / count.to_f # compute avg here
     end
     output
   end
 
+  def sum(a, b)
+    [a, b].compact.inject(:+)
+  end
+
+  def max(a, b)
+    [a, b].compact.max
+  end
+
+  def min(a, b)
+    [a, b].compact.min
+  end
+
+  # Store internal status into a file
+  #
+  # @param [String] file_path
+  def save_status(file_path)
+    return unless file_path
+
+    begin
+      Pathname.new(file_path).open('wb') do |f|
+        @saved_at = Fluent::Engine.now
+        @saved_duration = @saved_at - @last_checked
+        Marshal.dump({
+          :counts           => @counts,
+          :matches          => @matches,
+          :saved_at         => @saved_at,
+          :saved_duration   => @saved_duration,
+          :aggregate        => @aggregate,
+          :sum              => @sum,
+          :max              => @max,
+          :min              => @min,
+          :avg              => @avg,
+        }, f)
+      end
+    rescue => e
+      $log.warn "out_calc: Can't write store_file #{e.class} #{e.message}"
+    end
+  end
+
+  # Load internal status from a file
+  #
+  # @param [String] file_path
+  # @param [Interger] interval
+  def load_status(file_path, interval)
+    return unless (f = Pathname.new(file_path)).exist?
+
+    begin
+      f.open('rb') do |f|
+        stored = Marshal.load(f)
+        if stored[:aggregate] == @aggregate and
+          stored[:sum] == @sum and
+          stored[:max] == @max and
+          stored[:min] == @min and
+          stored[:avg] == @avg
+
+          if !stored[:matches].first[1].has_key?(:max)
+            $log.warn "out_calc: stored data does not have compatibility with the current version. ignore stored data"
+            return
+          end
+
+          if Fluent::Engine.now <= stored[:saved_at] + interval
+            @counts = stored[:counts]
+            @matches = stored[:matches]
+            @saved_at = stored[:saved_at]
+            @saved_duration = stored[:saved_duration]
+
+            # skip the saved duration to continue counting
+            @last_checked = Fluent::Engine.now - @saved_duration
+          else
+            $log.warn "out_calc: stored data is outdated. ignore stored data"
+          end
+        else
+          $log.warn "out_calc: configuration param was changed. ignore stored data"
+        end
+      end
+    rescue => e
+      $log.warn "out_calc: Can't load store_file #{e.class} #{e.message}"
+    end
+  end
+
 end

data/spec/out_calc_spec.rb

@@ -78,10 +78,6 @@ describe Fluent::CalcOutput do
       driver.instance.flush_emit(0)
     end
 
-    context 'interval' do
-      pending
-    end
-
     context 'sum/max/min/avg' do
       let(:config) do
         CONFIG + %[
@@ -100,15 +96,46 @@ describe Fluent::CalcOutput do
       it { emit }
     end
 
+    context 'sum/max/min/avg_suffix' do
+      let(:config) do
+        CONFIG + %[
+          sum ^(reqtime|reqsize)$
+          max ^reqtime$
+          min ^reqtime$
+          avg ^reqtime$
+          sum_suffix _sum
+          max_suffix _max
+          min_suffix _min
+          avg_suffix _avg
+        ]
+      end
+      let(:messages) do
+        [
+          {"reqtime"=>1.000,"reqsize"=>10},
+          {"reqtime"=>2.000,"reqsize"=>20},
+        ]
+    end
+      before do
+        Fluent::Engine.stub(:now).and_return(time)
+        Fluent::Engine.should_receive(:emit).with("calc.#{tag}", time, {
+          "reqtime_sum"=>3.000,"reqtime_max"=>2.000,"reqtime_min"=>1.000,"reqtime_avg"=>1.500,"reqsize_sum"=>30
+        })
+      end
+      it { emit }
+    end
+
     context 'tag' do
       let(:config) do
         CONFIG + %[
           tag foo
+          sum _count$
         ]
       end
       before do
         Fluent::Engine.stub(:now).and_return(time)
-        Fluent::Engine.should_receive(:emit).with("foo", time, {})
+        Fluent::Engine.should_receive(:emit).with("foo", time, {
+          "4xx_count"=>6,"5xx_count"=>6
+        })
       end
       it { emit }
     end
@@ -117,11 +144,14 @@ describe Fluent::CalcOutput do
       let(:config) do
         CONFIG + %[
           add_tag_prefix foo
+          sum _count$
         ]
       end
       before do
         Fluent::Engine.stub(:now).and_return(time)
-        Fluent::Engine.should_receive(:emit).with("foo.#{tag}", time, {})
+        Fluent::Engine.should_receive(:emit).with("foo.#{tag}", time, {
+          "4xx_count"=>6,"5xx_count"=>6
+        })
       end
       it { emit }
     end
@@ -176,6 +206,47 @@ describe Fluent::CalcOutput do
         it { emit }
       end
     end
+
+    describe "store_file" do
+      let(:store_file) do
+        dirname = "tmp"
+        Dir.mkdir dirname unless Dir.exist? dirname
+        filename = "#{dirname}/test.dat"
+        File.unlink filename if File.exist? filename
+        filename
+      end
+
+      let(:config) do
+        CONFIG + %[
+          sum _count$
+          store_file #{store_file}
+        ]
+      end
+
+      it 'stored_data and loaded_data should equal' do
+        driver.run { messages.each {|message| driver.emit(message, time) } }
+        driver.instance.shutdown
+        stored_counts = driver.instance.counts
+        stored_matches = driver.instance.matches
+        stored_saved_at = driver.instance.saved_at
+        stored_saved_duration = driver.instance.saved_duration
+        driver.instance.counts = {}
+        driver.instance.matches = {}
+        driver.instance.saved_at = nil
+        driver.instance.saved_duration = nil
+
+        driver.instance.start
+        loaded_counts = driver.instance.counts
+        loaded_matches = driver.instance.matches
+        loaded_saved_at = driver.instance.saved_at
+        loaded_saved_duration = driver.instance.saved_duration
+
+        loaded_counts.should == stored_counts
+        loaded_matches.should == stored_matches
+        loaded_saved_at.should == stored_saved_at
+        loaded_saved_duration.should == stored_saved_duration
+      end
+    end
   end
 end
 

data/spec/spec_helper.rb

@@ -4,9 +4,6 @@ require 'bundler'
 Bundler.setup(:default, :test)
 Bundler.require(:default, :test)
 
-require 'coveralls'
-Coveralls.wear!
-
 require 'fluent/test'
 require 'rspec'
 require 'pry'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-calc
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - Naotoshi SEO
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-06-18 00:00:00.000000000 Z
+date: 2013-08-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -67,7 +67,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: coveralls
+  name: pry-nav
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
@@ -127,3 +127,4 @@ summary: fluentd plugin to calc messages
 test_files:
 - spec/out_calc_spec.rb
 - spec/spec_helper.rb
+has_rdoc: