fluent-plugin-groupcounter 0.1.0 → 0.2.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 86658cfcab0fe4d17fcbafd52f8baa4063905316
+  data.tar.gz: bd59e1fe0c665c9c942519d1eb026e9f258113d3
+SHA512:
+  metadata.gz: 92bab15c3c80a69a66c131de63a99d3cfcc3e4a61eea8618603413e02cd142cd93c897cd87571db89faa7270eb0e4318d1d4ac97237bc3dd824ecdd05bbddf16
+  data.tar.gz: 99cd15d0539ec3a0d898bdf4f6c8937111e523dca1acade1151c0e200500c1868717bd8196d76276991d671da9ec3f24f7bf977dea378d251e763d2589ea8c1e

data/.pryrc

@@ -0,0 +1,3 @@
+Pry.commands.alias_command 'c', 'continue'
+Pry.commands.alias_command 's', 'step'
+Pry.commands.alias_command 'n', 'next'

data/.travis.yml

@@ -0,0 +1,6 @@
+rvm:
+  - 1.9.2
+  - 1.9.3
+  - 2.0.0
+gemfile:
+  - Gemfile

data/README.md

@@ -1,43 +1,124 @@
 # fluent-plugin-groupcounter
 
-## Component
-
-### GroupCounterOutput
-
-Fluentd plugin to count like COUNT(\*) GROUP BY
+Fluentd plugin to count like SELECT COUNT(\*) GROUP BY.
 
 ## Configuration
 
-## GroupCounterOutput
+Assume inputs are coming as followings:
+
+    apache.access: {"code":"200", "method":"GET", "path":"/index.html", "reqtime":"1.001" }
+    apache.access: {"code":"404", "method":"GET", "path":"/foo.html",   "reqtime":"2.002" }
+    apache.access: {"code":"200", "method":"GET", "path":"/index.html", "reqtime":"3.003" }
 
-    <source>
-      type tail
-      path /var/log/httpd-access.log
-      tag apache.access
-      format apache
-    </source>
+Think of quering `SELECT COUNT(\*) GROUP BY code,method,path`. Configuration becomes as below:
 
     <match apache.access>
       type groupcounter
-      count_interval 5s
       aggregate tag
       output_per_tag true
       tag_prefix groupcounter
       group_by_keys code,method,path
     </match>
 
-Output like below
+Output becomes like
+
+    groupcounter.apache.access: {"200_GET_/index.html_count":2, "404_GET_/foo.html_count":1}
+
+## Parameters
+
+* group\_by\_keys (semi-required)
+
+    Specify keys in the event record for grouping. `group_by_keys` or `group_by_expression` is required.
+
+* delimiter
+
+    Specify the delimiter to join `group_by_keys`. Default is '_'.
+
+* group\_by\_expression (semi-required)
+
+    Use an expression to group the event record. `group_by_keys` or `group_by_expression` is required.
+
+    For examples, for the exampled input above, the configuration as below
+
+        group_by_expression ${method}${path}/${code}
+
+    gives you an output like
+
+        groupcounter.apache.access: {"GET/index.html/200_count":1, "GET/foo.html/400_count":1}
+
+    SECRET TRICK: You can write a ruby code in the ${} placeholder like
+
+        group_by_expression ${method}${path.split(".")[0]}/${code[0]}xx
+
+    This gives an output like
+
+        groupcounter.apache.access: {"GET/index/2xx_count":1, "GET/foo/4xx_count":1}
+
+* tag
+
+    The output tag. Default is `groupcount`.
+
+* tag\_prefix
+
+    The prefix string which will be added to the input tag. `output_per_tag yes` must be specified together. 
+
+* input\_tag\_remove\_prefix
+
+    The prefix string which will be removed from the input tag.
+
+* count\_interval
+
+    The interval time to count in seconds. Default is `60`.
+
+* unit
+
+    The interval time to monitor specified an unit (either of `minute`, `hour`, or `day`).
+    Use either of `count_interval` or `unit`.
+
+* store\_file
+
+    Store internal data into a file of the given path on shutdown, and load on starting.
+
+* max\_key
+
+    Specify key name in the event record to do `SELECT COUNT(\*),MAX(key_name) GROUP BY`.
+
+    For examples, for the exampled input above, adding the configuration as below
+
+        max_key reqtime
+
+    gives you an output like
+
+        groupcounter.apache.access: {"200_GET_/index.html_reqtime_max":3.003, "404_GET_/foo.html_reqtime_max":2.002}
+
+* min\_key
+
+    Specify key name in the event record to do `SELECT COUNT(\*),MIN(key_name) GROUP BY`.
+
+* avg\_key
+
+    Specify key name in the event record to do `SELECT COUNT(\*),AVG(key_name) GROUP BY`.
+
+* count\_suffix
+
+    Default is `_count`
+
+* max\_suffix
+
+    Default is `_max`. Should be used with `max_key` option.
+
+* min\_suffix
 
-    groupcounter.apache.access: {"200_GET_/index.html_count":1,"200_GET_/index.html_rate":0.2,"200_GET_/index.html_percentage":100.0}
+    Default is `_min`. Should be used with `min_key` option.
 
-## TODO
+* avg\_suffix
 
-* tests
-* documents
+    Default is `_avg`. Should be used with `avg_key` option.
 
 ## Copyright
 
 * Copyright
   * Copyright (c) 2012- Ryosuke IWANAGA (riywo)
+  * Copyright (c) 2013- Naotoshi SEO (sonots)
 * License
   * Apache License, Version 2.0

data/Rakefile

@@ -1,10 +1,15 @@
 # encoding: utf-8
 require "bundler/gem_tasks"
 
+require 'rspec/core'
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = FileList['spec/**/*_spec.rb']
+end
+task :default => :spec
+
 desc 'Open an irb session preloaded with the gem library'
 task :console do
     sh 'irb -rubygems -I lib'
 end
-
 task :c => :console
-

data/fluent-plugin-groupcounter.gemspec

@@ -3,12 +3,12 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-groupcounter"
-  s.version     = "0.1.0"
+  s.version     = "0.2.0"
   s.authors     = ["Ryosuke IWANAGA", "Naotoshi SEO"]
-  s.email       = ["@riywo", "@sonots"]
+  s.email       = ["@riywo", "sonots@gmail.com"]
   s.homepage    = "https://github.com/riywo/fluent-plugin-groupcounter"
-  s.summary     = %q{Fluentd plugin to count like COUNT(\*) GROUP BY}
-  s.description = %q{Fluentd plugin to count like COUNT(\*) GROUP BY}
+  s.summary     = %q{Fluentd plugin to count like SELECT COUNT(\*) GROUP BY}
+  s.description = %q{Fluentd plugin to count like SELECT COUNT(\*) GROUP BY}
 
   s.rubyforge_project = "fluent-plugin-groupcounter"
 
@@ -18,6 +18,8 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   s.add_runtime_dependency "fluentd"
-  s.add_development_dependency "fluentd"
   s.add_development_dependency "rake"
+  s.add_development_dependency "rspec"
+  s.add_development_dependency "pry"
+  s.add_development_dependency "pry-nav"
 end

data/lib/fluent/plugin/out_groupcounter.rb

@@ -1,6 +1,11 @@
 class Fluent::GroupCounterOutput < Fluent::Output
   Fluent::Plugin.register_output('groupcounter', self)
 
+  def initialize
+    super
+    require 'pathname'
+  end
+
   config_param :count_interval, :time, :default => nil
   config_param :unit, :string, :default => 'minute'
   config_param :output_per_tag, :bool, :default => false
@@ -8,20 +13,35 @@ class Fluent::GroupCounterOutput < Fluent::Output
   config_param :tag, :string, :default => 'groupcount'
   config_param :tag_prefix, :string, :default => nil
   config_param :input_tag_remove_prefix, :string, :default => nil
-  config_param :group_by_keys, :string
-  config_param :output_messages, :bool, :default => false
+  config_param :group_by_keys, :string, :default => nil
+  config_param :group_by_expression, :string, :default => nil
+  config_param :max_key, :string, :default => nil
+  config_param :min_key, :string, :default => nil
+  config_param :avg_key, :string, :default => nil
+  config_param :delimiter, :string, :default => '_'
+  config_param :count_suffix, :string, :default => '_count'
+  config_param :max_suffix, :string, :default => '_max'
+  config_param :min_suffix, :string, :default => '_min'
+  config_param :avg_suffix, :string, :default => '_avg'
+  config_param :store_file, :string, :default => nil
 
-  attr_accessor :tick
+  attr_accessor :count_interval
   attr_accessor :counts
+  attr_accessor :saved_duration
+  attr_accessor :saved_at
   attr_accessor :last_checked
 
   def configure(conf)
     super
 
+    if @group_by_keys.nil? and @group_by_expression.nil?
+      raise Fluent::ConfigError, "Either of group_by_keys or group_by_expression must be specified"
+    end
+
     if @count_interval
-      @tick = @count_interval.to_i
+      @count_interval = @count_interval.to_i
     else
-      @tick = case @unit
+      @count_interval = case @unit
               when 'minute' then 60
               when 'hour' then 3600
               when 'day' then 86400
@@ -47,14 +67,23 @@ class Fluent::GroupCounterOutput < Fluent::Output
       @removed_length = @removed_prefix_string.length
     end
 
-    @group_by_keys = @group_by_keys.split(',')
+    @group_by_keys = @group_by_keys.split(',') if @group_by_keys
+
+    if @store_file
+      f = Pathname.new(@store_file)
+      if (f.exist? && !f.writable_real?) || (!f.exist? && !f.parent.writable_real?)
+        raise Fluent::ConfigError, "#{@store_file} is not writable"
+      end
+    end
 
     @counts = count_initialized
+    @hostname = Socket.gethostname
     @mutex = Mutex.new
   end
 
   def start
     super
+    load_status(@store_file, @count_interval) if @store_file
     start_watch
   end
 
@@ -62,102 +91,64 @@ class Fluent::GroupCounterOutput < Fluent::Output
     super
     @watcher.terminate
     @watcher.join
+    save_status(@store_file) if @store_file
   end
 
   def count_initialized
-    # counts['tag'][group_by_keys] = count
-    # counts['tag'][__sum] = sum
     {}
   end
 
-  def countups(tag, counts)
-    if @aggregate == :all
-      tag = 'all'
-    end
-    @counts[tag] ||= {}
-    
-    @mutex.synchronize {
-      sum = 0
-      counts.each do |key, count|
-        sum += count
-        @counts[tag][key] ||= 0
-        @counts[tag][key] += count
-      end
-      @counts[tag]['__sum'] ||= 0
-      @counts[tag]['__sum'] += sum
-    }
-  end
-
-  def stripped_tag(tag)
-    return tag unless @input_tag_remove_prefix
-    return tag[@removed_length..-1] if tag.start_with?(@removed_prefix_string) and tag.length > @removed_length
-    return tag[@removed_length..-1] if tag == @input_tag_remove_prefix
-    tag
-  end
-
-  def generate_fields(step, target_counts, attr_prefix, output)
-    return {} unless target_counts
-    sum = target_counts['__sum']
-    messages = target_counts.delete('__sum')
+  def generate_fields(counts_per_tag, output = {}, key_prefix = '')
+    return {} unless counts_per_tag
+    # total_count = counts_per_tag.delete('__total_count')
 
-    target_counts.each do |key, count|
-      output[attr_prefix + key + '_count'] = count
-      output[attr_prefix + key + '_rate'] = ((count * 100.0) / (1.00 * step)).floor / 100.0
-      output[attr_prefix + key + '_percentage'] = count * 100.0 / (1.00 * sum) if sum > 0
-      if @output_messages
-        output[attr_prefix + 'messages'] = messages
-      end
+    counts_per_tag.each do |group_key, count|
+      output[key_prefix + group_key + @count_suffix] = count[:count] if count[:count]
+      output[key_prefix + group_key + "#{@delimiter}#{@min_key}#{@min_suffix}"] = count[:min] if count[:min]
+      output[key_prefix + group_key + "#{@delimiter}#{@max_key}#{@max_suffix}"] = count[:max] if count[:max]
+      output[key_prefix + group_key + "#{@delimiter}#{@avg_key}#{@avg_suffix}"] = count[:sum] / (count[:count] * 1.0) if count[:sum] and count[:count] > 0
+      # output[key_prefix + group_key + "#{@delimiter}rate"] = ((count[:count] * 100.0) / (1.00 * step)).floor / 100.0
+      # output[key_prefix + group_key + "#{@delimiter}percentage"] = count[:count] * 100.0 / (1.00 * total_count) if total_count > 0
     end
 
     output
   end
 
-  def generate_output(counts, step)
-    if @aggregate == :all
-      return generate_fields(step, counts['all'], '', {})
-    end
+  def generate_output(counts)
+    if @output_per_tag # tag => output
+      return {'all' => generate_fields(counts['all'])} if @aggregate == :all
 
-    output = {}
-    counts.keys.each do |tag|
-      generate_fields(step, counts[tag], stripped_tag(tag) + '_', output)
-    end
-    output
-  end
-
-  def generate_output_per_tags(counts, step)
-    if @aggregate == :all
-      return {'all' => generate_fields(step, counts['all'], '', {})}
-    end
+      output_pairs = {}
+      counts.keys.each do |tag|
+        output_pairs[stripped_tag(tag)] = generate_fields(counts[tag])
+      end
+      output_pairs
+    else
+      return generate_fields(counts['all']) if @aggregate == :all
 
-    output_pairs = {}
-    counts.keys.each do |tag|
-      output_pairs[stripped_tag(tag)] = generate_fields(step, counts[tag], '', {})
+      output = {}
+      counts.keys.each do |tag|
+        generate_fields(counts[tag], output, stripped_tag(tag) + '_')
+      end
+      output
     end
-    output_pairs
-  end
-
-  def flush(step) # returns one message
-    flushed,@counts = @counts,count_initialized()
-    generate_output(flushed, step)
   end
 
-  def flush_per_tags(step) # returns map of tag - message
-    flushed,@counts = @counts,count_initialized()
-    generate_output_per_tags(flushed, step)
+  def flush
+    flushed, @counts = @counts, count_initialized()
+    generate_output(flushed)
   end
 
-  def flush_emit(step)
+  # this method emits messages (periodically called)
+  def flush_emit
+    time = Fluent::Engine.now
     if @output_per_tag
-      # tag - message maps
-      time = Fluent::Engine.now
-      flush_per_tags(step).each do |tag,message|
+      flush.each do |tag, message|
         Fluent::Engine.emit(@tag_prefix_string + tag, time, message)
       end
     else
-      message = flush(step)
-      if message.keys.size > 0
-        Fluent::Engine.emit(@tag, Fluent::Engine.now, message)
-      end
+      message = flush
+      Fluent::Engine.emit(@tag, time, message) unless message.empty?
     end
   end
 
@@ -168,34 +159,174 @@ class Fluent::GroupCounterOutput < Fluent::Output
   
   def watch
     # instance variable, and public accessable, for test
-    @last_checked = Fluent::Engine.now
+    @last_checked ||= Fluent::Engine.now
     while true
       sleep 0.5
-      if Fluent::Engine.now - @last_checked >= @tick
-        now = Fluent::Engine.now
-        flush_emit(now - @last_checked)
-        @last_checked = now
+      begin
+        if Fluent::Engine.now - @last_checked >= @count_interval
+          now = Fluent::Engine.now
+          flush_emit
+          @last_checked = now
+        end
+      rescue => e
+        $log.warn "#{e.class} #{e.message} #{e.backtrace.first}"
       end
     end
   end
 
+  # recieve messages at here
   def emit(tag, es, chain)
-    c = {}
-
-    es.each do |time,record|
-      values = []
-      @group_by_keys.each { |key|
-        v = record[key] || 'undef'
-        values.push(v)
-      }
-      value = values.join('_')
-
-      value = value.to_s.force_encoding('ASCII-8BIT')
-      c[value] ||= 0
-      c[value] += 1
+    group_counts = {}
+
+    tags = tag.split('.')
+    es.each do |time, record|
+      count = {}
+      count[:count] = 1
+      count[:sum] = record[@avg_key].to_f if @avg_key and record[@avg_key]
+      count[:max] = record[@max_key].to_f if @max_key and record[@max_key]
+      count[:min] = record[@min_key].to_f if @min_key and record[@min_key]
+
+      group_key = group_key(tag, time, record)
+
+      group_counts[group_key] ||= {}
+      countup(group_counts[group_key], count)
     end
-    countups(tag, c)
+    summarize_counts(tag, group_counts)
 
     chain.next
+  rescue => e
+    $log.warn "#{e.class} #{e.message} #{e.backtrace.first}"
+  end
+
+  # Summarize counts for each tag
+  def summarize_counts(tag, group_counts)
+    tag = 'all' if @aggregate == :all
+    @counts[tag] ||= {}
+    
+    @mutex.synchronize {
+      group_counts.each do |group_key, count|
+        @counts[tag][group_key] ||= {}
+        countup(@counts[tag][group_key], count)
+      end
+
+      # total_count = group_counts.map {|group_key, count| count[:count] }.inject(:+)
+      # @counts[tag]['__total_count'] = sum(@counts[tag]['__total_count'], total_count)
+    }
+  end
+
+  def countup(counts, count)
+    counts[:count] = sum(counts[:count], count[:count])
+    counts[:sum]   = sum(counts[:sum], count[:sum]) if @avg_key and count[:sum]
+    counts[:max]   = max(counts[:max], count[:max]) if @max_key and count[:max]
+    counts[:min]   = min(counts[:min], count[:min]) if @min_key and count[:min]
+  end
+
+  # Expand record with @group_by_keys, and get a value to be a group_key
+  def group_key(tag, time, record)
+    if @group_by_expression
+      tags = tag.split('.')
+      group_key = expand_placeholder(@group_by_expression, record, tag, tags, Time.at(time))
+    else # @group_by_keys
+      values = @group_by_keys.map {|key| record[key] || 'undef'}
+      group_key = values.join(@delimiter)
+    end
+    group_key = group_key.to_s.force_encoding('ASCII-8BIT')
+  end
+
+  def sum(a, b)
+    a ||= 0
+    b ||= 0
+    a + b
+  end
+
+  def max(a, b)
+    return b if a.nil?
+    return a if b.nil?
+    a > b ? a : b
+  end
+
+  def min(a, b)
+    return b if a.nil?
+    return a if b.nil?
+    a > b ? b : a
+  end
+
+  def stripped_tag(tag)
+    return tag unless @input_tag_remove_prefix
+    return tag[@removed_length..-1] if tag.start_with?(@removed_prefix_string) and tag.length > @removed_length
+    return tag[@removed_length..-1] if tag == @input_tag_remove_prefix
+    tag
+  end
+
+  # Store internal status into a file
+  #
+  # @param [String] file_path
+  def save_status(file_path)
+
+    begin
+      Pathname.new(file_path).open('wb') do |f|
+        @saved_at = Fluent::Engine.now
+        @saved_duration = @saved_at - @last_checked
+        Marshal.dump({
+          :counts           => @counts,
+          :saved_at         => @saved_at,
+          :saved_duration   => @saved_duration,
+          :aggregate        => @aggregate,
+          :group_by_keys    => @group_by_keys,
+        }, f)
+      end
+    rescue => e
+      $log.warn "out_groupcounter: Can't write store_file #{e.class} #{e.message}"
+    end
+  end
+
+  # Load internal status from a file
+  #
+  # @param [String] file_path
+  # @param [Interger] count_interval
+  def load_status(file_path, count_interval)
+    return unless (f = Pathname.new(file_path)).exist?
+
+    begin
+      f.open('rb') do |f|
+        stored = Marshal.load(f)
+        if stored[:aggregate] == @aggregate and
+          stored[:group_by_keys] == @group_by_keys and
+
+          if Fluent::Engine.now <= stored[:saved_at] + count_interval
+            @counts = stored[:counts]
+            @saved_at = stored[:saved_at]
+            @saved_duration = stored[:saved_duration]
+
+            # skip the saved duration to continue counting
+            @last_checked = Fluent::Engine.now - @saved_duration
+          else
+            $log.warn "out_groupcounter: stored data is outdated. ignore stored data"
+          end
+        else
+          $log.warn "out_groupcounter: configuration param was changed. ignore stored data"
+        end
+      end
+    rescue => e
+      $log.warn "out_groupcounter: Can't load store_file #{e.class} #{e.message}"
+    end
+  end
+
+  private
+
+  def expand_placeholder(str, record, tag, tags, time)
+    struct = UndefOpenStruct.new(record)
+    struct.tag  = tag
+    struct.tags = tags
+    struct.time = time
+    struct.hostname = @hostname
+    str = str.gsub(/\$\{([^}]+)\}/, '#{\1}') # ${..} => #{..}
+    eval "\"#{str}\"", struct.instance_eval { binding }
+  end
+
+  class UndefOpenStruct < OpenStruct
+    (Object.instance_methods).each do |m|
+      undef_method m unless m.to_s =~ /^__|respond_to_missing\?|object_id|public_methods|instance_eval|method_missing|define_singleton_method|respond_to\?|new_ostruct_member/
+    end
   end
 end

data/spec/out_groupcounter_spec.rb

@@ -0,0 +1,354 @@
+# encoding: UTF-8
+require_relative 'spec_helper'
+
+class Fluent::Test::OutputTestDriver
+  def emit_with_tag(record, time=Time.now, tag = nil)
+    @tag = tag if tag
+    emit(record, time)
+  end
+end
+
+describe Fluent::GroupCounterOutput do
+  before { Fluent::Test.setup }
+  CONFIG = %[
+    count_interval 5s
+    aggragate tag
+    output_per_tag true
+    tag_prefix count
+    group_by_keys code,method,path
+  ]
+
+  let(:tag) { 'test' }
+  let(:driver) { Fluent::Test::OutputTestDriver.new(Fluent::GroupCounterOutput, tag).configure(config) }
+
+  describe 'test configure' do
+    describe 'bad configuration' do
+      context 'test empty configuration' do
+        let(:config) { %[] }
+        it { expect { driver }.to raise_error(Fluent::ConfigError) }
+      end
+    end
+
+    describe 'good configuration' do
+      subject { driver.instance }
+
+      context "test least configuration" do
+        let(:config) { %[group_by_keys foo] }
+        its(:count_interval) { should == 60 }
+        its(:unit) { should == 'minute' }
+        its(:output_per_tag) { should == false }
+        its(:aggregate) { should == :tag }
+        its(:tag) { should == 'groupcount' }
+        its(:tag_prefix) { should be_nil }
+        its(:input_tag_remove_prefix) { should be_nil }
+        its(:group_by_keys) { should == %w[foo] }
+      end
+
+      context "test template configuration" do
+        let(:config) { CONFIG }
+        its(:count_interval) { should == 5 }
+        its(:unit) { should == 'minute' }
+        its(:output_per_tag) { should == true }
+        its(:aggregate) { should == :tag }
+        its(:tag) { should == 'groupcount' }
+        its(:tag_prefix) { should == 'count' }
+        its(:input_tag_remove_prefix) { should be_nil }
+        its(:group_by_keys) { should == %w[code method path] }
+      end
+    end
+  end
+
+  describe 'test emit' do
+    let(:time) { Time.now.to_i }
+    let(:emit) do
+      driver.run { messages.each {|message| driver.emit(message, time) } }
+      driver.instance.flush_emit
+    end
+
+    let(:messages) do
+      [
+        {"code" => 200, "method" => "GET",  "path" => "/ping", "reqtime" => "0.000" },
+        {"code" => 200, "method" => "POST", "path" => "/auth", "reqtime" => "1.001" },
+        {"code" => 200, "method" => "GET",  "path" => "/ping", "reqtime" => "2.002" },
+        {"code" => 400, "method" => "GET",  "path" => "/ping", "reqtime" => "3.003" },
+      ]
+    end
+    let(:expected) do
+      {
+        "200_GET_/ping_count"=>2,
+        "200_POST_/auth_count"=>1,
+        "400_GET_/ping_count"=>1,
+      }
+    end
+    let(:expected_with_tag) do
+      Hash[*(expected.map {|key, val| next ["test_#{key}", val] }.flatten)] 
+    end
+
+    context 'default' do
+      let(:config) { CONFIG }
+      before do
+        Fluent::Engine.stub(:now).and_return(time)
+        Fluent::Engine.should_receive(:emit).with("count.#{tag}", time, expected)
+      end
+      it { emit }
+    end
+
+    context 'delimiter' do
+      let(:config) { CONFIG + %[delimiter /] }
+      let(:expected) do
+        {
+          "200/GET//ping_count"=>2,
+          "200/POST//auth_count"=>1,
+          "400/GET//ping_count"=>1,
+        }
+      end
+      before do
+        Fluent::Engine.stub(:now).and_return(time)
+        Fluent::Engine.should_receive(:emit).with("count.#{tag}", time, expected)
+      end
+      it { emit }
+    end
+
+    context 'count_suffix' do
+      let(:config) { CONFIG + %[count_suffix /count] }
+      let(:expected) do
+        {
+          "200_GET_/ping/count"=>2,
+          "200_POST_/auth/count"=>1,
+          "400_GET_/ping/count"=>1,
+        }
+      end
+      before do
+        Fluent::Engine.stub(:now).and_return(time)
+        Fluent::Engine.should_receive(:emit).with("count.#{tag}", time, expected)
+      end
+      it { emit }
+    end
+
+    context 'max_key' do
+      let(:config) { CONFIG + %[max_key reqtime] }
+      let(:expected) do
+        {
+          "200_GET_/ping_count"=>2, "200_GET_/ping_reqtime_max"=>2.002,
+          "200_POST_/auth_count"=>1, "200_POST_/auth_reqtime_max"=>1.001,
+          "400_GET_/ping_count"=>1, "400_GET_/ping_reqtime_max"=>3.003,
+        }
+      end
+      before do
+        Fluent::Engine.stub(:now).and_return(time)
+        Fluent::Engine.should_receive(:emit).with("count.#{tag}", time, expected)
+      end
+      it { emit }
+    end
+
+    context 'max_suffix' do
+      let(:config) { CONFIG + %[max_key reqtime \n max_suffix /max] }
+      let(:expected) do
+        {
+          "200_GET_/ping_count"=>2, "200_GET_/ping_reqtime/max"=>2.002,
+          "200_POST_/auth_count"=>1, "200_POST_/auth_reqtime/max"=>1.001,
+          "400_GET_/ping_count"=>1, "400_GET_/ping_reqtime/max"=>3.003,
+        }
+      end
+      before do
+        Fluent::Engine.stub(:now).and_return(time)
+        Fluent::Engine.should_receive(:emit).with("count.#{tag}", time, expected)
+      end
+      it { emit }
+    end
+
+    context 'min_key' do
+      let(:config) { CONFIG + %[min_key reqtime] }
+      let(:expected) do
+        {
+          "200_GET_/ping_count"=>2, "200_GET_/ping_reqtime_min"=>0.000,
+          "200_POST_/auth_count"=>1, "200_POST_/auth_reqtime_min"=>1.001,
+          "400_GET_/ping_count"=>1, "400_GET_/ping_reqtime_min"=>3.003,
+        }
+      end
+      before do
+        Fluent::Engine.stub(:now).and_return(time)
+        Fluent::Engine.should_receive(:emit).with("count.#{tag}", time, expected)
+      end
+      it { emit }
+    end
+
+    context 'min_suffix' do
+      let(:config) { CONFIG + %[min_key reqtime \n min_suffix /min] }
+      let(:expected) do
+        {
+          "200_GET_/ping_count"=>2, "200_GET_/ping_reqtime/min"=>0.000,
+          "200_POST_/auth_count"=>1, "200_POST_/auth_reqtime/min"=>1.001,
+          "400_GET_/ping_count"=>1, "400_GET_/ping_reqtime/min"=>3.003,
+        }
+      end
+      before do
+        Fluent::Engine.stub(:now).and_return(time)
+        Fluent::Engine.should_receive(:emit).with("count.#{tag}", time, expected)
+      end
+      it { emit }
+    end
+
+    context 'avg_key' do
+      let(:config) { CONFIG + %[avg_key reqtime] }
+      let(:expected) do
+        {
+          "200_GET_/ping_count"=>2, "200_GET_/ping_reqtime_avg"=>1.001,
+          "200_POST_/auth_count"=>1, "200_POST_/auth_reqtime_avg"=>1.001,
+          "400_GET_/ping_count"=>1, "400_GET_/ping_reqtime_avg"=>3.003,
+        }
+      end
+      before do
+        Fluent::Engine.stub(:now).and_return(time)
+        Fluent::Engine.should_receive(:emit).with("count.#{tag}", time, expected)
+      end
+      it { emit }
+    end
+
+    context 'avg_suffix' do
+      let(:config) { CONFIG + %[avg_key reqtime \n avg_suffix /avg] }
+      let(:expected) do
+        {
+          "200_GET_/ping_count"=>2, "200_GET_/ping_reqtime/avg"=>1.001,
+          "200_POST_/auth_count"=>1, "200_POST_/auth_reqtime/avg"=>1.001,
+          "400_GET_/ping_count"=>1, "400_GET_/ping_reqtime/avg"=>3.003,
+        }
+      end
+      before do
+        Fluent::Engine.stub(:now).and_return(time)
+        Fluent::Engine.should_receive(:emit).with("count.#{tag}", time, expected)
+      end
+      it { emit }
+    end
+
+    context 'tag' do
+      context 'not effective if output_per_tag true' do
+        let(:config) do
+          CONFIG + %[
+          output_per_tag true
+          tag foo
+          ]
+        end
+        before do
+          Fluent::Engine.stub(:now).and_return(time)
+          Fluent::Engine.should_receive(:emit).with("count.#{tag}", time, expected)
+        end
+        it { emit }
+      end
+
+      context 'effective if output_per_tag false' do
+        let(:config) do
+          CONFIG + %[
+          output_per_tag false
+          tag foo
+          ]
+        end
+        before do
+          Fluent::Engine.stub(:now).and_return(time)
+          Fluent::Engine.should_receive(:emit).with("foo", time, expected_with_tag)
+        end
+        it { emit }
+      end
+    end
+
+    context 'tag_prefix' do
+      let(:config) do
+        CONFIG + %[
+          tag_prefix foo
+        ]
+      end
+      before do
+        Fluent::Engine.stub(:now).and_return(time)
+        Fluent::Engine.should_receive(:emit).with("foo.#{tag}", time, expected)
+      end
+      it { emit }
+    end
+
+    context 'aggregate all' do
+      let(:emit) do
+        driver.run { messages.each {|message| driver.emit_with_tag(message, time, 'foo.bar') } }
+        driver.run { messages.each {|message| driver.emit_with_tag(message, time, 'foo.bar2') } }
+        driver.instance.flush_emit
+      end
+
+      let(:config) do
+        CONFIG + %[
+          aggregate all
+          tag_prefix count
+        ]
+      end
+
+      before do
+        Fluent::Engine.stub(:now).and_return(time)
+        Fluent::Engine.should_receive(:emit).with("count.all", time, {
+          "200_GET_/ping_count"=>4, "200_POST_/auth_count"=>2, "400_GET_/ping_count"=>2
+        })
+      end
+      it { emit }
+    end
+
+    context "store_file" do
+      let(:store_file) do
+        dirname = "tmp"
+        Dir.mkdir dirname unless Dir.exist? dirname
+        filename = "#{dirname}/test.dat"
+        File.unlink filename if File.exist? filename
+        filename
+      end
+
+      let(:config) do
+        CONFIG + %[
+          store_file #{store_file}
+          ]
+      end
+
+      it 'stored_data and loaded_data should equal' do
+        driver.run { messages.each {|message| driver.emit({'message' => message}, time) } }
+        driver.instance.shutdown
+        stored_counts = driver.instance.counts
+        stored_saved_at = driver.instance.saved_at
+        stored_saved_duration = driver.instance.saved_duration
+        driver.instance.counts = {}
+        driver.instance.saved_at = nil
+        driver.instance.saved_duration = nil
+
+        driver.instance.start
+        loaded_counts = driver.instance.counts
+        loaded_saved_at = driver.instance.saved_at
+        loaded_saved_duration = driver.instance.saved_duration
+
+        loaded_counts.should == stored_counts
+        loaded_saved_at.should == stored_saved_at
+        loaded_saved_duration.should == stored_saved_duration
+      end
+    end
+
+    context 'group_by_expression' do
+      let(:config) { CONFIG + %[group_by_expression ${method}_${path.split("?")[0].split("/")[2]}/${code[0]}xx] }
+      let(:messages) do
+        [
+          {"code" => "200", "method" => "GET",  "path" => "/api/people/@me/@self?count=1", "reqtime" => 0.000 },
+          {"code" => "200", "method" => "POST", "path" => "/api/ngword?_method=check", "reqtime" => 1.001 },
+          {"code" => "400", "method" => "GET",  "path" => "/api/messages/@me/@outbox", "reqtime" => 2.002 },
+          {"code" => "201", "method" => "GET",  "path" => "/api/people/@me/@self", "reqtime" => 3.003 },
+        ]
+      end
+      let(:expected) do
+        {
+          "GET_people/2xx_count"=>2,
+          "POST_ngword/2xx_count"=>1,
+          "GET_messages/4xx_count"=>1,
+        }
+      end
+      before do
+        Fluent::Engine.stub(:now).and_return(time)
+        Fluent::Engine.should_receive(:emit).with("count.#{tag}", time, expected)
+      end
+      it { emit }
+    end
+
+  end
+end
+
+
+

data/spec/spec_helper.rb

@@ -0,0 +1,13 @@
+# encoding: UTF-8
+require 'rubygems'
+require 'bundler'
+Bundler.setup(:default, :test)
+Bundler.require(:default, :test)
+
+require 'fluent/test'
+require 'rspec'
+require 'pry'
+
+$TESTING=true
+$:.unshift File.join(File.dirname(__FILE__), '..', 'lib')
+require 'fluent/plugin/out_groupcounter'

metadata

@@ -1,8 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-groupcounter
 version: !ruby/object:Gem::Version
-  version: 0.1.0
-  prerelease: 
+  version: 0.2.0
 platform: ruby
 authors:
 - Ryosuke IWANAGA
@@ -10,98 +9,120 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-02-06 00:00:00.000000000 Z
+date: 2013-08-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: fluentd
+  name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-nav
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-description: Fluentd plugin to count like COUNT(\*) GROUP BY
+description: Fluentd plugin to count like SELECT COUNT(\*) GROUP BY
 email:
-- ! '@riywo'
-- ! '@sonots'
+- '@riywo'
+- sonots@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .pryrc
+- .travis.yml
 - Gemfile
 - README.md
 - Rakefile
 - fluent-plugin-groupcounter.gemspec
 - lib/fluent/plugin/out_groupcounter.rb
+- spec/out_groupcounter_spec.rb
+- spec/spec_helper.rb
 homepage: https://github.com/riywo/fluent-plugin-groupcounter
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: -1667358742827062990
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: -1667358742827062990
 requirements: []
 rubyforge_project: fluent-plugin-groupcounter
-rubygems_version: 1.8.23
+rubygems_version: 2.0.2
 signing_key: 
-specification_version: 3
-summary: Fluentd plugin to count like COUNT(\*) GROUP BY
-test_files: []
+specification_version: 4
+summary: Fluentd plugin to count like SELECT COUNT(\*) GROUP BY
+test_files:
+- spec/out_groupcounter_spec.rb
+- spec/spec_helper.rb
+has_rdoc: