fluent-plugin-grepcounter 0.0.1 → 0.1.0.pre

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e62f5b7c59d5196eb5a38630bb8a1292883f1d07
-  data.tar.gz: 64a2bede2b197b7f693207bc63b6a71101df8c50
+  metadata.gz: c342677be8075559528320aa8964dd2208f6d2cb
+  data.tar.gz: 28f6b36798b5023e180a9247ef6480b3997633c6
 SHA512:
-  metadata.gz: 49ae4eca6395c2b381631ced8bebdb30d3451cd4f504a817b2ec858a45c9acbd198385c465062f93968e547cf49c52ec6f124d6e198b9a72457c1048a37e9cd7
-  data.tar.gz: f9d7d8b9c50c6d45e22af946edef16d440e147c76f1e2507bf48dd272c837dd52282db09fa9d3411aa6cf44971c35f8c410a40622ac746b23118ec041286be79
+  metadata.gz: 96e1fd97aee7feeb3f2843cac84558b4163e95bb175750e766bf5ea66da275556f75b425669c505ab51468b61a1047e8517aa4605f139358acdae4e92d96a7bb
+  data.tar.gz: f6fc309ed2057081fe6a724a213f491c146dede6cad7d490ca92c186918a3e5e09874cce81d6e1da18c0de9feff0395956197b70f74d6bb0b2520eed842675e9

data/.coveralls.yml

@@ -0,0 +1 @@
+repo_token: i4DJCtdksuIwhBck1tukIjzKoMCxWIIvQ

data/.gitignore

@@ -8,4 +8,5 @@ Gemfile.lock
 vendor
 doc/*
 tmp/*
+coverage
 .yardoc

data/CHANGELOG.md

@@ -0,0 +1,16 @@
+## 0.1.0 (2013/05/01)
+
+Features:
+
+  - aggregate tag/all
+
+Changes:
+
+  - Remove `output_matched_message` option, and make it default. 
+  - Rename `output_with_joined_delimiter` to `output_delimiter`. 
+
+Bugfixes:
+
+## 0.0.1
+
+First version

data/Gemfile

@@ -1,3 +1,4 @@
 source "http://rubygems.org"
 
+gem 'simplecov', git: 'https://github.com/colszowka/simplecov.git'
 gemspec

data/README.md

@@ -1,15 +1,9 @@
-# fluent-plugin-grepcounter [![Build Status](https://secure.travis-ci.org/sonots/fluent-plugin-grepcounter.png?branch=master)](http://travis-ci.org/sonots/fluent-plugin-grepcounter) [![Dependency Status](https://gemnasium.com/sonots/fluent-plugin-grepcounter.png)](https://gemnasium.com/sonots/fluent-plugin-grepcounter)
+# fluent-plugin-grepcounter [![Build Status](https://secure.travis-ci.org/sonots/fluent-plugin-grepcounter.png?branch=master)](http://travis-ci.org/sonots/fluent-plugin-grepcounter) [![Dependency Status](https://gemnasium.com/sonots/fluent-plugin-grepcounter.png)](https://gemnasium.com/sonots/fluent-plugin-grepcounter) [![Coverage Status](https://coveralls.io/repos/sonots/fluent-plugin-grepcounter/badge.png?branch=master)](https://coveralls.io/r/sonots/fluent-plugin-grepcounter)
 
-## Component
-
-### GrepCounterOutput
-
-Fluentd plugin to count the number of matched messages
+Fluentd plugin to count the number of matched messages.
 
 ## Configuration
 
-## GrepCounterOutput
-
 Assume inputs from another plugin are as belows:
 
     syslog.host1: {"message":"2013/01/13T07:02:11.124202 INFO GET /ping" }
@@ -29,11 +23,11 @@ An example of grepcounter configuration:
       add_tag_prefix warn.count
     </source>
 
-Outputs as belows:
+Then, output bocomes as belows:
 
-    warn.count.syslog.host1: {"count":2,"input_tag":"syslog.host1","input_tag_last":"host1"}
+    warn.count.syslog.host1: {"count":2,"message":["2013/01/13T07:02:13.232645 WARN POST /auth","2013/01/13T07:02:43.632145 WARN POST /login"]}
 
-Another example of grepcounter configuration:
+Another example of grepcounter configuration to use `output_delimiter`:
 
     <match syslog.**>
       type grepcounter
@@ -43,13 +37,26 @@ Another example of grepcounter configuration:
       exclude favicon.ico
       threshold 1
       add_tag_prefix warn.count
-      output_matched_message true
-      output_with_joined_delimiter \n
+      output_delimiter \n
     </source>
 
-Outputs as belows:
+Then, output bocomes as belows. You can use the `message` field is joined with \n.
+
+    warn.count.syslog.host1: {"count":2,"message":"2013/01/13T07:02:13.232645 WARN POST /auth\n2013/01/13T07:02:43.632145 WARN POST /login"}
+
+## Parameters
+
+- aggregate
 
-    warn.count.syslog.host1: {"count":2,"input_tag":"syslog.host1","input_tag_last":"host1","message":"2013/01/13T07:02:13.232645 WARN POST /auth\n2013/01/13T07:02:43.632145 WARN POST /login"}
+    Count by each `tag` or `all`. The default value is `tag`. 
+
+- output_tag
+
+    The output tag. Required for aggregate `all`. 
+
+## ChaangeLog
+
+See [CHANGELOG.md](CHANGELOG.md) for details.
 
 ## Contributing
 
@@ -62,3 +69,4 @@ Outputs as belows:
 ## Copyright
 
 Copyright (c) 2013 Naotoshi SEO. See [LICENSE](LICENSE) for details.
+

data/fluent-plugin-grepcounter.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-grepcounter"
-  s.version     = "0.0.1"
+  s.version     = "0.1.0.pre"
   s.authors     = ["Naotoshi SEO"]
   s.email       = ["sonots@gmail.com"]
   s.homepage    = "https://github.com/sonots/fluent-plugin-grepcounter"
@@ -21,5 +21,6 @@ Gem::Specification.new do |s|
   s.add_development_dependency "rake"
   s.add_development_dependency "rspec"
   s.add_development_dependency "pry"
+  s.add_development_dependency 'coveralls'
   # s.add_development_dependency "delorean"
 end

data/lib/fluent/plugin/out_grepcounter.rb

@@ -1,15 +1,16 @@
+# encoding: UTF-8
 class Fluent::GrepCounterOutput < Fluent::Output
   Fluent::Plugin.register_output('grepcounter', self)
 
   config_param :input_key, :string
-  config_param :regexp, :string
+  config_param :regexp, :string, :default => nil
   config_param :count_interval, :time, :default => 5
   config_param :exclude, :string, :default => nil
   config_param :threshold, :integer, :default => 1
-  config_param :output_tag, :string, :default => 'count'
-  config_param :add_tag_prefix, :string, :default => nil
-  config_param :output_matched_message, :bool, :default => false
-  config_param :output_with_joined_delimiter, :string, :default => nil
+  config_param :output_tag, :string, :default => nil
+  config_param :add_tag_prefix, :string, :default => 'count'
+  config_param :output_delimiter, :string, :default => nil
+  config_param :aggregate, :string, :default => 'tag'
 
   attr_accessor :matches
   attr_accessor :last_checked
@@ -23,8 +24,15 @@ class Fluent::GrepCounterOutput < Fluent::Output
     @exclude = Regexp.compile(@exclude) if @exclude
     @threshold = @threshold.to_i
 
-    if @output_with_joined_delimiter and @output_matched_message == false
-      raise Fluent::ConfigError, "'output_matched_message' must be true to use 'output_with_joined_delimiter'"
+    unless ['tag', 'all'].include?(@aggregate)
+      raise Fluent::ConfigError, "grepcounter aggregate allows tag/all"
+    end
+
+    case @aggregate
+    when 'all'
+      raise Fluent::ConfigError, "output_tag must be specified with aggregate all" if @output_tag.nil?
+    when 'tag'
+      # raise Fluent::ConfigError, "add_tag_prefix must be specified with aggregate tag" if @add_tag_prefix.nil?
     end
 
     @matches = {}
@@ -49,9 +57,9 @@ class Fluent::GrepCounterOutput < Fluent::Output
     # filter out and insert
     es.each do |time,record|
       value = record[@input_key]
-      next unless @regexp and @regexp.match(value)
+      next if @regexp and !@regexp.match(value)
       next if @exclude and @exclude.match(value)
-      matches << value if @output_matched_message
+      matches << value
       count += 1
     end
     # thread safe merge
@@ -83,23 +91,34 @@ class Fluent::GrepCounterOutput < Fluent::Output
   def flush_emit(step)
     time = Fluent::Engine.now
     flushed_counts, flushed_matches, @counts, @matches = @counts, @matches, {}, {}
-    flushed_counts.keys.each do |tag|
-      count = flushed_counts[tag]
-      matches = flushed_matches[tag]
-      output = generate_output(tag, count, matches)
-      tag = @add_tag_prefix ? "#{@add_tag_prefix}.#{tag}" : @output_tag
-      Fluent::Engine.emit(tag, time, output) if output
+
+    if @aggregate == 'all'
+      count = 0; matches = []
+      flushed_counts.keys.each do |tag|
+        count += flushed_counts[tag]
+        matches += flushed_matches[tag]
+      end
+      output = generate_output(count, matches)
+      Fluent::Engine.emit(@output_tag, time, output) if output
+    else
+      flushed_counts.keys.each do |tag|
+        count = flushed_counts[tag]
+        matches = flushed_matches[tag]
+        output = generate_output(count, matches, tag)
+        tag = @output_tag ? @output_tag : "#{@add_tag_prefix}.#{tag}"
+        Fluent::Engine.emit(tag, time, output) if output
+      end
     end
   end
 
-  def generate_output(input_tag, count, matches)
+  def generate_output(count, matches, tag = nil)
     return nil if count < @threshold
     output = {}
     output['count'] = count
-    output['input_tag'] = input_tag
-    output['input_tag_last'] = input_tag.split(".").last
-    if @output_matched_message
-      output['message'] = @output_with_joined_delimiter.nil? ? matches : matches.join(@output_with_joined_delimiter)
+    output['message'] = @output_delimiter.nil? ? matches : matches.join(@output_delimiter)
+    if tag
+      output['input_tag'] = tag
+      output['input_tag_last'] = tag.split('.').last
     end
     output
   end

data/spec/out_grepcounter_spec.rb

@@ -5,7 +5,6 @@ describe Fluent::GrepCounterOutput do
   before { Fluent::Test.setup }
   CONFIG = %[
     input_key message
-    regexp WARN
   ]
   let(:tag) { 'syslog.host1' }
   let(:driver) { Fluent::Test::OutputTestDriver.new(Fluent::GrepCounterOutput, tag).configure(config) }
@@ -17,8 +16,21 @@ describe Fluent::GrepCounterOutput do
         it { expect { driver }.to raise_error(Fluent::ConfigError) }
       end
 
-      context "check output_with_joined_delimiter" do
-        let(:config) { CONFIG + %[ output_with_joined_delimiter \\n ] }
+      context 'invalid aggregate' do
+        let(:config) do
+          CONFIG + %[
+          aggregate foo
+          ]
+        end
+        it { expect { driver }.to raise_error(Fluent::ConfigError) }
+      end
+
+      context 'no tag for aggregate all' do
+        let(:config) do
+          CONFIG + %[
+          aggregate all
+          ]
+        end
         it { expect { driver }.to raise_error(Fluent::ConfigError) }
       end
     end
@@ -30,13 +42,11 @@ describe Fluent::GrepCounterOutput do
         let(:config) { CONFIG }
         its(:input_key) { should == "message" }
         its(:count_interval) { should == 5 }
-        its(:regexp) { should == /WARN/ }
+        its(:regexp) { should be_nil }
         its(:exclude) { should be_nil }
         its(:threshold) { should == 1 }
-        its(:output_tag) { should == 'count' }
-        its(:add_tag_prefix) { should be_nil }
-        its(:output_matched_message) { should be_false }
-        its(:output_with_joined_delimiter) { should be_nil }
+        its(:output_tag) { should be_nil }
+        its(:add_tag_prefix) { should == 'count' }
       end
     end
   end
@@ -60,11 +70,28 @@ describe Fluent::GrepCounterOutput do
       pending
     end
 
+    context 'default' do
+      let(:config) { CONFIG }
+      before do
+        Fluent::Engine.stub(:now).and_return(time)
+        Fluent::Engine.should_receive(:emit).with("count.#{tag}", time, {"count"=>4,
+          "message"=>["2013/01/13T07:02:11.124202 INFO GET /ping","2013/01/13T07:02:13.232645 WARN POST /auth","2013/01/13T07:02:21.542145 WARN GET /favicon.ico","2013/01/13T07:02:43.632145 WARN POST /login"],
+          "input_tag" => tag,
+          "input_tag_last" => tag.split('.').last,
+        })
+      end
+      it { emit }
+    end
+
     context 'regexp' do
       let(:config) { CONFIG + %[ regexp WARN ] }
       before do
         Fluent::Engine.stub(:now).and_return(time)
-        Fluent::Engine.should_receive(:emit).with("count", time, {"count"=>3, "input_tag"=>tag, "input_tag_last"=>tag.split(".").last})
+        Fluent::Engine.should_receive(:emit).with("count.#{tag}", time, {"count"=>3,
+          "message"=>["2013/01/13T07:02:13.232645 WARN POST /auth","2013/01/13T07:02:21.542145 WARN GET /favicon.ico","2013/01/13T07:02:43.632145 WARN POST /login"],
+          "input_tag" => tag,
+          "input_tag_last" => tag.split('.').last,
+        })
       end
       it { emit }
     end
@@ -72,12 +99,17 @@ describe Fluent::GrepCounterOutput do
     context 'exclude' do
       let(:config) do
         CONFIG + %[
+          regexp WARN
           exclude favicon
         ]
       end
       before do
         Fluent::Engine.stub(:now).and_return(time)
-        Fluent::Engine.should_receive(:emit).with("count", time, {"count"=>2, "input_tag"=>tag, "input_tag_last"=>tag.split(".").last})
+        Fluent::Engine.should_receive(:emit).with("count.#{tag}", time, {"count"=>2,
+          "message"=>["2013/01/13T07:02:13.232645 WARN POST /auth","2013/01/13T07:02:43.632145 WARN POST /login"],
+          "input_tag" => tag,
+          "input_tag_last" => tag.split('.').last,
+        })
       end
       it { emit }
     end
@@ -85,12 +117,17 @@ describe Fluent::GrepCounterOutput do
     context 'threshold (less than or equal to)' do
       let(:config) do
         CONFIG + %[
+          regexp WARN
           threshold 3
         ]
       end
       before do
         Fluent::Engine.stub(:now).and_return(time)
-        Fluent::Engine.should_receive(:emit).with("count", time, {"count"=>3, "input_tag"=>tag, "input_tag_last"=>tag.split(".").last})
+        Fluent::Engine.should_receive(:emit).with("count.#{tag}", time, {"count"=>3,
+          "message"=>["2013/01/13T07:02:13.232645 WARN POST /auth","2013/01/13T07:02:21.542145 WARN GET /favicon.ico","2013/01/13T07:02:43.632145 WARN POST /login"],
+          "input_tag" => tag,
+          "input_tag_last" => tag.split('.').last,
+        })
       end
       it { emit }
     end
@@ -98,6 +135,7 @@ describe Fluent::GrepCounterOutput do
     context 'threshold (greater)' do
       let(:config) do
         CONFIG + %[
+          regexp WARN
           threshold 4
         ]
       end
@@ -111,12 +149,17 @@ describe Fluent::GrepCounterOutput do
     context 'output_tag' do
       let(:config) do
         CONFIG + %[
+          regexp WARN
           output_tag foo
         ]
       end
       before do
         Fluent::Engine.stub(:now).and_return(time)
-        Fluent::Engine.should_receive(:emit).with("foo", time, {"count"=>3, "input_tag"=>tag, "input_tag_last"=>tag.split(".").last})
+        Fluent::Engine.should_receive(:emit).with("foo", time, {"count"=>3,
+          "message"=>["2013/01/13T07:02:13.232645 WARN POST /auth","2013/01/13T07:02:21.542145 WARN GET /favicon.ico","2013/01/13T07:02:43.632145 WARN POST /login"],
+          "input_tag" => tag,
+          "input_tag_last" => tag.split('.').last,
+        })
       end
       it { emit }
     end
@@ -124,45 +167,52 @@ describe Fluent::GrepCounterOutput do
     context 'add_tag_prefix' do
       let(:config) do
         CONFIG + %[
+          regexp WARN
           add_tag_prefix foo
         ]
       end
       before do
         Fluent::Engine.stub(:now).and_return(time)
-        Fluent::Engine.should_receive(:emit).with("foo.#{tag}", time, {"count"=>3, "input_tag"=>tag, "input_tag_last"=>tag.split(".").last})
+        Fluent::Engine.should_receive(:emit).with("foo.#{tag}", time, {"count"=>3,
+          "message"=>["2013/01/13T07:02:13.232645 WARN POST /auth","2013/01/13T07:02:21.542145 WARN GET /favicon.ico","2013/01/13T07:02:43.632145 WARN POST /login"],
+          "input_tag" => tag,
+          "input_tag_last" => tag.split('.').last,
+        })
       end
       it { emit }
     end
 
-    context 'output_matched_message' do
+    context 'output_delimiter' do
       let(:config) do
+        # \\n shall be \n in config file
         CONFIG + %[
-          output_matched_message true
+          regexp WARN
+          output_delimiter \\n
         ]
       end
       before do
         Fluent::Engine.stub(:now).and_return(time)
-        Fluent::Engine.should_receive(:emit).with("count", time, {
-          "count"=>3, "input_tag"=>tag, "input_tag_last"=>tag.split(".").last,
-          "message"=>["2013/01/13T07:02:13.232645 WARN POST /auth","2013/01/13T07:02:21.542145 WARN GET /favicon.ico","2013/01/13T07:02:43.632145 WARN POST /login"]
+        Fluent::Engine.should_receive(:emit).with("count.#{tag}", time, {"count"=>3, 
+          "message"=>"2013/01/13T07:02:13.232645 WARN POST /auth\\n2013/01/13T07:02:21.542145 WARN GET /favicon.ico\\n2013/01/13T07:02:43.632145 WARN POST /login",
+          "input_tag" => tag,
+          "input_tag_last" => tag.split('.').last,
         })
       end
       it { emit }
     end
 
-    context 'output_with_joined_delimiter' do
+    context 'aggregate all' do
       let(:config) do
-        # \\n shall be \n in config file
         CONFIG + %[
-          output_matched_message true
-          output_with_joined_delimiter \\n
+          regexp WARN
+          aggregate all
+          output_tag count
         ]
       end
       before do
         Fluent::Engine.stub(:now).and_return(time)
-        Fluent::Engine.should_receive(:emit).with("count", time, {
-          "count"=>3, "input_tag"=>tag, "input_tag_last"=>tag.split(".").last,
-          "message"=>"2013/01/13T07:02:13.232645 WARN POST /auth\\n2013/01/13T07:02:21.542145 WARN GET /favicon.ico\\n2013/01/13T07:02:43.632145 WARN POST /login"
+        Fluent::Engine.should_receive(:emit).with("count", time, {"count"=>3, 
+          "message"=>["2013/01/13T07:02:13.232645 WARN POST /auth","2013/01/13T07:02:21.542145 WARN GET /favicon.ico","2013/01/13T07:02:43.632145 WARN POST /login"],
         })
       end
       it { emit }

data/spec/spec_helper.rb

@@ -4,6 +4,9 @@ require 'bundler'
 Bundler.setup(:default, :test)
 Bundler.require(:default, :test)
 
+require 'coveralls'
+Coveralls.wear!
+
 require 'fluent/test'
 require 'rspec'
 require 'pry'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-grepcounter
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.0.pre
 platform: ruby
 authors:
 - Naotoshi SEO
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-11 00:00:00.000000000 Z
+date: 2013-05-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -66,6 +66,20 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: coveralls
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Count the number of matched messages
 email:
 - sonots@gmail.com
@@ -73,10 +87,12 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- .coveralls.yml
 - .gitignore
 - .rdebugrc
 - .rspec
 - .travis.yml
+- CHANGELOG.md
 - Gemfile
 - LICENSE
 - README.md
@@ -99,9 +115,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - '>'
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: fluent-plugin-grepcounter
 rubygems_version: 2.0.0