fluent-plugin-grepcounter 0.4.2 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3b6b29ceafed9d27f2a47aaae6f4509944a958c3
-  data.tar.gz: a7efb1d61d2a585f49a81a22b0c3d3dfa0b122f4
+  metadata.gz: 905021d7ccf6788eb182017c59884b5071bd9e87
+  data.tar.gz: 25501e79fe3b478a3cbc3a2b5fa96beae5bbbe6b
 SHA512:
-  metadata.gz: f6df4866a77929e798551961c6349ceb88d80b58f7959af60536b12cbcf4bd93e93311823da8abb882828f4cf6f3b313d2c50f34d682e700b6ae7f4f2054e19e
-  data.tar.gz: 78b66fd77923e02947ae8a1a45da3e89e5c30eba12bb96f7970a80f109f91f8956f6233c3c0e400958f2c9c8500b66f7238b6ec471f32273dd005a6c9d7f23dd
+  metadata.gz: 084d066b3dbce3bb8686955ad34822332c963f57239ac6459761b04605de0c3bfa61288cfbe5b8c1b725a777cafc007f7ee044e9214c5c55a05796143e7e0a20
+  data.tar.gz: 8351db08a7cdde4d550ca1303f3352c003c24d0dd1ca304a258fd999be2253535434de78d6554e7977965b85647d811421d07f45183d5fdca98f651af3b4af29

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.5.0 (2013/12/17)
+
+Features
+
+  - Add `regexpN` and `excludeN` options
+
 ## 0.4.2 (2013/12/12)
 
 Features

data/README.md

@@ -1,4 +1,4 @@
-# fluent-plugin-grepcounter [![Build Status](https://secure.travis-ci.org/sonots/fluent-plugin-grepcounter.png?branch=master)](http://travis-ci.org/sonots/fluent-plugin-grepcounter) [![Dependency Status](https://gemnasium.com/sonots/fluent-plugin-grepcounter.png)](https://gemnasium.com/sonots/fluent-plugin-grepcounter)
+# fluent-plugin-grepcounter [![Build Status](https://secure.travis-ci.org/sonots/fluent-plugin-grepcounter.png?branch=master)](http://travis-ci.org/sonots/fluent-plugin-grepcounter)
 
 Fluentd plugin to count the number of matched messages, and emit if exeeds the `threshold`. 
 
@@ -63,18 +63,26 @@ Then, output bocomes as belows (indented). You can see the `message` field is jo
 
     The interval time to count in seconds. Default is 60.
 
-- input\_key
+- input\_key *field\_key*
 
-    The target field key to grep out
+    The target field key to grep out. Use with regexp or exclude. 
 
-- regexp
+- regexp *regexp*
 
     The filtering regular expression
 
-- exclude
+- exclude *regexp*
 
     The excluding regular expression like grep -v
 
+- regexp[1-20] *field\_key* *regexp* (experimental)
+
+    The target field key and the filtering regular expression to grep out. 
+
+- exclude[1-20] *field_key* *regexp* (experimental)
+
+    The target field key and the excluding regular expression like grep -v
+
 - threshold
 
     The threshold number to emit. Emit if `count` value >= specified value.

data/fluent-plugin-grepcounter.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-grepcounter"
-  s.version     = "0.4.2"
+  s.version     = "0.5.0"
   s.authors     = ["Naotoshi Seo"]
   s.email       = ["sonots@gmail.com"]
   s.homepage    = "https://github.com/sonots/fluent-plugin-grepcounter"

data/lib/fluent/plugin/out_grepcounter.rb

@@ -2,16 +2,20 @@
 class Fluent::GrepCounterOutput < Fluent::Output
   Fluent::Plugin.register_output('grepcounter', self)
 
+  REGEXP_MAX_NUM = 20
+
   def initialize
     super
     require 'pathname'
   end
 
-  config_param :input_key, :string
+  config_param :input_key, :string, :default => nil
   config_param :regexp, :string, :default => nil
-  config_param :count_interval, :time, :default => 5
   config_param :exclude, :string, :default => nil
-  config_param :threshold, :integer, :default => nil # obsolete
+  (1..REGEXP_MAX_NUM).each {|i| config_param :"regexp#{i}",  :string, :default => nil }
+  (1..REGEXP_MAX_NUM).each {|i| config_param :"exclude#{i}", :string, :default => nil }
+  config_param :count_interval, :time, :default => 5
+  config_param :threshold, :integer, :default => nil # not obsolete, though
   config_param :comparator, :string, :default => '>=' # obsolete
   config_param :less_than, :float, :default => nil
   config_param :less_equal, :float, :default => nil
@@ -36,57 +40,56 @@ class Fluent::GrepCounterOutput < Fluent::Output
   def configure(conf)
     super
 
-    @count_interval = @count_interval.to_i
-    @input_key = @input_key.to_s
-    @regexp = Regexp.compile(@regexp) if @regexp
-    @exclude = Regexp.compile(@exclude) if @exclude
+    if @input_key
+      @regexp = Regexp.compile(@regexp) if @regexp
+      @exclude = Regexp.compile(@exclude) if @exclude
+    end
 
-    @threshold = @threshold.to_i if @threshold
+    @regexps = {}
+    (1..REGEXP_MAX_NUM).each do |i|
+      next unless conf["regexp#{i}"]
+      key, regexp = conf["regexp#{i}"].split(/ /, 2)
+      raise Fluent::ConfigError, "regexp#{i} does not contain 2 parameters" unless regexp
+      raise Fluent::ConfigError, "regexp#{i} contains a duplicated key, #{key}" if @regexps[key]
+      @regexps[key] = Regexp.compile(regexp)
+    end
 
-    unless ['>=', '<='].include?(@comparator)
-      raise Fluent::ConfigError, "grepcounter: comparator allows >=, <="
+    @excludes = {}
+    (1..REGEXP_MAX_NUM).each do |i|
+      next unless conf["exclude#{i}"]
+      key, exclude = conf["exclude#{i}"].split(/ /, 2)
+      raise Fluent::ConfigError, "exclude#{i} does not contain 2 parameters" unless exclude
+      raise Fluent::ConfigError, "exclude#{i} contains a duplicated key, #{key}" if @excludes[key]
+      @excludes[key] = Regexp.compile(exclude)
     end
 
+    if @input_key and (!@regexps.empty? or !@excludes.empty?)
+      raise Fluent::ConfigError, "Classic style `input_key`, and new style `regexpN`, `excludeN` can not be used together"
+    end
+
+    # to support obsolete options
+    @tag ||= @output_tag
+    @delimiter ||= @output_with_joined_delimiter
+
     # to support obsolete `threshold` and `comparator` options
     if @threshold.nil? and @less_than.nil? and @less_equal.nil? and @greater_than.nil? and @greater_equal.nil?
       @threshold = 1
     end
-    if @threshold and @comparator
-      if @comparator == '>='
+    unless %w[>= <=].include?(@comparator)
+      raise Fluent::ConfigError, "grepcounter: comparator allows >=, <="
+    end
+    if @threshold
+      case @comparator
+      when '>='
         @greater_equal = @threshold
       else
         @less_equal = @threshold
       end
     end
 
-    # to support osolete `output_tag` option
-    @tag = @output_tag if !@tag and @output_tag
-
-    # to support obsolete `output_with_joined_delimiter` option
-    @delimiter = @output_with_joined_delimiter if !@delimiter and @output_with_joined_delimiter
-
-    unless ['tag', 'all'].include?(@aggregate)
-      raise Fluent::ConfigError, "grepcounter: aggregate allows tag/all"
-    end
-
-    case @aggregate
-    when 'all'
-      raise Fluent::ConfigError, "grepcounter: output_tag must be specified with aggregate all" if @output_tag.nil?
-    when 'tag'
-      # raise Fluent::ConfigError, "grepcounter: add_tag_prefix must be specified with aggregate tag" if @add_tag_prefix.nil?
-    end
-
-    if @store_file
-      f = Pathname.new(@store_file)
-      if (f.exist? && !f.writable_real?) || (!f.exist? && !f.parent.writable_real?)
-        raise Fluent::ConfigError, "#{@store_file} is not writable"
-      end
-    end
-
     if @tag.nil? and @add_tag_prefix.nil? and @remove_tag_prefix.nil?
       @add_tag_prefix = 'count' # not ConfigError to support lower version compatibility
     end
-
     @tag_prefix = "#{@add_tag_prefix}." if @add_tag_prefix
     @tag_prefix_match = "#{@remove_tag_prefix}." if @remove_tag_prefix
     @tag_proc =
@@ -102,6 +105,22 @@ class Fluent::GrepCounterOutput < Fluent::Output
         Proc.new {|tag| tag }
       end
 
+    case @aggregate
+    when 'all'
+      raise Fluent::ConfigError, "grepcounter: `tag` must be specified with aggregate all" if @tag.nil?
+    when 'tag'
+      # raise Fluent::ConfigError, "grepcounter: add_tag_prefix must be specified with aggregate tag" if @add_tag_prefix.nil?
+    else
+      raise Fluent::ConfigError, "grepcounter: aggregate allows tag/all"
+    end
+
+    if @store_file
+      f = Pathname.new(@store_file)
+      if (f.exist? && !f.writable_real?) || (!f.exist? && !f.parent.writable_real?)
+        raise Fluent::ConfigError, "#{@store_file} is not writable"
+      end
+    end
+
     @matches = {}
     @counts  = {}
     @mutex = Mutex.new
@@ -125,10 +144,23 @@ class Fluent::GrepCounterOutput < Fluent::Output
     count = 0; matches = []
     # filter out and insert
     es.each do |time,record|
-      value = record[@input_key]
-      next unless match(value.to_s)
-      matches << value
-      count += 1
+      catch(:break_loop) do
+        if key = @input_key
+          value = record[key].to_s
+          throw :break_loop if @regexp and !match(@regexp, value)
+          throw :break_loop if @exclude and match(@exclude, value)
+          matches << value # old style stores as an array of values
+        else
+          @regexps.each do |key, regexp|
+            throw :break_loop unless match(regexp, record[key].to_s)
+          end
+          @excludes.each do |key, exclude|
+            throw :break_loop if match(exclude, record[key].to_s)
+          end
+          matches << record # new style stores as an array of hashes, but how to utilize it?
+        end
+        count += 1
+      end
     end
     # thread safe merge
     @counts[tag] ||= 0
@@ -196,7 +228,12 @@ class Fluent::GrepCounterOutput < Fluent::Output
     return nil if @greater_equal and count <  @greater_equal
     output = {}
     output['count'] = count
-    output['message'] = @delimiter.nil? ? matches : matches.join(@delimiter)
+    if @input_key
+      output['message'] = @delimiter ? matches.join(@delimiter) : matches
+    else
+      # I will think of good format later...
+      output['message'] = @delimiter ? matches.map{|hash| hash.to_hash}.join(@delimiter) : matches
+    end
     if tag
       output['input_tag'] = tag
       output['input_tag_last'] = tag.split('.').last
@@ -208,10 +245,9 @@ class Fluent::GrepCounterOutput < Fluent::Output
     string.index(substring) == 0 ? string[substring.size..-1] : string
   end
 
-  def match(string)
+  def match(regexp, string)
     begin
-      return false if @regexp and !@regexp.match(string)
-      return false if @exclude and @exclude.match(string)
+      return regexp.match(string)
     rescue ArgumentError => e
       raise e unless e.message.index("invalid byte sequence in") == 0
       string = replace_invalid_byte(string)

data/spec/out_grepcounter_spec.rb

@@ -18,8 +18,8 @@ describe Fluent::GrepCounterOutput do
 
   describe 'test configure' do
     describe 'bad configuration' do
-      context "lack of requirements" do
-        let(:config) { '' }
+      context 'should not use classic style and new style together' do
+        let(:config) { %[input_key message\nregexp1 message foo] }
         it { expect { driver }.to raise_error(Fluent::ConfigError) }
       end
 
@@ -109,6 +109,22 @@ describe Fluent::GrepCounterOutput do
       it { emit }
     end
 
+    context 'regexpN' do
+      let(:config) { %[ regexp1 message WARN ] }
+      before do
+        Fluent::Engine.stub(:now).and_return(time)
+        Fluent::Engine.should_receive(:emit).with("count.#{tag}", time, expected.merge({
+          "count"=>3,
+          "message"=>[
+            {"message"=>"2013/01/13T07:02:13.232645 WARN POST /auth"},
+            {"message"=>"2013/01/13T07:02:21.542145 WARN GET /favicon.ico"},
+            {"message"=>"2013/01/13T07:02:43.632145 WARN POST /login"},
+          ],
+        }))
+      end
+      it { emit }
+    end
+
     context 'exclude' do
       let(:config) { CONFIG + %[regexp WARN \n exclude favicon] }
       before do
@@ -124,6 +140,21 @@ describe Fluent::GrepCounterOutput do
       it { emit }
     end
 
+    context 'excludeN' do
+      let(:config) { %[regexp1 message WARN \n exclude1 message favicon] }
+      before do
+        Fluent::Engine.stub(:now).and_return(time)
+        Fluent::Engine.should_receive(:emit).with("count.#{tag}", time, expected.merge({
+          "count"=>2,
+          "message"=>[
+            {"message"=>"2013/01/13T07:02:13.232645 WARN POST /auth"},
+            {"message"=>"2013/01/13T07:02:43.632145 WARN POST /login"},
+          ],
+        }))
+      end
+      it { emit }
+    end
+
     context "threshold and comparator (obsolete)" do
       context '>= threshold' do
         let(:config) { CONFIG + %[threshold 4] }
@@ -301,7 +332,7 @@ describe Fluent::GrepCounterOutput do
       it { emit }
     end
 
-    context 'delimiter' do
+    context 'delimiter for old style (input_key)' do
       # \\n shall be \n in config file
       let(:config) { CONFIG + %[delimiter \\n] }
       before do
@@ -312,6 +343,18 @@ describe Fluent::GrepCounterOutput do
       it { emit }
     end
 
+    context 'delimiter for new style (regexpN or excludeN)' do
+      # \\n shall be \n in config file
+      let(:config) { %[regexp1 message .\ndelimiter \\n] }
+      before do
+        Fluent::Engine.stub(:now).and_return(time)
+        # I will think of good format later ...
+        message = "{\"message\"=>\"2013/01/13T07:02:11.124202 INFO GET /ping\"}\\n{\"message\"=>\"2013/01/13T07:02:13.232645 WARN POST /auth\"}\\n{\"message\"=>\"2013/01/13T07:02:21.542145 WARN GET /favicon.ico\"}\\n{\"message\"=>\"2013/01/13T07:02:43.632145 WARN POST /login\"}"
+        Fluent::Engine.should_receive(:emit).with("count.#{tag}", time, expected.merge("message" => message))
+      end
+      it { emit }
+    end
+
     context 'aggregate all' do
       let(:messages) { ['foobar', 'foobar'] }
       let(:emit) do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-grepcounter
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.5.0
 platform: ruby
 authors:
 - Naotoshi Seo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-12 00:00:00.000000000 Z
+date: 2013-12-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd