fluent-plugin-grep 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2e382e08c49b5adc65ff681356c292423a90eff1
-  data.tar.gz: 6eeb7e28f4e75a25a15ff816299062625fa5939d
+  metadata.gz: 01a71925261dd9465cd10f28acf6af28861d9f6b
+  data.tar.gz: cfacc3cb357efeacff1d486f0451ec3952db8a61
 SHA512:
-  metadata.gz: eb7f1ab442ae55877b383bfacb4e8b44d723052574fdd4d870f3c32b10ca19b209a8749eeb641779cbbc9f6851167586f97b9c0f487fdf2f9d63303ea7b1c912
-  data.tar.gz: 959c770dbfc9dc31326f311c1ade4d9c85b6831a3a192dcece72d17aa7ba951e53a9dbc8eec7f91a80a4ececa3c216be7bb7b2f18cc03e6dcfabe77c4d8c26d3
+  metadata.gz: d885ec8a7c2180fdbc581b821fa826240b5aef21c120a5425f29f283d491f300f2fb84ea7abbfd7b2e76b555f69aa951833da598b6d35ebc5ac3359861547111
+  data.tar.gz: 0c88d08c8f7e6a3f40191d54db854e59f47e13c2ec7259dac9de768f14eaf2df2c013d0eb4229aed50c186be623a2392fdb43d47e723dac135f9af9e60a641ac

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.3.0 (2013/12/15)
+
+Features:
+
+- Add `regexpN` and `excludeN` option. `input_key`, `regexp`, and `exclude` options are now obsolete.
+
 ## 0.2.1 (2013/12/12)
 
 Features:

data/README.md

@@ -1,4 +1,4 @@
-# fluent-plugin-grep [![Build Status](https://secure.travis-ci.org/sonots/fluent-plugin-grep.png?branch=master)](http://travis-ci.org/sonots/fluent-plugin-grep) [![Dependency Status](https://gemnasium.com/sonots/fluent-plugin-grep.png)](https://gemnasium.com/sonots/fluent-plugin-grep) [![Coverage Status](https://coveralls.io/repos/sonots/fluent-plugin-grep/badge.png?branch=master)](https://coveralls.io/r/sonots/fluent-plugin-grep)
+# fluent-plugin-grep [![Build Status](https://secure.travis-ci.org/sonots/fluent-plugin-grep.png?branch=master)](http://travis-ci.org/sonots/fluent-plugin-grep) [![Coverage Status](https://coveralls.io/repos/sonots/fluent-plugin-grep/badge.png?branch=master)](https://coveralls.io/r/sonots/fluent-plugin-grep)
 
 Fluentd plugin to grep messages.
 
@@ -6,9 +6,8 @@ Fluentd plugin to grep messages.
 
     <match foo.bar.**>
       type grep
-      input_key message
-      regexp WARN
-      exclude favicon.ico
+      regexp1 message WARN
+      exclude1 message favicon
       add_tag_prefix greped
     </source>
 
@@ -19,24 +18,20 @@ Assuming following inputs are coming:
     foo.bar: {"foo":"bar","message":"2013/01/13T07:02:21.542145 WARN GET /favicon.ico"}
     foo.bar: {"foo":"bar","message":"2013/01/13T07:02:43.632145 WARN POST /login"}
 
-then output bocomes as belows (like, | grep WARN | grep -v favicon.ico):
+then output bocomes as belows (like, | grep WARN | grep -v favicon):
 
     greped.foo.bar: {"foo":"bar","message":"2013/01/13T07:02:13.232645 WARN POST /auth"}
     greped.foo.bar: {"foo":"bar","message":"2013/01/13T07:02:43.632145 WARN POST /login"}
 
 ## Parameters
 
-- input_key
+- regexp[1-20] *field\_key* *regexp*
 
-    The target field key to grep out
+    The target field key and the filtering regular expression to grep out. 
 
-- regexp
+- exclude[1-20] *field_key* *regexp*
 
-    The filtering regular expression
-
-- exclude
-
-    The excluding regular expression like grep -v
+    The target field key and the excluding regular expression like grep -v
 
 - tag
 
@@ -54,6 +49,18 @@ then output bocomes as belows (like, | grep WARN | grep -v favicon.ico):
 
     Replace invalid byte sequence in UTF-8 with '?' character if `true`
 
+- input\_key *field\_key* (obsolete)
+
+    The target field key to grep out. Use with regexp or exclude. 
+
+- regexp *regexp* (obsolete)
+
+    The filtering regular expression
+
+- exclude *regexp* (obsolete)
+
+    The excluding regular expression like grep -v
+
 ## ChangeLog
 
 See [CHANGELOG.md](CHANGELOG.md) for details.

data/fluent-plugin-grep.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-grep"
-  s.version     = "0.2.1"
+  s.version     = "0.3.0"
   s.authors     = ["Naotoshi Seo"]
   s.email       = ["sonots@gmail.com"]
   s.homepage    = "https://github.com/sonots/fluent-plugin-grep"

data/lib/fluent/plugin/out_grep.rb

@@ -1,20 +1,38 @@
 class Fluent::GrepOutput < Fluent::Output
   Fluent::Plugin.register_output('grep', self)
 
-  config_param :input_key, :string
-  config_param :regexp, :string, :default => nil
-  config_param :exclude, :string, :default => nil
+  REGEXP_MAX_NUM = 20
+
+  config_param :input_key, :string, :default => nil # obsolete
+  config_param :regexp, :string, :default => nil # obsolete
+  config_param :exclude, :string, :default => nil # obsolete
   config_param :tag, :string, :default => nil
   config_param :add_tag_prefix, :string, :default => nil
   config_param :remove_tag_prefix, :string, :default => nil
   config_param :replace_invalid_sequence, :bool, :default => false
+  (1..REGEXP_MAX_NUM).each {|i| config_param :"regexp#{i}",  :string, :default => nil }
+  (1..REGEXP_MAX_NUM).each {|i| config_param :"exclude#{i}", :string, :default => nil }
 
   def configure(conf)
     super
 
-    @input_key = @input_key.to_s
-    @regexp = Regexp.compile(@regexp) if @regexp
-    @exclude = Regexp.compile(@exclude) if @exclude
+    @regexps = {}
+    @regexps[@input_key] = Regexp.compile(@regexp) if @input_key and @regexp
+    (1..REGEXP_MAX_NUM).each do |i|
+      next unless conf["regexp#{i}"]
+      key, regexp = conf["regexp#{i}"].split(/ +/, 2)
+      raise Fluent::ConfigError, "regexp#{i} does not contain 2 parameters" unless regexp
+      @regexps[key] = Regexp.compile(regexp)
+    end
+
+    @excludes = {}
+    @excludes[@input_key] = Regexp.compile(@exclude) if @input_key and @exclude
+    (1..REGEXP_MAX_NUM).each do |i|
+      next unless conf["exclude#{i}"]
+      key, exclude = conf["exclude#{i}"].split(/ +/, 2)
+      raise Fluent::ConfigError, "exclude#{i} does not contain 2 parameters" unless exclude
+      @excludes[key] = Regexp.compile(exclude)
+    end
 
     if @tag.nil? and @add_tag_prefix.nil? and @remove_tag_prefix.nil?
       @add_tag_prefix = 'greped' # not ConfigError to support lower version compatibility
@@ -40,9 +58,15 @@ class Fluent::GrepOutput < Fluent::Output
     emit_tag = @tag_proc.call(tag)
 
     es.each do |time,record|
-      value = record[@input_key]
-      next unless match(value.to_s)
-      Fluent::Engine.emit(emit_tag, time, record)
+      catch(:break_loop) do
+        @regexps.each do |key, regexp|
+          throw :break_loop unless match(regexp, record[key].to_s)
+        end
+        @excludes.each do |key, exclude|
+          throw :break_loop if match(exclude, record[key].to_s)
+        end
+        Fluent::Engine.emit(emit_tag, time, record)
+      end
     end
 
     chain.next
@@ -57,10 +81,9 @@ class Fluent::GrepOutput < Fluent::Output
     string.index(substring) == 0 ? string[substring.size..-1] : string
   end
 
-  def match(string)
+  def match(regexp, string)
     begin
-      return false if @regexp and !@regexp.match(string)
-      return false if @exclude and @exclude.match(string)
+      return regexp.match(string)
     rescue ArgumentError => e
       raise e unless e.message.index("invalid byte sequence in") == 0
       string = replace_invalid_byte(string)

data/spec/out_grep_bench.rb

@@ -25,7 +25,9 @@ end
 # BEFORE TAG_PROC
 #              user     system      total        real
 #          2.560000   0.030000   2.590000 (  3.169847)
-# AFTER  TAG_PROC
+# AFTER  TAG_PROC (0.2.1)
 #              user     system      total        real
 #          2.480000   0.040000   2.520000 (  3.085798)
-
+# AFTER  regexps, exludes (0.3.0) 
+#              user     system      total        real
+#          2.700000   0.050000   2.750000 (  3.340524)

data/spec/out_grep_spec.rb

@@ -11,10 +11,6 @@ describe Fluent::GrepOutput do
 
   describe 'test configure' do
     describe 'bad configuration' do
-      context "lack of requirements" do
-        let(:config) { '' }
-        it { expect { driver }.to raise_error(Fluent::ConfigError) }
-      end
     end
 
     describe 'good configuration' do
@@ -72,6 +68,21 @@ describe Fluent::GrepOutput do
       it { emit }
     end
 
+    context 'regexpN' do
+      let(:config) do
+        CONFIG + %[
+          regexp1 message WARN
+        ]
+      end
+      before do
+        Fluent::Engine.stub(:now).and_return(time)
+        Fluent::Engine.should_receive(:emit).with("greped.#{tag}", time, {'foo'=>'bar', 'message'=>"2013/01/13T07:02:13.232645 WARN POST /auth"})
+        Fluent::Engine.should_receive(:emit).with("greped.#{tag}", time, {'foo'=>'bar', 'message'=>"2013/01/13T07:02:21.542145 WARN GET /favicon.ico"})
+        Fluent::Engine.should_receive(:emit).with("greped.#{tag}", time, {'foo'=>'bar', 'message'=>"2013/01/13T07:02:43.632145 WARN POST /login"})
+      end
+      it { emit }
+    end
+
     context 'exclude' do
       let(:config) do
         CONFIG + %[
@@ -87,6 +98,21 @@ describe Fluent::GrepOutput do
       it { emit }
     end
 
+    context 'excludeN' do
+      let(:config) do
+        CONFIG + %[
+          exclude1 message favicon
+        ]
+      end
+      before do
+        Fluent::Engine.stub(:now).and_return(time)
+        Fluent::Engine.should_receive(:emit).with("greped.#{tag}", time, {'foo'=>'bar', 'message'=>"2013/01/13T07:02:11.124202 INFO GET /ping"})
+        Fluent::Engine.should_receive(:emit).with("greped.#{tag}", time, {'foo'=>'bar', 'message'=>"2013/01/13T07:02:13.232645 WARN POST /auth"})
+        Fluent::Engine.should_receive(:emit).with("greped.#{tag}", time, {'foo'=>'bar', 'message'=>"2013/01/13T07:02:43.632145 WARN POST /login"})
+      end
+      it { emit }
+    end
+
     context 'tag' do
       let(:config) do
         CONFIG + %[

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-grep
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Naotoshi Seo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-12 00:00:00.000000000 Z
+date: 2013-12-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd