fluent-plugin-grep 0.2.1 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 2e382e08c49b5adc65ff681356c292423a90eff1
4
- data.tar.gz: 6eeb7e28f4e75a25a15ff816299062625fa5939d
3
+ metadata.gz: 01a71925261dd9465cd10f28acf6af28861d9f6b
4
+ data.tar.gz: cfacc3cb357efeacff1d486f0451ec3952db8a61
5
5
  SHA512:
6
- metadata.gz: eb7f1ab442ae55877b383bfacb4e8b44d723052574fdd4d870f3c32b10ca19b209a8749eeb641779cbbc9f6851167586f97b9c0f487fdf2f9d63303ea7b1c912
7
- data.tar.gz: 959c770dbfc9dc31326f311c1ade4d9c85b6831a3a192dcece72d17aa7ba951e53a9dbc8eec7f91a80a4ececa3c216be7bb7b2f18cc03e6dcfabe77c4d8c26d3
6
+ metadata.gz: d885ec8a7c2180fdbc581b821fa826240b5aef21c120a5425f29f283d491f300f2fb84ea7abbfd7b2e76b555f69aa951833da598b6d35ebc5ac3359861547111
7
+ data.tar.gz: 0c88d08c8f7e6a3f40191d54db854e59f47e13c2ec7259dac9de768f14eaf2df2c013d0eb4229aed50c186be623a2392fdb43d47e723dac135f9af9e60a641ac
data/CHANGELOG.md CHANGED
@@ -1,3 +1,9 @@
1
+ ## 0.3.0 (2013/12/15)
2
+
3
+ Features:
4
+
5
+ - Add `regexpN` and `excludeN` option. `input_key`, `regexp`, and `exclude` options are now obsolete.
6
+
1
7
  ## 0.2.1 (2013/12/12)
2
8
 
3
9
  Features:
data/README.md CHANGED
@@ -1,4 +1,4 @@
1
- # fluent-plugin-grep [![Build Status](https://secure.travis-ci.org/sonots/fluent-plugin-grep.png?branch=master)](http://travis-ci.org/sonots/fluent-plugin-grep) [![Dependency Status](https://gemnasium.com/sonots/fluent-plugin-grep.png)](https://gemnasium.com/sonots/fluent-plugin-grep) [![Coverage Status](https://coveralls.io/repos/sonots/fluent-plugin-grep/badge.png?branch=master)](https://coveralls.io/r/sonots/fluent-plugin-grep)
1
+ # fluent-plugin-grep [![Build Status](https://secure.travis-ci.org/sonots/fluent-plugin-grep.png?branch=master)](http://travis-ci.org/sonots/fluent-plugin-grep) [![Coverage Status](https://coveralls.io/repos/sonots/fluent-plugin-grep/badge.png?branch=master)](https://coveralls.io/r/sonots/fluent-plugin-grep)
2
2
 
3
3
  Fluentd plugin to grep messages.
4
4
 
@@ -6,9 +6,8 @@ Fluentd plugin to grep messages.
6
6
 
7
7
  <match foo.bar.**>
8
8
  type grep
9
- input_key message
10
- regexp WARN
11
- exclude favicon.ico
9
+ regexp1 message WARN
10
+ exclude1 message favicon
12
11
  add_tag_prefix greped
13
12
  </source>
14
13
 
@@ -19,24 +18,20 @@ Assuming following inputs are coming:
19
18
  foo.bar: {"foo":"bar","message":"2013/01/13T07:02:21.542145 WARN GET /favicon.ico"}
20
19
  foo.bar: {"foo":"bar","message":"2013/01/13T07:02:43.632145 WARN POST /login"}
21
20
 
22
- then output bocomes as belows (like, | grep WARN | grep -v favicon.ico):
21
+ then output bocomes as belows (like, | grep WARN | grep -v favicon):
23
22
 
24
23
  greped.foo.bar: {"foo":"bar","message":"2013/01/13T07:02:13.232645 WARN POST /auth"}
25
24
  greped.foo.bar: {"foo":"bar","message":"2013/01/13T07:02:43.632145 WARN POST /login"}
26
25
 
27
26
  ## Parameters
28
27
 
29
- - input_key
28
+ - regexp[1-20] *field\_key* *regexp*
30
29
 
31
- The target field key to grep out
30
+ The target field key and the filtering regular expression to grep out.
32
31
 
33
- - regexp
32
+ - exclude[1-20] *field_key* *regexp*
34
33
 
35
- The filtering regular expression
36
-
37
- - exclude
38
-
39
- The excluding regular expression like grep -v
34
+ The target field key and the excluding regular expression like grep -v
40
35
 
41
36
  - tag
42
37
 
@@ -54,6 +49,18 @@ then output bocomes as belows (like, | grep WARN | grep -v favicon.ico):
54
49
 
55
50
  Replace invalid byte sequence in UTF-8 with '?' character if `true`
56
51
 
52
+ - input\_key *field\_key* (obsolete)
53
+
54
+ The target field key to grep out. Use with regexp or exclude.
55
+
56
+ - regexp *regexp* (obsolete)
57
+
58
+ The filtering regular expression
59
+
60
+ - exclude *regexp* (obsolete)
61
+
62
+ The excluding regular expression like grep -v
63
+
57
64
  ## ChangeLog
58
65
 
59
66
  See [CHANGELOG.md](CHANGELOG.md) for details.
@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
3
3
 
4
4
  Gem::Specification.new do |s|
5
5
  s.name = "fluent-plugin-grep"
6
- s.version = "0.2.1"
6
+ s.version = "0.3.0"
7
7
  s.authors = ["Naotoshi Seo"]
8
8
  s.email = ["sonots@gmail.com"]
9
9
  s.homepage = "https://github.com/sonots/fluent-plugin-grep"
@@ -1,20 +1,38 @@
1
1
  class Fluent::GrepOutput < Fluent::Output
2
2
  Fluent::Plugin.register_output('grep', self)
3
3
 
4
- config_param :input_key, :string
5
- config_param :regexp, :string, :default => nil
6
- config_param :exclude, :string, :default => nil
4
+ REGEXP_MAX_NUM = 20
5
+
6
+ config_param :input_key, :string, :default => nil # obsolete
7
+ config_param :regexp, :string, :default => nil # obsolete
8
+ config_param :exclude, :string, :default => nil # obsolete
7
9
  config_param :tag, :string, :default => nil
8
10
  config_param :add_tag_prefix, :string, :default => nil
9
11
  config_param :remove_tag_prefix, :string, :default => nil
10
12
  config_param :replace_invalid_sequence, :bool, :default => false
13
+ (1..REGEXP_MAX_NUM).each {|i| config_param :"regexp#{i}", :string, :default => nil }
14
+ (1..REGEXP_MAX_NUM).each {|i| config_param :"exclude#{i}", :string, :default => nil }
11
15
 
12
16
  def configure(conf)
13
17
  super
14
18
 
15
- @input_key = @input_key.to_s
16
- @regexp = Regexp.compile(@regexp) if @regexp
17
- @exclude = Regexp.compile(@exclude) if @exclude
19
+ @regexps = {}
20
+ @regexps[@input_key] = Regexp.compile(@regexp) if @input_key and @regexp
21
+ (1..REGEXP_MAX_NUM).each do |i|
22
+ next unless conf["regexp#{i}"]
23
+ key, regexp = conf["regexp#{i}"].split(/ +/, 2)
24
+ raise Fluent::ConfigError, "regexp#{i} does not contain 2 parameters" unless regexp
25
+ @regexps[key] = Regexp.compile(regexp)
26
+ end
27
+
28
+ @excludes = {}
29
+ @excludes[@input_key] = Regexp.compile(@exclude) if @input_key and @exclude
30
+ (1..REGEXP_MAX_NUM).each do |i|
31
+ next unless conf["exclude#{i}"]
32
+ key, exclude = conf["exclude#{i}"].split(/ +/, 2)
33
+ raise Fluent::ConfigError, "exclude#{i} does not contain 2 parameters" unless exclude
34
+ @excludes[key] = Regexp.compile(exclude)
35
+ end
18
36
 
19
37
  if @tag.nil? and @add_tag_prefix.nil? and @remove_tag_prefix.nil?
20
38
  @add_tag_prefix = 'greped' # not ConfigError to support lower version compatibility
@@ -40,9 +58,15 @@ class Fluent::GrepOutput < Fluent::Output
40
58
  emit_tag = @tag_proc.call(tag)
41
59
 
42
60
  es.each do |time,record|
43
- value = record[@input_key]
44
- next unless match(value.to_s)
45
- Fluent::Engine.emit(emit_tag, time, record)
61
+ catch(:break_loop) do
62
+ @regexps.each do |key, regexp|
63
+ throw :break_loop unless match(regexp, record[key].to_s)
64
+ end
65
+ @excludes.each do |key, exclude|
66
+ throw :break_loop if match(exclude, record[key].to_s)
67
+ end
68
+ Fluent::Engine.emit(emit_tag, time, record)
69
+ end
46
70
  end
47
71
 
48
72
  chain.next
@@ -57,10 +81,9 @@ class Fluent::GrepOutput < Fluent::Output
57
81
  string.index(substring) == 0 ? string[substring.size..-1] : string
58
82
  end
59
83
 
60
- def match(string)
84
+ def match(regexp, string)
61
85
  begin
62
- return false if @regexp and !@regexp.match(string)
63
- return false if @exclude and @exclude.match(string)
86
+ return regexp.match(string)
64
87
  rescue ArgumentError => e
65
88
  raise e unless e.message.index("invalid byte sequence in") == 0
66
89
  string = replace_invalid_byte(string)
@@ -25,7 +25,9 @@ end
25
25
  # BEFORE TAG_PROC
26
26
  # user system total real
27
27
  # 2.560000 0.030000 2.590000 ( 3.169847)
28
- # AFTER TAG_PROC
28
+ # AFTER TAG_PROC (0.2.1)
29
29
  # user system total real
30
30
  # 2.480000 0.040000 2.520000 ( 3.085798)
31
-
31
+ # AFTER regexps, exludes (0.3.0)
32
+ # user system total real
33
+ # 2.700000 0.050000 2.750000 ( 3.340524)
@@ -11,10 +11,6 @@ describe Fluent::GrepOutput do
11
11
 
12
12
  describe 'test configure' do
13
13
  describe 'bad configuration' do
14
- context "lack of requirements" do
15
- let(:config) { '' }
16
- it { expect { driver }.to raise_error(Fluent::ConfigError) }
17
- end
18
14
  end
19
15
 
20
16
  describe 'good configuration' do
@@ -72,6 +68,21 @@ describe Fluent::GrepOutput do
72
68
  it { emit }
73
69
  end
74
70
 
71
+ context 'regexpN' do
72
+ let(:config) do
73
+ CONFIG + %[
74
+ regexp1 message WARN
75
+ ]
76
+ end
77
+ before do
78
+ Fluent::Engine.stub(:now).and_return(time)
79
+ Fluent::Engine.should_receive(:emit).with("greped.#{tag}", time, {'foo'=>'bar', 'message'=>"2013/01/13T07:02:13.232645 WARN POST /auth"})
80
+ Fluent::Engine.should_receive(:emit).with("greped.#{tag}", time, {'foo'=>'bar', 'message'=>"2013/01/13T07:02:21.542145 WARN GET /favicon.ico"})
81
+ Fluent::Engine.should_receive(:emit).with("greped.#{tag}", time, {'foo'=>'bar', 'message'=>"2013/01/13T07:02:43.632145 WARN POST /login"})
82
+ end
83
+ it { emit }
84
+ end
85
+
75
86
  context 'exclude' do
76
87
  let(:config) do
77
88
  CONFIG + %[
@@ -87,6 +98,21 @@ describe Fluent::GrepOutput do
87
98
  it { emit }
88
99
  end
89
100
 
101
+ context 'excludeN' do
102
+ let(:config) do
103
+ CONFIG + %[
104
+ exclude1 message favicon
105
+ ]
106
+ end
107
+ before do
108
+ Fluent::Engine.stub(:now).and_return(time)
109
+ Fluent::Engine.should_receive(:emit).with("greped.#{tag}", time, {'foo'=>'bar', 'message'=>"2013/01/13T07:02:11.124202 INFO GET /ping"})
110
+ Fluent::Engine.should_receive(:emit).with("greped.#{tag}", time, {'foo'=>'bar', 'message'=>"2013/01/13T07:02:13.232645 WARN POST /auth"})
111
+ Fluent::Engine.should_receive(:emit).with("greped.#{tag}", time, {'foo'=>'bar', 'message'=>"2013/01/13T07:02:43.632145 WARN POST /login"})
112
+ end
113
+ it { emit }
114
+ end
115
+
90
116
  context 'tag' do
91
117
  let(:config) do
92
118
  CONFIG + %[
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-grep
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.1
4
+ version: 0.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Naotoshi Seo
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2013-12-12 00:00:00.000000000 Z
11
+ date: 2013-12-14 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd