fluent-plugin-grepcounter 0.1.4 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +7 -1
- data/Gemfile +0 -1
- data/README.md +24 -8
- data/fluent-plugin-grepcounter.gemspec +2 -2
- data/lib/fluent/plugin/out_grepcounter.rb +100 -5
- data/spec/out_grepcounter_spec.rb +206 -153
- data/spec/spec_helper.rb +0 -3
- metadata +4 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: bddb5d5327eec9e655863fc3cb7c6df468ebf818
|
4
|
+
data.tar.gz: c5d37235947a78bdf39cb45819434f2ae0357b06
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 978c44205ca7867084429682d76be4c380fee8c2fc2b34e32d8855cb78c8da232c4da6430dbd0b5b98c2209c6a4a19afcdd1dbddb1e05743c08da5065b9a5f74
|
7
|
+
data.tar.gz: d20233e52d1732422756df141fd456401b14a31621652fa2052ca7b45e6beff327b2ea7d1e0b17d01a678e779fee3afb35667164afc647bfcc1db36ee8a41f2d
|
data/CHANGELOG.md
CHANGED
data/Gemfile
CHANGED
data/README.md
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
# fluent-plugin-grepcounter [](http://travis-ci.org/sonots/fluent-plugin-grepcounter) [](https://gemnasium.com/sonots/fluent-plugin-grepcounter)
|
1
|
+
# fluent-plugin-grepcounter [](http://travis-ci.org/sonots/fluent-plugin-grepcounter) [](https://gemnasium.com/sonots/fluent-plugin-grepcounter)
|
2
2
|
|
3
3
|
Fluentd plugin to count the number of matched messages.
|
4
4
|
|
@@ -74,20 +74,28 @@ Then, output bocomes as belows (indented). You can see the `message` field is jo
|
|
74
74
|
|
75
75
|
- threshold
|
76
76
|
|
77
|
-
The threshold number to emit
|
77
|
+
The threshold number to emit. Emit if `count` value >= specified value.
|
78
78
|
|
79
|
-
- comparator
|
79
|
+
- comparator (obsolete from 0.2.0. Use greater\_equal or less\_equal instead)
|
80
80
|
|
81
81
|
The comparation operator for the threshold (either of `>=` or `<=`). Default is `>=`, i.e., emit if count >= threshold.
|
82
82
|
NOTE: 0 count message will not be emitted even if `<=` is specified because standby nodes receive no message usually.
|
83
83
|
|
84
|
-
-
|
84
|
+
- less\_than (from 0.2.0)
|
85
|
+
|
86
|
+
A `less than` threshold value, that is, emit if `count` value < specified value.
|
87
|
+
|
88
|
+
- less\_equal (from 0.2.0)
|
89
|
+
|
90
|
+
A `less than or eqaul` threshold value, that is, emit if `count` value <= specified value.
|
91
|
+
|
92
|
+
- greater\_than (from 0.2.0)
|
85
93
|
|
86
|
-
|
94
|
+
A `greater than` threshold value, that is, emit if `count` value > specified value.
|
87
95
|
|
88
|
-
-
|
96
|
+
- greater\_equal (from 0.2.0)
|
89
97
|
|
90
|
-
|
98
|
+
A `greater than or eqaul` threshold value, that is, emit if `count` value >= specified value. Same with `threshold` option.
|
91
99
|
|
92
100
|
- output\_tag
|
93
101
|
|
@@ -97,11 +105,19 @@ Then, output bocomes as belows (indented). You can see the `message` field is jo
|
|
97
105
|
|
98
106
|
Add tag prefix for output message
|
99
107
|
|
108
|
+
- output\_with\_joined\_delimiter
|
109
|
+
|
110
|
+
Output matched messages after `join`ed with the specified delimiter.
|
111
|
+
|
100
112
|
- replace\_invalid\_sequence
|
101
113
|
|
102
114
|
Replace invalid byte sequence in UTF-8 with '?' character if `true`
|
103
115
|
|
104
|
-
|
116
|
+
- store\_file
|
117
|
+
|
118
|
+
Store internal count data into a file of the given path on shutdown, and load on statring.
|
119
|
+
|
120
|
+
## ChangeLog
|
105
121
|
|
106
122
|
See [CHANGELOG.md](CHANGELOG.md) for details.
|
107
123
|
|
@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = "fluent-plugin-grepcounter"
|
6
|
-
s.version = "0.
|
6
|
+
s.version = "0.2.0"
|
7
7
|
s.authors = ["Naotoshi SEO"]
|
8
8
|
s.email = ["sonots@gmail.com"]
|
9
9
|
s.homepage = "https://github.com/sonots/fluent-plugin-grepcounter"
|
@@ -21,6 +21,6 @@ Gem::Specification.new do |s|
|
|
21
21
|
s.add_development_dependency "rake"
|
22
22
|
s.add_development_dependency "rspec"
|
23
23
|
s.add_development_dependency "pry"
|
24
|
-
s.add_development_dependency
|
24
|
+
s.add_development_dependency "pry-nav"
|
25
25
|
# s.add_development_dependency "delorean"
|
26
26
|
end
|
@@ -2,19 +2,32 @@
|
|
2
2
|
class Fluent::GrepCounterOutput < Fluent::Output
|
3
3
|
Fluent::Plugin.register_output('grepcounter', self)
|
4
4
|
|
5
|
+
def initialize
|
6
|
+
super
|
7
|
+
require 'pathname'
|
8
|
+
end
|
9
|
+
|
5
10
|
config_param :input_key, :string
|
6
11
|
config_param :regexp, :string, :default => nil
|
7
12
|
config_param :count_interval, :time, :default => 5
|
8
13
|
config_param :exclude, :string, :default => nil
|
9
|
-
config_param :threshold, :integer, :default =>
|
10
|
-
config_param :comparator, :string, :default => '>='
|
14
|
+
config_param :threshold, :integer, :default => nil # obsolete
|
15
|
+
config_param :comparator, :string, :default => '>=' # obsolete
|
16
|
+
config_param :less_than, :float, :default => nil
|
17
|
+
config_param :less_equal, :float, :default => nil
|
18
|
+
config_param :greater_than, :float, :default => nil
|
19
|
+
config_param :greater_equal, :float, :default => nil
|
11
20
|
config_param :output_tag, :string, :default => nil
|
12
21
|
config_param :add_tag_prefix, :string, :default => 'count'
|
13
22
|
config_param :output_with_joined_delimiter, :string, :default => nil
|
14
23
|
config_param :aggregate, :string, :default => 'tag'
|
15
24
|
config_param :replace_invalid_sequence, :bool, :default => false
|
25
|
+
config_param :store_file, :string, :default => nil
|
16
26
|
|
27
|
+
attr_accessor :counts
|
17
28
|
attr_accessor :matches
|
29
|
+
attr_accessor :saved_duration
|
30
|
+
attr_accessor :saved_at
|
18
31
|
attr_accessor :last_checked
|
19
32
|
|
20
33
|
def configure(conf)
|
@@ -24,12 +37,25 @@ class Fluent::GrepCounterOutput < Fluent::Output
|
|
24
37
|
@input_key = @input_key.to_s
|
25
38
|
@regexp = Regexp.compile(@regexp) if @regexp
|
26
39
|
@exclude = Regexp.compile(@exclude) if @exclude
|
27
|
-
|
40
|
+
|
41
|
+
@threshold = @threshold.to_i if @threshold
|
28
42
|
|
29
43
|
unless ['>=', '<='].include?(@comparator)
|
30
44
|
raise Fluent::ConfigError, "grepcounter: comparator allows >=, <="
|
31
45
|
end
|
32
46
|
|
47
|
+
# to support obsolete options
|
48
|
+
if @threshold.nil? and @less_than.nil? and @less_equal.nil? and @greater_than.nil? and @greater_equal.nil?
|
49
|
+
@threshold = 1
|
50
|
+
end
|
51
|
+
if @threshold and @comparator
|
52
|
+
if @comparator == '>='
|
53
|
+
@greater_equal = @threshold
|
54
|
+
else
|
55
|
+
@less_equal = @threshold
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
33
59
|
unless ['tag', 'all'].include?(@aggregate)
|
34
60
|
raise Fluent::ConfigError, "grepcounter: aggregate allows tag/all"
|
35
61
|
end
|
@@ -41,6 +67,13 @@ class Fluent::GrepCounterOutput < Fluent::Output
|
|
41
67
|
# raise Fluent::ConfigError, "grepcounter: add_tag_prefix must be specified with aggregate tag" if @add_tag_prefix.nil?
|
42
68
|
end
|
43
69
|
|
70
|
+
if @store_file
|
71
|
+
f = Pathname.new(@store_file)
|
72
|
+
if (f.exist? && !f.writable_real?) || (!f.exist? && !f.parent.writable_real?)
|
73
|
+
raise Fluent::ConfigError, "#{@store_file} is not writable"
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
44
77
|
@matches = {}
|
45
78
|
@counts = {}
|
46
79
|
@mutex = Mutex.new
|
@@ -48,6 +81,7 @@ class Fluent::GrepCounterOutput < Fluent::Output
|
|
48
81
|
|
49
82
|
def start
|
50
83
|
super
|
84
|
+
load_status(@store_file, @count_interval) if @store_file
|
51
85
|
@watcher = Thread.new(&method(:watcher))
|
52
86
|
end
|
53
87
|
|
@@ -55,6 +89,7 @@ class Fluent::GrepCounterOutput < Fluent::Output
|
|
55
89
|
super
|
56
90
|
@watcher.terminate
|
57
91
|
@watcher.join
|
92
|
+
save_status(@store_file) if @store_file
|
58
93
|
end
|
59
94
|
|
60
95
|
# Called when new line comes. This method actually does not emit
|
@@ -83,7 +118,7 @@ class Fluent::GrepCounterOutput < Fluent::Output
|
|
83
118
|
# thread callback
|
84
119
|
def watcher
|
85
120
|
# instance variable, and public accessable, for test
|
86
|
-
@last_checked
|
121
|
+
@last_checked ||= Fluent::Engine.now
|
87
122
|
while true
|
88
123
|
sleep 0.5
|
89
124
|
begin
|
@@ -125,7 +160,10 @@ class Fluent::GrepCounterOutput < Fluent::Output
|
|
125
160
|
def generate_output(count, matches, tag = nil)
|
126
161
|
return nil if count.nil?
|
127
162
|
return nil if count == 0 # ignore 0 because standby nodes receive no message usually
|
128
|
-
return nil
|
163
|
+
return nil if @less_than and @less_than <= count
|
164
|
+
return nil if @less_equal and @less_equal < count
|
165
|
+
return nil if @greater_than and count <= @greater_than
|
166
|
+
return nil if @greater_equal and count < @greater_equal
|
129
167
|
output = {}
|
130
168
|
output['count'] = count
|
131
169
|
output['message'] = @output_with_joined_delimiter.nil? ? matches : matches.join(@output_with_joined_delimiter)
|
@@ -157,4 +195,61 @@ class Fluent::GrepCounterOutput < Fluent::Output
|
|
157
195
|
temporal_encoding = (original_encoding == Encoding::UTF_8 ? Encoding::UTF_16BE : Encoding::UTF_8)
|
158
196
|
string.encode(temporal_encoding, original_encoding, replace_options).encode(original_encoding)
|
159
197
|
end
|
198
|
+
|
199
|
+
# Store internal status into a file
|
200
|
+
#
|
201
|
+
# @param [String] file_path
|
202
|
+
def save_status(file_path)
|
203
|
+
begin
|
204
|
+
Pathname.new(file_path).open('wb') do |f|
|
205
|
+
@saved_at = Fluent::Engine.now
|
206
|
+
@saved_duration = @saved_at - @last_checked
|
207
|
+
Marshal.dump({
|
208
|
+
:counts => @counts,
|
209
|
+
:matches => @matches,
|
210
|
+
:saved_at => @saved_at,
|
211
|
+
:saved_duration => @saved_duration,
|
212
|
+
:regexp => @regexp,
|
213
|
+
:exclude => @exclude,
|
214
|
+
:input_key => @input_key,
|
215
|
+
}, f)
|
216
|
+
end
|
217
|
+
rescue => e
|
218
|
+
$log.warn "out_grepcounter: Can't write store_file #{e.class} #{e.message}"
|
219
|
+
end
|
220
|
+
end
|
221
|
+
|
222
|
+
# Load internal status from a file
|
223
|
+
#
|
224
|
+
# @param [String] file_path
|
225
|
+
# @param [Interger] count_interval
|
226
|
+
def load_status(file_path, count_interval)
|
227
|
+
return unless (f = Pathname.new(file_path)).exist?
|
228
|
+
begin
|
229
|
+
f.open('rb') do |f|
|
230
|
+
stored = Marshal.load(f)
|
231
|
+
if stored[:regexp] == @regexp and
|
232
|
+
stored[:exclude] == @exclude and
|
233
|
+
stored[:input_key] == @input_key
|
234
|
+
|
235
|
+
if Fluent::Engine.now <= stored[:saved_at] + count_interval
|
236
|
+
@counts = stored[:counts]
|
237
|
+
@matches = stored[:matches]
|
238
|
+
@saved_at = stored[:saved_at]
|
239
|
+
@saved_duration = stored[:saved_duration]
|
240
|
+
|
241
|
+
# skip the saved duration to continue counting
|
242
|
+
@last_checked = Fluent::Engine.now - @saved_duration
|
243
|
+
else
|
244
|
+
$log.warn "out_grepcounter: stored data is outdated. ignore stored data"
|
245
|
+
end
|
246
|
+
else
|
247
|
+
$log.warn "out_grepcounter: configuration param was changed. ignore stored data"
|
248
|
+
end
|
249
|
+
end
|
250
|
+
rescue => e
|
251
|
+
$log.warn "out_grepcounter: Can't load store_file #{e.class} #{e.message}"
|
252
|
+
end
|
253
|
+
end
|
254
|
+
|
160
255
|
end
|
@@ -24,29 +24,17 @@ describe Fluent::GrepCounterOutput do
|
|
24
24
|
end
|
25
25
|
|
26
26
|
context 'invalid aggregate' do
|
27
|
-
let(:config)
|
28
|
-
CONFIG + %[
|
29
|
-
aggregate foo
|
30
|
-
]
|
31
|
-
end
|
27
|
+
let(:config) { CONFIG + %[aggregate foo] }
|
32
28
|
it { expect { driver }.to raise_error(Fluent::ConfigError) }
|
33
29
|
end
|
34
30
|
|
35
31
|
context 'no tag for aggregate all' do
|
36
|
-
let(:config)
|
37
|
-
CONFIG + %[
|
38
|
-
aggregate all
|
39
|
-
]
|
40
|
-
end
|
32
|
+
let(:config) { CONFIG + %[aggregate all] }
|
41
33
|
it { expect { driver }.to raise_error(Fluent::ConfigError) }
|
42
34
|
end
|
43
35
|
|
44
36
|
context 'invalid comparator' do
|
45
|
-
let(:config)
|
46
|
-
CONFIG + %[
|
47
|
-
comparator foo
|
48
|
-
]
|
49
|
-
end
|
37
|
+
let(:config) { CONFIG + %[comparator foo] }
|
50
38
|
it { expect { driver }.to raise_error(Fluent::ConfigError) }
|
51
39
|
end
|
52
40
|
end
|
@@ -82,20 +70,25 @@ describe Fluent::GrepCounterOutput do
|
|
82
70
|
driver.run { messages.each {|message| driver.emit({'message' => message}, time) } }
|
83
71
|
driver.instance.flush_emit(0)
|
84
72
|
end
|
85
|
-
|
86
|
-
|
87
|
-
|
73
|
+
let(:expected) do
|
74
|
+
{
|
75
|
+
"count"=>4,
|
76
|
+
"message"=>[
|
77
|
+
"2013/01/13T07:02:11.124202 INFO GET /ping",
|
78
|
+
"2013/01/13T07:02:13.232645 WARN POST /auth",
|
79
|
+
"2013/01/13T07:02:21.542145 WARN GET /favicon.ico",
|
80
|
+
"2013/01/13T07:02:43.632145 WARN POST /login",
|
81
|
+
],
|
82
|
+
"input_tag" => tag,
|
83
|
+
"input_tag_last" => tag.split('.').last,
|
84
|
+
}
|
88
85
|
end
|
89
86
|
|
90
87
|
context 'default' do
|
91
88
|
let(:config) { CONFIG }
|
92
89
|
before do
|
93
90
|
Fluent::Engine.stub(:now).and_return(time)
|
94
|
-
Fluent::Engine.should_receive(:emit).with("count.#{tag}", time,
|
95
|
-
"message"=>["2013/01/13T07:02:11.124202 INFO GET /ping","2013/01/13T07:02:13.232645 WARN POST /auth","2013/01/13T07:02:21.542145 WARN GET /favicon.ico","2013/01/13T07:02:43.632145 WARN POST /login"],
|
96
|
-
"input_tag" => tag,
|
97
|
-
"input_tag_last" => tag.split('.').last,
|
98
|
-
})
|
91
|
+
Fluent::Engine.should_receive(:emit).with("count.#{tag}", time, expected)
|
99
92
|
end
|
100
93
|
it { emit }
|
101
94
|
end
|
@@ -104,198 +97,258 @@ describe Fluent::GrepCounterOutput do
|
|
104
97
|
let(:config) { CONFIG + %[ regexp WARN ] }
|
105
98
|
before do
|
106
99
|
Fluent::Engine.stub(:now).and_return(time)
|
107
|
-
Fluent::Engine.should_receive(:emit).with("count.#{tag}", time, {
|
108
|
-
"
|
109
|
-
"
|
110
|
-
|
111
|
-
|
100
|
+
Fluent::Engine.should_receive(:emit).with("count.#{tag}", time, expected.merge({
|
101
|
+
"count"=>3,
|
102
|
+
"message"=>[
|
103
|
+
"2013/01/13T07:02:13.232645 WARN POST /auth",
|
104
|
+
"2013/01/13T07:02:21.542145 WARN GET /favicon.ico",
|
105
|
+
"2013/01/13T07:02:43.632145 WARN POST /login"
|
106
|
+
],
|
107
|
+
}))
|
112
108
|
end
|
113
109
|
it { emit }
|
114
110
|
end
|
115
111
|
|
116
112
|
context 'exclude' do
|
117
|
-
let(:config)
|
118
|
-
CONFIG + %[
|
119
|
-
regexp WARN
|
120
|
-
exclude favicon
|
121
|
-
]
|
122
|
-
end
|
113
|
+
let(:config) { CONFIG + %[regexp WARN \n exclude favicon] }
|
123
114
|
before do
|
124
115
|
Fluent::Engine.stub(:now).and_return(time)
|
125
|
-
Fluent::Engine.should_receive(:emit).with("count.#{tag}", time, {
|
126
|
-
"
|
127
|
-
"
|
128
|
-
|
129
|
-
|
116
|
+
Fluent::Engine.should_receive(:emit).with("count.#{tag}", time, expected.merge({
|
117
|
+
"count"=>2,
|
118
|
+
"message"=>[
|
119
|
+
"2013/01/13T07:02:13.232645 WARN POST /auth",
|
120
|
+
"2013/01/13T07:02:43.632145 WARN POST /login"
|
121
|
+
],
|
122
|
+
}))
|
130
123
|
end
|
131
124
|
it { emit }
|
132
125
|
end
|
133
126
|
|
134
|
-
context
|
135
|
-
|
136
|
-
CONFIG + %[
|
137
|
-
|
138
|
-
|
139
|
-
|
127
|
+
context "threshold and comparator" do
|
128
|
+
context '>= threshold' do
|
129
|
+
let(:config) { CONFIG + %[threshold 4] }
|
130
|
+
before do
|
131
|
+
Fluent::Engine.stub(:now).and_return(time)
|
132
|
+
Fluent::Engine.should_receive(:emit).with("count.#{tag}", time, expected)
|
133
|
+
end
|
134
|
+
it { emit }
|
140
135
|
end
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
136
|
+
|
137
|
+
context 'not >= threshold' do
|
138
|
+
let(:config) { CONFIG + %[threshold 5] }
|
139
|
+
before do
|
140
|
+
Fluent::Engine.stub(:now).and_return(time)
|
141
|
+
Fluent::Engine.should_not_receive(:emit)
|
142
|
+
end
|
143
|
+
it { emit }
|
144
|
+
end
|
145
|
+
|
146
|
+
context '<= threshold' do
|
147
|
+
let(:config) { CONFIG + %[threshold 4 \n comparator <=] }
|
148
|
+
before do
|
149
|
+
Fluent::Engine.stub(:now).and_return(time)
|
150
|
+
Fluent::Engine.should_receive(:emit).with("count.#{tag}", time, expected)
|
151
|
+
end
|
152
|
+
it { emit }
|
153
|
+
end
|
154
|
+
|
155
|
+
context 'not <= threshold' do
|
156
|
+
let(:config) { CONFIG + %[threshold 3 \n comparator <=] }
|
157
|
+
before do
|
158
|
+
Fluent::Engine.stub(:now).and_return(time)
|
159
|
+
Fluent::Engine.should_not_receive(:emit)
|
160
|
+
end
|
161
|
+
it { emit }
|
148
162
|
end
|
149
|
-
it { emit }
|
150
163
|
end
|
151
164
|
|
152
|
-
context
|
153
|
-
|
154
|
-
CONFIG + %[
|
155
|
-
|
156
|
-
|
157
|
-
|
165
|
+
context "less|greater_than|equal" do
|
166
|
+
context 'greater_equal' do
|
167
|
+
let(:config) { CONFIG + %[greater_equal 4] }
|
168
|
+
before do
|
169
|
+
Fluent::Engine.stub(:now).and_return(time)
|
170
|
+
Fluent::Engine.should_receive(:emit).with("count.#{tag}", time, expected)
|
171
|
+
end
|
172
|
+
it { emit }
|
158
173
|
end
|
159
|
-
|
160
|
-
|
161
|
-
|
174
|
+
|
175
|
+
context 'not greater_equal' do
|
176
|
+
let(:config) { CONFIG + %[greater_equal 5] }
|
177
|
+
before do
|
178
|
+
Fluent::Engine.stub(:now).and_return(time)
|
179
|
+
Fluent::Engine.should_not_receive(:emit)
|
180
|
+
end
|
181
|
+
it { emit }
|
182
|
+
end
|
183
|
+
|
184
|
+
context 'greater_than' do
|
185
|
+
let(:config) { CONFIG + %[greater_than 3] }
|
186
|
+
before do
|
187
|
+
Fluent::Engine.stub(:now).and_return(time)
|
188
|
+
Fluent::Engine.should_receive(:emit).with("count.#{tag}", time, expected)
|
189
|
+
end
|
190
|
+
it { emit }
|
191
|
+
end
|
192
|
+
|
193
|
+
context 'not greater_than' do
|
194
|
+
let(:config) { CONFIG + %[greater_than 4] }
|
195
|
+
before do
|
196
|
+
Fluent::Engine.stub(:now).and_return(time)
|
197
|
+
Fluent::Engine.should_not_receive(:emit)
|
198
|
+
end
|
199
|
+
it { emit }
|
200
|
+
end
|
201
|
+
|
202
|
+
context 'less_equal' do
|
203
|
+
let(:config) { CONFIG + %[less_equal 4] }
|
204
|
+
before do
|
205
|
+
Fluent::Engine.stub(:now).and_return(time)
|
206
|
+
Fluent::Engine.should_receive(:emit).with("count.#{tag}", time, expected)
|
207
|
+
end
|
208
|
+
it { emit }
|
209
|
+
end
|
210
|
+
|
211
|
+
context 'not less_equal' do
|
212
|
+
let(:config) { CONFIG + %[less_equal 3] }
|
213
|
+
before do
|
214
|
+
Fluent::Engine.stub(:now).and_return(time)
|
215
|
+
Fluent::Engine.should_not_receive(:emit)
|
216
|
+
end
|
217
|
+
it { emit }
|
218
|
+
end
|
219
|
+
|
220
|
+
context 'less_than' do
|
221
|
+
let(:config) { CONFIG + %[less_than 5] }
|
222
|
+
before do
|
223
|
+
Fluent::Engine.stub(:now).and_return(time)
|
224
|
+
Fluent::Engine.should_receive(:emit).with("count.#{tag}", time, expected)
|
225
|
+
end
|
226
|
+
it { emit }
|
227
|
+
end
|
228
|
+
|
229
|
+
context 'not less_than' do
|
230
|
+
let(:config) { CONFIG + %[less_than 4] }
|
231
|
+
before do
|
232
|
+
Fluent::Engine.stub(:now).and_return(time)
|
233
|
+
Fluent::Engine.should_not_receive(:emit)
|
234
|
+
end
|
235
|
+
it { emit }
|
236
|
+
end
|
237
|
+
|
238
|
+
context 'between' do
|
239
|
+
let(:config) { CONFIG + %[greater_than 1 \n less_than 5] }
|
240
|
+
before do
|
241
|
+
Fluent::Engine.stub(:now).and_return(time)
|
242
|
+
Fluent::Engine.should_receive(:emit).with("count.#{tag}", time, expected)
|
243
|
+
end
|
244
|
+
it { emit }
|
245
|
+
end
|
246
|
+
|
247
|
+
context 'not between' do
|
248
|
+
let(:config) { CONFIG + %[greater_than 1 \n less_than 4] }
|
249
|
+
before do
|
250
|
+
Fluent::Engine.stub(:now).and_return(time)
|
251
|
+
Fluent::Engine.should_not_receive(:emit)
|
252
|
+
end
|
253
|
+
it { emit }
|
162
254
|
end
|
163
|
-
it { emit }
|
164
255
|
end
|
165
256
|
|
166
257
|
context 'output_tag' do
|
167
|
-
let(:config)
|
168
|
-
CONFIG + %[
|
169
|
-
regexp WARN
|
170
|
-
output_tag foo
|
171
|
-
]
|
172
|
-
end
|
258
|
+
let(:config) { CONFIG + %[output_tag foo] }
|
173
259
|
before do
|
174
260
|
Fluent::Engine.stub(:now).and_return(time)
|
175
|
-
Fluent::Engine.should_receive(:emit).with("foo", time,
|
176
|
-
"message"=>["2013/01/13T07:02:13.232645 WARN POST /auth","2013/01/13T07:02:21.542145 WARN GET /favicon.ico","2013/01/13T07:02:43.632145 WARN POST /login"],
|
177
|
-
"input_tag" => tag,
|
178
|
-
"input_tag_last" => tag.split('.').last,
|
179
|
-
})
|
261
|
+
Fluent::Engine.should_receive(:emit).with("foo", time, expected)
|
180
262
|
end
|
181
263
|
it { emit }
|
182
264
|
end
|
183
265
|
|
184
266
|
context 'add_tag_prefix' do
|
185
|
-
let(:config)
|
186
|
-
CONFIG + %[
|
187
|
-
regexp WARN
|
188
|
-
add_tag_prefix foo
|
189
|
-
]
|
190
|
-
end
|
267
|
+
let(:config) { CONFIG + %[add_tag_prefix foo] }
|
191
268
|
before do
|
192
269
|
Fluent::Engine.stub(:now).and_return(time)
|
193
|
-
Fluent::Engine.should_receive(:emit).with("foo.#{tag}", time,
|
194
|
-
"message"=>["2013/01/13T07:02:13.232645 WARN POST /auth","2013/01/13T07:02:21.542145 WARN GET /favicon.ico","2013/01/13T07:02:43.632145 WARN POST /login"],
|
195
|
-
"input_tag" => tag,
|
196
|
-
"input_tag_last" => tag.split('.').last,
|
197
|
-
})
|
270
|
+
Fluent::Engine.should_receive(:emit).with("foo.#{tag}", time, expected)
|
198
271
|
end
|
199
272
|
it { emit }
|
200
273
|
end
|
201
274
|
|
202
275
|
context 'output_with_joined_delimiter' do
|
203
|
-
|
204
|
-
|
205
|
-
CONFIG + %[
|
206
|
-
regexp WARN
|
207
|
-
output_with_joined_delimiter \\n
|
208
|
-
]
|
209
|
-
end
|
276
|
+
# \\n shall be \n in config file
|
277
|
+
let(:config) { CONFIG + %[output_with_joined_delimiter \\n] }
|
210
278
|
before do
|
211
279
|
Fluent::Engine.stub(:now).and_return(time)
|
212
|
-
|
213
|
-
|
214
|
-
"input_tag" => tag,
|
215
|
-
"input_tag_last" => tag.split('.').last,
|
216
|
-
})
|
280
|
+
message = expected["message"].join('\n')
|
281
|
+
Fluent::Engine.should_receive(:emit).with("count.#{tag}", time, expected.merge("message" => message))
|
217
282
|
end
|
218
283
|
it { emit }
|
219
284
|
end
|
220
285
|
|
221
286
|
context 'aggregate all' do
|
287
|
+
let(:messages) { ['foobar', 'foobar'] }
|
222
288
|
let(:emit) do
|
223
289
|
driver.run { messages.each {|message| driver.emit_with_tag({'message' => message}, time, 'foo.bar') } }
|
224
290
|
driver.run { messages.each {|message| driver.emit_with_tag({'message' => message}, time, 'foo.bar2') } }
|
225
291
|
driver.instance.flush_emit(0)
|
226
292
|
end
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
output_tag count
|
233
|
-
]
|
293
|
+
let(:expected) do
|
294
|
+
{
|
295
|
+
"count"=>messages.size*2,
|
296
|
+
"message"=>messages*2,
|
297
|
+
}
|
234
298
|
end
|
299
|
+
|
300
|
+
let(:config) { CONFIG + %[aggregate all \n output_tag count] }
|
235
301
|
before do
|
236
302
|
Fluent::Engine.stub(:now).and_return(time)
|
237
|
-
Fluent::Engine.should_receive(:emit).with("count", time,
|
238
|
-
"message"=>["2013/01/13T07:02:13.232645 WARN POST /auth","2013/01/13T07:02:21.542145 WARN GET /favicon.ico","2013/01/13T07:02:43.632145 WARN POST /login"]*2,
|
239
|
-
})
|
303
|
+
Fluent::Engine.should_receive(:emit).with("count", time, expected)
|
240
304
|
end
|
241
305
|
it { emit }
|
242
306
|
end
|
243
307
|
|
244
308
|
context 'replace_invalid_sequence' do
|
245
|
-
let(:config)
|
246
|
-
|
247
|
-
regexp WARN
|
248
|
-
replace_invalid_sequence true
|
249
|
-
]
|
250
|
-
end
|
251
|
-
let(:messages) do
|
252
|
-
[
|
253
|
-
"\xff".force_encoding('UTF-8'),
|
254
|
-
]
|
255
|
-
end
|
309
|
+
let(:config) { CONFIG + %[regexp WARN \n replace_invalid_sequence true] }
|
310
|
+
let(:messages) { [ "\xff".force_encoding('UTF-8') ] }
|
256
311
|
before do
|
257
312
|
Fluent::Engine.stub(:now).and_return(time)
|
258
313
|
end
|
259
|
-
it { expect { emit }.not_to raise_error
|
314
|
+
it { expect { emit }.not_to raise_error }
|
260
315
|
end
|
261
316
|
|
262
|
-
describe "
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
|
267
|
-
|
268
|
-
|
269
|
-
]
|
270
|
-
end
|
271
|
-
before do
|
272
|
-
Fluent::Engine.stub(:now).and_return(time)
|
273
|
-
Fluent::Engine.should_receive(:emit).with("count.#{tag}", time, {"count"=>3,
|
274
|
-
"message"=>["2013/01/13T07:02:13.232645 WARN POST /auth","2013/01/13T07:02:21.542145 WARN GET /favicon.ico","2013/01/13T07:02:43.632145 WARN POST /login"],
|
275
|
-
"input_tag" => tag,
|
276
|
-
"input_tag_last" => tag.split('.').last,
|
277
|
-
})
|
278
|
-
end
|
279
|
-
it { emit }
|
317
|
+
describe "store_file" do
|
318
|
+
let(:store_file) do
|
319
|
+
dirname = "tmp"
|
320
|
+
Dir.mkdir dirname unless Dir.exist? dirname
|
321
|
+
filename = "#{dirname}/test.dat"
|
322
|
+
File.unlink filename if File.exist? filename
|
323
|
+
filename
|
280
324
|
end
|
281
325
|
|
282
|
-
|
283
|
-
|
284
|
-
|
285
|
-
|
286
|
-
|
287
|
-
|
288
|
-
|
289
|
-
|
290
|
-
|
291
|
-
|
292
|
-
|
293
|
-
|
294
|
-
|
326
|
+
let(:config) { CONFIG + %[store_file #{store_file}] }
|
327
|
+
|
328
|
+
it 'stored_data and loaded_data should equal' do
|
329
|
+
driver.run { messages.each {|message| driver.emit({'message' => message}, time) } }
|
330
|
+
driver.instance.shutdown
|
331
|
+
stored_counts = driver.instance.counts
|
332
|
+
stored_matches = driver.instance.matches
|
333
|
+
stored_saved_at = driver.instance.saved_at
|
334
|
+
stored_saved_duration = driver.instance.saved_duration
|
335
|
+
driver.instance.counts = {}
|
336
|
+
driver.instance.matches = {}
|
337
|
+
driver.instance.saved_at = nil
|
338
|
+
driver.instance.saved_duration = nil
|
339
|
+
|
340
|
+
driver.instance.start
|
341
|
+
loaded_counts = driver.instance.counts
|
342
|
+
loaded_matches = driver.instance.matches
|
343
|
+
loaded_saved_at = driver.instance.saved_at
|
344
|
+
loaded_saved_duration = driver.instance.saved_duration
|
345
|
+
|
346
|
+
loaded_counts.should == stored_counts
|
347
|
+
loaded_matches.should == stored_matches
|
348
|
+
loaded_saved_at.should == stored_saved_at
|
349
|
+
loaded_saved_duration.should == stored_saved_duration
|
295
350
|
end
|
296
351
|
end
|
352
|
+
|
297
353
|
end
|
298
354
|
end
|
299
|
-
|
300
|
-
|
301
|
-
|
data/spec/spec_helper.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-grepcounter
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Naotoshi SEO
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2013-
|
11
|
+
date: 2013-09-26 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -67,7 +67,7 @@ dependencies:
|
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: '0'
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
|
-
name:
|
70
|
+
name: pry-nav
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
72
72
|
requirements:
|
73
73
|
- - '>='
|
@@ -120,7 +120,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
120
120
|
version: '0'
|
121
121
|
requirements: []
|
122
122
|
rubyforge_project: fluent-plugin-grepcounter
|
123
|
-
rubygems_version: 2.0.
|
123
|
+
rubygems_version: 2.0.3
|
124
124
|
signing_key:
|
125
125
|
specification_version: 4
|
126
126
|
summary: Count the number of matched messages
|