fluent-plugin-numeric-counter 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +21 -0
- data/Gemfile +4 -0
- data/LICENSE +22 -0
- data/README.md +74 -0
- data/Rakefile +11 -0
- data/fluent-plugin-numeric-counter.gemspec +19 -0
- data/lib/fluent/plugin/out_numeric_counter.rb +220 -0
- data/test/helper.rb +28 -0
- data/test/plugin/test_out_numeric_counter.rb +190 -0
- metadata +88 -0
data/.gitignore
ADDED
data/Gemfile
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2012 TAGOMORI Satoshi
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,74 @@
|
|
1
|
+
# fluent-plugin-numeric-counter
|
2
|
+
|
3
|
+
## Component
|
4
|
+
|
5
|
+
### NumericCounterOutput
|
6
|
+
|
7
|
+
Fluentd plugin to count messages, matches for numeric range patterns, and emits its result (like fluent-plugin-datacounter).
|
8
|
+
|
9
|
+
- Counts per min/hour/day
|
10
|
+
- Counts per second (average every min/hour/day)
|
11
|
+
- Percentage of each numeric pattern in total counts of messages
|
12
|
+
|
13
|
+
NumericCounterOutput emits messages contains results data, so you can output these message (with 'numcount' tag by default) to any outputs you want.
|
14
|
+
|
15
|
+
output ex1 (aggregates all inputs): {"pattern1_count":20, "pattern1_rate":0.333, "pattern1_percentage":25.0, "pattern2_count":40, "pattern2_rate":0.666, "pattern2_percentage":50.0, "unmatched_count":20, "unmatched_rate":0.333, "unmatched_percentage":25.0}
|
16
|
+
output ex2 (aggregates per tag): {"test_pattern1_count":10, "test_pattern1_rate":0.333, "test_pattern1_percentage":25.0, "test_pattern2_count":40, "test_pattern2_rate":0.666, "test_pattern2_percentage":50.0, "test_unmatched_count":20, "test_unmatched_rate":0.333, "test_unmatched_percentage":25.0}
|
17
|
+
|
18
|
+
'input\_tag\_remove\_prefix' option available if you want to remove tag prefix from output field names.
|
19
|
+
|
20
|
+
If you want to omit 'unmatched' messages from percentage counting, specify 'outcast_unmatched yes'.
|
21
|
+
|
22
|
+
## Configuration
|
23
|
+
|
24
|
+
### NumericCounterOutput
|
25
|
+
|
26
|
+
Count messages that have attribute 'duration'(response time by microseconds), by several numeric ranges, per minutes.
|
27
|
+
|
28
|
+
<match accesslog.**>
|
29
|
+
type numeric_counter
|
30
|
+
unit minute # or 'count_interval 60s' or '45s', '3m' ... as you want
|
31
|
+
aggregate all # or 'tag'
|
32
|
+
count_key duration
|
33
|
+
|
34
|
+
# patternX: X(1-20)
|
35
|
+
# patternX NAME LOW HIGH #=> patternX matches N like LOW <= N < HIGH
|
36
|
+
|
37
|
+
pattern1 HIGHSPEED 0 10000 # under 10ms
|
38
|
+
pattern2 SEMIHIGHSPEED 10000 100000 # under 100ms
|
39
|
+
pattern3 NORMAL 100000 1000000 # under 1s
|
40
|
+
pattern4 STUPID 1000000 10000000 # under 10s!
|
41
|
+
|
42
|
+
# patternZ (Z is last number of specified patterns)
|
43
|
+
# patternZ NAME LOW #=> patternZ matches N like LOW <= N (upper threshold is unlimited)
|
44
|
+
patternZ MUSTDIE 10000000 # over 10s!
|
45
|
+
</match>
|
46
|
+
|
47
|
+
Size specifier (like 10k, 5M, 103g) available as 1024\*\*1, 1024\*\*2, 1024\*\*3 ...., for example, for bytes of access log.
|
48
|
+
|
49
|
+
<match accesslog.**>
|
50
|
+
type numeric_counter
|
51
|
+
unit hour
|
52
|
+
aggregate tag
|
53
|
+
count_key bytes
|
54
|
+
|
55
|
+
pattern1 SMALL 0 1k
|
56
|
+
pattern2 MIDDLE 1k 1m
|
57
|
+
pattern3 LARGE 1m 10m
|
58
|
+
pattern4 HUGE 10m 1g
|
59
|
+
pattern5 XXXX 1g
|
60
|
+
</match>
|
61
|
+
|
62
|
+
You can try to use negative numbers, and floating point numbers.... (not tested enough).
|
63
|
+
|
64
|
+
## TODO
|
65
|
+
|
66
|
+
* more tests
|
67
|
+
* more documents
|
68
|
+
|
69
|
+
## Copyright
|
70
|
+
|
71
|
+
* Copyright
|
72
|
+
* Copyright (c) 2012- TAGOMORI Satoshi (tagomoris)
|
73
|
+
* License
|
74
|
+
* Apache License, Version 2.0
|
data/Rakefile
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
|
3
|
+
Gem::Specification.new do |gem|
|
4
|
+
gem.name = "fluent-plugin-numeric-counter"
|
5
|
+
gem.version = "0.1.0"
|
6
|
+
gem.authors = ["TAGOMORI Satoshi"]
|
7
|
+
gem.email = ["tagomoris@gmail.com"]
|
8
|
+
gem.description = %q{Counts messages, with specified key and numeric value in specified range}
|
9
|
+
gem.summary = %q{Fluentd plugin to count messages with specified numeric values}
|
10
|
+
gem.homepage = "https://github.com/tagomoris/fluent-plugin-numeric-counter"
|
11
|
+
|
12
|
+
gem.files = `git ls-files`.split($\)
|
13
|
+
gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
14
|
+
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
15
|
+
gem.require_paths = ["lib"]
|
16
|
+
|
17
|
+
gem.add_development_dependency "fluentd"
|
18
|
+
gem.add_runtime_dependency "fluentd"
|
19
|
+
end
|
@@ -0,0 +1,220 @@
|
|
1
|
+
class Fluent::NumericCounterOutput < Fluent::Output
|
2
|
+
Fluent::Plugin.register_output('numeric_counter', self)
|
3
|
+
|
4
|
+
PATTERN_MAX_NUM = 20
|
5
|
+
|
6
|
+
config_param :count_interval, :time, :default => 60
|
7
|
+
config_param :unit, :string, :default => nil
|
8
|
+
config_param :aggregate, :string, :default => 'tag'
|
9
|
+
config_param :tag, :string, :default => 'numcount'
|
10
|
+
config_param :input_tag_remove_prefix, :string, :default => nil
|
11
|
+
config_param :count_key, :string
|
12
|
+
config_param :outcast_unmatched, :bool, :default => false
|
13
|
+
|
14
|
+
# pattern0 reserved as unmatched counts
|
15
|
+
config_param :pattern1, :string # string: NAME LOW HIGH
|
16
|
+
# LOW/HIGH allows size prefix (ex: 10k, 5M, 3500G)
|
17
|
+
(2..PATTERN_MAX_NUM).each do |i|
|
18
|
+
config_param ('pattern' + i.to_s).to_sym, :string, :default => nil
|
19
|
+
end
|
20
|
+
|
21
|
+
attr_accessor :counts, :last_checked
|
22
|
+
|
23
|
+
# for test
|
24
|
+
# attr_accessor :count_interval, :unit, :aggregate, :tag, :input_tag_remove_prefix, :count_key, :outcast_unmatched
|
25
|
+
attr_accessor :patterns
|
26
|
+
|
27
|
+
def parse_num(str)
|
28
|
+
if str.nil?
|
29
|
+
nil
|
30
|
+
elsif str =~ /^[-0-9]+$/
|
31
|
+
str.to_i
|
32
|
+
elsif str =~ /^[-.0-9]+$/
|
33
|
+
str.to_f
|
34
|
+
else
|
35
|
+
Fluent::Config.size_value(str)
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def configure(conf)
|
40
|
+
super
|
41
|
+
|
42
|
+
if @unit
|
43
|
+
@count_interval = case @unit
|
44
|
+
when 'minute' then 60
|
45
|
+
when 'hour' then 3600
|
46
|
+
when 'day' then 86400
|
47
|
+
else
|
48
|
+
raise Fluent::ConfigError, 'unit must be one of minute/hour/day'
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
@aggregate = @aggregate.to_sym
|
53
|
+
raise Fluent::ConfigError, "numeric_counter allows tag/all to aggregate unit" unless [:tag, :all].include?(@aggregate)
|
54
|
+
|
55
|
+
@patterns = [[0, 'unmatched', nil, nil]] # counts-index, name, low, high
|
56
|
+
pattern_names = ['unmatched']
|
57
|
+
|
58
|
+
invalids = conf.keys.select{|k| k =~ /^pattern(\d+)$/ and not (1..PATTERN_MAX_NUM).include?($1.to_i)}
|
59
|
+
if invalids.size > 0
|
60
|
+
$log.warn "invalid number patterns (valid pattern number:1-#{PATTERN_MAX_NUM}):" + invalids.join(",")
|
61
|
+
end
|
62
|
+
(1..PATTERN_MAX_NUM).each do |i|
|
63
|
+
next unless conf["pattern#{i}"]
|
64
|
+
name,low,high = conf["pattern#{i}"].split(/ +/, 3)
|
65
|
+
@patterns.push([i, name, parse_num(low), parse_num(high)])
|
66
|
+
pattern_names.push(name)
|
67
|
+
end
|
68
|
+
pattern_index_list = conf.keys.select{|s| s =~ /^pattern\d$/}.map{|v| (/^pattern(\d)$/.match(v))[1].to_i}
|
69
|
+
unless pattern_index_list.reduce(true){|v,i| v and @patterns[i]}
|
70
|
+
raise Fluent::ConfigError, "jump of pattern index found"
|
71
|
+
end
|
72
|
+
unless @patterns.length == pattern_names.uniq.length
|
73
|
+
raise Fluent::ConfigError, "duplicated pattern names found"
|
74
|
+
end
|
75
|
+
@patterns[1..-1].each do |index, name, low, high|
|
76
|
+
raise Fluent::ConfigError, "numbers of low/high missing" if low.nil?
|
77
|
+
raise Fluent::ConfigError, "unspecified high threshold allowed only in last pattern" if high.nil? and index != @patterns.length - 1
|
78
|
+
end
|
79
|
+
|
80
|
+
if @input_tag_remove_prefix
|
81
|
+
@removed_prefix_string = @input_tag_remove_prefix + '.'
|
82
|
+
@removed_length = @removed_prefix_string.length
|
83
|
+
end
|
84
|
+
|
85
|
+
@counts = count_initialized
|
86
|
+
@mutex = Mutex.new
|
87
|
+
end
|
88
|
+
|
89
|
+
def start
|
90
|
+
super
|
91
|
+
start_watch
|
92
|
+
end
|
93
|
+
|
94
|
+
def shutdown
|
95
|
+
super
|
96
|
+
@watcher.terminate
|
97
|
+
@watcher.join
|
98
|
+
end
|
99
|
+
|
100
|
+
def count_initialized(keys=nil)
|
101
|
+
# counts['tag'][pattern_index_num] = count
|
102
|
+
if @aggregate == :all
|
103
|
+
{'all' => Array.new(@patterns.length){|i| 0}}
|
104
|
+
elsif keys
|
105
|
+
values = Array.new(keys.length){|i|
|
106
|
+
Array.new(@patterns.length){|j| 0 }
|
107
|
+
}
|
108
|
+
Hash[[keys, values].transpose]
|
109
|
+
else
|
110
|
+
{}
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
def countups(tag, counts)
|
115
|
+
if @aggregate == :all
|
116
|
+
tag = 'all'
|
117
|
+
end
|
118
|
+
|
119
|
+
@mutex.synchronize {
|
120
|
+
@counts[tag] ||= [0] * @patterns.length
|
121
|
+
counts.each_with_index do |count, i|
|
122
|
+
@counts[tag][i] += count
|
123
|
+
end
|
124
|
+
}
|
125
|
+
end
|
126
|
+
|
127
|
+
def stripped_tag(tag)
|
128
|
+
return tag unless @input_tag_remove_prefix
|
129
|
+
return tag[@removed_length..-1] if tag.start_with?(@removed_prefix_string) and tag.length > @removed_length
|
130
|
+
return tag[@removed_length..-1] if tag == @input_tag_remove_prefix
|
131
|
+
tag
|
132
|
+
end
|
133
|
+
|
134
|
+
def generate_output(counts, step)
|
135
|
+
output = {}
|
136
|
+
|
137
|
+
if @aggregate == :all
|
138
|
+
# index 0 is unmatched
|
139
|
+
sum = if @outcast_unmatched
|
140
|
+
counts['all'][1..-1].inject(:+)
|
141
|
+
else
|
142
|
+
counts['all'].inject(:+)
|
143
|
+
end
|
144
|
+
counts['all'].each_with_index do |count,i|
|
145
|
+
name = @patterns[i][1]
|
146
|
+
output[name + '_count'] = count
|
147
|
+
output[name + '_rate'] = ((count * 100.0) / (1.00 * step)).floor / 100.0
|
148
|
+
unless i == 0 and @outcast_unmatched
|
149
|
+
output[name + '_percentage'] = count * 100.0 / (1.00 * sum) if sum > 0
|
150
|
+
end
|
151
|
+
end
|
152
|
+
return output
|
153
|
+
end
|
154
|
+
|
155
|
+
counts.keys.each do |tag|
|
156
|
+
t = stripped_tag(tag)
|
157
|
+
sum = if @outcast_unmatched
|
158
|
+
counts[tag][1..-1].inject(:+)
|
159
|
+
else
|
160
|
+
counts[tag].inject(:+)
|
161
|
+
end
|
162
|
+
counts[tag].each_with_index do |count,i|
|
163
|
+
name = @patterns[i][1]
|
164
|
+
output[t + '_' + name + '_count'] = count
|
165
|
+
output[t + '_' + name + '_rate'] = ((count * 100.0) / (1.00 * step)).floor / 100.0
|
166
|
+
unless i == 0 and @outcast_unmatched
|
167
|
+
output[t + '_' + name + '_percentage'] = count * 100.0 / (1.00 * sum) if sum > 0
|
168
|
+
end
|
169
|
+
end
|
170
|
+
end
|
171
|
+
output
|
172
|
+
end
|
173
|
+
|
174
|
+
def flush(step)
|
175
|
+
flushed,@counts = @counts,count_initialized(@counts.keys.dup)
|
176
|
+
generate_output(flushed, step)
|
177
|
+
end
|
178
|
+
|
179
|
+
def flush_emit(step)
|
180
|
+
Fluent::Engine.emit(@tag, Fluent::Engine.now, flush(step))
|
181
|
+
end
|
182
|
+
|
183
|
+
def start_watch
|
184
|
+
@watcher = Thread.new(&method(:watch))
|
185
|
+
end
|
186
|
+
|
187
|
+
def watch
|
188
|
+
@last_checked = Fluent::Engine.now
|
189
|
+
while true
|
190
|
+
sleep 0.5
|
191
|
+
if Fluent::Engine.now - @last_checked >= @count_interval
|
192
|
+
now = Fluent::Engine.now
|
193
|
+
flush_emit(now - @last_checked)
|
194
|
+
@last_checked = now
|
195
|
+
end
|
196
|
+
end
|
197
|
+
end
|
198
|
+
|
199
|
+
def emit(tag, es, chain)
|
200
|
+
c = [0] * @patterns.length
|
201
|
+
|
202
|
+
es.each do |time,record|
|
203
|
+
value = record[@count_key]
|
204
|
+
next if value.nil?
|
205
|
+
|
206
|
+
value = value.to_f
|
207
|
+
matched = false
|
208
|
+
@patterns.each do |index, name, low, high|
|
209
|
+
next if low.nil? or value < low or (not high.nil? and value >= high)
|
210
|
+
c[index] += 1
|
211
|
+
matched = true
|
212
|
+
break
|
213
|
+
end
|
214
|
+
c[0] += 1 unless matched
|
215
|
+
end
|
216
|
+
countups(tag, c)
|
217
|
+
|
218
|
+
chain.next
|
219
|
+
end
|
220
|
+
end
|
data/test/helper.rb
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'bundler'
|
3
|
+
begin
|
4
|
+
Bundler.setup(:default, :development)
|
5
|
+
rescue Bundler::BundlerError => e
|
6
|
+
$stderr.puts e.message
|
7
|
+
$stderr.puts "Run `bundle install` to install missing gems"
|
8
|
+
exit e.status_code
|
9
|
+
end
|
10
|
+
require 'test/unit'
|
11
|
+
|
12
|
+
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
13
|
+
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
14
|
+
require 'fluent/test'
|
15
|
+
unless ENV.has_key?('VERBOSE')
|
16
|
+
nulllogger = Object.new
|
17
|
+
nulllogger.instance_eval {|obj|
|
18
|
+
def method_missing(method, *args)
|
19
|
+
# pass
|
20
|
+
end
|
21
|
+
}
|
22
|
+
$log = nulllogger
|
23
|
+
end
|
24
|
+
|
25
|
+
require 'fluent/plugin/out_numeric_counter'
|
26
|
+
|
27
|
+
class Test::Unit::TestCase
|
28
|
+
end
|
@@ -0,0 +1,190 @@
|
|
1
|
+
require 'helper'
|
2
|
+
|
3
|
+
class NumericCounterOutputTest < Test::Unit::TestCase
|
4
|
+
def setup
|
5
|
+
Fluent::Test.setup
|
6
|
+
end
|
7
|
+
|
8
|
+
CONFIG = %[
|
9
|
+
count_interval 60
|
10
|
+
aggregate tag
|
11
|
+
input_tag_remove_prefix test
|
12
|
+
count_key target
|
13
|
+
pattern1 u100ms 0 100000
|
14
|
+
pattern2 u1s 100000 1000000
|
15
|
+
pattern3 u3s 1000000 3000000
|
16
|
+
]
|
17
|
+
|
18
|
+
def create_driver(conf=CONFIG, tag='test')
|
19
|
+
Fluent::Test::OutputTestDriver.new(Fluent::NumericCounterOutput, tag).configure(conf)
|
20
|
+
end
|
21
|
+
|
22
|
+
def test_parse_num
|
23
|
+
p = create_driver.instance
|
24
|
+
|
25
|
+
assert_equal 1, p.parse_num('1')
|
26
|
+
assert_equal -1, p.parse_num('-1')
|
27
|
+
assert_equal 1.0, p.parse_num('1.0')
|
28
|
+
assert_equal -2.0, p.parse_num('-2.0000')
|
29
|
+
assert_equal 1024, p.parse_num('1k')
|
30
|
+
end
|
31
|
+
|
32
|
+
def test_configure
|
33
|
+
d = create_driver %[
|
34
|
+
count_key field1
|
35
|
+
pattern1 smallnum 0.1 200
|
36
|
+
pattern2 subnum -500 -1
|
37
|
+
]
|
38
|
+
|
39
|
+
assert_equal 60, d.instance.count_interval
|
40
|
+
assert_equal :tag, d.instance.aggregate
|
41
|
+
assert_equal 'numcount', d.instance.tag
|
42
|
+
assert_nil d.instance.input_tag_remove_prefix
|
43
|
+
assert_equal false, d.instance.outcast_unmatched
|
44
|
+
assert_equal [[0, 'unmatched', nil, nil], [1, 'smallnum', 0.1, 200], [2, 'subnum', -500, -1]], d.instance.patterns
|
45
|
+
|
46
|
+
d = create_driver %[
|
47
|
+
count_key key1
|
48
|
+
pattern1 x 0.1 10
|
49
|
+
pattern2 y 10 11
|
50
|
+
pattern3 z 11
|
51
|
+
]
|
52
|
+
assert_equal [[0, 'unmatched', nil, nil], [1, 'x', 0.1, 10], [2, 'y', 10, 11], [3, 'z', 11, nil]], d.instance.patterns
|
53
|
+
end
|
54
|
+
|
55
|
+
def test_countups
|
56
|
+
d = create_driver
|
57
|
+
assert_nil d.instance.counts['test.input']
|
58
|
+
|
59
|
+
d.instance.countups('test.input', [0, 0, 0, 0])
|
60
|
+
assert_equal [0,0,0,0], d.instance.counts['test.input']
|
61
|
+
d.instance.countups('test.input', [1, 1, 1, 0])
|
62
|
+
assert_equal [1,1,1,0], d.instance.counts['test.input']
|
63
|
+
d.instance.countups('test.input', [0, 5, 1, 0])
|
64
|
+
assert_equal [1,6,2,0], d.instance.counts['test.input']
|
65
|
+
end
|
66
|
+
|
67
|
+
def test_generate_output
|
68
|
+
d = create_driver
|
69
|
+
# pattern1 u100ms 0 100000
|
70
|
+
# pattern2 u1s 100000 1000000
|
71
|
+
# pattern3 u3s 1000000 3000000
|
72
|
+
|
73
|
+
r1 = d.instance.generate_output({'test.input' => [60,240,180,120], 'test.input2' => [0,600,0,0]}, 60)
|
74
|
+
assert_equal 60, r1['input_unmatched_count']
|
75
|
+
assert_equal 1.0, r1['input_unmatched_rate']
|
76
|
+
assert_equal 10.0, r1['input_unmatched_percentage']
|
77
|
+
assert_equal 240, r1['input_u100ms_count']
|
78
|
+
assert_equal 4.0, r1['input_u100ms_rate']
|
79
|
+
assert_equal 40.0, r1['input_u100ms_percentage']
|
80
|
+
assert_equal 180, r1['input_u1s_count']
|
81
|
+
assert_equal 3.0, r1['input_u1s_rate']
|
82
|
+
assert_equal 30.0, r1['input_u1s_percentage']
|
83
|
+
assert_equal 120, r1['input_u3s_count']
|
84
|
+
assert_equal 2.0, r1['input_u3s_rate']
|
85
|
+
assert_equal 20.0, r1['input_u3s_percentage']
|
86
|
+
|
87
|
+
assert_equal 0, r1['input2_unmatched_count']
|
88
|
+
assert_equal 0.0, r1['input2_unmatched_rate']
|
89
|
+
assert_equal 0.0, r1['input2_unmatched_percentage']
|
90
|
+
assert_equal 600, r1['input2_u100ms_count']
|
91
|
+
assert_equal 10.0, r1['input2_u100ms_rate']
|
92
|
+
assert_equal 100.0, r1['input2_u100ms_percentage']
|
93
|
+
assert_equal 0, r1['input2_u1s_count']
|
94
|
+
assert_equal 0.0, r1['input2_u1s_rate']
|
95
|
+
assert_equal 0.0, r1['input2_u1s_percentage']
|
96
|
+
assert_equal 0, r1['input2_u3s_count']
|
97
|
+
assert_equal 0.0, r1['input2_u3s_rate']
|
98
|
+
assert_equal 0.0, r1['input2_u3s_percentage']
|
99
|
+
|
100
|
+
d = create_driver %[
|
101
|
+
aggregate all
|
102
|
+
count_key f1
|
103
|
+
pattern1 good 1 2
|
104
|
+
outcast_unmatched yes
|
105
|
+
]
|
106
|
+
r2 = d.instance.generate_output({'all' => [60,240]}, 60)
|
107
|
+
assert_equal 60, r2['unmatched_count']
|
108
|
+
assert_equal 1.0, r2['unmatched_rate']
|
109
|
+
assert_nil r2['unmatched_percentage']
|
110
|
+
assert_equal 240, r2['good_count']
|
111
|
+
assert_equal 4.0, r2['good_rate']
|
112
|
+
assert_equal 100.0, r2['good_percentage']
|
113
|
+
end
|
114
|
+
|
115
|
+
def test_pattern_num
|
116
|
+
assert_equal 20, Fluent::NumericCounterOutput::PATTERN_MAX_NUM
|
117
|
+
|
118
|
+
conf = %[
|
119
|
+
aggregate all
|
120
|
+
count_key field
|
121
|
+
]
|
122
|
+
(1..20).each do |i|
|
123
|
+
conf += "pattern#{i} name#{i} #{i} #{i+1}\n"
|
124
|
+
end
|
125
|
+
d = create_driver(conf, 'test.max')
|
126
|
+
d.run do
|
127
|
+
(0..21).each do |i|
|
128
|
+
d.emit({'field' => i})
|
129
|
+
end
|
130
|
+
end
|
131
|
+
r = d.instance.flush(60)
|
132
|
+
assert_equal 2, r['unmatched_count'] # 0 and 21
|
133
|
+
assert_equal 1, r['name1_count']
|
134
|
+
assert_equal 1, r['name2_count']
|
135
|
+
assert_equal 1, r['name3_count']
|
136
|
+
assert_equal 1, r['name4_count']
|
137
|
+
assert_equal 1, r['name5_count']
|
138
|
+
assert_equal 1, r['name6_count']
|
139
|
+
assert_equal 1, r['name7_count']
|
140
|
+
assert_equal 1, r['name8_count']
|
141
|
+
assert_equal 1, r['name9_count']
|
142
|
+
assert_equal 1, r['name10_count']
|
143
|
+
assert_equal 1, r['name11_count']
|
144
|
+
assert_equal 1, r['name12_count']
|
145
|
+
assert_equal 1, r['name13_count']
|
146
|
+
assert_equal 1, r['name14_count']
|
147
|
+
assert_equal 1, r['name15_count']
|
148
|
+
assert_equal 1, r['name16_count']
|
149
|
+
assert_equal 1, r['name17_count']
|
150
|
+
assert_equal 1, r['name18_count']
|
151
|
+
assert_equal 1, r['name19_count']
|
152
|
+
assert_equal 1, r['name20_count']
|
153
|
+
end
|
154
|
+
|
155
|
+
def test_emit
|
156
|
+
# CONFIG = %[
|
157
|
+
# count_interval 60
|
158
|
+
# aggregate tag
|
159
|
+
# input_tag_remove_prefix test
|
160
|
+
# count_key target
|
161
|
+
# pattern1 u100ms 0 100000
|
162
|
+
# pattern2 u1s 100000 1000000
|
163
|
+
# pattern3 u3s 1000000 3000000
|
164
|
+
# ]
|
165
|
+
d = create_driver(CONFIG, 'test.tag1')
|
166
|
+
d.run do
|
167
|
+
60.times do
|
168
|
+
d.emit({'target' => '50000'})
|
169
|
+
d.emit({'target' => '100000'})
|
170
|
+
d.emit({'target' => '100001'})
|
171
|
+
d.emit({'target' => '0.0'})
|
172
|
+
d.emit({'target' => '-1'})
|
173
|
+
end
|
174
|
+
end
|
175
|
+
r = d.instance.flush(60)
|
176
|
+
|
177
|
+
assert_equal 120, r['tag1_u100ms_count']
|
178
|
+
assert_equal 2.0, r['tag1_u100ms_rate']
|
179
|
+
assert_equal 40.0, r['tag1_u100ms_percentage']
|
180
|
+
assert_equal 120, r['tag1_u1s_count']
|
181
|
+
assert_equal 2.0, r['tag1_u1s_rate']
|
182
|
+
assert_equal 40, r['tag1_u1s_percentage']
|
183
|
+
assert_equal 0, r['tag1_u3s_count']
|
184
|
+
assert_equal 0, r['tag1_u3s_rate']
|
185
|
+
assert_equal 0, r['tag1_u3s_percentage']
|
186
|
+
assert_equal 60, r['tag1_unmatched_count']
|
187
|
+
assert_equal 1.0, r['tag1_unmatched_rate']
|
188
|
+
assert_equal 20, r['tag1_unmatched_percentage']
|
189
|
+
end
|
190
|
+
end
|
metadata
ADDED
@@ -0,0 +1,88 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-numeric-counter
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- TAGOMORI Satoshi
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2012-07-06 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: fluentd
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ! '>='
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '0'
|
22
|
+
type: :development
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ! '>='
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '0'
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: fluentd
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
34
|
+
requirements:
|
35
|
+
- - ! '>='
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: '0'
|
38
|
+
type: :runtime
|
39
|
+
prerelease: false
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ! '>='
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: '0'
|
46
|
+
description: Counts messages, with specified key and numeric value in specified range
|
47
|
+
email:
|
48
|
+
- tagomoris@gmail.com
|
49
|
+
executables: []
|
50
|
+
extensions: []
|
51
|
+
extra_rdoc_files: []
|
52
|
+
files:
|
53
|
+
- .gitignore
|
54
|
+
- Gemfile
|
55
|
+
- LICENSE
|
56
|
+
- README.md
|
57
|
+
- Rakefile
|
58
|
+
- fluent-plugin-numeric-counter.gemspec
|
59
|
+
- lib/fluent/plugin/out_numeric_counter.rb
|
60
|
+
- test/helper.rb
|
61
|
+
- test/plugin/test_out_numeric_counter.rb
|
62
|
+
homepage: https://github.com/tagomoris/fluent-plugin-numeric-counter
|
63
|
+
licenses: []
|
64
|
+
post_install_message:
|
65
|
+
rdoc_options: []
|
66
|
+
require_paths:
|
67
|
+
- lib
|
68
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
69
|
+
none: false
|
70
|
+
requirements:
|
71
|
+
- - ! '>='
|
72
|
+
- !ruby/object:Gem::Version
|
73
|
+
version: '0'
|
74
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
75
|
+
none: false
|
76
|
+
requirements:
|
77
|
+
- - ! '>='
|
78
|
+
- !ruby/object:Gem::Version
|
79
|
+
version: '0'
|
80
|
+
requirements: []
|
81
|
+
rubyforge_project:
|
82
|
+
rubygems_version: 1.8.21
|
83
|
+
signing_key:
|
84
|
+
specification_version: 3
|
85
|
+
summary: Fluentd plugin to count messages with specified numeric values
|
86
|
+
test_files:
|
87
|
+
- test/helper.rb
|
88
|
+
- test/plugin/test_out_numeric_counter.rb
|