fluent-plugin-numeric-counter 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +21 -0
- data/Gemfile +4 -0
- data/LICENSE +22 -0
- data/README.md +74 -0
- data/Rakefile +11 -0
- data/fluent-plugin-numeric-counter.gemspec +19 -0
- data/lib/fluent/plugin/out_numeric_counter.rb +220 -0
- data/test/helper.rb +28 -0
- data/test/plugin/test_out_numeric_counter.rb +190 -0
- metadata +88 -0
data/.gitignore
ADDED
data/Gemfile
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2012 TAGOMORI Satoshi
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,74 @@
|
|
1
|
+
# fluent-plugin-numeric-counter
|
2
|
+
|
3
|
+
## Component
|
4
|
+
|
5
|
+
### NumericCounterOutput
|
6
|
+
|
7
|
+
Fluentd plugin to count messages, matches for numeric range patterns, and emits its result (like fluent-plugin-datacounter).
|
8
|
+
|
9
|
+
- Counts per min/hour/day
|
10
|
+
- Counts per second (average every min/hour/day)
|
11
|
+
- Percentage of each numeric pattern in total counts of messages
|
12
|
+
|
13
|
+
NumericCounterOutput emits messages contains results data, so you can output these message (with 'numcount' tag by default) to any outputs you want.
|
14
|
+
|
15
|
+
output ex1 (aggregates all inputs): {"pattern1_count":20, "pattern1_rate":0.333, "pattern1_percentage":25.0, "pattern2_count":40, "pattern2_rate":0.666, "pattern2_percentage":50.0, "unmatched_count":20, "unmatched_rate":0.333, "unmatched_percentage":25.0}
|
16
|
+
output ex2 (aggregates per tag): {"test_pattern1_count":10, "test_pattern1_rate":0.333, "test_pattern1_percentage":25.0, "test_pattern2_count":40, "test_pattern2_rate":0.666, "test_pattern2_percentage":50.0, "test_unmatched_count":20, "test_unmatched_rate":0.333, "test_unmatched_percentage":25.0}
|
17
|
+
|
18
|
+
'input\_tag\_remove\_prefix' option available if you want to remove tag prefix from output field names.
|
19
|
+
|
20
|
+
If you want to omit 'unmatched' messages from percentage counting, specify 'outcast_unmatched yes'.
|
21
|
+
|
22
|
+
## Configuration
|
23
|
+
|
24
|
+
### NumericCounterOutput
|
25
|
+
|
26
|
+
Count messages that have attribute 'duration'(response time by microseconds), by several numeric ranges, per minutes.
|
27
|
+
|
28
|
+
<match accesslog.**>
|
29
|
+
type numeric_counter
|
30
|
+
unit minute # or 'count_interval 60s' or '45s', '3m' ... as you want
|
31
|
+
aggregate all # or 'tag'
|
32
|
+
count_key duration
|
33
|
+
|
34
|
+
# patternX: X(1-20)
|
35
|
+
# patternX NAME LOW HIGH #=> patternX matches N like LOW <= N < HIGH
|
36
|
+
|
37
|
+
pattern1 HIGHSPEED 0 10000 # under 10ms
|
38
|
+
pattern2 SEMIHIGHSPEED 10000 100000 # under 100ms
|
39
|
+
pattern3 NORMAL 100000 1000000 # under 1s
|
40
|
+
pattern4 STUPID 1000000 10000000 # under 10s!
|
41
|
+
|
42
|
+
# patternZ (Z is last number of specified patterns)
|
43
|
+
# patternZ NAME LOW #=> patternZ matches N like LOW <= N (upper threshold is unlimited)
|
44
|
+
patternZ MUSTDIE 10000000 # over 10s!
|
45
|
+
</match>
|
46
|
+
|
47
|
+
Size specifier (like 10k, 5M, 103g) available as 1024\*\*1, 1024\*\*2, 1024\*\*3 ...., for example, for bytes of access log.
|
48
|
+
|
49
|
+
<match accesslog.**>
|
50
|
+
type numeric_counter
|
51
|
+
unit hour
|
52
|
+
aggregate tag
|
53
|
+
count_key bytes
|
54
|
+
|
55
|
+
pattern1 SMALL 0 1k
|
56
|
+
pattern2 MIDDLE 1k 1m
|
57
|
+
pattern3 LARGE 1m 10m
|
58
|
+
pattern4 HUGE 10m 1g
|
59
|
+
pattern5 XXXX 1g
|
60
|
+
</match>
|
61
|
+
|
62
|
+
You can try to use negative numbers, and floating point numbers.... (not tested enough).
|
63
|
+
|
64
|
+
## TODO
|
65
|
+
|
66
|
+
* more tests
|
67
|
+
* more documents
|
68
|
+
|
69
|
+
## Copyright
|
70
|
+
|
71
|
+
* Copyright
|
72
|
+
* Copyright (c) 2012- TAGOMORI Satoshi (tagomoris)
|
73
|
+
* License
|
74
|
+
* Apache License, Version 2.0
|
data/Rakefile
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
|
3
|
+
Gem::Specification.new do |gem|
|
4
|
+
gem.name = "fluent-plugin-numeric-counter"
|
5
|
+
gem.version = "0.1.0"
|
6
|
+
gem.authors = ["TAGOMORI Satoshi"]
|
7
|
+
gem.email = ["tagomoris@gmail.com"]
|
8
|
+
gem.description = %q{Counts messages, with specified key and numeric value in specified range}
|
9
|
+
gem.summary = %q{Fluentd plugin to count messages with specified numeric values}
|
10
|
+
gem.homepage = "https://github.com/tagomoris/fluent-plugin-numeric-counter"
|
11
|
+
|
12
|
+
gem.files = `git ls-files`.split($\)
|
13
|
+
gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
14
|
+
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
15
|
+
gem.require_paths = ["lib"]
|
16
|
+
|
17
|
+
gem.add_development_dependency "fluentd"
|
18
|
+
gem.add_runtime_dependency "fluentd"
|
19
|
+
end
|
@@ -0,0 +1,220 @@
|
|
1
|
+
class Fluent::NumericCounterOutput < Fluent::Output
|
2
|
+
Fluent::Plugin.register_output('numeric_counter', self)
|
3
|
+
|
4
|
+
PATTERN_MAX_NUM = 20
|
5
|
+
|
6
|
+
config_param :count_interval, :time, :default => 60
|
7
|
+
config_param :unit, :string, :default => nil
|
8
|
+
config_param :aggregate, :string, :default => 'tag'
|
9
|
+
config_param :tag, :string, :default => 'numcount'
|
10
|
+
config_param :input_tag_remove_prefix, :string, :default => nil
|
11
|
+
config_param :count_key, :string
|
12
|
+
config_param :outcast_unmatched, :bool, :default => false
|
13
|
+
|
14
|
+
# pattern0 reserved as unmatched counts
|
15
|
+
config_param :pattern1, :string # string: NAME LOW HIGH
|
16
|
+
# LOW/HIGH allows size prefix (ex: 10k, 5M, 3500G)
|
17
|
+
(2..PATTERN_MAX_NUM).each do |i|
|
18
|
+
config_param ('pattern' + i.to_s).to_sym, :string, :default => nil
|
19
|
+
end
|
20
|
+
|
21
|
+
attr_accessor :counts, :last_checked
|
22
|
+
|
23
|
+
# for test
|
24
|
+
# attr_accessor :count_interval, :unit, :aggregate, :tag, :input_tag_remove_prefix, :count_key, :outcast_unmatched
|
25
|
+
attr_accessor :patterns
|
26
|
+
|
27
|
+
def parse_num(str)
|
28
|
+
if str.nil?
|
29
|
+
nil
|
30
|
+
elsif str =~ /^[-0-9]+$/
|
31
|
+
str.to_i
|
32
|
+
elsif str =~ /^[-.0-9]+$/
|
33
|
+
str.to_f
|
34
|
+
else
|
35
|
+
Fluent::Config.size_value(str)
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def configure(conf)
|
40
|
+
super
|
41
|
+
|
42
|
+
if @unit
|
43
|
+
@count_interval = case @unit
|
44
|
+
when 'minute' then 60
|
45
|
+
when 'hour' then 3600
|
46
|
+
when 'day' then 86400
|
47
|
+
else
|
48
|
+
raise Fluent::ConfigError, 'unit must be one of minute/hour/day'
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
@aggregate = @aggregate.to_sym
|
53
|
+
raise Fluent::ConfigError, "numeric_counter allows tag/all to aggregate unit" unless [:tag, :all].include?(@aggregate)
|
54
|
+
|
55
|
+
@patterns = [[0, 'unmatched', nil, nil]] # counts-index, name, low, high
|
56
|
+
pattern_names = ['unmatched']
|
57
|
+
|
58
|
+
invalids = conf.keys.select{|k| k =~ /^pattern(\d+)$/ and not (1..PATTERN_MAX_NUM).include?($1.to_i)}
|
59
|
+
if invalids.size > 0
|
60
|
+
$log.warn "invalid number patterns (valid pattern number:1-#{PATTERN_MAX_NUM}):" + invalids.join(",")
|
61
|
+
end
|
62
|
+
(1..PATTERN_MAX_NUM).each do |i|
|
63
|
+
next unless conf["pattern#{i}"]
|
64
|
+
name,low,high = conf["pattern#{i}"].split(/ +/, 3)
|
65
|
+
@patterns.push([i, name, parse_num(low), parse_num(high)])
|
66
|
+
pattern_names.push(name)
|
67
|
+
end
|
68
|
+
pattern_index_list = conf.keys.select{|s| s =~ /^pattern\d$/}.map{|v| (/^pattern(\d)$/.match(v))[1].to_i}
|
69
|
+
unless pattern_index_list.reduce(true){|v,i| v and @patterns[i]}
|
70
|
+
raise Fluent::ConfigError, "jump of pattern index found"
|
71
|
+
end
|
72
|
+
unless @patterns.length == pattern_names.uniq.length
|
73
|
+
raise Fluent::ConfigError, "duplicated pattern names found"
|
74
|
+
end
|
75
|
+
@patterns[1..-1].each do |index, name, low, high|
|
76
|
+
raise Fluent::ConfigError, "numbers of low/high missing" if low.nil?
|
77
|
+
raise Fluent::ConfigError, "unspecified high threshold allowed only in last pattern" if high.nil? and index != @patterns.length - 1
|
78
|
+
end
|
79
|
+
|
80
|
+
if @input_tag_remove_prefix
|
81
|
+
@removed_prefix_string = @input_tag_remove_prefix + '.'
|
82
|
+
@removed_length = @removed_prefix_string.length
|
83
|
+
end
|
84
|
+
|
85
|
+
@counts = count_initialized
|
86
|
+
@mutex = Mutex.new
|
87
|
+
end
|
88
|
+
|
89
|
+
def start
|
90
|
+
super
|
91
|
+
start_watch
|
92
|
+
end
|
93
|
+
|
94
|
+
def shutdown
|
95
|
+
super
|
96
|
+
@watcher.terminate
|
97
|
+
@watcher.join
|
98
|
+
end
|
99
|
+
|
100
|
+
def count_initialized(keys=nil)
|
101
|
+
# counts['tag'][pattern_index_num] = count
|
102
|
+
if @aggregate == :all
|
103
|
+
{'all' => Array.new(@patterns.length){|i| 0}}
|
104
|
+
elsif keys
|
105
|
+
values = Array.new(keys.length){|i|
|
106
|
+
Array.new(@patterns.length){|j| 0 }
|
107
|
+
}
|
108
|
+
Hash[[keys, values].transpose]
|
109
|
+
else
|
110
|
+
{}
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
def countups(tag, counts)
|
115
|
+
if @aggregate == :all
|
116
|
+
tag = 'all'
|
117
|
+
end
|
118
|
+
|
119
|
+
@mutex.synchronize {
|
120
|
+
@counts[tag] ||= [0] * @patterns.length
|
121
|
+
counts.each_with_index do |count, i|
|
122
|
+
@counts[tag][i] += count
|
123
|
+
end
|
124
|
+
}
|
125
|
+
end
|
126
|
+
|
127
|
+
def stripped_tag(tag)
|
128
|
+
return tag unless @input_tag_remove_prefix
|
129
|
+
return tag[@removed_length..-1] if tag.start_with?(@removed_prefix_string) and tag.length > @removed_length
|
130
|
+
return tag[@removed_length..-1] if tag == @input_tag_remove_prefix
|
131
|
+
tag
|
132
|
+
end
|
133
|
+
|
134
|
+
def generate_output(counts, step)
|
135
|
+
output = {}
|
136
|
+
|
137
|
+
if @aggregate == :all
|
138
|
+
# index 0 is unmatched
|
139
|
+
sum = if @outcast_unmatched
|
140
|
+
counts['all'][1..-1].inject(:+)
|
141
|
+
else
|
142
|
+
counts['all'].inject(:+)
|
143
|
+
end
|
144
|
+
counts['all'].each_with_index do |count,i|
|
145
|
+
name = @patterns[i][1]
|
146
|
+
output[name + '_count'] = count
|
147
|
+
output[name + '_rate'] = ((count * 100.0) / (1.00 * step)).floor / 100.0
|
148
|
+
unless i == 0 and @outcast_unmatched
|
149
|
+
output[name + '_percentage'] = count * 100.0 / (1.00 * sum) if sum > 0
|
150
|
+
end
|
151
|
+
end
|
152
|
+
return output
|
153
|
+
end
|
154
|
+
|
155
|
+
counts.keys.each do |tag|
|
156
|
+
t = stripped_tag(tag)
|
157
|
+
sum = if @outcast_unmatched
|
158
|
+
counts[tag][1..-1].inject(:+)
|
159
|
+
else
|
160
|
+
counts[tag].inject(:+)
|
161
|
+
end
|
162
|
+
counts[tag].each_with_index do |count,i|
|
163
|
+
name = @patterns[i][1]
|
164
|
+
output[t + '_' + name + '_count'] = count
|
165
|
+
output[t + '_' + name + '_rate'] = ((count * 100.0) / (1.00 * step)).floor / 100.0
|
166
|
+
unless i == 0 and @outcast_unmatched
|
167
|
+
output[t + '_' + name + '_percentage'] = count * 100.0 / (1.00 * sum) if sum > 0
|
168
|
+
end
|
169
|
+
end
|
170
|
+
end
|
171
|
+
output
|
172
|
+
end
|
173
|
+
|
174
|
+
def flush(step)
|
175
|
+
flushed,@counts = @counts,count_initialized(@counts.keys.dup)
|
176
|
+
generate_output(flushed, step)
|
177
|
+
end
|
178
|
+
|
179
|
+
def flush_emit(step)
|
180
|
+
Fluent::Engine.emit(@tag, Fluent::Engine.now, flush(step))
|
181
|
+
end
|
182
|
+
|
183
|
+
def start_watch
|
184
|
+
@watcher = Thread.new(&method(:watch))
|
185
|
+
end
|
186
|
+
|
187
|
+
def watch
|
188
|
+
@last_checked = Fluent::Engine.now
|
189
|
+
while true
|
190
|
+
sleep 0.5
|
191
|
+
if Fluent::Engine.now - @last_checked >= @count_interval
|
192
|
+
now = Fluent::Engine.now
|
193
|
+
flush_emit(now - @last_checked)
|
194
|
+
@last_checked = now
|
195
|
+
end
|
196
|
+
end
|
197
|
+
end
|
198
|
+
|
199
|
+
def emit(tag, es, chain)
|
200
|
+
c = [0] * @patterns.length
|
201
|
+
|
202
|
+
es.each do |time,record|
|
203
|
+
value = record[@count_key]
|
204
|
+
next if value.nil?
|
205
|
+
|
206
|
+
value = value.to_f
|
207
|
+
matched = false
|
208
|
+
@patterns.each do |index, name, low, high|
|
209
|
+
next if low.nil? or value < low or (not high.nil? and value >= high)
|
210
|
+
c[index] += 1
|
211
|
+
matched = true
|
212
|
+
break
|
213
|
+
end
|
214
|
+
c[0] += 1 unless matched
|
215
|
+
end
|
216
|
+
countups(tag, c)
|
217
|
+
|
218
|
+
chain.next
|
219
|
+
end
|
220
|
+
end
|
data/test/helper.rb
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'bundler'
|
3
|
+
begin
|
4
|
+
Bundler.setup(:default, :development)
|
5
|
+
rescue Bundler::BundlerError => e
|
6
|
+
$stderr.puts e.message
|
7
|
+
$stderr.puts "Run `bundle install` to install missing gems"
|
8
|
+
exit e.status_code
|
9
|
+
end
|
10
|
+
require 'test/unit'
|
11
|
+
|
12
|
+
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
13
|
+
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
14
|
+
require 'fluent/test'
|
15
|
+
unless ENV.has_key?('VERBOSE')
|
16
|
+
nulllogger = Object.new
|
17
|
+
nulllogger.instance_eval {|obj|
|
18
|
+
def method_missing(method, *args)
|
19
|
+
# pass
|
20
|
+
end
|
21
|
+
}
|
22
|
+
$log = nulllogger
|
23
|
+
end
|
24
|
+
|
25
|
+
require 'fluent/plugin/out_numeric_counter'
|
26
|
+
|
27
|
+
class Test::Unit::TestCase
|
28
|
+
end
|
@@ -0,0 +1,190 @@
|
|
1
|
+
require 'helper'
|
2
|
+
|
3
|
+
class NumericCounterOutputTest < Test::Unit::TestCase
|
4
|
+
def setup
|
5
|
+
Fluent::Test.setup
|
6
|
+
end
|
7
|
+
|
8
|
+
CONFIG = %[
|
9
|
+
count_interval 60
|
10
|
+
aggregate tag
|
11
|
+
input_tag_remove_prefix test
|
12
|
+
count_key target
|
13
|
+
pattern1 u100ms 0 100000
|
14
|
+
pattern2 u1s 100000 1000000
|
15
|
+
pattern3 u3s 1000000 3000000
|
16
|
+
]
|
17
|
+
|
18
|
+
def create_driver(conf=CONFIG, tag='test')
|
19
|
+
Fluent::Test::OutputTestDriver.new(Fluent::NumericCounterOutput, tag).configure(conf)
|
20
|
+
end
|
21
|
+
|
22
|
+
def test_parse_num
|
23
|
+
p = create_driver.instance
|
24
|
+
|
25
|
+
assert_equal 1, p.parse_num('1')
|
26
|
+
assert_equal -1, p.parse_num('-1')
|
27
|
+
assert_equal 1.0, p.parse_num('1.0')
|
28
|
+
assert_equal -2.0, p.parse_num('-2.0000')
|
29
|
+
assert_equal 1024, p.parse_num('1k')
|
30
|
+
end
|
31
|
+
|
32
|
+
def test_configure
|
33
|
+
d = create_driver %[
|
34
|
+
count_key field1
|
35
|
+
pattern1 smallnum 0.1 200
|
36
|
+
pattern2 subnum -500 -1
|
37
|
+
]
|
38
|
+
|
39
|
+
assert_equal 60, d.instance.count_interval
|
40
|
+
assert_equal :tag, d.instance.aggregate
|
41
|
+
assert_equal 'numcount', d.instance.tag
|
42
|
+
assert_nil d.instance.input_tag_remove_prefix
|
43
|
+
assert_equal false, d.instance.outcast_unmatched
|
44
|
+
assert_equal [[0, 'unmatched', nil, nil], [1, 'smallnum', 0.1, 200], [2, 'subnum', -500, -1]], d.instance.patterns
|
45
|
+
|
46
|
+
d = create_driver %[
|
47
|
+
count_key key1
|
48
|
+
pattern1 x 0.1 10
|
49
|
+
pattern2 y 10 11
|
50
|
+
pattern3 z 11
|
51
|
+
]
|
52
|
+
assert_equal [[0, 'unmatched', nil, nil], [1, 'x', 0.1, 10], [2, 'y', 10, 11], [3, 'z', 11, nil]], d.instance.patterns
|
53
|
+
end
|
54
|
+
|
55
|
+
def test_countups
|
56
|
+
d = create_driver
|
57
|
+
assert_nil d.instance.counts['test.input']
|
58
|
+
|
59
|
+
d.instance.countups('test.input', [0, 0, 0, 0])
|
60
|
+
assert_equal [0,0,0,0], d.instance.counts['test.input']
|
61
|
+
d.instance.countups('test.input', [1, 1, 1, 0])
|
62
|
+
assert_equal [1,1,1,0], d.instance.counts['test.input']
|
63
|
+
d.instance.countups('test.input', [0, 5, 1, 0])
|
64
|
+
assert_equal [1,6,2,0], d.instance.counts['test.input']
|
65
|
+
end
|
66
|
+
|
67
|
+
def test_generate_output
|
68
|
+
d = create_driver
|
69
|
+
# pattern1 u100ms 0 100000
|
70
|
+
# pattern2 u1s 100000 1000000
|
71
|
+
# pattern3 u3s 1000000 3000000
|
72
|
+
|
73
|
+
r1 = d.instance.generate_output({'test.input' => [60,240,180,120], 'test.input2' => [0,600,0,0]}, 60)
|
74
|
+
assert_equal 60, r1['input_unmatched_count']
|
75
|
+
assert_equal 1.0, r1['input_unmatched_rate']
|
76
|
+
assert_equal 10.0, r1['input_unmatched_percentage']
|
77
|
+
assert_equal 240, r1['input_u100ms_count']
|
78
|
+
assert_equal 4.0, r1['input_u100ms_rate']
|
79
|
+
assert_equal 40.0, r1['input_u100ms_percentage']
|
80
|
+
assert_equal 180, r1['input_u1s_count']
|
81
|
+
assert_equal 3.0, r1['input_u1s_rate']
|
82
|
+
assert_equal 30.0, r1['input_u1s_percentage']
|
83
|
+
assert_equal 120, r1['input_u3s_count']
|
84
|
+
assert_equal 2.0, r1['input_u3s_rate']
|
85
|
+
assert_equal 20.0, r1['input_u3s_percentage']
|
86
|
+
|
87
|
+
assert_equal 0, r1['input2_unmatched_count']
|
88
|
+
assert_equal 0.0, r1['input2_unmatched_rate']
|
89
|
+
assert_equal 0.0, r1['input2_unmatched_percentage']
|
90
|
+
assert_equal 600, r1['input2_u100ms_count']
|
91
|
+
assert_equal 10.0, r1['input2_u100ms_rate']
|
92
|
+
assert_equal 100.0, r1['input2_u100ms_percentage']
|
93
|
+
assert_equal 0, r1['input2_u1s_count']
|
94
|
+
assert_equal 0.0, r1['input2_u1s_rate']
|
95
|
+
assert_equal 0.0, r1['input2_u1s_percentage']
|
96
|
+
assert_equal 0, r1['input2_u3s_count']
|
97
|
+
assert_equal 0.0, r1['input2_u3s_rate']
|
98
|
+
assert_equal 0.0, r1['input2_u3s_percentage']
|
99
|
+
|
100
|
+
d = create_driver %[
|
101
|
+
aggregate all
|
102
|
+
count_key f1
|
103
|
+
pattern1 good 1 2
|
104
|
+
outcast_unmatched yes
|
105
|
+
]
|
106
|
+
r2 = d.instance.generate_output({'all' => [60,240]}, 60)
|
107
|
+
assert_equal 60, r2['unmatched_count']
|
108
|
+
assert_equal 1.0, r2['unmatched_rate']
|
109
|
+
assert_nil r2['unmatched_percentage']
|
110
|
+
assert_equal 240, r2['good_count']
|
111
|
+
assert_equal 4.0, r2['good_rate']
|
112
|
+
assert_equal 100.0, r2['good_percentage']
|
113
|
+
end
|
114
|
+
|
115
|
+
def test_pattern_num
|
116
|
+
assert_equal 20, Fluent::NumericCounterOutput::PATTERN_MAX_NUM
|
117
|
+
|
118
|
+
conf = %[
|
119
|
+
aggregate all
|
120
|
+
count_key field
|
121
|
+
]
|
122
|
+
(1..20).each do |i|
|
123
|
+
conf += "pattern#{i} name#{i} #{i} #{i+1}\n"
|
124
|
+
end
|
125
|
+
d = create_driver(conf, 'test.max')
|
126
|
+
d.run do
|
127
|
+
(0..21).each do |i|
|
128
|
+
d.emit({'field' => i})
|
129
|
+
end
|
130
|
+
end
|
131
|
+
r = d.instance.flush(60)
|
132
|
+
assert_equal 2, r['unmatched_count'] # 0 and 21
|
133
|
+
assert_equal 1, r['name1_count']
|
134
|
+
assert_equal 1, r['name2_count']
|
135
|
+
assert_equal 1, r['name3_count']
|
136
|
+
assert_equal 1, r['name4_count']
|
137
|
+
assert_equal 1, r['name5_count']
|
138
|
+
assert_equal 1, r['name6_count']
|
139
|
+
assert_equal 1, r['name7_count']
|
140
|
+
assert_equal 1, r['name8_count']
|
141
|
+
assert_equal 1, r['name9_count']
|
142
|
+
assert_equal 1, r['name10_count']
|
143
|
+
assert_equal 1, r['name11_count']
|
144
|
+
assert_equal 1, r['name12_count']
|
145
|
+
assert_equal 1, r['name13_count']
|
146
|
+
assert_equal 1, r['name14_count']
|
147
|
+
assert_equal 1, r['name15_count']
|
148
|
+
assert_equal 1, r['name16_count']
|
149
|
+
assert_equal 1, r['name17_count']
|
150
|
+
assert_equal 1, r['name18_count']
|
151
|
+
assert_equal 1, r['name19_count']
|
152
|
+
assert_equal 1, r['name20_count']
|
153
|
+
end
|
154
|
+
|
155
|
+
def test_emit
|
156
|
+
# CONFIG = %[
|
157
|
+
# count_interval 60
|
158
|
+
# aggregate tag
|
159
|
+
# input_tag_remove_prefix test
|
160
|
+
# count_key target
|
161
|
+
# pattern1 u100ms 0 100000
|
162
|
+
# pattern2 u1s 100000 1000000
|
163
|
+
# pattern3 u3s 1000000 3000000
|
164
|
+
# ]
|
165
|
+
d = create_driver(CONFIG, 'test.tag1')
|
166
|
+
d.run do
|
167
|
+
60.times do
|
168
|
+
d.emit({'target' => '50000'})
|
169
|
+
d.emit({'target' => '100000'})
|
170
|
+
d.emit({'target' => '100001'})
|
171
|
+
d.emit({'target' => '0.0'})
|
172
|
+
d.emit({'target' => '-1'})
|
173
|
+
end
|
174
|
+
end
|
175
|
+
r = d.instance.flush(60)
|
176
|
+
|
177
|
+
assert_equal 120, r['tag1_u100ms_count']
|
178
|
+
assert_equal 2.0, r['tag1_u100ms_rate']
|
179
|
+
assert_equal 40.0, r['tag1_u100ms_percentage']
|
180
|
+
assert_equal 120, r['tag1_u1s_count']
|
181
|
+
assert_equal 2.0, r['tag1_u1s_rate']
|
182
|
+
assert_equal 40, r['tag1_u1s_percentage']
|
183
|
+
assert_equal 0, r['tag1_u3s_count']
|
184
|
+
assert_equal 0, r['tag1_u3s_rate']
|
185
|
+
assert_equal 0, r['tag1_u3s_percentage']
|
186
|
+
assert_equal 60, r['tag1_unmatched_count']
|
187
|
+
assert_equal 1.0, r['tag1_unmatched_rate']
|
188
|
+
assert_equal 20, r['tag1_unmatched_percentage']
|
189
|
+
end
|
190
|
+
end
|
metadata
ADDED
@@ -0,0 +1,88 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-numeric-counter
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- TAGOMORI Satoshi
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2012-07-06 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: fluentd
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ! '>='
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '0'
|
22
|
+
type: :development
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ! '>='
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '0'
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: fluentd
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
34
|
+
requirements:
|
35
|
+
- - ! '>='
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: '0'
|
38
|
+
type: :runtime
|
39
|
+
prerelease: false
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ! '>='
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: '0'
|
46
|
+
description: Counts messages, with specified key and numeric value in specified range
|
47
|
+
email:
|
48
|
+
- tagomoris@gmail.com
|
49
|
+
executables: []
|
50
|
+
extensions: []
|
51
|
+
extra_rdoc_files: []
|
52
|
+
files:
|
53
|
+
- .gitignore
|
54
|
+
- Gemfile
|
55
|
+
- LICENSE
|
56
|
+
- README.md
|
57
|
+
- Rakefile
|
58
|
+
- fluent-plugin-numeric-counter.gemspec
|
59
|
+
- lib/fluent/plugin/out_numeric_counter.rb
|
60
|
+
- test/helper.rb
|
61
|
+
- test/plugin/test_out_numeric_counter.rb
|
62
|
+
homepage: https://github.com/tagomoris/fluent-plugin-numeric-counter
|
63
|
+
licenses: []
|
64
|
+
post_install_message:
|
65
|
+
rdoc_options: []
|
66
|
+
require_paths:
|
67
|
+
- lib
|
68
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
69
|
+
none: false
|
70
|
+
requirements:
|
71
|
+
- - ! '>='
|
72
|
+
- !ruby/object:Gem::Version
|
73
|
+
version: '0'
|
74
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
75
|
+
none: false
|
76
|
+
requirements:
|
77
|
+
- - ! '>='
|
78
|
+
- !ruby/object:Gem::Version
|
79
|
+
version: '0'
|
80
|
+
requirements: []
|
81
|
+
rubyforge_project:
|
82
|
+
rubygems_version: 1.8.21
|
83
|
+
signing_key:
|
84
|
+
specification_version: 3
|
85
|
+
summary: Fluentd plugin to count messages with specified numeric values
|
86
|
+
test_files:
|
87
|
+
- test/helper.rb
|
88
|
+
- test/plugin/test_out_numeric_counter.rb
|