fluent-plugin-numeric-counter 0.3.0 → 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +0 -1
- data/Rakefile +1 -1
- data/fluent-plugin-numeric-counter.gemspec +3 -3
- data/lib/fluent/plugin/out_numeric_counter.rb +116 -123
- data/test/helper.rb +0 -1
- data/test/plugin/test_out_numeric_counter.rb +146 -114
- metadata +15 -15
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b9ce759e247b27af5611e9bdd14e040c717b9736
|
4
|
+
data.tar.gz: 342f5c8542780534e8f930f2f6e4c4a749869406
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3992934eebeb1493bc35ccecd593d73329b64cf6b649db583e7d64f05c0c0bafbaa65e4a2f1ae8daad2bc2b2161592e511c4070cd9b7ce842174af19a50fccad
|
7
|
+
data.tar.gz: 0bdde9d5ef5f89e49ea6dec8cd170b55085c828a46ab1d82c792ec5785e951b4013d700e29567b66787cdd7051c45f0b16605e6c2b65ce0b5fe39438f4ac60a6
|
data/.travis.yml
CHANGED
data/Rakefile
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
|
3
3
|
Gem::Specification.new do |gem|
|
4
4
|
gem.name = "fluent-plugin-numeric-counter"
|
5
|
-
gem.version = "0.
|
5
|
+
gem.version = "1.0.0"
|
6
6
|
gem.authors = ["TAGOMORI Satoshi"]
|
7
7
|
gem.email = ["tagomoris@gmail.com"]
|
8
8
|
gem.description = %q{Counts messages, with specified key and numeric value in specified range}
|
@@ -15,8 +15,8 @@ Gem::Specification.new do |gem|
|
|
15
15
|
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
16
16
|
gem.require_paths = ["lib"]
|
17
17
|
|
18
|
+
gem.add_runtime_dependency "fluentd", ">= 0.14.0"
|
18
19
|
gem.add_development_dependency "rake"
|
19
|
-
gem.add_development_dependency "
|
20
|
+
gem.add_development_dependency "timecop"
|
20
21
|
gem.add_development_dependency "test-unit", ">= 3.1.0"
|
21
|
-
gem.add_runtime_dependency "fluentd", "< 0.14.0"
|
22
22
|
end
|
@@ -1,59 +1,65 @@
|
|
1
|
-
|
1
|
+
require 'fluent/plugin/output'
|
2
|
+
require 'pathname'
|
3
|
+
|
4
|
+
class Fluent::Plugin::NumericCounterOutput < Fluent::Plugin::Output
|
2
5
|
Fluent::Plugin.register_output('numeric_counter', self)
|
3
6
|
|
4
|
-
|
5
|
-
super
|
6
|
-
require 'pathname'
|
7
|
-
end
|
7
|
+
helpers :event_emitter, :storage, :timer
|
8
8
|
|
9
|
+
DEFAULT_STORAGE_TYPE = 'local'
|
9
10
|
PATTERN_MAX_NUM = 20
|
10
11
|
|
11
|
-
config_param :count_interval, :time, :
|
12
|
-
:
|
13
|
-
config_param :unit, :
|
14
|
-
:
|
12
|
+
config_param :count_interval, :time, default: 60,
|
13
|
+
desc: 'The interval time to count in seconds.'
|
14
|
+
config_param :unit, :enum, list: [:minute, :hour, :day], default: nil,
|
15
|
+
desc: <<-DESC
|
15
16
|
The interval time to monitor specified an unit (either of minute, hour, or day).
|
16
17
|
Use either of count_interval or unit.
|
17
18
|
DESC
|
18
|
-
config_param :output_per_tag, :bool, :
|
19
|
-
:
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
config_param :
|
24
|
-
:
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
:desc => <<-DESC
|
29
|
-
The prefix string which will be added to the input tag.
|
30
|
-
output_per_tag yes must be specified together.
|
31
|
-
DESC
|
32
|
-
config_param :input_tag_remove_prefix, :string, :default => nil,
|
33
|
-
:desc => 'The prefix string which will be removed from the input tag.'
|
19
|
+
config_param :output_per_tag, :bool, default: false,
|
20
|
+
desc: 'Produce counter result per input tags.'
|
21
|
+
|
22
|
+
config_param :aggregate, :enum, list: [:tag, :all], default: :tag,
|
23
|
+
desc: 'Calculate in each input tag separetely, or all records in a mass.'
|
24
|
+
config_param :tag, :string, default: 'numcount',
|
25
|
+
desc: 'The output tag.'
|
26
|
+
|
27
|
+
config_param :input_tag_remove_prefix, :string, default: nil,
|
28
|
+
desc: 'The prefix string which will be removed from the input tag.'
|
34
29
|
config_param :count_key, :string,
|
35
|
-
:
|
36
|
-
config_param :outcast_unmatched, :bool, :
|
37
|
-
:
|
30
|
+
desc: 'The key to count in the event record.'
|
31
|
+
config_param :outcast_unmatched, :bool, default: false,
|
32
|
+
desc: <<-DESC
|
38
33
|
Specify yes if you do not want to include 'unmatched' counts into percentage.
|
39
34
|
DESC
|
40
|
-
config_param :output_messages, :bool, :
|
41
|
-
:
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
35
|
+
config_param :output_messages, :bool, default: false,
|
36
|
+
desc: 'Specify yes if you want to get tested messages.'
|
37
|
+
|
38
|
+
config_param :store_file, :string, default: nil,
|
39
|
+
obsoleted: 'Use store_storage parameter instead.',
|
40
|
+
desc: 'Store internal data into a file of the given path on shutdown, and load on starting.'
|
41
|
+
config_param :store_storage, :bool, default: false,
|
42
|
+
desc: 'Store internal data into a storage on shutdown, and load on starting.'
|
46
43
|
|
47
44
|
# pattern0 reserved as unmatched counts
|
48
45
|
config_param :pattern1, :string,
|
49
|
-
:
|
46
|
+
desc: <<-DESC
|
50
47
|
string: NAME LOW HIGH
|
51
48
|
LOW/HIGH allows size prefix (ex: 10k, 5M, 3500G)
|
52
49
|
Note that pattern0 reserved as unmatched counts.
|
53
50
|
DESC
|
54
51
|
(2..PATTERN_MAX_NUM).each do |i|
|
55
|
-
config_param ('pattern' + i.to_s).to_sym, :string, :
|
56
|
-
:
|
52
|
+
config_param ('pattern' + i.to_s).to_sym, :string, default: nil,
|
53
|
+
desc: 'string: NAME LOW HIGH'
|
54
|
+
end
|
55
|
+
|
56
|
+
config_param :tag_prefix, :string, default: nil,
|
57
|
+
desc: 'The prefix string to be added to input tags. Use with "output_per_tag yes".',
|
58
|
+
deprecated: 'Use @label routing instead.'
|
59
|
+
|
60
|
+
config_section :storage do
|
61
|
+
config_set_default :usage, 'resume'
|
62
|
+
config_set_default :@type, DEFAULT_STORAGE_TYPE
|
57
63
|
end
|
58
64
|
|
59
65
|
attr_accessor :counts
|
@@ -62,16 +68,6 @@ DESC
|
|
62
68
|
attr_accessor :saved_at
|
63
69
|
attr_accessor :patterns
|
64
70
|
|
65
|
-
# Define `log` method for v0.10.42 or earlier
|
66
|
-
unless method_defined?(:log)
|
67
|
-
define_method("log") { $log }
|
68
|
-
end
|
69
|
-
|
70
|
-
# Define `router` method of v0.12 to support v0.10.57 or earlier
|
71
|
-
unless method_defined?(:router)
|
72
|
-
define_method("router") { Fluent::Engine }
|
73
|
-
end
|
74
|
-
|
75
71
|
def parse_num(str)
|
76
72
|
if str.nil?
|
77
73
|
nil
|
@@ -85,27 +81,26 @@ DESC
|
|
85
81
|
end
|
86
82
|
|
87
83
|
def configure(conf)
|
84
|
+
label_routing_specified = conf.has_key?('@label')
|
85
|
+
|
88
86
|
super
|
89
87
|
|
90
88
|
if @unit
|
91
89
|
@count_interval = case @unit
|
92
|
-
when
|
93
|
-
when
|
94
|
-
when
|
90
|
+
when :minute then 60
|
91
|
+
when :hour then 3600
|
92
|
+
when :day then 86400
|
95
93
|
else
|
96
|
-
raise
|
94
|
+
raise "unknown unit:#{@unit}"
|
97
95
|
end
|
98
96
|
end
|
99
97
|
|
100
|
-
@aggregate = @aggregate.to_sym
|
101
|
-
raise Fluent::ConfigError, "numeric_counter allows tag/all to aggregate unit" unless [:tag, :all].include?(@aggregate)
|
102
|
-
|
103
98
|
@patterns = [[0, 'unmatched', nil, nil]] # counts-index, name, low, high
|
104
99
|
pattern_names = ['unmatched']
|
105
100
|
|
106
101
|
invalids = conf.keys.select{|k| k =~ /^pattern(\d+)$/ and not (1..PATTERN_MAX_NUM).include?($1.to_i)}
|
107
102
|
if invalids.size > 0
|
108
|
-
log.warn "invalid number patterns (valid pattern number:1-#{PATTERN_MAX_NUM}):"
|
103
|
+
log.warn "invalid number patterns (valid pattern number:1-#{PATTERN_MAX_NUM}):", invalids: invalids
|
109
104
|
end
|
110
105
|
(1..PATTERN_MAX_NUM).each do |i|
|
111
106
|
next unless conf["pattern#{i}"]
|
@@ -124,10 +119,14 @@ DESC
|
|
124
119
|
raise Fluent::ConfigError, "numbers of low/high missing" if low.nil?
|
125
120
|
raise Fluent::ConfigError, "unspecified high threshold allowed only in last pattern" if high.nil? and index != @patterns.length - 1
|
126
121
|
end
|
127
|
-
|
128
|
-
if @output_per_tag
|
129
|
-
raise Fluent::ConfigError, "
|
122
|
+
|
123
|
+
if @output_per_tag && (!label_routing_specified && !@tag_prefix)
|
124
|
+
raise Fluent::ConfigError, "specify @label to route output events into other <label> sections."
|
125
|
+
end
|
126
|
+
if @output_per_tag && @tag_prefix
|
130
127
|
@tag_prefix_string = @tag_prefix + '.'
|
128
|
+
else
|
129
|
+
@tag_prefix_string = nil
|
131
130
|
end
|
132
131
|
|
133
132
|
if @input_tag_remove_prefix
|
@@ -135,28 +134,39 @@ DESC
|
|
135
134
|
@removed_length = @removed_prefix_string.length
|
136
135
|
end
|
137
136
|
|
138
|
-
if @
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
137
|
+
if @store_storage
|
138
|
+
@storage = storage_create(usage: 'resume')
|
139
|
+
end
|
140
|
+
|
141
|
+
if system_config.workers > 1
|
142
|
+
log.warn "Fluentd is now working with multi process workers, and numeric_counter plugin will produce counter results in each separeted processes."
|
143
143
|
end
|
144
144
|
|
145
145
|
@counts = count_initialized
|
146
146
|
@mutex = Mutex.new
|
147
147
|
end
|
148
148
|
|
149
|
+
def multi_workers_ready?
|
150
|
+
true
|
151
|
+
end
|
152
|
+
|
149
153
|
def start
|
150
154
|
super
|
151
|
-
|
152
|
-
|
155
|
+
|
156
|
+
load_status(@count_interval) if @store_storage
|
157
|
+
|
158
|
+
@last_checked = Fluent::Engine.now
|
159
|
+
|
160
|
+
timer_execute(:out_numeric_counter_timer, @count_interval) do
|
161
|
+
now = Fluent::Engine.now
|
162
|
+
flush_emit(now - @last_checked)
|
163
|
+
@last_checked = now
|
164
|
+
end
|
153
165
|
end
|
154
166
|
|
155
167
|
def shutdown
|
168
|
+
save_status() if @store_storage
|
156
169
|
super
|
157
|
-
@watcher.terminate
|
158
|
-
@watcher.join
|
159
|
-
save_status(@store_file) if @store_file
|
160
170
|
end
|
161
171
|
|
162
172
|
def count_initialized(keys=nil)
|
@@ -217,7 +227,7 @@ DESC
|
|
217
227
|
end
|
218
228
|
end
|
219
229
|
|
220
|
-
output
|
230
|
+
output
|
221
231
|
end
|
222
232
|
|
223
233
|
def generate_output(counts, step)
|
@@ -258,7 +268,11 @@ DESC
|
|
258
268
|
if @output_per_tag
|
259
269
|
time = Fluent::Engine.now
|
260
270
|
flush_per_tags(step).each do |tag,message|
|
261
|
-
|
271
|
+
if @tag_prefix_string
|
272
|
+
router.emit(@tag_prefix_string + tag, time, message)
|
273
|
+
else
|
274
|
+
router.emit(tag, time, message)
|
275
|
+
end
|
262
276
|
end
|
263
277
|
else
|
264
278
|
message = flush(step)
|
@@ -268,23 +282,7 @@ DESC
|
|
268
282
|
end
|
269
283
|
end
|
270
284
|
|
271
|
-
def
|
272
|
-
@watcher = Thread.new(&method(:watch))
|
273
|
-
end
|
274
|
-
|
275
|
-
def watch
|
276
|
-
@last_checked ||= Fluent::Engine.now
|
277
|
-
while true
|
278
|
-
sleep 0.5
|
279
|
-
if Fluent::Engine.now - @last_checked >= @count_interval
|
280
|
-
now = Fluent::Engine.now
|
281
|
-
flush_emit(now - @last_checked)
|
282
|
-
@last_checked = now
|
283
|
-
end
|
284
|
-
end
|
285
|
-
end
|
286
|
-
|
287
|
-
def emit(tag, es, chain)
|
285
|
+
def process(tag, es)
|
288
286
|
c = [0] * @patterns.length
|
289
287
|
|
290
288
|
es.each do |time,record|
|
@@ -302,62 +300,57 @@ DESC
|
|
302
300
|
c[0] += 1 unless matched
|
303
301
|
end
|
304
302
|
countups(tag, c)
|
305
|
-
|
306
|
-
chain.next
|
307
303
|
end
|
308
304
|
|
309
|
-
# Store internal status into a
|
305
|
+
# Store internal status into a storage
|
310
306
|
#
|
311
|
-
|
312
|
-
def save_status(file_path)
|
307
|
+
def save_status()
|
313
308
|
begin
|
314
|
-
|
315
|
-
|
316
|
-
|
317
|
-
|
318
|
-
|
319
|
-
|
320
|
-
|
321
|
-
|
322
|
-
|
323
|
-
|
324
|
-
|
325
|
-
end
|
309
|
+
@saved_at = Fluent::Engine.now
|
310
|
+
@saved_duration = @saved_at - @last_checked
|
311
|
+
value = {
|
312
|
+
"counts" => @counts,
|
313
|
+
"saved_at" => @saved_at,
|
314
|
+
"saved_duration" => @saved_duration,
|
315
|
+
"aggregate" => @aggregate.to_s,
|
316
|
+
"count_key" => @count_key,
|
317
|
+
"patterns" => @patterns,
|
318
|
+
}
|
319
|
+
@storage.put(:stored_value, value)
|
326
320
|
rescue => e
|
327
|
-
log.warn "
|
321
|
+
log.warn "Can't write store_storage", error: e
|
328
322
|
end
|
329
323
|
end
|
330
324
|
|
331
|
-
# Load internal status from a
|
325
|
+
# Load internal status from a storage
|
332
326
|
#
|
333
|
-
# @param [String] file_path
|
334
327
|
# @param [Interger] count_interval
|
335
|
-
def load_status(
|
336
|
-
|
328
|
+
def load_status(count_interval)
|
329
|
+
stored = @storage.get(:stored_value)
|
330
|
+
return unless stored
|
337
331
|
|
338
332
|
begin
|
339
|
-
|
340
|
-
stored
|
341
|
-
|
342
|
-
stored[:count_key] == @count_key and
|
343
|
-
stored[:patterns] == @patterns
|
333
|
+
if stored["aggregate"] == @aggregate.to_s and
|
334
|
+
stored["count_key"] == @count_key and
|
335
|
+
stored["patterns"] == @patterns
|
344
336
|
|
345
|
-
|
346
|
-
|
347
|
-
@
|
348
|
-
@
|
337
|
+
if Fluent::Engine.now <= stored["saved_at"] + count_interval
|
338
|
+
@mutex.synchronize {
|
339
|
+
@counts = stored["counts"]
|
340
|
+
@saved_at = stored["saved_at"]
|
341
|
+
@saved_duration = stored["saved_duration"]
|
349
342
|
|
350
343
|
# skip the saved duration to continue counting
|
351
344
|
@last_checked = Fluent::Engine.now - @saved_duration
|
352
|
-
|
353
|
-
log.warn "out_datacounter: stored data is outdated. ignore stored data"
|
354
|
-
end
|
345
|
+
}
|
355
346
|
else
|
356
|
-
log.warn "
|
347
|
+
log.warn "stored data is outdated. ignore stored data"
|
357
348
|
end
|
349
|
+
else
|
350
|
+
log.warn "configuration param was changed. ignore stored data"
|
358
351
|
end
|
359
352
|
rescue => e
|
360
|
-
log.warn "
|
353
|
+
log.warn "Can't load store_storage", error: e
|
361
354
|
end
|
362
355
|
end
|
363
356
|
|
data/test/helper.rb
CHANGED
@@ -1,10 +1,20 @@
|
|
1
1
|
require 'helper'
|
2
|
+
require 'fluent/test/helpers'
|
3
|
+
require 'fluent/test/driver/output'
|
4
|
+
require 'timecop'
|
5
|
+
require 'fileutils'
|
2
6
|
|
3
7
|
class NumericCounterOutputTest < Test::Unit::TestCase
|
8
|
+
include Fluent::Test::Helpers
|
9
|
+
|
4
10
|
def setup
|
5
11
|
Fluent::Test.setup
|
6
12
|
end
|
7
13
|
|
14
|
+
def teardown
|
15
|
+
Timecop.return
|
16
|
+
end
|
17
|
+
|
8
18
|
CONFIG = %[
|
9
19
|
count_interval 60
|
10
20
|
aggregate tag
|
@@ -28,17 +38,17 @@ class NumericCounterOutputTest < Test::Unit::TestCase
|
|
28
38
|
output_messages true
|
29
39
|
]
|
30
40
|
|
31
|
-
def create_driver(conf=CONFIG
|
32
|
-
Fluent::Test::
|
41
|
+
def create_driver(conf=CONFIG)
|
42
|
+
Fluent::Test::Driver::Output.new(Fluent::Plugin::NumericCounterOutput).configure(conf)
|
33
43
|
end
|
34
|
-
|
44
|
+
|
35
45
|
def test_parse_num
|
36
46
|
p = create_driver.instance
|
37
47
|
|
38
48
|
assert_equal 1, p.parse_num('1')
|
39
|
-
assert_equal -1, p.parse_num('-1')
|
49
|
+
assert_equal (-1), p.parse_num('-1')
|
40
50
|
assert_equal 1.0, p.parse_num('1.0')
|
41
|
-
assert_equal -2.0, p.parse_num('-2.0000')
|
51
|
+
assert_equal (-2.0), p.parse_num('-2.0000')
|
42
52
|
assert_equal 1024, p.parse_num('1k')
|
43
53
|
end
|
44
54
|
|
@@ -247,7 +257,7 @@ class NumericCounterOutputTest < Test::Unit::TestCase
|
|
247
257
|
end
|
248
258
|
|
249
259
|
def test_pattern_num
|
250
|
-
assert_equal 20, Fluent::NumericCounterOutput::PATTERN_MAX_NUM
|
260
|
+
assert_equal 20, Fluent::Plugin::NumericCounterOutput::PATTERN_MAX_NUM
|
251
261
|
|
252
262
|
conf = %[
|
253
263
|
aggregate all
|
@@ -256,10 +266,10 @@ class NumericCounterOutputTest < Test::Unit::TestCase
|
|
256
266
|
(1..20).each do |i|
|
257
267
|
conf += "pattern#{i} name#{i} #{i} #{i+1}\n"
|
258
268
|
end
|
259
|
-
d = create_driver(conf
|
260
|
-
d.run do
|
269
|
+
d = create_driver(conf)
|
270
|
+
d.run(default_tag: 'test.max') do
|
261
271
|
(0..21).each do |i|
|
262
|
-
d.
|
272
|
+
d.feed({'field' => i})
|
263
273
|
end
|
264
274
|
end
|
265
275
|
r = d.instance.flush(60)
|
@@ -296,14 +306,14 @@ class NumericCounterOutputTest < Test::Unit::TestCase
|
|
296
306
|
# pattern2 u1s 100000 1000000
|
297
307
|
# pattern3 u3s 1000000 3000000
|
298
308
|
# ]
|
299
|
-
d = create_driver(CONFIG
|
300
|
-
d.run do
|
309
|
+
d = create_driver(CONFIG)
|
310
|
+
d.run(default_tag: 'test.tag1') do
|
301
311
|
60.times do
|
302
|
-
d.
|
303
|
-
d.
|
304
|
-
d.
|
305
|
-
d.
|
306
|
-
d.
|
312
|
+
d.feed({'target' => '50000'})
|
313
|
+
d.feed({'target' => '100000'})
|
314
|
+
d.feed({'target' => '100001'})
|
315
|
+
d.feed({'target' => '0.0'})
|
316
|
+
d.feed({'target' => '-1'})
|
307
317
|
end
|
308
318
|
end
|
309
319
|
r = d.instance.flush(60)
|
@@ -321,20 +331,20 @@ class NumericCounterOutputTest < Test::Unit::TestCase
|
|
321
331
|
assert_equal 1.0, r['tag1_unmatched_rate']
|
322
332
|
assert_equal 20, r['tag1_unmatched_percentage']
|
323
333
|
|
324
|
-
d = create_driver(CONFIG
|
325
|
-
d.run do
|
334
|
+
d = create_driver(CONFIG)
|
335
|
+
d.run(default_tag: 'test.tag1') do
|
326
336
|
60.times do
|
327
|
-
d.
|
328
|
-
d.
|
329
|
-
d.
|
330
|
-
d.
|
331
|
-
d.
|
337
|
+
d.feed({'target' => '50000'})
|
338
|
+
d.feed({'target' => '100000'})
|
339
|
+
d.feed({'target' => '100001'})
|
340
|
+
d.feed({'target' => '0.0'})
|
341
|
+
d.feed({'target' => '-1'})
|
332
342
|
end
|
343
|
+
d.instance.flush_emit(60)
|
333
344
|
end
|
334
|
-
d.
|
335
|
-
|
336
|
-
|
337
|
-
data = emits[0]
|
345
|
+
events = d.events
|
346
|
+
assert_equal 1, events.length
|
347
|
+
data = events[0]
|
338
348
|
assert_equal 'numcount', data[0] # tag
|
339
349
|
r = data[2] # message
|
340
350
|
assert_equal 120, r['tag1_u100ms_count']
|
@@ -352,17 +362,18 @@ class NumericCounterOutputTest < Test::Unit::TestCase
|
|
352
362
|
end
|
353
363
|
|
354
364
|
def test_emit_output_per_tag
|
355
|
-
d = create_driver(CONFIG_OUTPUT_PER_TAG
|
356
|
-
|
365
|
+
d = create_driver(CONFIG_OUTPUT_PER_TAG)
|
366
|
+
r = {}
|
367
|
+
d.run(default_tag: 'test.tag1') do
|
357
368
|
60.times do
|
358
|
-
d.
|
359
|
-
d.
|
360
|
-
d.
|
361
|
-
d.
|
362
|
-
d.
|
369
|
+
d.feed({'target' => '50000'})
|
370
|
+
d.feed({'target' => '100000'})
|
371
|
+
d.feed({'target' => '100001'})
|
372
|
+
d.feed({'target' => '0.0'})
|
373
|
+
d.feed({'target' => '-1'})
|
363
374
|
end
|
375
|
+
r = d.instance.flush_per_tags(60)
|
364
376
|
end
|
365
|
-
r = d.instance.flush_per_tags(60)
|
366
377
|
assert_equal 1, r.keys.size
|
367
378
|
r1 = r['tag1']
|
368
379
|
assert_equal 120, r1['u100ms_count']
|
@@ -379,20 +390,20 @@ class NumericCounterOutputTest < Test::Unit::TestCase
|
|
379
390
|
assert_equal 20, r1['unmatched_percentage']
|
380
391
|
assert_equal 300, r1['messages']
|
381
392
|
|
382
|
-
d = create_driver(CONFIG_OUTPUT_PER_TAG
|
383
|
-
d.run do
|
393
|
+
d = create_driver(CONFIG_OUTPUT_PER_TAG)
|
394
|
+
d.run(default_tag: 'test.tag1') do
|
384
395
|
60.times do
|
385
|
-
d.
|
386
|
-
d.
|
387
|
-
d.
|
388
|
-
d.
|
389
|
-
d.
|
396
|
+
d.feed({'target' => '50000'})
|
397
|
+
d.feed({'target' => '100000'})
|
398
|
+
d.feed({'target' => '100001'})
|
399
|
+
d.feed({'target' => '0.0'})
|
400
|
+
d.feed({'target' => '-1'})
|
390
401
|
end
|
402
|
+
d.instance.flush_emit(60)
|
391
403
|
end
|
392
|
-
d.
|
393
|
-
|
394
|
-
|
395
|
-
data = emits[0]
|
404
|
+
events = d.events
|
405
|
+
assert_equal 1, events.length
|
406
|
+
data = events[0]
|
396
407
|
assert_equal 'n.tag1', data[0] # tag
|
397
408
|
r = data[2] # message
|
398
409
|
assert_equal 120, r['u100ms_count']
|
@@ -418,7 +429,7 @@ class NumericCounterOutputTest < Test::Unit::TestCase
|
|
418
429
|
['count', 'rate'].map{|a| p + '_' + a}
|
419
430
|
}.flatten
|
420
431
|
|
421
|
-
d = create_driver(CONFIG
|
432
|
+
d = create_driver(CONFIG)
|
422
433
|
# CONFIG = %[
|
423
434
|
# count_interval 60
|
424
435
|
# aggregate tag
|
@@ -428,28 +439,28 @@ class NumericCounterOutputTest < Test::Unit::TestCase
|
|
428
439
|
# pattern2 u1s 100000 1000000
|
429
440
|
# pattern3 u3s 1000000 3000000
|
430
441
|
# ]
|
431
|
-
d.run do
|
442
|
+
d.run(default_tag: 'test.tag1') do
|
432
443
|
60.times do
|
433
|
-
d.
|
434
|
-
d.
|
435
|
-
d.
|
436
|
-
d.
|
437
|
-
d.
|
444
|
+
d.feed({'target' => '50000'})
|
445
|
+
d.feed({'target' => '100000'})
|
446
|
+
d.feed({'target' => '100001'})
|
447
|
+
d.feed({'target' => '0.0'})
|
448
|
+
d.feed({'target' => '-1'})
|
438
449
|
end
|
450
|
+
d.instance.flush_emit(60)
|
451
|
+
assert_equal 1, d.events.size
|
452
|
+
r1 = d.events[0][2]
|
453
|
+
assert_equal fields, r1.keys
|
454
|
+
|
455
|
+
d.instance.flush_emit(60)
|
456
|
+
assert_equal 2, d.events.size # +1
|
457
|
+
r2 = d.events[1][2]
|
458
|
+
assert_equal fields_without_percentage, r2.keys
|
459
|
+
assert_equal [0]*8, r2.values
|
460
|
+
|
461
|
+
d.instance.flush_emit(60)
|
462
|
+
assert_equal 2, d.events.size # +0
|
439
463
|
end
|
440
|
-
d.instance.flush_emit(60)
|
441
|
-
assert_equal 1, d.emits.size
|
442
|
-
r1 = d.emits[0][2]
|
443
|
-
assert_equal fields, r1.keys
|
444
|
-
|
445
|
-
d.instance.flush_emit(60)
|
446
|
-
assert_equal 2, d.emits.size # +1
|
447
|
-
r2 = d.emits[1][2]
|
448
|
-
assert_equal fields_without_percentage, r2.keys
|
449
|
-
assert_equal [0]*8, r2.values
|
450
|
-
|
451
|
-
d.instance.flush_emit(60)
|
452
|
-
assert_equal 2, d.emits.size # +0
|
453
464
|
end
|
454
465
|
|
455
466
|
def test_zero_tags_per_tag
|
@@ -460,7 +471,7 @@ class NumericCounterOutputTest < Test::Unit::TestCase
|
|
460
471
|
['count', 'rate'].map{|a| p + '_' + a}
|
461
472
|
}.flatten + ['messages']).sort
|
462
473
|
|
463
|
-
d = create_driver(CONFIG_OUTPUT_PER_TAG
|
474
|
+
d = create_driver(CONFIG_OUTPUT_PER_TAG)
|
464
475
|
# CONFIG_OUTPUT_PER_TAG = %[
|
465
476
|
# count_interval 60
|
466
477
|
# aggregate tag
|
@@ -473,44 +484,63 @@ class NumericCounterOutputTest < Test::Unit::TestCase
|
|
473
484
|
# pattern3 u3s 1000000 3000000
|
474
485
|
# output_messages true
|
475
486
|
# ]
|
476
|
-
d.run do
|
487
|
+
d.run(default_tag: 'test.tag1') do
|
477
488
|
60.times do
|
478
|
-
d.
|
479
|
-
d.
|
480
|
-
d.
|
481
|
-
d.
|
482
|
-
d.
|
489
|
+
d.feed({'target' => '50000'})
|
490
|
+
d.feed({'target' => '100000'})
|
491
|
+
d.feed({'target' => '100001'})
|
492
|
+
d.feed({'target' => '0.0'})
|
493
|
+
d.feed({'target' => '-1'})
|
483
494
|
end
|
495
|
+
d.instance.flush_emit(60)
|
496
|
+
assert_equal 1, d.events.size
|
497
|
+
r1 = d.events[0][2]
|
498
|
+
assert_equal fields, r1.keys.sort
|
499
|
+
|
500
|
+
d.instance.flush_emit(60)
|
501
|
+
assert_equal 2, d.events.size # +1
|
502
|
+
r2 = d.events[1][2]
|
503
|
+
assert_equal fields_without_percentage, r2.keys.sort
|
504
|
+
assert_equal [0]*9, r2.values # (_count, _rate) x4 + messages
|
505
|
+
|
506
|
+
d.instance.flush_emit(60)
|
507
|
+
assert_equal 2, d.events.size # +0
|
484
508
|
end
|
485
|
-
d.instance.flush_emit(60)
|
486
|
-
assert_equal 1, d.emits.size
|
487
|
-
r1 = d.emits[0][2]
|
488
|
-
assert_equal fields, r1.keys.sort
|
489
|
-
|
490
|
-
d.instance.flush_emit(60)
|
491
|
-
assert_equal 2, d.emits.size # +1
|
492
|
-
r2 = d.emits[1][2]
|
493
|
-
assert_equal fields_without_percentage, r2.keys.sort
|
494
|
-
assert_equal [0]*9, r2.values # (_count, _rate) x4 + messages
|
495
|
-
|
496
|
-
d.instance.flush_emit(60)
|
497
|
-
assert_equal 2, d.emits.size # +0
|
498
509
|
end
|
499
510
|
|
500
|
-
def
|
511
|
+
def test_store_storage
|
501
512
|
dir = "test/tmp"
|
502
|
-
Dir.mkdir dir unless Dir.exist? dir
|
503
513
|
file = "#{dir}/test.dat"
|
504
|
-
|
505
|
-
|
514
|
+
FileUtils.rm_rf(file)
|
515
|
+
FileUtils.mkdir_p(dir)
|
516
|
+
|
517
|
+
config = {
|
518
|
+
"count_interval" => 60,
|
519
|
+
"aggregate" => "tag",
|
520
|
+
"input_tag_remove_prefix" => "test",
|
521
|
+
"count_key" => " target",
|
522
|
+
"pattern1" => "u100ms 0 100000",
|
523
|
+
"pattern2" => "u1s 100000 1000000",
|
524
|
+
"pattern3" => "u3s 1000000 3000000",
|
525
|
+
"store_storage" => true,
|
526
|
+
}
|
527
|
+
conf = config_element('ROOT', '', config, [
|
528
|
+
config_element(
|
529
|
+
'storage', '',
|
530
|
+
{'@type' => 'local',
|
531
|
+
'@id' => 'test-01',
|
532
|
+
'path' => "#{file}",
|
533
|
+
'persistent' => true,
|
534
|
+
})
|
535
|
+
])
|
506
536
|
# test store
|
507
|
-
d = create_driver(
|
508
|
-
|
537
|
+
d = create_driver(conf)
|
538
|
+
time = Fluent::Engine.now
|
539
|
+
d.run(default_tag: 'test') do
|
509
540
|
d.instance.flush_emit(60)
|
510
|
-
d.
|
511
|
-
d.
|
512
|
-
d.
|
513
|
-
d.instance.shutdown
|
541
|
+
d.feed(time, {'target' => 1})
|
542
|
+
d.feed(time, {'target' => 1})
|
543
|
+
d.feed(time, {'target' => 1})
|
514
544
|
end
|
515
545
|
stored_counts = d.instance.counts
|
516
546
|
stored_saved_at = d.instance.saved_at
|
@@ -518,40 +548,42 @@ class NumericCounterOutputTest < Test::Unit::TestCase
|
|
518
548
|
assert File.exist? file
|
519
549
|
|
520
550
|
# test load
|
521
|
-
d = create_driver(
|
522
|
-
|
551
|
+
d = create_driver(conf)
|
552
|
+
loaded_counts = 0
|
553
|
+
loaded_saved_at = 0
|
554
|
+
loaded_saved_duration = 0
|
555
|
+
d.run(default_tag: 'test') do
|
523
556
|
loaded_counts = d.instance.counts
|
524
557
|
loaded_saved_at = d.instance.saved_at
|
525
558
|
loaded_saved_duration = d.instance.saved_duration
|
526
|
-
assert_equal stored_counts, loaded_counts
|
527
|
-
assert_equal stored_saved_at, loaded_saved_at
|
528
|
-
assert_equal stored_saved_duration, loaded_saved_duration
|
529
559
|
end
|
560
|
+
assert_equal stored_counts, loaded_counts
|
561
|
+
assert_equal stored_saved_at, loaded_saved_at
|
562
|
+
assert_equal stored_saved_duration, loaded_saved_duration
|
530
563
|
|
531
564
|
# test not to load if config is changed
|
532
|
-
d = create_driver(
|
533
|
-
d.run do
|
565
|
+
d = create_driver(conf.merge("count_key" => "foobar", "store_storage" => true))
|
566
|
+
d.run(default_tag: 'test') do
|
534
567
|
loaded_counts = d.instance.counts
|
535
568
|
loaded_saved_at = d.instance.saved_at
|
536
569
|
loaded_saved_duration = d.instance.saved_duration
|
537
|
-
assert_equal({}, loaded_counts)
|
538
|
-
assert_equal(nil, loaded_saved_at)
|
539
|
-
assert_equal(nil, loaded_saved_duration)
|
540
570
|
end
|
571
|
+
assert_equal({}, loaded_counts)
|
572
|
+
assert_equal(nil, loaded_saved_at)
|
573
|
+
assert_equal(nil, loaded_saved_duration)
|
541
574
|
|
542
575
|
# test not to load if stored data is outdated.
|
543
|
-
|
544
|
-
d = create_driver(
|
545
|
-
d.run do
|
576
|
+
Timecop.freeze(Time.now + 61) # jump more than count_interval
|
577
|
+
d = create_driver(conf.merge("store_storage" => true))
|
578
|
+
d.run(default_tag: 'test') do
|
546
579
|
loaded_counts = d.instance.counts
|
547
580
|
loaded_saved_at = d.instance.saved_at
|
548
581
|
loaded_saved_duration = d.instance.saved_duration
|
549
|
-
assert_equal({}, loaded_counts)
|
550
|
-
assert_equal(nil, loaded_saved_at)
|
551
|
-
assert_equal(nil, loaded_saved_duration)
|
552
582
|
end
|
553
|
-
|
583
|
+
assert_equal({}, loaded_counts)
|
584
|
+
assert_equal(nil, loaded_saved_at)
|
585
|
+
assert_equal(nil, loaded_saved_duration)
|
554
586
|
|
555
|
-
|
587
|
+
FileUtils.rm_rf(file)
|
556
588
|
end
|
557
589
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-numeric-counter
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 1.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- TAGOMORI Satoshi
|
@@ -11,21 +11,21 @@ cert_chain: []
|
|
11
11
|
date: 2017-02-01 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
name:
|
14
|
+
name: fluentd
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
17
|
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version:
|
20
|
-
type: :
|
19
|
+
version: 0.14.0
|
20
|
+
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version:
|
26
|
+
version: 0.14.0
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
|
-
name:
|
28
|
+
name: rake
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
31
|
- - ">="
|
@@ -39,33 +39,33 @@ dependencies:
|
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '0'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
|
-
name:
|
42
|
+
name: timecop
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
45
|
- - ">="
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version:
|
47
|
+
version: '0'
|
48
48
|
type: :development
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
52
|
- - ">="
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version:
|
54
|
+
version: '0'
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
|
-
name:
|
56
|
+
name: test-unit
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
|
-
- - "
|
59
|
+
- - ">="
|
60
60
|
- !ruby/object:Gem::Version
|
61
|
-
version:
|
62
|
-
type: :
|
61
|
+
version: 3.1.0
|
62
|
+
type: :development
|
63
63
|
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
|
-
- - "
|
66
|
+
- - ">="
|
67
67
|
- !ruby/object:Gem::Version
|
68
|
-
version:
|
68
|
+
version: 3.1.0
|
69
69
|
description: Counts messages, with specified key and numeric value in specified range
|
70
70
|
email:
|
71
71
|
- tagomoris@gmail.com
|