fluent-plugin-numeric-counter 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2,7 +2,7 @@
|
|
2
2
|
|
3
3
|
Gem::Specification.new do |gem|
|
4
4
|
gem.name = "fluent-plugin-numeric-counter"
|
5
|
-
gem.version = "0.
|
5
|
+
gem.version = "0.2.0"
|
6
6
|
gem.authors = ["TAGOMORI Satoshi"]
|
7
7
|
gem.email = ["tagomoris@gmail.com"]
|
8
8
|
gem.description = %q{Counts messages, with specified key and numeric value in specified range}
|
@@ -105,11 +105,12 @@ class Fluent::NumericCounterOutput < Fluent::Output
|
|
105
105
|
|
106
106
|
def count_initialized(keys=nil)
|
107
107
|
# counts['tag'][pattern_index_num] = count
|
108
|
+
# counts['tag'][-1] = sum
|
108
109
|
if @aggregate == :all
|
109
|
-
{'all' => Array.new(@patterns.length){|i| 0}}
|
110
|
+
{'all' => Array.new(@patterns.length + 1){|i| 0}}
|
110
111
|
elsif keys
|
111
112
|
values = Array.new(keys.length){|i|
|
112
|
-
Array.new(@patterns.length){|j| 0 }
|
113
|
+
Array.new(@patterns.length + 1){|j| 0 }
|
113
114
|
}
|
114
115
|
Hash[[keys, values].transpose]
|
115
116
|
else
|
@@ -123,10 +124,13 @@ class Fluent::NumericCounterOutput < Fluent::Output
|
|
123
124
|
end
|
124
125
|
|
125
126
|
@mutex.synchronize {
|
126
|
-
@counts[tag] ||= [0] * @patterns.length
|
127
|
+
@counts[tag] ||= [0] * (@patterns.length + 1)
|
128
|
+
sum = 0
|
127
129
|
counts.each_with_index do |count, i|
|
130
|
+
sum += count
|
128
131
|
@counts[tag][i] += count
|
129
132
|
end
|
133
|
+
@counts[tag][-1] += sum
|
130
134
|
}
|
131
135
|
end
|
132
136
|
|
@@ -139,11 +143,11 @@ class Fluent::NumericCounterOutput < Fluent::Output
|
|
139
143
|
|
140
144
|
def generate_fields(step, target_counts, attr_prefix, output)
|
141
145
|
sum = if @outcast_unmatched
|
142
|
-
target_counts[1..-
|
146
|
+
target_counts[1..-2].inject(:+)
|
143
147
|
else
|
144
|
-
target_counts
|
148
|
+
target_counts[-1]
|
145
149
|
end
|
146
|
-
messages =
|
150
|
+
messages = target_counts.delete_at(-1)
|
147
151
|
|
148
152
|
target_counts.each_with_index do |count,i|
|
149
153
|
name = @patterns[i][1]
|
@@ -185,12 +189,12 @@ class Fluent::NumericCounterOutput < Fluent::Output
|
|
185
189
|
end
|
186
190
|
|
187
191
|
def flush(step) # returns one message
|
188
|
-
flushed,@counts = @counts,count_initialized(@counts.keys.dup)
|
192
|
+
flushed,@counts = @counts,count_initialized(@counts.keys.dup.select{|k| @counts[k][-1] > 0})
|
189
193
|
generate_output(flushed, step)
|
190
194
|
end
|
191
195
|
|
192
196
|
def flush_per_tags(step) # returns map of tag - message
|
193
|
-
flushed,@counts = @counts,count_initialized(@counts.keys.dup)
|
197
|
+
flushed,@counts = @counts,count_initialized(@counts.keys.dup.select{|k| @counts[k][-1] > 0})
|
194
198
|
generate_output_per_tags(flushed, step)
|
195
199
|
end
|
196
200
|
|
@@ -201,7 +205,10 @@ class Fluent::NumericCounterOutput < Fluent::Output
|
|
201
205
|
Fluent::Engine.emit(@tag_prefix_string + tag, time, message)
|
202
206
|
end
|
203
207
|
else
|
204
|
-
|
208
|
+
message = flush(step)
|
209
|
+
if message.keys.size > 0
|
210
|
+
Fluent::Engine.emit(@tag, Fluent::Engine.now, message)
|
211
|
+
end
|
205
212
|
end
|
206
213
|
end
|
207
214
|
|
@@ -119,11 +119,11 @@ class NumericCounterOutputTest < Test::Unit::TestCase
|
|
119
119
|
assert_nil d.instance.counts['test.input']
|
120
120
|
|
121
121
|
d.instance.countups('test.input', [0, 0, 0, 0])
|
122
|
-
assert_equal [0,0,0,0], d.instance.counts['test.input']
|
122
|
+
assert_equal [0,0,0,0,0], d.instance.counts['test.input']
|
123
123
|
d.instance.countups('test.input', [1, 1, 1, 0])
|
124
|
-
assert_equal [1,1,1,0], d.instance.counts['test.input']
|
124
|
+
assert_equal [1,1,1,0,3], d.instance.counts['test.input']
|
125
125
|
d.instance.countups('test.input', [0, 5, 1, 0])
|
126
|
-
assert_equal [1,6,2,0], d.instance.counts['test.input']
|
126
|
+
assert_equal [1,6,2,0,9], d.instance.counts['test.input']
|
127
127
|
end
|
128
128
|
|
129
129
|
def test_generate_output
|
@@ -132,7 +132,7 @@ class NumericCounterOutputTest < Test::Unit::TestCase
|
|
132
132
|
# pattern2 u1s 100000 1000000
|
133
133
|
# pattern3 u3s 1000000 3000000
|
134
134
|
|
135
|
-
r1 = d.instance.generate_output({'test.input' => [60,240,180,120], 'test.input2' => [0,600,0,0]}, 60)
|
135
|
+
r1 = d.instance.generate_output({'test.input' => [60,240,180,120,600], 'test.input2' => [0,600,0,0,600]}, 60)
|
136
136
|
assert_equal 60, r1['input_unmatched_count']
|
137
137
|
assert_equal 1.0, r1['input_unmatched_rate']
|
138
138
|
assert_equal 10.0, r1['input_unmatched_percentage']
|
@@ -163,7 +163,7 @@ class NumericCounterOutputTest < Test::Unit::TestCase
|
|
163
163
|
|
164
164
|
d = create_driver(CONFIG + "\n output_messages yes \n")
|
165
165
|
|
166
|
-
r1 = d.instance.generate_output({'test.input' => [60,240,180,120], 'test.input2' => [0,600,0,0]}, 60)
|
166
|
+
r1 = d.instance.generate_output({'test.input' => [60,240,180,120,600], 'test.input2' => [0,600,0,0,600]}, 60)
|
167
167
|
assert_equal 600, r1['input_messages']
|
168
168
|
assert_equal 600, r1['input2_messages']
|
169
169
|
|
@@ -174,7 +174,7 @@ class NumericCounterOutputTest < Test::Unit::TestCase
|
|
174
174
|
outcast_unmatched yes
|
175
175
|
output_messages true
|
176
176
|
]
|
177
|
-
r2 = d.instance.generate_output({'all' => [60,240]}, 60)
|
177
|
+
r2 = d.instance.generate_output({'all' => [60,240,300]}, 60)
|
178
178
|
assert_equal 60, r2['unmatched_count']
|
179
179
|
assert_equal 1.0, r2['unmatched_rate']
|
180
180
|
assert_nil r2['unmatched_percentage']
|
@@ -190,7 +190,7 @@ class NumericCounterOutputTest < Test::Unit::TestCase
|
|
190
190
|
# pattern2 u1s 100000 1000000
|
191
191
|
# pattern3 u3s 1000000 3000000
|
192
192
|
|
193
|
-
r1 = d.instance.generate_output_per_tags({'test.input' => [60,240,180,120], 'test.input2' => [0,600,0,0]}, 60)
|
193
|
+
r1 = d.instance.generate_output_per_tags({'test.input' => [60,240,180,120,600], 'test.input2' => [0,600,0,0,600]}, 60)
|
194
194
|
assert_equal 2, r1.keys.size
|
195
195
|
|
196
196
|
r = r1['input']
|
@@ -225,7 +225,7 @@ class NumericCounterOutputTest < Test::Unit::TestCase
|
|
225
225
|
|
226
226
|
d = create_driver(CONFIG_OUTPUT_PER_TAG)
|
227
227
|
|
228
|
-
r1 = d.instance.generate_output_per_tags({'test.input' => [60,240,180,120], 'test.input2' => [0,600,0,0]}, 60)
|
228
|
+
r1 = d.instance.generate_output_per_tags({'test.input' => [60,240,180,120,600], 'test.input2' => [0,600,0,0,600]}, 60)
|
229
229
|
assert_equal 600, r1['input']['messages']
|
230
230
|
assert_equal 600, r1['input2']['messages']
|
231
231
|
|
@@ -236,7 +236,7 @@ class NumericCounterOutputTest < Test::Unit::TestCase
|
|
236
236
|
outcast_unmatched yes
|
237
237
|
output_messages true
|
238
238
|
]
|
239
|
-
r2 = d.instance.generate_output_per_tags({'all' => [60,240]}, 60)
|
239
|
+
r2 = d.instance.generate_output_per_tags({'all' => [60,240,300]}, 60)
|
240
240
|
assert_equal 60, r2['all']['unmatched_count']
|
241
241
|
assert_equal 1.0, r2['all']['unmatched_rate']
|
242
242
|
assert_nil r2['all']['unmatched_percentage']
|
@@ -409,4 +409,91 @@ class NumericCounterOutputTest < Test::Unit::TestCase
|
|
409
409
|
assert_equal 20, r['unmatched_percentage']
|
410
410
|
assert_equal 300, r['messages']
|
411
411
|
end
|
412
|
+
|
413
|
+
def test_zero_tags
|
414
|
+
fields = ['unmatched','u100ms','u1s','u3s'].map{|k| 'tag1_' + k}.map{|p|
|
415
|
+
['count', 'rate', 'percentage'].map{|a| p + '_' + a}
|
416
|
+
}.flatten
|
417
|
+
fields_without_percentage = ['unmatched','u100ms','u1s','u3s'].map{|k| 'tag1_' + k}.map{|p|
|
418
|
+
['count', 'rate'].map{|a| p + '_' + a}
|
419
|
+
}.flatten
|
420
|
+
|
421
|
+
d = create_driver(CONFIG, 'test.tag1')
|
422
|
+
# CONFIG = %[
|
423
|
+
# count_interval 60
|
424
|
+
# aggregate tag
|
425
|
+
# input_tag_remove_prefix test
|
426
|
+
# count_key target
|
427
|
+
# pattern1 u100ms 0 100000
|
428
|
+
# pattern2 u1s 100000 1000000
|
429
|
+
# pattern3 u3s 1000000 3000000
|
430
|
+
# ]
|
431
|
+
d.run do
|
432
|
+
60.times do
|
433
|
+
d.emit({'target' => '50000'})
|
434
|
+
d.emit({'target' => '100000'})
|
435
|
+
d.emit({'target' => '100001'})
|
436
|
+
d.emit({'target' => '0.0'})
|
437
|
+
d.emit({'target' => '-1'})
|
438
|
+
end
|
439
|
+
end
|
440
|
+
d.instance.flush_emit(60)
|
441
|
+
assert_equal 1, d.emits.size
|
442
|
+
r1 = d.emits[0][2]
|
443
|
+
assert_equal fields, r1.keys
|
444
|
+
|
445
|
+
d.instance.flush_emit(60)
|
446
|
+
assert_equal 2, d.emits.size # +1
|
447
|
+
r2 = d.emits[1][2]
|
448
|
+
assert_equal fields_without_percentage, r2.keys
|
449
|
+
assert_equal [0]*8, r2.values
|
450
|
+
|
451
|
+
d.instance.flush_emit(60)
|
452
|
+
assert_equal 2, d.emits.size # +0
|
453
|
+
end
|
454
|
+
|
455
|
+
def test_zero_tags_per_tag
|
456
|
+
fields = (['unmatched','u100ms','u1s','u3s'].map{|p|
|
457
|
+
['count', 'rate', 'percentage'].map{|a| p + '_' + a}
|
458
|
+
}.flatten + ['messages']).sort
|
459
|
+
fields_without_percentage = (['unmatched','u100ms','u1s','u3s'].map{|p|
|
460
|
+
['count', 'rate'].map{|a| p + '_' + a}
|
461
|
+
}.flatten + ['messages']).sort
|
462
|
+
|
463
|
+
d = create_driver(CONFIG_OUTPUT_PER_TAG, 'test.tag1')
|
464
|
+
# CONFIG_OUTPUT_PER_TAG = %[
|
465
|
+
# count_interval 60
|
466
|
+
# aggregate tag
|
467
|
+
# output_per_tag true
|
468
|
+
# tag_prefix n
|
469
|
+
# input_tag_remove_prefix test
|
470
|
+
# count_key target
|
471
|
+
# pattern1 u100ms 0 100000
|
472
|
+
# pattern2 u1s 100000 1000000
|
473
|
+
# pattern3 u3s 1000000 3000000
|
474
|
+
# output_messages true
|
475
|
+
# ]
|
476
|
+
d.run do
|
477
|
+
60.times do
|
478
|
+
d.emit({'target' => '50000'})
|
479
|
+
d.emit({'target' => '100000'})
|
480
|
+
d.emit({'target' => '100001'})
|
481
|
+
d.emit({'target' => '0.0'})
|
482
|
+
d.emit({'target' => '-1'})
|
483
|
+
end
|
484
|
+
end
|
485
|
+
d.instance.flush_emit(60)
|
486
|
+
assert_equal 1, d.emits.size
|
487
|
+
r1 = d.emits[0][2]
|
488
|
+
assert_equal fields, r1.keys.sort
|
489
|
+
|
490
|
+
d.instance.flush_emit(60)
|
491
|
+
assert_equal 2, d.emits.size # +1
|
492
|
+
r2 = d.emits[1][2]
|
493
|
+
assert_equal fields_without_percentage, r2.keys.sort
|
494
|
+
assert_equal [0]*9, r2.values # (_count, _rate) x4 + messages
|
495
|
+
|
496
|
+
d.instance.flush_emit(60)
|
497
|
+
assert_equal 2, d.emits.size # +0
|
498
|
+
end
|
412
499
|
end
|