fluent-plugin-datacounter 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = "fluent-plugin-datacounter"
|
6
|
-
s.version = "0.
|
6
|
+
s.version = "0.4.0"
|
7
7
|
s.authors = ["TAGOMORI Satoshi"]
|
8
8
|
s.email = ["tagomoris@gmail.com"]
|
9
9
|
s.homepage = "https://github.com/tagomoris/fluent-plugin-datacounter"
|
@@ -95,11 +95,12 @@ class Fluent::DataCounterOutput < Fluent::Output
|
|
95
95
|
|
96
96
|
def count_initialized(keys=nil)
|
97
97
|
# counts['tag'][num] = count
|
98
|
+
# counts['tag'][-1] = sum
|
98
99
|
if @aggregate == :all
|
99
|
-
{'all' => ([0] * @patterns.length)}
|
100
|
+
{'all' => ([0] * (@patterns.length + 1))}
|
100
101
|
elsif keys
|
101
102
|
values = Array.new(keys.length) {|i|
|
102
|
-
Array.new(@patterns.length){|j| 0 }
|
103
|
+
Array.new(@patterns.length + 1){|j| 0 }
|
103
104
|
}
|
104
105
|
Hash[[keys, values].transpose]
|
105
106
|
else
|
@@ -113,10 +114,13 @@ class Fluent::DataCounterOutput < Fluent::Output
|
|
113
114
|
end
|
114
115
|
|
115
116
|
@mutex.synchronize {
|
116
|
-
@counts[tag] ||= [0] * @patterns.length
|
117
|
+
@counts[tag] ||= [0] * (@patterns.length + 1)
|
118
|
+
sum = 0
|
117
119
|
counts.each_with_index do |count, i|
|
120
|
+
sum += count
|
118
121
|
@counts[tag][i] += count
|
119
122
|
end
|
123
|
+
@counts[tag][-1] += sum
|
120
124
|
}
|
121
125
|
end
|
122
126
|
|
@@ -129,11 +133,11 @@ class Fluent::DataCounterOutput < Fluent::Output
|
|
129
133
|
|
130
134
|
def generate_fields(step, target_counts, attr_prefix, output)
|
131
135
|
sum = if @outcast_unmatched
|
132
|
-
target_counts[1..-
|
136
|
+
target_counts[1..-2].inject(:+)
|
133
137
|
else
|
134
|
-
target_counts
|
138
|
+
target_counts[-1]
|
135
139
|
end
|
136
|
-
messages =
|
140
|
+
messages = target_counts.delete_at(-1)
|
137
141
|
|
138
142
|
target_counts.each_with_index do |count,i|
|
139
143
|
name = @patterns[i][1]
|
@@ -175,12 +179,12 @@ class Fluent::DataCounterOutput < Fluent::Output
|
|
175
179
|
end
|
176
180
|
|
177
181
|
def flush(step) # returns one message
|
178
|
-
flushed,@counts = @counts,count_initialized(@counts.keys.dup)
|
182
|
+
flushed,@counts = @counts,count_initialized(@counts.keys.dup.select{|k| @counts[k][-1] > 0})
|
179
183
|
generate_output(flushed, step)
|
180
184
|
end
|
181
185
|
|
182
186
|
def flush_per_tags(step) # returns map of tag - message
|
183
|
-
flushed,@counts = @counts,count_initialized(@counts.keys.dup)
|
187
|
+
flushed,@counts = @counts,count_initialized(@counts.keys.dup.select{|k| @counts[k][-1] > 0})
|
184
188
|
generate_output_per_tags(flushed, step)
|
185
189
|
end
|
186
190
|
|
@@ -192,7 +196,10 @@ class Fluent::DataCounterOutput < Fluent::Output
|
|
192
196
|
Fluent::Engine.emit(@tag_prefix_string + tag, time, message)
|
193
197
|
end
|
194
198
|
else
|
195
|
-
|
199
|
+
message = flush(step)
|
200
|
+
if message.keys.size > 0
|
201
|
+
Fluent::Engine.emit(@tag, Fluent::Engine.now, message)
|
202
|
+
end
|
196
203
|
end
|
197
204
|
end
|
198
205
|
|
@@ -148,7 +148,7 @@ class DataCounterOutputTest < Test::Unit::TestCase
|
|
148
148
|
pattern1 hoge 1\d\d
|
149
149
|
pattern2 moge 2\d\d
|
150
150
|
]
|
151
|
-
assert_equal [0,0,0], d.instance.counts['all']
|
151
|
+
assert_equal [0,0,0,0], d.instance.counts['all']
|
152
152
|
end
|
153
153
|
|
154
154
|
def test_countups
|
@@ -156,11 +156,11 @@ class DataCounterOutputTest < Test::Unit::TestCase
|
|
156
156
|
assert_nil d.instance.counts['test.input']
|
157
157
|
|
158
158
|
d.instance.countups('test.input', [0, 0, 0, 0, 0])
|
159
|
-
assert_equal [0,0,0,0,0], d.instance.counts['test.input']
|
159
|
+
assert_equal [0,0,0,0,0,0], d.instance.counts['test.input']
|
160
160
|
d.instance.countups('test.input', [1, 1, 1, 0, 0])
|
161
|
-
assert_equal [1,1,1,0,0], d.instance.counts['test.input']
|
161
|
+
assert_equal [1,1,1,0,0,3], d.instance.counts['test.input']
|
162
162
|
d.instance.countups('test.input', [0, 5, 1, 0, 0])
|
163
|
-
assert_equal [1,6,2,0,0], d.instance.counts['test.input']
|
163
|
+
assert_equal [1,6,2,0,0,9], d.instance.counts['test.input']
|
164
164
|
end
|
165
165
|
|
166
166
|
def test_stripped_tag
|
@@ -172,7 +172,7 @@ class DataCounterOutputTest < Test::Unit::TestCase
|
|
172
172
|
|
173
173
|
def test_generate_output
|
174
174
|
d = create_driver
|
175
|
-
r1 = d.instance.generate_output({'test.input' => [60,240,120,180,0], 'test.input2' => [0,600,0,0,0]}, 60)
|
175
|
+
r1 = d.instance.generate_output({'test.input' => [60,240,120,180,0,600], 'test.input2' => [0,600,0,0,0,600]}, 60)
|
176
176
|
assert_equal 60, r1['input_unmatched_count']
|
177
177
|
assert_equal 1.0, r1['input_unmatched_rate']
|
178
178
|
assert_equal 10.0, r1['input_unmatched_percentage']
|
@@ -208,7 +208,7 @@ class DataCounterOutputTest < Test::Unit::TestCase
|
|
208
208
|
assert_nil r1['input2_messages']
|
209
209
|
|
210
210
|
d = create_driver (CONFIG + "\n output_messages true\n")
|
211
|
-
r1 = d.instance.generate_output({'test.input' => [60,240,120,180,0], 'test.input2' => [0,600,0,0,0]}, 60)
|
211
|
+
r1 = d.instance.generate_output({'test.input' => [60,240,120,180,0,600], 'test.input2' => [0,600,0,0,0,600]}, 60)
|
212
212
|
assert_equal 600, r1['input_messages']
|
213
213
|
assert_equal 600, r1['input2_messages']
|
214
214
|
|
@@ -217,7 +217,7 @@ class DataCounterOutputTest < Test::Unit::TestCase
|
|
217
217
|
count_key field
|
218
218
|
pattern1 hoge xxx\d\d
|
219
219
|
]
|
220
|
-
r2 = d.instance.generate_output({'all' => [60,240]}, 60)
|
220
|
+
r2 = d.instance.generate_output({'all' => [60,240,300]}, 60)
|
221
221
|
assert_equal 60, r2['unmatched_count']
|
222
222
|
assert_equal 1.0, r2['unmatched_rate']
|
223
223
|
assert_equal 20.0, r2['unmatched_percentage']
|
@@ -231,7 +231,7 @@ class DataCounterOutputTest < Test::Unit::TestCase
|
|
231
231
|
pattern1 hoge xxx\d\d
|
232
232
|
output_messages yes
|
233
233
|
]
|
234
|
-
r2 = d.instance.generate_output({'all' => [60,240]}, 60)
|
234
|
+
r2 = d.instance.generate_output({'all' => [60,240,300]}, 60)
|
235
235
|
assert_equal 60, r2['unmatched_count']
|
236
236
|
assert_equal 1.0, r2['unmatched_rate']
|
237
237
|
assert_equal 20.0, r2['unmatched_percentage']
|
@@ -243,7 +243,7 @@ class DataCounterOutputTest < Test::Unit::TestCase
|
|
243
243
|
|
244
244
|
def test_generate_output_per_tag
|
245
245
|
d = create_driver(CONFIG_OUTPUT_PER_TAG)
|
246
|
-
result = d.instance.generate_output_per_tags({'test.input' => [60,240,120,180,0], 'test.input2' => [0,600,0,0,0]}, 60)
|
246
|
+
result = d.instance.generate_output_per_tags({'test.input' => [60,240,120,180,0,600], 'test.input2' => [0,600,0,0,0,600]}, 60)
|
247
247
|
assert_equal 60, result['input']['unmatched_count']
|
248
248
|
assert_equal 1.0, result['input']['unmatched_rate']
|
249
249
|
assert_equal 10.0, result['input']['unmatched_percentage']
|
@@ -285,7 +285,7 @@ class DataCounterOutputTest < Test::Unit::TestCase
|
|
285
285
|
output_per_tag yes
|
286
286
|
tag_prefix d
|
287
287
|
]
|
288
|
-
r = d.instance.generate_output_per_tags({'all' => [60,240]}, 60)
|
288
|
+
r = d.instance.generate_output_per_tags({'all' => [60,240,300]}, 60)
|
289
289
|
assert_equal 1, r.keys.size
|
290
290
|
assert_equal 60, r['all']['unmatched_count']
|
291
291
|
assert_equal 1.0, r['all']['unmatched_rate']
|
@@ -567,4 +567,90 @@ class DataCounterOutputTest < Test::Unit::TestCase
|
|
567
567
|
assert_equal 60, data[2]['redirect_count']
|
568
568
|
assert_equal 50.0, data[2]['redirect_percentage']
|
569
569
|
end
|
570
|
+
|
571
|
+
def test_zero_tags
|
572
|
+
fields = ['unmatched','status2xx','status3xx','status4xx','status5xx'].map{|k| 'tag1_' + k}.map{|p|
|
573
|
+
['count', 'rate', 'percentage'].map{|a| p + '_' + a}
|
574
|
+
}.flatten
|
575
|
+
fields_without_percentage = ['unmatched','status2xx','status3xx','status4xx','status5xx'].map{|k| 'tag1_' + k}.map{|p|
|
576
|
+
['count', 'rate'].map{|a| p + '_' + a}
|
577
|
+
}.flatten
|
578
|
+
|
579
|
+
d = create_driver(CONFIG, 'test.tag1')
|
580
|
+
# CONFIG = %[
|
581
|
+
# unit minute
|
582
|
+
# aggregate tag
|
583
|
+
# input_tag_remove_prefix test
|
584
|
+
# count_key target
|
585
|
+
# pattern1 status2xx ^2\\d\\d$
|
586
|
+
# pattern2 status3xx ^3\\d\\d$
|
587
|
+
# pattern3 status4xx ^4\\d\\d$
|
588
|
+
# pattern4 status5xx ^5\\d\\d$
|
589
|
+
# ]
|
590
|
+
d.run do
|
591
|
+
60.times do
|
592
|
+
d.emit({'target' => '200'})
|
593
|
+
d.emit({'target' => '100'})
|
594
|
+
d.emit({'target' => '200'})
|
595
|
+
d.emit({'target' => '400'})
|
596
|
+
end
|
597
|
+
end
|
598
|
+
d.instance.flush_emit(60)
|
599
|
+
assert_equal 1, d.emits.size
|
600
|
+
r1 = d.emits[0][2]
|
601
|
+
assert_equal fields, r1.keys
|
602
|
+
|
603
|
+
d.instance.flush_emit(60)
|
604
|
+
assert_equal 2, d.emits.size # +1
|
605
|
+
r2 = d.emits[1][2]
|
606
|
+
assert_equal fields_without_percentage, r2.keys
|
607
|
+
assert_equal [0]*10, r2.values
|
608
|
+
|
609
|
+
d.instance.flush_emit(60)
|
610
|
+
assert_equal 2, d.emits.size # +0
|
611
|
+
end
|
612
|
+
def test_zer_tags_per_tag
|
613
|
+
fields = (['unmatched','status2xx','status3xx','status4xx','status5xx'].map{|p|
|
614
|
+
['count', 'rate', 'percentage'].map{|a| p + '_' + a}
|
615
|
+
}.flatten + ['messages']).sort
|
616
|
+
fields_without_percentage = (['unmatched','status2xx','status3xx','status4xx','status5xx'].map{|p|
|
617
|
+
['count', 'rate'].map{|a| p + '_' + a}
|
618
|
+
}.flatten + ['messages']).sort
|
619
|
+
|
620
|
+
d = create_driver(CONFIG_OUTPUT_PER_TAG, 'test.tag1')
|
621
|
+
# CONFIG_OUTPUT_PER_TAG = %[
|
622
|
+
# unit minute
|
623
|
+
# aggregate tag
|
624
|
+
# output_per_tag yes
|
625
|
+
# tag_prefix d
|
626
|
+
# input_tag_remove_prefix test
|
627
|
+
# count_key target
|
628
|
+
# pattern1 status2xx ^2\\d\\d$
|
629
|
+
# pattern2 status3xx ^3\\d\\d$
|
630
|
+
# pattern3 status4xx ^4\\d\\d$
|
631
|
+
# pattern4 status5xx ^5\\d\\d$
|
632
|
+
# output_messages yes
|
633
|
+
# ]
|
634
|
+
d.run do
|
635
|
+
60.times do
|
636
|
+
d.emit({'target' => '200'})
|
637
|
+
d.emit({'target' => '100'})
|
638
|
+
d.emit({'target' => '200'})
|
639
|
+
d.emit({'target' => '400'})
|
640
|
+
end
|
641
|
+
end
|
642
|
+
d.instance.flush_emit(60)
|
643
|
+
assert_equal 1, d.emits.size
|
644
|
+
r1 = d.emits[0][2]
|
645
|
+
assert_equal fields, r1.keys.sort
|
646
|
+
|
647
|
+
d.instance.flush_emit(60)
|
648
|
+
assert_equal 2, d.emits.size # +1
|
649
|
+
r2 = d.emits[1][2]
|
650
|
+
assert_equal fields_without_percentage, r2.keys.sort
|
651
|
+
assert_equal [0]*11, r2.values # (_count, _rate)x5 + messages
|
652
|
+
|
653
|
+
d.instance.flush_emit(60)
|
654
|
+
assert_equal 2, d.emits.size # +0
|
655
|
+
end
|
570
656
|
end
|