fluent-plugin-datacounter 0.3.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
3
3
 
4
4
  Gem::Specification.new do |s|
5
5
  s.name = "fluent-plugin-datacounter"
6
- s.version = "0.3.1"
6
+ s.version = "0.4.0"
7
7
  s.authors = ["TAGOMORI Satoshi"]
8
8
  s.email = ["tagomoris@gmail.com"]
9
9
  s.homepage = "https://github.com/tagomoris/fluent-plugin-datacounter"
@@ -95,11 +95,12 @@ class Fluent::DataCounterOutput < Fluent::Output
95
95
 
96
96
  def count_initialized(keys=nil)
97
97
  # counts['tag'][num] = count
98
+ # counts['tag'][-1] = sum
98
99
  if @aggregate == :all
99
- {'all' => ([0] * @patterns.length)}
100
+ {'all' => ([0] * (@patterns.length + 1))}
100
101
  elsif keys
101
102
  values = Array.new(keys.length) {|i|
102
- Array.new(@patterns.length){|j| 0 }
103
+ Array.new(@patterns.length + 1){|j| 0 }
103
104
  }
104
105
  Hash[[keys, values].transpose]
105
106
  else
@@ -113,10 +114,13 @@ class Fluent::DataCounterOutput < Fluent::Output
113
114
  end
114
115
 
115
116
  @mutex.synchronize {
116
- @counts[tag] ||= [0] * @patterns.length
117
+ @counts[tag] ||= [0] * (@patterns.length + 1)
118
+ sum = 0
117
119
  counts.each_with_index do |count, i|
120
+ sum += count
118
121
  @counts[tag][i] += count
119
122
  end
123
+ @counts[tag][-1] += sum
120
124
  }
121
125
  end
122
126
 
@@ -129,11 +133,11 @@ class Fluent::DataCounterOutput < Fluent::Output
129
133
 
130
134
  def generate_fields(step, target_counts, attr_prefix, output)
131
135
  sum = if @outcast_unmatched
132
- target_counts[1..-1].inject(:+)
136
+ target_counts[1..-2].inject(:+)
133
137
  else
134
- target_counts.inject(:+)
138
+ target_counts[-1]
135
139
  end
136
- messages = sum + (@outcast_unmatched ? target_counts[0] : 0)
140
+ messages = target_counts.delete_at(-1)
137
141
 
138
142
  target_counts.each_with_index do |count,i|
139
143
  name = @patterns[i][1]
@@ -175,12 +179,12 @@ class Fluent::DataCounterOutput < Fluent::Output
175
179
  end
176
180
 
177
181
  def flush(step) # returns one message
178
- flushed,@counts = @counts,count_initialized(@counts.keys.dup)
182
+ flushed,@counts = @counts,count_initialized(@counts.keys.dup.select{|k| @counts[k][-1] > 0})
179
183
  generate_output(flushed, step)
180
184
  end
181
185
 
182
186
  def flush_per_tags(step) # returns map of tag - message
183
- flushed,@counts = @counts,count_initialized(@counts.keys.dup)
187
+ flushed,@counts = @counts,count_initialized(@counts.keys.dup.select{|k| @counts[k][-1] > 0})
184
188
  generate_output_per_tags(flushed, step)
185
189
  end
186
190
 
@@ -192,7 +196,10 @@ class Fluent::DataCounterOutput < Fluent::Output
192
196
  Fluent::Engine.emit(@tag_prefix_string + tag, time, message)
193
197
  end
194
198
  else
195
- Fluent::Engine.emit(@tag, Fluent::Engine.now, flush(step))
199
+ message = flush(step)
200
+ if message.keys.size > 0
201
+ Fluent::Engine.emit(@tag, Fluent::Engine.now, message)
202
+ end
196
203
  end
197
204
  end
198
205
 
@@ -148,7 +148,7 @@ class DataCounterOutputTest < Test::Unit::TestCase
148
148
  pattern1 hoge 1\d\d
149
149
  pattern2 moge 2\d\d
150
150
  ]
151
- assert_equal [0,0,0], d.instance.counts['all']
151
+ assert_equal [0,0,0,0], d.instance.counts['all']
152
152
  end
153
153
 
154
154
  def test_countups
@@ -156,11 +156,11 @@ class DataCounterOutputTest < Test::Unit::TestCase
156
156
  assert_nil d.instance.counts['test.input']
157
157
 
158
158
  d.instance.countups('test.input', [0, 0, 0, 0, 0])
159
- assert_equal [0,0,0,0,0], d.instance.counts['test.input']
159
+ assert_equal [0,0,0,0,0,0], d.instance.counts['test.input']
160
160
  d.instance.countups('test.input', [1, 1, 1, 0, 0])
161
- assert_equal [1,1,1,0,0], d.instance.counts['test.input']
161
+ assert_equal [1,1,1,0,0,3], d.instance.counts['test.input']
162
162
  d.instance.countups('test.input', [0, 5, 1, 0, 0])
163
- assert_equal [1,6,2,0,0], d.instance.counts['test.input']
163
+ assert_equal [1,6,2,0,0,9], d.instance.counts['test.input']
164
164
  end
165
165
 
166
166
  def test_stripped_tag
@@ -172,7 +172,7 @@ class DataCounterOutputTest < Test::Unit::TestCase
172
172
 
173
173
  def test_generate_output
174
174
  d = create_driver
175
- r1 = d.instance.generate_output({'test.input' => [60,240,120,180,0], 'test.input2' => [0,600,0,0,0]}, 60)
175
+ r1 = d.instance.generate_output({'test.input' => [60,240,120,180,0,600], 'test.input2' => [0,600,0,0,0,600]}, 60)
176
176
  assert_equal 60, r1['input_unmatched_count']
177
177
  assert_equal 1.0, r1['input_unmatched_rate']
178
178
  assert_equal 10.0, r1['input_unmatched_percentage']
@@ -208,7 +208,7 @@ class DataCounterOutputTest < Test::Unit::TestCase
208
208
  assert_nil r1['input2_messages']
209
209
 
210
210
  d = create_driver (CONFIG + "\n output_messages true\n")
211
- r1 = d.instance.generate_output({'test.input' => [60,240,120,180,0], 'test.input2' => [0,600,0,0,0]}, 60)
211
+ r1 = d.instance.generate_output({'test.input' => [60,240,120,180,0,600], 'test.input2' => [0,600,0,0,0,600]}, 60)
212
212
  assert_equal 600, r1['input_messages']
213
213
  assert_equal 600, r1['input2_messages']
214
214
 
@@ -217,7 +217,7 @@ class DataCounterOutputTest < Test::Unit::TestCase
217
217
  count_key field
218
218
  pattern1 hoge xxx\d\d
219
219
  ]
220
- r2 = d.instance.generate_output({'all' => [60,240]}, 60)
220
+ r2 = d.instance.generate_output({'all' => [60,240,300]}, 60)
221
221
  assert_equal 60, r2['unmatched_count']
222
222
  assert_equal 1.0, r2['unmatched_rate']
223
223
  assert_equal 20.0, r2['unmatched_percentage']
@@ -231,7 +231,7 @@ class DataCounterOutputTest < Test::Unit::TestCase
231
231
  pattern1 hoge xxx\d\d
232
232
  output_messages yes
233
233
  ]
234
- r2 = d.instance.generate_output({'all' => [60,240]}, 60)
234
+ r2 = d.instance.generate_output({'all' => [60,240,300]}, 60)
235
235
  assert_equal 60, r2['unmatched_count']
236
236
  assert_equal 1.0, r2['unmatched_rate']
237
237
  assert_equal 20.0, r2['unmatched_percentage']
@@ -243,7 +243,7 @@ class DataCounterOutputTest < Test::Unit::TestCase
243
243
 
244
244
  def test_generate_output_per_tag
245
245
  d = create_driver(CONFIG_OUTPUT_PER_TAG)
246
- result = d.instance.generate_output_per_tags({'test.input' => [60,240,120,180,0], 'test.input2' => [0,600,0,0,0]}, 60)
246
+ result = d.instance.generate_output_per_tags({'test.input' => [60,240,120,180,0,600], 'test.input2' => [0,600,0,0,0,600]}, 60)
247
247
  assert_equal 60, result['input']['unmatched_count']
248
248
  assert_equal 1.0, result['input']['unmatched_rate']
249
249
  assert_equal 10.0, result['input']['unmatched_percentage']
@@ -285,7 +285,7 @@ class DataCounterOutputTest < Test::Unit::TestCase
285
285
  output_per_tag yes
286
286
  tag_prefix d
287
287
  ]
288
- r = d.instance.generate_output_per_tags({'all' => [60,240]}, 60)
288
+ r = d.instance.generate_output_per_tags({'all' => [60,240,300]}, 60)
289
289
  assert_equal 1, r.keys.size
290
290
  assert_equal 60, r['all']['unmatched_count']
291
291
  assert_equal 1.0, r['all']['unmatched_rate']
@@ -567,4 +567,90 @@ class DataCounterOutputTest < Test::Unit::TestCase
567
567
  assert_equal 60, data[2]['redirect_count']
568
568
  assert_equal 50.0, data[2]['redirect_percentage']
569
569
  end
570
+
571
+ def test_zero_tags
572
+ fields = ['unmatched','status2xx','status3xx','status4xx','status5xx'].map{|k| 'tag1_' + k}.map{|p|
573
+ ['count', 'rate', 'percentage'].map{|a| p + '_' + a}
574
+ }.flatten
575
+ fields_without_percentage = ['unmatched','status2xx','status3xx','status4xx','status5xx'].map{|k| 'tag1_' + k}.map{|p|
576
+ ['count', 'rate'].map{|a| p + '_' + a}
577
+ }.flatten
578
+
579
+ d = create_driver(CONFIG, 'test.tag1')
580
+ # CONFIG = %[
581
+ # unit minute
582
+ # aggregate tag
583
+ # input_tag_remove_prefix test
584
+ # count_key target
585
+ # pattern1 status2xx ^2\\d\\d$
586
+ # pattern2 status3xx ^3\\d\\d$
587
+ # pattern3 status4xx ^4\\d\\d$
588
+ # pattern4 status5xx ^5\\d\\d$
589
+ # ]
590
+ d.run do
591
+ 60.times do
592
+ d.emit({'target' => '200'})
593
+ d.emit({'target' => '100'})
594
+ d.emit({'target' => '200'})
595
+ d.emit({'target' => '400'})
596
+ end
597
+ end
598
+ d.instance.flush_emit(60)
599
+ assert_equal 1, d.emits.size
600
+ r1 = d.emits[0][2]
601
+ assert_equal fields, r1.keys
602
+
603
+ d.instance.flush_emit(60)
604
+ assert_equal 2, d.emits.size # +1
605
+ r2 = d.emits[1][2]
606
+ assert_equal fields_without_percentage, r2.keys
607
+ assert_equal [0]*10, r2.values
608
+
609
+ d.instance.flush_emit(60)
610
+ assert_equal 2, d.emits.size # +0
611
+ end
612
+ def test_zer_tags_per_tag
613
+ fields = (['unmatched','status2xx','status3xx','status4xx','status5xx'].map{|p|
614
+ ['count', 'rate', 'percentage'].map{|a| p + '_' + a}
615
+ }.flatten + ['messages']).sort
616
+ fields_without_percentage = (['unmatched','status2xx','status3xx','status4xx','status5xx'].map{|p|
617
+ ['count', 'rate'].map{|a| p + '_' + a}
618
+ }.flatten + ['messages']).sort
619
+
620
+ d = create_driver(CONFIG_OUTPUT_PER_TAG, 'test.tag1')
621
+ # CONFIG_OUTPUT_PER_TAG = %[
622
+ # unit minute
623
+ # aggregate tag
624
+ # output_per_tag yes
625
+ # tag_prefix d
626
+ # input_tag_remove_prefix test
627
+ # count_key target
628
+ # pattern1 status2xx ^2\\d\\d$
629
+ # pattern2 status3xx ^3\\d\\d$
630
+ # pattern3 status4xx ^4\\d\\d$
631
+ # pattern4 status5xx ^5\\d\\d$
632
+ # output_messages yes
633
+ # ]
634
+ d.run do
635
+ 60.times do
636
+ d.emit({'target' => '200'})
637
+ d.emit({'target' => '100'})
638
+ d.emit({'target' => '200'})
639
+ d.emit({'target' => '400'})
640
+ end
641
+ end
642
+ d.instance.flush_emit(60)
643
+ assert_equal 1, d.emits.size
644
+ r1 = d.emits[0][2]
645
+ assert_equal fields, r1.keys.sort
646
+
647
+ d.instance.flush_emit(60)
648
+ assert_equal 2, d.emits.size # +1
649
+ r2 = d.emits[1][2]
650
+ assert_equal fields_without_percentage, r2.keys.sort
651
+ assert_equal [0]*11, r2.values # (_count, _rate)x5 + messages
652
+
653
+ d.instance.flush_emit(60)
654
+ assert_equal 2, d.emits.size # +0
655
+ end
570
656
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-datacounter
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.1
4
+ version: 0.4.0
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors: