fluent-plugin-numeric-counter 0.3.0 → 1.0.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: bf6633c31e441b5d191a04289f68514a451cce3e
4
- data.tar.gz: 44937da1b98030c2e20e66694d6b47310a3f4b2a
3
+ metadata.gz: b9ce759e247b27af5611e9bdd14e040c717b9736
4
+ data.tar.gz: 342f5c8542780534e8f930f2f6e4c4a749869406
5
5
  SHA512:
6
- metadata.gz: fd6abbbf507db891618579ea508736e59b489839420a3f04458843f3ccbb40bf420a86204c51b572e07fc8e837e4002a9b74d8aa58e9461854c48a64eb93091b
7
- data.tar.gz: e9778122e8cb4caad8c24c4e5f513844dca18006c9b84c81d4f7e9c8383e1595c3f0f9c49c5f09846b1f5652f7ff37b3d3ee7cd5b6bf91001a37b06f3c86c59d
6
+ metadata.gz: 3992934eebeb1493bc35ccecd593d73329b64cf6b649db583e7d64f05c0c0bafbaa65e4a2f1ae8daad2bc2b2161592e511c4070cd9b7ce842174af19a50fccad
7
+ data.tar.gz: 0bdde9d5ef5f89e49ea6dec8cd170b55085c828a46ab1d82c792ec5785e951b4013d700e29567b66787cdd7051c45f0b16605e6c2b65ce0b5fe39438f4ac60a6
@@ -1,7 +1,6 @@
1
1
  language: ruby
2
2
  sudo: false
3
3
  rvm:
4
- - 2.0.0
5
4
  - 2.1.8
6
5
  - 2.2.4
7
6
  - 2.3.0
data/Rakefile CHANGED
@@ -8,4 +8,4 @@ Rake::TestTask.new(:test) do |test|
8
8
  test.verbose = true
9
9
  end
10
10
 
11
- task :default => :test
11
+ task default: :test
@@ -2,7 +2,7 @@
2
2
 
3
3
  Gem::Specification.new do |gem|
4
4
  gem.name = "fluent-plugin-numeric-counter"
5
- gem.version = "0.3.0"
5
+ gem.version = "1.0.0"
6
6
  gem.authors = ["TAGOMORI Satoshi"]
7
7
  gem.email = ["tagomoris@gmail.com"]
8
8
  gem.description = %q{Counts messages, with specified key and numeric value in specified range}
@@ -15,8 +15,8 @@ Gem::Specification.new do |gem|
15
15
  gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
16
16
  gem.require_paths = ["lib"]
17
17
 
18
+ gem.add_runtime_dependency "fluentd", ">= 0.14.0"
18
19
  gem.add_development_dependency "rake"
19
- gem.add_development_dependency "delorean"
20
+ gem.add_development_dependency "timecop"
20
21
  gem.add_development_dependency "test-unit", ">= 3.1.0"
21
- gem.add_runtime_dependency "fluentd", "< 0.14.0"
22
22
  end
@@ -1,59 +1,65 @@
1
- class Fluent::NumericCounterOutput < Fluent::Output
1
+ require 'fluent/plugin/output'
2
+ require 'pathname'
3
+
4
+ class Fluent::Plugin::NumericCounterOutput < Fluent::Plugin::Output
2
5
  Fluent::Plugin.register_output('numeric_counter', self)
3
6
 
4
- def initialize
5
- super
6
- require 'pathname'
7
- end
7
+ helpers :event_emitter, :storage, :timer
8
8
 
9
+ DEFAULT_STORAGE_TYPE = 'local'
9
10
  PATTERN_MAX_NUM = 20
10
11
 
11
- config_param :count_interval, :time, :default => 60,
12
- :desc => 'The interval time to count in seconds.'
13
- config_param :unit, :string, :default => nil,
14
- :desc => <<-DESC
12
+ config_param :count_interval, :time, default: 60,
13
+ desc: 'The interval time to count in seconds.'
14
+ config_param :unit, :enum, list: [:minute, :hour, :day], default: nil,
15
+ desc: <<-DESC
15
16
  The interval time to monitor specified an unit (either of minute, hour, or day).
16
17
  Use either of count_interval or unit.
17
18
  DESC
18
- config_param :output_per_tag, :bool, :default => false,
19
- :desc => <<-DESC
20
- Emit for each input tag.
21
- tag_prefix must be specified together.
22
- DESC
23
- config_param :aggregate, :string, :default => 'tag',
24
- :desc => 'Calculate in each input tag separetely, or all records in a mass.'
25
- config_param :tag, :string, :default => 'numcount',
26
- :desc => 'The output tag.'
27
- config_param :tag_prefix, :string, :default => nil,
28
- :desc => <<-DESC
29
- The prefix string which will be added to the input tag.
30
- output_per_tag yes must be specified together.
31
- DESC
32
- config_param :input_tag_remove_prefix, :string, :default => nil,
33
- :desc => 'The prefix string which will be removed from the input tag.'
19
+ config_param :output_per_tag, :bool, default: false,
20
+ desc: 'Produce counter result per input tags.'
21
+
22
+ config_param :aggregate, :enum, list: [:tag, :all], default: :tag,
23
+ desc: 'Calculate in each input tag separetely, or all records in a mass.'
24
+ config_param :tag, :string, default: 'numcount',
25
+ desc: 'The output tag.'
26
+
27
+ config_param :input_tag_remove_prefix, :string, default: nil,
28
+ desc: 'The prefix string which will be removed from the input tag.'
34
29
  config_param :count_key, :string,
35
- :desc => 'The key to count in the event record.'
36
- config_param :outcast_unmatched, :bool, :default => false,
37
- :desc => <<-DESC
30
+ desc: 'The key to count in the event record.'
31
+ config_param :outcast_unmatched, :bool, default: false,
32
+ desc: <<-DESC
38
33
  Specify yes if you do not want to include 'unmatched' counts into percentage.
39
34
  DESC
40
- config_param :output_messages, :bool, :default => false,
41
- :desc => 'Specify yes if you want to get tested messages.'
42
- config_param :store_file, :string, :default => nil,
43
- :desc => <<-DESC
44
- Store internal data into a file of the given path on shutdown, and load on starting.
45
- DESC
35
+ config_param :output_messages, :bool, default: false,
36
+ desc: 'Specify yes if you want to get tested messages.'
37
+
38
+ config_param :store_file, :string, default: nil,
39
+ obsoleted: 'Use store_storage parameter instead.',
40
+ desc: 'Store internal data into a file of the given path on shutdown, and load on starting.'
41
+ config_param :store_storage, :bool, default: false,
42
+ desc: 'Store internal data into a storage on shutdown, and load on starting.'
46
43
 
47
44
  # pattern0 reserved as unmatched counts
48
45
  config_param :pattern1, :string,
49
- :desc => <<-DESC
46
+ desc: <<-DESC
50
47
  string: NAME LOW HIGH
51
48
  LOW/HIGH allows size prefix (ex: 10k, 5M, 3500G)
52
49
  Note that pattern0 reserved as unmatched counts.
53
50
  DESC
54
51
  (2..PATTERN_MAX_NUM).each do |i|
55
- config_param ('pattern' + i.to_s).to_sym, :string, :default => nil,
56
- :desc => 'string: NAME LOW HIGH'
52
+ config_param ('pattern' + i.to_s).to_sym, :string, default: nil,
53
+ desc: 'string: NAME LOW HIGH'
54
+ end
55
+
56
+ config_param :tag_prefix, :string, default: nil,
57
+ desc: 'The prefix string to be added to input tags. Use with "output_per_tag yes".',
58
+ deprecated: 'Use @label routing instead.'
59
+
60
+ config_section :storage do
61
+ config_set_default :usage, 'resume'
62
+ config_set_default :@type, DEFAULT_STORAGE_TYPE
57
63
  end
58
64
 
59
65
  attr_accessor :counts
@@ -62,16 +68,6 @@ DESC
62
68
  attr_accessor :saved_at
63
69
  attr_accessor :patterns
64
70
 
65
- # Define `log` method for v0.10.42 or earlier
66
- unless method_defined?(:log)
67
- define_method("log") { $log }
68
- end
69
-
70
- # Define `router` method of v0.12 to support v0.10.57 or earlier
71
- unless method_defined?(:router)
72
- define_method("router") { Fluent::Engine }
73
- end
74
-
75
71
  def parse_num(str)
76
72
  if str.nil?
77
73
  nil
@@ -85,27 +81,26 @@ DESC
85
81
  end
86
82
 
87
83
  def configure(conf)
84
+ label_routing_specified = conf.has_key?('@label')
85
+
88
86
  super
89
87
 
90
88
  if @unit
91
89
  @count_interval = case @unit
92
- when 'minute' then 60
93
- when 'hour' then 3600
94
- when 'day' then 86400
90
+ when :minute then 60
91
+ when :hour then 3600
92
+ when :day then 86400
95
93
  else
96
- raise Fluent::ConfigError, 'unit must be one of minute/hour/day'
94
+ raise "unknown unit:#{@unit}"
97
95
  end
98
96
  end
99
97
 
100
- @aggregate = @aggregate.to_sym
101
- raise Fluent::ConfigError, "numeric_counter allows tag/all to aggregate unit" unless [:tag, :all].include?(@aggregate)
102
-
103
98
  @patterns = [[0, 'unmatched', nil, nil]] # counts-index, name, low, high
104
99
  pattern_names = ['unmatched']
105
100
 
106
101
  invalids = conf.keys.select{|k| k =~ /^pattern(\d+)$/ and not (1..PATTERN_MAX_NUM).include?($1.to_i)}
107
102
  if invalids.size > 0
108
- log.warn "invalid number patterns (valid pattern number:1-#{PATTERN_MAX_NUM}):" + invalids.join(",")
103
+ log.warn "invalid number patterns (valid pattern number:1-#{PATTERN_MAX_NUM}):", invalids: invalids
109
104
  end
110
105
  (1..PATTERN_MAX_NUM).each do |i|
111
106
  next unless conf["pattern#{i}"]
@@ -124,10 +119,14 @@ DESC
124
119
  raise Fluent::ConfigError, "numbers of low/high missing" if low.nil?
125
120
  raise Fluent::ConfigError, "unspecified high threshold allowed only in last pattern" if high.nil? and index != @patterns.length - 1
126
121
  end
127
-
128
- if @output_per_tag
129
- raise Fluent::ConfigError, "tag_prefix must be specified with output_per_tag" unless @tag_prefix
122
+
123
+ if @output_per_tag && (!label_routing_specified && !@tag_prefix)
124
+ raise Fluent::ConfigError, "specify @label to route output events into other <label> sections."
125
+ end
126
+ if @output_per_tag && @tag_prefix
130
127
  @tag_prefix_string = @tag_prefix + '.'
128
+ else
129
+ @tag_prefix_string = nil
131
130
  end
132
131
 
133
132
  if @input_tag_remove_prefix
@@ -135,28 +134,39 @@ DESC
135
134
  @removed_length = @removed_prefix_string.length
136
135
  end
137
136
 
138
- if @store_file
139
- f = Pathname.new(@store_file)
140
- if (f.exist? && !f.writable_real?) || (!f.exist? && !f.parent.writable_real?)
141
- raise Fluent::ConfigError, "#{@store_file} is not writable"
142
- end
137
+ if @store_storage
138
+ @storage = storage_create(usage: 'resume')
139
+ end
140
+
141
+ if system_config.workers > 1
142
+ log.warn "Fluentd is now working with multi process workers, and numeric_counter plugin will produce counter results in each separeted processes."
143
143
  end
144
144
 
145
145
  @counts = count_initialized
146
146
  @mutex = Mutex.new
147
147
  end
148
148
 
149
+ def multi_workers_ready?
150
+ true
151
+ end
152
+
149
153
  def start
150
154
  super
151
- load_status(@store_file, @count_interval) if @store_file
152
- start_watch
155
+
156
+ load_status(@count_interval) if @store_storage
157
+
158
+ @last_checked = Fluent::Engine.now
159
+
160
+ timer_execute(:out_numeric_counter_timer, @count_interval) do
161
+ now = Fluent::Engine.now
162
+ flush_emit(now - @last_checked)
163
+ @last_checked = now
164
+ end
153
165
  end
154
166
 
155
167
  def shutdown
168
+ save_status() if @store_storage
156
169
  super
157
- @watcher.terminate
158
- @watcher.join
159
- save_status(@store_file) if @store_file
160
170
  end
161
171
 
162
172
  def count_initialized(keys=nil)
@@ -217,7 +227,7 @@ DESC
217
227
  end
218
228
  end
219
229
 
220
- output
230
+ output
221
231
  end
222
232
 
223
233
  def generate_output(counts, step)
@@ -258,7 +268,11 @@ DESC
258
268
  if @output_per_tag
259
269
  time = Fluent::Engine.now
260
270
  flush_per_tags(step).each do |tag,message|
261
- router.emit(@tag_prefix_string + tag, time, message)
271
+ if @tag_prefix_string
272
+ router.emit(@tag_prefix_string + tag, time, message)
273
+ else
274
+ router.emit(tag, time, message)
275
+ end
262
276
  end
263
277
  else
264
278
  message = flush(step)
@@ -268,23 +282,7 @@ DESC
268
282
  end
269
283
  end
270
284
 
271
- def start_watch
272
- @watcher = Thread.new(&method(:watch))
273
- end
274
-
275
- def watch
276
- @last_checked ||= Fluent::Engine.now
277
- while true
278
- sleep 0.5
279
- if Fluent::Engine.now - @last_checked >= @count_interval
280
- now = Fluent::Engine.now
281
- flush_emit(now - @last_checked)
282
- @last_checked = now
283
- end
284
- end
285
- end
286
-
287
- def emit(tag, es, chain)
285
+ def process(tag, es)
288
286
  c = [0] * @patterns.length
289
287
 
290
288
  es.each do |time,record|
@@ -302,62 +300,57 @@ DESC
302
300
  c[0] += 1 unless matched
303
301
  end
304
302
  countups(tag, c)
305
-
306
- chain.next
307
303
  end
308
304
 
309
- # Store internal status into a file
305
+ # Store internal status into a storage
310
306
  #
311
- # @param [String] file_path
312
- def save_status(file_path)
307
+ def save_status()
313
308
  begin
314
- Pathname.new(file_path).open('wb') do |f|
315
- @saved_at = Fluent::Engine.now
316
- @saved_duration = @saved_at - @last_checked
317
- Marshal.dump({
318
- :counts => @counts,
319
- :saved_at => @saved_at,
320
- :saved_duration => @saved_duration,
321
- :aggregate => @aggregate,
322
- :count_key => @count_key,
323
- :patterns => @patterns,
324
- }, f)
325
- end
309
+ @saved_at = Fluent::Engine.now
310
+ @saved_duration = @saved_at - @last_checked
311
+ value = {
312
+ "counts" => @counts,
313
+ "saved_at" => @saved_at,
314
+ "saved_duration" => @saved_duration,
315
+ "aggregate" => @aggregate.to_s,
316
+ "count_key" => @count_key,
317
+ "patterns" => @patterns,
318
+ }
319
+ @storage.put(:stored_value, value)
326
320
  rescue => e
327
- log.warn "out_datacounter: Can't write store_file #{e.class} #{e.message}"
321
+ log.warn "Can't write store_storage", error: e
328
322
  end
329
323
  end
330
324
 
331
- # Load internal status from a file
325
+ # Load internal status from a storage
332
326
  #
333
- # @param [String] file_path
334
327
  # @param [Interger] count_interval
335
- def load_status(file_path, count_interval)
336
- return unless (f = Pathname.new(file_path)).exist?
328
+ def load_status(count_interval)
329
+ stored = @storage.get(:stored_value)
330
+ return unless stored
337
331
 
338
332
  begin
339
- f.open('rb') do |f|
340
- stored = Marshal.load(f)
341
- if stored[:aggregate] == @aggregate and
342
- stored[:count_key] == @count_key and
343
- stored[:patterns] == @patterns
333
+ if stored["aggregate"] == @aggregate.to_s and
334
+ stored["count_key"] == @count_key and
335
+ stored["patterns"] == @patterns
344
336
 
345
- if Fluent::Engine.now <= stored[:saved_at] + count_interval
346
- @counts = stored[:counts]
347
- @saved_at = stored[:saved_at]
348
- @saved_duration = stored[:saved_duration]
337
+ if Fluent::Engine.now <= stored["saved_at"] + count_interval
338
+ @mutex.synchronize {
339
+ @counts = stored["counts"]
340
+ @saved_at = stored["saved_at"]
341
+ @saved_duration = stored["saved_duration"]
349
342
 
350
343
  # skip the saved duration to continue counting
351
344
  @last_checked = Fluent::Engine.now - @saved_duration
352
- else
353
- log.warn "out_datacounter: stored data is outdated. ignore stored data"
354
- end
345
+ }
355
346
  else
356
- log.warn "out_datacounter: configuration param was changed. ignore stored data"
347
+ log.warn "stored data is outdated. ignore stored data"
357
348
  end
349
+ else
350
+ log.warn "configuration param was changed. ignore stored data"
358
351
  end
359
352
  rescue => e
360
- log.warn "out_datacounter: Can't load store_file #{e.class} #{e.message}"
353
+ log.warn "Can't load store_storage", error: e
361
354
  end
362
355
  end
363
356
 
@@ -22,7 +22,6 @@ unless ENV.has_key?('VERBOSE')
22
22
  $log = nulllogger
23
23
  end
24
24
 
25
- require 'delorean'
26
25
  require 'fluent/plugin/out_numeric_counter'
27
26
 
28
27
  class Test::Unit::TestCase
@@ -1,10 +1,20 @@
1
1
  require 'helper'
2
+ require 'fluent/test/helpers'
3
+ require 'fluent/test/driver/output'
4
+ require 'timecop'
5
+ require 'fileutils'
2
6
 
3
7
  class NumericCounterOutputTest < Test::Unit::TestCase
8
+ include Fluent::Test::Helpers
9
+
4
10
  def setup
5
11
  Fluent::Test.setup
6
12
  end
7
13
 
14
+ def teardown
15
+ Timecop.return
16
+ end
17
+
8
18
  CONFIG = %[
9
19
  count_interval 60
10
20
  aggregate tag
@@ -28,17 +38,17 @@ class NumericCounterOutputTest < Test::Unit::TestCase
28
38
  output_messages true
29
39
  ]
30
40
 
31
- def create_driver(conf=CONFIG, tag='test')
32
- Fluent::Test::OutputTestDriver.new(Fluent::NumericCounterOutput, tag).configure(conf)
41
+ def create_driver(conf=CONFIG)
42
+ Fluent::Test::Driver::Output.new(Fluent::Plugin::NumericCounterOutput).configure(conf)
33
43
  end
34
-
44
+
35
45
  def test_parse_num
36
46
  p = create_driver.instance
37
47
 
38
48
  assert_equal 1, p.parse_num('1')
39
- assert_equal -1, p.parse_num('-1')
49
+ assert_equal (-1), p.parse_num('-1')
40
50
  assert_equal 1.0, p.parse_num('1.0')
41
- assert_equal -2.0, p.parse_num('-2.0000')
51
+ assert_equal (-2.0), p.parse_num('-2.0000')
42
52
  assert_equal 1024, p.parse_num('1k')
43
53
  end
44
54
 
@@ -247,7 +257,7 @@ class NumericCounterOutputTest < Test::Unit::TestCase
247
257
  end
248
258
 
249
259
  def test_pattern_num
250
- assert_equal 20, Fluent::NumericCounterOutput::PATTERN_MAX_NUM
260
+ assert_equal 20, Fluent::Plugin::NumericCounterOutput::PATTERN_MAX_NUM
251
261
 
252
262
  conf = %[
253
263
  aggregate all
@@ -256,10 +266,10 @@ class NumericCounterOutputTest < Test::Unit::TestCase
256
266
  (1..20).each do |i|
257
267
  conf += "pattern#{i} name#{i} #{i} #{i+1}\n"
258
268
  end
259
- d = create_driver(conf, 'test.max')
260
- d.run do
269
+ d = create_driver(conf)
270
+ d.run(default_tag: 'test.max') do
261
271
  (0..21).each do |i|
262
- d.emit({'field' => i})
272
+ d.feed({'field' => i})
263
273
  end
264
274
  end
265
275
  r = d.instance.flush(60)
@@ -296,14 +306,14 @@ class NumericCounterOutputTest < Test::Unit::TestCase
296
306
  # pattern2 u1s 100000 1000000
297
307
  # pattern3 u3s 1000000 3000000
298
308
  # ]
299
- d = create_driver(CONFIG, 'test.tag1')
300
- d.run do
309
+ d = create_driver(CONFIG)
310
+ d.run(default_tag: 'test.tag1') do
301
311
  60.times do
302
- d.emit({'target' => '50000'})
303
- d.emit({'target' => '100000'})
304
- d.emit({'target' => '100001'})
305
- d.emit({'target' => '0.0'})
306
- d.emit({'target' => '-1'})
312
+ d.feed({'target' => '50000'})
313
+ d.feed({'target' => '100000'})
314
+ d.feed({'target' => '100001'})
315
+ d.feed({'target' => '0.0'})
316
+ d.feed({'target' => '-1'})
307
317
  end
308
318
  end
309
319
  r = d.instance.flush(60)
@@ -321,20 +331,20 @@ class NumericCounterOutputTest < Test::Unit::TestCase
321
331
  assert_equal 1.0, r['tag1_unmatched_rate']
322
332
  assert_equal 20, r['tag1_unmatched_percentage']
323
333
 
324
- d = create_driver(CONFIG, 'test.tag1')
325
- d.run do
334
+ d = create_driver(CONFIG)
335
+ d.run(default_tag: 'test.tag1') do
326
336
  60.times do
327
- d.emit({'target' => '50000'})
328
- d.emit({'target' => '100000'})
329
- d.emit({'target' => '100001'})
330
- d.emit({'target' => '0.0'})
331
- d.emit({'target' => '-1'})
337
+ d.feed({'target' => '50000'})
338
+ d.feed({'target' => '100000'})
339
+ d.feed({'target' => '100001'})
340
+ d.feed({'target' => '0.0'})
341
+ d.feed({'target' => '-1'})
332
342
  end
343
+ d.instance.flush_emit(60)
333
344
  end
334
- d.instance.flush_emit(60)
335
- emits = d.emits
336
- assert_equal 1, emits.length
337
- data = emits[0]
345
+ events = d.events
346
+ assert_equal 1, events.length
347
+ data = events[0]
338
348
  assert_equal 'numcount', data[0] # tag
339
349
  r = data[2] # message
340
350
  assert_equal 120, r['tag1_u100ms_count']
@@ -352,17 +362,18 @@ class NumericCounterOutputTest < Test::Unit::TestCase
352
362
  end
353
363
 
354
364
  def test_emit_output_per_tag
355
- d = create_driver(CONFIG_OUTPUT_PER_TAG, 'test.tag1')
356
- d.run do
365
+ d = create_driver(CONFIG_OUTPUT_PER_TAG)
366
+ r = {}
367
+ d.run(default_tag: 'test.tag1') do
357
368
  60.times do
358
- d.emit({'target' => '50000'})
359
- d.emit({'target' => '100000'})
360
- d.emit({'target' => '100001'})
361
- d.emit({'target' => '0.0'})
362
- d.emit({'target' => '-1'})
369
+ d.feed({'target' => '50000'})
370
+ d.feed({'target' => '100000'})
371
+ d.feed({'target' => '100001'})
372
+ d.feed({'target' => '0.0'})
373
+ d.feed({'target' => '-1'})
363
374
  end
375
+ r = d.instance.flush_per_tags(60)
364
376
  end
365
- r = d.instance.flush_per_tags(60)
366
377
  assert_equal 1, r.keys.size
367
378
  r1 = r['tag1']
368
379
  assert_equal 120, r1['u100ms_count']
@@ -379,20 +390,20 @@ class NumericCounterOutputTest < Test::Unit::TestCase
379
390
  assert_equal 20, r1['unmatched_percentage']
380
391
  assert_equal 300, r1['messages']
381
392
 
382
- d = create_driver(CONFIG_OUTPUT_PER_TAG, 'test.tag1')
383
- d.run do
393
+ d = create_driver(CONFIG_OUTPUT_PER_TAG)
394
+ d.run(default_tag: 'test.tag1') do
384
395
  60.times do
385
- d.emit({'target' => '50000'})
386
- d.emit({'target' => '100000'})
387
- d.emit({'target' => '100001'})
388
- d.emit({'target' => '0.0'})
389
- d.emit({'target' => '-1'})
396
+ d.feed({'target' => '50000'})
397
+ d.feed({'target' => '100000'})
398
+ d.feed({'target' => '100001'})
399
+ d.feed({'target' => '0.0'})
400
+ d.feed({'target' => '-1'})
390
401
  end
402
+ d.instance.flush_emit(60)
391
403
  end
392
- d.instance.flush_emit(60)
393
- emits = d.emits
394
- assert_equal 1, emits.length
395
- data = emits[0]
404
+ events = d.events
405
+ assert_equal 1, events.length
406
+ data = events[0]
396
407
  assert_equal 'n.tag1', data[0] # tag
397
408
  r = data[2] # message
398
409
  assert_equal 120, r['u100ms_count']
@@ -418,7 +429,7 @@ class NumericCounterOutputTest < Test::Unit::TestCase
418
429
  ['count', 'rate'].map{|a| p + '_' + a}
419
430
  }.flatten
420
431
 
421
- d = create_driver(CONFIG, 'test.tag1')
432
+ d = create_driver(CONFIG)
422
433
  # CONFIG = %[
423
434
  # count_interval 60
424
435
  # aggregate tag
@@ -428,28 +439,28 @@ class NumericCounterOutputTest < Test::Unit::TestCase
428
439
  # pattern2 u1s 100000 1000000
429
440
  # pattern3 u3s 1000000 3000000
430
441
  # ]
431
- d.run do
442
+ d.run(default_tag: 'test.tag1') do
432
443
  60.times do
433
- d.emit({'target' => '50000'})
434
- d.emit({'target' => '100000'})
435
- d.emit({'target' => '100001'})
436
- d.emit({'target' => '0.0'})
437
- d.emit({'target' => '-1'})
444
+ d.feed({'target' => '50000'})
445
+ d.feed({'target' => '100000'})
446
+ d.feed({'target' => '100001'})
447
+ d.feed({'target' => '0.0'})
448
+ d.feed({'target' => '-1'})
438
449
  end
450
+ d.instance.flush_emit(60)
451
+ assert_equal 1, d.events.size
452
+ r1 = d.events[0][2]
453
+ assert_equal fields, r1.keys
454
+
455
+ d.instance.flush_emit(60)
456
+ assert_equal 2, d.events.size # +1
457
+ r2 = d.events[1][2]
458
+ assert_equal fields_without_percentage, r2.keys
459
+ assert_equal [0]*8, r2.values
460
+
461
+ d.instance.flush_emit(60)
462
+ assert_equal 2, d.events.size # +0
439
463
  end
440
- d.instance.flush_emit(60)
441
- assert_equal 1, d.emits.size
442
- r1 = d.emits[0][2]
443
- assert_equal fields, r1.keys
444
-
445
- d.instance.flush_emit(60)
446
- assert_equal 2, d.emits.size # +1
447
- r2 = d.emits[1][2]
448
- assert_equal fields_without_percentage, r2.keys
449
- assert_equal [0]*8, r2.values
450
-
451
- d.instance.flush_emit(60)
452
- assert_equal 2, d.emits.size # +0
453
464
  end
454
465
 
455
466
  def test_zero_tags_per_tag
@@ -460,7 +471,7 @@ class NumericCounterOutputTest < Test::Unit::TestCase
460
471
  ['count', 'rate'].map{|a| p + '_' + a}
461
472
  }.flatten + ['messages']).sort
462
473
 
463
- d = create_driver(CONFIG_OUTPUT_PER_TAG, 'test.tag1')
474
+ d = create_driver(CONFIG_OUTPUT_PER_TAG)
464
475
  # CONFIG_OUTPUT_PER_TAG = %[
465
476
  # count_interval 60
466
477
  # aggregate tag
@@ -473,44 +484,63 @@ class NumericCounterOutputTest < Test::Unit::TestCase
473
484
  # pattern3 u3s 1000000 3000000
474
485
  # output_messages true
475
486
  # ]
476
- d.run do
487
+ d.run(default_tag: 'test.tag1') do
477
488
  60.times do
478
- d.emit({'target' => '50000'})
479
- d.emit({'target' => '100000'})
480
- d.emit({'target' => '100001'})
481
- d.emit({'target' => '0.0'})
482
- d.emit({'target' => '-1'})
489
+ d.feed({'target' => '50000'})
490
+ d.feed({'target' => '100000'})
491
+ d.feed({'target' => '100001'})
492
+ d.feed({'target' => '0.0'})
493
+ d.feed({'target' => '-1'})
483
494
  end
495
+ d.instance.flush_emit(60)
496
+ assert_equal 1, d.events.size
497
+ r1 = d.events[0][2]
498
+ assert_equal fields, r1.keys.sort
499
+
500
+ d.instance.flush_emit(60)
501
+ assert_equal 2, d.events.size # +1
502
+ r2 = d.events[1][2]
503
+ assert_equal fields_without_percentage, r2.keys.sort
504
+ assert_equal [0]*9, r2.values # (_count, _rate) x4 + messages
505
+
506
+ d.instance.flush_emit(60)
507
+ assert_equal 2, d.events.size # +0
484
508
  end
485
- d.instance.flush_emit(60)
486
- assert_equal 1, d.emits.size
487
- r1 = d.emits[0][2]
488
- assert_equal fields, r1.keys.sort
489
-
490
- d.instance.flush_emit(60)
491
- assert_equal 2, d.emits.size # +1
492
- r2 = d.emits[1][2]
493
- assert_equal fields_without_percentage, r2.keys.sort
494
- assert_equal [0]*9, r2.values # (_count, _rate) x4 + messages
495
-
496
- d.instance.flush_emit(60)
497
- assert_equal 2, d.emits.size # +0
498
509
  end
499
510
 
500
- def test_store_file
511
+ def test_store_storage
501
512
  dir = "test/tmp"
502
- Dir.mkdir dir unless Dir.exist? dir
503
513
  file = "#{dir}/test.dat"
504
- File.unlink file if File.exist? file
505
-
514
+ FileUtils.rm_rf(file)
515
+ FileUtils.mkdir_p(dir)
516
+
517
+ config = {
518
+ "count_interval" => 60,
519
+ "aggregate" => "tag",
520
+ "input_tag_remove_prefix" => "test",
521
+ "count_key" => " target",
522
+ "pattern1" => "u100ms 0 100000",
523
+ "pattern2" => "u1s 100000 1000000",
524
+ "pattern3" => "u3s 1000000 3000000",
525
+ "store_storage" => true,
526
+ }
527
+ conf = config_element('ROOT', '', config, [
528
+ config_element(
529
+ 'storage', '',
530
+ {'@type' => 'local',
531
+ '@id' => 'test-01',
532
+ 'path' => "#{file}",
533
+ 'persistent' => true,
534
+ })
535
+ ])
506
536
  # test store
507
- d = create_driver(CONFIG + %[store_file #{file}])
508
- d.run do
537
+ d = create_driver(conf)
538
+ time = Fluent::Engine.now
539
+ d.run(default_tag: 'test') do
509
540
  d.instance.flush_emit(60)
510
- d.emit({'target' => 1})
511
- d.emit({'target' => 1})
512
- d.emit({'target' => 1})
513
- d.instance.shutdown
541
+ d.feed(time, {'target' => 1})
542
+ d.feed(time, {'target' => 1})
543
+ d.feed(time, {'target' => 1})
514
544
  end
515
545
  stored_counts = d.instance.counts
516
546
  stored_saved_at = d.instance.saved_at
@@ -518,40 +548,42 @@ class NumericCounterOutputTest < Test::Unit::TestCase
518
548
  assert File.exist? file
519
549
 
520
550
  # test load
521
- d = create_driver(CONFIG + %[store_file #{file}])
522
- d.run do
551
+ d = create_driver(conf)
552
+ loaded_counts = 0
553
+ loaded_saved_at = 0
554
+ loaded_saved_duration = 0
555
+ d.run(default_tag: 'test') do
523
556
  loaded_counts = d.instance.counts
524
557
  loaded_saved_at = d.instance.saved_at
525
558
  loaded_saved_duration = d.instance.saved_duration
526
- assert_equal stored_counts, loaded_counts
527
- assert_equal stored_saved_at, loaded_saved_at
528
- assert_equal stored_saved_duration, loaded_saved_duration
529
559
  end
560
+ assert_equal stored_counts, loaded_counts
561
+ assert_equal stored_saved_at, loaded_saved_at
562
+ assert_equal stored_saved_duration, loaded_saved_duration
530
563
 
531
564
  # test not to load if config is changed
532
- d = create_driver(CONFIG + %[count_key foobar store_file #{file}])
533
- d.run do
565
+ d = create_driver(conf.merge("count_key" => "foobar", "store_storage" => true))
566
+ d.run(default_tag: 'test') do
534
567
  loaded_counts = d.instance.counts
535
568
  loaded_saved_at = d.instance.saved_at
536
569
  loaded_saved_duration = d.instance.saved_duration
537
- assert_equal({}, loaded_counts)
538
- assert_equal(nil, loaded_saved_at)
539
- assert_equal(nil, loaded_saved_duration)
540
570
  end
571
+ assert_equal({}, loaded_counts)
572
+ assert_equal(nil, loaded_saved_at)
573
+ assert_equal(nil, loaded_saved_duration)
541
574
 
542
575
  # test not to load if stored data is outdated.
543
- Delorean.jump 61 # jump more than count_interval
544
- d = create_driver(CONFIG + %[store_file #{file}])
545
- d.run do
576
+ Timecop.freeze(Time.now + 61) # jump more than count_interval
577
+ d = create_driver(conf.merge("store_storage" => true))
578
+ d.run(default_tag: 'test') do
546
579
  loaded_counts = d.instance.counts
547
580
  loaded_saved_at = d.instance.saved_at
548
581
  loaded_saved_duration = d.instance.saved_duration
549
- assert_equal({}, loaded_counts)
550
- assert_equal(nil, loaded_saved_at)
551
- assert_equal(nil, loaded_saved_duration)
552
582
  end
553
- Delorean.back_to_the_present
583
+ assert_equal({}, loaded_counts)
584
+ assert_equal(nil, loaded_saved_at)
585
+ assert_equal(nil, loaded_saved_duration)
554
586
 
555
- File.unlink file
587
+ FileUtils.rm_rf(file)
556
588
  end
557
589
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-numeric-counter
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.0
4
+ version: 1.0.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - TAGOMORI Satoshi
@@ -11,21 +11,21 @@ cert_chain: []
11
11
  date: 2017-02-01 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
- name: rake
14
+ name: fluentd
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
17
  - - ">="
18
18
  - !ruby/object:Gem::Version
19
- version: '0'
20
- type: :development
19
+ version: 0.14.0
20
+ type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - ">="
25
25
  - !ruby/object:Gem::Version
26
- version: '0'
26
+ version: 0.14.0
27
27
  - !ruby/object:Gem::Dependency
28
- name: delorean
28
+ name: rake
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
31
  - - ">="
@@ -39,33 +39,33 @@ dependencies:
39
39
  - !ruby/object:Gem::Version
40
40
  version: '0'
41
41
  - !ruby/object:Gem::Dependency
42
- name: test-unit
42
+ name: timecop
43
43
  requirement: !ruby/object:Gem::Requirement
44
44
  requirements:
45
45
  - - ">="
46
46
  - !ruby/object:Gem::Version
47
- version: 3.1.0
47
+ version: '0'
48
48
  type: :development
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
52
  - - ">="
53
53
  - !ruby/object:Gem::Version
54
- version: 3.1.0
54
+ version: '0'
55
55
  - !ruby/object:Gem::Dependency
56
- name: fluentd
56
+ name: test-unit
57
57
  requirement: !ruby/object:Gem::Requirement
58
58
  requirements:
59
- - - "<"
59
+ - - ">="
60
60
  - !ruby/object:Gem::Version
61
- version: 0.14.0
62
- type: :runtime
61
+ version: 3.1.0
62
+ type: :development
63
63
  prerelease: false
64
64
  version_requirements: !ruby/object:Gem::Requirement
65
65
  requirements:
66
- - - "<"
66
+ - - ">="
67
67
  - !ruby/object:Gem::Version
68
- version: 0.14.0
68
+ version: 3.1.0
69
69
  description: Counts messages, with specified key and numeric value in specified range
70
70
  email:
71
71
  - tagomoris@gmail.com