fluent-plugin-numeric-counter 0.3.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bf6633c31e441b5d191a04289f68514a451cce3e
-  data.tar.gz: 44937da1b98030c2e20e66694d6b47310a3f4b2a
+  metadata.gz: b9ce759e247b27af5611e9bdd14e040c717b9736
+  data.tar.gz: 342f5c8542780534e8f930f2f6e4c4a749869406
 SHA512:
-  metadata.gz: fd6abbbf507db891618579ea508736e59b489839420a3f04458843f3ccbb40bf420a86204c51b572e07fc8e837e4002a9b74d8aa58e9461854c48a64eb93091b
-  data.tar.gz: e9778122e8cb4caad8c24c4e5f513844dca18006c9b84c81d4f7e9c8383e1595c3f0f9c49c5f09846b1f5652f7ff37b3d3ee7cd5b6bf91001a37b06f3c86c59d
+  metadata.gz: 3992934eebeb1493bc35ccecd593d73329b64cf6b649db583e7d64f05c0c0bafbaa65e4a2f1ae8daad2bc2b2161592e511c4070cd9b7ce842174af19a50fccad
+  data.tar.gz: 0bdde9d5ef5f89e49ea6dec8cd170b55085c828a46ab1d82c792ec5785e951b4013d700e29567b66787cdd7051c45f0b16605e6c2b65ce0b5fe39438f4ac60a6

data/.travis.yml

@@ -1,7 +1,6 @@
 language: ruby
 sudo: false
 rvm:
-  - 2.0.0
   - 2.1.8
   - 2.2.4
   - 2.3.0

data/Rakefile

@@ -8,4 +8,4 @@ Rake::TestTask.new(:test) do |test|
   test.verbose = true
 end
 
-task :default => :test
+task default: :test

data/fluent-plugin-numeric-counter.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-numeric-counter"
-  gem.version       = "0.3.0"
+  gem.version       = "1.0.0"
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
   gem.description   = %q{Counts messages, with specified key and numeric value in specified range}
@@ -15,8 +15,8 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
 
+  gem.add_runtime_dependency "fluentd", ">= 0.14.0"
   gem.add_development_dependency "rake"
-  gem.add_development_dependency "delorean"
+  gem.add_development_dependency "timecop"
   gem.add_development_dependency "test-unit", ">= 3.1.0"
-  gem.add_runtime_dependency "fluentd", "< 0.14.0"
 end

data/lib/fluent/plugin/out_numeric_counter.rb

@@ -1,59 +1,65 @@
-class Fluent::NumericCounterOutput < Fluent::Output
+require 'fluent/plugin/output'
+require 'pathname'
+
+class Fluent::Plugin::NumericCounterOutput < Fluent::Plugin::Output
   Fluent::Plugin.register_output('numeric_counter', self)
 
-  def initialize
-    super
-    require 'pathname'
-  end
+  helpers :event_emitter, :storage, :timer
 
+  DEFAULT_STORAGE_TYPE = 'local'
   PATTERN_MAX_NUM = 20
 
-  config_param :count_interval, :time, :default => 60,
-               :desc => 'The interval time to count in seconds.'
-  config_param :unit, :string, :default => nil,
-               :desc => <<-DESC
+  config_param :count_interval, :time, default: 60,
+               desc: 'The interval time to count in seconds.'
+  config_param :unit, :enum, list: [:minute, :hour, :day], default: nil,
+               desc: <<-DESC
 The interval time to monitor specified an unit (either of minute, hour, or day).
 Use either of count_interval or unit.
 DESC
-  config_param :output_per_tag, :bool, :default => false,
-               :desc => <<-DESC
-Emit for each input tag.
-tag_prefix must be specified together.
-DESC
-  config_param :aggregate, :string, :default => 'tag',
-               :desc => 'Calculate in each input tag separetely, or all records in a mass.'
-  config_param :tag, :string, :default => 'numcount',
-               :desc => 'The output tag.'
-  config_param :tag_prefix, :string, :default => nil,
-               :desc => <<-DESC
-The prefix string which will be added to the input tag.
-output_per_tag yes must be specified together.
-DESC
-  config_param :input_tag_remove_prefix, :string, :default => nil,
-               :desc => 'The prefix string which will be removed from the input tag.'
+  config_param :output_per_tag, :bool, default: false,
+               desc: 'Produce counter result per input tags.'
+
+  config_param :aggregate, :enum, list: [:tag, :all], default: :tag,
+               desc: 'Calculate in each input tag separetely, or all records in a mass.'
+  config_param :tag, :string, default: 'numcount',
+               desc: 'The output tag.'
+
+  config_param :input_tag_remove_prefix, :string, default: nil,
+               desc: 'The prefix string which will be removed from the input tag.'
   config_param :count_key, :string,
-               :desc => 'The key to count in the event record.'
-  config_param :outcast_unmatched, :bool, :default => false,
-               :desc => <<-DESC
+               desc: 'The key to count in the event record.'
+  config_param :outcast_unmatched, :bool, default: false,
+               desc: <<-DESC
 Specify yes if you do not want to include 'unmatched' counts into percentage.
 DESC
-  config_param :output_messages, :bool, :default => false,
-               :desc => 'Specify yes if you want to get tested messages.'
-  config_param :store_file, :string, :default => nil,
-               :desc => <<-DESC
-Store internal data into a file of the given path on shutdown, and load on starting.
-DESC
+  config_param :output_messages, :bool, default: false,
+               desc: 'Specify yes if you want to get tested messages.'
+
+  config_param :store_file, :string, default: nil,
+               obsoleted: 'Use store_storage parameter instead.',
+               desc: 'Store internal data into a file of the given path on shutdown, and load on starting.'
+  config_param :store_storage, :bool, default: false,
+               desc: 'Store internal data into a storage on shutdown, and load on starting.'
 
   # pattern0 reserved as unmatched counts
   config_param :pattern1, :string,
-               :desc => <<-DESC
+               desc: <<-DESC
 string: NAME LOW HIGH
 LOW/HIGH allows size prefix (ex: 10k, 5M, 3500G)
 Note that pattern0 reserved as unmatched counts.
 DESC
   (2..PATTERN_MAX_NUM).each do |i|
-    config_param ('pattern' + i.to_s).to_sym, :string, :default => nil,
-                 :desc => 'string: NAME LOW HIGH'
+    config_param ('pattern' + i.to_s).to_sym, :string, default: nil,
+                 desc: 'string: NAME LOW HIGH'
+  end
+
+  config_param :tag_prefix, :string, default: nil,
+               desc: 'The prefix string to be added to input tags. Use with "output_per_tag yes".',
+               deprecated: 'Use @label routing instead.'
+
+  config_section :storage do
+    config_set_default :usage, 'resume'
+    config_set_default :@type, DEFAULT_STORAGE_TYPE
   end
 
   attr_accessor :counts
@@ -62,16 +68,6 @@ DESC
   attr_accessor :saved_at
   attr_accessor :patterns
 
-  # Define `log` method for v0.10.42 or earlier
-  unless method_defined?(:log)
-    define_method("log") { $log }
-  end
-
-  # Define `router` method of v0.12 to support v0.10.57 or earlier
-  unless method_defined?(:router)
-    define_method("router") { Fluent::Engine }
-  end
-
   def parse_num(str)
     if str.nil?
       nil
@@ -85,27 +81,26 @@ DESC
   end
 
   def configure(conf)
+    label_routing_specified = conf.has_key?('@label')
+
     super
 
     if @unit
       @count_interval = case @unit
-                        when 'minute' then 60
-                        when 'hour' then 3600
-                        when 'day' then 86400
+                        when :minute then 60
+                        when :hour then 3600
+                        when :day then 86400
                         else
-                          raise Fluent::ConfigError, 'unit must be one of minute/hour/day'
+                          raise "unknown unit:#{@unit}"
                         end
     end
 
-    @aggregate = @aggregate.to_sym
-    raise Fluent::ConfigError, "numeric_counter allows tag/all to aggregate unit" unless [:tag, :all].include?(@aggregate)
-
     @patterns = [[0, 'unmatched', nil, nil]] # counts-index, name, low, high
     pattern_names = ['unmatched']
 
     invalids = conf.keys.select{|k| k =~ /^pattern(\d+)$/ and not (1..PATTERN_MAX_NUM).include?($1.to_i)}
     if invalids.size > 0
-      log.warn "invalid number patterns (valid pattern number:1-#{PATTERN_MAX_NUM}):" + invalids.join(",")
+      log.warn "invalid number patterns (valid pattern number:1-#{PATTERN_MAX_NUM}):", invalids: invalids
     end
     (1..PATTERN_MAX_NUM).each do |i|
       next unless conf["pattern#{i}"]
@@ -124,10 +119,14 @@ DESC
       raise Fluent::ConfigError, "numbers of low/high missing" if low.nil?
       raise Fluent::ConfigError, "unspecified high threshold allowed only in last pattern" if high.nil? and index != @patterns.length - 1
     end
-    
-    if @output_per_tag
-      raise Fluent::ConfigError, "tag_prefix must be specified with output_per_tag" unless @tag_prefix
+
+    if @output_per_tag && (!label_routing_specified && !@tag_prefix)
+      raise Fluent::ConfigError, "specify @label to route output events into other <label> sections."
+    end
+    if @output_per_tag && @tag_prefix
       @tag_prefix_string = @tag_prefix + '.'
+    else
+      @tag_prefix_string = nil
     end
 
     if @input_tag_remove_prefix
@@ -135,28 +134,39 @@ DESC
       @removed_length = @removed_prefix_string.length
     end
 
-    if @store_file
-      f = Pathname.new(@store_file)
-      if (f.exist? && !f.writable_real?) || (!f.exist? && !f.parent.writable_real?)
-        raise Fluent::ConfigError, "#{@store_file} is not writable"
-      end
+    if @store_storage
+      @storage = storage_create(usage: 'resume')
+    end
+
+    if system_config.workers > 1
+      log.warn "Fluentd is now working with multi process workers, and numeric_counter plugin will produce counter results in each separeted processes."
     end
 
     @counts = count_initialized
     @mutex = Mutex.new
   end
 
+  def multi_workers_ready?
+    true
+  end
+
   def start
     super
-    load_status(@store_file, @count_interval) if @store_file
-    start_watch
+
+    load_status(@count_interval) if @store_storage
+
+    @last_checked = Fluent::Engine.now
+
+    timer_execute(:out_numeric_counter_timer, @count_interval) do
+      now = Fluent::Engine.now
+      flush_emit(now - @last_checked)
+      @last_checked = now
+    end
   end
 
   def shutdown
+    save_status() if @store_storage
     super
-    @watcher.terminate
-    @watcher.join
-    save_status(@store_file) if @store_file
   end
 
   def count_initialized(keys=nil)
@@ -217,7 +227,7 @@ DESC
       end
     end
 
-    output    
+    output
   end
 
   def generate_output(counts, step)
@@ -258,7 +268,11 @@ DESC
     if @output_per_tag
       time = Fluent::Engine.now
       flush_per_tags(step).each do |tag,message|
-        router.emit(@tag_prefix_string + tag, time, message)
+        if @tag_prefix_string
+          router.emit(@tag_prefix_string + tag, time, message)
+        else
+          router.emit(tag, time, message)
+        end
       end
     else
       message = flush(step)
@@ -268,23 +282,7 @@ DESC
     end
   end
 
-  def start_watch
-    @watcher = Thread.new(&method(:watch))
-  end
-
-  def watch
-    @last_checked ||= Fluent::Engine.now
-    while true
-      sleep 0.5
-      if Fluent::Engine.now - @last_checked >= @count_interval
-        now = Fluent::Engine.now
-        flush_emit(now - @last_checked)
-        @last_checked = now
-      end
-    end
-  end
-
-  def emit(tag, es, chain)
+  def process(tag, es)
     c = [0] * @patterns.length
 
     es.each do |time,record|
@@ -302,62 +300,57 @@ DESC
       c[0] += 1 unless matched
     end
     countups(tag, c)
-
-    chain.next
   end
 
-  # Store internal status into a file
+  # Store internal status into a storage
   #
-  # @param [String] file_path
-  def save_status(file_path)
+  def save_status()
     begin
-      Pathname.new(file_path).open('wb') do |f|
-        @saved_at = Fluent::Engine.now
-        @saved_duration = @saved_at - @last_checked
-        Marshal.dump({
-          :counts           => @counts,
-          :saved_at         => @saved_at,
-          :saved_duration   => @saved_duration,
-          :aggregate        => @aggregate,
-          :count_key        => @count_key,
-          :patterns         => @patterns,
-        }, f)
-      end
+      @saved_at = Fluent::Engine.now
+      @saved_duration = @saved_at - @last_checked
+      value = {
+        "counts"         => @counts,
+        "saved_at"       => @saved_at,
+        "saved_duration" => @saved_duration,
+        "aggregate"      => @aggregate.to_s,
+        "count_key"      => @count_key,
+        "patterns"       => @patterns,
+      }
+      @storage.put(:stored_value, value)
     rescue => e
-      log.warn "out_datacounter: Can't write store_file #{e.class} #{e.message}"
+      log.warn "Can't write store_storage", error: e
     end
   end
 
-  # Load internal status from a file
+  # Load internal status from a storage
   #
-  # @param [String] file_path
   # @param [Interger] count_interval
-  def load_status(file_path, count_interval)
-    return unless (f = Pathname.new(file_path)).exist?
+  def load_status(count_interval)
+    stored = @storage.get(:stored_value)
+    return unless stored
 
     begin
-      f.open('rb') do |f|
-        stored = Marshal.load(f)
-        if stored[:aggregate] == @aggregate and
-          stored[:count_key] == @count_key and
-          stored[:patterns]  == @patterns
+      if stored["aggregate"] == @aggregate.to_s and
+        stored["count_key"] == @count_key and
+        stored["patterns"] == @patterns
 
-          if Fluent::Engine.now <= stored[:saved_at] + count_interval
-            @counts = stored[:counts]
-            @saved_at = stored[:saved_at]
-            @saved_duration = stored[:saved_duration]
+        if Fluent::Engine.now <= stored["saved_at"] + count_interval
+          @mutex.synchronize {
+            @counts = stored["counts"]
+            @saved_at = stored["saved_at"]
+            @saved_duration = stored["saved_duration"]
 
             # skip the saved duration to continue counting
             @last_checked = Fluent::Engine.now - @saved_duration
-          else
-            log.warn "out_datacounter: stored data is outdated. ignore stored data"
-          end
+          }
         else
-          log.warn "out_datacounter: configuration param was changed. ignore stored data"
+          log.warn "stored data is outdated. ignore stored data"
         end
+      else
+        log.warn "configuration param was changed. ignore stored data"
       end
     rescue => e
-      log.warn "out_datacounter: Can't load store_file #{e.class} #{e.message}"
+      log.warn "Can't load store_storage", error: e
     end
   end
 

data/test/helper.rb

@@ -22,7 +22,6 @@ unless ENV.has_key?('VERBOSE')
   $log = nulllogger
 end
 
-require 'delorean'
 require 'fluent/plugin/out_numeric_counter'
 
 class Test::Unit::TestCase

data/test/plugin/test_out_numeric_counter.rb

@@ -1,10 +1,20 @@
 require 'helper'
+require 'fluent/test/helpers'
+require 'fluent/test/driver/output'
+require 'timecop'
+require 'fileutils'
 
 class NumericCounterOutputTest < Test::Unit::TestCase
+  include Fluent::Test::Helpers
+
   def setup
     Fluent::Test.setup
   end
 
+  def teardown
+    Timecop.return
+  end
+
   CONFIG = %[
     count_interval 60
     aggregate tag
@@ -28,17 +38,17 @@ class NumericCounterOutputTest < Test::Unit::TestCase
     output_messages true
   ]
 
-  def create_driver(conf=CONFIG, tag='test')
-    Fluent::Test::OutputTestDriver.new(Fluent::NumericCounterOutput, tag).configure(conf)
+  def create_driver(conf=CONFIG)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::NumericCounterOutput).configure(conf)
   end
-  
+
   def test_parse_num
     p = create_driver.instance
 
     assert_equal 1, p.parse_num('1')
-    assert_equal -1, p.parse_num('-1')
+    assert_equal (-1), p.parse_num('-1')
     assert_equal 1.0, p.parse_num('1.0')
-    assert_equal -2.0, p.parse_num('-2.0000')
+    assert_equal (-2.0), p.parse_num('-2.0000')
     assert_equal 1024, p.parse_num('1k')
   end
 
@@ -247,7 +257,7 @@ class NumericCounterOutputTest < Test::Unit::TestCase
   end
 
   def test_pattern_num
-    assert_equal 20, Fluent::NumericCounterOutput::PATTERN_MAX_NUM
+    assert_equal 20, Fluent::Plugin::NumericCounterOutput::PATTERN_MAX_NUM
 
     conf = %[
       aggregate all
@@ -256,10 +266,10 @@ class NumericCounterOutputTest < Test::Unit::TestCase
     (1..20).each do |i|
       conf += "pattern#{i} name#{i} #{i} #{i+1}\n"
     end
-    d = create_driver(conf, 'test.max')
-    d.run do
+    d = create_driver(conf)
+    d.run(default_tag: 'test.max') do
       (0..21).each do |i|
-        d.emit({'field' => i})
+        d.feed({'field' => i})
       end
     end
     r = d.instance.flush(60)
@@ -296,14 +306,14 @@ class NumericCounterOutputTest < Test::Unit::TestCase
   #   pattern2 u1s 100000 1000000
   #   pattern3 u3s 1000000 3000000
   # ]
-    d = create_driver(CONFIG, 'test.tag1')
-    d.run do
+    d = create_driver(CONFIG)
+    d.run(default_tag: 'test.tag1') do
       60.times do
-        d.emit({'target' =>  '50000'})
-        d.emit({'target' => '100000'})
-        d.emit({'target' => '100001'})
-        d.emit({'target' => '0.0'})
-        d.emit({'target' => '-1'})
+        d.feed({'target' =>  '50000'})
+        d.feed({'target' => '100000'})
+        d.feed({'target' => '100001'})
+        d.feed({'target' => '0.0'})
+        d.feed({'target' => '-1'})
       end
     end
     r = d.instance.flush(60)
@@ -321,20 +331,20 @@ class NumericCounterOutputTest < Test::Unit::TestCase
     assert_equal 1.0, r['tag1_unmatched_rate']
     assert_equal 20, r['tag1_unmatched_percentage']
 
-    d = create_driver(CONFIG, 'test.tag1')
-    d.run do
+    d = create_driver(CONFIG)
+    d.run(default_tag: 'test.tag1') do
       60.times do
-        d.emit({'target' =>  '50000'})
-        d.emit({'target' => '100000'})
-        d.emit({'target' => '100001'})
-        d.emit({'target' => '0.0'})
-        d.emit({'target' => '-1'})
+        d.feed({'target' =>  '50000'})
+        d.feed({'target' => '100000'})
+        d.feed({'target' => '100001'})
+        d.feed({'target' => '0.0'})
+        d.feed({'target' => '-1'})
       end
+      d.instance.flush_emit(60)
     end
-    d.instance.flush_emit(60)
-    emits = d.emits
-    assert_equal 1, emits.length
-    data = emits[0]
+    events = d.events
+    assert_equal 1, events.length
+    data = events[0]
     assert_equal 'numcount', data[0] # tag
     r = data[2] # message
     assert_equal 120, r['tag1_u100ms_count']
@@ -352,17 +362,18 @@ class NumericCounterOutputTest < Test::Unit::TestCase
   end
 
   def test_emit_output_per_tag
-    d = create_driver(CONFIG_OUTPUT_PER_TAG, 'test.tag1')
-    d.run do
+    d = create_driver(CONFIG_OUTPUT_PER_TAG)
+    r = {}
+    d.run(default_tag: 'test.tag1') do
       60.times do
-        d.emit({'target' =>  '50000'})
-        d.emit({'target' => '100000'})
-        d.emit({'target' => '100001'})
-        d.emit({'target' => '0.0'})
-        d.emit({'target' => '-1'})
+        d.feed({'target' =>  '50000'})
+        d.feed({'target' => '100000'})
+        d.feed({'target' => '100001'})
+        d.feed({'target' => '0.0'})
+        d.feed({'target' => '-1'})
       end
+      r = d.instance.flush_per_tags(60)
     end
-    r = d.instance.flush_per_tags(60)
     assert_equal 1, r.keys.size
     r1 = r['tag1']
     assert_equal 120, r1['u100ms_count']
@@ -379,20 +390,20 @@ class NumericCounterOutputTest < Test::Unit::TestCase
     assert_equal 20, r1['unmatched_percentage']
     assert_equal 300, r1['messages']
 
-    d = create_driver(CONFIG_OUTPUT_PER_TAG, 'test.tag1')
-    d.run do
+    d = create_driver(CONFIG_OUTPUT_PER_TAG)
+    d.run(default_tag: 'test.tag1') do
       60.times do
-        d.emit({'target' =>  '50000'})
-        d.emit({'target' => '100000'})
-        d.emit({'target' => '100001'})
-        d.emit({'target' => '0.0'})
-        d.emit({'target' => '-1'})
+        d.feed({'target' =>  '50000'})
+        d.feed({'target' => '100000'})
+        d.feed({'target' => '100001'})
+        d.feed({'target' => '0.0'})
+        d.feed({'target' => '-1'})
       end
+      d.instance.flush_emit(60)
     end
-    d.instance.flush_emit(60)
-    emits = d.emits
-    assert_equal 1, emits.length
-    data = emits[0]
+    events = d.events
+    assert_equal 1, events.length
+    data = events[0]
     assert_equal 'n.tag1', data[0] # tag
     r = data[2] # message
     assert_equal 120, r['u100ms_count']
@@ -418,7 +429,7 @@ class NumericCounterOutputTest < Test::Unit::TestCase
       ['count', 'rate'].map{|a| p + '_' + a}
     }.flatten
 
-    d = create_driver(CONFIG, 'test.tag1')
+    d = create_driver(CONFIG)
     # CONFIG = %[
     #   count_interval 60
     #   aggregate tag
@@ -428,28 +439,28 @@ class NumericCounterOutputTest < Test::Unit::TestCase
     #   pattern2 u1s 100000 1000000
     #   pattern3 u3s 1000000 3000000
     # ]
-    d.run do
+    d.run(default_tag: 'test.tag1') do
       60.times do
-        d.emit({'target' =>  '50000'})
-        d.emit({'target' => '100000'})
-        d.emit({'target' => '100001'})
-        d.emit({'target' => '0.0'})
-        d.emit({'target' => '-1'})
+        d.feed({'target' =>  '50000'})
+        d.feed({'target' => '100000'})
+        d.feed({'target' => '100001'})
+        d.feed({'target' => '0.0'})
+        d.feed({'target' => '-1'})
       end
+      d.instance.flush_emit(60)
+      assert_equal 1, d.events.size
+      r1 = d.events[0][2]
+      assert_equal fields, r1.keys
+
+      d.instance.flush_emit(60)
+      assert_equal 2, d.events.size # +1
+      r2 = d.events[1][2]
+      assert_equal fields_without_percentage, r2.keys
+      assert_equal [0]*8, r2.values
+
+      d.instance.flush_emit(60)
+      assert_equal 2, d.events.size # +0
     end
-    d.instance.flush_emit(60)
-    assert_equal 1, d.emits.size
-    r1 = d.emits[0][2]
-    assert_equal fields, r1.keys
-
-    d.instance.flush_emit(60)
-    assert_equal 2, d.emits.size # +1
-    r2 = d.emits[1][2]
-    assert_equal fields_without_percentage, r2.keys
-    assert_equal [0]*8, r2.values
-
-    d.instance.flush_emit(60)
-    assert_equal 2, d.emits.size # +0
   end
 
   def test_zero_tags_per_tag
@@ -460,7 +471,7 @@ class NumericCounterOutputTest < Test::Unit::TestCase
         ['count', 'rate'].map{|a| p + '_' + a}
       }.flatten + ['messages']).sort
 
-    d = create_driver(CONFIG_OUTPUT_PER_TAG, 'test.tag1')
+    d = create_driver(CONFIG_OUTPUT_PER_TAG)
     # CONFIG_OUTPUT_PER_TAG = %[
     #   count_interval 60
     #   aggregate tag
@@ -473,44 +484,63 @@ class NumericCounterOutputTest < Test::Unit::TestCase
     #   pattern3 u3s 1000000 3000000
     #   output_messages true
     # ]
-    d.run do
+    d.run(default_tag: 'test.tag1') do
       60.times do
-        d.emit({'target' =>  '50000'})
-        d.emit({'target' => '100000'})
-        d.emit({'target' => '100001'})
-        d.emit({'target' => '0.0'})
-        d.emit({'target' => '-1'})
+        d.feed({'target' =>  '50000'})
+        d.feed({'target' => '100000'})
+        d.feed({'target' => '100001'})
+        d.feed({'target' => '0.0'})
+        d.feed({'target' => '-1'})
       end
+      d.instance.flush_emit(60)
+      assert_equal 1, d.events.size
+      r1 = d.events[0][2]
+      assert_equal fields, r1.keys.sort
+
+      d.instance.flush_emit(60)
+      assert_equal 2, d.events.size # +1
+      r2 = d.events[1][2]
+      assert_equal fields_without_percentage, r2.keys.sort
+      assert_equal [0]*9, r2.values # (_count, _rate) x4 + messages
+
+      d.instance.flush_emit(60)
+      assert_equal 2, d.events.size # +0
     end
-    d.instance.flush_emit(60)
-    assert_equal 1, d.emits.size
-    r1 = d.emits[0][2]
-    assert_equal fields, r1.keys.sort
-
-    d.instance.flush_emit(60)
-    assert_equal 2, d.emits.size # +1
-    r2 = d.emits[1][2]
-    assert_equal fields_without_percentage, r2.keys.sort
-    assert_equal [0]*9, r2.values # (_count, _rate) x4 + messages
-
-    d.instance.flush_emit(60)
-    assert_equal 2, d.emits.size # +0
   end
 
-  def test_store_file
+  def test_store_storage
     dir = "test/tmp"
-    Dir.mkdir dir unless Dir.exist? dir
     file = "#{dir}/test.dat"
-    File.unlink file if File.exist? file
-
+    FileUtils.rm_rf(file)
+    FileUtils.mkdir_p(dir)
+
+    config = {
+      "count_interval" => 60,
+      "aggregate" => "tag",
+      "input_tag_remove_prefix" => "test",
+      "count_key" => " target",
+      "pattern1" => "u100ms 0 100000",
+      "pattern2" => "u1s 100000 1000000",
+      "pattern3" => "u3s 1000000 3000000",
+      "store_storage" => true,
+    }
+    conf = config_element('ROOT', '', config, [
+                            config_element(
+                              'storage', '',
+                              {'@type' => 'local',
+                               '@id' => 'test-01',
+                               'path' => "#{file}",
+                               'persistent' => true,
+                              })
+                          ])
     # test store
-    d = create_driver(CONFIG + %[store_file #{file}])
-    d.run do
+    d = create_driver(conf)
+    time = Fluent::Engine.now
+    d.run(default_tag: 'test') do
       d.instance.flush_emit(60)
-      d.emit({'target' => 1})
-      d.emit({'target' => 1})
-      d.emit({'target' => 1})
-      d.instance.shutdown
+      d.feed(time, {'target' => 1})
+      d.feed(time, {'target' => 1})
+      d.feed(time, {'target' => 1})
     end
     stored_counts = d.instance.counts
     stored_saved_at = d.instance.saved_at
@@ -518,40 +548,42 @@ class NumericCounterOutputTest < Test::Unit::TestCase
     assert File.exist? file
 
     # test load
-    d = create_driver(CONFIG + %[store_file #{file}])
-    d.run do
+    d = create_driver(conf)
+    loaded_counts = 0
+    loaded_saved_at = 0
+    loaded_saved_duration = 0
+    d.run(default_tag: 'test') do
       loaded_counts = d.instance.counts
       loaded_saved_at = d.instance.saved_at
       loaded_saved_duration = d.instance.saved_duration
-      assert_equal stored_counts, loaded_counts
-      assert_equal stored_saved_at, loaded_saved_at
-      assert_equal stored_saved_duration, loaded_saved_duration
     end
+    assert_equal stored_counts, loaded_counts
+    assert_equal stored_saved_at, loaded_saved_at
+    assert_equal stored_saved_duration, loaded_saved_duration
 
     # test not to load if config is changed
-    d = create_driver(CONFIG + %[count_key foobar store_file #{file}])
-    d.run do
+    d = create_driver(conf.merge("count_key" => "foobar", "store_storage" => true))
+    d.run(default_tag: 'test') do
       loaded_counts = d.instance.counts
       loaded_saved_at = d.instance.saved_at
       loaded_saved_duration = d.instance.saved_duration
-      assert_equal({}, loaded_counts)
-      assert_equal(nil, loaded_saved_at)
-      assert_equal(nil, loaded_saved_duration)
     end
+    assert_equal({}, loaded_counts)
+    assert_equal(nil, loaded_saved_at)
+    assert_equal(nil, loaded_saved_duration)
 
     # test not to load if stored data is outdated.
-    Delorean.jump 61 # jump more than count_interval
-    d = create_driver(CONFIG + %[store_file #{file}])
-    d.run do
+    Timecop.freeze(Time.now + 61) # jump more than count_interval
+    d = create_driver(conf.merge("store_storage" => true))
+    d.run(default_tag: 'test') do
       loaded_counts = d.instance.counts
       loaded_saved_at = d.instance.saved_at
       loaded_saved_duration = d.instance.saved_duration
-      assert_equal({}, loaded_counts)
-      assert_equal(nil, loaded_saved_at)
-      assert_equal(nil, loaded_saved_duration)
     end
-    Delorean.back_to_the_present
+    assert_equal({}, loaded_counts)
+    assert_equal(nil, loaded_saved_at)
+    assert_equal(nil, loaded_saved_duration)
 
-    File.unlink file
+    FileUtils.rm_rf(file)
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-numeric-counter
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 1.0.0
 platform: ruby
 authors:
 - TAGOMORI Satoshi
@@ -11,21 +11,21 @@ cert_chain: []
 date: 2017-02-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
+        version: 0.14.0
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.14.0
 - !ruby/object:Gem::Dependency
-  name: delorean
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -39,33 +39,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: test-unit
+  name: timecop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.1.0
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.1.0
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: fluentd
+  name: test-unit
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "<"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.14.0
-  type: :runtime
+        version: 3.1.0
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "<"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.14.0
+        version: 3.1.0
 description: Counts messages, with specified key and numeric value in specified range
 email:
 - tagomoris@gmail.com