fluent-plugin-datacounter 0.5.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 91d0b353b01ecba75105027f3ea432cb54659dda
-  data.tar.gz: 07d98018128765756fb478638531f778c5138f50
+  metadata.gz: 658db36e09471b16f21cba480b36b5168818daf4
+  data.tar.gz: e798289a5f33cd2bf8a85f3e8c0efa941d2b1439
 SHA512:
-  metadata.gz: 3a765c1d921fa731f6fdfb5beca42e6522a52585542b1d23a6ad2f07b859e2f2756d8b1407379169c20709ce2ecf07a49446248b78b9f32b674a9d8d26954c24
-  data.tar.gz: 8f4dc5d9acc4ff7f325647c3da6c0af16b36ca0574d990e0b79d34fa6de6e22d08aac8863b2df73e994df20ef058515be6f025ea142f8434dda7ef1b454e6073
+  metadata.gz: 77682c560a84a722ec7b2edb6eb831295ffdfca1d0ba228cd79fefc33ae9a3d93f35ffc26d95a49cd66b54f2b6ecb1215b8527852f0b4a4834f75a56731c34b1
+  data.tar.gz: a575acce7199476d09caff16a83a7323134b869e88f534bc11c9e9c83af5090d11ab0508a2494084a51f3717a81061ce49b86ca79cda2d22e99a87eab81f0357

data/.travis.yml

@@ -1,6 +1,7 @@
 language: ruby
 rvm:
-  - 2.0.0
   - 2.1.8
   - 2.2.4
-
+  - 2.3.3
+before_install:
+  - gem update bundler

data/Rakefile

@@ -7,5 +7,5 @@ Rake::TestTask.new(:test) do |test|
   test.verbose = true
 end
 
-task :default => :test
+task default: :test
 

data/fluent-plugin-datacounter.gemspec

@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |gem|
   gem.name        = "fluent-plugin-datacounter"
-  gem.version     = "0.5.0"
+  gem.version     = "1.0.0"
   gem.authors     = ["TAGOMORI Satoshi"]
   gem.email       = ["tagomoris@gmail.com"]
   gem.homepage    = "https://github.com/tagomoris/fluent-plugin-datacounter"
@@ -14,10 +14,10 @@ Gem::Specification.new do |gem|
   gem.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   gem.require_paths = ["lib"]
 
-  gem.add_runtime_dependency "fluentd", "< 0.14.0"
+  gem.add_runtime_dependency "fluentd", [">= 0.14.8", "< 2"]
 
   gem.add_development_dependency "bundler"
   gem.add_development_dependency "rake"
   gem.add_development_dependency "test-unit"
-  gem.add_development_dependency "delorean"
+  gem.add_development_dependency "timecop"
 end

data/lib/fluent/plugin/out_datacounter.rb

@@ -1,52 +1,51 @@
-class Fluent::DataCounterOutput < Fluent::Output
+require 'pathname'
+require 'fluent/plugin/output'
+
+class Fluent::Plugin::DataCounterOutput < Fluent::Plugin::Output
   Fluent::Plugin.register_output('datacounter', self)
 
-  # Define `log` method for v0.10.42 or earlier
-  unless method_defined?(:log)
-    define_method("log") { $log }
-  end
-
-  # Define `router` method of v0.12 to support v0.10 or earlier
-  unless method_defined?(:router)
-    define_method("router") { Fluent::Engine }
-  end
-
-  def initialize
-    super
-    require 'pathname'
-  end
+  helpers :event_emitter, :storage, :timer
 
+  DEFAULT_STORAGE_TYPE = 'local'
   PATTERN_MAX_NUM = 20
 
-  config_param :count_interval, :time, :default => nil,
-               :desc => 'The interval time to count in seconds.'
-  config_param :unit, :string, :default => 'minute',
-               :desc => 'The interval time to monitor specified an unit (either of minute, hour, or day).'
-  config_param :output_per_tag, :bool, :default => false,
-               :desc => 'Emit for each input tag. tag_prefix must be specified together.'
-  config_param :aggregate, :string, :default => 'tag',
-               :desc => 'Calculate in each input tag separetely, or all records in a mass.'
-  config_param :tag, :string, :default => 'datacount',
-               :desc => 'The output tag.'
-  config_param :tag_prefix, :string, :default => nil,
-               :desc => 'The prefix string which will be added to the input tag.'
-  config_param :input_tag_remove_prefix, :string, :default => nil,
-               :desc => 'The prefix string which will be removed from the input tag.'
+  config_param :count_interval, :time, default: nil,
+               desc: 'The interval time to count in seconds.'
+  config_param :unit, :enum, list: [:minute, :hour, :day], default: :minute,
+               desc: 'The interval time to monitor specified an unit (either of minute, hour, or day).'
+  config_param :output_per_tag, :bool, default: false,
+               desc: 'Emit for each input tag. tag_prefix must be specified together.'
+  config_param :aggregate, :enum, list: [:tag, :all], default: :tag,
+               desc: 'Calculate in each input tag separetely, or all records in a mass.'
+  config_param :tag, :string, default: 'datacount',
+               desc: 'The output tag.'
+  config_param :tag_prefix, :string, default: nil,
+               desc: 'The prefix string which will be added to the input tag.'
+  config_param :input_tag_remove_prefix, :string, default: nil,
+               desc: 'The prefix string which will be removed from the input tag.'
   config_param :count_key, :string,
-               :desc => 'The key to count in the event record.'
-  config_param :outcast_unmatched, :bool, :default => false,
-               :desc => 'Specify yes if you do not want to include \'unmatched\' counts into percentage. '
-  config_param :output_messages, :bool, :default => false,
-               :desc => 'Specify yes if you want to get tested messages.'
-  config_param :store_file, :string, :default => nil,
-               :desc => 'Store internal data into a file of the given path on shutdown, and load on starting.'
+               desc: 'The key to count in the event record.'
+  config_param :outcast_unmatched, :bool, default: false,
+               desc: 'Specify yes if you do not want to include \'unmatched\' counts into percentage. '
+  config_param :output_messages, :bool, default: false,
+               desc: 'Specify yes if you want to get tested messages.'
+  config_param :store_file, :string, default: nil,
+               obsoleted: 'Use store_storage parameter instead.',
+               desc: 'Store internal data into a file of the given path on shutdown, and load on starting.'
+  config_param :store_storage, :bool, default: false,
+               desc: 'Store internal data into a storage on shutdown, and load on starting.'
 
   # pattern0 reserved as unmatched counts
   config_param :pattern1, :string, # string: NAME REGEXP
-               :desc => 'Specify RegexpName and Regexp. format: NAME REGEXP'
+               desc: 'Specify RegexpName and Regexp. format: NAME REGEXP'
   (2..PATTERN_MAX_NUM).each do |i|
-    config_param ('pattern' + i.to_s).to_sym, :string, :default => nil, # NAME REGEXP
-                 :desc => "Specify RegexpName and Regexp. format: NAME REGEXP"
+    config_param ('pattern' + i.to_s).to_sym, :string, default: nil, # NAME REGEXP
+                 desc: "Specify RegexpName and Regexp. format: NAME REGEXP"
+  end
+
+  config_section :storage do
+    config_set_default :usage, 'resume'
+    config_set_default :@type, DEFAULT_STORAGE_TYPE
   end
 
   attr_accessor :tick
@@ -62,28 +61,21 @@ class Fluent::DataCounterOutput < Fluent::Output
       @tick = @count_interval.to_i
     else
       @tick = case @unit
-              when 'minute' then 60
-              when 'hour' then 3600
-              when 'day' then 86400
+              when :minute then 60
+              when :hour then 3600
+              when :day then 86400
               else
-                raise RuntimeError, "@unit must be one of minute/hour/day"
+                raise RuntimeError, "BUG: unknown unit: #{@unit}"
               end
     end
 
-    @aggregate = case @aggregate
-                 when 'tag' then :tag
-                 when 'all' then :all
-                 else
-                   raise Fluent::ConfigError, "datacounter aggregate allows tag/all"
-                 end
-
     @patterns = [[0, 'unmatched', nil]]
     pattern_names = ['unmatched']
 
     pattern_keys = conf.keys.select{|k| k =~ /^pattern(\d+)$/}
     invalids = pattern_keys.select{|arg| arg =~ /^pattern(\d+)/ and not (1..PATTERN_MAX_NUM).include?($1.to_i)}
     if invalids.size > 0
-      log.warn "invalid number patterns (valid pattern number:1-20):" + invalids.join(",")
+      log.warn "invalid number patterns (valid pattern number:1-20)", invalids: invalids
     end
     (1..PATTERN_MAX_NUM).each do |i|
       next unless conf["pattern#{i}"]
@@ -109,28 +101,34 @@ class Fluent::DataCounterOutput < Fluent::Output
       @removed_length = @removed_prefix_string.length
     end
 
-    if @store_file
-      f = Pathname.new(@store_file)
-      if (f.exist? && !f.writable_real?) || (!f.exist? && !f.parent.writable_real?)
-        raise Fluent::ConfigError, "#{@store_file} is not writable"
-      end
+    if @store_storage
+      @storage = storage_create(usage: 'resume')
+    end
+
+    if system_config.workers > 1
+      log.warn "Fluentd is now working with multi process workers, and datacounter plugin will produce counter results in each separeted processes."
     end
 
     @counts = count_initialized
     @mutex = Mutex.new
   end
 
+  def multi_workers_ready?
+    true
+  end
+
   def start
     super
-    load_status(@store_file, @tick) if @store_file
-    start_watch
+
+    load_status(@tick) if @store_storage
+
+    @last_checked = Fluent::Engine.now
+    timer_execute(:out_datacounter_timer, @tick, &method(:watch))
   end
 
   def shutdown
+    save_status() if @store_storage
     super
-    @watcher.terminate
-    @watcher.join
-    save_status(@store_file) if @store_file
   end
 
   def count_initialized(keys=nil)
@@ -235,39 +233,28 @@ class Fluent::DataCounterOutput < Fluent::Output
   end
 
   def flush_emit(step)
+    time = Fluent::Engine.now
     if @output_per_tag
       # tag - message maps
-      time = Fluent::Engine.now
       flush_per_tags(step).each do |tag,message|
         router.emit(@tag_prefix_string + tag, time, message)
       end
     else
       message = flush(step)
       if message.keys.size > 0
-        router.emit(@tag, Fluent::Engine.now, message)
+        router.emit(@tag, time, message)
       end
     end
   end
 
-  def start_watch
-    # for internal, or tests only
-    @watcher = Thread.new(&method(:watch))
-  end
-
   def watch
     # instance variable, and public accessable, for test
-    @last_checked ||= Fluent::Engine.now
-    while true
-      sleep 0.5
-      if Fluent::Engine.now - @last_checked >= @tick
-        now = Fluent::Engine.now
-        flush_emit(now - @last_checked)
-        @last_checked = now
-      end
-    end
+    now = Fluent::Engine.now
+    flush_emit(now - @last_checked)
+    @last_checked = now
   end
 
-  def emit(tag, es, chain)
+  def process(tag, es)
     c = [0] * @patterns.length
 
     es.each do |time,record|
@@ -285,65 +272,72 @@ class Fluent::DataCounterOutput < Fluent::Output
       c[0] += 1 unless matched
     end
     countups(tag, c)
-
-    chain.next
   end
 
-  # Store internal status into a file
+  # Store internal status into a storage
   #
-  # @param [String] file_path
-  def save_status(file_path)
+  def save_status()
     begin
-      Pathname.new(file_path).open('wb') do |f|
-        @saved_at = Fluent::Engine.now
-        @saved_duration = @saved_at - @last_checked
-        Marshal.dump({
-          :counts           => @counts,
-          :saved_at        => @saved_at,
-          :saved_duration  => @saved_duration,
-          :aggregate        => @aggregate,
-          :count_key        => @count_key,
-          :patterns         => @patterns,
-        }, f)
-      end
+      @saved_at = Fluent::Engine.now
+      @saved_duration = @saved_at - @last_checked
+      patterns = @patterns.map{|idx, label, regexp|
+        if regexp
+          [idx, label, regexp.source]
+        else
+          [idx, label, regexp]
+        end
+      }
+      value = {
+        "counts"         => @counts,
+        "saved_at"       => @saved_at,
+        "saved_duration" => @saved_duration,
+        "aggregate"      => @aggregate,
+        "count_key"      => @count_key.to_s,
+        "patterns"       => patterns,
+      }
+      @storage.put(:stored_value, value)
     rescue => e
-      log.warn "out_datacounter: Can't write store_file #{e.class} #{e.message}"
+      log.warn "Can't write store_storage", error: e
     end
   end
 
-  # Load internal status from a file
+  # Load internal status from a storage
   #
-  # @param [String] file_path
   # @param [Interger] tick The count interval
-  def load_status(file_path, tick)
-    return unless (f = Pathname.new(file_path)).exist?
+  def load_status(tick)
+    stored = @storage.get(:stored_value)
+    return unless stored
 
     begin
-      f.open('rb') do |f|
-        stored = Marshal.load(f)
-        if stored[:aggregate] == @aggregate and
-          stored[:count_key] == @count_key and
-          stored[:patterns]  == @patterns
-
-          if Fluent::Engine.now <= stored[:saved_at] + tick
-            @mutex.synchronize {
-              @counts = stored[:counts]
-              @saved_at = stored[:saved_at]
-              @saved_duration = stored[:saved_duration]
-
-              # skip the saved duration to continue counting
-              @last_checked = Fluent::Engine.now - @saved_duration
-            }
-          else
-            log.warn "out_datacounter: stored data is outdated. ignore stored data"
-          end
+      patterns = stored["patterns"].map{|idx, label, regexp|
+        if regexp
+          [idx, label, Regexp.compile(regexp)]
         else
-          log.warn "out_datacounter: configuration param was changed. ignore stored data"
+          [idx, label, regexp]
         end
+      }
+
+      if stored["aggregate"] == @aggregate.to_s and
+        stored["count_key"] == @count_key and
+        patterns == @patterns
+
+        if Fluent::Engine.now <= stored["saved_at"] + tick
+          @mutex.synchronize {
+            @counts = stored["counts"]
+            @saved_at = stored["saved_at"]
+            @saved_duration = stored["saved_duration"]
+
+            # skip the saved duration to continue counting
+            @last_checked = Fluent::Engine.now - @saved_duration
+          }
+        else
+          log.warn "stored data is outdated. ignore stored data"
+        end
+      else
+        log.warn "configuration param was changed. ignore stored data"
       end
     rescue => e
-      log.warn "out_datacounter: Can't load store_file #{e.class} #{e.message}"
+      log.warn "Can't load store_storage", error: e
     end
   end
-
 end

data/test/helper.rb

@@ -22,7 +22,6 @@ unless ENV.has_key?('VERBOSE')
   $log = nulllogger
 end
 
-require 'delorean'
 require 'fluent/plugin/out_datacounter'
 
 class Test::Unit::TestCase

data/test/plugin/test_out_datacounter.rb

@@ -1,10 +1,22 @@
 require 'helper'
+require 'fluent/test/driver/output'
+require 'fileutils'
+require 'timecop'
+
 
 class DataCounterOutputTest < Test::Unit::TestCase
   def setup
     Fluent::Test.setup
   end
 
+  def teardown
+    Timecop.return
+  end
+
+  def config_element(name = 'test', argument = '', params = {}, elements = [])
+    Fluent::Config::Element.new(name, argument, params, elements)
+  end
+
   CONFIG = %[
     unit minute
     aggregate tag
@@ -30,39 +42,39 @@ class DataCounterOutputTest < Test::Unit::TestCase
     output_messages yes
   ]
 
-  def create_driver(conf = CONFIG, tag='test.input')
-    Fluent::Test::OutputTestDriver.new(Fluent::DataCounterOutput, tag).configure(conf)
+  def create_driver(conf = CONFIG)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::DataCounterOutput).configure(conf)
   end
 
   def test_configure
     assert_raise(Fluent::ConfigError) {
-      d = create_driver('')
+      create_driver('')
     }
     assert_raise(Fluent::ConfigError) {
-      d = create_driver %[
+      create_driver %[
         count_key field
       ]
     }
     assert_raise(Fluent::ConfigError) {
-      d = create_driver %[
+      create_driver %[
         pattern1 hoge ^1\\d\\d$
       ]
     }
     assert_raise(Fluent::ConfigError) {
-      d = create_driver %[
+      create_driver %[
         count_key field
         pattern2 hoge ^1\\d\\d$
       ]
     }
     assert_raise(Fluent::ConfigError) {
-      d = create_driver %[
+      create_driver %[
         count_key field
         pattern1 hoge ^1\\d\\d$
         pattern4 pos  ^4\\d\\d$
       ]
     }
     assert_raise(Fluent::ConfigError) {
-      d = create_driver %[
+      create_driver %[
         count_key field
         pattern1 hoge ^1\\d\\d$
         pattern2 hoge ^4\\d\\d$
@@ -91,7 +103,7 @@ class DataCounterOutputTest < Test::Unit::TestCase
       pattern1 ok ^2\\d\\d$
     ]
     assert_equal d1.instance.tick, d2.instance.tick
-    
+
     d = create_driver %[
       count_interval 5m
       count_key field
@@ -296,7 +308,7 @@ class DataCounterOutputTest < Test::Unit::TestCase
   end
 
   def test_pattern_num
-    assert_equal 20, Fluent::DataCounterOutput::PATTERN_MAX_NUM
+    assert_equal 20, Fluent::Plugin::DataCounterOutput::PATTERN_MAX_NUM
 
     conf = %[
       aggregate all
@@ -305,10 +317,10 @@ class DataCounterOutputTest < Test::Unit::TestCase
     (1..20).each do |i|
       conf += "pattern#{i} name#{i} ^#{i}$\n"
     end
-    d = create_driver(conf, 'test.max')
-    d.run do
+    d = create_driver(conf)
+    d.run(default_tag: 'test.max') do
       (1..20).each do |j|
-        d.emit({'field' => j})
+        d.feed({'field' => j})
       end
     end
     r = d.instance.flush(60)
@@ -335,13 +347,13 @@ class DataCounterOutputTest < Test::Unit::TestCase
   end
 
   def test_emit
-    d1 = create_driver(CONFIG, 'test.tag1')
-    d1.run do
+    d1 = create_driver(CONFIG)
+    d1.run(default_tag: 'test.tag1') do
       60.times do
-        d1.emit({'target' => '200'})
-        d1.emit({'target' => '100'})
-        d1.emit({'target' => '200'})
-        d1.emit({'target' => '400'})
+        d1.feed({'target' => '200'})
+        d1.feed({'target' => '100'})
+        d1.feed({'target' => '200'})
+        d1.feed({'target' => '400'})
       end
     end
     r1 = d1.instance.flush(60)
@@ -352,11 +364,11 @@ class DataCounterOutputTest < Test::Unit::TestCase
     assert_equal 60, r1['tag1_status4xx_count']
     assert_equal 1.0, r1['tag1_status4xx_rate']
     assert_equal 25.0, r1['tag1_status4xx_percentage']
-    
+
     assert_equal 60, r1['tag1_unmatched_count']
     assert_equal 1.0, r1['tag1_unmatched_rate']
     assert_equal 25.0, r1['tag1_unmatched_percentage']
-    
+
     assert_equal 0, r1['tag1_status3xx_count']
     assert_equal 0.0, r1['tag1_status3xx_rate']
     assert_equal 0.0, r1['tag1_status3xx_percentage']
@@ -370,17 +382,17 @@ class DataCounterOutputTest < Test::Unit::TestCase
       pattern1 ok 2\\d\\d
       pattern2 redirect 3\\d\\d
       output_messages yes
-    ], 'test.tag2')
-    d2.run do
+    ])
+    d2.run(default_tag: 'test.tag2') do
       60.times do
-        d2.emit({'target' => '200'})
-        d2.emit({'target' => '300 200'})
+        d2.feed({'target' => '200'})
+        d2.feed({'target' => '300 200'})
       end
+      d2.instance.flush_emit(120)
     end
-    d2.instance.flush_emit(120)
-    emits = d2.emits
-    assert_equal 1, emits.length
-    data = emits[0]
+    events = d2.events
+    assert_equal 1, events.length
+    data = events[0]
     assert_equal 'datacount', data[0] # tag
     assert_equal 120, data[2]['ok_count']
     assert_equal 1.0, data[2]['ok_rate']
@@ -399,18 +411,18 @@ class DataCounterOutputTest < Test::Unit::TestCase
       pattern1 ok 2\\d\\d
       pattern2 redirect 3\\d\\d
       outcast_unmatched yes
-    ], 'test.tag2')
-    d3.run do
+    ])
+    d3.run(default_tag: 'test.tag2') do
       60.times do
-        d3.emit({'target' => '200'})
-        d3.emit({'target' => '300'})
-        d3.emit({'target' => '400'})
+        d3.feed({'target' => '200'})
+        d3.feed({'target' => '300'})
+        d3.feed({'target' => '400'})
       end
+      d3.instance.flush_emit(180)
     end
-    d3.instance.flush_emit(180)
-    emits = d3.emits
-    assert_equal 1, emits.length
-    data = emits[0]
+    events = d3.events
+    assert_equal 1, events.length
+    data = events[0]
     assert_equal 'datacount', data[0] # tag
     assert_equal 60, data[2]['tag2_unmatched_count']
     assert_nil data[2]['tag2_unmatched_percentage']
@@ -426,18 +438,18 @@ class DataCounterOutputTest < Test::Unit::TestCase
       pattern2 redirect 3\\d\\d
       outcast_unmatched true
       output_messages true
-    ], 'test.tag2')
-    d3.run do
+    ])
+    d3.run(default_tag: 'test.tag2') do
       60.times do
-        d3.emit({'target' => '200'})
-        d3.emit({'target' => '300'})
-        d3.emit({'target' => '400'})
+        d3.feed({'target' => '200'})
+        d3.feed({'target' => '300'})
+        d3.feed({'target' => '400'})
       end
+      d3.instance.flush_emit(180)
     end
-    d3.instance.flush_emit(180)
-    emits = d3.emits
-    assert_equal 1, emits.length
-    data = emits[0]
+    events = d3.events
+    assert_equal 1, events.length
+    data = events[0]
     assert_equal 'datacount', data[0] # tag
     assert_equal 60, data[2]['unmatched_count']
     assert_nil data[2]['unmatched_percentage']
@@ -449,13 +461,13 @@ class DataCounterOutputTest < Test::Unit::TestCase
   end
 
   def test_emit_output_per_tag
-    d1 = create_driver(CONFIG_OUTPUT_PER_TAG, 'test.tag1')
-    d1.run do
+    d1 = create_driver(CONFIG_OUTPUT_PER_TAG)
+    d1.run(default_tag: 'test.tag1') do
       60.times do
-        d1.emit({'target' => '200'})
-        d1.emit({'target' => '100'})
-        d1.emit({'target' => '200'})
-        d1.emit({'target' => '400'})
+        d1.feed({'target' => '200'})
+        d1.feed({'target' => '100'})
+        d1.feed({'target' => '200'})
+        d1.feed({'target' => '400'})
       end
     end
     r1 = d1.instance.flush_per_tags(60)
@@ -468,11 +480,11 @@ class DataCounterOutputTest < Test::Unit::TestCase
     assert_equal 60, r['status4xx_count']
     assert_equal 1.0, r['status4xx_rate']
     assert_equal 25.0, r['status4xx_percentage']
-    
+
     assert_equal 60, r['unmatched_count']
     assert_equal 1.0, r['unmatched_rate']
     assert_equal 25.0, r['unmatched_percentage']
-    
+
     assert_equal 0, r['status3xx_count']
     assert_equal 0.0, r['status3xx_rate']
     assert_equal 0.0, r['status3xx_percentage']
@@ -489,17 +501,17 @@ class DataCounterOutputTest < Test::Unit::TestCase
       pattern2 redirect 3\\d\\d
       output_per_tag yes
       tag_prefix d
-    ], 'test.tag2')
-    d2.run do
+    ])
+    d2.run(default_tag: 'test.tag2') do
       60.times do
-        d2.emit({'target' => '200'})
-        d2.emit({'target' => '300 200'})
+        d2.feed({'target' => '200'})
+        d2.feed({'target' => '300 200'})
       end
+      d2.instance.flush_emit(120)
     end
-    d2.instance.flush_emit(120)
-    emits = d2.emits
-    assert_equal 1, emits.length
-    data = emits[0]
+    events = d2.events
+    assert_equal 1, events.length
+    data = events[0]
     assert_equal 'd.all', data[0] # tag
     assert_equal 120, data[2]['ok_count']
     assert_equal 1.0, data[2]['ok_rate']
@@ -519,18 +531,18 @@ class DataCounterOutputTest < Test::Unit::TestCase
       outcast_unmatched yes
       output_per_tag yes
       tag_prefix d
-    ], 'test.tag2')
-    d3.run do
+    ])
+    d3.run(default_tag: 'test.tag2') do
       60.times do
-        d3.emit({'target' => '200'})
-        d3.emit({'target' => '300'})
-        d3.emit({'target' => '400'})
+        d3.feed({'target' => '200'})
+        d3.feed({'target' => '300'})
+        d3.feed({'target' => '400'})
       end
+      d3.instance.flush_emit(180)
     end
-    d3.instance.flush_emit(180)
-    emits = d3.emits
-    assert_equal 1, emits.length
-    data = emits[0]
+    events = d3.events
+    assert_equal 1, events.length
+    data = events[0]
     assert_equal 'd.tag2', data[0] # tag
     assert_equal 60, data[2]['unmatched_count']
     assert_nil data[2]['unmatched_percentage']
@@ -547,18 +559,18 @@ class DataCounterOutputTest < Test::Unit::TestCase
       outcast_unmatched true
       output_per_tag yes
       tag_prefix ddd
-    ], 'test.tag2')
-    d3.run do
+    ])
+    d3.run(default_tag: 'test.tag2') do
       60.times do
-        d3.emit({'target' => '200'})
-        d3.emit({'target' => '300'})
-        d3.emit({'target' => '400'})
+        d3.feed({'target' => '200'})
+        d3.feed({'target' => '300'})
+        d3.feed({'target' => '400'})
       end
+      d3.instance.flush_emit(180)
     end
-    d3.instance.flush_emit(180)
-    emits = d3.emits
-    assert_equal 1, emits.length
-    data = emits[0]
+    events = d3.events
+    assert_equal 1, events.length
+    data = events[0]
     assert_equal 'ddd.all', data[0] # tag
     assert_equal 60, data[2]['unmatched_count']
     assert_nil data[2]['unmatched_percentage']
@@ -576,7 +588,7 @@ class DataCounterOutputTest < Test::Unit::TestCase
       ['count', 'rate'].map{|a| p + '_' + a}
     }.flatten
 
-    d = create_driver(CONFIG, 'test.tag1')
+    d = create_driver(CONFIG)
     # CONFIG = %[
     #   unit minute
     #   aggregate tag
@@ -587,27 +599,27 @@ class DataCounterOutputTest < Test::Unit::TestCase
     #   pattern3 status4xx ^4\\d\\d$
     #   pattern4 status5xx ^5\\d\\d$
     # ]
-    d.run do
+    d.run(default_tag: 'test.tag1') do
       60.times do
-        d.emit({'target' => '200'})
-        d.emit({'target' => '100'})
-        d.emit({'target' => '200'})
-        d.emit({'target' => '400'})
+        d.feed({'target' => '200'})
+        d.feed({'target' => '100'})
+        d.feed({'target' => '200'})
+        d.feed({'target' => '400'})
       end
+      d.instance.flush_emit(60)
+      assert_equal 1, d.events.size
+      r1 = d.events[0][2]
+      assert_equal fields, r1.keys
+
+      d.instance.flush_emit(60)
+      assert_equal 2, d.events.size # +1
+      r2 = d.events[1][2]
+      assert_equal fields_without_percentage, r2.keys
+      assert_equal [0]*10, r2.values
+
+      d.instance.flush_emit(60)
+      assert_equal 2, d.events.size # +0
     end
-    d.instance.flush_emit(60)
-    assert_equal 1, d.emits.size
-    r1 = d.emits[0][2]
-    assert_equal fields, r1.keys
-
-    d.instance.flush_emit(60)
-    assert_equal 2, d.emits.size # +1
-    r2 = d.emits[1][2]
-    assert_equal fields_without_percentage, r2.keys
-    assert_equal [0]*10, r2.values
-
-    d.instance.flush_emit(60)
-    assert_equal 2, d.emits.size # +0
   end
   def test_zer_tags_per_tag
     fields = (['unmatched','status2xx','status3xx','status4xx','status5xx'].map{|p|
@@ -617,7 +629,7 @@ class DataCounterOutputTest < Test::Unit::TestCase
       ['count', 'rate'].map{|a| p + '_' + a}
     }.flatten + ['messages']).sort
 
-    d = create_driver(CONFIG_OUTPUT_PER_TAG, 'test.tag1')
+    d = create_driver(CONFIG_OUTPUT_PER_TAG)
     # CONFIG_OUTPUT_PER_TAG = %[
     #   unit minute
     #   aggregate tag
@@ -631,84 +643,99 @@ class DataCounterOutputTest < Test::Unit::TestCase
     #   pattern4 status5xx ^5\\d\\d$
     #   output_messages yes
     # ]
-    d.run do
+    d.run(default_tag: 'test.tag1') do
       60.times do
-        d.emit({'target' => '200'})
-        d.emit({'target' => '100'})
-        d.emit({'target' => '200'})
-        d.emit({'target' => '400'})
+        d.feed({'target' => '200'})
+        d.feed({'target' => '100'})
+        d.feed({'target' => '200'})
+        d.feed({'target' => '400'})
       end
+      d.instance.flush_emit(60)
+      assert_equal 1, d.events.size
+      r1 = d.events[0][2]
+      assert_equal fields, r1.keys.sort
+
+      d.instance.flush_emit(60)
+      assert_equal 2, d.events.size # +1
+      r2 = d.events[1][2]
+      assert_equal fields_without_percentage, r2.keys.sort
+      assert_equal [0]*11, r2.values # (_count, _rate)x5 + messages
+
+      d.instance.flush_emit(60)
+      assert_equal 2, d.events.size # +0
     end
-    d.instance.flush_emit(60)
-    assert_equal 1, d.emits.size
-    r1 = d.emits[0][2]
-    assert_equal fields, r1.keys.sort
-
-    d.instance.flush_emit(60)
-    assert_equal 2, d.emits.size # +1
-    r2 = d.emits[1][2]
-    assert_equal fields_without_percentage, r2.keys.sort
-    assert_equal [0]*11, r2.values # (_count, _rate)x5 + messages
-
-    d.instance.flush_emit(60)
-    assert_equal 2, d.emits.size # +0
   end
 
-  def test_store_file
+  def test_store_storage
     dir = "test/tmp"
     Dir.mkdir dir unless Dir.exist? dir
-    file = "#{dir}/test.dat"
-    File.unlink file if File.exist? file
-
+    storage_path = "#{dir}/test.dat"
+    FileUtils.rm_rf(storage_path)
+
+    config = {
+      "unit" =>  "minute",
+      "aggregate" => "tag",
+      "input_tag_remove_prefix" => "test",
+      "count_key" =>  " target",
+      "pattern1" => "status2xx ^2\\d\\d$",
+      "pattern2" => "status3xx ^3\\d\\d$",
+      "pattern3" => "status4xx ^4\\d\\d$",
+      "pattern4" => "status5xx ^5\\d\\d$",
+      "store_storage" => true
+    }
+    storage_conf = config_element('storage', 'resume', {'@type' => 'local', '@id' => 'test-01', 'path' => storage_path, 'persistent' => 'true'})
+    conf = config_element('ROOT', '', config, [storage_conf])
     # test store
-    d = create_driver(CONFIG + %[store_file #{file}])
-    d.run do
+    d = create_driver(conf)
+    time = Fluent::Engine.now
+    d.run(default_tag: 'test.input') do
       d.instance.flush_emit(60)
-      d.emit({'target' => 1})
-      d.emit({'target' => 1})
-      d.emit({'target' => 1})
-      d.instance.shutdown
+      d.feed(time, {'target' => 1})
+      d.feed(time, {'target' => 1})
+      d.feed(time, {'target' => 1})
     end
     stored_counts = d.instance.counts
     stored_saved_at = d.instance.saved_at
     stored_saved_duration = d.instance.saved_duration
-    assert File.exist? file
+    assert File.exist?(storage_path)
 
     # test load
-    d = create_driver(CONFIG + %[store_file #{file}])
-    d.run do
+    d = create_driver(conf)
+    loaded_counts = {}
+    loaded_saved_at = nil
+    loaded_saved_duration = nil
+    d.run(default_tag: 'test.input') do
       loaded_counts = d.instance.counts
       loaded_saved_at = d.instance.saved_at
       loaded_saved_duration = d.instance.saved_duration
-      assert_equal stored_counts, loaded_counts
-      assert_equal stored_saved_at, loaded_saved_at
-      assert_equal stored_saved_duration, loaded_saved_duration
     end
+    assert_equal stored_counts, loaded_counts
+    assert_equal stored_saved_at, loaded_saved_at
+    assert_equal stored_saved_duration, loaded_saved_duration
 
     # test not to load if config is changed
-    d = create_driver(CONFIG + %[count_key foobar store_file #{file}])
-    d.run do
+    d = create_driver(conf.merge("count_key" => "foobar", "store_storage" => true))
+    d.run(default_tag: 'test.input') do
       loaded_counts = d.instance.counts
       loaded_saved_at = d.instance.saved_at
       loaded_saved_duration = d.instance.saved_duration
-      assert_equal({}, loaded_counts)
-      assert_equal(nil, loaded_saved_at)
-      assert_equal(nil, loaded_saved_duration)
     end
+    assert_equal({}, loaded_counts)
+    assert_equal(nil, loaded_saved_at)
+    assert_equal(nil, loaded_saved_duration)
 
     # test not to load if stored data is outdated.
-    Delorean.jump 61 # jump more than count_interval
-    d = create_driver(CONFIG + %[store_file #{file}])
-    d.run do
+    Timecop.freeze(Time.now + 61) # jump more than count_interval
+    d = create_driver(conf.merge("store_storage" => true))
+    d.run(default_tag: 'test.input') do
       loaded_counts = d.instance.counts
       loaded_saved_at = d.instance.saved_at
       loaded_saved_duration = d.instance.saved_duration
-      assert_equal({}, loaded_counts)
-      assert_equal(nil, loaded_saved_at)
-      assert_equal(nil, loaded_saved_duration)
     end
-    Delorean.back_to_the_present
+    assert_equal({}, loaded_counts)
+    assert_equal(nil, loaded_saved_at)
+    assert_equal(nil, loaded_saved_duration)
 
-    File.unlink file
+    FileUtils.rm_rf(storage_path)
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-datacounter
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 1.0.0
 platform: ruby
 authors:
 - TAGOMORI Satoshi
@@ -14,16 +14,22 @@ dependencies:
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.8
     - - "<"
       - !ruby/object:Gem::Version
-        version: 0.14.0
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.8
     - - "<"
       - !ruby/object:Gem::Version
-        version: 0.14.0
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -67,7 +73,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: delorean
+  name: timecop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="