fluent-plugin-flowcounter 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 873629b9804e92822b8a2b854a5df3da2798bf45
-  data.tar.gz: f3d79ae17a63763f9dc0ca8073982022d8b0759e
+  metadata.gz: 2d5e0837aff2b5b9122daa256b9eac720886e653
+  data.tar.gz: 5af91ae11eb092d76b87dc3669af751b6141cca7
 SHA512:
-  metadata.gz: 9b92f7be19ad17f125244c10efab034df307f0581f6db4ba82d06551eee513e3002453080221494b8ec5e746601ee3a55a5540f87e93ded866e9222df1391d33
-  data.tar.gz: dd56f20d75dfb69f309170fdb59a520a69da0fca58e5188c46cc98106c8448e1c20d7c00e79ea4fbf98aeccda6835d98ca3caddfe322cb440ea26f8a83958b9f
+  metadata.gz: 95de380ed6362234b5053bd341df61e24040c174f638cc486267a1a98a2a0d286da701f552d37223655726b45859a6499da95f9ec39a7a5d99e7ee8130a54a12
+  data.tar.gz: a4180307283337fb425e9aee62cb4b803c420aef397d1da4022e5d13fbb4d7f8fadb0631100bddb879bb78a184594eac6c8f394d80615a7622cbbe97e8b26b67

data/README.md

@@ -96,6 +96,29 @@ Counts active tag, stop count records if the tag message stoped(when aggragates
       delete_idle true
     </match>
 
+### Generate Output at Zero Time
+
+If you want to generate count results at every 0 second (for unit:minute), at every 00:00 (for unit:hour) or at every 00:00:00 (for unit:day), specify `timestamp_counting true` in your configuration.
+
+    <match target.**>
+      @type flowcounter
+      count_keys *
+      aggregate all
+      unit hour
+      timestamp_counting true
+    </match>
+
+The configuration above emits output at 00:00:00, 01:00:00, 02:00:00, .... every day. In this use case, `unit: day` requires to configure `timestamp_timezone` to set the timezone to determine the beginning of the day.
+
+    <match target.**>
+      @type flowcounter
+      count_keys *
+      aggregate all
+      unit day
+      timestamp_counting true
+      timestamp_timezone -03:00
+    </match>
+
 ### Embedding Hostname
 
 The current version of this plugin doesn't support `${hostname}` placeholders. Use ruby code embedding for such purpose:

data/fluent-plugin-flowcounter.gemspec

@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |gem|
   gem.name        = "fluent-plugin-flowcounter"
-  gem.version     = "1.0.0"
+  gem.version     = "1.1.0"
   gem.authors     = ["TAGOMORI Satoshi"]
   gem.email       = ["tagomoris@gmail.com"]
   gem.summary     = %q{Fluent plugin to count message flow}
@@ -16,5 +16,6 @@ Gem::Specification.new do |gem|
 
   gem.add_development_dependency "rake"
   gem.add_development_dependency "test-unit"
+  gem.add_development_dependency "timecop"
   gem.add_runtime_dependency "fluentd", "~> 0.14.0"
 end

data/lib/fluent/plugin/out_flowcounter.rb

@@ -1,4 +1,5 @@
 require 'fluent/plugin/output'
+require 'time'
 
 class Fluent::Plugin::FlowCounterOutput < Fluent::Plugin::Output
   Fluent::Plugin.register_output('flowcounter', self)
@@ -6,11 +7,13 @@ class Fluent::Plugin::FlowCounterOutput < Fluent::Plugin::Output
   helpers :event_emitter, :timer
 
   config_param :unit, :enum, list: [:second, :minute, :hour, :day], default: :minute
+  config_param :timestamp_counting, :bool, default: false
+  config_param :timestamp_timezone, :string, default: nil
   config_param :aggregate, :enum, list: [:tag, :all], default: :tag
   config_param :output_style, :enum, list: [:joined, :tagged], default: :joined
   config_param :tag, :string, default: 'flowcount'
   config_param :input_tag_remove_prefix, :string, default: nil
-  config_param :count_keys, :string, default: nil
+  config_param :count_keys, :array, value_type: :string, default: []
   config_param :delimiter, :string, default: '_'
   config_param :delete_idle, :bool, default: false
 
@@ -38,21 +41,51 @@ class Fluent::Plugin::FlowCounterOutput < Fluent::Plugin::Output
       @removed_length = @removed_prefix_string.length
     end
     @count_all = false
-    if @count_keys
-      @count_keys = @count_keys.split(',').map(&:strip)
+    if @count_keys && !@count_keys.empty?
       @count_all = (@count_keys == ['*'])
       @count_bytes = true
     else
       @count_bytes = false
     end
 
+    if @timestamp_counting
+      @timestamp_timezone_offset = 0
+      if @unit == :second
+        raise Fluent::ConfigError, "timestamp_counting cannot be enabled with unit: second"
+      elsif @unit == :day
+        unless @timestamp_timezone
+          raise Fluent::ConfigError, "timestamp_counting requires timestamp_timezone to be configured (e.g., '-0700') if unit is day"
+        end
+        @timestamp_timezone_offset = Time.zone_offset(@timestamp_timezone)
+        unless @timestamp_timezone_offset
+          raise Fluent::ConfigError, "invalid timestamp_timezone value (specify like '-0700')"
+        end
+      end
+      @last_checked = nil
+      @initializer = ->{ now = Fluent::EventTime.now.to_i; @last_checked = now - (now % @tick) - @timestamp_timezone_offset }
+      @checker = ->{ Fluent::EventTime.now.to_i - @last_checked >= @tick }
+      @updater = ->{ @last_checked += @tick; return Fluent::EventTime.new(@last_checked, 0), @tick }
+    else
+      @last_checked = nil
+      @initializer = ->{ @last_checked = Fluent::Clock.now }
+      @checker = ->{ Fluent::Clock.now - @last_checked >= @tick }
+      @updater = ->{ prev = @last_checked; @last_checked = Fluent::Clock.now; return Fluent::EventTime.now, @last_checked - prev }
+    end
+
     @counts = count_initialized
     @mutex = Mutex.new
   end
 
   def start
     super
-    start_watch
+
+    @initializer.call
+    timer_execute(:out_flowcounter_watcher, 0.5) do
+      if @checker.call
+        now, interval = @updater.call
+        flush_emit(now, interval)
+      end
+    end
   end
 
   def count_initialized(keys=nil)
@@ -114,27 +147,13 @@ class Fluent::Plugin::FlowCounterOutput < Fluent::Plugin::Output
     }
   end
 
-  def flush_emit(step)
+  def flush_emit(now, step)
     if @output_style == :tagged
       tagged_flush(step).each do |data|
-        router.emit(@tag, Fluent::Engine.now, data)
+        router.emit(@tag, now, data)
       end
     else
-      router.emit(@tag, Fluent::Engine.now, flush(step))
-    end
-  end
-
-  def start_watch
-    @last_checked = Fluent::Engine.now
-    # for internal, or tests only
-    timer_execute(:out_flowcounter_watcher, 0.5, &method(:watch))
-  end
-
-  def watch
-    if Fluent::Engine.now - @last_checked >= @tick
-      now = Fluent::Engine.now
-      flush_emit(now - @last_checked)
-      @last_checked = now
+      router.emit(@tag, now, flush(step))
     end
   end
 

data/test/helper.rb

@@ -17,3 +17,13 @@ require 'fluent/plugin/out_flowcounter'
 
 class Test::Unit::TestCase
 end
+
+def waiting(seconds)
+  begin
+    Timeout.timeout(seconds) do
+      yield
+    end
+  rescue Timeout::Error
+    raise "Timed out with timeout second #{seconds}"
+  end
+end

data/test/plugin/test_out_flowcounter.rb

@@ -1,10 +1,16 @@
 require 'helper'
+require 'timecop'
 
 class FlowCounterOutputTest < Test::Unit::TestCase
   def setup
     Fluent::Test.setup
   end
 
+  def teardown
+    Fluent::Clock.return
+    Timecop.return
+  end
+
   CONFIG = %[
 unit day
 aggregate tag
@@ -245,7 +251,7 @@ count_keys message
         d2.feed(time, {'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
         d2.feed(time, {'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
       end
-      d2.instance.flush_emit(60)
+      d2.instance.flush_emit(Fluent::EventTime.now, 60)
     end
     events = d2.events
     assert_equal 1, events.length
@@ -272,7 +278,7 @@ count_keys message
         d3.feed(time, {'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
         d3.feed(time, {'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
       end
-      d3.instance.flush_emit(60)
+      d3.instance.flush_emit(Fluent::EventTime.now, 60)
     end
     events = d3.events
     assert_equal 1, events.length
@@ -364,7 +370,7 @@ count_keys message
         d3.feed(time, {'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
         d3.feed(time, {'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
       end
-      d3.instance.flush_emit(60)
+      d3.instance.flush_emit(Fluent::EventTime.now, 60)
     end
     events = d3.events
     assert_equal 1, events.length
@@ -373,4 +379,198 @@ count_keys message
     assert_equal 60*5, data[2]['count']
     assert_equal 0, data[2]['bytes']
   end
+
+  data(
+    minute: ["minute", 59, 61, 90, 100, 110, 122],
+    hour: ["hour", 3500, 3601, 3800, 6900, 7180, 7202],
+    day: ["day", 86350, 86401, 86400+20000, 86400+65000, 86400+86350, 86400+86402],
+  )
+  test 'emit timing' do |data|
+    unit, time1, time2, time3, time4, time5, time6 = data
+
+    d = create_driver %[
+      unit #{unit}
+      timestamp_counting false
+      aggregate all
+      tag flowcount
+      count_keys message
+    ]
+
+    start = Fluent::Clock.now
+    Fluent::Clock.freeze(start)
+
+    first_emit_before = nil
+    first_emit_after = nil
+    second_emit_before = nil
+    second_emit_after = nil
+
+    prev_emit_count = 0
+    wait_next_emit = ->{
+      waiting(10) do
+        sleep 0.1 while d.emit_count == prev_emit_count
+        prev_emit_count = d.emit_count
+      end
+    }
+
+    d.run(default_tag: 'test', timeout: 300_000) do
+      3600.times do
+        d.feed({'message'=> 'a' * 100})
+        d.feed({'message'=> 'b' * 100})
+        d.feed({'message'=> 'c' * 100})
+      end
+
+      Fluent::Clock.freeze(start + time1)
+      sleep 2 # nothing emitted here - clock is frozen
+      assert_equal 0, d.emit_count
+
+      3600.times do
+        d.feed({'message'=> 'a' * 100})
+        d.feed({'message'=> 'b' * 100})
+        d.feed({'message'=> 'c' * 100})
+      end
+
+      first_emit_before = Fluent::EventTime.now
+      Fluent::Clock.freeze(start + time2)
+      wait_next_emit.call # emitted from 7200 * 3 events
+      first_emit_after = Fluent::EventTime.now
+
+      Fluent::Clock.freeze(start + time3)
+
+      3600.times do
+        d.feed({'message'=> 'a' * 100})
+        d.feed({'message'=> 'b' * 100})
+        d.feed({'message'=> 'c' * 100})
+      end
+
+      Fluent::Clock.freeze(start + time4)
+      sleep 1
+      assert_equal prev_emit_count, d.emit_count
+
+      Fluent::Clock.freeze(start + time5)
+      sleep 1
+      assert_equal prev_emit_count, d.emit_count
+
+      second_emit_before = Fluent::EventTime.now
+      Fluent::Clock.freeze(start + time6)
+      wait_next_emit.call # emitted from 3600 events
+      second_emit_after = Fluent::EventTime.now
+    end
+
+    events = d.events
+    assert_equal 2, events.size
+    tag, t1, r1 = events[0]
+    assert{ t1 >= first_emit_before }
+    assert{ t1 <= first_emit_after }
+    assert_equal 3600*3*2, r1["count"]
+    tag, t2, r2 = events[1]
+    assert{ t2 >= second_emit_before }
+    assert{ t2 <= second_emit_after }
+    assert_equal 3600*3, r2["count"]
+  end
+
+  data(
+    minute: [
+      "minute",
+      "",
+      Time.utc(2017,7,26,13,49, 3),
+      Time.utc(2017,7,26,13,49,55),
+      Time.utc(2017,7,26,13,50, 0),
+      Time.utc(2017,7,26,13,50, 3),
+      Time.utc(2017,7,26,13,50,49),
+      Time.utc(2017,7,26,13,51, 0)],
+    hour: [
+      "hour",
+      "",
+      Time.utc(2017,7,26,13,49, 3),
+      Time.utc(2017,7,26,13,59,58),
+      Time.utc(2017,7,26,14, 0, 0),
+      Time.utc(2017,7,26,14,15,49),
+      Time.utc(2017,7,26,14,58, 0),
+      Time.utc(2017,7,26,15, 0, 0)],
+    day: [
+      "day",
+      "timestamp_timezone +0000",
+      Time.utc(2017,7,26,13,49, 3),
+      Time.utc(2017,7,26,23,59,58),
+      Time.utc(2017,7,27, 0, 0, 0),
+      Time.utc(2017,7,27, 1,30,10),
+      Time.utc(2017,7,27,23,38,50),
+      Time.utc(2017,7,28, 0, 0, 0)],
+    dayz: [
+      "day",
+      "timestamp_timezone -0700",
+      Time.new(2017,7,26,13,49, 3, "-07:00"),
+      Time.new(2017,7,26,23,59,58, "-07:00"),
+      Time.new(2017,7,27, 0, 0, 0, "-07:00"),
+      Time.new(2017,7,27, 1,30,10, "-07:00"),
+      Time.new(2017,7,27,23,38,50, "-07:00"),
+      Time.new(2017,7,28, 0, 0, 0, "-07:00")],
+  )
+  test 'emit timing with timestamp_counting' do |data|
+    unit, conf, start, time1, time2, time3, time4, time5 = data
+
+    d = create_driver %[
+      unit #{unit}
+      timestamp_counting true
+      #{conf}
+      aggregate all
+      tag flowcount
+      count_keys message
+    ]
+
+    Timecop.freeze(start)
+
+    prev_emit_count = 0
+    wait_next_emit = ->{
+      waiting(10) do
+        sleep 0.1 while d.emit_count == prev_emit_count
+        prev_emit_count = d.emit_count
+      end
+    }
+
+    d.run(default_tag: 'test', timeout: 300_000) do
+      3600.times do
+        d.feed({'message'=> 'a' * 100})
+        d.feed({'message'=> 'b' * 100})
+        d.feed({'message'=> 'c' * 100})
+      end
+
+      Timecop.freeze(time1)
+      sleep 2 # nothing emitted here - time is frozen
+      assert_equal 0, d.emit_count
+
+      3600.times do
+        d.feed({'message'=> 'a' * 100})
+        d.feed({'message'=> 'b' * 100})
+        d.feed({'message'=> 'c' * 100})
+      end
+
+      Timecop.freeze(time2)
+      wait_next_emit.call # emitted from 7200 * 3 events
+
+      Timecop.freeze(time3)
+
+      3600.times do
+        d.feed({'message'=> 'a' * 100})
+        d.feed({'message'=> 'b' * 100})
+        d.feed({'message'=> 'c' * 100})
+      end
+
+      Timecop.freeze(time4)
+      sleep 1
+      assert_equal prev_emit_count, d.emit_count
+
+      Timecop.freeze(time5)
+      wait_next_emit.call # emitted from 3600 events
+    end
+
+    events = d.events
+    assert_equal 2, events.size
+    tag, t1, r1 = events[0]
+    assert_equal Fluent::EventTime.new(time2.to_i, 0), t1
+    assert_equal 3600*3*2, r1["count"]
+    tag, t2, r2 = events[1]
+    assert_equal Fluent::EventTime.new(time5.to_i, 0), t2
+    assert_equal 3600*3, r2["count"]
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-flowcounter
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - TAGOMORI Satoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-07-25 00:00:00.000000000 Z
+date: 2017-07-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -38,6 +38,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement