fluent-plugin-flowcounter 0.4.3 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8e19ecb9672a7f5b42584af6344941594f39e790
-  data.tar.gz: f8d1c5957950a8bc7f342e3c257bbeda53722fbe
+  metadata.gz: 873629b9804e92822b8a2b854a5df3da2798bf45
+  data.tar.gz: f3d79ae17a63763f9dc0ca8073982022d8b0759e
 SHA512:
-  metadata.gz: 42681b5d601e6862247cd7e81d4b5ccf604d4213b11f08ef6d5888d81017b516808dbd8b641d69d4c89961cce01dfa7d556342daf035d301fea2bd9d60ab6169
-  data.tar.gz: f1bb4e7bae3ede0f43c7c8cb08311b9d4892b5cff9a3b4b98a940549b4c48df294b7f7bb9f2bc3a046fb039cf5ef971740046d826f95f2ea324fd4d667048d6e
+  metadata.gz: 9b92f7be19ad17f125244c10efab034df307f0581f6db4ba82d06551eee513e3002453080221494b8ec5e746601ee3a55a5540f87e93ded866e9222df1391d33
+  data.tar.gz: dd56f20d75dfb69f309170fdb59a520a69da0fca58e5188c46cc98106c8448e1c20d7c00e79ea4fbf98aeccda6835d98ca3caddfe322cb440ea26f8a83958b9f

data/.travis.yml

@@ -4,3 +4,4 @@ rvm:
   - 2.1
   - 2.2
   - 2.3.0
+  - 2.4.1

data/README.md

@@ -25,7 +25,7 @@ If you want to count only records, omit `count_keys` configuration.
 
 ## Configuration
 
-Counts from fields 'field1' and 'field2', per minute(default), aggregates per tags(default), output with tag 'flowcount'(default).
+Counts from fields 'field1' and 'field2', per minute(default), aggregates per tags(default), output with tag 'flowcount'(default). It is strongly recommended to specify `@label` to control event stream routing.
 
     <match **>
       @type copy
@@ -34,13 +34,16 @@ Counts from fields 'field1' and 'field2', per minute(default), aggregates per ta
       </store>
       <store>
         @type flowcounter
+        @label @counts
         count_keys field1,field2
       </store>
     </match>
     
-    <match flowcount>
-      # output configurations where to send count results
-    </match>
+    <label @counts>
+      <match flowcount>
+        # output configurations where to send count results
+      </match>
+    </label>
 
 Counts from field 'message', per hour, aggregates all tags, output with tag 'fluentd.traffic'.
 
@@ -51,6 +54,7 @@ Counts from field 'message', per hour, aggregates all tags, output with tag 'flu
       </store>
       <store>
         @type flowcounter
+        @label @counts
         count_keys message
         unit       hour
         aggregate  all
@@ -58,9 +62,11 @@ Counts from field 'message', per hour, aggregates all tags, output with tag 'flu
       </store>
     </match>
     
-    <match fluentd.traffic>
-      # output configurations where to send count results
-    </match>
+    <label @counts>
+      <match fluentd.traffic>
+        # output configurations where to send count results
+      </match>
+    </label>
 
 To count with all fields in messages, specify 'count_keys *'.
 
@@ -81,27 +87,31 @@ To count records only (without bytes), omit `count_keys` (it runs in better perf
       tag fluentd.traffic
     </match>
 
-Use '${hostname}' if you want your hostname in tag.
+Counts active tag, stop count records if the tag message stoped(when aggragates per tag).
 
     <match target.**>
       @type flowcounter
       count_keys *
-      tag fluentd.node.${hostname}
+      aggregate tag
+      delete_idle true
     </match>
 
-Counts active tag, stop count records if the tag message stoped(when aggragates per tag).
+### Embedding Hostname
+
+The current version of this plugin doesn't support `${hostname}` placeholders. Use ruby code embedding for such purpose:
 
     <match target.**>
       @type flowcounter
       count_keys *
-      aggregate tag
-      delete_idle true
+      tag "fluentd.node.#{Socket.gethostname}"
     </match>
 
+See [Fluentd document page](https://docs.fluentd.org/articles/config-file#embedded-ruby-code) for further details.
+
 ## TODO
 
-* consider what to do next
-* patches welcome!
+* Support Counter API when it's supported in Fluentd core
+* Patches welcome!
 
 ## Copyright
 

data/fluent-plugin-flowcounter.gemspec

@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |gem|
   gem.name        = "fluent-plugin-flowcounter"
-  gem.version     = "0.4.3"
+  gem.version     = "1.0.0"
   gem.authors     = ["TAGOMORI Satoshi"]
   gem.email       = ["tagomoris@gmail.com"]
   gem.summary     = %q{Fluent plugin to count message flow}
@@ -16,6 +16,5 @@ Gem::Specification.new do |gem|
 
   gem.add_development_dependency "rake"
   gem.add_development_dependency "test-unit"
-  gem.add_runtime_dependency "fluentd", ['>= 0.10.59', "< 0.14.0"]
-  gem.add_runtime_dependency "fluent-mixin-config-placeholders", ">= 0.3.0"
+  gem.add_runtime_dependency "fluentd", "~> 0.14.0"
 end

data/lib/fluent/plugin/out_flowcounter.rb

@@ -1,29 +1,19 @@
-require 'fluent/mixin/config_placeholders'
+require 'fluent/plugin/output'
 
-class Fluent::FlowCounterOutput < Fluent::Output
+class Fluent::Plugin::FlowCounterOutput < Fluent::Plugin::Output
   Fluent::Plugin.register_output('flowcounter', self)
 
-  # Define `log` method for v0.10.42 or earlier
-  unless method_defined?(:log)
-    define_method("log") { $log }
-  end
-
-  # Define `router` method of v0.12 to support v0.10 or earlier
-  unless method_defined?(:router)
-    define_method("router") { ::Fluent::Engine }
-  end
+  helpers :event_emitter, :timer
 
-  config_param :unit, :string, default: 'minute'
-  config_param :aggregate, :string, default: 'tag'
-  config_param :output_style, :string, default: 'joined'
+  config_param :unit, :enum, list: [:second, :minute, :hour, :day], default: :minute
+  config_param :aggregate, :enum, list: [:tag, :all], default: :tag
+  config_param :output_style, :enum, list: [:joined, :tagged], default: :joined
   config_param :tag, :string, default: 'flowcount'
   config_param :input_tag_remove_prefix, :string, default: nil
   config_param :count_keys, :string, default: nil
   config_param :delimiter, :string, default: '_'
   config_param :delete_idle, :bool, default: false
 
-  include Fluent::Mixin::ConfigPlaceholders
-
   attr_accessor :counts
   attr_accessor :last_checked
   attr_accessor :count_all
@@ -32,14 +22,6 @@ class Fluent::FlowCounterOutput < Fluent::Output
   def configure(conf)
     super
 
-    @unit = case @unit
-            when 'second' then :second
-            when 'minute' then :minute
-            when 'hour' then :hour
-            when 'day' then :day
-            else
-              raise Fluent::ConfigError, "flowcounter unit allows second/minute/hour/day"
-            end
     @tick = case @unit
             when :second then 1
             when :minute then 60
@@ -48,18 +30,6 @@ class Fluent::FlowCounterOutput < Fluent::Output
             else
               raise Fluent::ConfigError, "flowcounter unit allows second/minute/hour/day"
             end
-    @aggregate = case @aggregate
-                 when 'tag' then :tag
-                 when 'all' then :all
-                 else
-                   raise Fluent::ConfigError, "flowcounter aggregate allows tag/all"
-                 end
-    @output_style = case @output_style
-                 when 'joined' then :joined
-                 when 'tagged' then :tagged
-                 else
-                   raise Fluent::ConfigError, "flowcounter output_style allows joined/tagged"
-                 end
     if @output_style == :tagged and @aggregate != :tag
       raise Fluent::ConfigError, "flowcounter aggregate must be 'tag' when output_style is 'tagged'"
     end
@@ -67,6 +37,7 @@ class Fluent::FlowCounterOutput < Fluent::Output
       @removed_prefix_string = @input_tag_remove_prefix + '.'
       @removed_length = @removed_prefix_string.length
     end
+    @count_all = false
     if @count_keys
       @count_keys = @count_keys.split(',').map(&:strip)
       @count_all = (@count_keys == ['*'])
@@ -84,12 +55,6 @@ class Fluent::FlowCounterOutput < Fluent::Output
     start_watch
   end
 
-  def shutdown
-    super
-    @watcher.terminate
-    @watcher.join
-  end
-
   def count_initialized(keys=nil)
     if @aggregate == :all
       if @count_bytes
@@ -160,26 +125,22 @@ class Fluent::FlowCounterOutput < Fluent::Output
   end
 
   def start_watch
+    @last_checked = Fluent::Engine.now
     # for internal, or tests only
-    @watcher = Thread.new(&method(:watch))
+    timer_execute(:out_flowcounter_watcher, 0.5, &method(:watch))
   end
 
   def watch
-    # instance variable, and public accessable, for test
-    @last_checked = Fluent::Engine.now
-    while true
-      sleep 0.5
-      if Fluent::Engine.now - @last_checked >= @tick
-        now = Fluent::Engine.now
-        flush_emit(now - @last_checked)
-        @last_checked = now
-      end
+    if Fluent::Engine.now - @last_checked >= @tick
+      now = Fluent::Engine.now
+      flush_emit(now - @last_checked)
+      @last_checked = now
     end
   end
 
   FOR_MISSING = ''
 
-  def emit(tag, es, chain)
+  def process(tag, es)
     name = tag
     if @input_tag_remove_prefix and
         ( (tag.start_with?(@removed_prefix_string) and tag.length > @removed_length) or tag == @input_tag_remove_prefix)
@@ -198,7 +159,5 @@ class Fluent::FlowCounterOutput < Fluent::Output
       }
     end
     countup(name, c, b)
-
-    chain.next
   end
 end

data/test/helper.rb

@@ -12,6 +12,7 @@ require 'test/unit'
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 $LOAD_PATH.unshift(File.dirname(__FILE__))
 require 'fluent/test'
+require 'fluent/test/driver/output'
 require 'fluent/plugin/out_flowcounter'
 
 class Test::Unit::TestCase

data/test/plugin/test_out_flowcounter.rb

@@ -13,8 +13,8 @@ input_tag_remove_prefix test
 count_keys message
   ]
 
-  def create_driver(conf=CONFIG,tag='test')
-    Fluent::Test::OutputTestDriver.new(Fluent::FlowCounterOutput, tag).configure(conf)
+  def create_driver(conf=CONFIG)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::FlowCounterOutput).configure(conf)
   end
 
   def test_configure
@@ -128,15 +128,6 @@ count_keys message
     assert d.instance.count_all
   end
 
-  def test_configure_placeholders
-    d = create_driver %[
-      hostname testing.node.local
-      tag test.flowcount.${hostname}
-      count_keys *
-    ]
-    assert_equal 'test.flowcount.testing.node.local', d.instance.tag
-  end
-
   def test_count_initialized
     d = create_driver %[
       aggregate all
@@ -203,13 +194,13 @@ count_keys message
   end
 
   def test_emit
-    d1 = create_driver(CONFIG, 'test.tag1')
+    d1 = create_driver(CONFIG)
     time = Time.parse("2012-01-02 13:14:15").to_i
-    d1.run do
+    d1.run(default_tag: 'test.tag1') do
       3600.times do
-        d1.emit({'message'=> 'a' * 100})
-        d1.emit({'message'=> 'b' * 100})
-        d1.emit({'message'=> 'c' * 100})
+        d1.feed(time, {'message'=> 'a' * 100})
+        d1.feed(time, {'message'=> 'b' * 100})
+        d1.feed(time, {'message'=> 'c' * 100})
       end
     end
     r1 = d1.instance.flush(3600 * 24)
@@ -223,11 +214,11 @@ count_keys message
       aggregate all
       tag flow
       count_keys f1,f2,f3
-    ], 'test.tag1')
+    ])
     time = Time.parse("2012-01-02 13:14:15").to_i
-    d3.run do
+    d3.run(default_tag: 'test.tag1') do
       60.times do
-        d3.emit({'f1'=>'1'*10, 'f2'=>'2'*20, 'f3'=>'3'*10})
+        d3.feed({'f1'=>'1'*10, 'f2'=>'2'*20, 'f3'=>'3'*10})
       end
     end
     r3 = d3.instance.flush(60)
@@ -244,21 +235,21 @@ count_keys message
       tag  flowcount
       input_tag_remove_prefix test
       count_keys f1,f2,f3
-    ], 'test.tag2')
+    ])
     time = Time.now.to_i
-    d2.run do
+    d2.run(default_tag: 'test.tag2') do
       60.times do
-        d2.emit({'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
-        d2.emit({'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
-        d2.emit({'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
-        d2.emit({'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
-        d2.emit({'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
+        d2.feed(time, {'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
+        d2.feed(time, {'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
+        d2.feed(time, {'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
+        d2.feed(time, {'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
+        d2.feed(time, {'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
       end
+      d2.instance.flush_emit(60)
     end
-    d2.instance.flush_emit(60)
-    emits = d2.emits
-    assert_equal 1, emits.length
-    data = emits[0]
+    events = d2.events
+    assert_equal 1, events.length
+    data = events[0]
     assert_equal 'flowcount', data[0] # tag
     assert_equal 60*5, data[2]['count']
     assert_equal 60*5*20, data[2]['bytes']
@@ -271,21 +262,21 @@ count_keys message
       tag  flowcount
       input_tag_remove_prefix test
       count_keys *
-    ], 'test.tag3')
+    ])
     time = Time.now.to_i
-    d3.run do
+    d3.run(default_tag: 'test.tag3') do
       60.times do
-        d3.emit({'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
-        d3.emit({'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
-        d3.emit({'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
-        d3.emit({'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
-        d3.emit({'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
+        d3.feed(time, {'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
+        d3.feed(time, {'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
+        d3.feed(time, {'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
+        d3.feed(time, {'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
+        d3.feed(time, {'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
       end
+      d3.instance.flush_emit(60)
     end
-    d3.instance.flush_emit(60)
-    emits = d3.emits
-    assert_equal 1, emits.length
-    data = emits[0]
+    events = d3.events
+    assert_equal 1, events.length
+    data = events[0]
     assert_equal 'flowcount', data[0] # tag
     assert_equal 60*5, data[2]['count']
     msgpack_size = {'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'}.to_msgpack.bytesize * 5 * 60
@@ -300,11 +291,11 @@ count_keys message
       tag flow
       input_tag_remove_prefix test
       count_keys *
-    ], 'test.tag1')
+    ])
     time = Time.parse("2012-01-02 13:14:15").to_i
-    d1.run do
+    d1.run(default_tag: 'test.tag1') do
       60.times do
-        d1.emit({'message'=> 'hello'})
+        d1.feed(time, {'message'=> 'hello'})
       end
     end
     r1 = d1.instance.tagged_flush(60)
@@ -322,16 +313,17 @@ count_keys message
       aggregate tag
       tag  flowcount
       input_tag_remove_prefix test
-    ], 'test.tag1')
+    ])
     time = Time.parse("2012-01-02 13:14:15").to_i
-    d1.run do
+    r1 = {}
+    d1.run(default_tag: 'test.tag1') do
       3600.times do
-        d1.emit({'message'=> 'a' * 100})
-        d1.emit({'message'=> 'b' * 100})
-        d1.emit({'message'=> 'c' * 100})
+        d1.feed(time, {'message'=> 'a' * 100})
+        d1.feed(time, {'message'=> 'b' * 100})
+        d1.feed(time, {'message'=> 'c' * 100})
       end
+      r1 = d1.instance.flush(3600 * 24)
     end
-    r1 = d1.instance.flush(3600 * 24)
     assert_equal 3600*3, r1['tag1_count']
     assert_nil r1['tag1_bytes']
     assert_equal (300/24.0).floor / 100.0, r1['tag1_count_rate'] # 3 * 3600 / (60 * 60 * 24) as xx.xx
@@ -341,11 +333,11 @@ count_keys message
       unit minute
       aggregate all
       tag flow
-    ], 'test.tag1')
+    ])
     time = Time.parse("2012-01-02 13:14:15").to_i
-    d3.run do
+    d3.run(default_tag: 'test.tag1') do
       60.times do
-        d3.emit({'f1'=>'1'*10, 'f2'=>'2'*20, 'f3'=>'3'*10})
+        d3.feed({'f1'=>'1'*10, 'f2'=>'2'*20, 'f3'=>'3'*10})
       end
     end
     r3 = d3.instance.flush(60)
@@ -362,21 +354,21 @@ count_keys message
       tag  flowcount
       input_tag_remove_prefix test
       count_keys f4
-    ], 'test.tag4')
+    ])
     time = Time.now.to_i
-    d3.run do
+    d3.run(default_tag: 'test.tag4', expect_emits: 1) do
       60.times do
-        d3.emit({'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
-        d3.emit({'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
-        d3.emit({'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
-        d3.emit({'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
-        d3.emit({'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
+        d3.feed(time, {'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
+        d3.feed(time, {'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
+        d3.feed(time, {'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
+        d3.feed(time, {'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
+        d3.feed(time, {'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
       end
+      d3.instance.flush_emit(60)
     end
-    d3.instance.flush_emit(60)
-    emits = d3.emits
-    assert_equal 1, emits.length
-    data = emits[0]
+    events = d3.events
+    assert_equal 1, events.length
+    data = events[0]
     assert_equal 'flowcount', data[0]
     assert_equal 60*5, data[2]['count']
     assert_equal 0, data[2]['bytes']

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-flowcounter
 version: !ruby/object:Gem::Version
-  version: 0.4.3
+  version: 1.0.0
 platform: ruby
 authors:
 - TAGOMORI Satoshi
@@ -42,36 +42,16 @@ dependencies:
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.10.59
-    - - "<"
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.14.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.10.59
-    - - "<"
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.14.0
-- !ruby/object:Gem::Dependency
-  name: fluent-mixin-config-placeholders
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.3.0
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.3.0
 description: Plugin to counts messages/bytes that matches, per minutes/hours/days
 email:
 - tagomoris@gmail.com