fluent-plugin-flowcounter 0.3.2 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 373e8131d63b116d0e7bb4364fd93a19356bb141
-  data.tar.gz: 2000acaab8a219b70da435c4f77469155cbdd300
+  metadata.gz: 416a10332d7ab178acadf8d6894305482d798040
+  data.tar.gz: c6a281878e6a7a9a8ae2cf8449945e8acdd7bbb0
 SHA512:
-  metadata.gz: 715cd4f1dfeb133f64f334608df853f8efc2cf0debd31a87f63642dd0b82cf4c774126935e821a345bce8f348e8aea71a56409646b93c35f6cf51edb72df4a8a
-  data.tar.gz: 96e775ec5a88dc718a21676fe630236b50b7eba183a3bfeac47a0e66883292dd3b29b9ca16a261673eaf14340db56fd5ec48aa1f36341e1f3851568b8d0c5339
+  metadata.gz: 8908f3cab634a1c7b728fd9f469993ba9e4e34194f6d6c6751a40c0d2bcdc1eb0e38317bc1b2f6f72aa489d9693d323690049b93b3c0d894cb552be86c710aac
+  data.tar.gz: e40bc4a1c056d4372df6172e303d65966f1c4d2d0286e444f6d02c2e561a103807962d53f3eb393d6777393cc0a28fdc49e2ca3be9f150516b7f78edd887b54a

data/.travis.yml

@@ -4,3 +4,4 @@ rvm:
   - 2.0.0
   - 2.1
   - 2.2
+  - 2.3.0

data/README.md

@@ -3,9 +3,9 @@
 Count metrics below about matches. This is a plugin for [Fluentd](http://fluentd.org)
 
 * Messages per second/minute/hour/day
-* Bytes per second/minute/hour/day
+* Bytes per second/minute/hour/day (optional)
 * Messages per second (average every second/minute/hour/day)
-* Bytes per second (average every second/minute/hour/day)
+* Bytes per second (average every second/minute/hour/day) (optional)
 
 FlowCounterOutput emits messages contains results data, so you can output these message (with 'flowcount' tag by default) to any outputs you want.
 
@@ -19,17 +19,21 @@ Or, output result data with for each tags (with `output_style tagged`)
 
 `input_tag_remove_prefix` option available if you want to remove tag prefix from output field names.
 
+If you want to count only records, omit `count_keys` configuration.
+
+    {"tag":"test", "count":300, "count_rate":5}
+
 ## Configuration
 
 Counts from fields 'field1' and 'field2', per minute(default), aggregates per tags(default), output with tag 'flowcount'(default).
 
     <match **>
-      type copy
+      @type copy
       <store>
         # original output configurations...
       </store>
       <store>
-        type flowcounter
+        @type flowcounter
         count_keys field1,field2
       </store>
     </match>
@@ -41,12 +45,12 @@ Counts from fields 'field1' and 'field2', per minute(default), aggregates per ta
 Counts from field 'message', per hour, aggregates all tags, output with tag 'fluentd.traffic'.
 
     <match **>
-      type copy
+      @type copy
       <store>
         # original output configurations...
       </store>
       <store>
-        type flowcounter
+        @type flowcounter
         count_keys message
         unit       hour
         aggregate  all
@@ -61,17 +65,26 @@ Counts from field 'message', per hour, aggregates all tags, output with tag 'flu
 To count with all fields in messages, specify 'count_keys *'.
 
     <match target.**>
-      type flowcounter
+      @type flowcounter
       count_keys *
       unit hour
       aggregate all
       tag fluentd.traffic
     </match>
 
+To count records only (without bytes), omit `count_keys` (it runs in better performance.)
+
+    <match target.**>
+      @type flowcounter
+      unit hour
+      aggregate all
+      tag fluentd.traffic
+    </match>
+
 Use '${hostname}' if you want your hostname in tag.
 
     <match target.**>
-      type flowcounter
+      @type flowcounter
       count_keys *
       tag fluentd.node.${hostname}
     </match>

data/fluent-plugin-flowcounter.gemspec

@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |gem|
   gem.name        = "fluent-plugin-flowcounter"
-  gem.version     = "0.3.2"
+  gem.version     = "0.4.0"
   gem.authors     = ["TAGOMORI Satoshi"]
   gem.email       = ["tagomoris@gmail.com"]
   gem.summary     = %q{Fluent plugin to count message flow}

data/lib/fluent/plugin/out_flowcounter.rb

@@ -13,13 +13,13 @@ class Fluent::FlowCounterOutput < Fluent::Output
     define_method("router") { ::Fluent::Engine }
   end
 
-  config_param :unit, :string, :default => 'minute'
-  config_param :aggregate, :string, :default => 'tag'
-  config_param :output_style, :string, :default => 'joined'
-  config_param :tag, :string, :default => 'flowcount'
-  config_param :input_tag_remove_prefix, :string, :default => nil
-  config_param :count_keys, :string
-  config_param :delimiter, :string, :default => '_'
+  config_param :unit, :string, default: 'minute'
+  config_param :aggregate, :string, default: 'tag'
+  config_param :output_style, :string, default: 'joined'
+  config_param :tag, :string, default: 'flowcount'
+  config_param :input_tag_remove_prefix, :string, default: nil
+  config_param :count_keys, :string, default: nil
+  config_param :delimiter, :string, default: '_'
 
   include Fluent::Mixin::ConfigPlaceholders
 
@@ -66,8 +66,13 @@ class Fluent::FlowCounterOutput < Fluent::Output
       @removed_prefix_string = @input_tag_remove_prefix + '.'
       @removed_length = @removed_prefix_string.length
     end
-    @count_keys = @count_keys.split(',')
-    @count_all = (@count_keys == ['*'])
+    if @count_keys
+      @count_keys = @count_keys.split(',')
+      @count_all = (@count_keys == ['*'])
+      @count_bytes = true
+    else
+      @count_bytes = false
+    end
 
     @counts = count_initialized
     @mutex = Mutex.new
@@ -86,7 +91,11 @@ class Fluent::FlowCounterOutput < Fluent::Output
 
   def count_initialized(keys=nil)
     if @aggregate == :all
-      {'count' => 0, 'bytes' => 0}
+      if @count_bytes
+        {'count' => 0, 'bytes' => 0}
+      else
+        {'count' => 0}
+      end
     elsif keys
       values = Array.new(keys.length){|i| 0 }
       Hash[[keys, values].transpose]
@@ -100,11 +109,11 @@ class Fluent::FlowCounterOutput < Fluent::Output
     b = 'bytes'
     if @aggregate == :tag
       c = name + delimiter + 'count'
-      b = name + delimiter + 'bytes'
+      b = name + delimiter + 'bytes' if @count_bytes
     end
     @mutex.synchronize {
       @counts[c] = (@counts[c] || 0) + counts
-      @counts[b] = (@counts[b] || 0) + bytes
+      @counts[b] = (@counts[b] || 0) + bytes if @count_bytes
     }
   end
 
@@ -127,8 +136,10 @@ class Fluent::FlowCounterOutput < Fluent::Output
     names.map {|name|
       counts = {
         'count' => flushed[name + delimiter + 'count'],
-        'bytes' => flushed[name + delimiter + 'bytes'],
       }
+      if @count_bytes
+        counts['bytes'] = flushed[name + delimiter + 'bytes']
+      end
       data = generate_output(counts, step)
       data['tag'] = name
       data
@@ -175,12 +186,12 @@ class Fluent::FlowCounterOutput < Fluent::Output
     if @count_all
       es.each {|time,record|
         c += 1
-        b += record.to_msgpack.bytesize
+        b += record.to_msgpack.bytesize if @count_bytes
       }
     else
       es.each {|time,record|
         c += 1
-        b += @count_keys.inject(0){|s,k| s + (record[k] || FOR_MISSING).bytesize}
+        b += @count_keys.inject(0){|s,k| s + (record[k] || FOR_MISSING).bytesize} if @count_bytes
       }
     end
     countup(name, c, b)

data/test/plugin/test_out_flowcounter.rb

@@ -4,7 +4,7 @@ class FlowCounterOutputTest < Test::Unit::TestCase
   def setup
     Fluent::Test.setup
   end
-  
+
   CONFIG = %[
 unit day
 aggregate tag
@@ -18,9 +18,9 @@ count_keys message
   end
 
   def test_configure
-    assert_raise(Fluent::ConfigError) {
-      d = create_driver('')
-    }
+    d = create_driver('')
+    assert !(d.instance.instance_eval{ @count_bytes })
+
     assert_raise(Fluent::ConfigError) {
       d = create_driver %[
         count_keys message,message2
@@ -235,7 +235,7 @@ count_keys message
     ], 'test.tag2')
     time = Time.now.to_i
     d2.run do
-      60.times do 
+      60.times do
         d2.emit({'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
         d2.emit({'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
         d2.emit({'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
@@ -262,7 +262,7 @@ count_keys message
     ], 'test.tag3')
     time = Time.now.to_i
     d3.run do
-      60.times do 
+      60.times do
         d3.emit({'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
         d3.emit({'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
         d3.emit({'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
@@ -304,6 +304,45 @@ count_keys message
     assert_equal 15.0, r1[0]['bytes_rate']
   end
 
+  def test_emit_not_to_count_bytes
+    d1 = create_driver( %[
+      unit day
+      aggregate tag
+      tag  flowcount
+      input_tag_remove_prefix test
+    ], 'test.tag1')
+    time = Time.parse("2012-01-02 13:14:15").to_i
+    d1.run do
+      3600.times do
+        d1.emit({'message'=> 'a' * 100})
+        d1.emit({'message'=> 'b' * 100})
+        d1.emit({'message'=> 'c' * 100})
+      end
+    end
+    r1 = d1.instance.flush(3600 * 24)
+    assert_equal 3600*3, r1['tag1_count']
+    assert_nil r1['tag1_bytes']
+    assert_equal (300/24.0).floor / 100.0, r1['tag1_count_rate'] # 3 * 3600 / (60 * 60 * 24) as xx.xx
+    assert_nil r1['tag1_bytes_rate']
+
+    d3 = create_driver( %[
+      unit minute
+      aggregate all
+      tag flow
+    ], 'test.tag1')
+    time = Time.parse("2012-01-02 13:14:15").to_i
+    d3.run do
+      60.times do
+        d3.emit({'f1'=>'1'*10, 'f2'=>'2'*20, 'f3'=>'3'*10})
+      end
+    end
+    r3 = d3.instance.flush(60)
+    assert_equal 60, r3['count']
+    assert_nil r3['bytes']
+    assert_equal 1.0, r3['count_rate']
+    assert_nil r3['bytes_rate']
+  end
+
   def test_emit_records_without_specified_field
     d3 = create_driver( %[
       unit minute

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-flowcounter
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.4.0
 platform: ruby
 authors:
 - TAGOMORI Satoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-12-31 00:00:00.000000000 Z
+date: 2016-03-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -105,7 +105,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Fluent plugin to count message flow