fluent-plugin-flowcounter 0.1.8 → 0.1.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bfdce38cafd1a126c924aaf6e5d77c2e2c3d63da
-  data.tar.gz: 05cda19ad49378a4207a69d0a05c1dcb11e2542b
+  metadata.gz: 3639330b2bfa7435b5a737363733c630d12a8d6c
+  data.tar.gz: f32013615c744fa27fcd6e2014ef055753c99ec9
 SHA512:
-  metadata.gz: 685895bfeb32f9e77a6b2a4cc960eff33be04ed060d119186e8e7fdd0976dbfc93300d9797066dc3b7befc0635d29e62a737b948a79e378fccc70b6d3271603c
-  data.tar.gz: ff5151d8a32dab10545d168ada66a5a36a9fd36941908c561c85394b1586b7a83d502efc6502cf9e80e849666eb7cf8ed98eed98797bf173c32d87d858c31cd4
+  metadata.gz: 4c9b9cbde88a539fac3d12dcc6983f33061b1e9c1e39f36cef2ac8e109726b683a9b343ff2123619f2d72a273741b205547f8025349be80270438b34254e6911
+  data.tar.gz: 526a506b799a7167e88543281fe2f5c5547614b09018a0c8b8ef2a45281a315a5093fdf8e55be231db5a16aaa06f1a158a215d64beb2891d3d5638a5d3cfab99

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+rvm:
+  - 1.9.3
+  - 2.0.0

data/{README.rdoc → README.md}

@@ -1,26 +1,25 @@
-= fluent-plugin-flowcounter
-
-== Component
-
-=== FlowCounterOutput
+# fluent-plugin-flowcounter
 
 Count metricses below about matches.
 
-- Messages per minute/hour/day
-- Bytes per minute/hour/day
-- Messages per second (average every minute/hour/day)
-- Bytes per second (average every minute/hour/day)
+* Messages per minute/hour/day
+* Bytes per minute/hour/day
+* Messages per second (average every minute/hour/day)
+* Bytes per second (average every minute/hour/day)
 
 FlowCounterOutput emits messages contains results data, so you can output these message (with 'flowcount' tag by default) to any outputs you want.
 
     output ex1 (aggragates all inputs): {"count":300, "bytes":3660, "count_rate":5, "bytes_rate":61}
     output ex2 (aggragates per tag): {"test_count":300, "test_bytes":3660, "test_count_rate":5, "test_bytes_rate":61, "service1_count":180, "service1_bytes":7260, "service1_count_rate":3, "service1_bytes_rate":121}
 
-'input_tag_remove_prefix' option available if you want to remove tag prefix from output field names.
+Or, output result data with for each tags (with `output_style tagged`)
+
+    {"tag":"test", "count":300, "bytes":3660, "count_rate":5, "bytes_rate":61}
+    {"tag":"service1", "count":180, "bytes":7260, "count_rate":3, "bytes_rate":121}
 
-== Configuration
+`input_tag_remove_prefix` option available if you want to remove tag prefix from output field names.
 
-=== FlowCounterOutput
+## Configuration
 
 Counts from fields 'field1' and 'field2', per minute(default), aggregates per tags(default), output with tag 'flowcount'(default).
 
@@ -77,12 +76,14 @@ Use '${hostname}' if you want your hostname in tag.
       tag fluentd.node.${hostname}
     </match>
 
-== TODO
+## TODO
 
-- consider what to do next
-- patches welcome!
+* consider what to do next
+* patches welcome!
 
-== Copyright
+## Copyright
 
-Copyright:: Copyright (c) 2012- TAGOMORI Satoshi (tagomoris)
-License::   Apache License, Version 2.0
+* Copyright
+  * Copyright (c) 2012- TAGOMORI Satoshi (tagomoris)
+* License
+  * Apache License, Version 2.0

data/fluent-plugin-flowcounter.gemspec

@@ -1,12 +1,13 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |gem|
   gem.name        = "fluent-plugin-flowcounter"
-  gem.version     = "0.1.8"
+  gem.version     = "0.1.9"
   gem.authors     = ["TAGOMORI Satoshi"]
   gem.email       = ["tagomoris@gmail.com"]
   gem.summary     = %q{Fluent plugin to count message flow}
   gem.description = %q{Plugin to counts messages/bytes that matches, per minutes/hours/days}
   gem.homepage    = "https://github.com/tagomoris/fluent-plugin-flowcounter"
+  gem.license     = "APLv2"
 
   gem.files         = `git ls-files`.split($\)
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
@@ -14,7 +15,6 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
 
   gem.add_development_dependency "rake"
-  gem.add_development_dependency "shoulda"
   gem.add_runtime_dependency "fluentd"
   gem.add_runtime_dependency "fluent-mixin-config-placeholders"
 end

data/lib/fluent/plugin/out_flowcounter.rb

@@ -5,6 +5,7 @@ class Fluent::FlowCounterOutput < Fluent::Output
 
   config_param :unit, :string, :default => 'minute'
   config_param :aggregate, :string, :default => 'tag'
+  config_param :output_style, :string, :default => 'joined'
   config_param :tag, :string, :default => 'flowcount'
   config_param :input_tag_remove_prefix, :string, :default => nil
   config_param :count_keys, :string
@@ -31,6 +32,15 @@ class Fluent::FlowCounterOutput < Fluent::Output
                  else
                    raise Fluent::ConfigError, "flowcounter aggregate allows tag/all"
                  end
+    @output_style = case @output_style
+                 when 'joined' then :joined
+                 when 'tagged' then :tagged
+                 else
+                   raise Fluent::ConfigError, "flowcounter output_style allows joined/tagged"
+                 end
+    if @output_style == :tagged and @aggregate != :tag
+      raise Fluent::ConfigError, "flowcounter aggregate must be 'tag' when output_style is 'tagged'"
+    end
     if @input_tag_remove_prefix
       @removed_prefix_string = @input_tag_remove_prefix + '.'
       @removed_length = @removed_prefix_string.length
@@ -90,8 +100,28 @@ class Fluent::FlowCounterOutput < Fluent::Output
     generate_output(flushed, step)
   end
 
+  def tagged_flush(step)
+    flushed,@counts = @counts,count_initialized(@counts.keys)
+    names = flushed.keys.select {|x| x.end_with?('_count')}.map {|x| x.chomp('_count')}
+    names.map {|name|
+      counts = {
+        'count' => flushed[name + '_count'],
+        'bytes' => flushed[name + '_bytes'],
+      }
+      data = generate_output(counts, step)
+      data['tag'] = name
+      data
+    }
+  end
+
   def flush_emit(step)
-    Fluent::Engine.emit(@tag, Fluent::Engine.now, flush(step))
+    if @output_style == :tagged
+      tagged_flush(step).each do |data|
+        Fluent::Engine.emit(@tag, Fluent::Engine.now, data)
+      end
+    else
+      Fluent::Engine.emit(@tag, Fluent::Engine.now, flush(step))
+    end
   end
 
   def start_watch

data/test/helper.rb

@@ -8,7 +8,6 @@ rescue Bundler::BundlerError => e
   exit e.status_code
 end
 require 'test/unit'
-require 'shoulda'
 
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 $LOAD_PATH.unshift(File.dirname(__FILE__))

data/test/plugin/test_out_flowcounter.rb

@@ -32,6 +32,7 @@ count_keys message
     ]
     assert_equal :minute, d.instance.unit
     assert_equal :tag, d.instance.aggregate
+    assert_equal :joined, d.instance.output_style
     assert_equal 'flowcount', d.instance.tag
     assert_nil d.instance.input_tag_remove_prefix
     assert_equal ['message'], d.instance.count_keys
@@ -41,6 +42,7 @@ count_keys message
     ]
     assert_equal :minute, d.instance.unit
     assert_equal :tag, d.instance.aggregate
+    assert_equal :joined, d.instance.output_style
     assert_equal 'flowcount', d.instance.tag
     assert_nil d.instance.input_tag_remove_prefix
     assert_equal ['field1', 'field2'], d.instance.count_keys
@@ -51,6 +53,18 @@ count_keys message
     ]
     assert_equal :hour, d.instance.unit
     assert_equal :tag, d.instance.aggregate
+    assert_equal :joined, d.instance.output_style
+    assert_equal 'flowcount', d.instance.tag
+    assert_nil d.instance.input_tag_remove_prefix
+    assert_equal ['message'], d.instance.count_keys
+
+    d = create_driver %[
+      output_style tagged
+      count_keys message
+    ]
+    assert_equal :minute, d.instance.unit
+    assert_equal :tag, d.instance.aggregate
+    assert_equal :tagged, d.instance.output_style
     assert_equal 'flowcount', d.instance.tag
     assert_nil d.instance.input_tag_remove_prefix
     assert_equal ['message'], d.instance.count_keys
@@ -64,6 +78,7 @@ count_keys message
     ]
     assert_equal :day, d.instance.unit
     assert_equal :all, d.instance.aggregate
+    assert_equal :joined, d.instance.output_style
     assert_equal 'test.flowcount', d.instance.tag
     assert_equal 'test', d.instance.input_tag_remove_prefix
     assert_equal ['message'], d.instance.count_keys
@@ -77,6 +92,7 @@ count_keys message
     ]
     assert_equal :day, d.instance.unit
     assert_equal :all, d.instance.aggregate
+    assert_equal :joined, d.instance.output_style
     assert_equal 'test.flowcount', d.instance.tag
     assert_equal 'test', d.instance.input_tag_remove_prefix
     assert d.instance.count_all
@@ -245,4 +261,28 @@ count_keys message
     msgpack_size = {'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'}.to_msgpack.bytesize * 5 * 60
     assert_equal msgpack_size, data[2]['bytes']
   end
+
+  def test_emit_tagged
+    d1 = create_driver( %[
+      unit minute
+      aggregate tag
+      output_style tagged
+      tag flow
+      input_tag_remove_prefix test
+      count_keys *
+    ], 'test.tag1')
+    time = Time.parse("2012-01-02 13:14:15").to_i
+    d1.run do
+      60.times do
+        d1.emit({'message'=> 'hello'})
+      end
+    end
+    r1 = d1.instance.tagged_flush(60)
+    assert_equal 1, r1.length
+    assert_equal 'tag1', r1[0]['tag']
+    assert_equal 60, r1[0]['count']
+    assert_equal 60*15, r1[0]['bytes']
+    assert_equal 1.0, r1[0]['count_rate']
+    assert_equal 15.0, r1[0]['bytes_rate']
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-flowcounter
 version: !ruby/object:Gem::Version
-  version: 0.1.8
+  version: 0.1.9
 platform: ruby
 authors:
 - TAGOMORI Satoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-04-10 00:00:00.000000000 Z
+date: 2013-09-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -24,20 +24,6 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: shoulda
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
@@ -75,17 +61,19 @@ extra_rdoc_files: []
 files:
 - .document
 - .gitignore
+- .travis.yml
 - AUTHORS
 - Gemfile
 - LICENSE.txt
-- README.rdoc
+- README.md
 - Rakefile
 - fluent-plugin-flowcounter.gemspec
 - lib/fluent/plugin/out_flowcounter.rb
 - test/helper.rb
 - test/plugin/test_out_flowcounter.rb
 homepage: https://github.com/tagomoris/fluent-plugin-flowcounter
-licenses: []
+licenses:
+- APLv2
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -103,10 +91,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.0
+rubygems_version: 2.0.3
 signing_key: 
 specification_version: 4
 summary: Fluent plugin to count message flow
 test_files:
 - test/helper.rb
 - test/plugin/test_out_flowcounter.rb
+has_rdoc: