fluent-plugin-flowcounter 0.1.0

data/.document

@@ -0,0 +1,5 @@
+lib/**/*.rb
+bin/*
+- 
+features/**/*.feature
+LICENSE.txt

data/.gitignore

@@ -0,0 +1,51 @@
+# rcov generated
+coverage
+
+# rdoc generated
+rdoc
+
+# yard generated
+doc
+.yardoc
+
+# bundler
+.bundle
+
+# jeweler generated
+pkg
+
+# Have editor/IDE/OS specific files you need to ignore? Consider using a global gitignore: 
+#
+# * Create a file at ~/.gitignore
+# * Include files you want ignored
+# * Run: git config --global core.excludesfile ~/.gitignore
+#
+# After doing this, these files will be ignored in all your git projects,
+# saving you from having to 'pollute' every project you touch with them
+#
+# Not sure what to needs to be ignored for particular editors/OSes? Here's some ideas to get you started. (Remember, remove the leading # of the line)
+#
+# For MacOS:
+#
+.DS_Store
+
+# For TextMate
+*.tmproj
+tmtags
+
+# For emacs:
+*~
+\#*
+.\#*
+
+# For vim:
+*.swp
+
+# For redcar:
+#.redcar
+
+# For rubinius:
+#*.rbc
+
+Gemfile.lock
+vendor

data/AUTHORS

@@ -0,0 +1 @@
+TAGOMORI Satoshi <tagomoris _at_ gmail.com>

data/Gemfile

@@ -0,0 +1,16 @@
+source "http://rubygems.org"
+# Add dependencies required to use your gem here.
+# Example:
+#   gem "activesupport", ">= 2.3.5"
+
+# Add dependencies to develop your gem here.
+# Include everything needed to run rake, tests, features, etc.
+group :development do
+  gem "shoulda", ">= 0"
+  gem "bundler", "~> 1.0.0"
+  gem "jeweler", "~> 1.6.4"
+  gem "simplecov", ">= 0"
+  gem "rdoc"
+end
+
+gem "fluentd"

data/LICENSE.txt

@@ -0,0 +1,13 @@
+Copyright (c) 2012- TAGOMORI Satoshi
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/README.rdoc

@@ -0,0 +1,70 @@
+= fluent-plugin-flowcounter
+
+== Component
+
+=== FlowCounterOutput
+
+Count metricses below about matches.
+
+- Messages per min/hour/day
+- Bytes per min/hour/day
+- Messages per second (average every min/hour/day)
+- Bytes per second (average every min/hour/day)
+
+FlowCounterOutput emits messages contains results data, so you can output these message (with 'flowcount' tag by default) to any outputs you want.
+
+    output ex1 (aggragates all inputs): {"count":300, "bytes":3660, "count_rate":5, "bytes_rate":61}
+    output ex2 (aggragates per tag): {"test_count":300, "test_bytes":3660, "test_count_rate":5, "test_bytes_rate":61, "service1_count":180, "service1_bytes":7260, "service1_count_rate":3, "service1_bytes_rate":121}
+
+'input_tag_remove_prefix' option available if you want to remove tag prefix from output field names.
+
+== Configuration
+
+=== FlowCounterOutput
+
+Counts from fields 'field1' and 'field2', per minute(default), aggregates per tags(default), output with tag 'flowcount'(default).
+
+    <match **>
+      copy
+      <store>
+        # original output configurations...
+      </store>
+      <store>
+        type flowcounter
+        count_keys field1,field2
+      </store>
+    </match>
+    
+    <match flowcount>
+      # output configurations where to send count results
+    </match>
+
+Counts from field 'message', per hour, aggregates all tags, output with tag 'fluentd.traffic'.
+
+    <match **>
+      copy
+      <store>
+        # original output configurations...
+      </store>
+      <store>
+        type flowcounter
+        count_keys message
+        unit       hour
+        aggregate  all
+        tag        fluentd.traffic
+      </store>
+    </match>
+    
+    <match flowcount>
+      # output configurations where to send count results
+    </match>
+
+== TODO
+
+- consider what to do next
+- patches welcome!
+
+== Copyright
+
+Copyright:: Copyright (c) 2012- TAGOMORI Satoshi (tagomoris)
+License::   Apache License, Version 2.0

data/Rakefile

@@ -0,0 +1,52 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "fluent-plugin-flowcounter"
+  gem.description = "Plugin to counts messages/bytes that matches, per minutes/hours/days"
+  gem.homepage = "http://github.com/tagomoris/fluent-plugin-flowcounter"
+  gem.summary = gem.description
+  gem.email = "tagomoris@gmail.com"
+  gem.authors = ["TAGOMORI Satoshi"]
+  gem.has_rdoc = "false"
+  # dependencies defined in Gemfile
+  gem.files       = `git ls-files`.split("\n")
+  gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
+  gem.add_dependency "fluentd", "~> 0.10.8"
+  gem.add_development_dependency "rake", ">= 0.9.2"
+  gem.add_development_dependency "simplecov", ">= 0.5.4"
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test
+
+require 'rdoc/task'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "fluent-plugin-flowcounter #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/lib/fluent/plugin/out_flowcounter.rb

@@ -0,0 +1,128 @@
+class Fluent::FlowCounterOutput < Fluent::Output
+  Fluent::Plugin.register_output('flowcounter', self)
+
+  config_param :unit, :string, :default => 'minute'
+  config_param :aggregate, :string, :default => 'tag'
+  config_param :tag, :string, :default => 'flowcount'
+  config_param :input_tag_remove_prefix, :string, :default => nil
+  config_param :count_keys, :string
+
+  attr_accessor :counts
+  attr_accessor :last_checked
+
+  def configure(conf)
+    super
+
+    @unit = case @unit
+            when 'minute' then :minute
+            when 'hour' then :hour
+            when 'day' then :day
+            else
+              raise Fluent::ConfigError, "flowcounter unit allows minute/hour/day"
+            end
+    @aggregate = case @aggregate
+                 when 'tag' then :tag
+                 when 'all' then :all
+                 else
+                   raise Fluent::ConfigError, "flowcounter aggregate allows tag/all"
+                 end
+    if @input_tag_remove_prefix
+      @removed_prefix_string = @input_tag_remove_prefix + '.'
+      @removed_length = @removed_prefix_string.length
+    end
+    @count_keys = @count_keys.split(',')
+
+    @counts = count_initialized
+  end
+
+  def start
+    super
+    @counts = count_initialized
+    start_watch
+  end
+
+  def shutdown
+    super
+    @watcher.terminate
+    @watcher.join
+  end
+
+  def count_initialized(keys=nil)
+    if @aggregate == :all
+      {'count' => 0, 'bytes' => 0}
+    elsif keys
+      Hash[[keys, [0]*(keys.length)].transpose]
+    else
+      {}
+    end
+  end
+
+  def countup(name, counts, bytes)
+    c = 'count'
+    b = 'bytes'
+    if @aggregate == :tag
+      c = name + '_count'
+      b = name + '_bytes'
+    end
+    @counts[c] = (@counts[c] || 0) + counts
+    @counts[b] = (@counts[b] || 0) + bytes
+  end
+
+  def generate_output(counts, step)
+    rates = {}
+    counts.keys.each {|key|
+      rates[key + '_rate'] = ((counts[key] * 100.0) / (1.00 * step)).floor / 100.0
+    }
+    counts.update(rates)
+  end
+
+  def flush(step)
+    flushed,@counts = @counts,count_initialized(@counts.keys)
+    generate_output(flushed, step)
+  end
+
+  def flush_emit(step)
+    Fluent::Engine.emit(@tag, Fluent::Engine.now, flush(step))
+  end
+
+  def start_watch
+    # for internal, or tests only
+    @watcher = Thread.new(&method(:watch))
+  end
+
+  def watch
+    # instance variable, and public accessable, for test
+    @last_checked = Fluent::Engine.now
+    tick = case @unit
+           when :minute then 60
+           when :hour then 3600
+           when :day then 86400
+           else
+             raise RuntimeError, "@unit must be one of minute/hour/day"
+           end
+    while true
+      sleep 0.5
+      if Fluent::Engine.now - @last_checked >= tick
+        now = Fluent::Engine.now
+        flush_emit(now - @last_checked)
+        @last_checked = now
+      end
+    end
+  end
+
+  def emit(tag, es, chain)
+    name = tag
+    if @input_tag_remove_prefix and
+        ( (tag.start_with?(@removed_prefix_string) and tag.length > @removed_length) or tag == @input_tag_remove_prefix)
+      name = tag[@removed_length..-1]
+    end
+    c,b = 0,0
+    es.each {|time,record|
+      c += 1
+      b += @count_keys.inject(0){|s,k| s + record[k].bytesize}
+    }
+    countup(name, c, b)
+
+    chain.next
+  end
+end

data/test/helper.rb

@@ -0,0 +1,19 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+require 'shoulda'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'fluent/test'
+require 'fluent/plugin/out_flowcounter'
+
+class Test::Unit::TestCase
+end

data/test/plugin/test_out_flowcounter.rb

@@ -0,0 +1,199 @@
+require 'helper'
+
+class FlowCounterOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+  
+  CONFIG = %[
+unit day
+aggregate tag
+tag  flowcount
+input_tag_remove_prefix test
+count_keys message
+  ]
+
+  def create_driver(conf=CONFIG,tag='test')
+    Fluent::Test::OutputTestDriver.new(Fluent::FlowCounterOutput, tag).configure(conf)
+  end
+
+  def test_configure
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver('')
+    }
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver %[
+        count_keys message,message2
+        unit week
+      ]
+    }
+    d = create_driver %[
+      count_keys message
+    ]
+    assert_equal :minute, d.instance.unit
+    assert_equal :tag, d.instance.aggregate
+    assert_equal 'flowcount', d.instance.tag
+    assert_nil d.instance.input_tag_remove_prefix
+    assert_equal ['message'], d.instance.count_keys
+
+    d = create_driver %[
+      count_keys field1,field2
+    ]
+    assert_equal :minute, d.instance.unit
+    assert_equal :tag, d.instance.aggregate
+    assert_equal 'flowcount', d.instance.tag
+    assert_nil d.instance.input_tag_remove_prefix
+    assert_equal ['field1', 'field2'], d.instance.count_keys
+
+    d = create_driver %[
+      unit hour
+      count_keys message
+    ]
+    assert_equal :hour, d.instance.unit
+    assert_equal :tag, d.instance.aggregate
+    assert_equal 'flowcount', d.instance.tag
+    assert_nil d.instance.input_tag_remove_prefix
+    assert_equal ['message'], d.instance.count_keys
+
+    d = create_driver %[
+      unit day
+      aggregate all
+      tag test.flowcount
+      input_tag_remove_prefix test
+      count_keys message
+    ]
+    assert_equal :day, d.instance.unit
+    assert_equal :all, d.instance.aggregate
+    assert_equal 'test.flowcount', d.instance.tag
+    assert_equal 'test', d.instance.input_tag_remove_prefix
+    assert_equal ['message'], d.instance.count_keys
+  end
+
+  def test_count_initialized
+    d = create_driver %[
+      aggregate all
+      count_keys f1,f2,f3
+    ]
+    assert_equal 0, d.instance.counts['count']
+    assert_equal 0, d.instance.counts['bytes']
+  end
+
+  def test_countup
+    d = create_driver
+    assert_nil d.instance.counts['message_count']
+    assert_nil d.instance.counts['message_bytes']
+
+    d.instance.countup('message', 30, 50)
+    assert_equal 30, d.instance.counts['message_count']
+    assert_equal 50, d.instance.counts['message_bytes']
+
+    d.instance.countup('message', 10, 70)
+    assert_equal 40, d.instance.counts['message_count']
+    assert_equal 120, d.instance.counts['message_bytes']
+
+    d = create_driver %[
+      aggregate all
+      count_keys message,field
+    ]
+    assert_equal 0, d.instance.counts['count']
+    assert_equal 0, d.instance.counts['bytes']
+
+    d.instance.countup('message', 30, 50)
+    assert_equal 30, d.instance.counts['count']
+    assert_equal 50, d.instance.counts['bytes']
+
+    d.instance.countup('field', 10, 70)
+    assert_equal 40, d.instance.counts['count']
+    assert_equal 120, d.instance.counts['bytes']
+  end
+
+  def test_generate_output
+    d = create_driver %[
+       unit minute
+       count_keys message
+    ]
+    r1 = d.instance.generate_output({'count' => 600, 'bytes' => 18000}, 60)
+    assert_equal 10.00, r1['count_rate']
+    assert_equal 300.00, r1['bytes_rate']
+    r2 = d.instance.generate_output({'count' => 100, 'bytes' => 1000}, 60)
+    assert_equal 1.66, r2['count_rate']
+    assert_equal 16.66, r2['bytes_rate']
+
+    d = create_driver %[
+      unit hour
+      count_keys f1,f2
+    ]
+    r3 = d.instance.generate_output({'xx_count' => 1800, 'xx_bytes' => 600000}, 3600)
+    assert_equal 0.50, r3['xx_count_rate']
+    assert_equal 166.66, r3['xx_bytes_rate']
+
+    r4 = d.instance.generate_output({'t1_count' => 7200, 't1_bytes' => 14400, 't2_count' => 14400, 't2_bytes' => 288000}, 3600)
+    assert_equal 2.00, r4['t1_count_rate']
+    assert_equal 4.00, r4['t1_bytes_rate']
+    assert_equal 4.00, r4['t2_count_rate']
+    assert_equal 80.00, r4['t2_bytes_rate']
+  end
+
+  def test_emit
+    d1 = create_driver(CONFIG, 'test.tag1')
+    time = Time.parse("2012-01-02 13:14:15").to_i
+    d1.run do
+      3600.times do
+        d1.emit({'message'=> 'a' * 100})
+        d1.emit({'message'=> 'b' * 100})
+        d1.emit({'message'=> 'c' * 100})
+      end
+    end
+    r1 = d1.instance.flush(3600 * 24)
+    assert_equal 3600*3, r1['tag1_count']
+    assert_equal 3600*3*100, r1['tag1_bytes']
+    assert_equal (300/24.0).floor / 100.0, r1['tag1_count_rate'] # 3 * 3600 / (60 * 60 * 24) as xx.xx
+    assert_equal (30000/24.0).floor / 100.0, r1['tag1_bytes_rate'] # 300 * 3600 / (60 * 60 * 24) xx.xx
+
+    d3 = create_driver( %[
+      unit minute
+      aggregate all
+      tag flow
+      count_keys f1,f2,f3
+    ], 'test.tag1')
+    time = Time.parse("2012-01-02 13:14:15").to_i
+    d3.run do
+      60.times do
+        d3.emit({'f1'=>'1'*10, 'f2'=>'2'*20, 'f3'=>'3'*10})
+      end
+    end
+    r3 = d3.instance.flush(60)
+    assert_equal 60, r3['count']
+    assert_equal 60*40, r3['bytes']
+    assert_equal 1.0, r3['count_rate']
+    assert_equal 40.0, r3['bytes_rate']
+  end
+
+  def test_emit2
+    d2 = create_driver( %[
+      unit minute
+      aggregate all
+      tag  flowcount
+      input_tag_remove_prefix test
+      count_keys f1,f2,f3
+    ], 'test.tag2')
+    time = Time.now.to_i
+    d2.run do
+      60.times do 
+        d2.emit({'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
+        d2.emit({'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
+        d2.emit({'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
+        d2.emit({'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
+        d2.emit({'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
+      end
+    end
+    d2.instance.flush_emit(60)
+    emits = d2.emits
+    assert_equal 1, emits.length
+    p emits[0].to_json
+    data = emits[0]
+    assert_equal 'flowcount', data[0] # tag
+    assert_equal 60*5, data[2]['count']
+    assert_equal 60*5*20, data[2]['bytes']
+  end
+end

metadata

@@ -0,0 +1,161 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-flowcounter
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- TAGOMORI Satoshi
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-02-18 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: &2152296260 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *2152296260
+- !ruby/object:Gem::Dependency
+  name: shoulda
+  requirement: &2152295780 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *2152295780
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: &2152295300 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :development
+  prerelease: false
+  version_requirements: *2152295300
+- !ruby/object:Gem::Dependency
+  name: jeweler
+  requirement: &2152294820 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.6.4
+  type: :development
+  prerelease: false
+  version_requirements: *2152294820
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: &2152294340 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *2152294340
+- !ruby/object:Gem::Dependency
+  name: rdoc
+  requirement: &2152293860 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *2152293860
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: &2152293380 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.10.8
+  type: :runtime
+  prerelease: false
+  version_requirements: *2152293380
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: &2152292900 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: *2152292900
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: &2152292420 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.5.4
+  type: :development
+  prerelease: false
+  version_requirements: *2152292420
+description: Plugin to counts messages/bytes that matches, per minutes/hours/days
+email: tagomoris@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- LICENSE.txt
+- README.rdoc
+files:
+- .document
+- .gitignore
+- AUTHORS
+- Gemfile
+- LICENSE.txt
+- README.rdoc
+- Rakefile
+- VERSION
+- lib/fluent/plugin/out_flowcounter.rb
+- test/helper.rb
+- test/plugin/test_out_flowcounter.rb
+homepage: http://github.com/tagomoris/fluent-plugin-flowcounter
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 3390482931615998912
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.6
+signing_key: 
+specification_version: 3
+summary: Plugin to counts messages/bytes that matches, per minutes/hours/days
+test_files:
+- test/helper.rb
+- test/plugin/test_out_flowcounter.rb