fluent-plugin-lossycount 0.0.1

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    MjA2OWZjMWIwY2QxZTE0OTI2NGMzODkxMjJiYjg4OGQ2Mzg0YjZhYQ==
+  data.tar.gz: !binary |-
+    MmFiMTk3Nzk0ODZlZTY3YTViMmQ4NjU1MWQ0ZTk1ZjhiYzEwYjRkZA==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    ZGZiM2M2Y2MzZjk4NDk2NTU4ZjMwZmQyYzMxMzhiNWEwMTViNGNkZTAzZTYx
+    ZjViNjRkZTExNDY2YzMzNmI1Y2EzODIyMGIwNTFjNTFiZmZhYTQzZjE1YTE4
+    ZWY3MDk0ZmYyZDUzNjI4MTEyMTc2MDRiZWM1OWJhYTVlYzYwYTA=
+  data.tar.gz: !binary |-
+    Y2EyMzg0ZjcwYzQ3ZWMzNmZlNTZhMTZjMDI1MTEyZWVlZGJlMjYyZTY0MWU1
+    OTMzNzQ1NTE2MDk3YWU2ZmViMzlmNThmYzk0MjQzYjNhZTdkZDg3ODAyMWM1
+    NzM5YmE3OGFmY2U2YWFhNjFiMmYzZTVhNTExNmEwOWM0MjRiN2M=

data/.gitignore

@@ -0,0 +1,3 @@
+Gemfile.lock
+source
+dump.rdb

data/AUTHORS

@@ -0,0 +1 @@
+moaikids

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+
+gemspec

data/README.rdoc

@@ -0,0 +1,30 @@
+= Lossy count output plugin for Fluentd
+
+== Overview
+
+*lossycount* output plugin is a counter logic.
+
+== Installation
+
+== Configuration
+
+    <match pattern>
+        type lossycount
+        gamma LOSSY_COUNT_GAMMA
+        epsilon LOSSY_COUNT_EPSILON
+        key_name INPUT_KEY_NAME
+        time_windows SECONDS
+        output_tag OUTPUT_TAG_NAME
+        output_key_name OUTPUT_KEY_NAME
+        output_timestamp_name OUTPUT_TIMESTAMP_NAME
+        output_value_name OUTPUT_VALUE_NAME
+        enable_metrics true
+        metrics_tag METRICS_OUTPUT_TAG_NAME
+        verbose false
+    </match>
+
+== Copyright
+
+Copyright:: Copyright (c) 2013 moaikids
+License::   Apache License, Version 2.0
+

data/Rakefile

@@ -0,0 +1,11 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+    test.libs << 'lib' << 'test'
+    test.pattern = 'test/**/test_*.rb'
+    test.verbose = true
+end
+
+task :default => :test

data/fluent-plugin-lossycount.gemspec

@@ -0,0 +1,19 @@
+# -*- encoding: utf-8 -*-
+Gem::Specification.new do |gem|
+    gem.name        = "fluent-plugin-lossycount"
+    gem.version     = "0.0.1"
+    gem.authors     = ["moaikids"]
+    gem.licenses    = ["Apache License Version 2.0"]
+    gem.summary     = %q{Lossy count output plugin for Fluentd}
+    gem.description = %q{Lossy count output plugin for Fluentd...}
+    gem.homepage    = "https://github.com/moaikids/fluent-plugin-lossycount"
+
+    gem.files         = `git ls-files`.split($\)
+    gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+    gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+    gem.require_paths = ["lib"]
+
+    gem.add_development_dependency "rake"
+    gem.add_development_dependency "shoulda"
+    gem.add_runtime_dependency "fluentd"
+end

data/lib/fluent/plugin/out_lossycount.rb

@@ -0,0 +1,195 @@
+class Fluent::LossyCountOutput < Fluent::Output
+    Fluent::Plugin.register_output('lossycount', self)
+
+    config_param :key_name, :string, :default => nil
+    config_param :time_windows, :time, :default => 60
+    config_param :output_tag, :string, :default => nil
+    config_param :output_key_name, :string, :default => 'key'
+    config_param :output_timestamp_name, :string, :default => 'timestamp'
+    config_param :output_value_name, :string, :default => 'value'
+    config_param :gamma, :float, :default => 0.005
+    config_param :epsilon, :float, :default => 0.004
+    config_param :enable_metrics, :bool, :default => false
+    config_param :metrics_tag, :string, :default => 'counter.metrics'
+    config_param :verbose, :bool, :default => false
+
+    def configure(config)
+        super
+        unless config.has_key?('key_name')
+            raise Fluent::ConfigError, "you must set 'key_name'"
+        end
+        unless config.has_key?('output_tag')
+            raise Fluent::ConfigError, "you must set 'output_tag'"
+        end
+
+        @mutex = Mutex.new
+        @sleep_wait = 0.4
+        init_counter()
+    end
+
+    def start
+        super
+        start_watch
+    end
+
+    def shutdown
+        super
+        @watcher.terminate
+        @watcher.join
+    end
+
+    def init_counter()
+        @counter = Counter::LossyCounter.new({:gamma => @gamma, :epsilon => @epsilon})
+    end
+
+    def flush_counter(step)
+        freq_counter = {}
+        metrics = {}
+        @mutex.synchronize {
+            freq_counter = @counter.get()
+            if @enable_metrics
+                metrics = @counter.get_metrics()
+            end 
+            init_counter()
+        }
+        flush_time = Fluent::Engine.now.to_i
+        if @verbose
+            $log.info "flushtime : " + flush_time.to_s
+            $log.info "{ "
+        end
+        freq_counter.each_pair { |key, value|
+            map = {@output_key_name => key, @output_timestamp_name => flush_time, @output_value_name => value}
+            if @verbose
+                $log.info map
+            end
+            Fluent::Engine.emit(@output_tag, Fluent::Engine.now, map)
+        }
+        if @verbose
+            $log.info "}"
+        end
+
+        if @enable_metrics
+            if @verbose
+                $log.info "metrics : " + metrics.to_s
+            end
+            Fluent::Engine.emit(@metrics_tag, Fluent::Engine.now, metrics)
+        end
+    end
+
+    def start_watch
+        @watcher = Thread.new{
+            @last_checked = Fluent::Engine.now.to_i
+            while true
+                sleep @sleep_wait
+                now = Fluent::Engine.now.to_i
+                if now - @last_checked >= @time_windows
+                    flush_counter(now - @last_checked)
+                    @last_checked = now
+                end
+            end
+        }
+    end
+
+    def emit(tag, es, chain)
+        es.each {|time,record|
+            k = traverse(record, @key_name)
+            if k
+                if k.is_a?(Array)
+                    k.each{ |v|
+                        @counter.add(v.to_s)
+                    }
+                else
+                    @counter.add(k.to_s)
+                end
+            end
+        }
+        chain.next
+    end
+
+    def traverse(data, key)
+        val = data
+        key.split('.').each{ |k|
+            if val.has_key?(k)
+                val = val[k]
+            else
+                return nil
+            end
+        }
+        return val
+    end
+end
+
+
+module Counter
+    class LossyCounter
+        def initialize(config)
+            @gamma = config.has_key?(:gamma) ? config[:gamma].to_f : 0.005
+            @epsilon = config.has_key?(:epsilon) ? config[:epsilon].to_f : 0.001
+            @current = 1
+            @freq_counter = {}
+            @delta_counter = {}
+            @num = 0
+            @max_size = -1
+        end
+
+        def add(key)
+            if @freq_counter.has_key?(key)
+                @freq_counter[key] += 1
+            else
+                @freq_counter[key] = 1
+                @delta_counter[key] = @current - 1
+            end
+            @num += 1
+            if @num % (1 / @epsilon).to_i == 0
+                sweep()
+            end
+        end
+
+        def sweep()
+            length = @freq_counter.length
+            if @max_size < length
+                @max_size = length
+            end
+
+            @freq_counter.each_pair { |key, value|
+                if value <= (@current - @delta_counter[key])
+                    @freq_counter.delete(key)
+                    @delta_counter.delete(key)
+                end
+            }
+            @current += 1
+        end
+
+        def get()
+            buf = {}
+            @freq_counter.each_pair { |key, value|
+                if value > (@num * (@gamma - @epsilon) ).to_i
+                    buf[key] = value
+                end
+            }
+            return buf
+        end
+        
+        def get_num()
+            return @num
+        end 
+
+        def get_current_max_size()
+            return @max_size
+        end
+
+        def get_num_x_gamma()
+            return @num.to_f * @gamma
+        end
+
+        def get_num_x_gamma_d_epsilon()
+            return @num.to_f * (@gamma - @epsilon)
+        end
+
+        def get_metrics()
+            return {'num' => get_num(), 'max_size' => get_current_max_size(), 'current_size' => @freq_counter.size() , 'reduced_size' => get().size(), "gamma" => @gamma , "epsilon" => @epsilon , "n_x_gamma" => get_num_x_gamma(), "n_x_gamma-epsilon" => get_num_x_gamma_d_epsilon() }
+        end
+
+    end
+end
+

data/test/counter/test_lossy_counter.rb

@@ -0,0 +1,73 @@
+require 'helper'
+
+class LossyCounterTest < Test::Unit::TestCase
+    def setup
+        @gamma   = 0.005
+        @epsilon = 0.0045
+    end
+
+    def test_add_naive()
+        counter = Counter::LossyCounter.new({:gamma => @gamma, :epsilon => @epsilon})
+        counter.add('a')
+        counter.add('b')
+        counter.add('c')
+        counter.add('b')
+        counter.add('a')
+        counter.add('b')
+
+        freq_counter = counter.get()
+
+        p freq_counter
+        assert_equal freq_counter.has_key?('a') , true
+        assert_equal freq_counter.has_key?('b') , true
+        assert_equal freq_counter.has_key?('c') , true
+        assert_equal freq_counter.has_key?('d') , false
+        assert_equal freq_counter['a'] , 2
+        assert_equal freq_counter['b'] , 3
+        assert_equal freq_counter['c'] , 1
+        assert_equal freq_counter['d'] , nil
+
+        p counter.get_metrics()
+    end
+
+    def test_add_and_sweep()
+        counter = Counter::LossyCounter.new({:gamma => @gamma, :epsilon => @epsilon})
+        buf = {}
+        radix = [10000000, 1000000, 100000, 10000, 1000, 100, 10]
+        l = 100000
+
+        (0..(l - 1)).step(1) do |i|
+            now = ((Time.now.to_f * 10000000) % 10000000).to_i
+            for r in radix
+                key = now & r
+                counter.add(key.to_s)
+                if buf.has_key?(key.to_s)
+                    buf[key] += 1
+                else
+                    buf[key] = 1
+                end
+            end
+        end
+
+        n = counter.get_num()
+        g = counter.get_num_x_gamma().to_i
+        e = (n * @epsilon).to_i
+        ge = counter.get_num_x_gamma_d_epsilon().to_i
+        freq_count = counter.get()
+
+        p counter.get_metrics()
+
+        buf.each_pair { |key, value|
+            if freq_count.has_key?(key)
+                if freq_count[key] >= g.to_i
+                    assert_equal value , freq_count[key]
+                else
+                    assert_operator e.to_i , :>= , (freq_count[key] - value).abs
+                end
+            else 
+                assert_operator e.to_i , :>= , value
+            end
+        }
+
+    end
+end

data/test/helper.rb

@@ -0,0 +1,20 @@
+require 'rubygems'
+require 'bundler'
+begin
+    Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+    $stderr.puts e.message
+    $stderr.puts "Run `bundle install` to install missing gems"
+    exit e.status_code
+end
+
+require 'test/unit'
+require 'shoulda'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'fluent/test'
+require 'fluent/plugin/out_lossycount'
+
+class Test::Unit::TestCase
+end

metadata

@@ -0,0 +1,96 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-lossycount
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- moaikids
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-08-31 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: shoulda
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Lossy count output plugin for Fluentd...
+email: 
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- AUTHORS
+- Gemfile
+- README.rdoc
+- Rakefile
+- fluent-plugin-lossycount.gemspec
+- lib/fluent/plugin/out_lossycount.rb
+- test/counter/test_lossy_counter.rb
+- test/helper.rb
+homepage: https://github.com/moaikids/fluent-plugin-lossycount
+licenses:
+- Apache License Version 2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.7
+signing_key: 
+specification_version: 4
+summary: Lossy count output plugin for Fluentd
+test_files:
+- test/counter/test_lossy_counter.rb
+- test/helper.rb