fluent-plugin-redis-multi-type-counter 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: f1e9b2d54944b6575cabdad55ce1250d61c3163a
+  data.tar.gz: 692d3df513e0f23ccc6019f7547092f8a38e2e69
+SHA512:
+  metadata.gz: c750a05f9e29ce9b9a925f055039c35e4e1e010dff5db0fb7f6195974e4a025427257b43eb970501d3b60a2570312aff3315eb42cd534000fa6908be3601c77a
+  data.tar.gz: 6d71cc09fce6a3d3c91c1f92174bd36469602c650d1aaf397e007083b7640f95f2c5df2fee87d63f3d0758f389db860bbdcf69159f5d07303a874ac68c6230f5

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/.travis.yml

@@ -0,0 +1,16 @@
+language: ruby
+
+rvm:
+  - 1.9.2
+  - 1.9.3
+
+services:
+  - redis-server
+
+script: bundle exec ruby -S -Itest test/plugin/out_redis_counter.rb
+
+branches:
+  only:
+    - master
+    - develop
+

data/CHANGELOG.md

@@ -0,0 +1,26 @@
+## 0.1.0
+* support hash, sorted set type
+
+######################################################################
+## fluent-plugin-redis-counter
+## 0.3.2
+* add password support for Redis connection.
+
+## 0.3.1
+* improve performance
+
+## 0.3.0
+* count value can be picked up from JSON data contents.
+
+## 0.2.0
+* count key can be built from JSON data contents.
+
+## 0.1.1
+* add 'count_key_format' in order to make time-dependent redis keyname
+* bug fix
+
+## 0.1.0
+* design changed.
+
+## 0.0.1
+* first version

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in fluent-plugin-redis-multi-type-counter.gemspec
+gemspec

data/README.md

@@ -0,0 +1,119 @@
+# Redis multi-type counter plugin for fluent [![Build Status](https://secure.travis-ci.org/heartsavior/fluent-plugin-redis-multi-type-counter.png)](http://travis-ci.org/heartsavior/fluent-plugin-redis-multi-type-counter)
+
+fluent-plugin-redis-multi-type-counter is a fluent plugin to count-up/down redis keys, hash, sorted set.
+
+# Installation
+
+fluent-plugin-redis-multi-type-counter is hosted by [RubyGems.org](https://rubygems.org/).
+
+    $fluent-gem install fluent-plugin-redis-multi-type-counter
+
+# Configuration
+
+    <match redis_counter.**>
+      type redis_counter
+
+      host localhost
+      port 6379
+
+      # database number is optional.
+      db_number 0        # 0 is default
+
+      # match condition
+      # this pattern matches {"status": "200", "url": "http://foo.example.com"} and then,
+      # increment Redis key "foo-status2xx" by calling Redis::incrby( "foo-status2xx", 1 ).
+      <pattern>
+        match_status ^2[0-9][0-9]$        # matches with {"status": "200", ...
+        match_url ^http:\/\/foo\.         # matches with {"url": "http://foo.example.com", ...
+        count_key foo-status2xx           # key-name for Redis
+        count_value 1                     # count-up amount(default: 1, negative value is allowed)
+      </pattern>
+
+      # time-dependent redis keyname
+      # for example, "foo-status2xx-%Y-%m-%d" will be formatted to "foo-status2xx-2012-06-21".
+      # rules for placeholder(%Y, etc.) is similar to out_file plugin.
+      <pattern>
+        match_status ^2[0-9][0-9]$
+        count_key_format foo-statux2xx-%d
+        localtime                         # time-zone(default: localtime, it can be "utc" or "localtime")
+      </pattern>
+
+      # you can also use values from the matched JSON record in the key name, by using the 
+      # syntax foo%_{key1}-%_{key2}, where key1 and key2 are keys in the JSON record that 
+      # was received by fluentd.
+      # for example, "customer:%_{customer_id}:status2xx" will be formatted to 
+      # "customer:123:status2xx" if the JSON record contains a key named "customer_id" 
+      # with value 123, like so: {"status": 200, "customer_id": 123 ... }.
+      # these can be combined with the time formatting options in the previous example.
+      <pattern>
+        match_status ^2[0-9][0-9]$
+        count_key_format customer:%_{customer_id}:status2xx-%Y-%m-%d
+      </pattern>
+
+      # you can also sum up key in hash, by configuring count_hash_key_format
+      # syntax is same to count_key_format
+      # for example, {"custom_id": 123, "date": "20131219" ...}.
+      # HINCRBY item_count:123 20131219 1
+      <pattern>
+        count_key_format item_count:%_{item_id}
+        count_hash_key_format %_{date}
+      <pattern>
+
+      # you can also sum up key in sorted set(zset), by configuring count_zset_key_format
+      # syntax and usage is same to count_hash_key_format
+      # for example, {"custom_id": 123, "date": "20131219" ...}.
+      # ZINCRBY item_count:123 1 20131219
+      <pattern>
+        count_key_format item_count:%_{item_id}
+        count_zset_key_format %_{date}
+      <pattern>
+
+      # you can also sum up specified key with count_value_key option.
+      # for example, {"count": 321, "customer_id": 123 ... }.
+      # INCRBY item_count:123 321.
+      <pattern>
+        count_key_format item_count:%_{item_id}
+        count_value_key count
+      </pattern>
+    </match>
+
+# Example
+
+prepare a conf file ("fluent.conf") in current directory like this:
+
+    <source>
+      type forward
+    </source>
+    <match debug.**>
+      type redis_counter
+      host localhost
+      port 6379
+      db_number 0
+      <pattern>
+        match_status ^2[0-9][0-9]$
+        match_url ^http:\/\/foo\.
+        count_key foo
+      </pattern>
+    </match>
+
+run commands for test:
+
+    $redis-server 2>&1 >/dev/null &
+    [1] 879
+    $echo del foo | redis-cli -h localhost -p 6379 -n 0
+    (integer) 0
+    $fluentd -c ./fluent.conf 2>&1 >/dev/null &
+    [2] 889
+    $echo {\"status\": \"200\", \"url\": \"http://foo.example.com\"} | fluent-cat debug
+    $echo {\"status\": \"500\", \"url\": \"http://foo.example.com\"} | fluent-cat debug
+    $kill -s HUP 889
+    $echo get foo | redis-cli -h localhost -p 6379 -n 0
+    "1"
+
+# Copyright
+- Copyright © 2014      Jungtaek Lim
+- Copyright © 2012-2014 Buntaro Okada
+- Copyright © 2011-2012 Yuki Nishijima
+
+# License
+- Apache License, Version 2.0

data/Rakefile

@@ -0,0 +1,2 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/benchmark/performance_test.rb

@@ -0,0 +1,99 @@
+require 'fluent/test'
+require 'benchmark'
+
+if ENV["PROFILE"]
+require 'ruby-prof'
+end
+
+$LOAD_PATH.unshift(File.expand_path(File.join(File.dirname(__FILE__), "..", "lib")))
+require 'fluent/plugin/out_redis_counter'
+
+module PerformanceTestApp
+  class Benchmarker
+    def initialize
+      Fluent::Test.setup
+      @fluent = Fluent::Test::BufferedOutputTestDriver.new(Fluent::RedisCounterOutput).configure(%[
+          host localhost
+          port 6379
+          db_number 1
+          <pattern>
+            match_status ^2[0-9]{2}$
+            count_key_format status-normal:item_id:%_{item_id}
+            count_value 1
+          </pattern>
+      ])
+    end
+
+    def prepare
+      10_0000.times do
+        item_id = ( rand * 10_0000 ).to_i
+        @fluent.emit({ "status" => "200", "item_id" => item_id })
+      end
+    end
+
+    def run
+      result = nil
+      profile = nil
+      Benchmark.bm do |x|
+        result = x.report {
+          if ENV["PROFILE"]
+            profile = RubyProf.profile do
+              @fluent.run
+            end
+          else
+              @fluent.run
+          end
+        }
+      end
+
+      if ENV["PROFILE"]
+        profile_printer = RubyProf::GraphPrinter.new(profile)
+        profile_printer.print(STDOUT, {})
+      end
+      $log.info("benchmark result: #{result}")
+    end
+  end
+
+  class MemoryWatcher
+    def initialize
+    end
+
+    def run(&block)
+      start_memory = get_memory(Process.pid)
+      pid = Process.fork do
+        block.call
+      end
+      @max_memory = 0
+      th = Thread.new do
+        begin
+          loop do
+            sleep(0.01)
+            msize = get_memory(pid)
+            if msize > @max_memory
+              @max_memory = msize
+            end
+          end
+        ensure
+          $log.info("start memory size:\t#{sprintf("%#10d", start_memory / 1024)}KB")
+          $log.info("max memory size:\t#{sprintf("%#10d", @max_memory / 1024)}KB")
+        end
+      end
+      Process.waitall
+      th.kill
+    end
+
+    def get_memory(pid)
+      `ps -h -o rss= -p #{pid}`.to_i * 1024
+    end
+  end
+end
+
+if $0 == __FILE__
+  parent = PerformanceTestApp::MemoryWatcher.new
+  app = PerformanceTestApp::Benchmarker.new
+  parent.run do
+    app.prepare
+    app.run
+  end
+end
+

data/fluent-plugin-redis-multi-type-counter.gemspec

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+
+Gem::Specification.new do |s|
+  s.name        = "fluent-plugin-redis-multi-type-counter"
+  s.version     = "0.1.0"
+  s.description = "fluent-plugin-redis-multi-type-counter is a fluent plugin to count-up/down redis keys, hash keys, zset keys"
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Jungtaek Lim"]
+  s.date        = %q{2014-01-07}
+  s.email       = "kabhwan@gmail.com"
+  s.homepage    = "https://github.com/heartsavior/fluent-plugin-redis-multi-type-counter"
+  s.summary     = "Redis multi type counter plugin for fluent"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_dependency %q<fluentd>, ["~> 0.10.0"]
+  s.add_dependency %q<redis>, [">= 2.2.2"]
+
+  s.license = 'APACHE2'
+end

data/lib/fluent/plugin/out_redis_counter.rb

@@ -0,0 +1,247 @@
+module Fluent
+  class RedisCounterOutput < BufferedOutput
+    Fluent::Plugin.register_output('redis_counter', self)
+    attr_reader :host, :port, :db_number, :password, :redis, :patterns
+
+    config_param :max_pipelining, :integer, :default => 1000
+
+    def initialize
+      super
+      require 'redis'
+      require 'msgpack'
+    end
+
+    def configure(conf)
+      super
+      @host = conf.has_key?('host') ? conf['host'] : 'localhost'
+      @port = conf.has_key?('port') ? conf['port'].to_i : 6379
+      @password = conf.has_key?('password') ? conf['password'] : nil
+      @db_number = conf.has_key?('db_number') ? conf['db_number'].to_i : nil
+      @patterns = []
+      conf.elements.select { |element|
+        element.name == 'pattern'
+      }.each { |element|
+        begin
+          @patterns << Pattern.new(element)
+        rescue RedisCounterException => e
+          raise Fluent::ConfigError, e.message
+        end
+      }
+    end
+
+    def start
+      super
+      @redis = Redis.new(
+        :host => @host, :port => @port,
+        :password => @password,
+        :thread_safe => true, :db => @db_number
+      )
+    end
+
+    def shutdown
+      @redis.quit
+    end
+
+    def format(tag, time, record)
+      [tag, time, record].to_msgpack
+    end
+
+    def write(chunk)
+      table = {}
+      table.default = 0
+      chunk.open { |io|
+        begin
+          MessagePack::Unpacker.new(io).each { |message|
+            (tag, time, record) = message
+            @patterns.select { |pattern|
+              pattern.is_match?(record)
+            }.each{ |pattern|
+              count_key = pattern.get_count_key(time, record)
+              count_hash_key = pattern.get_count_hash_key(record)
+              count_zset_key = pattern.get_count_zset_key(record)
+
+              key = RecordKey.new(count_key, count_hash_key, count_zset_key)
+              table[key] += pattern.get_count_value(record)
+            }
+          }
+        rescue EOFError
+          # EOFError always occured when reached end of chunk.
+        end
+      }
+
+      table.each_pair.select { |key, value|
+        value != 0
+      }.each_slice(@max_pipelining) { |items|
+        @redis.pipelined do
+          items.each do |key, value|
+            if key.count_hash_key != nil
+              @redis.hincrby(key.count_key, key.count_hash_key, value)
+            elsif key.count_zset_key != nil
+              @redis.zincrby(key.count_key, value, key.count_zset_key)
+            else
+              @redis.incrby(key.count_key, value)
+            end
+          end
+        end
+      }
+    end
+
+    class RecordKey
+      attr_reader :count_key, :count_hash_key, :count_zset_key
+
+      def initialize(count_key, count_hash_key, count_zset_key)
+        @count_key = count_key
+        @count_hash_key = count_hash_key
+        @count_zset_key = count_zset_key
+      end
+
+      def hash
+        hash_key = ""
+
+        keys = [@count_key, @count_hash_key, @count_zset_key]
+        keys.select { |key| 
+          key != nil
+        }.each { |key|
+          hash_key += ("@@@@" + key)
+        }
+
+        hash_key.hash
+      end
+
+      def eql?(other)
+        return @count_key.eql?(other.count_key) && @count_hash_key.eql?(other.count_hash_key) && 
+          @count_zset_key.eql?(other.count_zset_key)
+      end
+    end
+
+    class RedisCounterException < Exception
+    end
+
+    class RecordValueFormatter
+      attr_reader :format
+      def initialize(format)
+        @format = format
+      end
+
+      CUSTOM_KEY_EXPRESSION_RE = /(%_\{([^\}]+)\})/
+
+      def key(record)
+        @format.gsub(CUSTOM_KEY_EXPRESSION_RE) do |s|
+          record[$2]
+        end
+      end
+    end
+
+    class Pattern
+      attr_reader :matches, :count_value, :count_value_key
+
+      def initialize(conf_element)
+        if !conf_element.has_key?('count_key') && !conf_element.has_key?('count_key_format')
+          raise RedisCounterException, '"count_key" or "count_key_format" is required.'
+        end
+        if conf_element.has_key?('count_key') && conf_element.has_key?('count_key_format')
+          raise RedisCounterException, 'both "count_key" and "count_key_format" are specified.'
+        end
+
+        if conf_element.has_key?('count_key')
+          @count_key = conf_element['count_key']
+        else
+          if conf_element.has_key?('localtime') && conf_element.has_key?('utc')
+            raise RedisCounterException, 'both "localtime" and "utc" are specified.'
+          end
+          is_localtime = true
+          if conf_element.has_key?('utc')
+            is_localtime = false
+          end
+          @count_key_format = [conf_element['count_key_format'], is_localtime]
+          @record_formatter_for_count_key = RecordValueFormatter.new(@count_key_format[0])
+        end
+
+        if conf_element.has_key?('count_hash_key_format') && conf_element.has_key?('count_zset_key_format')
+          raise RedisCounterException, 'both "count_hash_key_format" "count_zset_key_format" are specified.'
+        end
+
+        if conf_element.has_key?('count_hash_key_format')
+          @count_hash_key_format = conf_element['count_hash_key_format']
+          @record_formatter_for_count_hash_key = RecordValueFormatter.new(@count_hash_key_format)
+        else
+          @count_hash_key_format = nil
+        end
+
+        if conf_element.has_key?('count_zset_key_format')
+          @count_zset_key_format = conf_element['count_zset_key_format']
+          @record_formatter_for_count_zset_key = RecordValueFormatter.new(@count_zset_key_format)
+        else
+          @count_zset_key_format = nil
+        end
+
+        if conf_element.has_key?('count_value_key')
+          @count_value_key = conf_element['count_value_key']
+        else
+          @count_value = 1
+          if conf_element.has_key?('count_value')
+            begin
+              @count_value = Integer(conf_element['count_value'])
+            rescue
+              raise RedisCounterException, 'invalid "count_value", integer required.'
+            end
+          end
+        end
+
+        @matches = {}
+        conf_element.each_pair.select { |key, value|
+          key =~ /^match_/
+        }.each { |key, value|
+          name = key['match_'.size .. key.size]
+          @matches[name] = Regexp.new(value)
+        }
+      end
+
+      def is_match?(record)
+        @matches.each_pair{ |key, value|
+          if !record.has_key?(key) || !(record[key] =~ value)
+            return false
+          end
+        }
+        return true
+      end
+
+      def get_count_key(time, record)
+        if @count_key_format == nil
+          @count_key
+        else
+          count_key = @record_formatter_for_count_key.key(record)
+          formatter = TimeFormatter.new(count_key, @count_key_format[1])
+          formatter.format(time)
+        end
+      end
+
+      def get_count_hash_key(record)
+        if @count_hash_key_format == nil
+          return nil
+        else
+          return @record_formatter_for_count_hash_key.key(record)
+        end
+      end
+
+      def get_count_zset_key(record)
+        if @count_zset_key_format == nil
+          return nil
+        else
+          return @record_formatter_for_count_zset_key.key(record)
+        end
+      end
+
+      def get_count_value(record)
+        if @count_value_key
+          ret = record[@count_value_key] || 0
+          return ret.kind_of?(Integer) ? ret : 0
+        else
+          if @count_value
+            return @count_value
+          end
+        end
+      end
+    end
+  end
+end

data/test/plugin/out_redis_counter.rb

@@ -0,0 +1,372 @@
+require 'fluent/test'
+
+class RedisCounterTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+    require 'fluent/plugin/out_redis_counter'
+
+    @d = create_driver %[
+      host localhost
+      port 6379
+      db_number 1
+    ]
+    redis = Redis.new(
+      :host => "localhost", :port => 6379,
+      :thread_safe => true, :db => 1
+    )
+    redis.del("a")
+    redis.del("b")
+    redis.del("foo-2012-06-21")
+    redis.del("item_sum_count:200")
+    redis.del("item_sum_count_by_hash:200")
+    redis.del("item_sum_count_by_zset:200")
+    redis.quit
+  end
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::BufferedOutputTestDriver.new(Fluent::RedisCounterOutput).configure(conf)
+  end
+
+  def test_configure
+    assert_equal 'localhost', @d.instance.host
+    assert_equal 6379, @d.instance.port
+    assert_equal 1, @d.instance.db_number
+    assert_equal 0, @d.instance.patterns.size
+  end
+
+  def test_configure_pattern
+    driver = create_driver %[
+      host localhost
+      port 6379
+      db_number 1
+      <pattern>
+        match_status ^2[0-9]{2}$
+        match_url ^https
+        count_key status-normal
+      </pattern>
+      <pattern>
+        count_key foo
+        count_value 2
+      </pattern>
+    ]
+    assert_equal 'localhost', driver.instance.host
+    assert_equal 6379, driver.instance.port
+    assert_equal 1, driver.instance.db_number
+    assert_equal 2, driver.instance.patterns.size
+
+    assert_equal 2, driver.instance.patterns[0].matches.size
+    assert_equal Regexp.new('^2[0-9]{2}$'), driver.instance.patterns[0].matches['status']
+    assert_equal Regexp.new('^https'), driver.instance.patterns[0].matches['url']
+    assert_equal 'status-normal', driver.instance.patterns[0].get_count_key(Time.now.to_i, {})
+    assert_equal 1, driver.instance.patterns[0].count_value
+
+    assert_equal 0, driver.instance.patterns[1].matches.size
+    assert_equal 'foo', driver.instance.patterns[1].get_count_key(Time.now.to_i, {})
+    assert_equal 2, driver.instance.patterns[1].count_value
+  end
+
+  def test_configure_count_key_required
+    begin
+      create_driver %[
+        <pattern>
+          count_value 1
+        </pattern>
+      ]
+      flunk
+    rescue Fluent::ConfigError => e
+      assert_equal '"count_key" or "count_key_format" is required.', e.message
+    end
+  end
+
+  def test_configure_count_key_duplicated
+    begin
+      create_driver %[
+        <pattern>
+          count_key foo
+          count_key_format foo-%Y
+        </pattern>
+      ]
+      flunk
+    rescue Fluent::ConfigError => e
+      assert_equal 'both "count_key" and "count_key_format" are specified.', e.message
+    end
+  end
+
+  def test_configure_count_key_format_utc
+    driver = create_driver %[
+      <pattern>
+        count_key_format foo-%Y-%m-%d-%H-%M-%S
+        utc
+      </pattern>
+    ]
+    time = Time.parse('2011-06-21 03:12:01 UTC').to_i
+    assert_equal 'foo-2011-06-21-03-12-01', driver.instance.patterns[0].get_count_key(time, {})
+  end
+
+  def test_configure_count_key_format_localtime
+    driver = create_driver %[
+      <pattern>
+        count_key_format foo-%Y-%m-%d-%H-%M-%S
+        localtime
+      </pattern>
+    ]
+    local_time = Time.parse('2012-06-21 03:12:00').to_i
+    assert_equal 'foo-2012-06-21-03-12-00', driver.instance.patterns[0].get_count_key(local_time, {})
+  end
+
+  def test_configure_duplicated_timezone
+    begin
+      create_driver %[
+        <pattern>
+          count_key_format foo%Y
+          localtime
+          utc
+        </pattern>
+      ]
+      flunk
+    rescue Fluent::ConfigError => e
+      assert_equal 'both "localtime" and "utc" are specified.', e.message
+    end
+  end
+
+  def test_configure_count_key_format_with_record_value_formatter
+    driver = create_driver %[
+      <pattern>
+        count_key_format %_{prefix}-foo-%Y-%m-%_{type}-%_{customer_id}
+        localtime
+      </pattern>
+    ]
+    local_time = Time.parse('2012-06-21 03:12:00').to_i
+    record = {'prefix' => 'pre', 'type' => 'bar', 'customer_id' => 321}
+    assert_equal 'pre-foo-2012-06-bar-321', driver.instance.patterns[0].get_count_key(local_time, record)
+  end
+
+  def test_configure_count_hash_key
+    driver = create_driver %[
+      <pattern>
+        count_key_format %_{prefix}-foo-bar
+        count_hash_key_format %_{host}-hash-key
+      </pattern>
+    ]
+    record = {'prefix' => 'pre', 'host' => 'localhost'}
+    local_time = Time.parse('2012-06-21 03:12:00').to_i
+    assert_equal 'pre-foo-bar', driver.instance.patterns[0].get_count_key(local_time, record)
+    assert_equal 'localhost-hash-key', driver.instance.patterns[0].get_count_hash_key(record)
+  end
+
+  def test_configure_count_zset_key
+    driver = create_driver %[
+      <pattern>
+        count_key_format %_{prefix}-foo-bar
+        count_zset_key_format %_{host}-zset-key
+      </pattern>
+    ]
+    record = {'prefix' => 'pre', 'host' => 'localhost'}
+    local_time = Time.parse('2012-06-21 03:12:00').to_i
+    assert_equal 'pre-foo-bar', driver.instance.patterns[0].get_count_key(local_time, record)
+    assert_equal 'localhost-zset-key', driver.instance.patterns[0].get_count_zset_key(record)
+  end
+
+  def test_configure_both_specified_hash_key_and_zset_key
+    begin
+      create_driver %[
+        <pattern>
+          count_key_format %_{prefix}-foo-bar
+          count_hash_key_format %_{host}-hash-key
+          count_zset_key_format %_{host}-zset-key
+        </pattern>
+      ]
+      flunk
+    rescue Fluent::ConfigError => e
+      assert_equal 'both "count_hash_key_format" "count_zset_key_format" are specified.', e.message
+    end
+  end
+
+  def test_configure_invalid_count_value
+    begin
+      create_driver %[
+        <pattern>
+          count_key foo
+          count_value a
+        </pattern>
+      ]
+      flunk
+    rescue Fluent::ConfigError => e
+      assert_equal 'invalid "count_value", integer required.', e.message
+    end
+  end
+
+  def test_get_count_value_with_count_value_key
+    driver = create_driver %[
+      <pattern>
+        count_key_format %_{customer_id}
+        count_value_key count
+      </pattern>
+    ]
+    record = {'count' => 123, 'customer_id' => 321}
+    assert_equal 123, driver.instance.patterns[0].get_count_value(record)
+  end
+
+  def test_get_count_value_without_count_value_key
+    driver = create_driver %[
+      <pattern>
+        count_key_format %_{customer_id}
+      </pattern>
+    ]
+    record = {'count' => 123, 'customer_id' => 321}
+    assert_equal 1, driver.instance.patterns[0].get_count_value(record)
+  end
+
+  def test_format
+    time = Time.parse('2012-06-21 01:55:00 UTC').to_i
+    @d.emit({"a" => 1}, time)
+    @d.expect_format(['test', time, {"a" => 1}].to_msgpack)
+    @d.run
+  end
+
+  def test_write
+    driver = create_driver %[
+      db_number 1
+      <pattern>
+        match_a ^2[0-9][0-9]$
+        count_key a
+        count_value 2
+      </pattern>
+    ]
+    driver.emit({"a" => "value-of-a"})
+    driver.emit({"a" => "200"})
+    driver.emit({"b" => "200"})
+    driver.emit({"aa" => "200"})
+    driver.emit({"a" => "2000"})
+    driver.run
+
+    assert_equal '2', driver.instance.redis.get("a")
+    assert_nil driver.instance.redis.get("b")
+  end
+
+  def test_write_with_timeformat
+    driver = create_driver %[
+      db_number 1
+      <pattern>
+        match_a ^2[0-9][0-9]$
+        count_key_format foo-%Y-%m-%d
+        count_value 2
+      </pattern>
+    ]
+    time = Time.parse('2012-06-21 03:01:00 UTC').to_i
+    driver.emit({"a" => "200"}, time)
+    driver.run
+
+    assert_equal '2', driver.instance.redis.get("foo-2012-06-21")
+  end
+
+  def test_write_with_count_hash_key
+    def create_driver_for_test
+      driver = create_driver %[
+        db_number 1
+        <pattern>
+          count_key_format item_sum_count_by_hash:%_{item_id}
+          count_hash_key_format %_{from}
+        </pattern>
+      ]
+    end
+    
+    driver = create_driver_for_test
+
+    time = Time.parse('2012-06-21 03:01:00 UTC').to_i
+    driver.emit({"item_id" => 200, "from" => "US"}, time)
+    driver.run
+
+    assert_equal '1', driver.instance.redis.hget("item_sum_count_by_hash:200", "US")
+
+    driver = create_driver_for_test
+
+    driver.emit({"item_id" => 200, "from" => "CHINA"}, time)
+    driver.run
+
+    assert_equal '1', driver.instance.redis.hget("item_sum_count_by_hash:200", "CHINA")
+    assert_equal '1', driver.instance.redis.hget("item_sum_count_by_hash:200", "US")
+
+    driver = create_driver_for_test
+
+    driver.emit({"item_id" => 200, "from" => "US"}, time)
+    driver.run
+
+    assert_equal '2', driver.instance.redis.hget("item_sum_count_by_hash:200", "US")
+  end
+
+  def test_write_with_count_zset_key
+    def create_driver_for_test
+      driver = create_driver %[
+        db_number 1
+        <pattern>
+          count_key_format item_sum_count_by_zset:%_{item_id}
+          count_zset_key_format %_{from}
+        </pattern>
+      ]
+    end
+    
+    driver = create_driver_for_test
+
+    time = Time.parse('2012-06-21 03:01:00 UTC').to_i
+    driver.emit({"item_id" => 200, "from" => "US"}, time)
+    driver.run
+
+    assert_equal 1, driver.instance.redis.zscore("item_sum_count_by_zset:200", "US")
+
+    driver = create_driver_for_test
+
+    driver.emit({"item_id" => 200, "from" => "CHINA"}, time)
+    driver.run
+
+    assert_equal 1, driver.instance.redis.zscore("item_sum_count_by_zset:200", "CHINA")
+    assert_equal 1, driver.instance.redis.zscore("item_sum_count_by_zset:200", "US")
+
+    driver = create_driver_for_test
+
+    driver.emit({"item_id" => 200, "from" => "US"}, time)
+    driver.run
+
+    assert_equal 2, driver.instance.redis.zscore("item_sum_count_by_zset:200", "US")
+  end
+
+
+  def test_write_with_count_value_key
+    driver = create_driver %[
+      db_number 1
+      <pattern>
+        count_key_format item_sum_count:%_{item_id}
+        count_value_key count
+      </pattern>
+    ]
+
+    time = Time.parse('2012-06-21 03:01:00 UTC').to_i
+    driver.emit({"item_id" => 200, "count" => 123}, time)
+    driver.run
+
+    assert_equal '123', driver.instance.redis.get("item_sum_count:200")
+
+    driver = create_driver %[
+      db_number 1
+      <pattern>
+        count_key_format item_sum_count:%_{item_id}
+        count_value_key count
+      </pattern>
+    ]
+    driver.emit({"item_id" => 200, }, time)
+    driver.run
+    assert_equal '123', driver.instance.redis.get("item_sum_count:200"), "it should be ignore when count_value_key does not exists."
+
+    driver = create_driver %[
+      db_number 1
+      <pattern>
+        count_key_format item_sum_count:%_{item_id}
+        count_value_key count
+      </pattern>
+    ]
+    driver.emit({"item_id" => 200, "count" => "111aaa"}, time)
+    driver.run
+    assert_equal '123', driver.instance.redis.get("item_sum_count:200"), "it should be ignore when count_value_key is not number"
+  end
+
+end

metadata

@@ -0,0 +1,84 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-redis-multi-type-counter
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Jungtaek Lim
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-01-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.10.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.10.0
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 2.2.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 2.2.2
+description: fluent-plugin-redis-multi-type-counter is a fluent plugin to count-up/down
+  redis keys, hash keys, zset keys
+email: kabhwan@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .travis.yml
+- CHANGELOG.md
+- Gemfile
+- README.md
+- Rakefile
+- VERSION
+- benchmark/performance_test.rb
+- fluent-plugin-redis-multi-type-counter.gemspec
+- lib/fluent/plugin/out_redis_counter.rb
+- test/plugin/out_redis_counter.rb
+homepage: https://github.com/heartsavior/fluent-plugin-redis-multi-type-counter
+licenses:
+- APACHE2
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: Redis multi type counter plugin for fluent
+test_files:
+- test/plugin/out_redis_counter.rb