fluent-plugin-redis-counter 0.0.1 → 0.1.0

data/.travis.yml

@@ -1,5 +1,11 @@
 language: ruby
+
 rvm:
   - 1.9.2
   - 1.9.3
+
 script: bundle exec ruby -S -Itest test/plugin/out_redis_counter.rb
+
+branches:
+  only:
+    - master

data/CHANGELOG.md

@@ -1,2 +1,5 @@
+## 0.1.0
+* design changed.
+
 ## 0.0.1
 * first version

data/README.md

@@ -2,6 +2,12 @@
 
 fluent-plugin-redis-counter is a fluent plugin to count-up/down redis keys.
 
+# Installation
+
+fluent-plugin-redis-counter is hosted by [RubyGems.org](https://rubygems.org/).
+
+    $fluent-gem install fluent-plugin-redis-counter
+
 # Configuration
 
     <match redis_counter.**>
@@ -12,6 +18,16 @@ fluent-plugin-redis-counter is a fluent plugin to count-up/down redis keys.
 
       # database number is optional.
       db_number 0        # 0 is default
+
+      # match condition
+      # this pattern matches {"status": "200", "url": "http://foo.example.com"} and then,
+      # increment Redis key "foo-status2xx" by calling Redis::incrby( "foo-status2xx", 1 ).
+      <pattern>
+        match_status ^2[0-9][0-9]$        # matches with {"status": "200", ...
+        match_url ^http:\/\/foo\.         # matches with {"url": "http://foo.example.com", ...
+        count_key foo-status2xx           # key-name for Redis
+        count_value 1                     # count-up amount(default: 1, negative value is allowed)
+      </pattern>
     </match>
 
 # Example
@@ -26,29 +42,30 @@ prepare a conf file ("fluent.conf") in current directory like this:
       host localhost
       port 6379
       db_number 0
+      <pattern>
+        match_status ^2[0-9][0-9]$
+        match_url ^http:\/\/foo\.
+        count_key foo
+      </pattern>
     </match>
 
 run commands for test:
 
     $redis-server 2>&1 >/dev/null &
     [1] 879
-    $redis-cli
-    redis 127.0.0.1:6379>del foo
+    $echo del foo | redis-cli -h localhost -p 6379 -n 0
     (integer) 0
-    redis 127.0.0.1:6379>exit
     $fluentd -c ./fluent.conf 2>&1 >/dev/null &
     [2] 889
-    $echo {\"foo\":5} | fluent-cat debug
-    $echo {\"foo\":-2} | fluent-cat debug
+    $echo {\"status\": \"200\", \"url\": \"http://foo.example.com\"} | fluent-cat debug
+    $echo {\"status\": \"500\", \"url\": \"http://foo.example.com\"} | fluent-cat debug
     $kill -s HUP 889
-    $redis-cli
-    redis 127.0.0.1:6379>get foo
-    "3"
-    redis 127.0.0.1:6379>
+    $echo get foo | redis-cli -h localhost -p 6379 -n 0
+    "1"
 
 # Copyright
-- Copyright(C) 2012 Buntaro Okada
-- Copyright(C) 2011-2012 Yuki Nishijima
+- Copyright © 2012 Buntaro Okada
+- Copyright © 2011-2012 Yuki Nishijima
 
 # License
 - Apache License, Version 2.0

data/VERSION

@@ -1 +1 @@
-0.0.1
+0.1.0

data/fluent-plugin-redis-counter.gemspec

@@ -3,11 +3,11 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-redis-counter"
-  s.version     = "0.0.1"
+  s.version     = "0.1.0"
   s.description = "fluent-plugin-redis-counter is a fluent plugin to count-up/down redis keys."
   s.platform    = Gem::Platform::RUBY
   s.authors     = ["Buntaro Okada"]
-  s.date        = %q{2012-06-15}
+  s.date        = %q{2012-06-19}
   s.email       = "kbinani.bt@gmail.com"
   s.homepage    = "https://github.com/kbinani/fluent-plugin-redis-counter"
   s.summary     = "Redis counter plugin for fluent"
@@ -18,5 +18,5 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   s.add_dependency %q<fluentd>, ["~> 0.10.0"]
-  s.add_dependency %q<redis>, ["~> 2.2.2"]
+  s.add_dependency %q<redis>, [">= 2.2.2"]
 end

data/lib/fluent/plugin/out_redis_counter.rb

@@ -1,7 +1,7 @@
 module Fluent
   class RedisCounterOutput < BufferedOutput
     Fluent::Plugin.register_output('redis_counter', self)
-    attr_reader :host, :port, :db_number, :redis
+    attr_reader :host, :port, :db_number, :redis, :patterns
 
     def initialize
       super
@@ -14,6 +14,34 @@ module Fluent
       @host = conf.has_key?('host') ? conf['host'] : 'localhost'
       @port = conf.has_key?('port') ? conf['port'].to_i : 6379
       @db_number = conf.has_key?('db_number') ? conf['db_number'].to_i : nil
+      @patterns = []
+      conf.elements.select { |e|
+        e.name == 'pattern'
+      }.each { |e|
+        if e.has_key?('count_key') == false
+          raise Fluent::ConfigError, '"count_key" is required.'
+        end
+        count_value = 1
+        if e.has_key?('count_value')
+          begin
+            count_value = Integer(e['count_value'])
+          rescue
+            raise Fluent::ConfigError, 'invalid "count_value", integer required.'
+          end
+        end
+        matches = {}
+        e.each_key { |key|
+          if key =~ /^match_/
+            name = key['match_'.size .. key.size]
+            matches[name] = Regexp.new(e[key])
+          end
+        }
+        @patterns << {
+          'matches' => matches,
+          'count_key' => e['count_key'],
+          'count_value' => count_value
+        }
+      }
     end
 
     def start
@@ -38,26 +66,42 @@ module Fluent
       chunk.open { |io|
         begin
           MessagePack::Unpacker.new(io).each { |record|
-            record.each_key { |key|
-              begin
-                value = Integer(record[key])
-                if value != 0
-                  table[key] += value
-                end
-              rescue ArgumentError
-                # convert to integer failed, do nothing.
-              rescue TypeError
-                # object can't convert to integer, do nothing.
-              end
-            }
+            matched_pattern = get_matched_pattern(record)
+            if matched_pattern != nil && matched_pattern['count_value'] != 0
+              table[matched_pattern['count_key']] += matched_pattern['count_value']
+            end
           }
         rescue EOFError
           # EOFError always occured when reached end of chunk.
         end
       }
       table.each_key { |key|
-        @redis.incrby(key, table[key])
+        if (value = table[key]) != 0
+          @redis.incrby(key, value)
+        end
+      }
+    end
+
+    private
+    def get_matched_pattern(record)
+      @patterns.each { |pattern|
+        all_matched = true
+        pattern['matches'].each_key{ |key|
+          if !record.has_key?(key)
+            all_matched = false
+            break
+          else
+            if !(record[key] =~ pattern['matches'][key])
+              all_matched = false
+              break
+            end
+          end
+        }
+        if all_matched
+          return pattern
+        end
       }
+      return nil
     end
   end
 end

data/test/plugin/out_redis_counter.rb

@@ -27,39 +27,91 @@ class RedisCounterTest < Test::Unit::TestCase
     assert_equal 'localhost', @d.instance.host
     assert_equal 6379, @d.instance.port
     assert_equal 1, @d.instance.db_number
+    assert_equal 0, @d.instance.patterns.size
   end
 
-  def test_format
-    @d.emit({"a" => 1})
-    @d.expect_format({"a" => 1}.to_msgpack)
-    @d.run
-  end
+  def test_configure_pattern
+    driver = create_driver %[
+      host localhost
+      port 6379
+      db_number 1
+      <pattern>
+        match_status ^2[0-9]{2}$
+        match_url ^https
+        count_key status-normal
+      </pattern>
+      <pattern>
+        count_key foo
+        count_value 2
+      </pattern>
+    ]
+    assert_equal 'localhost', driver.instance.host
+    assert_equal 6379, driver.instance.port
+    assert_equal 1, driver.instance.db_number
+    assert_equal 2, driver.instance.patterns.size
 
-  def test_write
-    @d.emit({"a" => 2})
-    @d.emit({"a" => 3})
-    @d.emit({"a" => "foo"})
-    @d.emit({"a" => -1, "b" => 1})
-    @d.run
+    assert_equal 2, driver.instance.patterns[0]['matches'].size
+    assert_equal Regexp.new('^2[0-9]{2}$'), driver.instance.patterns[0]['matches']['status']
+    assert_equal Regexp.new('^https'), driver.instance.patterns[0]['matches']['url']
+    assert_equal 'status-normal', driver.instance.patterns[0]['count_key']
+    assert_equal 1, driver.instance.patterns[0]['count_value']
 
-    assert_equal "4", @d.instance.redis.get("a")
-    assert_equal "1", @d.instance.redis.get("b")
+    assert_equal 0, driver.instance.patterns[1]['matches'].size
+    assert_equal 'foo', driver.instance.patterns[1]['count_key']
+    assert_equal 2, driver.instance.patterns[1]['count_value']
   end
 
-  def test_write_with_float
-    @d.emit({"a" => "1.1"})
-    @d.emit({"a" => "2"})
-    @d.run
+  def test_configure_count_key_required
+    begin
+      create_driver %[
+        <pattern>
+          count_value 1
+        </pattern>
+      ]
+      flunk
+    rescue Fluent::ConfigError => e
+      assert_equal '"count_key" is required.', e.message
+    end
+  end
 
-    assert_equal "2", @d.instance.redis.get("a")
+  def test_configure_invalid_count_value
+    begin
+      create_driver %[
+        <pattern>
+          count_key foo
+          count_value a
+        </pattern>
+      ]
+      flunk
+    rescue Fluent::ConfigError => e
+      assert_equal 'invalid "count_value", integer required.', e.message
+    end
   end
 
-  def test_write_with_object
+  def test_format
     @d.emit({"a" => 1})
-    @d.emit({"a" => {"foo" => 1}})
+    @d.expect_format({"a" => 1}.to_msgpack)
     @d.run
+  end
+
+  def test_write
+    driver = create_driver %[
+      db_number 1
+      <pattern>
+        match_a ^2[0-9][0-9]$
+        count_key a
+        count_value 2
+      </pattern>
+    ]
+    driver.emit({"a" => "value-of-a"})
+    driver.emit({"a" => "200"})
+    driver.emit({"b" => "200"})
+    driver.emit({"aa" => "200"})
+    driver.emit({"a" => "2000"})
+    driver.run
 
-    assert_equal "1", @d.instance.redis.get("a")
+    assert_equal '2', driver.instance.redis.get("a")
+    assert_nil driver.instance.redis.get("b")
   end
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-redis-counter
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-06-15 00:00:00.000000000 Z
+date: 2012-06-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -32,7 +32,7 @@ dependencies:
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ~>
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 2.2.2
   type: :runtime
@@ -40,7 +40,7 @@ dependencies:
   version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ~>
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 2.2.2
 description: fluent-plugin-redis-counter is a fluent plugin to count-up/down redis