fluent-plugin-redis-counter 0.1.0 → 0.1.1

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.1.1
+* add 'count_key_format' in order to make time-dependent redis keyname
+* bug fix
+
 ## 0.1.0
 * design changed.
 

data/README.md

@@ -28,6 +28,15 @@ fluent-plugin-redis-counter is hosted by [RubyGems.org](https://rubygems.org/).
         count_key foo-status2xx           # key-name for Redis
         count_value 1                     # count-up amount(default: 1, negative value is allowed)
       </pattern>
+
+      # time-dependent redis keyname
+      # for example, "foo-status2xx-%Y-%m-%d" will be formatted to "foo-status2xx-2012-06-21".
+      # rules for placeholder(%Y, etc.) is similar to out_file plugin.
+      <pattern>
+        match_status ^2[0-9][0-9]$
+        count_key_format foo-statux2xx-%d
+        localtime                          # time-zone(default: localtime, it can be "utc" or "localtime")
+      </pattern>
     </match>
 
 # Example

data/VERSION

@@ -1 +1 @@
-0.1.0
+0.1.1

data/fluent-plugin-redis-counter.gemspec

@@ -3,11 +3,11 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-redis-counter"
-  s.version     = "0.1.0"
+  s.version     = "0.1.1"
   s.description = "fluent-plugin-redis-counter is a fluent plugin to count-up/down redis keys."
   s.platform    = Gem::Platform::RUBY
   s.authors     = ["Buntaro Okada"]
-  s.date        = %q{2012-06-19}
+  s.date        = %q{2012-06-21}
   s.email       = "kbinani.bt@gmail.com"
   s.homepage    = "https://github.com/kbinani/fluent-plugin-redis-counter"
   s.summary     = "Redis counter plugin for fluent"

data/lib/fluent/plugin/out_redis_counter.rb

@@ -15,32 +15,14 @@ module Fluent
       @port = conf.has_key?('port') ? conf['port'].to_i : 6379
       @db_number = conf.has_key?('db_number') ? conf['db_number'].to_i : nil
       @patterns = []
-      conf.elements.select { |e|
-        e.name == 'pattern'
-      }.each { |e|
-        if e.has_key?('count_key') == false
-          raise Fluent::ConfigError, '"count_key" is required.'
-        end
-        count_value = 1
-        if e.has_key?('count_value')
-          begin
-            count_value = Integer(e['count_value'])
-          rescue
-            raise Fluent::ConfigError, 'invalid "count_value", integer required.'
-          end
+      conf.elements.select { |element|
+        element.name == 'pattern'
+      }.each { |element|
+        begin
+          @patterns << Pattern.new(element)
+        rescue RedisCounterException => e
+          raise Fluent::ConfigError, e.message
         end
-        matches = {}
-        e.each_key { |key|
-          if key =~ /^match_/
-            name = key['match_'.size .. key.size]
-            matches[name] = Regexp.new(e[key])
-          end
-        }
-        @patterns << {
-          'matches' => matches,
-          'count_key' => e['count_key'],
-          'count_value' => count_value
-        }
       }
     end
 
@@ -57,7 +39,7 @@ module Fluent
     end
 
     def format(tag, time, record)
-      record.to_msgpack
+      [tag, time, record].to_msgpack
     end
 
     def write(chunk)
@@ -65,43 +47,86 @@ module Fluent
       table.default = 0
       chunk.open { |io|
         begin
-          MessagePack::Unpacker.new(io).each { |record|
-            matched_pattern = get_matched_pattern(record)
-            if matched_pattern != nil && matched_pattern['count_value'] != 0
-              table[matched_pattern['count_key']] += matched_pattern['count_value']
-            end
+          MessagePack::Unpacker.new(io).each { |message|
+            (tag, time, record) = message
+            @patterns.select { |pattern|
+              pattern.is_match?(record)
+            }.each{ |pattern|
+              table[pattern.get_count_key(time)] += pattern.count_value
+            }
           }
         rescue EOFError
           # EOFError always occured when reached end of chunk.
         end
       }
-      table.each_key { |key|
-        if (value = table[key]) != 0
-          @redis.incrby(key, value)
-        end
+      table.each_pair.select { |key, value|
+        value != 0
+      }.each { |key, value|
+        @redis.incrby(key, value)
       }
     end
 
-    private
-    def get_matched_pattern(record)
-      @patterns.each { |pattern|
-        all_matched = true
-        pattern['matches'].each_key{ |key|
-          if !record.has_key?(key)
-            all_matched = false
-            break
-          else
-            if !(record[key] =~ pattern['matches'][key])
-              all_matched = false
-              break
-            end
+    class RedisCounterException < Exception
+    end
+
+    class Pattern
+      attr_reader :matches, :count_value
+
+      def initialize(conf_element)
+        if !conf_element.has_key?('count_key') && !conf_element.has_key?('count_key_format')
+          raise RedisCounterException, '"count_key" or "count_key_format" is required.'
+        end
+        if conf_element.has_key?('count_key') && conf_element.has_key?('count_key_format')
+          raise RedisCounterException, 'both "count_key" and "count_key_format" are specified.'
+        end
+
+        if conf_element.has_key?('count_key')
+          @count_key = conf_element['count_key']
+        else
+          if conf_element.has_key?('localtime') && conf_element.has_key?('utc')
+            raise RedisCounterException, 'both "localtime" and "utc" are specified.'
+          end
+          is_localtime = true
+          if conf_element.has_key?('utc')
+            is_localtime = false
+          end
+          @count_key_format = TimeFormatter.new(conf_element['count_key_format'], is_localtime)
+        end
+
+        @count_value = 1
+        if conf_element.has_key?('count_value')
+          begin
+            @count_value = Integer(conf_element['count_value'])
+          rescue
+            raise RedisCounterException, 'invalid "count_value", integer required.'
+          end
+        end
+
+        @matches = {}
+        conf_element.each_pair.select { |key, value|
+          key =~ /^match_/
+        }.each { |key, value|
+          name = key['match_'.size .. key.size]
+          @matches[name] = Regexp.new(value)
+        }
+      end
+
+      def is_match?(record)
+        @matches.each_pair{ |key, value|
+          if !record.has_key?(key) || !(record[key] =~ value)
+            return false
           end
         }
-        if all_matched
-          return pattern
+        return true
+      end
+
+      def get_count_key(time)
+        if @count_key_format == nil
+          @count_key
+        else
+          @count_key_format.format(time)
         end
-      }
-      return nil
+      end
     end
   end
 end

data/test/plugin/out_redis_counter.rb

@@ -16,6 +16,7 @@ class RedisCounterTest < Test::Unit::TestCase
     )
     redis.del("a")
     redis.del("b")
+    redis.del("foo-2012-06-21")
     redis.quit
   end
 
@@ -50,15 +51,15 @@ class RedisCounterTest < Test::Unit::TestCase
     assert_equal 1, driver.instance.db_number
     assert_equal 2, driver.instance.patterns.size
 
-    assert_equal 2, driver.instance.patterns[0]['matches'].size
-    assert_equal Regexp.new('^2[0-9]{2}$'), driver.instance.patterns[0]['matches']['status']
-    assert_equal Regexp.new('^https'), driver.instance.patterns[0]['matches']['url']
-    assert_equal 'status-normal', driver.instance.patterns[0]['count_key']
-    assert_equal 1, driver.instance.patterns[0]['count_value']
+    assert_equal 2, driver.instance.patterns[0].matches.size
+    assert_equal Regexp.new('^2[0-9]{2}$'), driver.instance.patterns[0].matches['status']
+    assert_equal Regexp.new('^https'), driver.instance.patterns[0].matches['url']
+    assert_equal 'status-normal', driver.instance.patterns[0].get_count_key(Time.now.to_i)
+    assert_equal 1, driver.instance.patterns[0].count_value
 
-    assert_equal 0, driver.instance.patterns[1]['matches'].size
-    assert_equal 'foo', driver.instance.patterns[1]['count_key']
-    assert_equal 2, driver.instance.patterns[1]['count_value']
+    assert_equal 0, driver.instance.patterns[1].matches.size
+    assert_equal 'foo', driver.instance.patterns[1].get_count_key(Time.now.to_i)
+    assert_equal 2, driver.instance.patterns[1].count_value
   end
 
   def test_configure_count_key_required
@@ -70,7 +71,58 @@ class RedisCounterTest < Test::Unit::TestCase
       ]
       flunk
     rescue Fluent::ConfigError => e
-      assert_equal '"count_key" is required.', e.message
+      assert_equal '"count_key" or "count_key_format" is required.', e.message
+    end
+  end
+
+  def test_configure_count_key_duplicated
+    begin
+      create_driver %[
+        <pattern>
+          count_key foo
+          count_key_format foo-%Y
+        </pattern>
+      ]
+      flunk
+    rescue Fluent::ConfigError => e
+      assert_equal 'both "count_key" and "count_key_format" are specified.', e.message
+    end
+  end
+
+  def test_configure_count_key_format_utc
+    driver = create_driver %[
+      <pattern>
+        count_key_format foo-%Y-%m-%d-%H-%M-%S
+        utc
+      </pattern>
+    ]
+    time = Time.parse('2011-06-21 03:12:01 UTC').to_i
+    assert_equal 'foo-2011-06-21-03-12-01', driver.instance.patterns[0].get_count_key(time)
+  end
+
+  def test_configure_count_key_format_localtime
+    driver = create_driver %[
+      <pattern>
+        count_key_format foo-%Y-%m-%d-%H-%M-%S
+        localtime
+      </pattern>
+    ]
+    local_time = Time.parse('2012-06-21 03:12:00').to_i
+    assert_equal 'foo-2012-06-21-03-12-00', driver.instance.patterns[0].get_count_key(local_time)
+  end
+
+  def test_configure_duplicated_timezone
+    begin
+      create_driver %[
+        <pattern>
+          count_key_format foo%Y
+          localtime
+          utc
+        </pattern>
+      ]
+      flunk
+    rescue Fluent::ConfigError => e
+      assert_equal 'both "localtime" and "utc" are specified.', e.message
     end
   end
 
@@ -89,8 +141,9 @@ class RedisCounterTest < Test::Unit::TestCase
   end
 
   def test_format
-    @d.emit({"a" => 1})
-    @d.expect_format({"a" => 1}.to_msgpack)
+    time = Time.parse('2012-06-21 01:55:00 UTC').to_i
+    @d.emit({"a" => 1}, time)
+    @d.expect_format(['test', time, {"a" => 1}].to_msgpack)
     @d.run
   end
 
@@ -114,4 +167,20 @@ class RedisCounterTest < Test::Unit::TestCase
     assert_nil driver.instance.redis.get("b")
   end
 
+  def test_write_with_timeformat
+    driver = create_driver %[
+      db_number 1
+      <pattern>
+        match_a ^2[0-9][0-9]$
+        count_key_format foo-%Y-%m-%d
+        count_value 2
+      </pattern>
+    ]
+    time = Time.parse('2012-06-21 03:01:00 UTC').to_i
+    driver.emit({"a" => "200"}, time)
+    driver.run
+
+    assert_equal '2', driver.instance.redis.get("foo-2012-06-21")
+  end
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-redis-counter
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-06-19 00:00:00.000000000 Z
+date: 2012-06-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd