fluent-plugin-redis-counter 0.3.0 → 0.3.1

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    YjcyZmZkMWJmZWM1Y2IyMDg5YzY5OTc0Y2M2NDEyOWJjMDdlODBhNw==
+    YjI2ZDVjZjQ2ZjkyZGE4N2VlZDIxOTdhMGQzNWQ5MTUzYzUwYmIwYw==
   data.tar.gz: !binary |-
-    ODk1ZmUyNWRlNjQzMWI1OWJjZTYyMDdlN2Y1ZjBkOTI3MWVmNGMzMw==
+    MTEyN2U5MDlhZmQ0NmZlNjhiMDEzNmFhNmYwYzdhMjA5NDliMDEwNw==
 !binary "U0hBNTEy":
   metadata.gz: !binary |-
-    M2FjODc0ZjUwNzU1M2E3M2VmYWIzYmUyOTE5N2U3ZWU0ZDAxZjRhMzhjM2Yw
-    ZjExNmE5OTVkNTk2YTZhODJiYTdhNjBhMmU0MzAwMDk0ZWU1ODZmNjBmMDUz
-    NDc3MGE0ODRkYzZhYTkwZWIyYWYzNWNjZDRhMjI2NjhlOWQxMjE=
+    MmEwMjA0YTkxM2NhNWZiMWY5NmY1MzdjNTg0NDA3ZWI5YzQ3NTI5ZjhmNzQz
+    MGVkZDkxYmZlYjlhM2NjNDY4ZWMwNGE2MWFlODhkNjgzMThjYTliYWFlN2Vm
+    YmJmOGI2OGRmZGQ5ZmY5YzY4ZTlmYjVhMzIwY2UwNTRlYTYzMmE=
   data.tar.gz: !binary |-
-    MjMyNmM4OWFmNzg4YmYzNGFlMWE5Yjk5OTBkYWQ0YzFmNTg5NTJlYTZmZDdh
-    ZDBlY2EwMzI0YjlkZDE5YjVjZWE2MDk1OTE2Mjc2MzI3ZjBlMGZlMzQ2MjZl
-    MThlNzQ2N2FhZmQyMTI1NzUyODUzNDIzMjA1NWI1YWFhOWM3NmE=
+    ZGUyYTI3MjYxZWRmYWVlY2M5OWEzZTI0ODM0MzNlODk4MmI1MGQ2NjU2Mzdj
+    MTdiYWEyNjAyOTA2MWQwNzQ3NmQxNDhkY2RkOGJmYjc2ZTMzODAzNTk0M2Iz
+    NWJkMjY3ZWJiZmZmMjRhMTY4NDNmMjEwNzlhMGUyZDA2NTdjMGI=

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 0.3.1
+* improve performance
+
 ## 0.3.0
 * count value can be picked up from JSON data contents.
 

data/VERSION

@@ -1 +1 @@
-0.3.0
+0.3.1

data/benchmark/performance_test.rb

@@ -0,0 +1,99 @@
+require 'fluent/test'
+require 'benchmark'
+
+if ENV["PROFILE"]
+require 'ruby-prof'
+end
+
+$LOAD_PATH.unshift(File.expand_path(File.join(File.dirname(__FILE__), "..", "lib")))
+require 'fluent/plugin/out_redis_counter'
+
+module PerformanceTestApp
+  class Benchmarker
+    def initialize
+      Fluent::Test.setup
+      @fluent = Fluent::Test::BufferedOutputTestDriver.new(Fluent::RedisCounterOutput).configure(%[
+          host localhost
+          port 6379
+          db_number 1
+          <pattern>
+            match_status ^2[0-9]{2}$
+            count_key_format status-normal:item_id:%_{item_id}
+            count_value 1
+          </pattern>
+      ])
+    end
+
+    def prepare
+      10_0000.times do
+        item_id = ( rand * 10_0000 ).to_i
+        @fluent.emit({ "status" => "200", "item_id" => item_id })
+      end
+    end
+
+    def run
+      result = nil
+      profile = nil
+      Benchmark.bm do |x|
+        result = x.report {
+          if ENV["PROFILE"]
+            profile = RubyProf.profile do
+              @fluent.run
+            end
+          else
+              @fluent.run
+          end
+        }
+      end
+
+      if ENV["PROFILE"]
+        profile_printer = RubyProf::GraphPrinter.new(profile)
+        profile_printer.print(STDOUT, {})
+      end
+      $log.info("benchmark result: #{result}")
+    end
+  end
+
+  class MemoryWatcher
+    def initialize
+    end
+
+    def run(&block)
+      start_memory = get_memory(Process.pid)
+      pid = Process.fork do
+        block.call
+      end
+      @max_memory = 0
+      th = Thread.new do
+        begin
+          loop do
+            sleep(0.01)
+            msize = get_memory(pid)
+            if msize > @max_memory
+              @max_memory = msize
+            end
+          end
+        ensure
+          $log.info("start memory size:\t#{sprintf("%#10d", start_memory / 1024)}KB")
+          $log.info("max memory size:\t#{sprintf("%#10d", @max_memory / 1024)}KB")
+        end
+      end
+      Process.waitall
+      th.kill
+    end
+
+    def get_memory(pid)
+      `ps -h -o rss= -p #{pid}`.to_i * 1024
+    end
+  end
+end
+
+if $0 == __FILE__
+  parent = PerformanceTestApp::MemoryWatcher.new
+  app = PerformanceTestApp::Benchmarker.new
+  parent.run do
+    app.prepare
+    app.run
+  end
+end
+

data/fluent-plugin-redis-counter.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-redis-counter"
-  s.version     = "0.3.0"
+  s.version     = "0.3.1"
   s.description = "fluent-plugin-redis-counter is a fluent plugin to count-up/down redis keys."
   s.platform    = Gem::Platform::RUBY
   s.authors     = ["Buntaro Okada"]

data/lib/fluent/plugin/out_redis_counter.rb

@@ -3,6 +3,8 @@ module Fluent
     Fluent::Plugin.register_output('redis_counter', self)
     attr_reader :host, :port, :db_number, :redis, :patterns
 
+    config_param :max_pipelining, :integer, :default => 1000
+
     def initialize
       super
       require 'redis'
@@ -61,8 +63,12 @@ module Fluent
       }
       table.each_pair.select { |key, value|
         value != 0
-      }.each { |key, value|
-        @redis.incrby(key, value)
+      }.each_slice(@max_pipelining) { |items|
+        @redis.pipelined do
+          items.each do |key, value|
+            @redis.incrby(key, value)
+          end
+        end
       }
     end
 
@@ -75,10 +81,11 @@ module Fluent
         @format = format
       end
 
+      CUSTOM_KEY_EXPRESSION_RE = /(%_\{([^\}]+)\})/
+
       def key(record)
-        @format.gsub(/(%_\{[^\}]+\})/) do |s|
-          key = s.match(/\{([^\}]+)\}/)[1]
-          record[key]
+        @format.gsub(CUSTOM_KEY_EXPRESSION_RE) do |s|
+          record[$2]
         end
       end
     end
@@ -105,6 +112,7 @@ module Fluent
             is_localtime = false
           end
           @count_key_format = [conf_element['count_key_format'], is_localtime]
+          @record_value_formatter = RecordValueFormatter.new(@count_key_format[0])
         end
 
         if conf_element.has_key?('count_value_key')
@@ -142,7 +150,7 @@ module Fluent
         if @count_key_format == nil
           @count_key
         else
-          count_key = RecordValueFormatter.new(@count_key_format[0]).key(record)
+          count_key = @record_value_formatter.key(record)
           formatter = TimeFormatter.new(count_key, @count_key_format[1])
           formatter.format(time)
         end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-redis-counter
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors:
 - Buntaro Okada
@@ -52,6 +52,7 @@ files:
 - README.md
 - Rakefile
 - VERSION
+- benchmark/performance_test.rb
 - fluent-plugin-redis-counter.gemspec
 - lib/fluent/plugin/out_redis_counter.rb
 - test/plugin/out_redis_counter.rb