fluent-plugin-redisstore 0.0.3 → 0.0.4

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+vendor/bundle

data/README.rdoc

@@ -28,6 +28,14 @@ this sources folked 'fluent-plugin-redis'. (https://github.com/yuki24/fluent-plu
       order asc(asc|desc, zset/list, nil)
     </match>
 
+    # use hiredis for better performance
+    # this configuration requires "gem install hiredis"
+    <match pattern>
+      type redisstore
+      driver hiredis
+      ...
+    </match>
+
 == Copyright
 
 Copyright:: Copyright (c) 2013 moaikids

data/benchmarking/.gitignore

@@ -0,0 +1 @@
+dummy.log

data/benchmarking/Gemfile

@@ -0,0 +1,5 @@
+source 'https://rubygems.org'
+
+gem 'fluentd'
+gem 'fluent-plugin-redisstore', :path => '../'
+gem 'dummer'

data/benchmarking/README.rdoc

@@ -0,0 +1,15 @@
+= Benchmark
+
+== worker
+
+    bundle exec fluentd -c default.conf
+    # or
+    bundle exec fluentd -c hiredis.conf
+
+== dummer
+
+    bundle exec dummer -c dummer.conf
+
+== monitor
+
+    bundle exec ruby monitor.rb

data/benchmarking/default.conf

@@ -0,0 +1,21 @@
+<source>
+  type tail
+  path dummy.log
+  pos_file /var/tmp/_var_log_dummy.pos
+  format none
+  tag dummy
+</source>
+<match dummy>
+  type redisstore
+  host 127.0.0.1
+  timeout 15.0
+  store_type zset
+  key_name method
+  key_suffix _suffix
+  score_name uri
+  value_name reqtime
+  key_expire 604800
+  value_expire 259200
+  value_length 10
+  flush_interval 0.1
+</match>

data/benchmarking/dummer.conf

@@ -0,0 +1,13 @@
+configure 'sample' do
+  output "dummy.log"
+  rate 100000
+  delimiter "\t"
+  labeled true
+  field :id, type: :integer, countup: true, format: "%04d"
+  field :time, type: :datetime, format: "[%Y-%m-%d %H:%M:%S]", random: false
+  field :level, type: :string, any: %w[DEBUG INFO WARN ERROR]
+  field :method, type: :string, any: %w[GET POST PUT]
+  field :uri, type: :string, any: %w[/api/v1/people /api/v1/textdata /api/v1/messages]
+  field :reqtime, type: :float, range: 0.1..5.0
+  field :foobar, type: :string, length: 8
+end 

data/benchmarking/hiredis.conf

@@ -0,0 +1,22 @@
+<source>
+  type tail
+  path dummy.log
+  pos_file /var/tmp/_var_log_dummy.pos
+  format none
+  tag dummy
+</source>
+<match dummy>
+  type redisstore
+  driver hiredis
+  host 127.0.0.1
+  timeout 15.0
+  store_type zset
+  key_name method
+  key_suffix _suffix
+  score_name uri
+  value_name reqtime
+  key_expire 604800
+  value_expire 259200
+  value_length 10
+  flush_interval 0.1
+</match>

data/benchmarking/monitor.rb

@@ -0,0 +1,30 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'terminal-table'
+require 'pp'
+require 'redis'
+require 'hiredis'
+
+class RedisInfo
+  def initialize(host,port)
+    @redis = Redis.new(:host => host, :port => port, driver: :hiredis)
+  end
+  def get_total_commands_processed
+    @redis.info["total_commands_processed"].to_i
+  end
+end
+
+chk = RedisInfo.new("127.0.0.1","6379")
+previous = 0
+
+res = []
+(1..30).each do |i|
+  current = chk.get_total_commands_processed
+  res << current - previous
+  previous = current
+  sleep 1
+end
+
+res.shift
+puts "#{1.0 * res.inject(:+) / res.size} req/s"

data/fluent-plugin-redisstore.gemspec

@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |gem|
     gem.name        = "fluent-plugin-redisstore"
-    gem.version     = "0.0.3"
+    gem.version     = "0.0.4"
     gem.authors     = ["moaikids"]
     gem.licenses    = ["Apache License Version 2.0"]
     gem.summary     = %q{Redis(zset/set/list/string) output plugin for Fluentd}

data/lib/fluent/plugin/out_redisstore.rb

@@ -12,6 +12,8 @@ module Fluent
     def configure(conf)
       super
 
+      @driver = conf.has_key?('driver') ? conf['driver'] : nil
+
       @host = conf.has_key?('host') ? conf['host'] : 'localhost'
       @port = conf.has_key?('port') ? conf['port'].to_i : 6379
       @db_number = conf.has_key?('db_number') ? conf['db_number'].to_i : nil
@@ -32,9 +34,26 @@ module Fluent
 
     def start
       super
+
+      opt = {
+        :host => @host,
+        :port => @port,
+        :db => @db_number,
+        :timeout => @timeout,
+        :thread_safe => true,
+      }
+
+      if @driver
+        opt[:driver] = @driver.to_sym
+      end
+
+      @redis = Redis.new(opt)
     end
 
     def shutdown
+      super
+
+      @redis.quit
     end
 
     def format(tag, time, record)
@@ -43,9 +62,6 @@ module Fluent
     end
 
     def write(chunk)
-      @redis = Redis.new(:host => @host, :port => @port, :timeout => @timeout,
-                         :thread_safe => true, :db => @db_number)
-
       @redis.pipelined {
         chunk.open { |io|
           begin
@@ -70,8 +86,6 @@ module Fluent
           end
         }
       }
-
-      @redis.quit
     end
 
     def operation_for_zset(record)
@@ -88,7 +102,7 @@ module Fluent
       end
       v = traverse(record, @value_name)
       sk = @key_prefix + k + @key_suffix
-      
+
       @redis.zadd sk , s, v
       if @key_expire > 0
         @redis.expire sk , @key_expire
@@ -110,7 +124,7 @@ module Fluent
       end
       v = traverse(record, @value_name)
       sk = @key_prefix + k + @key_suffix
-              
+
       @redis.sadd sk, v
       if @key_expire > 0
         @redis.expire sk, @key_expire
@@ -130,14 +144,14 @@ module Fluent
         @redis.rpush sk, v
       else
         @redis.lpush sk, v
-      end             
+      end
       if @key_expire > 0
         @redis.expire sk, @key_expire
       end
       if @value_length > 0
         script = generate_ltrim_script(sk, @value_length, @order)
         @redis.eval script
-      end 
+      end
     end
 
     def operation_for_string(record)
@@ -148,7 +162,7 @@ module Fluent
       end
       v = traverse(record, @value_name)
       sk = @key_prefix + k + @key_suffix
-         
+
       @redis.set sk, v
       if @key_expire > 0
         @redis.expire sk, @key_expire

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-redisstore
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-08-20 00:00:00.000000000 Z
+date: 2016-01-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -81,10 +81,18 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- .gitignore
 - AUTHORS
 - Gemfile
 - README.rdoc
 - Rakefile
+- benchmarking/.gitignore
+- benchmarking/Gemfile
+- benchmarking/README.rdoc
+- benchmarking/default.conf
+- benchmarking/dummer.conf
+- benchmarking/hiredis.conf
+- benchmarking/monitor.rb
 - fluent-plugin-redisstore.gemspec
 - lib/fluent/plugin/out_redisstore.rb
 homepage: https://github.com/moaikids/fluent-plugin-redisstore
@@ -102,7 +110,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -1048563133641840096
+      hash: 2621668993962248223
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -111,7 +119,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -1048563133641840096
+      hash: 2621668993962248223
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.23