fluent-plugin-redis-render 0.2.2 → 0.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f88cba02f14dcd45add8833b10f8db1c1abe9e8be4deed2a0c890ff6dd30f4f3
-  data.tar.gz: 2ec2723f356b89cbb3e791e691e4bc40398e3c9be005e33d03d6201253c8c6cf
+  metadata.gz: ad7d27266ac4898fde77f36b733641955f84660b946f51c86df5e617c9ca93cd
+  data.tar.gz: 6ade50a10cebc63054d8908192f3c79cd27429f1db5551bc4d8080387f98fd41
 SHA512:
-  metadata.gz: 86ba7d0f40051f12910dba8104d5dcdde6de5db27b16cc4d793bbc4570ea94fdd2acc4be39be88998fefcd1c1fc1ebd773e09d368e19195a95f38055395e980a
-  data.tar.gz: 2d2be36e3c733b27a49f380529c05bc67409765e0ce4e15a9f380d8bc477cf3c1708f83072be5fabfc371decfe5b7da1fd4c2f2c45512f644bdbe303df63107d
+  metadata.gz: 0d4d7e9ce6eecae1e13760b4a508e0418bc445e00e472cdc7cf7fb90c5dea7e3a0601eb3ced523d2b2663a3430b9f9b55a480105d9e3d1891c7f6d3e5229617d
+  data.tar.gz: 4f289bd0c1a16a6de06797069a68eb29693bb8dc783b5df3a2e385b4b24b2174b19218d0ab3e1177bf1afe44a0b7aa50989aff3f1813d5c4c5d92fa63e5d234c

data/.gitignore

@@ -1 +1,4 @@
 *.gem
+.idea/
+vendor/
+.bundle/

data/Gemfile.lock

@@ -1,41 +1,39 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-redis-render (0.2.1)
+    fluent-plugin-redis-render (0.2.4)
       fluentd (>= 0.14.15, < 2)
       redis
 
 GEM
   remote: http://rubygems.org/
   specs:
-    cool.io (1.5.4)
-    dig_rb (1.0.1)
-    fluentd (1.4.2)
+    concurrent-ruby (1.1.6)
+    cool.io (1.6.0)
+    fluentd (1.10.3)
       cool.io (>= 1.4.5, < 2.0.0)
-      dig_rb (~> 1.0.0)
       http_parser.rb (>= 0.5.1, < 0.7.0)
-      msgpack (>= 0.7.0, < 2.0.0)
+      msgpack (>= 1.3.1, < 2.0.0)
       serverengine (>= 2.0.4, < 3.0.0)
       sigdump (~> 0.2.2)
       strptime (>= 0.2.2, < 1.0.0)
-      tzinfo (~> 1.0)
+      tzinfo (>= 1.0, < 3.0)
       tzinfo-data (~> 1.0)
       yajl-ruby (~> 1.0)
     http_parser.rb (0.6.0)
-    msgpack (1.2.10)
+    msgpack (1.3.3)
     power_assert (1.1.4)
     rake (12.3.2)
-    redis (4.1.0)
-    serverengine (2.1.0)
+    redis (4.1.4)
+    serverengine (2.2.1)
       sigdump (~> 0.2.2)
     sigdump (0.2.4)
     strptime (0.2.3)
     test-unit (3.3.2)
       power_assert
-    thread_safe (0.3.6)
-    tzinfo (1.2.5)
-      thread_safe (~> 0.1)
-    tzinfo-data (1.2019.1)
+    tzinfo (2.0.2)
+      concurrent-ruby (~> 1.0)
+    tzinfo-data (1.2020.1)
       tzinfo (>= 1.0.0)
     yajl-ruby (1.4.1)
 

data/fluent-plugin-redis-store.gemspec

@@ -1,13 +1,13 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |gem|
     gem.name        = "fluent-plugin-redis-render"
-    gem.email       = "hanai@pokelabo.co.jp"
-    gem.version     = "0.2.2"
+    gem.email       = "dev@render.com"
+    gem.version     = "0.2.4"
     gem.authors     = ["moaikids", "HANAI Tohru aka pokehanai"]
     gem.licenses    = ["Apache License Version 2.0"]
     gem.summary     = %q{Redis(zset/set/list/string/publish) output plugin for Fluentd}
     gem.description = %q{Redis(zset/set/list/string/publish) output plugin for Fluentd...}
-    gem.homepage    = "https://github.com/renderinc/fluent-plugin-redis-store"
+    gem.homepage    = "https://github.com/pokehanai/fluent-plugin-redis-store"
 
     gem.files         = `git ls-files`.split($\)
     gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }

data/lib/fluent/plugin/out_redis_store.rb

@@ -9,28 +9,28 @@ module Fluent::Plugin
     DEFAULT_BUFFER_TYPE = "memory"
 
     # redis connection
-    config_param :host, :string, :default => '127.0.0.1'
-    config_param :port, :integer, :default => 6379
-    config_param :path, :string, :default => nil
-    config_param :password, :string, :default => nil
-    config_param :db, :integer, :default => 0
-    config_param :timeout, :float, :default => 5.0
-    config_param :cluster_nodes, :string, :default => nil
+    config_param :host,      :string,  :default => '127.0.0.1'
+    config_param :port,      :integer, :default => 6379
+    config_param :path,      :string,  :default => nil
+    config_param :password,  :string,  :default => nil
+    config_param :db,        :integer, :default => 0
+    config_param :timeout,   :float,   :default => 5.0
 
     # redis command and parameters
-    config_param :format_type, :string, :default => 'json'
-    config_param :store_type, :string, :default => 'zset'
-    config_param :key_prefix, :string, :default => ''
-    config_param :key_suffix, :string, :default => ''
-    config_param :key, :string, :default => nil
-    config_param :key_path, :string, :default => nil
-    config_param :score_path, :string, :default => nil
-    config_param :value_path, :string, :default => ''
-    config_param :key_expire, :integer, :default => -1
-    config_param :value_expire, :integer, :default => -1
-    config_param :value_length, :integer, :default => -1
-    config_param :order, :string, :default => 'asc'
-    config_param :collision_policy, :string, :default => nil
+    config_param :format_type,       :string,  :default => 'json'
+    config_param :store_type,        :string,  :default => 'zset'
+    config_param :key_prefix,        :string,  :default => ''
+    config_param :key_suffix,        :string,  :default => ''
+    config_param :key,               :string,  :default => nil
+    config_param :key_path,          :string,  :default => nil
+    config_param :score_path,        :string,  :default => nil
+    config_param :value_path,        :string,  :default => ''
+    config_param :key_expire,        :integer, :default => -1
+    config_param :value_expire,      :integer, :default => -1
+    config_param :value_length,      :integer, :default => -1
+    config_param :max_list_length,   :integer, :default => 1000
+    config_param :order,             :string,  :default => 'asc'
+    config_param :collision_policy,  :string,  :default => nil
     config_set_default :flush_interval, 1
 
     config_section :buffer do
@@ -57,10 +57,6 @@ module Fluent::Plugin
       if @path
         @redis = Redis.new(:path => @path, :password => @password,
                            :timeout => @timeout, :thread_safe => true, :db => @db)
-      elsif @cluster_nodes
-        nodes = @cluster_nodes.split(' ')
-        @redis = Redis.new(:cluster => nodes, :password => @password,
-                           :timeout => @timeout, :thread_safe => true, :db => @db)
       else
         @redis = Redis.new(:host => @host, :port => @port, :password => @password,
                            :timeout => @timeout, :thread_safe => true, :db => @db)
@@ -85,34 +81,36 @@ module Fluent::Plugin
     end
 
     def write(chunk)
-      chunk.open {|io|
-        begin
-          MessagePack::Unpacker.new(io).each {|message|
-            begin
-              (_, time, record) = message
-              case @store_type
-              when 'zset'
-                operation_for_zset(record, time)
-              when 'set'
-                operation_for_set(record)
-              when 'list'
-                operation_for_list(record)
-              when 'string'
-                operation_for_string(record)
-              when 'publish'
-                operation_for_publish(record)
+      @redis.pipelined {
+        chunk.open { |io|
+          begin
+            MessagePack::Unpacker.new(io).each { |message|
+              begin
+                (_, time, record) = message
+                case @store_type
+                when 'zset'
+                  operation_for_zset(record, time)
+                when 'set'
+                  operation_for_set(record)
+                when 'list'
+                  operation_for_list(record)
+                when 'string'
+                  operation_for_string(record)
+                when 'publish'
+                  operation_for_publish(record)
+                end
+              rescue NoMethodError => e
+                puts e
+              rescue Encoding::UndefinedConversionError => e
+                log.error "Plugin error: " + e.to_s
+                log.error "Original record: " + record.to_s
+                puts e
               end
-            rescue NoMethodError => e
-              puts e
-            rescue Encoding::UndefinedConversionError => e
-              log.error "Plugin error: " + e.to_s
-              log.error "Original record: " + record.to_s
-              puts e
-            end
-          }
-        rescue EOFError
-          # EOFError always occured when reached end of chunk.
-        end
+            }
+          rescue EOFError
+            # EOFError always occured when reached end of chunk.
+          end
+        }
       }
     end
 
@@ -133,7 +131,7 @@ module Fluent::Plugin
       set_key_expire key
       if 0 < @value_expire
         now = Time.now.to_i
-        @redis.zremrangebyscore key, '-inf', (now - @value_expire)
+        @redis.zremrangebyscore key , '-inf' , (now - @value_expire)
       end
       if 0 < @value_length
         script = generate_zremrangebyrank_script(key, @value_length, @order)
@@ -157,6 +155,9 @@ module Fluent::Plugin
       else
         @redis.lpush key, value
       end
+
+      @redis.ltrim key, 0, @max_list_length - 1 if 0 < @max_list_length
+
       set_key_expire key
       if 0 < @value_length
         script = generate_ltrim_script(key, @value_length, @order)
@@ -179,7 +180,7 @@ module Fluent::Plugin
     end
 
     def generate_zremrangebyrank_script(key, maxlen, order)
-      script = "local key = '" + key.to_s + "'\n"
+      script  = "local key = '" + key.to_s + "'\n"
       script += "local maxlen = " + maxlen.to_s + "\n"
       script += "local order ='" + order.to_s + "'\n"
       script += "local len = tonumber(redis.call('ZCOUNT', key, '-inf', '+inf'))\n"
@@ -197,7 +198,7 @@ module Fluent::Plugin
     end
 
     def generate_ltrim_script(key, maxlen, order)
-      script = "local key = '" + key.to_s + "'\n"
+      script  = "local key = '" + key.to_s + "'\n"
       script += "local maxlen = " + maxlen.to_s + "\n"
       script += "local order ='" + order.to_s + "'\n"
       script += "local len = tonumber(redis.call('LLEN', key))\n"
@@ -214,7 +215,7 @@ module Fluent::Plugin
 
     def traverse(data, key)
       val = data
-      key.split('.').each {|k|
+      key.split('.').each{ |k|
         if val.has_key?(k)
           val = val[k]
         else
@@ -257,9 +258,7 @@ module Fluent::Plugin
     end
 
     def set_key_expire(key)
-      if 0 < @key_expire
-        @redis.expire key, @key_expire
-      end
+      @redis.expire key, @key_expire if 0 < @key_expire
     end
 
   end

data/test/plugin/test_out_redis_publish.rb

@@ -29,6 +29,13 @@ class Redis
     $message = message
   end
 
+  def ltrim(key, start, stop)
+    $command = :ltrim
+    $key = key
+    $start = start
+    $stop = stop
+  end
+
   def sadd(key, message)
     $command = :sadd
     $key = key
@@ -91,7 +98,7 @@ class RedisStoreOutputTest < Test::Unit::TestCase
     assert_equal(-1, d.instance.value_length)
     assert_equal('asc', d.instance.order)
     assert_equal(nil, d.instance.collision_policy)
-    assert_equal(nil, d.instance.cluster_nodes)
+    assert_equal(1000, d.instance.max_list_length)
   end
 
   def test_configure_host_port_db
@@ -237,6 +244,7 @@ class RedisStoreOutputTest < Test::Unit::TestCase
       format_type plain
       store_type list
       key_path   user
+      max_list_length 0
     ]
     d = create_driver(config)
     message = {
@@ -268,7 +276,7 @@ class RedisStoreOutputTest < Test::Unit::TestCase
       d.feed(get_time, message)
     end
 
-    assert_equal :lpush, $command
+    assert_equal :ltrim, $command
     assert_equal "george", $key
     assert_equal message, $message
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-redis-render
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.4
 platform: ruby
 authors:
 - moaikids
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-04-22 00:00:00.000000000 Z
+date: 2020-05-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -74,7 +74,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 description: Redis(zset/set/list/string/publish) output plugin for Fluentd...
-email: hanai@pokelabo.co.jp
+email: dev@render.com
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -89,7 +89,7 @@ files:
 - lib/fluent/plugin/out_redis_store.rb
 - test/helpers.rb
 - test/plugin/test_out_redis_publish.rb
-homepage: https://github.com/renderinc/fluent-plugin-redis-store
+homepage: https://github.com/pokehanai/fluent-plugin-redis-store
 licenses:
 - Apache License Version 2.0
 metadata: {}