fluent-plugin-redis-store 0.1.2 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b6d5c8e74473ada6d1883030ccd153f6a246a0cc
-  data.tar.gz: '029eda873dfb56bbf74a7cbadc3edb17797cfd75'
+  metadata.gz: 8055a7eb193c6aba6625c44505354898b2ccb147
+  data.tar.gz: d173101f80f617b44bb57fff38fb43b954e32e94
 SHA512:
-  metadata.gz: ad5023960c0b604a27e5efe343127be8544d5fc1a6906fceba10c573f9f5c845e095b38488826ee6f0dd0483ecde09636907c833459ef08e10b003948d406a54
-  data.tar.gz: 78d5c22c9c417a9ce4a3f5679263fdc95173792dbe19e695cec53618f9740204ae96e5fd2b010965c886d2346ae5f8c4e63ee9e302b83a89c5a3e49bde6e6085
+  metadata.gz: f72795d871fc200ca056b3efce938a68c855665b090f7a26f3979e7314776be8fc3af69169b91b954d294586083c7a8c1774e4016f5bf1ee7fe74a4c0847d574
+  data.tar.gz: bc433609670879c4a05524bdcc0315aec37b1623a0fcd9cdb40b33375de849f15d722181255812b52783338f81e091a736bbe98ed1dbbd50e80c2d1912a8c0c1

data/.gitignore

@@ -0,0 +1 @@
+*.gem

data/README.md

@@ -43,7 +43,7 @@ For _key_, the plugin supports either way;
    type redis_store
    key userdata
    ```
-   
+
 2. Lookup a key string in every event data by a lookup path.  
    If event data have structured data like
 
@@ -83,7 +83,12 @@ To determine what _value_ in every event data to be srtored, you have two option
 Installation
 ------------
 
-    /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-redis-store
+```bash
+fluent-gem install fluent-plugin-redis-store
+
+# or if you are using td-agent:
+td-agent-gem install fluent-plugin-redis-store
+```
 
 Configuration
 -------------
@@ -92,7 +97,7 @@ Configuration
 
 | Key        | Type   | Required?   |                  Default | Description                                       |
 | :----      | :----- | :---------- | :----------------------- | :------------                                     |
-| `host`      | string | Optional    | 127.0.0.1               | host name of Redis server                               |
+| `host`     | string | Optional    |                127.0.0.1 | host name of Redis server                         |
 | `port`     | int    | Optional    |                     6379 | port number of Redis server                       |
 | `password` | string | Optional    |                          | password for Redis connection                     |
 | `path`     | string | Optional    |                          | To connect via Unix socket, try '/tmp/redis.sock' |
@@ -101,17 +106,17 @@ Configuration
 
 ### common options for storages
 
-| Key           | Type   | Default                  | Description                                          |
-| :----         | :----- | :----------------------- | :------------                                        |
-| `key`         | string |                          | Fixed _key_ used to store(publish) in Redis          |
-| `key_path`    | string |                          | path to lookup for _key_ in the event data           |
-| `key_prefix`  | string |                          | prefix of _key_                                      |
-| `key_suffix`  | string |                          | suffix of _key_                                      |
-| `value_path`  | string | (whole event data)       | path to lookup for _value_ in the event data         |
-| `store_type`  | string | zset                     | `string`/`list`/`set`/`zset`/`publish`               |
-| `format_type` | string | plain                    | format type for _value_ (`plain`/`json`/`msgpack`)   |
-| `key_expire`  | int    | -1                       | If set, the key will be expired in specified seconds |
-| `flush_interval`  | time    | 1                       | Time interval which events will be flushed to Redis |
+| Key              | Type   | Default                  | Description                                          |
+| :----            | :----- | :----------------------- | :------------                                        |
+| `key`            | string |                          | Fixed _key_ used to store(publish) in Redis          |
+| `key_path`       | string |                          | path to lookup for _key_ in the event data           |
+| `key_prefix`     | string |                          | prefix of _key_                                      |
+| `key_suffix`     | string |                          | suffix of _key_                                      |
+| `value_path`     | string | (whole event data)       | path to lookup for _value_ in the event data         |
+| `store_type`     | string | zset                     | `string`/`list`/`set`/`zset`/`publish`               |
+| `format_type`    | string | plain                    | format type for _value_ (`plain`/`json`/`msgpack`)   |
+| `key_expire`     | int    | -1                       | If set, the key will be expired in specified seconds |
+| `flush_interval` | time   | 1                        | Time interval which events will be flushed to Redis  |
 
 Note: either `key` or `key_path` is required.
 
@@ -134,10 +139,11 @@ No more options than common options.
 
 ### `zset` storage specific options
 
-| Key            | Type   | Default                  | Description                                  |
-| :----          | :----- | :----------------------- | :------------                                |
-| `score_path`   | string | (_time_ of log event)    | path to lookup for _score_ in the event data |
-| `value_expire` | int    |                          | value expiration in seconds                  |
+| Key                 | Type   | Default                  | Description                                                             |
+| :----               | :----- | :----------------------- | :------------                                                           |
+| `score_path`        | string | (_time_ of log event)    | path to lookup for _score_ in the event data                            |
+| `collision_policy`  | string | (nil)                    | Only update elements that already exist (XX) or add a new element (NX)  |
+| `value_expire`      | int    |                          | value expiration in seconds                                             |
 
 If `value_expire` is set, the plugin assumes that the _score_ in the **SortedSet** is
 based on *timestamp* and it deletes expired _members_ every after new event data arrives.
@@ -147,6 +153,17 @@ based on *timestamp* and it deletes expired _members_ every after new event data
 No more options than common options.
 
 
+Contributors
+------------
+
+moaikids  
+HANAI tohru  
+Mohit Khanna  
+yamada-shinji  
+Heitor de Souza  
+Hiroshi Hatake  
+Gabriel Bordeaux  
+
 Copyright
 ---------
 

data/fluent-plugin-redis-store.gemspec

@@ -2,7 +2,7 @@
 Gem::Specification.new do |gem|
     gem.name        = "fluent-plugin-redis-store"
     gem.email       = "hanai@pokelabo.co.jp"
-    gem.version     = "0.1.2"
+    gem.version     = "0.2.0"
     gem.authors     = ["moaikids", "HANAI Tohru aka pokehanai"]
     gem.licenses    = ["Apache License Version 2.0"]
     gem.summary     = %q{Redis(zset/set/list/string/publish) output plugin for Fluentd}
@@ -16,6 +16,6 @@ Gem::Specification.new do |gem|
 
     gem.add_development_dependency "rake"
     gem.add_development_dependency "test-unit"
-    gem.add_runtime_dependency "fluentd", "~> 0.12.0"
+    gem.add_runtime_dependency "fluentd", [">= 0.14.15", "< 2"]
     gem.add_runtime_dependency "redis"
 end

data/lib/fluent/plugin/out_redis_store.rb

@@ -1,7 +1,13 @@
-module Fluent
-  class RedisStoreOutput < BufferedOutput
+require 'fluent/plugin/output'
+
+module Fluent::Plugin
+  class RedisStoreOutput < Output
     Fluent::Plugin.register_output('redis_store', self)
 
+    helpers :compat_parameters
+
+    DEFAULT_BUFFER_TYPE = "memory"
+
     # redis connection
     config_param :host,      :string,  :default => '127.0.0.1'
     config_param :port,      :integer, :default => 6379
@@ -11,27 +17,33 @@ module Fluent
     config_param :timeout,   :float,   :default => 5.0
 
     # redis command and parameters
-    config_param :format_type,    :string,  :default => 'json'
-    config_param :store_type,     :string,  :default => 'zset'
-    config_param :key_prefix,     :string,  :default => ''
-    config_param :key_suffix,     :string,  :default => ''
-    config_param :key,            :string,  :default => nil
-    config_param :key_path,       :string,  :default => nil
-    config_param :score_path,     :string,  :default => nil
-    config_param :value_path,     :string,  :default => ''
-    config_param :key_expire,     :integer, :default => -1
-    config_param :value_expire,   :integer, :default => -1
-    config_param :value_length,   :integer, :default => -1
-    config_param :order,          :string,  :default => 'asc'
+    config_param :format_type,       :string,  :default => 'json'
+    config_param :store_type,        :string,  :default => 'zset'
+    config_param :key_prefix,        :string,  :default => ''
+    config_param :key_suffix,        :string,  :default => ''
+    config_param :key,               :string,  :default => nil
+    config_param :key_path,          :string,  :default => nil
+    config_param :score_path,        :string,  :default => nil
+    config_param :value_path,        :string,  :default => ''
+    config_param :key_expire,        :integer, :default => -1
+    config_param :value_expire,      :integer, :default => -1
+    config_param :value_length,      :integer, :default => -1
+    config_param :order,             :string,  :default => 'asc'
+    config_param :collision_policy,  :string,  :default => nil
     config_set_default :flush_interval, 1
 
+    config_section :buffer do
+      config_set_default :@type, DEFAULT_BUFFER_TYPE
+    end
+
     def initialize
       super
-      require 'redis'
+      require 'redis' unless defined?(Redis) == 'constant'
       require 'msgpack'
     end
 
     def configure(conf)
+      compat_parameters_convert(conf, :buffer)
       super
 
       if @key_path == nil and @key == nil
@@ -52,10 +64,19 @@ module Fluent
 
     def shutdown
       @redis.quit
+      super
     end
 
     def format(tag, time, record)
-      [tag, time, record].to_msgpack
+      [tag, time.to_f, record].to_msgpack
+    end
+
+    def formatted_to_msgpack_binary?
+      true
+    end
+
+    def multi_workers_ready?
+      true
     end
 
     def write(chunk)
@@ -64,7 +85,7 @@ module Fluent
           begin
             MessagePack::Unpacker.new(io).each { |message|
               begin
-                (tag, time, record) = message
+                (_, time, record) = message
                 case @store_type
                 when 'zset'
                   operation_for_zset(record, time)
@@ -79,6 +100,10 @@ module Fluent
                 end
               rescue NoMethodError => e
                 puts e
+              rescue Encoding::UndefinedConversionError => e
+                log.error "Plugin error: " + e.to_s
+                log.error "Original record: " + record.to_s
+                puts e
               end
             }
           rescue EOFError
@@ -92,7 +117,15 @@ module Fluent
       key = get_key_from(record)
       value = get_value_from(record)
       score = get_score_from(record, time)
-      @redis.zadd key, score, value
+      if @collision_policy
+        if @collision_policy == 'NX'
+          @redis.zadd(key, score, value, :nx => true)
+        elsif @collision_policy == 'XX'
+          @redis.zadd(key, score, value, :xx => true)
+        end
+      else
+        @redis.zadd(key, score, value)
+      end
 
       set_key_expire key
       if 0 < @value_expire

data/test/helpers.rb

@@ -12,6 +12,8 @@ require 'test/unit'
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 $LOAD_PATH.unshift(File.dirname(__FILE__))
 require 'fluent/test'
+require 'fluent/test/driver/output'
+require 'fluent/test/helpers'
 unless ENV.has_key?('VERBOSE')
   nulllogger = Object.new
   nulllogger.instance_eval {|obj|
@@ -25,4 +27,5 @@ end
 require 'fluent/plugin/out_redis_store'
 
 class Test::Unit::TestCase
+  include Fluent::Test::Helpers
 end

data/test/plugin/test_out_redis_publish.rb

@@ -1,7 +1,5 @@
 require 'helpers'
 
-require 'redis'
-
 $channel = nil
 $message = nil
 
@@ -65,7 +63,7 @@ class RedisStoreOutputTest < Test::Unit::TestCase
   end
 
   def create_driver(conf)
-    Fluent::Test::BufferedOutputTestDriver.new(Fluent::RedisStoreOutput).configure(conf)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::RedisStoreOutput).configure(conf)
   end
 
   def test_configure_defaults
@@ -74,24 +72,25 @@ class RedisStoreOutputTest < Test::Unit::TestCase
       score_path b
     ]
     d = create_driver(config)
-    assert_equal "127.0.0.1", d.instance.host
-    assert_equal 6379, d.instance.port
-    assert_equal nil, d.instance.path
-    assert_equal nil, d.instance.password
-    assert_equal 0, d.instance.db
-    assert_equal 5.0, d.instance.timeout
-    assert_equal 'json', d.instance.format_type
-    assert_equal '', d.instance.key_prefix
-    assert_equal '', d.instance.key_suffix
-    assert_equal 'zset', d.instance.store_type
-    assert_equal 'a', d.instance.key_path
-    assert_equal nil, d.instance.key
-    assert_equal 'b', d.instance.score_path
-    assert_equal '', d.instance.value_path
-    assert_equal -1, d.instance.key_expire
-    assert_equal -1, d.instance.value_expire
-    assert_equal -1, d.instance.value_length
-    assert_equal 'asc', d.instance.order
+    assert_equal("127.0.0.1", d.instance.host)
+    assert_equal(6379, d.instance.port)
+    assert_equal(nil, d.instance.path)
+    assert_equal(nil, d.instance.password)
+    assert_equal(0, d.instance.db)
+    assert_equal(5.0, d.instance.timeout)
+    assert_equal('json', d.instance.format_type)
+    assert_equal('', d.instance.key_prefix)
+    assert_equal('', d.instance.key_suffix)
+    assert_equal('zset', d.instance.store_type)
+    assert_equal('a', d.instance.key_path)
+    assert_equal(nil, d.instance.key)
+    assert_equal('b', d.instance.score_path)
+    assert_equal('', d.instance.value_path)
+    assert_equal(-1, d.instance.key_expire)
+    assert_equal(-1, d.instance.value_expire)
+    assert_equal(-1, d.instance.value_length)
+    assert_equal('asc', d.instance.order)
+    assert_equal(nil, d.instance.collision_policy)
   end
 
   def test_configure_host_port_db
@@ -134,16 +133,17 @@ class RedisStoreOutputTest < Test::Unit::TestCase
 #  def test_write
 #    d = create_driver(CONFIG1)
 #
-#    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-#    d.emit({ "foo" => "bar" }, time)
-#    d.run
+#    time = event_time("2011-01-02 13:14:15 UTC")
+#    d.run(default_tag 'test') do
+#      d.feed({ "foo" => "bar" }, time)
+#    end
 #
 #    assert_equal "test", $channel
 #    assert_equal(%Q[{"foo":"bar","time":#{time}}], $message)
 #  end
 
   def get_time
-    Time.parse("2011-01-02 13:14:15 UTC").to_i
+    event_time("2011-01-02 13:14:15 UTC")
   end
 
   # it should return whole message
@@ -159,8 +159,9 @@ class RedisStoreOutputTest < Test::Unit::TestCase
       'stat' => { 'attack' => 7 }
     }
     $ttl = nil
-    d.emit(message, get_time)
-    d.run
+    d.run(default_tag: 'test') do
+      d.feed(get_time, message)
+    end
 
     assert_equal "george", $key
     assert_equal message, $message
@@ -181,8 +182,9 @@ class RedisStoreOutputTest < Test::Unit::TestCase
       'stat' => { 'attack' => 7 }
     }
     $ttl = nil
-    d.emit(message, get_time)
-    d.run
+    d.run(default_tag: 'test') do
+      d.feed(get_time, message)
+    end
 
     assert_equal "george", $key
     assert_equal 7, $message
@@ -201,8 +203,9 @@ class RedisStoreOutputTest < Test::Unit::TestCase
       'stat' => { 'attack' => 7 }
     }
     $ttl = nil
-    d.emit(message, get_time)
-    d.run
+    d.run(default_tag: 'test') do
+      d.feed(get_time, message)
+    end
 
     assert_equal "george", $key
     assert_equal message.to_json, $message
@@ -220,8 +223,9 @@ class RedisStoreOutputTest < Test::Unit::TestCase
       'stat' => { 'attack' => 7 }
     }
     $ttl = nil
-    d.emit(message, get_time)
-    d.run
+    d.run(default_tag: 'test') do
+      d.feed(get_time, message)
+    end
 
     assert_equal "george", $key
     assert_equal message.to_msgpack, $message
@@ -238,8 +242,9 @@ class RedisStoreOutputTest < Test::Unit::TestCase
       'user' => 'george',
       'stat' => { 'attack' => 7 }
     }
-    d.emit(message, get_time)
-    d.run
+    d.run(default_tag: 'test') do
+      d.feed(get_time, message)
+    end
 
     assert_equal :rpush, $command
     assert_equal "george", $key
@@ -258,8 +263,9 @@ class RedisStoreOutputTest < Test::Unit::TestCase
       'user' => 'george',
       'stat' => { 'attack' => 7 }
     }
-    d.emit(message, get_time)
-    d.run
+    d.run(default_tag: 'test') do
+      d.feed(get_time, message)
+    end
 
     assert_equal :lpush, $command
     assert_equal "george", $key
@@ -278,8 +284,9 @@ class RedisStoreOutputTest < Test::Unit::TestCase
       'user' => 'george',
       'stat' => { 'attack' => 7 }
     }
-    d.emit(message, get_time)
-    d.run
+    d.run(default_tag: 'test') do
+      d.feed(get_time, message)
+    end
 
     assert_equal :sadd, $command
     assert_equal "george", $key
@@ -300,8 +307,9 @@ class RedisStoreOutputTest < Test::Unit::TestCase
       'stat' => { 'attack' => 7 },
       'result' => 81
     }
-    d.emit(message, get_time)
-    d.run
+    d.run(default_tag: 'test') do
+      d.feed(get_time, message)
+    end
 
     assert_equal :zadd, $command
     assert_equal "george", $key
@@ -322,8 +330,9 @@ class RedisStoreOutputTest < Test::Unit::TestCase
       'stat' => { 'attack' => 7 },
       'result' => 81
     }
-    d.emit(message, get_time)
-    d.run
+    d.run(default_tag: 'test') do
+      d.feed(get_time, message)
+    end
 
     assert_equal :zadd, $command
     assert_equal "george", $key
@@ -342,8 +351,9 @@ class RedisStoreOutputTest < Test::Unit::TestCase
     message = {
       'user' => 'george'
     }
-    d.emit(message, get_time)
-    d.run
+    d.run(default_tag: 'test') do
+      d.feed(get_time, message)
+    end
 
     assert_equal :publish, $command
     assert_equal "george", $channel
@@ -359,9 +369,30 @@ class RedisStoreOutputTest < Test::Unit::TestCase
 
     d = create_driver(config)
     message = { 'user' => 'george' }
-    d.emit(message, get_time)
-    assert_raise(Fluent::ConfigError) do
-      d.run
+    suppress_output do
+      assert_raise(Fluent::ConfigError) do
+        d.run(default_tag: 'test') do
+          d.feed(get_time, message)
+        end
+      end
+    end
+  end
+
+  def suppress_output
+    begin
+      original_stderr = $stderr.clone
+      original_stdout = $stdout.clone
+      $stderr.reopen(File.new('/dev/null', 'w'))
+      $stdout.reopen(File.new('/dev/null', 'w'))
+      retval = yield
+    rescue Exception => e
+      $stdout.reopen(original_stdout)
+      $stderr.reopen(original_stderr)
+      raise e
+    ensure
+      $stdout.reopen(original_stdout)
+      $stderr.reopen(original_stderr)
     end
+    retval
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-redis-store
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.2.0
 platform: ruby
 authors:
 - moaikids
@@ -43,16 +43,22 @@ dependencies:
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.15
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 0.12.0
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.15
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 0.12.0
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: redis
   requirement: !ruby/object:Gem::Requirement
@@ -73,6 +79,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".gitignore"
 - AUTHORS
 - Gemfile
 - README.md