fluent-plugin-redis 0.2.3 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f837fbede1a7e35897c7f8dd4b6f525f026c4f98
-  data.tar.gz: e382edd5215f8fcd18741da1dfff0586f889833d
+  metadata.gz: 76ed9616d363f33920c8a1dd8e4e5596f2a35902
+  data.tar.gz: 5784039d3c0b6f7ee7d4c7e28a054d54faf91220
 SHA512:
-  metadata.gz: 43b836f1368931c4ed2076a739fac49dc847296c7dd94d321f3ae5e91658f14da8f2aa416288eb48a6a7f0cdbe7777ea17b38e1dbc4ad86d917ba6d73c7af8ae
-  data.tar.gz: a1496462b70612c6eb5c35ab534b9c327603cc03b5e7507a05855fd0117904872fc9f562f3e565369b1a59e530b64c79e5ce58bd17eedd2fe76a498e0f27bf3a
+  metadata.gz: c31c34bbd4d1f441b463a256fbee00a34bee3886fbf24619ad7940685e16965db707864831a372195f63476bd6956e4294d26dde7dea6cc4a549f0dcfdee8951
+  data.tar.gz: e76f25044068976638e066fd2dfc55622cb02bdfb2a1fff5048cd1d8909c8ade7e1250fe942d00d414720d47fc7332fcbc2da28618974a4e0b9ffe600fa3c5c1

data/.travis.yml

@@ -6,6 +6,7 @@ rvm:
   - 2.2
   - 2.3.3
 gemfile:
-  - gemfiles/fluentd_v0.12.gemfile
+  - gemfiles/fluentd_v0.14.gemfile
+  - Gemfile
 before_install:
   - gem update bundler

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 0.3.0
+
+* Migrate v0.14 API based plugin
+* Allow duplicate insert key to support update values functionality
+* Use v0.14's built-in placeholder functionality
+  * Enabled to specify more flexible tag and time format for identifier
+
 ## 0.2.2
 
 * Use redis-rb 3.2.x

data/README.rdoc

@@ -2,6 +2,16 @@
 
 fluent-plugin-redis is a fluent plugin to output to redis.
 
+== Requirements
+
+    |fluent-plugin-redis|     fluentd      |  ruby  |
+    |-------------------|------------------|--------|
+    |     >= 0.3.0      | >= 0.14.8        | >= 2.1 |
+    |     == 0.2.3      | ~> 0.12.0 *      | >= 1.9 |
+    |      < 0.2.3      | >= 0.10.0, < 2 * | >= 1.9 |
+
+    * May not support all future fluentd features
+
 == Installation
 
 What you have to do is only installing like this:
@@ -13,7 +23,7 @@ Then fluent automatically loads the plugin installed.
 == Configuration
 
     <match redis.**>
-      type redis
+      @type redis
 
       host localhost
       port 6379
@@ -22,9 +32,23 @@ Then fluent automatically loads the plugin installed.
       db_number 0        # 0 is default
       # If requirepass is set, please specify this.
       # password hogefuga
+      # Users can set '${tag}' or ${tag[0]}.${tag[1]} or ...?
+      # insert_key_prefix '${tag}'
+      # Users can set strftime format.
+      # "%s" will be replaced into unixtime.
+      # "%Y%m%d.%H%M%S" will be replaced like as 20161202.112335.
+      # strftime_format "%s"
+      # Allow insert key duplicate. It will work as update values.
+      # allow_duplicate_key true
     </match>
 
 
+=== Notice
+
+<em>insert_key_prefix</em>, <em>strftime_format</em>, and <em>allow_duplicate_key</em> are newly added config parameters.
+
+They can use v0.3.0 or later. To use this parameters, users must update Fluentd to v0.14 or later and this plugin to v0.3.0 or later.
+
 == Contributing to fluent-plugin-redis
  
 * Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet

data/Rakefile

@@ -7,4 +7,4 @@ Rake::TestTask.new(:test) do |test|
   test.verbose = true
 end
 
-task :default => :test
+task default: :test

data/fluent-plugin-redis.gemspec

@@ -3,10 +3,10 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-redis"
-  s.version     = "0.2.3"
+  s.version     = "0.3.0"
   s.platform    = Gem::Platform::RUBY
   s.authors     = ["Yuki Nishijima", "Hiroshi Hatake", "Kenji Okimoto"]
-  s.date        = %q{2016-12-01}
+  s.date        = %q{2016-12-06}
   s.email       = ["mail@yukinishijima.net", "fluent@clear-code.com"]
   s.homepage    = "http://github.com/yuki24/fluent-plugin-redis"
   s.summary     = "Redis output plugin for Fluent"
@@ -16,7 +16,7 @@ Gem::Specification.new do |s|
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib"]
 
-  s.add_dependency %q<fluentd>, ["~> 0.12.0"]
+  s.add_dependency %q<fluentd>, [">= 0.14.8", "< 2"]
   s.add_dependency %q<redis>, ["~> 3.3.0"]
   s.add_development_dependency %q<rake>, [">= 11.3.0"]
   s.add_development_dependency %q<bundler>

data/lib/fluent/plugin/out_redis.rb

@@ -1,40 +1,53 @@
-module Fluent
-  class RedisOutput < BufferedOutput
+require 'redis'
+require 'msgpack'
+require 'fluent/msgpack_factory'
+require 'fluent/plugin/output'
+
+module Fluent::Plugin
+  class RedisOutput < Output
+    include Fluent::MessagePackFactory::Mixin
+
     Fluent::Plugin.register_output('redis', self)
-    attr_reader :redis
 
-    config_param :host, :string, :default => 'localhost'
-    config_param :port, :integer, :default => 6379
-    config_param :db_number, :integer, :default => 0
-    config_param :password, :string, :default => nil, :secret => true
+    helpers :compat_parameters, :inject
 
-    # To support log_level option implemented by Fluentd v0.10.43
-    unless method_defined?(:log)
-      define_method("log") { $log }
-    end
+    DEFAULT_BUFFER_TYPE = "memory"
 
-    def initialize
-      super
-      require 'redis'
-      require 'msgpack'
+    attr_reader :redis
+
+    config_param :host, :string, default: 'localhost'
+    config_param :port, :integer, default: 6379
+    config_param :db_number, :integer, default: 0
+    config_param :password, :string, default: nil, secret: true
+    config_param :insert_key_prefix, :string, default: "${tag}"
+    config_param :strftime_format, :string, default: "%s"
+    config_param :allow_duplicate_key, :bool, default: false
+
+    config_section :buffer do
+      config_set_default :@type, DEFAULT_BUFFER_TYPE
+      config_set_default :chunk_keys, ['tag', 'time']
+      config_set_default :timekey, 60
     end
 
     def configure(conf)
+      compat_parameters_convert(conf, :buffer, :inject)
       super
 
       if conf.has_key?('namespace')
         log.warn "namespace option has been removed from fluent-plugin-redis 0.1.3. Please add or remove the namespace '#{conf['namespace']}' manually."
       end
+      raise Fluent::ConfigError, "'tag' in chunk_keys is required." if not @chunk_key_tag
+      raise Fluent::ConfigError, "'time' in chunk_keys is required." if not @chunk_key_time
     end
 
     def start
       super
 
       options = {
-        :host => @host,
-        :port => @port,
-        :thread_safe => true,
-        :db => @db_number
+        host: @host,
+        port: @port,
+        thread_safe: true,
+        db: @db_number
       }
       options[:password] = @password if @password
 
@@ -47,22 +60,42 @@ module Fluent
     end
 
     def format(tag, time, record)
-      identifier = [tag, time].join(".")
-      [identifier, record].to_msgpack
+      record = inject_values_to_record(tag, time, record)
+      [tag, time, record].to_msgpack
+    end
+
+    def formatted_to_msgpack_binary
+      true
     end
 
     def write(chunk)
+      tag, time = expand_placeholders(chunk.metadata)
       @redis.pipelined {
-        chunk.open { |io|
-          begin
-            MessagePack::Unpacker.new(io).each.each_with_index { |record, index|
-              @redis.mapped_hmset "#{record[0]}.#{index}", record[1]
-            }
-          rescue EOFError
-            # EOFError always occured when reached end of chunk.
+        unless @allow_duplicate_key
+          chunk.open { |io|
+            begin
+              msgpack_unpacker(io).each.with_index { |record, index|
+                identifier = [tag, time].join(".")
+                @redis.mapped_hmset "#{identifier}.#{index}", record[2]
+              }
+            rescue EOFError
+              # EOFError always occured when reached end of chunk.
+            end
+          }
+        else
+          chunk.each do |_tag, _time, record|
+            @redis.mapped_hmset "#{tag}", record
           end
-        }
+        end
       }
     end
+
+    private
+
+    def expand_placeholders(metadata)
+      tag = extract_placeholders(@insert_key_prefix, metadata)
+      time = extract_placeholders(@strftime_format, metadata)
+      return tag, time
+    end
   end
 end

data/test/plugin/test_out_redis.rb

@@ -1,20 +1,31 @@
 require 'fluent/test'
+require 'fluent/test/helpers'
+require 'fluent/test/driver/output'
 require 'fluent/plugin/out_redis'
+require 'fluent/time' # Fluent::TimeFormatter
 
 class FileOutputTest < Test::Unit::TestCase
+  include Fluent::Test::Helpers
+
+  CONFIG = %[
+    host localhost
+    port 6379
+    db_number 1
+  ]
+
   def setup
     Fluent::Test.setup
 
-    @d = create_driver %[
-      host localhost
-      port 6379
-      db_number 1
-    ]
-    @time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    @d = create_driver
+    @time = event_time("2011-01-02 13:14:15 UTC")
   end
 
   def create_driver(conf = CONFIG)
-    Fluent::Test::BufferedOutputTestDriver.new(Fluent::RedisOutput).configure(conf)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::RedisOutput).configure(conf)
+  end
+
+  def time_formatter(inject_config)
+    Fluent::TimeFormatter.new(inject_config.time_format, inject_config.localtime, inject_config.timezone)
   end
 
   def test_configure
@@ -22,13 +33,13 @@ class FileOutputTest < Test::Unit::TestCase
     assert_equal 6379, @d.instance.port
     assert_equal 1, @d.instance.db_number
     assert_nil @d.instance.password
+    assert_equal '${tag}', @d.instance.insert_key_prefix
+    assert_equal '%s', @d.instance.strftime_format
+    assert_false @d.instance.allow_duplicate_key
   end
 
   def test_configure_with_password
-    d = create_driver %[
-      host localhost
-      port 6379
-      db_number 1
+    d = create_driver CONFIG + %[
       password testpass
     ]
     assert_equal 'localhost', d.instance.host
@@ -37,18 +48,104 @@ class FileOutputTest < Test::Unit::TestCase
     assert_equal 'testpass', d.instance.password
   end
 
+  def test_configure_without_tag_chunk_key
+    config = config_element('ROOT', '', {
+                              "host" => "localhost",
+                              "port" =>  6379,
+                              "db_number" => 1,
+                            }, [
+                              config_element('buffer', 'time', {
+                                               'chunk_keys' => 'time',
+                                             })
+                            ])
+    assert_raise Fluent::ConfigError do
+      create_driver(config)
+    end
+  end
+
   def test_format
-    @d.emit({"a"=>1}, @time)
-    @d.expect_format(["test.#{@time}", {"a"=>1}].to_msgpack)
-    @d.run
+    @d.run(default_tag: 'test') do
+      @d.feed(@time, {"a"=>1})
+    end
+    assert_equal [["test", @time, {"a"=>1}].to_msgpack], @d.formatted
+  end
+
+  class InjectTest < self
+    def test_format_inject_tag_keys
+      d = create_driver CONFIG + %[
+        include_tag_key true
+      ]
+      d.run(default_tag: 'test') do
+        d.feed(@time, {"a"=>1})
+      end
+      assert_equal [["test", @time, {"a"=>1, "tag" => "test"}].to_msgpack], d.formatted
+    end
+
+    def test_format_inject_time_keys
+      d = create_driver CONFIG + %[
+        include_time_key true
+      ]
+      timef = time_formatter(d.instance.inject_config)
+      d.run(default_tag: 'test') do
+        d.feed(@time, {"a"=>1})
+      end
+      assert_equal [["test", @time, {"a"=>1, "time" => timef.call(@time)}].to_msgpack], d.formatted
+    end
   end
 
-  def test_write
-    @d.emit({"a"=>2}, @time)
-    @d.emit({"a"=>3}, @time)
-    @d.run
+  class WriteTest < self
+    def test_write
+      d = create_driver
+      time = event_time("2011-01-02 13:14:00 UTC")
+      d.run(default_tag: 'test') do
+        d.feed(time, {"a"=>2})
+        d.feed(time, {"a"=>3})
+      end
+
+      assert_equal "2", d.instance.redis.hget("test.#{time}.0", "a")
+      assert_equal "3", d.instance.redis.hget("test.#{time}.1", "a")
+    end
+
+    def test_write_with_insert_key_prefix
+      d = create_driver CONFIG + %[
+        insert_key_prefix "${tag[1]}.${tag[2]}"
+      ]
+      time = event_time("2011-01-02 13:14:00 UTC")
+      d.run(default_tag: 'prefix.insert.test') do
+        d.feed(time, {"a"=>2})
+        d.feed(time, {"a"=>3})
+      end
+
+      assert_equal "2", d.instance.redis.hget("insert.test.#{time}.0", "a")
+      assert_equal "3", d.instance.redis.hget("insert.test.#{time}.1", "a")
+    end
+
+    def test_write_with_custom_strftime_format
+      d = create_driver CONFIG + %[
+        strftime_format "%Y%m%d.%H%M%S"
+      ]
+      time = event_time("2011-01-02 13:14:00 UTC")
+      strtime = Time.at(time.to_r).strftime("%Y%m%d.%H%M%S")
+      d.run(default_tag: 'test') do
+        d.feed(time, {"a"=>4})
+        d.feed(time, {"a"=>5})
+      end
+
+      assert_equal "4", d.instance.redis.hget("test.#{strtime}.0", "a")
+      assert_equal "5", d.instance.redis.hget("test.#{strtime}.1", "a")
+    end
+
+    def test_write_with_allow_duplicate
+      d = create_driver CONFIG + %[
+        allow_duplicate_key true
+      ]
+      time = event_time("2011-01-02 13:14:00 UTC")
+      d.run(default_tag: 'test.duplicate') do
+        d.feed(time, {"a"=>6})
+        d.feed(time, {"a"=>7})
+      end
 
-    assert_equal "2", @d.instance.redis.hget("test.#{@time}.0", "a")
-    assert_equal "3", @d.instance.redis.hget("test.#{@time}.1", "a")
+      assert_equal "7", d.instance.redis.hget("test.duplicate", "a")
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-redis
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.3.0
 platform: ruby
 authors:
 - Yuki Nishijima
@@ -10,22 +10,28 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-12-01 00:00:00.000000000 Z
+date: 2016-12-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.8
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 0.12.0
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.8
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 0.12.0
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: redis
   requirement: !ruby/object:Gem::Requirement