fluent-plugin-redis 0.3.0 → 0.3.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 76ed9616d363f33920c8a1dd8e4e5596f2a35902
-  data.tar.gz: 5784039d3c0b6f7ee7d4c7e28a054d54faf91220
+SHA256:
+  metadata.gz: f4d9ddd2edc9aeb9d4c68c945677d964ee4c25bbfc308ec658b1098186adb175
+  data.tar.gz: 9a39e25338b0036b237bb08e5cfc033984bd1fc33d7ba39661e44a2b692a9ee2
 SHA512:
-  metadata.gz: c31c34bbd4d1f441b463a256fbee00a34bee3886fbf24619ad7940685e16965db707864831a372195f63476bd6956e4294d26dde7dea6cc4a549f0dcfdee8951
-  data.tar.gz: e76f25044068976638e066fd2dfc55622cb02bdfb2a1fff5048cd1d8909c8ade7e1250fe942d00d414720d47fc7332fcbc2da28618974a4e0b9ffe600fa3c5c1
+  metadata.gz: c66ca67dd12946c06f1a23479522a65ac5eef9f0b8b0b162380eb8eb5f12f19fe4dec0a209b0154ce73226e293ee1c089d32801adef8d72cd1dfd9daa9dcd94e
+  data.tar.gz: 159d112866122d37e233b37baa1aaea78371c350494ec816454e574c38f0701e1e0f797fd4a1eb288584f2826aab613bed41b6646d9d5ffa589f9a2a9286d8a0

data/.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,19 @@
+#### Problem
+
+...
+
+#### Steps to replicate
+
+Provide example config and message
+
+#### Expected Behavior
+
+...
+
+#### Your environment
+
+* OS version
+* paste result of ``fluentd --version`` or ``td-agent --version``
+* plugin version
+  * paste boot log of fluentd or td-agent
+  * paste result of ``fluent-gem list``, ``td-agent-gem list`` or your Gemfile.lock

data/.github/workflows/linux.yml

@@ -0,0 +1,31 @@
+name: Testing on Ubuntu
+on:
+  - push
+  - pull_request
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    services:
+      redis:
+        image: redis
+        ports:
+          - 6379:6379
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [ '2.5', '2.6', '2.7', '3.0' ]
+        os:
+          - ubuntu-latest
+    name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v2
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+    - name: unit testing
+      env:
+        CI: true
+      run: |
+        gem install bundler rake
+        bundle install --jobs 4 --retry 3
+        bundle exec rake test

data/CHANGELOG.md

@@ -1,3 +1,20 @@
+## 0.3.5
+* Confirm Ruby 3.0 ready
+
+## 0.3.4
+* Pass chunk directly into builtin placeholders
+
+## 0.3.3
+* Add multi workers environment considaration
+
+## 0.3.2
+* Add TTL support
+
+## 0.3.1
+
+* Use public msgpack APIs instead of its internal APIs
+* Remove needless rescue block
+
 ## 0.3.0
 
 * Migrate v0.14 API based plugin
@@ -38,4 +55,3 @@
 ## 0.0.1
 
 * This initial plugin is released.
-

data/{README.rdoc → README.md}

@@ -1,18 +1,18 @@
-= Redis output plugin for Fluent
+# Redis output plugin for Fluent
 
 fluent-plugin-redis is a fluent plugin to output to redis.
 
-== Requirements
+## Requirements
 
-    |fluent-plugin-redis|     fluentd      |  ruby  |
-    |-------------------|------------------|--------|
-    |     >= 0.3.0      | >= 0.14.8        | >= 2.1 |
-    |     == 0.2.3      | ~> 0.12.0 *      | >= 1.9 |
-    |      < 0.2.3      | >= 0.10.0, < 2 * | >= 1.9 |
+|fluent-plugin-redis|     fluentd      |  ruby  |
+|-------------------|------------------|--------|
+|     >= 0.3.0      | >= 0.14.8        | >= 2.1 |
+|     == 0.2.3      | ~> 0.12.0 *      | >= 1.9 |
+|      < 0.2.3      | >= 0.10.0, < 2 * | >= 1.9 |
 
-    * May not support all future fluentd features
+* May not support all future fluentd features
 
-== Installation
+## Installation
 
 What you have to do is only installing like this:
 
@@ -20,7 +20,7 @@ What you have to do is only installing like this:
 
 Then fluent automatically loads the plugin installed.
 
-== Configuration
+## Configuration
 
     <match redis.**>
       @type redis
@@ -40,17 +40,30 @@ Then fluent automatically loads the plugin installed.
       # strftime_format "%s"
       # Allow insert key duplicate. It will work as update values.
       # allow_duplicate_key true
+      # ttl 300 # If 0 or negative value is set, ttl is not set in each key.
     </match>
 
+### With multi workers
 
-=== Notice
+fluent-plugin-redis can handle <em>multi workers</em>.
+This feature can be enabled with the following configuration:
+
+    <system>
+      workers n # where n >= 2.
+    </system>
+
+### Notice
 
 <em>insert_key_prefix</em>, <em>strftime_format</em>, and <em>allow_duplicate_key</em> are newly added config parameters.
 
 They can use v0.3.0 or later. To use this parameters, users must update Fluentd to v0.14 or later and this plugin to v0.3.0 or later.
 
-== Contributing to fluent-plugin-redis
- 
+<em>multi workers</em> are newly introduced feature in Fluentd v0.14.
+
+It can use this feature in this plugin in v0.3.3 or later.
+
+## Contributing to fluent-plugin-redis
+
 * Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet
 * Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it
 * Fork the project
@@ -59,8 +72,10 @@ They can use v0.3.0 or later. To use this parameters, users must update Fluentd
 * Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
 * Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
 
+## Copyright
+
+Copyright (c) 2011- Yuki Nishijima
 
-== Copyright
+## License
 
-Copyright:: Copyright (c) 2011- Yuki Nishijima
-License::   Apache License, Version 2.0
+Apache License, Version 2.0

data/VERSION

@@ -1 +1 @@
-0.2.2
+0.3.1

data/fluent-plugin-redis.gemspec

@@ -3,12 +3,12 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-redis"
-  s.version     = "0.3.0"
+  s.version     = "0.3.5"
   s.platform    = Gem::Platform::RUBY
   s.authors     = ["Yuki Nishijima", "Hiroshi Hatake", "Kenji Okimoto"]
-  s.date        = %q{2016-12-06}
+  s.date        = %q{2019-02-26}
   s.email       = ["mail@yukinishijima.net", "fluent@clear-code.com"]
-  s.homepage    = "http://github.com/yuki24/fluent-plugin-redis"
+  s.homepage    = "https://github.com/fluent-plugins-nursery/fluent-plugin-redis"
   s.summary     = "Redis output plugin for Fluent"
 
   s.files         = `git ls-files`.split("\n")
@@ -16,10 +16,11 @@ Gem::Specification.new do |s|
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib"]
 
-  s.add_dependency %q<fluentd>, [">= 0.14.8", "< 2"]
+  s.add_dependency %q<fluentd>, [">= 0.14.22", "< 2"]
   s.add_dependency %q<redis>, ["~> 3.3.0"]
   s.add_development_dependency %q<rake>, [">= 11.3.0"]
   s.add_development_dependency %q<bundler>
-  s.add_development_dependency %q<test-unit>, ["~> 3.1.0"]
+  s.add_development_dependency %q<test-unit>, ["~> 3.3.9"]
   s.add_development_dependency %q<appraisal>, ["~> 2.1.0"]
+  s.add_development_dependency %q<webrick>
 end

data/lib/fluent/plugin/out_redis.rb

@@ -1,17 +1,15 @@
 require 'redis'
 require 'msgpack'
-require 'fluent/msgpack_factory'
 require 'fluent/plugin/output'
 
 module Fluent::Plugin
   class RedisOutput < Output
-    include Fluent::MessagePackFactory::Mixin
-
     Fluent::Plugin.register_output('redis', self)
 
     helpers :compat_parameters, :inject
 
     DEFAULT_BUFFER_TYPE = "memory"
+    DEFAULT_TTL_VALUE = -1
 
     attr_reader :redis
 
@@ -22,6 +20,7 @@ module Fluent::Plugin
     config_param :insert_key_prefix, :string, default: "${tag}"
     config_param :strftime_format, :string, default: "%s"
     config_param :allow_duplicate_key, :bool, default: false
+    config_param :ttl, :integer, default: DEFAULT_TTL_VALUE
 
     config_section :buffer do
       config_set_default :@type, DEFAULT_BUFFER_TYPE
@@ -31,6 +30,7 @@ module Fluent::Plugin
 
     def configure(conf)
       compat_parameters_convert(conf, :buffer, :inject)
+      @running_multi_workers = system_config.workers > 1
       super
 
       if conf.has_key?('namespace')
@@ -38,6 +38,7 @@ module Fluent::Plugin
       end
       raise Fluent::ConfigError, "'tag' in chunk_keys is required." if not @chunk_key_tag
       raise Fluent::ConfigError, "'time' in chunk_keys is required." if not @chunk_key_time
+      @unpacker = Fluent::Engine.msgpack_factory.unpacker
     end
 
     def start
@@ -68,23 +69,32 @@ module Fluent::Plugin
       true
     end
 
+    def multi_workers_ready?
+      true
+    end
+
     def write(chunk)
-      tag, time = expand_placeholders(chunk.metadata)
+      tag, time = expand_placeholders(chunk)
       @redis.pipelined {
         unless @allow_duplicate_key
-          chunk.open { |io|
-            begin
-              msgpack_unpacker(io).each.with_index { |record, index|
-                identifier = [tag, time].join(".")
-                @redis.mapped_hmset "#{identifier}.#{index}", record[2]
-              }
-            rescue EOFError
-              # EOFError always occured when reached end of chunk.
+          stream = chunk.to_msgpack_stream
+          @unpacker.feed_each(stream).with_index { |record, index|
+            identifier = if @running_multi_workers
+                           [tag, time, fluentd_worker_id].join(".")
+                         else
+                           [tag, time].join(".")
+                         end
+            @redis.multi do
+              @redis.mapped_hmset "#{identifier}.#{index}", record[2]
+              @redis.expire "#{identifier}.#{index}", @ttl if @ttl > 0
             end
           }
         else
           chunk.each do |_tag, _time, record|
-            @redis.mapped_hmset "#{tag}", record
+            @redis.multi do
+              @redis.mapped_hmset "#{tag}", record
+              @redis.expire "#{tag}", @ttl if @ttl > 0
+            end
           end
         end
       }
@@ -92,9 +102,9 @@ module Fluent::Plugin
 
     private
 
-    def expand_placeholders(metadata)
-      tag = extract_placeholders(@insert_key_prefix, metadata)
-      time = extract_placeholders(@strftime_format, metadata)
+    def expand_placeholders(chunk)
+      tag = extract_placeholders(@insert_key_prefix, chunk)
+      time = extract_placeholders(@strftime_format, chunk)
       return tag, time
     end
   end

data/test/plugin/test_out_redis.rb

@@ -36,6 +36,7 @@ class FileOutputTest < Test::Unit::TestCase
     assert_equal '${tag}', @d.instance.insert_key_prefix
     assert_equal '%s', @d.instance.strftime_format
     assert_false @d.instance.allow_duplicate_key
+    assert_equal Fluent::Plugin::RedisOutput::DEFAULT_TTL_VALUE, @d.instance.ttl
   end
 
   def test_configure_with_password
@@ -116,8 +117,25 @@ class FileOutputTest < Test::Unit::TestCase
         d.feed(time, {"a"=>3})
       end
 
-      assert_equal "2", d.instance.redis.hget("insert.test.#{time}.0", "a")
-      assert_equal "3", d.instance.redis.hget("insert.test.#{time}.1", "a")
+      assert_equal "2", d.instance.redis.hget("test.#{time}.0", "a")
+      assert_equal "3", d.instance.redis.hget("test.#{time}.1", "a")
+    end
+
+    def test_write_with_ttl
+      ttl = 2
+      d = create_driver CONFIG + %[
+        ttl #{ttl}
+        allow_duplicate_key true
+      ]
+      time = event_time("2011-01-02 13:14:00 UTC")
+      d.run(default_tag: 'ttl.insert.test') do
+        d.feed(time, {"a"=>2})
+      end
+
+      assert_in_delta 2.0, d.instance.redis.ttl("ttl.insert.test"), 1.0
+
+      sleep ttl+1
+      assert_equal -2, d.instance.redis.ttl("ttl.insert.test")
     end
 
     def test_write_with_custom_strftime_format

metadata

@@ -1,16 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-redis
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.5
 platform: ruby
 authors:
 - Yuki Nishijima
 - Hiroshi Hatake
 - Kenji Okimoto
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-12-06 00:00:00.000000000 Z
+date: 2019-02-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -18,7 +18,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.14.8
+        version: 0.14.22
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -28,7 +28,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.14.8
+        version: 0.14.22
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -80,14 +80,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.1.0
+        version: 3.3.9
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.1.0
+        version: 3.3.9
 - !ruby/object:Gem::Dependency
   name: appraisal
   requirement: !ruby/object:Gem::Requirement
@@ -102,7 +102,21 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 2.1.0
-description: 
+- !ruby/object:Gem::Dependency
+  name: webrick
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description:
 email:
 - mail@yukinishijima.net
 - fluent@clear-code.com
@@ -110,11 +124,12 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/ISSUE_TEMPLATE.md"
+- ".github/workflows/linux.yml"
 - ".gitignore"
-- ".travis.yml"
 - CHANGELOG.md
 - Gemfile
-- README.rdoc
+- README.md
 - Rakefile
 - VERSION
 - fluent-plugin-redis.gemspec
@@ -123,10 +138,10 @@ files:
 - gemfiles/fluentd_v0.14.gemfile
 - lib/fluent/plugin/out_redis.rb
 - test/plugin/test_out_redis.rb
-homepage: http://github.com/yuki24/fluent-plugin-redis
+homepage: https://github.com/fluent-plugins-nursery/fluent-plugin-redis
 licenses: []
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -141,9 +156,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.1
-signing_key: 
+rubygems_version: 3.1.2
+signing_key:
 specification_version: 4
 summary: Redis output plugin for Fluent
 test_files:

data/.travis.yml

@@ -1,12 +0,0 @@
-language: ruby
-services:
-  - redis-server
-rvm:
-  - 2.1
-  - 2.2
-  - 2.3.3
-gemfile:
-  - gemfiles/fluentd_v0.14.gemfile
-  - Gemfile
-before_install:
-  - gem update bundler