fluent-plugin-ddl-redis-store 0.2.1 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1f204dfec7c332be1ea6915acb4a008cb957e10bf1686116403fe717fbd535d7
-  data.tar.gz: 281b6c3ce41822f415bb1774b3e9e3729cd96604ac15c2838207adbbad7854f6
+  metadata.gz: af55b99d427e9fe7a98c858d8e654677f2fe707694db40fcaf20b327a96f7ed1
+  data.tar.gz: c5e088703cf5d27aa7bbbad6cd6de33beed94027b86df0683e63fe845a2fc873
 SHA512:
-  metadata.gz: c7fb2cfb2a447f2a1d90e9b1d60467c5c24f5794c57b115b6cd522d2126ee66c911945eaea79533f023f2f4e67c1d68dd766b7d7bf6588434dc0d0f623575d64
-  data.tar.gz: d6fcfc4cb3e89b2602f327a43fb73aca598edcad5e6d9071748dac6455a71b7b2ecf3fbf15bc3c7b52daa0a5578d58668e452920ce8e17d9d955b754ed18d4eb
+  metadata.gz: f138decaa02bfdb0fe9410c70a14def933cce2ceb9484fba1d7d6e38a367fe311fd24a4d8060c1a916ea2e5fdaf8a07cd438fe1930572468cea1398c8155c59d
+  data.tar.gz: 42f7c294e76ab3bf5ad43e930fe73638cb4a5906d005b71fef0f04d3a0a2a945d22231a4e15a3c408ff43b18ba508b080a2e8daf0ec8c841683012dcac2a9700

data/.github/CODEOWNERS

@@ -0,0 +1 @@
+*   @cerebrotech/eng-distributions

data/.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,25 @@
+## PR Type
+What kind of change does this PR introduce?
+
+<!-- Please check the one that applies to this PR using "x". -->
+- [ ] Bugfix
+- [ ] Feature
+- [ ] Code style update (formatting)
+- [ ] Refactoring (no functional changes)
+- [ ] CI-related changes
+- [ ] Other... Please describe:
+
+## What is the current behavior?
+<!-- Please describe the current behavior that you are modifying, or link to a relevant issue. -->
+
+Issue Number: https://dominodatalab.atlassian.net/browse/PLAT-XYZ
+
+## What is the new behavior?
+
+## Does this PR introduce a breaking change?
+- [ ] Yes
+- [ ] No
+
+<!-- If this PR contains a breaking change, please describe the impact and migration path for existing applications below. -->
+
+## Other information

data/.github/workflows/test.yml

@@ -0,0 +1,44 @@
+name: test
+on:
+  push:
+    branches: [master]
+  pull_request:
+
+jobs:
+  unit:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby: ["3.2", "3.3", "3.4"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+      - run: bundle exec rake test
+
+  integration:
+    runs-on: ubuntu-latest
+    services:
+      redis:
+        image: redis:7-alpine
+        ports: ["6379:6379"]
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 5s
+          --health-timeout 3s
+          --health-retries 5
+    strategy:
+      matrix:
+        ruby: ["3.2", "3.3", "3.4"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+      - run: bundle exec rake test:integration
+        env:
+          REDIS_HOST: 127.0.0.1
+          REDIS_PORT: 6379

data/.gitignore

@@ -1 +1,8 @@
 *.gem
+pkg/
+.bundle/
+vendor/
+Gemfile.lock
+*.log
+.DS_Store
+coverage/

data/README.md

@@ -10,15 +10,31 @@ Requirements
 
 | fluent-plugin-redis-store | fluentd | ruby |
 |------------------------|---------|------|
-| >= 0.2.0 | >= v0.14.15 | >= 2.1 |
+| >= 0.3.0 | >= v0.14.15 | >= 3.2 |
+| >= 0.2.0, < 0.3.0 | >= v0.14.15 | >= 2.1 |
 |  < 0.2.0 | >= v0.12.0 | >= 1.9 |
 
 Background
 ----------
 
-This is folked project from [fluent-plugin-redisstore][].
+This is a forked project from [fluent-plugin-redis-store][].
+
+[fluent-plugin-redis-store]: https://github.com/pokehanai/fluent-plugin-redis-store/
+
+Release Process
+---------------
+1. Bump `version` in `fluent-plugin-ddl-redis-store.gemspec`
+2. Bump `version` in `domino.yml` to match. It is **not** kept in sync automatically —
+   unlike sibling `domino.yml`s that track an external base image via Renovate, this one
+   tracks the gem's own release number, which nothing here polls or rewrites for you.
+3. `gem build fluent-plugin-ddl-redis-store.gemspec`
+4. `gem push <.gem file>` (with the gem file generated in the previous step)  
+   Ex: `gem push fluent-plugin-ddl-redis-store-0.3.1.gem`
+
+The `push` command needs to be authenticated with [RubyGems.org](https://rubygems.org/).
+
+Latest release: https://rubygems.org/gems/fluent-plugin-ddl-redis-store
 
-[fluent-plugin-redisstore]: https://github.com/moaikids/fluent-plugin-redisstore
 
 Features
 --------
@@ -92,10 +108,10 @@ Installation
 ------------
 
 ```bash
-fluent-gem install fluent-plugin-redis-store
+fluent-gem install fluent-plugin-ddl-redis-store
 
 # or if you are using td-agent:
-td-agent-gem install fluent-plugin-redis-store
+td-agent-gem install fluent-plugin-ddl-redis-store
 ```
 
 Configuration
@@ -161,6 +177,31 @@ based on *timestamp* and it deletes expired _members_ every after new event data
 No more options than common options.
 
 
+Testing
+-------
+
+Unit tests (`test/plugin/`) stub out the `Redis` class entirely and only exercise
+config parsing and key/value traversal -- they cannot detect a redis-rb client API
+break.
+
+```bash
+bundle exec rake test
+```
+
+Integration tests (`test/integration/`) run against a real Redis server and assert
+on actual `ZCARD`/`LLEN`/`GET`/etc. Start one first:
+
+```bash
+docker run -d --rm -p 6379:6379 redis:7-alpine
+bundle exec rake test:integration
+```
+
+They read `REDIS_HOST`/`REDIS_PORT` from the environment (default `127.0.0.1:6379`)
+and are skipped with a message if no Redis is reachable.
+
+Note: redis-rb 6.x requires Ruby 3.2+; if your local Ruby is older, run both tasks
+inside a container that matches the target fluentd image's Ruby/redis-rb version.
+
 Contributors
 ------------
 

data/Rakefile

@@ -4,7 +4,14 @@ require "bundler/gem_tasks"
 require 'rake/testtask'
 Rake::TestTask.new(:test) do |test|
     test.libs << 'lib' << 'test'
-    test.pattern = 'test/**/test_*.rb'
+    test.pattern = 'test/plugin/**/test_*.rb'
+    test.verbose = true
+end
+
+desc "Integration tests against a real Redis server (see test/integration for setup)"
+Rake::TestTask.new('test:integration') do |test|
+    test.libs << 'lib' << 'test'
+    test.pattern = 'test/integration/**/test_*.rb'
     test.verbose = true
 end
 

data/domino.yml

@@ -0,0 +1,7 @@
+workflowVersion: 0
+name: fluent-plugin-redis-store
+type: library
+version: 0.3.1
+versionFormat: semver
+language:
+  name: shell

data/fluent-plugin-ddl-redis-store.gemspec

@@ -1,13 +1,13 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |gem|
     gem.name        = "fluent-plugin-ddl-redis-store"
-    gem.email       = "hanai@pokelabo.co.jp"
-    gem.version     = "0.2.1"
+    gem.email       = "builds@dominodatalab.com"
+    gem.version     = "0.3.1"
     gem.authors     = ["moaikids", "HANAI Tohru aka pokehanai"]
     gem.licenses    = ["Apache License Version 2.0"]
     gem.summary     = %q{Redis(zset/set/list/string/publish) output plugin for Fluentd}
     gem.description = %q{Redis(zset/set/list/string/publish) output plugin for Fluentd...}
-    gem.homepage    = "https://github.com/pokehanai/fluent-plugin-redis-store"
+    gem.homepage    = "https://github.com/dominodatalab/fluent-plugin-redis-store"
 
     gem.files         = `git ls-files`.split($\)
     gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
@@ -17,5 +17,5 @@ Gem::Specification.new do |gem|
     gem.add_development_dependency "rake"
     gem.add_development_dependency "test-unit"
     gem.add_runtime_dependency "fluentd", [">= 0.14.15", "< 2"]
-    gem.add_runtime_dependency "redis"
+    gem.add_runtime_dependency "redis", [">= 5.0", "< 7"]
 end

data/lib/fluent/plugin/out_redis_store.rb

@@ -55,10 +55,10 @@ module Fluent::Plugin
       super
       if @path
         @redis = Redis.new(:path => @path, :password => @password,
-                           :timeout => @timeout, :thread_safe => true, :db => @db)
+                           :timeout => @timeout, :db => @db)
       else
         @redis = Redis.new(:host => @host, :port => @port, :password => @password,
-                           :timeout => @timeout, :thread_safe => true, :db => @db)
+                           :timeout => @timeout, :db => @db)
       end
     end
 
@@ -80,7 +80,7 @@ module Fluent::Plugin
     end
 
     def write(chunk)
-      @redis.pipelined {
+      @redis.pipelined { |pipeline|
         chunk.open { |io|
           begin
             MessagePack::Unpacker.new(io).each { |message|
@@ -88,15 +88,15 @@ module Fluent::Plugin
                 (_, time, record) = message
                 case @store_type
                 when 'zset'
-                  operation_for_zset(record, time)
+                  operation_for_zset(pipeline, record, time)
                 when 'set'
-                  operation_for_set(record)
+                  operation_for_set(pipeline, record)
                 when 'list'
-                  operation_for_list(record)
+                  operation_for_list(pipeline, record)
                 when 'string'
-                  operation_for_string(record)
+                  operation_for_string(pipeline, record)
                 when 'publish'
-                  operation_for_publish(record)
+                  operation_for_publish(pipeline, record)
                 end
               rescue NoMethodError => e
                 puts e
@@ -113,66 +113,66 @@ module Fluent::Plugin
       }
     end
 
-    def operation_for_zset(record, time)
+    def operation_for_zset(pipeline, record, time)
       key = get_key_from(record)
       value = get_value_from(record)
       score = get_score_from(record, time)
       if @collision_policy
         if @collision_policy == 'NX'
-          @redis.zadd(key, score, value, :nx => true)
+          pipeline.zadd(key, score, value, :nx => true)
         elsif @collision_policy == 'XX'
-          @redis.zadd(key, score, value, :xx => true)
+          pipeline.zadd(key, score, value, :xx => true)
         end
       else
-        @redis.zadd(key, score, value)
+        pipeline.zadd(key, score, value)
       end
 
-      set_key_expire key
+      set_key_expire pipeline, key
       if 0 < @value_expire
         now = Time.now.to_i
-        @redis.zremrangebyscore key , '-inf' , (now - @value_expire)
+        pipeline.zremrangebyscore key , '-inf' , (now - @value_expire)
       end
       if 0 < @value_length
         script = generate_zremrangebyrank_script(key, @value_length, @order)
-        @redis.eval script
+        pipeline.eval script
       end
     end
 
-    def operation_for_set(record)
+    def operation_for_set(pipeline, record)
       key = get_key_from(record)
       value = get_value_from(record)
-      @redis.sadd key, value
-      set_key_expire key
+      pipeline.sadd key, value
+      set_key_expire pipeline, key
     end
 
-    def operation_for_list(record)
+    def operation_for_list(pipeline, record)
       key = get_key_from(record)
       value = get_value_from(record)
 
       if @order == 'asc'
-        @redis.rpush key, value
+        pipeline.rpush key, value
       else
-        @redis.lpush key, value
+        pipeline.lpush key, value
       end
-      set_key_expire key
+      set_key_expire pipeline, key
       if 0 < @value_length
         script = generate_ltrim_script(key, @value_length, @order)
-        @redis.eval script
+        pipeline.eval script
       end
     end
 
-    def operation_for_string(record)
+    def operation_for_string(pipeline, record)
       key = get_key_from(record)
       value = get_value_from(record)
-      @redis.set key, value
+      pipeline.set key, value
 
-      set_key_expire key
+      set_key_expire pipeline, key
     end
 
-    def operation_for_publish(record)
+    def operation_for_publish(pipeline, record)
       key = get_key_from(record)
       value = get_value_from(record)
-      @redis.publish key, value
+      pipeline.publish key, value
     end
 
     def generate_zremrangebyrank_script(key, maxlen, order)
@@ -253,9 +253,9 @@ module Fluent::Plugin
       end
     end
 
-    def set_key_expire(key)
+    def set_key_expire(pipeline, key)
       if 0 < @key_expire
-        @redis.expire key, @key_expire
+        pipeline.expire key, @key_expire
       end
     end
 

data/renovate.json

@@ -0,0 +1,19 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "description": "Renovate config",
+  "extends": [
+    "github>cerebrotech/renovate-config"
+  ],
+  "packageRules": [
+    {
+      "description": "Require manual review for major bumps of the redis gem; a two-major-version unattended jump (redis-rb 4.8 -> 6.0) previously broke fluentd at boot (Redis.new ArgumentError) and silently stopped batching in write() -- see the matching rule in the sibling operations-fluentd-docker renovate config",
+      "matchManagers": ["bundler"],
+      "matchPackageNames": ["redis"],
+      "matchUpdateTypes": ["major"],
+      "enabled": false
+    }
+  ],
+  "vulnerabilityAlerts": {
+    "enabled": true
+  }
+}

data/test/integration/test_out_redis_store_integration.rb

@@ -0,0 +1,151 @@
+require 'helpers'
+require 'redis'
+require 'securerandom'
+
+# These tests exercise the plugin against a real Redis server, unlike
+# test/plugin/test_out_redis_publish.rb which stubs out the entire Redis
+# class and therefore cannot detect a redis-rb client API break (e.g. the
+# `pipelined` block-argument change between redis-rb 4.x and 5.x/6.x).
+#
+# Run a real Redis first:
+#   docker run -d --rm -p 6379:6379 redis:7-alpine
+# then:
+#   bundle exec rake test:integration
+class RedisStoreOutputIntegrationTest < Test::Unit::TestCase
+  REDIS_HOST = ENV['REDIS_HOST'] || '127.0.0.1'
+  REDIS_PORT = (ENV['REDIS_PORT'] || 6379).to_i
+
+  def setup
+    Fluent::Test.setup
+    @redis = Redis.new(host: REDIS_HOST, port: REDIS_PORT, timeout: 2.0)
+    begin
+      @redis.ping
+    rescue StandardError => e
+      omit "no Redis reachable at #{REDIS_HOST}:#{REDIS_PORT} (#{e.class}: #{e.message});" \
+        " start one with `docker run -d --rm -p 6379:6379 redis:7-alpine` to run integration tests"
+    end
+    @key = "fluent-plugin-redis-store-test:#{SecureRandom.hex(8)}"
+  end
+
+  def teardown
+    @redis.del(@key) if @redis
+  end
+
+  def create_driver(conf)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::RedisStoreOutput).configure(conf)
+  end
+
+  def get_time
+    event_time("2011-01-02 13:14:15 UTC")
+  end
+
+  def test_zset_batches_many_records_through_one_pipeline
+    config = %[
+      host       #{REDIS_HOST}
+      port       #{REDIS_PORT}
+      store_type  zset
+      key         #{@key}
+      score_path  result
+    ]
+    d = create_driver(config)
+    d.run(default_tag: 'test') do
+      50.times { |i| d.feed(get_time, { 'result' => i }) }
+    end
+
+    assert_equal 50, @redis.zcard(@key)
+  end
+
+  def test_zset_collision_policy_nx_does_not_overwrite
+    @redis.zadd(@key, 1, 'v')
+    config = %[
+      host             #{REDIS_HOST}
+      port             #{REDIS_PORT}
+      store_type       zset
+      key              #{@key}
+      score_path       result
+      collision_policy NX
+    ]
+    d = create_driver(config)
+    d.run(default_tag: 'test') do
+      d.feed(get_time, { 'result' => 99 })
+    end
+
+    assert_equal 1.0, @redis.zscore(@key, 'v')
+  end
+
+  def test_zset_value_length_trims_via_lua_eval
+    config = %[
+      host          #{REDIS_HOST}
+      port          #{REDIS_PORT}
+      store_type    zset
+      key           #{@key}
+      score_path    result
+      value_length  3
+    ]
+    d = create_driver(config)
+    d.run(default_tag: 'test') do
+      10.times { |i| d.feed(get_time, { 'result' => i }) }
+    end
+
+    # generate_zremrangebyrank_script removes one more entry than value_length
+    # on the 'asc' path (a pre-existing off-by-one, not introduced by the
+    # pipeline fix) -- this asserts current behavior, not a spec.
+    assert_equal 2, @redis.zcard(@key)
+  end
+
+  def test_list_rpush_asc
+    config = %[
+      host        #{REDIS_HOST}
+      port        #{REDIS_PORT}
+      format_type plain
+      store_type  list
+      key         #{@key}
+      value_path  v
+    ]
+    d = create_driver(config)
+    d.run(default_tag: 'test') do
+      5.times { |i| d.feed(get_time, { 'v' => i.to_s }) }
+    end
+
+    assert_equal 5, @redis.llen(@key)
+    assert_equal %w[0 1 2 3 4], @redis.lrange(@key, 0, -1)
+  end
+
+  def test_set_sadd
+    config = %[
+      host        #{REDIS_HOST}
+      port        #{REDIS_PORT}
+      format_type plain
+      store_type  set
+      key         #{@key}
+      value_path  v
+    ]
+    d = create_driver(config)
+    d.run(default_tag: 'test') do
+      3.times { |i| d.feed(get_time, { 'v' => "member-#{i}" }) }
+    end
+
+    assert_equal 3, @redis.scard(@key)
+    assert_equal true, @redis.sismember(@key, 'member-1')
+  end
+
+  def test_string_set_and_key_expire
+    config = %[
+      host        #{REDIS_HOST}
+      port        #{REDIS_PORT}
+      format_type plain
+      store_type  string
+      key         #{@key}
+      value_path  v
+      key_expire  30
+    ]
+    d = create_driver(config)
+    d.run(default_tag: 'test') do
+      d.feed(get_time, { 'v' => 'hello' })
+    end
+
+    assert_equal 'hello', @redis.get(@key)
+    ttl = @redis.ttl(@key)
+    assert ttl > 0 && ttl <= 30, "expected 0 < ttl <= 30, got #{ttl}"
+  end
+end

data/test/plugin/test_out_redis_publish.rb

@@ -210,7 +210,7 @@ class RedisStoreOutputTest < Test::Unit::TestCase
     assert_equal "george", $key
     assert_equal message.to_json, $message
   end
-
+  
   def test_json_with_malformed_utf8
     config = %[
       format_type json
@@ -221,16 +221,14 @@ class RedisStoreOutputTest < Test::Unit::TestCase
     message = {
       'user' => 'george',
       'data' => ([0x80, 0x80].map {|x| x.chr}.join).force_encoding('UTF-8')
-
     }
     $ttl = nil
     d.run(default_tag: 'test') do
       d.feed(get_time, message)
     end
-
     assert_equal message, Yajl.load($message, :check_utf8 => false)
   end
-  
+
   def test_msgpack
     config = %[
       format_type msgpack

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-ddl-redis-store
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.1
 platform: ruby
 authors:
 - moaikids
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-11-20 00:00:00.000000000 Z
+date: 2026-09-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -65,30 +65,42 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '5.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '5.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7'
 description: Redis(zset/set/list/string/publish) output plugin for Fluentd...
-email: hanai@pokelabo.co.jp
+email: builds@dominodatalab.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/CODEOWNERS"
+- ".github/PULL_REQUEST_TEMPLATE.md"
+- ".github/workflows/test.yml"
 - ".gitignore"
 - AUTHORS
 - Gemfile
 - README.md
 - Rakefile
+- domino.yml
 - fluent-plugin-ddl-redis-store.gemspec
 - lib/fluent/plugin/out_redis_store.rb
+- renovate.json
 - test/helpers.rb
+- test/integration/test_out_redis_store_integration.rb
 - test/plugin/test_out_redis_publish.rb
-homepage: https://github.com/pokehanai/fluent-plugin-redis-store
+homepage: https://github.com/dominodatalab/fluent-plugin-redis-store
 licenses:
 - Apache License Version 2.0
 metadata: {}
@@ -113,4 +125,5 @@ specification_version: 4
 summary: Redis(zset/set/list/string/publish) output plugin for Fluentd
 test_files:
 - test/helpers.rb
+- test/integration/test_out_redis_store_integration.rb
 - test/plugin/test_out_redis_publish.rb