fluent-plugin-ddl-redis-store 0.3.0 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e297bfd4b33f894d418409472cfe990597772affc569fecb1d1a371d9c45b19e
-  data.tar.gz: ec63c76ec0074f18da13daa82a4ca82a9a5b93d5e312eb2415d850d1095ff71c
+  metadata.gz: 88744c7b86deb03ab629da9f4cf4ffbb771e915ed4e92f6d135d92d2ca4e857f
+  data.tar.gz: 63a52311c27e78ded58f72a048966cc48d451652da1b06f0a9b025532f27b4d3
 SHA512:
-  metadata.gz: 5720f4a0aad3fe67fff9fcd8fee15f61a58e08e2868cdd4f2dfeb1e8ee46644ee950f81c6d99345737bcef8ac668ce2fb5927b605a2dd7b11567866cecf9b05d
-  data.tar.gz: 5d28612ac90fcdc9614b170eb706fa673cc0be75d7394e6e1394ffae0d042049c9c055960ae91dbea2a0872d7bcecfb6cb8cc0dbc75621ec3be5722173309312
+  metadata.gz: c911933d1521381f057b5ab875d4bf7a14b238c05336ec7c6f2c14a3f92e3419a1a0340ea5954a495037f53b6f2c68a1d16be2e1897b5f1e6df3be9ef5036b4e
+  data.tar.gz: 55462b877d5cd62505d5bff524de7487b24e4c50a56ee6a33f1607c55211e870d2948b26d9ca9fe4ed5f83e4b00215e0864b94471a6c736ce75f3542c4eb69cf

data/.github/CODEOWNERS

@@ -0,0 +1 @@
+*   @cerebrotech/eng-distributions

data/.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,25 @@
+## PR Type
+What kind of change does this PR introduce?
+
+<!-- Please check the one that applies to this PR using "x". -->
+- [ ] Bugfix
+- [ ] Feature
+- [ ] Code style update (formatting)
+- [ ] Refactoring (no functional changes)
+- [ ] CI-related changes
+- [ ] Other... Please describe:
+
+## What is the current behavior?
+<!-- Please describe the current behavior that you are modifying, or link to a relevant issue. -->
+
+Issue Number: https://dominodatalab.atlassian.net/browse/PLAT-XYZ
+
+## What is the new behavior?
+
+## Does this PR introduce a breaking change?
+- [ ] Yes
+- [ ] No
+
+<!-- If this PR contains a breaking change, please describe the impact and migration path for existing applications below. -->
+
+## Other information

data/.github/workflows/test.yml

@@ -0,0 +1,44 @@
+name: test
+on:
+  push:
+    branches: [master]
+  pull_request:
+
+jobs:
+  unit:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby: ["3.2", "3.3", "3.4"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+      - run: bundle exec rake test
+
+  integration:
+    runs-on: ubuntu-latest
+    services:
+      redis:
+        image: redis:7-alpine
+        ports: ["6379:6379"]
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 5s
+          --health-timeout 3s
+          --health-retries 5
+    strategy:
+      matrix:
+        ruby: ["3.2", "3.3", "3.4"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+      - run: bundle exec rake test:integration
+        env:
+          REDIS_HOST: 127.0.0.1
+          REDIS_PORT: 6379

data/.gitignore

@@ -1 +1,8 @@
 *.gem
+pkg/
+.bundle/
+vendor/
+Gemfile.lock
+*.log
+.DS_Store
+coverage/

data/README.md

@@ -10,15 +10,31 @@ Requirements
 
 | fluent-plugin-redis-store | fluentd | ruby |
 |------------------------|---------|------|
-| >= 0.2.0 | >= v0.14.15 | >= 2.1 |
+| >= 0.3.0 | >= v0.14.15 | >= 3.2 |
+| >= 0.2.0, < 0.3.0 | >= v0.14.15 | >= 2.1 |
 |  < 0.2.0 | >= v0.12.0 | >= 1.9 |
 
 Background
 ----------
 
-This is folked project from [fluent-plugin-redisstore][].
+This is a forked project from [fluent-plugin-redis-store][].
+
+[fluent-plugin-redis-store]: https://github.com/pokehanai/fluent-plugin-redis-store/
+
+Release Process
+---------------
+1. Bump `version` in `fluent-plugin-ddl-redis-store.gemspec`
+2. Bump `version` in `domino.yml` to match. It is **not** kept in sync automatically —
+   unlike sibling `domino.yml`s that track an external base image via Renovate, this one
+   tracks the gem's own release number, which nothing here polls or rewrites for you.
+3. `gem build fluent-plugin-ddl-redis-store.gemspec`
+4. `gem push <.gem file>` (with the gem file generated in the previous step)  
+   Ex: `gem push fluent-plugin-ddl-redis-store-0.3.1.gem`
+
+The `push` command needs to be authenticated with [RubyGems.org](https://rubygems.org/).
+
+Latest release: https://rubygems.org/gems/fluent-plugin-ddl-redis-store
 
-[fluent-plugin-redisstore]: https://github.com/moaikids/fluent-plugin-redisstore
 
 Features
 --------
@@ -92,10 +108,10 @@ Installation
 ------------
 
 ```bash
-fluent-gem install fluent-plugin-redis-store
+fluent-gem install fluent-plugin-ddl-redis-store
 
 # or if you are using td-agent:
-td-agent-gem install fluent-plugin-redis-store
+td-agent-gem install fluent-plugin-ddl-redis-store
 ```
 
 Configuration
@@ -161,6 +177,31 @@ based on *timestamp* and it deletes expired _members_ every after new event data
 No more options than common options.
 
 
+Testing
+-------
+
+Unit tests (`test/plugin/`) stub out the `Redis` class entirely and only exercise
+config parsing and key/value traversal -- they cannot detect a redis-rb client API
+break.
+
+```bash
+bundle exec rake test
+```
+
+Integration tests (`test/integration/`) run against a real Redis server and assert
+on actual `ZCARD`/`LLEN`/`GET`/etc. Start one first:
+
+```bash
+docker run -d --rm -p 6379:6379 redis:7-alpine
+bundle exec rake test:integration
+```
+
+They read `REDIS_HOST`/`REDIS_PORT` from the environment (default `127.0.0.1:6379`)
+and are skipped with a message if no Redis is reachable.
+
+Note: redis-rb 6.x requires Ruby 3.2+; if your local Ruby is older, run both tasks
+inside a container that matches the target fluentd image's Ruby/redis-rb version.
+
 Contributors
 ------------
 

data/Rakefile

@@ -4,7 +4,14 @@ require "bundler/gem_tasks"
 require 'rake/testtask'
 Rake::TestTask.new(:test) do |test|
     test.libs << 'lib' << 'test'
-    test.pattern = 'test/**/test_*.rb'
+    test.pattern = 'test/plugin/**/test_*.rb'
+    test.verbose = true
+end
+
+desc "Integration tests against a real Redis server (see test/integration for setup)"
+Rake::TestTask.new('test:integration') do |test|
+    test.libs << 'lib' << 'test'
+    test.pattern = 'test/integration/**/test_*.rb'
     test.verbose = true
 end
 

data/domino.yml

@@ -0,0 +1,7 @@
+workflowVersion: 0
+name: fluent-plugin-redis-store
+type: library
+version: 0.3.2
+versionFormat: semver
+language:
+  name: shell

data/fluent-plugin-ddl-redis-store.gemspec

@@ -2,7 +2,7 @@
 Gem::Specification.new do |gem|
     gem.name        = "fluent-plugin-ddl-redis-store"
     gem.email       = "builds@dominodatalab.com"
-    gem.version     = "0.3.0"
+    gem.version     = "0.3.2"
     gem.authors     = ["moaikids", "HANAI Tohru aka pokehanai"]
     gem.licenses    = ["Apache License Version 2.0"]
     gem.summary     = %q{Redis(zset/set/list/string/publish) output plugin for Fluentd}
@@ -17,5 +17,6 @@ Gem::Specification.new do |gem|
     gem.add_development_dependency "rake"
     gem.add_development_dependency "test-unit"
     gem.add_runtime_dependency "fluentd", [">= 0.14.15", "< 2"]
-    gem.add_runtime_dependency "redis"
+    gem.add_runtime_dependency "redis", [">= 5.0", "< 7"]
+    gem.add_runtime_dependency "yajl-ruby", "~> 1.4"
 end

data/lib/fluent/plugin/out_redis_store.rb

@@ -1,4 +1,5 @@
 require 'fluent/plugin/output'
+require 'yajl'
 
 module Fluent::Plugin
   class RedisStoreOutput < Output
@@ -55,10 +56,10 @@ module Fluent::Plugin
       super
       if @path
         @redis = Redis.new(:path => @path, :password => @password,
-                           :timeout => @timeout, :thread_safe => true, :db => @db)
+                           :timeout => @timeout, :db => @db)
       else
         @redis = Redis.new(:host => @host, :port => @port, :password => @password,
-                           :timeout => @timeout, :thread_safe => true, :db => @db)
+                           :timeout => @timeout, :db => @db)
       end
     end
 
@@ -80,7 +81,7 @@ module Fluent::Plugin
     end
 
     def write(chunk)
-      @redis.pipelined {
+      @redis.pipelined { |pipeline|
         chunk.open { |io|
           begin
             MessagePack::Unpacker.new(io).each { |message|
@@ -88,15 +89,15 @@ module Fluent::Plugin
                 (_, time, record) = message
                 case @store_type
                 when 'zset'
-                  operation_for_zset(record, time)
+                  operation_for_zset(pipeline, record, time)
                 when 'set'
-                  operation_for_set(record)
+                  operation_for_set(pipeline, record)
                 when 'list'
-                  operation_for_list(record)
+                  operation_for_list(pipeline, record)
                 when 'string'
-                  operation_for_string(record)
+                  operation_for_string(pipeline, record)
                 when 'publish'
-                  operation_for_publish(record)
+                  operation_for_publish(pipeline, record)
                 end
               rescue NoMethodError => e
                 puts e
@@ -113,66 +114,66 @@ module Fluent::Plugin
       }
     end
 
-    def operation_for_zset(record, time)
+    def operation_for_zset(pipeline, record, time)
       key = get_key_from(record)
       value = get_value_from(record)
       score = get_score_from(record, time)
       if @collision_policy
         if @collision_policy == 'NX'
-          @redis.zadd(key, score, value, :nx => true)
+          pipeline.zadd(key, score, value, :nx => true)
         elsif @collision_policy == 'XX'
-          @redis.zadd(key, score, value, :xx => true)
+          pipeline.zadd(key, score, value, :xx => true)
         end
       else
-        @redis.zadd(key, score, value)
+        pipeline.zadd(key, score, value)
       end
 
-      set_key_expire key
+      set_key_expire pipeline, key
       if 0 < @value_expire
         now = Time.now.to_i
-        @redis.zremrangebyscore key , '-inf' , (now - @value_expire)
+        pipeline.zremrangebyscore key , '-inf' , (now - @value_expire)
       end
       if 0 < @value_length
         script = generate_zremrangebyrank_script(key, @value_length, @order)
-        @redis.eval script
+        pipeline.eval script
       end
     end
 
-    def operation_for_set(record)
+    def operation_for_set(pipeline, record)
       key = get_key_from(record)
       value = get_value_from(record)
-      @redis.sadd key, value
-      set_key_expire key
+      pipeline.sadd key, value
+      set_key_expire pipeline, key
     end
 
-    def operation_for_list(record)
+    def operation_for_list(pipeline, record)
       key = get_key_from(record)
       value = get_value_from(record)
 
       if @order == 'asc'
-        @redis.rpush key, value
+        pipeline.rpush key, value
       else
-        @redis.lpush key, value
+        pipeline.lpush key, value
       end
-      set_key_expire key
+      set_key_expire pipeline, key
       if 0 < @value_length
         script = generate_ltrim_script(key, @value_length, @order)
-        @redis.eval script
+        pipeline.eval script
       end
     end
 
-    def operation_for_string(record)
+    def operation_for_string(pipeline, record)
       key = get_key_from(record)
       value = get_value_from(record)
-      @redis.set key, value
+      pipeline.set key, value
 
-      set_key_expire key
+      set_key_expire pipeline, key
     end
 
-    def operation_for_publish(record)
+    def operation_for_publish(pipeline, record)
       key = get_key_from(record)
       value = get_value_from(record)
-      @redis.publish key, value
+      pipeline.publish key, value
     end
 
     def generate_zremrangebyrank_script(key, maxlen, order)
@@ -253,9 +254,9 @@ module Fluent::Plugin
       end
     end
 
-    def set_key_expire(key)
+    def set_key_expire(pipeline, key)
       if 0 < @key_expire
-        @redis.expire key, @key_expire
+        pipeline.expire key, @key_expire
       end
     end
 

data/renovate.json

@@ -0,0 +1,20 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "description": "Renovate config",
+  "extends": [
+    "github>cerebrotech/renovate-config"
+  ],
+  "packageRules": [
+    {
+      "description": "Require manual review for major bumps of the redis gem; a two-major-version unattended jump (redis-rb 4.8 -> 6.0) previously broke fluentd at boot (Redis.new ArgumentError) and silently stopped batching in write() -- see the matching rule in the sibling operations-fluentd-docker renovate config",
+      "matchManagers": ["bundler"],
+      "matchPackageNames": ["redis"],
+      "matchUpdateTypes": ["major"],
+      "enabled": false
+    }
+  ],
+  "vulnerabilityAlerts": {
+    "enabled": true
+  }
+}
+

data/test/integration/test_out_redis_store_integration.rb

@@ -0,0 +1,151 @@
+require 'helpers'
+require 'redis'
+require 'securerandom'
+
+# These tests exercise the plugin against a real Redis server, unlike
+# test/plugin/test_out_redis_publish.rb which stubs out the entire Redis
+# class and therefore cannot detect a redis-rb client API break (e.g. the
+# `pipelined` block-argument change between redis-rb 4.x and 5.x/6.x).
+#
+# Run a real Redis first:
+#   docker run -d --rm -p 6379:6379 redis:7-alpine
+# then:
+#   bundle exec rake test:integration
+class RedisStoreOutputIntegrationTest < Test::Unit::TestCase
+  REDIS_HOST = ENV['REDIS_HOST'] || '127.0.0.1'
+  REDIS_PORT = (ENV['REDIS_PORT'] || 6379).to_i
+
+  def setup
+    Fluent::Test.setup
+    @redis = Redis.new(host: REDIS_HOST, port: REDIS_PORT, timeout: 2.0)
+    begin
+      @redis.ping
+    rescue StandardError => e
+      omit "no Redis reachable at #{REDIS_HOST}:#{REDIS_PORT} (#{e.class}: #{e.message});" \
+        " start one with `docker run -d --rm -p 6379:6379 redis:7-alpine` to run integration tests"
+    end
+    @key = "fluent-plugin-redis-store-test:#{SecureRandom.hex(8)}"
+  end
+
+  def teardown
+    @redis.del(@key) if @redis
+  end
+
+  def create_driver(conf)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::RedisStoreOutput).configure(conf)
+  end
+
+  def get_time
+    event_time("2011-01-02 13:14:15 UTC")
+  end
+
+  def test_zset_batches_many_records_through_one_pipeline
+    config = %[
+      host       #{REDIS_HOST}
+      port       #{REDIS_PORT}
+      store_type  zset
+      key         #{@key}
+      score_path  result
+    ]
+    d = create_driver(config)
+    d.run(default_tag: 'test') do
+      50.times { |i| d.feed(get_time, { 'result' => i }) }
+    end
+
+    assert_equal 50, @redis.zcard(@key)
+  end
+
+  def test_zset_collision_policy_nx_does_not_overwrite
+    @redis.zadd(@key, 1, 'v')
+    config = %[
+      host             #{REDIS_HOST}
+      port             #{REDIS_PORT}
+      store_type       zset
+      key              #{@key}
+      score_path       result
+      collision_policy NX
+    ]
+    d = create_driver(config)
+    d.run(default_tag: 'test') do
+      d.feed(get_time, { 'result' => 99 })
+    end
+
+    assert_equal 1.0, @redis.zscore(@key, 'v')
+  end
+
+  def test_zset_value_length_trims_via_lua_eval
+    config = %[
+      host          #{REDIS_HOST}
+      port          #{REDIS_PORT}
+      store_type    zset
+      key           #{@key}
+      score_path    result
+      value_length  3
+    ]
+    d = create_driver(config)
+    d.run(default_tag: 'test') do
+      10.times { |i| d.feed(get_time, { 'result' => i }) }
+    end
+
+    # generate_zremrangebyrank_script removes one more entry than value_length
+    # on the 'asc' path (a pre-existing off-by-one, not introduced by the
+    # pipeline fix) -- this asserts current behavior, not a spec.
+    assert_equal 2, @redis.zcard(@key)
+  end
+
+  def test_list_rpush_asc
+    config = %[
+      host        #{REDIS_HOST}
+      port        #{REDIS_PORT}
+      format_type plain
+      store_type  list
+      key         #{@key}
+      value_path  v
+    ]
+    d = create_driver(config)
+    d.run(default_tag: 'test') do
+      5.times { |i| d.feed(get_time, { 'v' => i.to_s }) }
+    end
+
+    assert_equal 5, @redis.llen(@key)
+    assert_equal %w[0 1 2 3 4], @redis.lrange(@key, 0, -1)
+  end
+
+  def test_set_sadd
+    config = %[
+      host        #{REDIS_HOST}
+      port        #{REDIS_PORT}
+      format_type plain
+      store_type  set
+      key         #{@key}
+      value_path  v
+    ]
+    d = create_driver(config)
+    d.run(default_tag: 'test') do
+      3.times { |i| d.feed(get_time, { 'v' => "member-#{i}" }) }
+    end
+
+    assert_equal 3, @redis.scard(@key)
+    assert_equal true, @redis.sismember(@key, 'member-1')
+  end
+
+  def test_string_set_and_key_expire
+    config = %[
+      host        #{REDIS_HOST}
+      port        #{REDIS_PORT}
+      format_type plain
+      store_type  string
+      key         #{@key}
+      value_path  v
+      key_expire  30
+    ]
+    d = create_driver(config)
+    d.run(default_tag: 'test') do
+      d.feed(get_time, { 'v' => 'hello' })
+    end
+
+    assert_equal 'hello', @redis.get(@key)
+    ttl = @redis.ttl(@key)
+    assert ttl > 0 && ttl <= 30, "expected 0 < ttl <= 30, got #{ttl}"
+  end
+end

metadata

@@ -1,15 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-ddl-redis-store
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.2
 platform: ruby
 authors:
 - moaikids
 - HANAI Tohru aka pokehanai
-autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-11-21 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -65,34 +64,59 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '5.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '5.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7'
+- !ruby/object:Gem::Dependency
+  name: yajl-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
 description: Redis(zset/set/list/string/publish) output plugin for Fluentd...
 email: builds@dominodatalab.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/CODEOWNERS"
+- ".github/PULL_REQUEST_TEMPLATE.md"
+- ".github/workflows/test.yml"
 - ".gitignore"
 - AUTHORS
 - Gemfile
 - README.md
 - Rakefile
+- domino.yml
 - fluent-plugin-ddl-redis-store.gemspec
 - lib/fluent/plugin/out_redis_store.rb
+- renovate.json
 - test/helpers.rb
+- test/integration/test_out_redis_store_integration.rb
 - test/plugin/test_out_redis_publish.rb
 homepage: https://github.com/dominodatalab/fluent-plugin-redis-store
 licenses:
 - Apache License Version 2.0
 metadata: {}
-post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -107,10 +131,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3.1
-signing_key: 
+rubygems_version: 3.6.9
 specification_version: 4
 summary: Redis(zset/set/list/string/publish) output plugin for Fluentd
 test_files:
 - test/helpers.rb
+- test/integration/test_out_redis_store_integration.rb
 - test/plugin/test_out_redis_publish.rb