waterdrop 2.8.1 → 2.8.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7d2fa38e1362c3069581533bd8da5ee508f00b9fea08c9a4c56ae16780c38c68
-  data.tar.gz: 0c861f372aeae735d0033b35bc9224e45025c03b2084d796108660b976e74eb3
+  metadata.gz: c2e1bd150394cd2ff84114c1d42f88c1e1d2dc0aac023244e561eee406b04700
+  data.tar.gz: 99bd1741ea65d1cc9cda5a94d0d04979dbfadbf74f2de79f9e93790292d6cea5
 SHA512:
-  metadata.gz: 80ae3a51e3aeaa19e875eddfefb16e415b61f7ef44861d97043b060604fc9ea06d774b9e308f198d89eb813fa405f7f650e0516c8ce5688933ffce5b1b6533d8
-  data.tar.gz: 2da33c249c8f4e761876a18d7d95745490ed05d296a736fdf39a4ec0b84ba84f3239b8528cb91e6e08cdfe729acabda530cb464a5f8eba779373d113b5b3be94
+  metadata.gz: fd7f0772a35bd4e421fd46aa54cef6626f9660ad4cecbd9c03d9a912d68dabe994c84f44e7895099008cad4166ffec77fafda3b06aeacec27a707a4a7ed803a4
+  data.tar.gz: 8c27c3bf8db0be08f137b0f2649586cf885a9f340b381a075e3d0d20ecad38ce6f3b0ce2a1e40a5131adf67b46f3005664fd9f4b16f91c1cbf6ab93d1713bf38

data/.github/CODEOWNERS

@@ -0,0 +1,3 @@
+/.github @mensfeld
+/.github/workflows/ @mensfeld
+/.github/actions/ @mensfeld

data/.github/workflows/ci.yml

@@ -6,12 +6,18 @@ concurrency:
 
 on:
   pull_request:
+    branches: [ main, master ]
   push:
+    branches: [ main, master ]
   schedule:
     - cron:  '0 1 * * *'
 
+permissions:
+  contents: read
+
 jobs:
   specs:
+    timeout-minutes: 15
     runs-on: ubuntu-latest
     needs: diffend
     strategy:
@@ -26,31 +32,31 @@ jobs:
           - ruby: '3.4'
             coverage: 'true'
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          fetch-depth: 0
 
       - name: Install package dependencies
         run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
 
       - name: Set up Ruby
-        uses: ruby/setup-ruby@v1
+        uses: ruby/setup-ruby@354a1ad156761f5ee2b7b13fa8e09943a5e8d252 # v1.229.0
         with:
           ruby-version: ${{matrix.ruby}}
           bundler-cache: true
           bundler: 'latest'
 
-      - name: Run Kafka with docker compose
+      - name: Run Kafka with Docker Compose
         run: |
           docker compose up -d || (sleep 5 && docker compose up -d)
 
       - name: Wait for Kafka
-        run: |
-          sleep 15
+        run: sleep 15
 
       - name: Install latest bundler
         run: |
           gem install bundler --no-document
           gem update --system --no-document
-
           bundle config set without 'tools benchmarks docs'
 
       - name: Bundle install
@@ -63,16 +69,23 @@ jobs:
           GITHUB_COVERAGE: ${{matrix.coverage}}
         run: bundle exec rspec
 
+      - name: Check Kafka logs for unexpected warnings
+        run: bin/verify_kafka_warnings
+
+      - name: Check test topics naming convention
+        run: bin/verify_topics_naming
+
   diffend:
+    timeout-minutes: 5
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           fetch-depth: 0
       - name: Set up Ruby
-        uses: ruby/setup-ruby@v1
+        uses: ruby/setup-ruby@354a1ad156761f5ee2b7b13fa8e09943a5e8d252 # v1.229.0
         with:
           ruby-version: 3.4
       - name: Install latest bundler
@@ -83,12 +96,25 @@ jobs:
         run: bundle secure
 
   coditsu:
+    timeout-minutes: 5
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           fetch-depth: 0
+      - name: Download Coditsu script
+        run: |
+          curl -sSL https://api.coditsu.io/run/ci -o coditsu_script.sh
+          chmod +x coditsu_script.sh
+      - name: Verify Coditsu script checksum
+        run: |
+          EXPECTED_SHA256="0aecc5aa010f53fca264548a41467a2b0a1208d750ce1da3e98a217304cacbbc"
+          ACTUAL_SHA256=$(sha256sum coditsu_script.sh | awk '{ print $1 }')
+          if [ "$ACTUAL_SHA256" != "$EXPECTED_SHA256" ]; then
+            echo "::error::Checksum verification failed. Expected $EXPECTED_SHA256 but got $ACTUAL_SHA256."
+            exit 1
+          fi
       - name: Run Coditsu
-        run: \curl -sSL https://api.coditsu.io/run/ci | bash
+        run: ./coditsu_script.sh

data/.github/workflows/verify-action-pins.yml

@@ -0,0 +1,16 @@
+name: Verify Action Pins
+on:
+  pull_request:
+    paths:
+      - '.github/workflows/**'
+jobs:
+  verify:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - name: Check SHA pins
+        run: |
+          if grep -E -r "uses: .*/.*@(v[0-9]+|main|master)($|[[:space:]]|$)" --include="*.yml" --include="*.yaml" .github/workflows/ | grep -v "#"; then
+            echo "::error::Actions should use SHA pins, not tags or branch names"
+            exit 1
+          fi

data/.ruby-version

@@ -1 +1 @@
-3.4.1
+3.4.2

data/CHANGELOG.md

@@ -1,5 +1,16 @@
 # WaterDrop changelog
 
+## 2.8.3 (2025-04-08)
+- [Enhancement] Support producing messages with arrays of strings in headers (KIP-82).
+- [Refactor] Introduce a `bin/verify_topics_naming` script to ensure proper test topics naming convention.
+- [Refactor] Remove factory bot and active support requirement in tests/dev.
+- [Change] Require `karafka-rdkafka` `>= 0.19.1` due to KIP-82.
+
+## 2.8.2 (2025-02-13)
+- [Feature] Allow for tagging of producer instances similar to how consumers can be tagged.
+- [Refactor] Ensure all test topics in the test suite start with "it-" prefix. 
+- [Refactor] Introduce a `bin/verify_kafka_warnings` script to clean Kafka from temporary test-suite topics.
+
 ## 2.8.1 (2024-12-26)
 - [Enhancement] Raise `WaterDrop::ProducerNotTransactionalError` when attempting to use transactions on a non-transactional producer.
 - [Fix] Disallow closing a producer from within a transaction.

data/Gemfile

@@ -6,12 +6,14 @@ plugin 'diffend'
 
 gemspec
 
+# Relaxed from 2.7 because we support Ruby 3.1
+gem 'zeitwerk', '~> 2.6.18'
+
 group :development do
   gem 'byebug'
 end
 
 group :test do
-  gem 'factory_bot'
   gem 'ostruct'
   gem 'rspec'
   gem 'simplecov'

data/Gemfile.lock

@@ -1,84 +1,80 @@
 PATH
   remote: .
   specs:
-    waterdrop (2.8.1)
-      karafka-core (>= 2.4.3, < 3.0.0)
-      karafka-rdkafka (>= 0.17.5)
+    waterdrop (2.8.3)
+      karafka-core (>= 2.4.9, < 3.0.0)
+      karafka-rdkafka (>= 0.19.1)
       zeitwerk (~> 2.3)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (7.2.1)
-      base64
-      bigdecimal
-      concurrent-ruby (~> 1.0, >= 1.3.1)
-      connection_pool (>= 2.2.5)
-      drb
-      i18n (>= 1.6, < 2)
-      logger (>= 1.4.2)
-      minitest (>= 5.1)
-      securerandom (>= 0.3)
-      tzinfo (~> 2.0, >= 2.0.5)
-    base64 (0.2.0)
-    bigdecimal (3.1.8)
-    byebug (11.1.3)
-    concurrent-ruby (1.3.4)
-    connection_pool (2.4.1)
-    diff-lcs (1.5.1)
+    byebug (12.0.0)
+    diff-lcs (1.6.0)
     docile (1.4.1)
-    drb (2.2.1)
-    factory_bot (6.5.0)
-      activesupport (>= 5.0.0)
-    ffi (1.17.0)
-    i18n (1.14.5)
-      concurrent-ruby (~> 1.0)
-    karafka-core (2.4.4)
-      karafka-rdkafka (>= 0.15.0, < 0.18.0)
-    karafka-rdkafka (0.17.6)
+    ffi (1.17.1)
+    ffi (1.17.1-aarch64-linux-gnu)
+    ffi (1.17.1-aarch64-linux-musl)
+    ffi (1.17.1-arm-linux-gnu)
+    ffi (1.17.1-arm-linux-musl)
+    ffi (1.17.1-arm64-darwin)
+    ffi (1.17.1-x86-linux-gnu)
+    ffi (1.17.1-x86-linux-musl)
+    ffi (1.17.1-x86_64-darwin)
+    ffi (1.17.1-x86_64-linux-gnu)
+    ffi (1.17.1-x86_64-linux-musl)
+    karafka-core (2.4.10)
+      karafka-rdkafka (>= 0.17.6, < 0.20.0)
+      logger (>= 1.6.0)
+    karafka-rdkafka (0.19.1)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
-    logger (1.6.1)
-    mini_portile2 (2.8.7)
-    minitest (5.25.1)
+    logger (1.6.6)
+    mini_portile2 (2.8.8)
     ostruct (0.6.1)
     rake (13.2.1)
     rspec (3.13.0)
       rspec-core (~> 3.13.0)
       rspec-expectations (~> 3.13.0)
       rspec-mocks (~> 3.13.0)
-    rspec-core (3.13.1)
+    rspec-core (3.13.3)
       rspec-support (~> 3.13.0)
     rspec-expectations (3.13.3)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
-    rspec-mocks (3.13.1)
+    rspec-mocks (3.13.2)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
-    rspec-support (3.13.1)
-    securerandom (0.3.1)
+    rspec-support (3.13.2)
     simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.13.1)
     simplecov_json_formatter (0.1.4)
-    tzinfo (2.0.6)
-      concurrent-ruby (~> 1.0)
     zeitwerk (2.6.18)
 
 PLATFORMS
+  aarch64-linux-gnu
+  aarch64-linux-musl
+  arm-linux-gnu
+  arm-linux-musl
+  arm64-darwin
   ruby
-  x86_64-linux
+  x86-linux-gnu
+  x86-linux-musl
+  x86_64-darwin
+  x86_64-linux-gnu
+  x86_64-linux-musl
 
 DEPENDENCIES
   byebug
-  factory_bot
   ostruct
   rspec
   simplecov
   waterdrop!
+  zeitwerk (~> 2.6.18)
 
 BUNDLED WITH
-   2.5.16
+   2.6.2

data/bin/verify_kafka_warnings

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+# Checks Kafka logs for unsupported warning patterns
+# Only specified warnings are allowed, all others should trigger failure
+
+allowed_patterns=(
+   "Performing controller activation"
+   "registered with feature metadata.version"
+)
+
+# Get all warnings
+warnings=$(docker logs --since=0 kafka | grep WARN)
+exit_code=0
+
+while IFS= read -r line; do
+   allowed=0
+   for pattern in "${allowed_patterns[@]}"; do
+       if echo "$line" | grep -q "$pattern"; then
+           allowed=1
+           break
+       fi
+   done
+
+   if [ $allowed -eq 0 ]; then
+       echo "Unexpected warning found:"
+       echo "$line"
+       exit_code=1
+   fi
+done <<< "$warnings"
+
+exit $exit_code

data/bin/verify_topics_naming

@@ -0,0 +1,30 @@
+#!/usr/bin/env ruby
+
+# This script verifies that we do not create (except few needed exceptions) test topics that do
+# not start with the "it-" prefix which is our standard.
+#
+# This ensures that we can clearly identify all test topics for removal in case of doing dev work
+# on a long-lived Kafka cluster without option to fully reset it.
+#
+# It also ensures we have one convention that we can follow.
+
+require 'bundler'
+Bundler.setup(:default, :test, :integrations)
+require 'rdkafka'
+
+admin = Rdkafka::Config.new('bootstrap.servers': 'localhost:9092').admin
+
+invalid = admin
+          .metadata
+          .topics
+          .map { |topic| topic[:topic_name] }
+          .select { |topic| !topic.start_with?('it-') }
+          .select { |topic| topic.length <= 6 }
+
+admin.close
+
+invalid.each do |invalid_name|
+  puts "#{invalid_name} does not start with the \"it-\" prefix"
+end
+
+exit invalid.empty? ? 0 : 1

data/config/locales/errors.yml

@@ -39,7 +39,7 @@ en:
       key_format: must be a non-empty string
       payload_max_size: is more than `max_payload_size` config value
       headers_invalid_key_type: all headers keys need to be of type String
-      headers_invalid_value_type: all headers values need to be of type String
+      headers_invalid_value_type: all headers values need to be strings or arrays of strings
 
     transactional_offset:
       consumer_format: 'must respond to #consumer_group_metadata_pointer method'

data/docker-compose.yml

@@ -1,7 +1,7 @@
 services:
   kafka:
     container_name: kafka
-    image: confluentinc/cp-kafka:7.8.0
+    image: confluentinc/cp-kafka:7.9.0
 
     ports:
       - 9092:9092

data/lib/waterdrop/contracts/message.rb

@@ -46,7 +46,12 @@ module WaterDrop
 
         message.fetch(:headers).each do |key, value|
           errors << [%i[headers], :invalid_key_type] unless key.is_a?(String)
-          errors << [%i[headers], :invalid_value_type] unless value.is_a?(String)
+
+          # Headers can be either strings or arrays of strings
+          next if value.is_a?(String)
+          next if value.is_a?(Array) && value.all? { |value| value.is_a?(String) }
+
+          errors << [%i[headers], :invalid_value_type]
         end
 
         errors

data/lib/waterdrop/producer.rb

@@ -9,6 +9,7 @@ module WaterDrop
     include Buffer
     include Transactions
     include ::Karafka::Core::Helpers::Time
+    include ::Karafka::Core::Taggable
 
     # Local storage for given thread waterdrop client references for variants
     ::Fiber.send(:attr_accessor, :waterdrop_clients)

data/lib/waterdrop/version.rb

@@ -3,5 +3,5 @@
 # WaterDrop library
 module WaterDrop
   # Current WaterDrop version
-  VERSION = '2.8.1'
+  VERSION = '2.8.3'
 end

data/renovate.json

@@ -1,6 +1,18 @@
 {
   "$schema": "https://docs.renovatebot.com/renovate-schema.json",
   "extends": [
-    "config:base"
+    "config:recommended"
+  ],
+  "github-actions": {
+    "enabled": true,
+    "pinDigests": true
+  },
+  "packageRules": [
+    {
+      "matchManagers": [
+        "github-actions"
+      ],
+      "minimumReleaseAge": "7 days"
+    }
   ]
 }

data/waterdrop.gemspec

@@ -16,8 +16,8 @@ Gem::Specification.new do |spec|
   spec.description   = spec.summary
   spec.licenses      = %w[LGPL-3.0-only Commercial]
 
-  spec.add_dependency 'karafka-core', '>= 2.4.3', '< 3.0.0'
-  spec.add_dependency 'karafka-rdkafka', '>= 0.17.5'
+  spec.add_dependency 'karafka-core', '>= 2.4.9', '< 3.0.0'
+  spec.add_dependency 'karafka-rdkafka', '>= 0.19.1'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 
   spec.required_ruby_version = '>= 3.1.0'
@@ -28,7 +28,7 @@ Gem::Specification.new do |spec|
 
   spec.cert_chain    = %w[certs/cert.pem]
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec)/}) }
-  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.executables   = []
   spec.require_paths = %w[lib]
 
   spec.metadata = {

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: waterdrop
 version: !ruby/object:Gem::Version
-  version: 2.8.1
+  version: 2.8.3
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -34,7 +34,7 @@ cert_chain:
   i9zWxov0mr44TWegTVeypcWGd/0nxu1+QHVNHJrpqlPBRvwQsUm7fwmRInGpcaB8
   ap8wNYvryYzrzvzUxIVFBVM5PacgkFqRmolCa8I7tdKQN+R1
   -----END CERTIFICATE-----
-date: 2024-12-26 00:00:00.000000000 Z
+date: 2025-04-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -42,7 +42,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.4.3
+        version: 2.4.9
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -52,7 +52,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.4.3
+        version: 2.4.9
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -62,14 +62,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.17.5
+        version: 0.19.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.17.5
+        version: 0.19.1
 - !ruby/object:Gem::Dependency
   name: zeitwerk
   requirement: !ruby/object:Gem::Requirement
@@ -93,10 +93,12 @@ extra_rdoc_files: []
 files:
 - ".coditsu/ci.yml"
 - ".diffend.yml"
+- ".github/CODEOWNERS"
 - ".github/FUNDING.yml"
 - ".github/ISSUE_TEMPLATE/bug_report.md"
 - ".github/ISSUE_TEMPLATE/feature_request.md"
 - ".github/workflows/ci.yml"
+- ".github/workflows/verify-action-pins.yml"
 - ".gitignore"
 - ".rspec"
 - ".ruby-gemset"
@@ -106,6 +108,8 @@ files:
 - Gemfile.lock
 - LICENSE
 - README.md
+- bin/verify_kafka_warnings
+- bin/verify_topics_naming
 - certs/cert.pem
 - config/locales/errors.yml
 - docker-compose.yml