karafka-rdkafka 0.19.0 → 0.19.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5a1e9fa0ca2b5dd14aed77c653fc4d154bb566113cac11c04d51cccc4e1d9fb7
-  data.tar.gz: c513b0b82bdae4d9a16251a0abcd4b73a79ba21d2833814bccddcd8f6446151a
+  metadata.gz: 390b5be16a78ebe2b6f994429cfe32e51bc11d88735ef28bfa57b9b8ba34d73e
+  data.tar.gz: 66003597faaddea33ab464aa3ec9dd906f3676f85e12e1e7c7bc2379e35feef4
 SHA512:
-  metadata.gz: 53bee0b1c513f6947ca657ca3836df05e6de31ba441aa6d85d71f523c28cad7b996ec14fae798b50bbaafb09eb00367bbc4298da5a926e3fae61cc94cb5179bb
-  data.tar.gz: 51e903bb75f34fa7f49a8ebd6cdff193b2ee916fd0f4823145aae6ee219bf45e51ce538f3c9c3e4de9ba58c659fa4ecaf494768df1b83f998c9d2509cda58074
+  metadata.gz: b7856bee34f2d3d4abe28013d3a8905d4166067f5ce4a1cddc4dc6572d0ebfb03f412a3d4a53ac515a3523cbc8088774699da8fa2fad8d88272db4d08e2ec81e
+  data.tar.gz: 0d1fbbfce6be2bb41cfc7db66f53030e907f4c9c7f8e2251aabdeb7ca9786d120e22e971eebf426dbdd57dcb60c198506204cb2b4443333744c65339d9956746

data/.github/CODEOWNERS

@@ -0,0 +1,3 @@
+/.github @mensfeld
+/.github/workflows/ @mensfeld
+/.github/actions/ @mensfeld

data/.github/workflows/ci.yml

@@ -6,9 +6,14 @@ concurrency:
 
 on:
   pull_request:
+    branches: [ main, master ]
   push:
+    branches: [ main, master ]
   schedule:
-    - cron:  '0 1 * * *'
+    - cron: '0 1 * * *'
+
+permissions:
+  contents: read
 
 env:
   BUNDLE_RETRY: 6
@@ -30,16 +35,19 @@ jobs:
           - ruby: '3.4'
             coverage: 'true'
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          fetch-depth: 0
+
       - name: Install package dependencies
         run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
 
-      - name: Start Kafka with docker compose
+      - name: Start Kafka with Docker Compose
         run: |
           docker compose up -d || (sleep 5 && docker compose up -d)
 
       - name: Set up Ruby
-        uses: ruby/setup-ruby@v1
+        uses: ruby/setup-ruby@354a1ad156761f5ee2b7b13fa8e09943a5e8d252 # v1.229.0
         with:
           ruby-version: ${{matrix.ruby}}
           bundler-cache: true
@@ -47,15 +55,13 @@ jobs:
       - name: Run all specs
         env:
           GITHUB_COVERAGE: ${{matrix.coverage}}
-
         run: |
           set -e
-          bundle install --path vendor/bundle
+          bundle install --jobs 4 --retry 3
           cd ext && bundle exec rake
           cd ..
           bundle exec rspec
 
-
   macos_build:
     timeout-minutes: 30
     runs-on: macos-latest
@@ -68,10 +74,10 @@ jobs:
           - '3.2'
           - '3.1'
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - name: Set up Ruby
-        uses: ruby/setup-ruby@v1
+        uses: ruby/setup-ruby@354a1ad156761f5ee2b7b13fa8e09943a5e8d252 # v1.229.0
         with:
           ruby-version: ${{matrix.ruby}}
           bundler-cache: false
@@ -79,5 +85,5 @@ jobs:
       - name: Build rdkafka-ruby
         run: |
           set -e
-          bundle install --path vendor/bundle
+          bundle install --jobs 4 --retry 3
           cd ext && bundle exec rake

data/.github/workflows/verify-action-pins.yml

@@ -0,0 +1,16 @@
+name: Verify Action Pins
+on:
+  pull_request:
+    paths:
+      - '.github/workflows/**'
+jobs:
+  verify:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - name: Check SHA pins
+        run: |
+          if grep -E -r "uses: .*/.*@(v[0-9]+|main|master)($|[[:space:]]|$)" --include="*.yml" --include="*.yaml" .github/workflows/ | grep -v "#"; then
+            echo "::error::Actions should use SHA pins, not tags or branch names"
+            exit 1
+          fi

data/.ruby-version

@@ -1 +1 @@
-3.4.1
+3.4.2

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Rdkafka Changelog
 
+## 0.19.1 (2025-04-07)
+- [Enhancement] Support producing and consuming of headers with mulitple values (KIP-82).
+- [Enhancement] Allow native Kafka customization poll time.
+
 ## 0.19.0 (2025-01-20)
 - **[Breaking]** Deprecate and remove `#each_batch` due to data consistency concerns.
 - [Enhancement] Bump librdkafka to 2.8.0

data/README.md

@@ -163,6 +163,7 @@ bundle exec rake produce_messages
 
 | rdkafka-ruby | librdkafka | patches |
 |-|-|-|
+| 0.19.1 (2025-04-07) | 2.8.0 (2025-01-07) | yes |
 | 0.19.0 (2025-01-20) | 2.8.0 (2025-01-07) | yes |
 | 0.18.0 (2024-11-26) | 2.6.1 (2024-11-18) | yes |
 | 0.17.4 (2024-09-02) | 2.5.3 (2024-09-02) | yes |

data/docker-compose.yml

@@ -1,7 +1,7 @@
 services:
   kafka:
     container_name: kafka
-    image: confluentinc/cp-kafka:7.8.0
+    image: confluentinc/cp-kafka:7.9.0
 
     ports:
       - 9092:9092

data/lib/rdkafka/config.rb

@@ -233,11 +233,12 @@ module Rdkafka
     #
     # @param native_kafka_auto_start [Boolean] should the native kafka operations be started
     #   automatically. Defaults to true. Set to false only when doing complex initialization.
+    # @param native_kafka_poll_timeout_ms [Integer] ms poll time of the native Kafka
     # @return [Producer] The created producer
     #
     # @raise [ConfigError] When the configuration contains invalid options
     # @raise [ClientCreationError] When the native client cannot be created
-    def producer(native_kafka_auto_start: true)
+    def producer(native_kafka_auto_start: true, native_kafka_poll_timeout_ms: 100)
       # Create opaque
       opaque = Opaque.new
       # Create Kafka config
@@ -254,7 +255,8 @@ module Rdkafka
           kafka,
           run_polling_thread: true,
           opaque: opaque,
-          auto_start: native_kafka_auto_start
+          auto_start: native_kafka_auto_start,
+          timeout_ms: native_kafka_poll_timeout_ms
         ),
         partitioner_name
       ).tap do |producer|
@@ -266,11 +268,12 @@ module Rdkafka
     #
     # @param native_kafka_auto_start [Boolean] should the native kafka operations be started
     #   automatically. Defaults to true. Set to false only when doing complex initialization.
+    # @param native_kafka_poll_timeout_ms [Integer] ms poll time of the native Kafka
     # @return [Admin] The created admin instance
     #
     # @raise [ConfigError] When the configuration contains invalid options
     # @raise [ClientCreationError] When the native client cannot be created
-    def admin(native_kafka_auto_start: true)
+    def admin(native_kafka_auto_start: true, native_kafka_poll_timeout_ms: 100)
       opaque = Opaque.new
       config = native_config(opaque)
       Rdkafka::Bindings.rd_kafka_conf_set_background_event_cb(config, Rdkafka::Callbacks::BackgroundEventCallbackFunction)
@@ -282,7 +285,8 @@ module Rdkafka
           kafka,
           run_polling_thread: true,
           opaque: opaque,
-          auto_start: native_kafka_auto_start
+          auto_start: native_kafka_auto_start,
+          timeout_ms: native_kafka_poll_timeout_ms
         )
       )
     end

data/lib/rdkafka/consumer/headers.rb

@@ -7,11 +7,13 @@ module Rdkafka
       EMPTY_HEADERS = {}.freeze
 
       # Reads a librdkafka native message's headers and returns them as a Ruby Hash
+      # where each key maps to either a String (single value) or Array<String> (multiple values)
+      # to support duplicate headers per KIP-82
       #
       # @private
       #
       # @param [Rdkafka::Bindings::Message] native_message
-      # @return [Hash<String, String>] headers Hash for the native_message
+      # @return [Hash<String, String|Array<String>>] headers Hash for the native_message
       # @raise [Rdkafka::RdkafkaError] when fail to read headers
       def self.from_native(native_message)
         headers_ptrptr = FFI::MemoryPointer.new(:pointer)
@@ -53,10 +55,19 @@ module Rdkafka
           size = size_ptr[:value]
 
           value_ptr = value_ptrptr.read_pointer
-
           value = value_ptr.read_string(size)
 
-          headers[name] = value
+          if headers.key?(name)
+            # If we've seen this header before, convert to array if needed and append
+            if headers[name].is_a?(Array)
+              headers[name] << value
+            else
+              headers[name] = [headers[name], value]
+            end
+          else
+            # First occurrence - store as single value
+            headers[name] = value
+          end
 
           idx += 1
         end

data/lib/rdkafka/native_kafka.rb

@@ -4,7 +4,7 @@ module Rdkafka
   # @private
   # A wrapper around a native kafka that polls and cleanly exits
   class NativeKafka
-    def initialize(inner, run_polling_thread:, opaque:, auto_start: true)
+    def initialize(inner, run_polling_thread:, opaque:, auto_start: true, timeout_ms: 100)
       @inner = inner
       @opaque = opaque
       # Lock around external access
@@ -30,6 +30,8 @@ module Rdkafka
 
       @run_polling_thread = run_polling_thread
 
+      @timeout_ms = timeout_ms
+
       start if auto_start
 
       @closing = false
@@ -50,7 +52,7 @@ module Rdkafka
           @polling_thread = Thread.new do
             loop do
               @poll_mutex.synchronize do
-                Rdkafka::Bindings.rd_kafka_poll(@inner, 100)
+                Rdkafka::Bindings.rd_kafka_poll(@inner, @timeout_ms)
               end
 
               # Exit thread if closing and the poll queue is empty

data/lib/rdkafka/producer.rb

@@ -309,7 +309,7 @@ module Rdkafka
     # @param partition [Integer,nil] Optional partition to produce to
     # @param partition_key [String, nil] Optional partition key based on which partition assignment can happen
     # @param timestamp [Time,Integer,nil] Optional timestamp of this message. Integer timestamp is in milliseconds since Jan 1 1970.
-    # @param headers [Hash<String,String>] Optional message headers
+    # @param headers [Hash<String,String|Array<String>>] Optional message headers. Values can be either a single string or an array of strings to support duplicate headers per KIP-82
     # @param label [Object, nil] a label that can be assigned when producing a message that will be part of the delivery handle and the delivery report
     # @param topic_config [Hash] topic config for given message dispatch. Allows to send messages to topics with different configuration
     #
@@ -401,11 +401,23 @@ module Rdkafka
       if headers
         headers.each do |key0, value0|
           key = key0.to_s
-          value = value0.to_s
-          args << :int << Rdkafka::Bindings::RD_KAFKA_VTYPE_HEADER
-          args << :string << key
-          args << :pointer << value
-          args << :size_t << value.bytesize
+          if value0.is_a?(Array)
+            # Handle array of values per KIP-82
+            value0.each do |value|
+              value = value.to_s
+              args << :int << Rdkafka::Bindings::RD_KAFKA_VTYPE_HEADER
+              args << :string << key
+              args << :pointer << value
+              args << :size_t << value.bytesize
+            end
+          else
+            # Handle single value
+            value = value0.to_s
+            args << :int << Rdkafka::Bindings::RD_KAFKA_VTYPE_HEADER
+            args << :string << key
+            args << :pointer << value
+            args << :size_t << value.bytesize
+          end
         end
       end
 

data/lib/rdkafka/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Rdkafka
-  VERSION = "0.19.0"
+  VERSION = "0.19.1"
   LIBRDKAFKA_VERSION = "2.8.0"
   LIBRDKAFKA_SOURCE_SHA256 = "5bd1c46f63265f31c6bfcedcde78703f77d28238eadf23821c2b43fc30be3e25"
 end

data/renovate.json

@@ -1,6 +1,18 @@
 {
   "$schema": "https://docs.renovatebot.com/renovate-schema.json",
   "extends": [
-    "config:base"
+    "config:recommended"
+  ],
+  "github-actions": {
+    "enabled": true,
+    "pinDigests": true
+  },
+  "packageRules": [
+    {
+      "matchManagers": [
+        "github-actions"
+      ],
+      "minimumReleaseAge": "7 days"
+    }
   ]
 }

data/spec/rdkafka/consumer/headers_spec.rb

@@ -3,7 +3,7 @@
 describe Rdkafka::Consumer::Headers do
   let(:headers) do
     { # Note String keys!
-      "version" => "2.1.3",
+      "version" => ["2.1.3", "2.1.4"],
       "type" => "String"
     }
   end
@@ -17,27 +17,39 @@ describe Rdkafka::Consumer::Headers do
         Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
       end
 
+      # First version header
       expect(Rdkafka::Bindings).to \
         receive(:rd_kafka_header_get_all)
           .with(headers_ptr, 0, anything, anything, anything) do |_, _, name_ptrptr, value_ptrptr, size_ptr|
-              expect(name_ptrptr).to receive(:read_pointer).and_return(double("pointer 0", read_string_to_null: headers.keys[0]))
-              expect(size_ptr).to receive(:[]).with(:value).and_return(headers.keys[0].size)
-              expect(value_ptrptr).to receive(:read_pointer).and_return(double("value pointer 0", read_string: headers.values[0]))
+              expect(name_ptrptr).to receive(:read_pointer).and_return(double("pointer 0", read_string_to_null: "version"))
+              expect(size_ptr).to receive(:[]).with(:value).and_return(headers["version"][0].size)
+              expect(value_ptrptr).to receive(:read_pointer).and_return(double("value pointer 0", read_string: headers["version"][0]))
               Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
             end
 
+      # Second version header
       expect(Rdkafka::Bindings).to \
         receive(:rd_kafka_header_get_all)
           .with(headers_ptr, 1, anything, anything, anything) do |_, _, name_ptrptr, value_ptrptr, size_ptr|
-              expect(name_ptrptr).to receive(:read_pointer).and_return(double("pointer 1", read_string_to_null: headers.keys[1]))
-              expect(size_ptr).to receive(:[]).with(:value).and_return(headers.keys[1].size)
-              expect(value_ptrptr).to receive(:read_pointer).and_return(double("value pointer 1", read_string: headers.values[1]))
+              expect(name_ptrptr).to receive(:read_pointer).and_return(double("pointer 1", read_string_to_null: "version"))
+              expect(size_ptr).to receive(:[]).with(:value).and_return(headers["version"][1].size)
+              expect(value_ptrptr).to receive(:read_pointer).and_return(double("value pointer 1", read_string: headers["version"][1]))
               Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
             end
 
+      # Single type header
       expect(Rdkafka::Bindings).to \
         receive(:rd_kafka_header_get_all)
-          .with(headers_ptr, 2, anything, anything, anything)
+          .with(headers_ptr, 2, anything, anything, anything) do |_, _, name_ptrptr, value_ptrptr, size_ptr|
+              expect(name_ptrptr).to receive(:read_pointer).and_return(double("pointer 2", read_string_to_null: "type"))
+              expect(size_ptr).to receive(:[]).with(:value).and_return(headers["type"].size)
+              expect(value_ptrptr).to receive(:read_pointer).and_return(double("value pointer 2", read_string: headers["type"]))
+              Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+            end
+
+      expect(Rdkafka::Bindings).to \
+        receive(:rd_kafka_header_get_all)
+          .with(headers_ptr, 3, anything, anything, anything)
           .and_return(Rdkafka::Bindings::RD_KAFKA_RESP_ERR__NOENT)
     end
 
@@ -46,8 +58,12 @@ describe Rdkafka::Consumer::Headers do
     it { is_expected.to eq(headers) }
     it { is_expected.to be_frozen }
 
-    it 'allows String key' do
-      expect(subject['version']).to eq("2.1.3")
+    it 'returns array for duplicate headers' do
+      expect(subject['version']).to eq(["2.1.3", "2.1.4"])
+    end
+
+    it 'returns string for single headers' do
+      expect(subject['type']).to eq("String")
     end
 
     it 'does not support symbols mappings' do

data/spec/rdkafka/producer_spec.rb

@@ -1002,4 +1002,44 @@ describe Rdkafka::Producer do
       end
     end
   end
+
+  describe "#produce with headers" do
+    it "should produce a message with array headers" do
+      headers = {
+        "version" => ["2.1.3", "2.1.4"],
+        "type" => "String"
+      }
+
+      report = producer.produce(
+        topic:     "consume_test_topic",
+        key:       "key headers",
+        headers:   headers
+      ).wait
+
+      message = wait_for_message(topic: "consume_test_topic", consumer: consumer, delivery_report: report)
+      expect(message).to be
+      expect(message.key).to eq('key headers')
+      expect(message.headers['type']).to eq('String')
+      expect(message.headers['version']).to eq(["2.1.3", "2.1.4"])
+    end
+
+    it "should produce a message with single value headers" do
+      headers = {
+        "version" => "2.1.3",
+        "type" => "String"
+      }
+
+      report = producer.produce(
+        topic:     "consume_test_topic",
+        key:       "key headers",
+        headers:   headers
+      ).wait
+
+      message = wait_for_message(topic: "consume_test_topic", consumer: consumer, delivery_report: report)
+      expect(message).to be
+      expect(message.key).to eq('key headers')
+      expect(message.headers['type']).to eq('String')
+      expect(message.headers['version']).to eq('2.1.3')
+    end
+  end
 end

data.tar.gz.sig

@@ -1,2 +1,3 @@
-<����T��HT9wKo�s<��
-��Y@�J�w�vr���|���%W��6^[�G����q�3�D���ڟ0�:d6����#�ă�C*�Ƌ{����Lw�~��A�m�;6Ey��$0�D��@����`�-5@�	�V�%'��׏
󸤆L��'$��s����X|�!b*�[�,�����6�$Zli�Ek��¹g�}M��fi�0:���?����UЖJ��ki>i�$u*�����R&�܏�Q�P"š՚|$�cyJ��U"Ф�]QB�L4��|�������R�vq�ۃW'F�17(��ǻX���@]J�~�F�"��q�Ŝ���OJ�09�D�1Pq��>Q�/��4�d��O��oa/$/C��'h��Lr�(��\
+��E�d/�;��u'q�������ˏn)��}-^ �V�l-���^@M��%b���s��{i�l�ý��-��?�L���1�l�s��c��Z$xɪW�Ƚ��l(��D�|[����)�5V̚��k%�X~�Ÿ��%
+���1}kڃ��29���ؔ�1!T�ޯ[��8��	���^�Ԥ�gk�	3�kP��o�&)�=`
]c|u(�$
+!%������,Ȓ �������`��V�m+Y.�бt�M�������z�hc,���

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.19.0
+  version: 0.19.1
 platform: ruby
 authors:
 - Thijs Cadier
@@ -35,7 +35,7 @@ cert_chain:
   i9zWxov0mr44TWegTVeypcWGd/0nxu1+QHVNHJrpqlPBRvwQsUm7fwmRInGpcaB8
   ap8wNYvryYzrzvzUxIVFBVM5PacgkFqRmolCa8I7tdKQN+R1
   -----END CERTIFICATE-----
-date: 2025-01-20 00:00:00.000000000 Z
+date: 2025-04-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -171,8 +171,10 @@ extensions:
 - ext/Rakefile
 extra_rdoc_files: []
 files:
+- ".github/CODEOWNERS"
 - ".github/FUNDING.yml"
 - ".github/workflows/ci.yml"
+- ".github/workflows/verify-action-pins.yml"
 - ".gitignore"
 - ".rspec"
 - ".ruby-gemset"