karafka-rdkafka 0.18.1 → 0.19.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7270222d1729d0c663af135797273621a62399112b338104a6025a5bf316206d
-  data.tar.gz: 50631f1eb4a61b39e2d7e63e4e0eef6cc5c9d78082779846c0f673d533ac4744
+  metadata.gz: 390b5be16a78ebe2b6f994429cfe32e51bc11d88735ef28bfa57b9b8ba34d73e
+  data.tar.gz: 66003597faaddea33ab464aa3ec9dd906f3676f85e12e1e7c7bc2379e35feef4
 SHA512:
-  metadata.gz: 6e4d5898c45c77268d387381d39d8d983d93a9e41afd6800f5e4f84ef8fef1f88432dff88ced1636c042d1a03c0688a9b34a06e98ee7fb7c4939db69f0c61332
-  data.tar.gz: fe401c8c8ed8ae2b31078206ba311f932ae2ae90857a5b75cb619c689a8e894ea9cc662d88084c832f00a3e78740b13435f03930c38aa87c6efdf245f087be7b
+  metadata.gz: b7856bee34f2d3d4abe28013d3a8905d4166067f5ce4a1cddc4dc6572d0ebfb03f412a3d4a53ac515a3523cbc8088774699da8fa2fad8d88272db4d08e2ec81e
+  data.tar.gz: 0d1fbbfce6be2bb41cfc7db66f53030e907f4c9c7f8e2251aabdeb7ca9786d120e22e971eebf426dbdd57dcb60c198506204cb2b4443333744c65339d9956746

data/.github/CODEOWNERS

@@ -0,0 +1,3 @@
+/.github @mensfeld
+/.github/workflows/ @mensfeld
+/.github/actions/ @mensfeld

data/.github/workflows/ci.yml

@@ -6,9 +6,14 @@ concurrency:
 
 on:
   pull_request:
+    branches: [ main, master ]
   push:
+    branches: [ main, master ]
   schedule:
-    - cron:  '0 1 * * *'
+    - cron: '0 1 * * *'
+
+permissions:
+  contents: read
 
 env:
   BUNDLE_RETRY: 6
@@ -22,24 +27,27 @@ jobs:
       fail-fast: false
       matrix:
         ruby:
-          - '3.4.0-preview2'
+          - '3.4'
           - '3.3'
           - '3.2'
           - '3.1'
         include:
-          - ruby: '3.3'
+          - ruby: '3.4'
             coverage: 'true'
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          fetch-depth: 0
+
       - name: Install package dependencies
         run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
 
-      - name: Start Kafka with docker compose
+      - name: Start Kafka with Docker Compose
         run: |
           docker compose up -d || (sleep 5 && docker compose up -d)
 
       - name: Set up Ruby
-        uses: ruby/setup-ruby@v1
+        uses: ruby/setup-ruby@354a1ad156761f5ee2b7b13fa8e09943a5e8d252 # v1.229.0
         with:
           ruby-version: ${{matrix.ruby}}
           bundler-cache: true
@@ -47,10 +55,35 @@ jobs:
       - name: Run all specs
         env:
           GITHUB_COVERAGE: ${{matrix.coverage}}
-
         run: |
           set -e
-          bundle install --path vendor/bundle
+          bundle install --jobs 4 --retry 3
           cd ext && bundle exec rake
           cd ..
           bundle exec rspec
+
+  macos_build:
+    timeout-minutes: 30
+    runs-on: macos-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby:
+          - '3.4'
+          - '3.3'
+          - '3.2'
+          - '3.1'
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@354a1ad156761f5ee2b7b13fa8e09943a5e8d252 # v1.229.0
+        with:
+          ruby-version: ${{matrix.ruby}}
+          bundler-cache: false
+
+      - name: Build rdkafka-ruby
+        run: |
+          set -e
+          bundle install --jobs 4 --retry 3
+          cd ext && bundle exec rake

data/.github/workflows/verify-action-pins.yml

@@ -0,0 +1,16 @@
+name: Verify Action Pins
+on:
+  pull_request:
+    paths:
+      - '.github/workflows/**'
+jobs:
+  verify:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - name: Check SHA pins
+        run: |
+          if grep -E -r "uses: .*/.*@(v[0-9]+|main|master)($|[[:space:]]|$)" --include="*.yml" --include="*.yaml" .github/workflows/ | grep -v "#"; then
+            echo "::error::Actions should use SHA pins, not tags or branch names"
+            exit 1
+          fi

data/.ruby-version

@@ -1 +1 @@
-3.3.6
+3.4.2

data/CHANGELOG.md

@@ -1,5 +1,14 @@
 # Rdkafka Changelog
 
+## 0.19.1 (2025-04-07)
+- [Enhancement] Support producing and consuming of headers with mulitple values (KIP-82).
+- [Enhancement] Allow native Kafka customization poll time.
+
+## 0.19.0 (2025-01-20)
+- **[Breaking]** Deprecate and remove `#each_batch` due to data consistency concerns.
+- [Enhancement] Bump librdkafka to 2.8.0
+- [Fix] Restore `Rdkafka::Bindings.rd_kafka_global_init` as it was not the source of the original issue.
+
 ## 0.18.1 (2024-12-04)
 - [Fix] Do not run `Rdkafka::Bindings.rd_kafka_global_init` on require to prevent some of macos versions from hanging on Puma fork.
 

data/README.md

@@ -163,6 +163,8 @@ bundle exec rake produce_messages
 
 | rdkafka-ruby | librdkafka | patches |
 |-|-|-|
+| 0.19.1 (2025-04-07) | 2.8.0 (2025-01-07) | yes |
+| 0.19.0 (2025-01-20) | 2.8.0 (2025-01-07) | yes |
 | 0.18.0 (2024-11-26) | 2.6.1 (2024-11-18) | yes |
 | 0.17.4 (2024-09-02) | 2.5.3 (2024-09-02) | yes |
 | 0.17.0 (2024-08-01) | 2.5.0 (2024-07-10) | yes |

data/docker-compose.yml

@@ -1,9 +1,7 @@
-version: '2'
-
 services:
   kafka:
     container_name: kafka
-    image: confluentinc/cp-kafka:7.8.0
+    image: confluentinc/cp-kafka:7.9.0
 
     ports:
       - 9092:9092

data/ext/Rakefile

@@ -16,11 +16,11 @@ task :default => :clean do
     require "mini_portile2"
     recipe = MiniPortile.new("librdkafka", Rdkafka::LIBRDKAFKA_VERSION)
 
-    # Use default homebrew openssl if we're on mac and the directory exists
-    # and each of flags is not empty
-    if recipe.host&.include?("darwin") && system("which brew &> /dev/null") && Dir.exist?("#{homebrew_prefix = %x(brew --prefix openssl).strip}")
-      ENV["CPPFLAGS"] = "-I#{homebrew_prefix}/include" unless ENV["CPPFLAGS"]
-      ENV["LDFLAGS"] = "-L#{homebrew_prefix}/lib" unless ENV["LDFLAGS"]
+    # Use default homebrew openssl if we're on mac and the directory exists, is not using nix-prepared libraries
+    # and each of flags is not already set
+    if recipe.host&.include?("darwin") && system("which brew &> /dev/null") && Dir.exist?("#{homebrew_prefix = %x(brew --prefix openssl).strip}") && ENV.key?("NIX_LDFLAGS")
+      ENV["CPPFLAGS"] = "-I#{homebrew_prefix}/include" unless ENV.key?("CPPFLAGS")
+      ENV["LDFLAGS"] = "-L#{homebrew_prefix}/lib" unless ENV.key?("LDFLAGS")
     end
 
     releases = File.expand_path(File.join(File.dirname(__FILE__), '../dist'))

data/lib/rdkafka/config.rb

@@ -233,11 +233,12 @@ module Rdkafka
     #
     # @param native_kafka_auto_start [Boolean] should the native kafka operations be started
     #   automatically. Defaults to true. Set to false only when doing complex initialization.
+    # @param native_kafka_poll_timeout_ms [Integer] ms poll time of the native Kafka
     # @return [Producer] The created producer
     #
     # @raise [ConfigError] When the configuration contains invalid options
     # @raise [ClientCreationError] When the native client cannot be created
-    def producer(native_kafka_auto_start: true)
+    def producer(native_kafka_auto_start: true, native_kafka_poll_timeout_ms: 100)
       # Create opaque
       opaque = Opaque.new
       # Create Kafka config
@@ -254,7 +255,8 @@ module Rdkafka
           kafka,
           run_polling_thread: true,
           opaque: opaque,
-          auto_start: native_kafka_auto_start
+          auto_start: native_kafka_auto_start,
+          timeout_ms: native_kafka_poll_timeout_ms
         ),
         partitioner_name
       ).tap do |producer|
@@ -266,11 +268,12 @@ module Rdkafka
     #
     # @param native_kafka_auto_start [Boolean] should the native kafka operations be started
     #   automatically. Defaults to true. Set to false only when doing complex initialization.
+    # @param native_kafka_poll_timeout_ms [Integer] ms poll time of the native Kafka
     # @return [Admin] The created admin instance
     #
     # @raise [ConfigError] When the configuration contains invalid options
     # @raise [ClientCreationError] When the native client cannot be created
-    def admin(native_kafka_auto_start: true)
+    def admin(native_kafka_auto_start: true, native_kafka_poll_timeout_ms: 100)
       opaque = Opaque.new
       config = native_config(opaque)
       Rdkafka::Bindings.rd_kafka_conf_set_background_event_cb(config, Rdkafka::Callbacks::BackgroundEventCallbackFunction)
@@ -282,7 +285,8 @@ module Rdkafka
           kafka,
           run_polling_thread: true,
           opaque: opaque,
-          auto_start: native_kafka_auto_start
+          auto_start: native_kafka_auto_start,
+          timeout_ms: native_kafka_poll_timeout_ms
         )
       )
     end

data/lib/rdkafka/consumer/headers.rb

@@ -7,11 +7,13 @@ module Rdkafka
       EMPTY_HEADERS = {}.freeze
 
       # Reads a librdkafka native message's headers and returns them as a Ruby Hash
+      # where each key maps to either a String (single value) or Array<String> (multiple values)
+      # to support duplicate headers per KIP-82
       #
       # @private
       #
       # @param [Rdkafka::Bindings::Message] native_message
-      # @return [Hash<String, String>] headers Hash for the native_message
+      # @return [Hash<String, String|Array<String>>] headers Hash for the native_message
       # @raise [Rdkafka::RdkafkaError] when fail to read headers
       def self.from_native(native_message)
         headers_ptrptr = FFI::MemoryPointer.new(:pointer)
@@ -53,10 +55,19 @@ module Rdkafka
           size = size_ptr[:value]
 
           value_ptr = value_ptrptr.read_pointer
-
           value = value_ptr.read_string(size)
 
-          headers[name] = value
+          if headers.key?(name)
+            # If we've seen this header before, convert to array if needed and append
+            if headers[name].is_a?(Array)
+              headers[name] << value
+            else
+              headers[name] = [headers[name], value]
+            end
+          else
+            # First occurrence - store as single value
+            headers[name] = value
+          end
 
           idx += 1
         end

data/lib/rdkafka/consumer.rb

@@ -619,87 +619,23 @@ module Rdkafka
       end
     end
 
-    # Poll for new messages and yield them in batches that may contain
-    # messages from more than one partition.
-    #
-    # Rather than yield each message immediately as soon as it is received,
-    # each_batch will attempt to wait for as long as `timeout_ms` in order
-    # to create a batch of up to but no more than `max_items` in size.
-    #
-    # Said differently, if more than `max_items` are available within
-    # `timeout_ms`, then `each_batch` will yield early with `max_items` in the
-    # array, but if `timeout_ms` passes by with fewer messages arriving, it
-    # will yield an array of fewer messages, quite possibly zero.
-    #
-    # In order to prevent wrongly auto committing many messages at once across
-    # possibly many partitions, callers must explicitly indicate which messages
-    # have been successfully processed as some consumed messages may not have
-    # been yielded yet. To do this, the caller should set
-    # `enable.auto.offset.store` to false and pass processed messages to
-    # {store_offset}. It is also possible, though more complex, to set
-    # 'enable.auto.commit' to false and then pass a manually assembled
-    # TopicPartitionList to {commit}.
-    #
-    # As with `each`, iteration will end when the consumer is closed.
-    #
-    # Exception behavior is more complicated than with `each`, in that if
-    # :yield_on_error is true, and an exception is raised during the
-    # poll, and messages have already been received, they will be yielded to
-    # the caller before the exception is allowed to propagate.
-    #
-    # If you are setting either auto.commit or auto.offset.store to false in
-    # the consumer configuration, then you should let yield_on_error keep its
-    # default value of false because you are guaranteed to see these messages
-    # again. However, if both auto.commit and auto.offset.store are set to
-    # true, you should set yield_on_error to true so you can process messages
-    # that you may or may not see again.
-    #
-    # @param max_items [Integer] Maximum size of the yielded array of messages
-    # @param bytes_threshold [Integer] Threshold number of total message bytes in the yielded array of messages
-    # @param timeout_ms [Integer] max time to wait for up to max_items
-    #
-    # @yieldparam messages [Array] An array of received Message
-    # @yieldparam pending_exception [Exception] normally nil, or an exception
-    #
-    # @yield [messages, pending_exception]
-    # which will be propagated after processing of the partial batch is complete.
-    #
-    # @return [nil]
-    #
-    # @raise [RdkafkaError] When polling fails
+    # Deprecated. Please read the error message for more details.
     def each_batch(max_items: 100, bytes_threshold: Float::INFINITY, timeout_ms: 250, yield_on_error: false, &block)
-      closed_consumer_check(__method__)
-      slice = []
-      bytes = 0
-      end_time = monotonic_now + timeout_ms / 1000.0
-      loop do
-        break if closed?
-        max_wait = end_time - monotonic_now
-        max_wait_ms = if max_wait <= 0
-                        0  # should not block, but may retrieve a message
-                      else
-                        (max_wait * 1000).floor
-                      end
-        message = nil
-        begin
-          message = poll max_wait_ms
-        rescue Rdkafka::RdkafkaError => error
-          raise unless yield_on_error
-          raise if slice.empty?
-          yield slice.dup, error
-          raise
-        end
-        if message
-          slice << message
-          bytes += message.payload.bytesize if message.payload
-        end
-        if slice.size == max_items || bytes >= bytes_threshold || monotonic_now >= end_time - 0.001
-          yield slice.dup, nil
-          slice.clear
-          bytes = 0
-          end_time = monotonic_now + timeout_ms / 1000.0
-        end
-      end
+      raise NotImplementedError, <<~ERROR
+        `each_batch` has been removed due to data consistency concerns.
+
+        This method was removed because it did not properly handle partition reassignments,
+        which could lead to processing messages from partitions that were no longer owned
+        by this consumer, resulting in duplicate message processing and data inconsistencies.
+
+        Recommended alternatives:
+
+        1. Implement your own batching logic using rebalance callbacks to properly handle
+           partition revocations and ensure message processing correctness.
+
+        2. Use a high-level batching library that supports proper partition reassignment
+           handling out of the box (such as the Karafka framework).
+      ERROR
     end
 
     # Returns pointer to the consumer group metadata. It is used only in the context of

data/lib/rdkafka/native_kafka.rb

@@ -4,7 +4,7 @@ module Rdkafka
   # @private
   # A wrapper around a native kafka that polls and cleanly exits
   class NativeKafka
-    def initialize(inner, run_polling_thread:, opaque:, auto_start: true)
+    def initialize(inner, run_polling_thread:, opaque:, auto_start: true, timeout_ms: 100)
       @inner = inner
       @opaque = opaque
       # Lock around external access
@@ -30,6 +30,8 @@ module Rdkafka
 
       @run_polling_thread = run_polling_thread
 
+      @timeout_ms = timeout_ms
+
       start if auto_start
 
       @closing = false
@@ -50,7 +52,7 @@ module Rdkafka
           @polling_thread = Thread.new do
             loop do
               @poll_mutex.synchronize do
-                Rdkafka::Bindings.rd_kafka_poll(@inner, 100)
+                Rdkafka::Bindings.rd_kafka_poll(@inner, @timeout_ms)
               end
 
               # Exit thread if closing and the poll queue is empty

data/lib/rdkafka/producer.rb

@@ -309,7 +309,7 @@ module Rdkafka
     # @param partition [Integer,nil] Optional partition to produce to
     # @param partition_key [String, nil] Optional partition key based on which partition assignment can happen
     # @param timestamp [Time,Integer,nil] Optional timestamp of this message. Integer timestamp is in milliseconds since Jan 1 1970.
-    # @param headers [Hash<String,String>] Optional message headers
+    # @param headers [Hash<String,String|Array<String>>] Optional message headers. Values can be either a single string or an array of strings to support duplicate headers per KIP-82
     # @param label [Object, nil] a label that can be assigned when producing a message that will be part of the delivery handle and the delivery report
     # @param topic_config [Hash] topic config for given message dispatch. Allows to send messages to topics with different configuration
     #
@@ -401,11 +401,23 @@ module Rdkafka
       if headers
         headers.each do |key0, value0|
           key = key0.to_s
-          value = value0.to_s
-          args << :int << Rdkafka::Bindings::RD_KAFKA_VTYPE_HEADER
-          args << :string << key
-          args << :pointer << value
-          args << :size_t << value.bytesize
+          if value0.is_a?(Array)
+            # Handle array of values per KIP-82
+            value0.each do |value|
+              value = value.to_s
+              args << :int << Rdkafka::Bindings::RD_KAFKA_VTYPE_HEADER
+              args << :string << key
+              args << :pointer << value
+              args << :size_t << value.bytesize
+            end
+          else
+            # Handle single value
+            value = value0.to_s
+            args << :int << Rdkafka::Bindings::RD_KAFKA_VTYPE_HEADER
+            args << :string << key
+            args << :pointer << value
+            args << :size_t << value.bytesize
+          end
         end
       end
 

data/lib/rdkafka/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Rdkafka
-  VERSION = "0.18.1"
-  LIBRDKAFKA_VERSION = "2.6.1"
-  LIBRDKAFKA_SOURCE_SHA256 = "0ddf205ad8d36af0bc72a2fec20639ea02e1d583e353163bf7f4683d949e901b"
+  VERSION = "0.19.1"
+  LIBRDKAFKA_VERSION = "2.8.0"
+  LIBRDKAFKA_SOURCE_SHA256 = "5bd1c46f63265f31c6bfcedcde78703f77d28238eadf23821c2b43fc30be3e25"
 end

data/lib/rdkafka.rb

@@ -49,3 +49,5 @@ require "rdkafka/producer/delivery_report"
 # Main Rdkafka namespace of this gem
 module Rdkafka
 end
+
+Rdkafka::Bindings.rd_kafka_global_init

data/renovate.json

@@ -1,6 +1,18 @@
 {
   "$schema": "https://docs.renovatebot.com/renovate-schema.json",
   "extends": [
-    "config:base"
+    "config:recommended"
+  ],
+  "github-actions": {
+    "enabled": true,
+    "pinDigests": true
+  },
+  "packageRules": [
+    {
+      "matchManagers": [
+        "github-actions"
+      ],
+      "minimumReleaseAge": "7 days"
+    }
   ]
 }

data/spec/rdkafka/consumer/headers_spec.rb

@@ -3,7 +3,7 @@
 describe Rdkafka::Consumer::Headers do
   let(:headers) do
     { # Note String keys!
-      "version" => "2.1.3",
+      "version" => ["2.1.3", "2.1.4"],
       "type" => "String"
     }
   end
@@ -17,27 +17,39 @@ describe Rdkafka::Consumer::Headers do
         Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
       end
 
+      # First version header
       expect(Rdkafka::Bindings).to \
         receive(:rd_kafka_header_get_all)
           .with(headers_ptr, 0, anything, anything, anything) do |_, _, name_ptrptr, value_ptrptr, size_ptr|
-              expect(name_ptrptr).to receive(:read_pointer).and_return(double("pointer 0", read_string_to_null: headers.keys[0]))
-              expect(size_ptr).to receive(:[]).with(:value).and_return(headers.keys[0].size)
-              expect(value_ptrptr).to receive(:read_pointer).and_return(double("value pointer 0", read_string: headers.values[0]))
+              expect(name_ptrptr).to receive(:read_pointer).and_return(double("pointer 0", read_string_to_null: "version"))
+              expect(size_ptr).to receive(:[]).with(:value).and_return(headers["version"][0].size)
+              expect(value_ptrptr).to receive(:read_pointer).and_return(double("value pointer 0", read_string: headers["version"][0]))
               Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
             end
 
+      # Second version header
       expect(Rdkafka::Bindings).to \
         receive(:rd_kafka_header_get_all)
           .with(headers_ptr, 1, anything, anything, anything) do |_, _, name_ptrptr, value_ptrptr, size_ptr|
-              expect(name_ptrptr).to receive(:read_pointer).and_return(double("pointer 1", read_string_to_null: headers.keys[1]))
-              expect(size_ptr).to receive(:[]).with(:value).and_return(headers.keys[1].size)
-              expect(value_ptrptr).to receive(:read_pointer).and_return(double("value pointer 1", read_string: headers.values[1]))
+              expect(name_ptrptr).to receive(:read_pointer).and_return(double("pointer 1", read_string_to_null: "version"))
+              expect(size_ptr).to receive(:[]).with(:value).and_return(headers["version"][1].size)
+              expect(value_ptrptr).to receive(:read_pointer).and_return(double("value pointer 1", read_string: headers["version"][1]))
               Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
             end
 
+      # Single type header
       expect(Rdkafka::Bindings).to \
         receive(:rd_kafka_header_get_all)
-          .with(headers_ptr, 2, anything, anything, anything)
+          .with(headers_ptr, 2, anything, anything, anything) do |_, _, name_ptrptr, value_ptrptr, size_ptr|
+              expect(name_ptrptr).to receive(:read_pointer).and_return(double("pointer 2", read_string_to_null: "type"))
+              expect(size_ptr).to receive(:[]).with(:value).and_return(headers["type"].size)
+              expect(value_ptrptr).to receive(:read_pointer).and_return(double("value pointer 2", read_string: headers["type"]))
+              Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+            end
+
+      expect(Rdkafka::Bindings).to \
+        receive(:rd_kafka_header_get_all)
+          .with(headers_ptr, 3, anything, anything, anything)
           .and_return(Rdkafka::Bindings::RD_KAFKA_RESP_ERR__NOENT)
     end
 
@@ -46,8 +58,12 @@ describe Rdkafka::Consumer::Headers do
     it { is_expected.to eq(headers) }
     it { is_expected.to be_frozen }
 
-    it 'allows String key' do
-      expect(subject['version']).to eq("2.1.3")
+    it 'returns array for duplicate headers' do
+      expect(subject['version']).to eq(["2.1.3", "2.1.4"])
+    end
+
+    it 'returns string for single headers' do
+      expect(subject['type']).to eq("String")
     end
 
     it 'does not support symbols mappings' do

data/spec/rdkafka/consumer_spec.rb

@@ -948,236 +948,10 @@ describe Rdkafka::Consumer do
   end
 
   describe "#each_batch" do
-    let(:message_payload) { 'a' * 10 }
-
-    before do
-      @topic = SecureRandom.base64(10).tr('+=/', '')
-    end
-
-    after do
-      @topic = nil
-    end
-
-    def topic_name
-      @topic
-    end
-
-    def produce_n(n)
-      handles = []
-      n.times do |i|
-        handles << producer.produce(
-          topic:     topic_name,
-          payload:   i % 10 == 0 ? nil : Time.new.to_f.to_s,
-          key:       i.to_s,
-          partition: 0
-        )
-      end
-      handles.each(&:wait)
-    end
-
-    def new_message
-      instance_double("Rdkafka::Consumer::Message").tap do |message|
-        allow(message).to receive(:payload).and_return(message_payload)
-      end
-    end
-
-    it "retrieves messages produced into a topic" do
-      # This is the only each_batch test that actually produces real messages
-      # into a topic in the real kafka of the container.
-      #
-      # The other tests stub 'poll' which makes them faster and more reliable,
-      # but it makes sense to keep a single test with a fully integrated flow.
-      # This will help to catch breaking changes in the behavior of 'poll',
-      # libdrkafka, or Kafka.
-      #
-      # This is, in effect, an integration test and the subsequent specs are
-      # unit tests.
-      admin = rdkafka_config.admin
-      create_topic_handle = admin.create_topic(topic_name, 1, 1)
-      create_topic_handle.wait(max_wait_timeout: 15.0)
-      consumer.subscribe(topic_name)
-      produce_n 42
-      all_yields = []
-      consumer.each_batch(max_items: 10) do |batch|
-        all_yields << batch
-        break if all_yields.flatten.size >= 42
-      end
-      expect(all_yields.flatten.first).to be_a Rdkafka::Consumer::Message
-      expect(all_yields.flatten.size).to eq 42
-      expect(all_yields.size).to be > 4
-      expect(all_yields.flatten.map(&:key)).to eq (0..41).map { |x| x.to_s }
-      admin.close
-    end
-
-    it "should batch poll results and yield arrays of messages" do
-      consumer.subscribe(topic_name)
-      all_yields = []
-      expect(consumer)
-        .to receive(:poll)
-        .exactly(10).times
-        .and_return(new_message)
-      consumer.each_batch(max_items: 10) do |batch|
-        all_yields << batch
-        break if all_yields.flatten.size >= 10
-      end
-      expect(all_yields.first).to be_instance_of(Array)
-      expect(all_yields.flatten.size).to eq 10
-      non_empty_yields = all_yields.reject { |batch| batch.empty? }
-      expect(non_empty_yields.size).to be < 10
-    end
-
-    it "should yield a partial batch if the timeout is hit with some messages" do
-      consumer.subscribe(topic_name)
-      poll_count = 0
-      expect(consumer)
-        .to receive(:poll)
-        .at_least(3).times do
-        poll_count = poll_count + 1
-        if poll_count > 2
-          sleep 0.1
-          nil
-        else
-          new_message
-        end
-      end
-      all_yields = []
-      consumer.each_batch(max_items: 10) do |batch|
-        all_yields << batch
-        break if all_yields.flatten.size >= 2
-      end
-      expect(all_yields.flatten.size).to eq 2
-    end
-
-    it "should yield [] if nothing is received before the timeout" do
-      admin = rdkafka_config.admin
-      create_topic_handle = admin.create_topic(topic_name, 1, 1)
-      create_topic_handle.wait(max_wait_timeout: 15.0)
-      consumer.subscribe(topic_name)
-      consumer.each_batch do |batch|
-        expect(batch).to eq([])
-        break
-      end
-      admin.close
-    end
-
-    it "should yield batchs of max_items in size if messages are already fetched" do
-      yielded_batches = []
-      expect(consumer)
-        .to receive(:poll)
-        .with(anything)
-        .exactly(20).times
-        .and_return(new_message)
-
-      consumer.each_batch(max_items: 10, timeout_ms: 500) do |batch|
-        yielded_batches << batch
-        break if yielded_batches.flatten.size >= 20
-        break if yielded_batches.size >= 20 # so failure doesn't hang
-      end
-      expect(yielded_batches.size).to eq 2
-      expect(yielded_batches.map(&:size)).to eq 2.times.map { 10 }
-    end
-
-    it "should yield batchs as soon as bytes_threshold is hit" do
-      yielded_batches = []
-      expect(consumer)
-        .to receive(:poll)
-        .with(anything)
-        .exactly(20).times
-        .and_return(new_message)
-
-      consumer.each_batch(bytes_threshold: message_payload.size * 4, timeout_ms: 500) do |batch|
-        yielded_batches << batch
-        break if yielded_batches.flatten.size >= 20
-        break if yielded_batches.size >= 20 # so failure doesn't hang
-      end
-      expect(yielded_batches.size).to eq 5
-      expect(yielded_batches.map(&:size)).to eq 5.times.map { 4 }
-    end
-
-    context "error raised from poll and yield_on_error is true" do
-      it "should yield buffered exceptions on rebalance, then break" do
-        config = rdkafka_consumer_config(
-          {
-            :"enable.auto.commit" => false,
-            :"enable.auto.offset.store" => false
-          }
-        )
-        consumer = config.consumer
-        consumer.subscribe(topic_name)
-        batches_yielded = []
-        exceptions_yielded = []
-        each_batch_iterations = 0
-        poll_count = 0
-        expect(consumer)
-          .to receive(:poll)
-          .with(anything)
-          .exactly(3).times
-          .and_wrap_original do |method, *args|
-              poll_count = poll_count + 1
-              if poll_count == 3
-                raise Rdkafka::RdkafkaError.new(27,
-                    "partitions ... too ... heavy ... must ... rebalance")
-              else
-                new_message
-              end
-          end
-        expect {
-          consumer.each_batch(max_items: 30, yield_on_error: true) do |batch, pending_error|
-            batches_yielded << batch
-            exceptions_yielded << pending_error
-            each_batch_iterations = each_batch_iterations + 1
-          end
-        }.to raise_error(Rdkafka::RdkafkaError)
-        expect(poll_count).to eq 3
-        expect(each_batch_iterations).to eq 1
-        expect(batches_yielded.size).to eq 1
-        expect(batches_yielded.first.size).to eq 2
-        expect(exceptions_yielded.flatten.size).to eq 1
-        expect(exceptions_yielded.flatten.first).to be_instance_of(Rdkafka::RdkafkaError)
-        consumer.close
-      end
-    end
-
-    context "error raised from poll and yield_on_error is false" do
-      it "should yield buffered exceptions on rebalance, then break" do
-        config = rdkafka_consumer_config(
-          {
-            :"enable.auto.commit" => false,
-            :"enable.auto.offset.store" => false
-          }
-        )
-        consumer = config.consumer
-        consumer.subscribe(topic_name)
-        batches_yielded = []
-        exceptions_yielded = []
-        each_batch_iterations = 0
-        poll_count = 0
-        expect(consumer)
-          .to receive(:poll)
-          .with(anything)
-          .exactly(3).times
-          .and_wrap_original do |method, *args|
-              poll_count = poll_count + 1
-              if poll_count == 3
-                raise Rdkafka::RdkafkaError.new(27,
-                    "partitions ... too ... heavy ... must ... rebalance")
-              else
-                new_message
-              end
-          end
-        expect {
-          consumer.each_batch(max_items: 30, yield_on_error: false) do |batch, pending_error|
-            batches_yielded << batch
-            exceptions_yielded << pending_error
-            each_batch_iterations = each_batch_iterations + 1
-          end
-        }.to raise_error(Rdkafka::RdkafkaError)
-        expect(poll_count).to eq 3
-        expect(each_batch_iterations).to eq 0
-        expect(batches_yielded.size).to eq 0
-        expect(exceptions_yielded.size).to eq 0
-        consumer.close
-      end
+    it 'expect to raise an error' do
+      expect do
+        consumer.each_batch {}
+      end.to raise_error(NotImplementedError)
     end
   end
 
@@ -1344,7 +1118,6 @@ describe Rdkafka::Consumer do
     {
         :subscribe               => [ nil ],
         :unsubscribe             => nil,
-        :each_batch              => nil,
         :pause                   => [ nil ],
         :resume                  => [ nil ],
         :subscription            => nil,

data/spec/rdkafka/producer_spec.rb

@@ -1002,4 +1002,44 @@ describe Rdkafka::Producer do
       end
     end
   end
+
+  describe "#produce with headers" do
+    it "should produce a message with array headers" do
+      headers = {
+        "version" => ["2.1.3", "2.1.4"],
+        "type" => "String"
+      }
+
+      report = producer.produce(
+        topic:     "consume_test_topic",
+        key:       "key headers",
+        headers:   headers
+      ).wait
+
+      message = wait_for_message(topic: "consume_test_topic", consumer: consumer, delivery_report: report)
+      expect(message).to be
+      expect(message.key).to eq('key headers')
+      expect(message.headers['type']).to eq('String')
+      expect(message.headers['version']).to eq(["2.1.3", "2.1.4"])
+    end
+
+    it "should produce a message with single value headers" do
+      headers = {
+        "version" => "2.1.3",
+        "type" => "String"
+      }
+
+      report = producer.produce(
+        topic:     "consume_test_topic",
+        key:       "key headers",
+        headers:   headers
+      ).wait
+
+      message = wait_for_message(topic: "consume_test_topic", consumer: consumer, delivery_report: report)
+      expect(message).to be
+      expect(message.key).to eq('key headers')
+      expect(message.headers['type']).to eq('String')
+      expect(message.headers['version']).to eq('2.1.3')
+    end
+  end
 end

data.tar.gz.sig

@@ -1,3 +1,3 @@
-E��~����4G�ڂ��m9:b�%�Ȉc�>��S��u��(?S��;8`���cΪ�1e�.Eh�s�a[�s�Q�RW!���D0�{�g�Q;��'���k.J�x�2�uޥ�B�uA��W���\w�"Wo�s����BU�L���Y=q`*	����[��C-�C*D�i�$h�.�9���b�,�'�`
-бC��2����w3��E��:1���R��q]4s�p��L
-l�ᔕ�J�p�W�Zo#��Yߢ����SAXCu�5��o����k�ua�߮�r�}F��leg�h�S.�oV*.���;'k��^�c7�z8�WjՊ!~�
+��E�d/�;��u'q�������ˏn)��}-^ �V�l-���^@M��%b���s��{i�l�ý��-��?�L���1�l�s��c��Z$xɪW�Ƚ��l(��D�|[����)�5V̚��k%�X~�Ÿ��%
+���1}kڃ��29���ؔ�1!T�ޯ[��8��	���^�Ԥ�gk�	3�kP��o�&)�=`
]c|u(�$
+!%������,Ȓ �������`��V�m+Y.�бt�M�������z�hc,���

metadata

@@ -1,12 +1,11 @@
 --- !ruby/object:Gem::Specification
 name: karafka-rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.18.1
+  version: 0.19.1
 platform: ruby
 authors:
 - Thijs Cadier
 - Maciej Mensfeld
-autorequire: 
 bindir: bin
 cert_chain:
 - |
@@ -36,7 +35,7 @@ cert_chain:
   i9zWxov0mr44TWegTVeypcWGd/0nxu1+QHVNHJrpqlPBRvwQsUm7fwmRInGpcaB8
   ap8wNYvryYzrzvzUxIVFBVM5PacgkFqRmolCa8I7tdKQN+R1
   -----END CERTIFICATE-----
-date: 2024-12-04 00:00:00.000000000 Z
+date: 2025-04-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -172,8 +171,10 @@ extensions:
 - ext/Rakefile
 extra_rdoc_files: []
 files:
+- ".github/CODEOWNERS"
 - ".github/FUNDING.yml"
 - ".github/workflows/ci.yml"
+- ".github/workflows/verify-action-pins.yml"
 - ".gitignore"
 - ".rspec"
 - ".ruby-gemset"
@@ -186,7 +187,7 @@ files:
 - README.md
 - Rakefile
 - certs/cert.pem
-- dist/librdkafka-2.6.1.tar.gz
+- dist/librdkafka-2.8.0.tar.gz
 - dist/patches/rdkafka_global_init.patch
 - docker-compose.yml
 - ext/README.md
@@ -261,7 +262,6 @@ files:
 - spec/rdkafka/producer/delivery_report_spec.rb
 - spec/rdkafka/producer_spec.rb
 - spec/spec_helper.rb
-homepage: 
 licenses:
 - MIT
 metadata:
@@ -272,7 +272,6 @@ metadata:
   source_code_uri: https://github.com/karafka/karafka-rdkafka
   documentation_uri: https://karafka.io/docs
   rubygems_mfa_required: 'true'
-post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -287,8 +286,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.22
-signing_key: 
+rubygems_version: 3.6.2
 specification_version: 4
 summary: The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka.
   It wraps the production-ready C client using the ffi gem and targets Kafka 1.0+