karafka-rdkafka 0.18.1 → 0.19.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7270222d1729d0c663af135797273621a62399112b338104a6025a5bf316206d
-  data.tar.gz: 50631f1eb4a61b39e2d7e63e4e0eef6cc5c9d78082779846c0f673d533ac4744
+  metadata.gz: 5a1e9fa0ca2b5dd14aed77c653fc4d154bb566113cac11c04d51cccc4e1d9fb7
+  data.tar.gz: c513b0b82bdae4d9a16251a0abcd4b73a79ba21d2833814bccddcd8f6446151a
 SHA512:
-  metadata.gz: 6e4d5898c45c77268d387381d39d8d983d93a9e41afd6800f5e4f84ef8fef1f88432dff88ced1636c042d1a03c0688a9b34a06e98ee7fb7c4939db69f0c61332
-  data.tar.gz: fe401c8c8ed8ae2b31078206ba311f932ae2ae90857a5b75cb619c689a8e894ea9cc662d88084c832f00a3e78740b13435f03930c38aa87c6efdf245f087be7b
+  metadata.gz: 53bee0b1c513f6947ca657ca3836df05e6de31ba441aa6d85d71f523c28cad7b996ec14fae798b50bbaafb09eb00367bbc4298da5a926e3fae61cc94cb5179bb
+  data.tar.gz: 51e903bb75f34fa7f49a8ebd6cdff193b2ee916fd0f4823145aae6ee219bf45e51ce538f3c9c3e4de9ba58c659fa4ecaf494768df1b83f998c9d2509cda58074

data/.github/workflows/ci.yml

@@ -22,12 +22,12 @@ jobs:
       fail-fast: false
       matrix:
         ruby:
-          - '3.4.0-preview2'
+          - '3.4'
           - '3.3'
           - '3.2'
           - '3.1'
         include:
-          - ruby: '3.3'
+          - ruby: '3.4'
             coverage: 'true'
     steps:
       - uses: actions/checkout@v4
@@ -54,3 +54,30 @@ jobs:
           cd ext && bundle exec rake
           cd ..
           bundle exec rspec
+
+
+  macos_build:
+    timeout-minutes: 30
+    runs-on: macos-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby:
+          - '3.4'
+          - '3.3'
+          - '3.2'
+          - '3.1'
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{matrix.ruby}}
+          bundler-cache: false
+
+      - name: Build rdkafka-ruby
+        run: |
+          set -e
+          bundle install --path vendor/bundle
+          cd ext && bundle exec rake

data/.ruby-version

@@ -1 +1 @@
-3.3.6
+3.4.1

data/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Rdkafka Changelog
 
+## 0.19.0 (2025-01-20)
+- **[Breaking]** Deprecate and remove `#each_batch` due to data consistency concerns.
+- [Enhancement] Bump librdkafka to 2.8.0
+- [Fix] Restore `Rdkafka::Bindings.rd_kafka_global_init` as it was not the source of the original issue.
+
 ## 0.18.1 (2024-12-04)
 - [Fix] Do not run `Rdkafka::Bindings.rd_kafka_global_init` on require to prevent some of macos versions from hanging on Puma fork.
 

data/README.md

@@ -163,6 +163,7 @@ bundle exec rake produce_messages
 
 | rdkafka-ruby | librdkafka | patches |
 |-|-|-|
+| 0.19.0 (2025-01-20) | 2.8.0 (2025-01-07) | yes |
 | 0.18.0 (2024-11-26) | 2.6.1 (2024-11-18) | yes |
 | 0.17.4 (2024-09-02) | 2.5.3 (2024-09-02) | yes |
 | 0.17.0 (2024-08-01) | 2.5.0 (2024-07-10) | yes |

data/docker-compose.yml

@@ -1,5 +1,3 @@
-version: '2'
-
 services:
   kafka:
     container_name: kafka

data/ext/Rakefile

@@ -16,11 +16,11 @@ task :default => :clean do
     require "mini_portile2"
     recipe = MiniPortile.new("librdkafka", Rdkafka::LIBRDKAFKA_VERSION)
 
-    # Use default homebrew openssl if we're on mac and the directory exists
-    # and each of flags is not empty
-    if recipe.host&.include?("darwin") && system("which brew &> /dev/null") && Dir.exist?("#{homebrew_prefix = %x(brew --prefix openssl).strip}")
-      ENV["CPPFLAGS"] = "-I#{homebrew_prefix}/include" unless ENV["CPPFLAGS"]
-      ENV["LDFLAGS"] = "-L#{homebrew_prefix}/lib" unless ENV["LDFLAGS"]
+    # Use default homebrew openssl if we're on mac and the directory exists, is not using nix-prepared libraries
+    # and each of flags is not already set
+    if recipe.host&.include?("darwin") && system("which brew &> /dev/null") && Dir.exist?("#{homebrew_prefix = %x(brew --prefix openssl).strip}") && ENV.key?("NIX_LDFLAGS")
+      ENV["CPPFLAGS"] = "-I#{homebrew_prefix}/include" unless ENV.key?("CPPFLAGS")
+      ENV["LDFLAGS"] = "-L#{homebrew_prefix}/lib" unless ENV.key?("LDFLAGS")
     end
 
     releases = File.expand_path(File.join(File.dirname(__FILE__), '../dist'))

data/lib/rdkafka/consumer.rb

@@ -619,87 +619,23 @@ module Rdkafka
       end
     end
 
-    # Poll for new messages and yield them in batches that may contain
-    # messages from more than one partition.
-    #
-    # Rather than yield each message immediately as soon as it is received,
-    # each_batch will attempt to wait for as long as `timeout_ms` in order
-    # to create a batch of up to but no more than `max_items` in size.
-    #
-    # Said differently, if more than `max_items` are available within
-    # `timeout_ms`, then `each_batch` will yield early with `max_items` in the
-    # array, but if `timeout_ms` passes by with fewer messages arriving, it
-    # will yield an array of fewer messages, quite possibly zero.
-    #
-    # In order to prevent wrongly auto committing many messages at once across
-    # possibly many partitions, callers must explicitly indicate which messages
-    # have been successfully processed as some consumed messages may not have
-    # been yielded yet. To do this, the caller should set
-    # `enable.auto.offset.store` to false and pass processed messages to
-    # {store_offset}. It is also possible, though more complex, to set
-    # 'enable.auto.commit' to false and then pass a manually assembled
-    # TopicPartitionList to {commit}.
-    #
-    # As with `each`, iteration will end when the consumer is closed.
-    #
-    # Exception behavior is more complicated than with `each`, in that if
-    # :yield_on_error is true, and an exception is raised during the
-    # poll, and messages have already been received, they will be yielded to
-    # the caller before the exception is allowed to propagate.
-    #
-    # If you are setting either auto.commit or auto.offset.store to false in
-    # the consumer configuration, then you should let yield_on_error keep its
-    # default value of false because you are guaranteed to see these messages
-    # again. However, if both auto.commit and auto.offset.store are set to
-    # true, you should set yield_on_error to true so you can process messages
-    # that you may or may not see again.
-    #
-    # @param max_items [Integer] Maximum size of the yielded array of messages
-    # @param bytes_threshold [Integer] Threshold number of total message bytes in the yielded array of messages
-    # @param timeout_ms [Integer] max time to wait for up to max_items
-    #
-    # @yieldparam messages [Array] An array of received Message
-    # @yieldparam pending_exception [Exception] normally nil, or an exception
-    #
-    # @yield [messages, pending_exception]
-    # which will be propagated after processing of the partial batch is complete.
-    #
-    # @return [nil]
-    #
-    # @raise [RdkafkaError] When polling fails
+    # Deprecated. Please read the error message for more details.
     def each_batch(max_items: 100, bytes_threshold: Float::INFINITY, timeout_ms: 250, yield_on_error: false, &block)
-      closed_consumer_check(__method__)
-      slice = []
-      bytes = 0
-      end_time = monotonic_now + timeout_ms / 1000.0
-      loop do
-        break if closed?
-        max_wait = end_time - monotonic_now
-        max_wait_ms = if max_wait <= 0
-                        0  # should not block, but may retrieve a message
-                      else
-                        (max_wait * 1000).floor
-                      end
-        message = nil
-        begin
-          message = poll max_wait_ms
-        rescue Rdkafka::RdkafkaError => error
-          raise unless yield_on_error
-          raise if slice.empty?
-          yield slice.dup, error
-          raise
-        end
-        if message
-          slice << message
-          bytes += message.payload.bytesize if message.payload
-        end
-        if slice.size == max_items || bytes >= bytes_threshold || monotonic_now >= end_time - 0.001
-          yield slice.dup, nil
-          slice.clear
-          bytes = 0
-          end_time = monotonic_now + timeout_ms / 1000.0
-        end
-      end
+      raise NotImplementedError, <<~ERROR
+        `each_batch` has been removed due to data consistency concerns.
+
+        This method was removed because it did not properly handle partition reassignments,
+        which could lead to processing messages from partitions that were no longer owned
+        by this consumer, resulting in duplicate message processing and data inconsistencies.
+
+        Recommended alternatives:
+
+        1. Implement your own batching logic using rebalance callbacks to properly handle
+           partition revocations and ensure message processing correctness.
+
+        2. Use a high-level batching library that supports proper partition reassignment
+           handling out of the box (such as the Karafka framework).
+      ERROR
     end
 
     # Returns pointer to the consumer group metadata. It is used only in the context of

data/lib/rdkafka/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Rdkafka
-  VERSION = "0.18.1"
-  LIBRDKAFKA_VERSION = "2.6.1"
-  LIBRDKAFKA_SOURCE_SHA256 = "0ddf205ad8d36af0bc72a2fec20639ea02e1d583e353163bf7f4683d949e901b"
+  VERSION = "0.19.0"
+  LIBRDKAFKA_VERSION = "2.8.0"
+  LIBRDKAFKA_SOURCE_SHA256 = "5bd1c46f63265f31c6bfcedcde78703f77d28238eadf23821c2b43fc30be3e25"
 end

data/lib/rdkafka.rb

@@ -49,3 +49,5 @@ require "rdkafka/producer/delivery_report"
 # Main Rdkafka namespace of this gem
 module Rdkafka
 end
+
+Rdkafka::Bindings.rd_kafka_global_init

data/spec/rdkafka/consumer_spec.rb

@@ -948,236 +948,10 @@ describe Rdkafka::Consumer do
   end
 
   describe "#each_batch" do
-    let(:message_payload) { 'a' * 10 }
-
-    before do
-      @topic = SecureRandom.base64(10).tr('+=/', '')
-    end
-
-    after do
-      @topic = nil
-    end
-
-    def topic_name
-      @topic
-    end
-
-    def produce_n(n)
-      handles = []
-      n.times do |i|
-        handles << producer.produce(
-          topic:     topic_name,
-          payload:   i % 10 == 0 ? nil : Time.new.to_f.to_s,
-          key:       i.to_s,
-          partition: 0
-        )
-      end
-      handles.each(&:wait)
-    end
-
-    def new_message
-      instance_double("Rdkafka::Consumer::Message").tap do |message|
-        allow(message).to receive(:payload).and_return(message_payload)
-      end
-    end
-
-    it "retrieves messages produced into a topic" do
-      # This is the only each_batch test that actually produces real messages
-      # into a topic in the real kafka of the container.
-      #
-      # The other tests stub 'poll' which makes them faster and more reliable,
-      # but it makes sense to keep a single test with a fully integrated flow.
-      # This will help to catch breaking changes in the behavior of 'poll',
-      # libdrkafka, or Kafka.
-      #
-      # This is, in effect, an integration test and the subsequent specs are
-      # unit tests.
-      admin = rdkafka_config.admin
-      create_topic_handle = admin.create_topic(topic_name, 1, 1)
-      create_topic_handle.wait(max_wait_timeout: 15.0)
-      consumer.subscribe(topic_name)
-      produce_n 42
-      all_yields = []
-      consumer.each_batch(max_items: 10) do |batch|
-        all_yields << batch
-        break if all_yields.flatten.size >= 42
-      end
-      expect(all_yields.flatten.first).to be_a Rdkafka::Consumer::Message
-      expect(all_yields.flatten.size).to eq 42
-      expect(all_yields.size).to be > 4
-      expect(all_yields.flatten.map(&:key)).to eq (0..41).map { |x| x.to_s }
-      admin.close
-    end
-
-    it "should batch poll results and yield arrays of messages" do
-      consumer.subscribe(topic_name)
-      all_yields = []
-      expect(consumer)
-        .to receive(:poll)
-        .exactly(10).times
-        .and_return(new_message)
-      consumer.each_batch(max_items: 10) do |batch|
-        all_yields << batch
-        break if all_yields.flatten.size >= 10
-      end
-      expect(all_yields.first).to be_instance_of(Array)
-      expect(all_yields.flatten.size).to eq 10
-      non_empty_yields = all_yields.reject { |batch| batch.empty? }
-      expect(non_empty_yields.size).to be < 10
-    end
-
-    it "should yield a partial batch if the timeout is hit with some messages" do
-      consumer.subscribe(topic_name)
-      poll_count = 0
-      expect(consumer)
-        .to receive(:poll)
-        .at_least(3).times do
-        poll_count = poll_count + 1
-        if poll_count > 2
-          sleep 0.1
-          nil
-        else
-          new_message
-        end
-      end
-      all_yields = []
-      consumer.each_batch(max_items: 10) do |batch|
-        all_yields << batch
-        break if all_yields.flatten.size >= 2
-      end
-      expect(all_yields.flatten.size).to eq 2
-    end
-
-    it "should yield [] if nothing is received before the timeout" do
-      admin = rdkafka_config.admin
-      create_topic_handle = admin.create_topic(topic_name, 1, 1)
-      create_topic_handle.wait(max_wait_timeout: 15.0)
-      consumer.subscribe(topic_name)
-      consumer.each_batch do |batch|
-        expect(batch).to eq([])
-        break
-      end
-      admin.close
-    end
-
-    it "should yield batchs of max_items in size if messages are already fetched" do
-      yielded_batches = []
-      expect(consumer)
-        .to receive(:poll)
-        .with(anything)
-        .exactly(20).times
-        .and_return(new_message)
-
-      consumer.each_batch(max_items: 10, timeout_ms: 500) do |batch|
-        yielded_batches << batch
-        break if yielded_batches.flatten.size >= 20
-        break if yielded_batches.size >= 20 # so failure doesn't hang
-      end
-      expect(yielded_batches.size).to eq 2
-      expect(yielded_batches.map(&:size)).to eq 2.times.map { 10 }
-    end
-
-    it "should yield batchs as soon as bytes_threshold is hit" do
-      yielded_batches = []
-      expect(consumer)
-        .to receive(:poll)
-        .with(anything)
-        .exactly(20).times
-        .and_return(new_message)
-
-      consumer.each_batch(bytes_threshold: message_payload.size * 4, timeout_ms: 500) do |batch|
-        yielded_batches << batch
-        break if yielded_batches.flatten.size >= 20
-        break if yielded_batches.size >= 20 # so failure doesn't hang
-      end
-      expect(yielded_batches.size).to eq 5
-      expect(yielded_batches.map(&:size)).to eq 5.times.map { 4 }
-    end
-
-    context "error raised from poll and yield_on_error is true" do
-      it "should yield buffered exceptions on rebalance, then break" do
-        config = rdkafka_consumer_config(
-          {
-            :"enable.auto.commit" => false,
-            :"enable.auto.offset.store" => false
-          }
-        )
-        consumer = config.consumer
-        consumer.subscribe(topic_name)
-        batches_yielded = []
-        exceptions_yielded = []
-        each_batch_iterations = 0
-        poll_count = 0
-        expect(consumer)
-          .to receive(:poll)
-          .with(anything)
-          .exactly(3).times
-          .and_wrap_original do |method, *args|
-              poll_count = poll_count + 1
-              if poll_count == 3
-                raise Rdkafka::RdkafkaError.new(27,
-                    "partitions ... too ... heavy ... must ... rebalance")
-              else
-                new_message
-              end
-          end
-        expect {
-          consumer.each_batch(max_items: 30, yield_on_error: true) do |batch, pending_error|
-            batches_yielded << batch
-            exceptions_yielded << pending_error
-            each_batch_iterations = each_batch_iterations + 1
-          end
-        }.to raise_error(Rdkafka::RdkafkaError)
-        expect(poll_count).to eq 3
-        expect(each_batch_iterations).to eq 1
-        expect(batches_yielded.size).to eq 1
-        expect(batches_yielded.first.size).to eq 2
-        expect(exceptions_yielded.flatten.size).to eq 1
-        expect(exceptions_yielded.flatten.first).to be_instance_of(Rdkafka::RdkafkaError)
-        consumer.close
-      end
-    end
-
-    context "error raised from poll and yield_on_error is false" do
-      it "should yield buffered exceptions on rebalance, then break" do
-        config = rdkafka_consumer_config(
-          {
-            :"enable.auto.commit" => false,
-            :"enable.auto.offset.store" => false
-          }
-        )
-        consumer = config.consumer
-        consumer.subscribe(topic_name)
-        batches_yielded = []
-        exceptions_yielded = []
-        each_batch_iterations = 0
-        poll_count = 0
-        expect(consumer)
-          .to receive(:poll)
-          .with(anything)
-          .exactly(3).times
-          .and_wrap_original do |method, *args|
-              poll_count = poll_count + 1
-              if poll_count == 3
-                raise Rdkafka::RdkafkaError.new(27,
-                    "partitions ... too ... heavy ... must ... rebalance")
-              else
-                new_message
-              end
-          end
-        expect {
-          consumer.each_batch(max_items: 30, yield_on_error: false) do |batch, pending_error|
-            batches_yielded << batch
-            exceptions_yielded << pending_error
-            each_batch_iterations = each_batch_iterations + 1
-          end
-        }.to raise_error(Rdkafka::RdkafkaError)
-        expect(poll_count).to eq 3
-        expect(each_batch_iterations).to eq 0
-        expect(batches_yielded.size).to eq 0
-        expect(exceptions_yielded.size).to eq 0
-        consumer.close
-      end
+    it 'expect to raise an error' do
+      expect do
+        consumer.each_batch {}
+      end.to raise_error(NotImplementedError)
     end
   end
 
@@ -1344,7 +1118,6 @@ describe Rdkafka::Consumer do
     {
         :subscribe               => [ nil ],
         :unsubscribe             => nil,
-        :each_batch              => nil,
         :pause                   => [ nil ],
         :resume                  => [ nil ],
         :subscription            => nil,

data.tar.gz.sig

@@ -1,3 +1,2 @@
-E��~����4G�ڂ��m9:b�%�Ȉc�>��S��u��(?S��;8`���cΪ�1e�.Eh�s�a[�s�Q�RW!���D0�{�g�Q;��'���k.J�x�2�uޥ�B�uA��W���\w�"Wo�s����BU�L���Y=q`*	����[��C-�C*D�i�$h�.�9���b�,�'�`
-бC��2����w3��E��:1���R��q]4s�p��L
-l�ᔕ�J�p�W�Zo#��Yߢ����SAXCu�5��o����k�ua�߮�r�}F��leg�h�S.�oV*.���;'k��^�c7�z8�WjՊ!~�
+<����T��HT9wKo�s<��
+��Y@�J�w�vr���|���%W��6^[�G����q�3�D���ڟ0�:d6����#�ă�C*�Ƌ{����Lw�~��A�m�;6Ey��$0�D��@����`�-5@�	�V�%'��׏
󸤆L��'$��s����X|�!b*�[�,�����6�$Zli�Ek��¹g�}M��fi�0:���?����UЖJ��ki>i�$u*�����R&�܏�Q�P"š՚|$�cyJ��U"Ф�]QB�L4��|�������R�vq�ۃW'F�17(��ǻX���@]J�~�F�"��q�Ŝ���OJ�09�D�1Pq��>Q�/��4�d��O��oa/$/C��'h��Lr�(��\

metadata

@@ -1,12 +1,11 @@
 --- !ruby/object:Gem::Specification
 name: karafka-rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.18.1
+  version: 0.19.0
 platform: ruby
 authors:
 - Thijs Cadier
 - Maciej Mensfeld
-autorequire: 
 bindir: bin
 cert_chain:
 - |
@@ -36,7 +35,7 @@ cert_chain:
   i9zWxov0mr44TWegTVeypcWGd/0nxu1+QHVNHJrpqlPBRvwQsUm7fwmRInGpcaB8
   ap8wNYvryYzrzvzUxIVFBVM5PacgkFqRmolCa8I7tdKQN+R1
   -----END CERTIFICATE-----
-date: 2024-12-04 00:00:00.000000000 Z
+date: 2025-01-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -186,7 +185,7 @@ files:
 - README.md
 - Rakefile
 - certs/cert.pem
-- dist/librdkafka-2.6.1.tar.gz
+- dist/librdkafka-2.8.0.tar.gz
 - dist/patches/rdkafka_global_init.patch
 - docker-compose.yml
 - ext/README.md
@@ -261,7 +260,6 @@ files:
 - spec/rdkafka/producer/delivery_report_spec.rb
 - spec/rdkafka/producer_spec.rb
 - spec/spec_helper.rb
-homepage: 
 licenses:
 - MIT
 metadata:
@@ -272,7 +270,6 @@ metadata:
   source_code_uri: https://github.com/karafka/karafka-rdkafka
   documentation_uri: https://karafka.io/docs
   rubygems_mfa_required: 'true'
-post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -287,8 +284,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.22
-signing_key: 
+rubygems_version: 3.6.2
 specification_version: 4
 summary: The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka.
   It wraps the production-ready C client using the ffi gem and targets Kafka 1.0+