karafka-rdkafka 0.13.2 → 0.13.9

data/spec/rdkafka/producer_spec.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
 require "zlib"
 
 describe Rdkafka::Producer do
@@ -559,6 +558,22 @@ describe Rdkafka::Producer do
     end
   end
 
+  context "when not being able to deliver the message" do
+    let(:producer) do
+      rdkafka_producer_config(
+        "bootstrap.servers": "localhost:9093",
+        "message.timeout.ms": 100
+      ).producer
+    end
+
+    it "should contain the error in the response when not deliverable" do
+      handler = producer.produce(topic: 'produce_test_topic', payload: nil)
+      # Wait for the async callbacks and delivery registry to update
+      sleep(2)
+      expect(handler.create_result.error).to be_a(Rdkafka::RdkafkaError)
+    end
+  end
+
   describe '#partition_count' do
     it { expect(producer.partition_count('example_topic')).to eq(1) }
 
@@ -627,4 +642,281 @@ describe Rdkafka::Producer do
       end
     end
   end
+
+  describe '#flush' do
+    it "should return flush when it can flush all outstanding messages or when no messages" do
+      producer.produce(
+        topic:     "produce_test_topic",
+        payload:   "payload headers",
+        key:       "key headers",
+        headers:   {}
+      )
+
+      expect(producer.flush(5_000)).to eq(true)
+    end
+
+    context 'when it cannot flush due to a timeout' do
+      let(:producer) do
+        rdkafka_producer_config(
+          "bootstrap.servers": "localhost:9093",
+          "message.timeout.ms": 2_000
+        ).producer
+      end
+
+      after do
+        # Allow rdkafka to evict message preventing memory-leak
+        sleep(2)
+      end
+
+      it "should return false on flush when cannot deliver and beyond timeout" do
+        producer.produce(
+          topic:     "produce_test_topic",
+          payload:   "payload headers",
+          key:       "key headers",
+          headers:   {}
+        )
+
+        expect(producer.flush(1_000)).to eq(false)
+      end
+    end
+
+    context 'when there is a different error' do
+      before { allow(Rdkafka::Bindings).to receive(:rd_kafka_flush).and_return(-199) }
+
+      it 'should raise it' do
+        expect { producer.flush }.to raise_error(Rdkafka::RdkafkaError)
+      end
+    end
+  end
+
+  describe '#purge' do
+    context 'when no outgoing messages' do
+      it { expect(producer.purge).to eq(true) }
+    end
+
+    context 'when librdkafka purge returns an error' do
+      before { expect(Rdkafka::Bindings).to receive(:rd_kafka_purge).and_return(-153) }
+
+      it 'expect to raise an error' do
+        expect { producer.purge }.to raise_error(Rdkafka::RdkafkaError, /retry/)
+      end
+    end
+
+    context 'when there are outgoing things in the queue' do
+      let(:producer) do
+        rdkafka_producer_config(
+          "bootstrap.servers": "localhost:9093",
+          "message.timeout.ms": 2_000
+        ).producer
+      end
+
+      it "should should purge and move forward" do
+        producer.produce(
+          topic:     "produce_test_topic",
+          payload:   "payload headers"
+        )
+
+        expect(producer.purge).to eq(true)
+        expect(producer.flush(1_000)).to eq(true)
+      end
+
+      it "should materialize the delivery handles" do
+        handle = producer.produce(
+          topic:     "produce_test_topic",
+          payload:   "payload headers"
+        )
+
+        expect(producer.purge).to eq(true)
+
+        expect { handle.wait }.to raise_error(Rdkafka::RdkafkaError, /purge_queue/)
+      end
+
+      context "when using delivery_callback" do
+        let(:delivery_reports) { [] }
+
+        let(:delivery_callback) do
+          ->(delivery_report) { delivery_reports << delivery_report }
+        end
+
+        before { producer.delivery_callback = delivery_callback }
+
+        it "should run the callback" do
+          handle = producer.produce(
+            topic:     "produce_test_topic",
+            payload:   "payload headers"
+          )
+
+          expect(producer.purge).to eq(true)
+          # queue purge
+          expect(delivery_reports[0].error).to eq(-152)
+        end
+      end
+    end
+  end
+
+  context 'when working with transactions' do
+    let(:producer) do
+      rdkafka_producer_config(
+        'transactional.id': SecureRandom.uuid,
+        'transaction.timeout.ms': 5_000
+      ).producer
+    end
+
+    it 'expect not to allow to produce without transaction init' do
+      expect do
+        producer.produce(topic: 'produce_test_topic', payload: 'data')
+      end.to raise_error(Rdkafka::RdkafkaError, /Erroneous state \(state\)/)
+    end
+
+    it 'expect to raise error when transactions are initialized but producing not in one' do
+      producer.init_transactions
+
+      expect do
+        producer.produce(topic: 'produce_test_topic', payload: 'data')
+      end.to raise_error(Rdkafka::RdkafkaError, /Erroneous state \(state\)/)
+    end
+
+    it 'expect to allow to produce within a transaction, finalize and ship data' do
+      producer.init_transactions
+      producer.begin_transaction
+      handle1 = producer.produce(topic: 'produce_test_topic', payload: 'data1', partition: 1)
+      handle2 = producer.produce(topic: 'example_topic', payload: 'data2', partition: 0)
+      producer.commit_transaction
+
+      report1 = handle1.wait(max_wait_timeout: 15)
+      report2 = handle2.wait(max_wait_timeout: 15)
+
+      message1 = wait_for_message(
+        topic: "produce_test_topic",
+        delivery_report: report1,
+        consumer: consumer
+      )
+
+      expect(message1.partition).to eq 1
+      expect(message1.payload).to eq "data1"
+      expect(message1.timestamp).to be_within(10).of(Time.now)
+
+      message2 = wait_for_message(
+        topic: "example_topic",
+        delivery_report: report2,
+        consumer: consumer
+      )
+
+      expect(message2.partition).to eq 0
+      expect(message2.payload).to eq "data2"
+      expect(message2.timestamp).to be_within(10).of(Time.now)
+    end
+
+    it 'expect not to send data and propagate purge queue error on abort' do
+      producer.init_transactions
+      producer.begin_transaction
+      handle1 = producer.produce(topic: 'produce_test_topic', payload: 'data1', partition: 1)
+      handle2 = producer.produce(topic: 'example_topic', payload: 'data2', partition: 0)
+      producer.abort_transaction
+
+      expect { handle1.wait(max_wait_timeout: 15) }
+        .to raise_error(Rdkafka::RdkafkaError, /Purged in queue \(purge_queue\)/)
+      expect { handle2.wait(max_wait_timeout: 15) }
+        .to raise_error(Rdkafka::RdkafkaError, /Purged in queue \(purge_queue\)/)
+    end
+
+    it 'expect to have non retryable, non abortable and not fatal error on abort' do
+      producer.init_transactions
+      producer.begin_transaction
+      handle = producer.produce(topic: 'produce_test_topic', payload: 'data1', partition: 1)
+      producer.abort_transaction
+
+      response = handle.wait(raise_response_error: false)
+
+      expect(response.error).to be_a(Rdkafka::RdkafkaError)
+      expect(response.error.retryable?).to eq(false)
+      expect(response.error.fatal?).to eq(false)
+      expect(response.error.abortable?).to eq(false)
+    end
+
+    # This may not always crash, depends on load but no other way to check it
+    context 'when timeout is too short and error occurs and we can abort' do
+      let(:producer) do
+        rdkafka_producer_config(
+          'transactional.id': SecureRandom.uuid,
+          'transaction.timeout.ms': 1_000
+        ).producer
+      end
+
+      it 'expect to allow to produce within a transaction, finalize and not ship data' do
+        producer.init_transactions
+        producer.begin_transaction
+
+        sleep(5)
+
+        handle1 = producer.produce(topic: 'produce_test_topic', payload: 'data1', partition: 1)
+        handle2 = producer.produce(topic: 'example_topic', payload: 'data2', partition: 0)
+
+        begin
+          producer.commit_transaction(15_000)
+        rescue Rdkafka::RdkafkaError => e
+          next unless e.abortable?
+
+          begin
+            producer.abort_transaction(15_000)
+          rescue Rdkafka::RdkafkaError => e
+            nil
+          end
+
+          expect { handle1.wait(max_wait_timeout: 15) }.to raise_error(Rdkafka::RdkafkaError)
+          expect { handle2.wait(max_wait_timeout: 15) }.to raise_error(Rdkafka::RdkafkaError)
+        end
+      end
+    end
+
+    context 'fencing against previous active producer with same transactional id' do
+      let(:transactional_id) { SecureRandom.uuid }
+
+      let(:producer1) do
+        rdkafka_producer_config(
+          'transactional.id': transactional_id,
+          'transaction.timeout.ms': 10_000
+        ).producer
+      end
+
+      let(:producer2) do
+        rdkafka_producer_config(
+          'transactional.id': transactional_id,
+          'transaction.timeout.ms': 10_000
+        ).producer
+      end
+
+      after do
+        producer1.close
+        producer2.close
+      end
+
+      it 'expect older producer not to be able to commit when fanced out' do
+        producer1.init_transactions
+        producer1.begin_transaction
+        producer1.produce(topic: 'produce_test_topic', payload: 'data1', partition: 1)
+
+        producer2.init_transactions
+        producer2.begin_transaction
+        producer2.produce(topic: 'produce_test_topic', payload: 'data1', partition: 1)
+
+        expect { producer1.commit_transaction }
+          .to raise_error(Rdkafka::RdkafkaError, /This instance has been fenced/)
+
+        error = false
+
+        begin
+          producer1.commit_transaction
+        rescue Rdkafka::RdkafkaError => e
+          error = e
+        end
+
+        expect(error.fatal?).to eq(true)
+        expect(error.abortable?).to eq(false)
+        expect(error.retryable?).to eq(false)
+
+        expect { producer2.commit_transaction }.not_to raise_error
+      end
+    end
+  end
 end

data/spec/spec_helper.rb

@@ -11,6 +11,7 @@ require "pry"
 require "rspec"
 require "rdkafka"
 require "timeout"
+require 'securerandom'
 
 def rdkafka_base_config
   {
@@ -35,7 +36,7 @@ def rdkafka_consumer_config(config_overrides={})
   # Add consumer specific fields to it
   config[:"auto.offset.reset"] = "earliest"
   config[:"enable.partition.eof"] = false
-  config[:"group.id"] = "ruby-test-#{Random.new.rand(0..1_000_000)}"
+  config[:"group.id"] = "ruby-test-#{SecureRandom.uuid}"
   # Enable debug mode if required
   if ENV["DEBUG_CONSUMER"]
     config[:debug] = "cgrp,topic,fetch"
@@ -106,6 +107,10 @@ def wait_for_unassignment(consumer)
   end
 end
 
+def objects_of_type_count(type)
+  ObjectSpace.each_object(type).count
+end
+
 def notify_listener(listener, &block)
   # 1. subscribe and poll
   consumer.subscribe("consume_test_topic")
@@ -134,6 +139,7 @@ RSpec.configure do |config|
         rake_test_topic:         3,
         watermarks_test_topic:   3,
         partitioner_test_topic: 25,
+        example_topic:           1
     }.each do |topic, partitions|
       create_topic_handle = admin.create_topic(topic.to_s, partitions, 1)
       begin

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.13.2
+  version: 0.13.9
 platform: ruby
 authors:
 - Thijs Cadier
@@ -12,30 +12,30 @@ cert_chain:
   -----BEGIN CERTIFICATE-----
   MIIEcDCCAtigAwIBAgIBATANBgkqhkiG9w0BAQsFADA/MRAwDgYDVQQDDAdjb250
   YWN0MRcwFQYKCZImiZPyLGQBGRYHa2FyYWZrYTESMBAGCgmSJomT8ixkARkWAmlv
-  MB4XDTIyMDgxOTE3MjEzN1oXDTIzMDgxOTE3MjEzN1owPzEQMA4GA1UEAwwHY29u
+  MB4XDTIzMDgyMTA3MjU1NFoXDTI0MDgyMDA3MjU1NFowPzEQMA4GA1UEAwwHY29u
   dGFjdDEXMBUGCgmSJomT8ixkARkWB2thcmFma2ExEjAQBgoJkiaJk/IsZAEZFgJp
-  bzCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAODzeO3L6lxdATzMHKNW
-  jFA/GGunoPuylO/BMzy8RiQHh7VIvysAKs0tHhTx3g2D0STDpF+hcQcPELFikiT2
-  F+1wOHj/SsrK7VKqfA8+gq04hKc5sQoX2Egf9k3V0YJ3eZ6R/koHkQ8A0TVt0w6F
-  ZQckoV4MqnEAx0g/FZN3mnHTlJ3VFLSBqJEIe+S6FZMl92mSv+hTrlUG8VaYxSfN
-  lTCvnKk284F6QZq5XIENLRmcDd/3aPBLnLwNnyMyhB+6gK8cUO+CFlDO5tjo/aBA
-  rUnl++wGG0JooF1ed0v+evOn9KoMBG6rHewcf79qJbVOscbD8qSAmo+sCXtcFryr
-  KRMTB8gNbowJkFRJDEe8tfRy11u1fYzFg/qNO82FJd62rKAw2wN0C29yCeQOPRb1
-  Cw9Y4ZwK9VFNEcV9L+3pHTHn2XfuZHtDaG198VweiF6raFO4yiEYccodH/USP0L5
-  cbcCFtmu/4HDSxL1ByQXO84A0ybJuk3/+aPUSXe9C9U8fwIDAQABo3cwdTAJBgNV
-  HRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQUSlcEakb7gfn/5E2WY6z73BF/
-  iZkwHQYDVR0RBBYwFIESY29udGFjdEBrYXJhZmthLmlvMB0GA1UdEgQWMBSBEmNv
-  bnRhY3RAa2FyYWZrYS5pbzANBgkqhkiG9w0BAQsFAAOCAYEA1aS+E7RXJ1w9g9mJ
-  G0NzFxe64OEuENosNlvYQCbRKGCXAU1qqelYkBQHseRgRKxLICrnypRo9IEobyHa
-  vDnJ4r7Tsb34dleqQW2zY/obG+cia3Ym2JsegXWF7dDOzCXJ4FN8MFoT2jHlqLLw
-  yrap0YO5zx0GSQ0Dwy8h2n2v2vanMEeCx7iNm3ERgR5WuN5sjzWoz2A/JLEEcK0C
-  EnAGKCWAd1fuG8IemDjT1edsd5FyYR4bIX0m+99oDuFZyPiiIbalmyYiSBBp59Yb
-  Q0P8zeBi4OfwCZNcxqz0KONmw9JLNv6DgyEAH5xe/4JzhMEgvIRiPj0pHfA7oqQF
-  KUNqvD1KlxbEC+bZfE5IZhnqYLdld/Ksqd22FI1RBhiS1Ejfsj99LVIm9cBuZEY2
-  Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
-  MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
+  bzCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAOuZpyQKEwsTG9plLat7
+  8bUaNuNBEnouTsNMr6X+XTgvyrAxTuocdsyP1sNCjdS1B8RiiDH1/Nt9qpvlBWon
+  sdJ1SYhaWNVfqiYStTDnCx3PRMmHRdD4KqUWKpN6VpZ1O/Zu+9Mw0COmvXgZuuO9
+  wMSJkXRo6dTCfMedLAIxjMeBIxtoLR2e6Jm6MR8+8WYYVWrO9kSOOt5eKQLBY7aK
+  b/Dc40EcJKPg3Z30Pia1M9ZyRlb6SOj6SKpHRqc7vbVQxjEw6Jjal1lZ49m3YZMd
+  ArMAs9lQZNdSw5/UX6HWWURLowg6k10RnhTUtYyzO9BFev0JFJftHnmuk8vtb+SD
+  5VPmjFXg2VOcw0B7FtG75Vackk8QKfgVe3nSPhVpew2CSPlbJzH80wChbr19+e3+
+  YGr1tOiaJrL6c+PNmb0F31NXMKpj/r+n15HwlTMRxQrzFcgjBlxf2XFGnPQXHhBm
+  kp1OFnEq4GG9sON4glRldkwzi/f/fGcZmo5fm3d+0ZdNgwIDAQABo3cwdTAJBgNV
+  HRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQUPVH5+dLA80A1kJ2Uz5iGwfOa
+  1+swHQYDVR0RBBYwFIESY29udGFjdEBrYXJhZmthLmlvMB0GA1UdEgQWMBSBEmNv
+  bnRhY3RAa2FyYWZrYS5pbzANBgkqhkiG9w0BAQsFAAOCAYEAnpa0jcN7JzREHMTQ
+  bfZ+xcvlrzuROMY6A3zIZmQgbnoZZNuX4cMRrT1p1HuwXpxdpHPw7dDjYqWw3+1h
+  3mXLeMuk7amjQpYoSWU/OIZMhIsARra22UN8qkkUlUj3AwTaChVKN/bPJOM2DzfU
+  kz9vUgLeYYFfQbZqeI6SsM7ltilRV4W8D9yNUQQvOxCFxtLOetJ00fC/E7zMUzbK
+  IBwYFQYsbI6XQzgAIPW6nGSYKgRhkfpmquXSNKZRIQ4V6bFrufa+DzD0bt2ZA3ah
+  fMmJguyb5L2Gf1zpDXzFSPMG7YQFLzwYz1zZZvOU7/UCpQsHpID/YxqDp4+Dgb+Y
+  qma0whX8UG/gXFV2pYWpYOfpatvahwi+A1TwPQsuZwkkhi1OyF1At3RY+hjSXyav
+  AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
+  msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2023-06-28 00:00:00.000000000 Z
+date: 2023-11-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -165,7 +165,7 @@ dependencies:
         version: '0'
 description: Modern Kafka client library for Ruby based on librdkafka
 email:
-- thijs@appsignal.com
+- contact@karafka.io
 executables: []
 extensions:
 - ext/Rakefile
@@ -174,15 +174,16 @@ files:
 - ".github/workflows/ci.yml"
 - ".gitignore"
 - ".rspec"
+- ".ruby-gemset"
+- ".ruby-version"
 - ".yardopts"
 - CHANGELOG.md
 - Gemfile
 - Guardfile
-- LICENSE
+- MIT-LICENSE
 - README.md
 - Rakefile
 - certs/cert_chain.pem
-- certs/karafka-pro.pem
 - docker-compose.yml
 - ext/README.md
 - ext/Rakefile
@@ -205,12 +206,14 @@ files:
 - lib/rdkafka/consumer/partition.rb
 - lib/rdkafka/consumer/topic_partition_list.rb
 - lib/rdkafka/error.rb
+- lib/rdkafka/helpers/time.rb
 - lib/rdkafka/metadata.rb
 - lib/rdkafka/native_kafka.rb
 - lib/rdkafka/producer.rb
 - lib/rdkafka/producer/delivery_handle.rb
 - lib/rdkafka/producer/delivery_report.rb
 - lib/rdkafka/version.rb
+- renovate.json
 - spec/rdkafka/abstract_handle_spec.rb
 - spec/rdkafka/admin/create_topic_handle_spec.rb
 - spec/rdkafka/admin/create_topic_report_spec.rb
@@ -251,14 +254,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.6'
+      version: '2.7'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.4
+rubygems_version: 3.4.19
 signing_key: 
 specification_version: 4
 summary: The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka.

data/certs/karafka-pro.pem

@@ -1,11 +0,0 @@
------BEGIN RSA PUBLIC KEY-----
-MIIBigKCAYEApcd6ybskiNs9WUvBGVUE8GdWDehjZ9TyjSj/fDl/UcMYqY0R5YX9
-tnYxEwZZRMdVltKWxr88Qmshh1IQz6CpJVbcfYjt/158pSGPm+AUua6tkLqIvZDM
-ocFOMafmroI+BMuL+Zu5QH7HC2tkT16jclGYfMQkJjXVUQTk2UZr+94+8RlUz/CH
-Y6hPA7xPgIyPfyPCxz1VWzAwXwT++NCJQPBr5MqT84LNSEzUSlR9pFNShf3UCUT+
-8LWOvjFSNGmMMSsbo2T7/+dz9/FM02YG00EO0x04qteggwcaEYLFrigDN6/fM0ih
-BXZILnMUqC/qrfW2YFg4ZqKZJuxaALqqkPxrkBDYqoqcAloqn36jBSke6tc/2I/J
-2Afq3r53UoAbUH7h5I/L8YeaiA4MYjAuq724lHlrOmIr4D6yjYC0a1LGlPjLk869
-2nsVXNgomhVb071E6amR+rJJnfvkdZgCmEBFnqnBV5A1u4qgNsa2rVcD+gJRvb2T
-aQtjlQWKPx5xAgMBAAE=
------END RSA PUBLIC KEY-----