karafka-rdkafka 0.13.2 → 0.13.9

data/lib/rdkafka/producer.rb

@@ -1,10 +1,10 @@
 # frozen_string_literal: true
 
-require "objspace"
-
 module Rdkafka
   # A producer for Kafka messages. To create a producer set up a {Config} and call {Config#producer producer} on that.
   class Producer
+    include Helpers::Time
+
     # Cache partitions count for 30 seconds
     PARTITIONS_COUNT_TTL = 30
 
@@ -63,6 +63,47 @@ module Rdkafka
       @delivery_callback_arity = arity(callback)
     end
 
+    # Init transactions
+    # Run once per producer
+    def init_transactions
+      closed_producer_check(__method__)
+
+      @native_kafka.with_inner do |inner|
+        response_ptr = Rdkafka::Bindings.rd_kafka_init_transactions(inner, -1)
+
+        Rdkafka::RdkafkaError.validate!(response_ptr) || true
+      end
+    end
+
+    def begin_transaction
+      closed_producer_check(__method__)
+
+      @native_kafka.with_inner do |inner|
+        response_ptr = Rdkafka::Bindings.rd_kafka_begin_transaction(inner)
+
+        Rdkafka::RdkafkaError.validate!(response_ptr)
+      end
+    end
+
+    def commit_transaction(timeout_ms = -1)
+      closed_producer_check(__method__)
+
+      @native_kafka.with_inner do |inner|
+        response_ptr = Rdkafka::Bindings.rd_kafka_commit_transaction(inner, timeout_ms)
+
+        Rdkafka::RdkafkaError.validate!(response_ptr)
+      end
+    end
+
+    def abort_transaction(timeout_ms = -1)
+      closed_producer_check(__method__)
+
+      @native_kafka.with_inner do |inner|
+        response_ptr = Rdkafka::Bindings.rd_kafka_abort_transaction(inner, timeout_ms)
+        Rdkafka::RdkafkaError.validate!(response_ptr)
+      end
+    end
+
     # Close this producer and wait for the internal poll queue to empty.
     def close
       return if closed?
@@ -79,27 +120,63 @@ module Rdkafka
     # in seconds. Call this before closing a producer to ensure delivery of all messages.
     #
     # @param timeout_ms [Integer] how long should we wait for flush of all messages
+    # @return [Boolean] true if no more data and all was flushed, false in case there are still
+    #   outgoing messages after the timeout
+    #
+    # @note We raise an exception for other errors because based on the librdkafka docs, there
+    #   should be no other errors.
+    #
+    # @note For `timed_out` we do not raise an error to keep it backwards compatible
     def flush(timeout_ms=5_000)
       closed_producer_check(__method__)
 
+      error = @native_kafka.with_inner do |inner|
+        response = Rdkafka::Bindings.rd_kafka_flush(inner, timeout_ms)
+        Rdkafka::RdkafkaError.build(response)
+      end
+
+      # Early skip not to build the error message
+      return true unless error
+      return false if error.code == :timed_out
+
+      raise(error)
+    end
+
+    # Purges the outgoing queue and releases all resources.
+    #
+    # Useful when closing the producer with outgoing messages to unstable clusters or when for
+    # any other reasons waiting cannot go on anymore. This purges both the queue and all the
+    # inflight requests + updates the delivery handles statuses so they can be materialized into
+    # `purge_queue` errors.
+    def purge
+      closed_producer_check(__method__)
+
       @native_kafka.with_inner do |inner|
-        Rdkafka::Bindings.rd_kafka_flush(inner, timeout_ms)
+        response = Bindings.rd_kafka_purge(
+          inner,
+          Bindings::RD_KAFKA_PURGE_F_QUEUE | Bindings::RD_KAFKA_PURGE_F_INFLIGHT
+        )
+
+        Rdkafka::RdkafkaError.validate!(response)
       end
+
+      # Wait for the purge to affect everything
+      sleep(0.001) until flush(100)
+
+      true
     end
 
     # Partition count for a given topic.
-    # NOTE: If 'allow.auto.create.topics' is set to true in the broker, the topic will be auto-created after returning nil.
     #
     # @param topic [String] The topic name.
+    # @return [Integer] partition count for a given topic
     #
-    # @return partition count [Integer,nil]
-    #
-    # We cache the partition count for a given topic for given time
-    # This prevents us in case someone uses `partition_key` from querying for the count with
-    # each message. Instead we query once every 30 seconds at most
+    # @note If 'allow.auto.create.topics' is set to true in the broker, the topic will be
+    #   auto-created after returning nil.
     #
-    # @param topic [String] topic name
-    # @return [Integer] partition count for a given topic
+    # @note We cache the partition count for a given topic for given time.
+    #   This prevents us in case someone uses `partition_key` from querying for the count with
+    #   each message. Instead we query once every 30 seconds at most
     def partition_count(topic)
       closed_producer_check(__method__)
 
@@ -208,7 +285,7 @@ module Rdkafka
       # Raise error if the produce call was not successful
       if response != 0
         DeliveryHandle.remove(delivery_handle.to_ptr.address)
-        raise RdkafkaError.new(response)
+        Rdkafka::RdkafkaError.validate!(response)
       end
 
       delivery_handle
@@ -229,11 +306,6 @@ module Rdkafka
 
     private
 
-    def monotonic_now
-      # needed because Time.now can go backwards
-      Process.clock_gettime(Process::CLOCK_MONOTONIC)
-    end
-
     def closed_producer_check(method)
       raise Rdkafka::ClosedProducerError.new(method) if closed?
     end

data/lib/rdkafka/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Rdkafka
-  VERSION = "0.13.2"
-  LIBRDKAFKA_VERSION = "2.1.1"
-  LIBRDKAFKA_SOURCE_SHA256 = "7be1fc37ab10ebdc037d5c5a9b35b48931edafffae054b488faaff99e60e0108"
+  VERSION = "0.13.9"
+  LIBRDKAFKA_VERSION = "2.2.0"
+  LIBRDKAFKA_SOURCE_SHA256 = "af9a820cbecbc64115629471df7c7cecd40403b6c34bfdbb9223152677a47226"
 end

data/lib/rdkafka.rb

@@ -1,7 +1,12 @@
 # frozen_string_literal: true
 
-require "rdkafka/version"
+require "logger"
+require "objspace"
+require "ffi"
+require "json"
 
+require "rdkafka/version"
+require "rdkafka/helpers/time"
 require "rdkafka/abstract_handle"
 require "rdkafka/admin"
 require "rdkafka/admin/create_topic_handle"
@@ -24,3 +29,7 @@ require "rdkafka/native_kafka"
 require "rdkafka/producer"
 require "rdkafka/producer/delivery_handle"
 require "rdkafka/producer/delivery_report"
+
+# Main Rdkafka namespace of this gem
+module Rdkafka
+end

data/renovate.json

@@ -0,0 +1,6 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": [
+    "config:base"
+  ]
+}

data/spec/rdkafka/abstract_handle_spec.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
-
 describe Rdkafka::AbstractHandle do
   let(:response) { 0 }
   let(:result) { -1 }

data/spec/rdkafka/admin/create_topic_handle_spec.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
-
 describe Rdkafka::Admin::CreateTopicHandle do
   let(:response) { 0 }
 
@@ -45,10 +43,12 @@ describe Rdkafka::Admin::CreateTopicHandle do
   describe "#raise_error" do
     let(:pending_handle) { false }
 
-    it "should raise the appropriate error" do
+    before { subject[:response] = -1 }
+
+    it "should raise the appropriate error when there is an error" do
       expect {
         subject.raise_error
-      }.to raise_exception(Rdkafka::RdkafkaError, /Success \(no_error\)/)
+      }.to raise_exception(Rdkafka::RdkafkaError, /Unknown broker error \(unknown\)/)
     end
   end
 end

data/spec/rdkafka/admin/create_topic_report_spec.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
-
 describe Rdkafka::Admin::CreateTopicReport do
   subject { Rdkafka::Admin::CreateTopicReport.new(
       FFI::MemoryPointer.from_string("error string"),

data/spec/rdkafka/admin/delete_topic_handle_spec.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
-
 describe Rdkafka::Admin::DeleteTopicHandle do
   let(:response) { 0 }
 
@@ -45,10 +43,12 @@ describe Rdkafka::Admin::DeleteTopicHandle do
   describe "#raise_error" do
     let(:pending_handle) { false }
 
+    before { subject[:response] = -1 }
+
     it "should raise the appropriate error" do
       expect {
         subject.raise_error
-      }.to raise_exception(Rdkafka::RdkafkaError, /Success \(no_error\)/)
+      }.to raise_exception(Rdkafka::RdkafkaError, /Unknown broker error \(unknown\)/)
     end
   end
 end

data/spec/rdkafka/admin/delete_topic_report_spec.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
-
 describe Rdkafka::Admin::DeleteTopicReport do
   subject { Rdkafka::Admin::DeleteTopicReport.new(
       FFI::MemoryPointer.from_string("error string"),

data/spec/rdkafka/admin_spec.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
 require "ostruct"
 
 describe Rdkafka::Admin do
@@ -33,7 +32,7 @@ describe Rdkafka::Admin do
           }.to raise_exception { |ex|
             expect(ex).to be_a(Rdkafka::RdkafkaError)
             expect(ex.message).to match(/Broker: Invalid topic \(topic_exception\)/)
-            expect(ex.broker_message).to match(/Topic name.*is illegal, it contains a character other than ASCII alphanumerics/)
+            expect(ex.broker_message).to match(/Topic name.*is invalid: .* contains one or more characters other than ASCII alphanumerics, '.', '_' and '-'/)
           }
         end
       end

data/spec/rdkafka/bindings_spec.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
 require 'zlib'
 
 describe Rdkafka::Bindings do

data/spec/rdkafka/callbacks_spec.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
-
 describe Rdkafka::Callbacks do
 
   # The code in the call back functions is 100% covered by other specs.  Due to

data/spec/rdkafka/config_spec.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
-
 describe Rdkafka::Config do
   context "logger" do
     it "should have a default logger" do
@@ -115,6 +113,14 @@ describe Rdkafka::Config do
       consumer.close
     end
 
+    it "should create a consumer with consumer_poll_set set to false" do
+      config = rdkafka_consumer_config
+      config.consumer_poll_set = false
+      consumer = config.consumer
+      expect(consumer).to be_a Rdkafka::Consumer
+      consumer.close
+    end
+
     it "should raise an error when creating a consumer with invalid config" do
       config = Rdkafka::Config.new('invalid.key' => 'value')
       expect {

data/spec/rdkafka/consumer/headers_spec.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
-
 describe Rdkafka::Consumer::Headers do
   let(:headers) do
     { # Note String keys!

data/spec/rdkafka/consumer/message_spec.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
-
 describe Rdkafka::Consumer::Message do
   let(:native_client) { new_native_client }
   let(:native_topic) { new_native_topic(native_client: native_client) }

data/spec/rdkafka/consumer/partition_spec.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
-
 describe Rdkafka::Consumer::Partition do
   let(:offset) { 100 }
   let(:err) { 0 }

data/spec/rdkafka/consumer/topic_partition_list_spec.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
-
 describe Rdkafka::Consumer::TopicPartitionList do
   it "should create a new list and add unassigned topics" do
     list = Rdkafka::Consumer::TopicPartitionList.new

data/spec/rdkafka/consumer_spec.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
 require "ostruct"
 require 'securerandom'
 
@@ -55,6 +54,30 @@ describe Rdkafka::Consumer do
         consumer.subscription
       }.to raise_error(Rdkafka::RdkafkaError)
     end
+
+    context "when using consumer without the poll set" do
+      let(:consumer) do
+        config = rdkafka_consumer_config
+        config.consumer_poll_set = false
+        config.consumer
+      end
+
+      it "should subscribe, unsubscribe and return the subscription" do
+        expect(consumer.subscription).to be_empty
+
+        consumer.subscribe("consume_test_topic")
+
+        expect(consumer.subscription).not_to be_empty
+        expected_subscription = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
+          list.add_topic("consume_test_topic")
+        end
+        expect(consumer.subscription).to eq expected_subscription
+
+        consumer.unsubscribe
+
+        expect(consumer.subscription).to be_empty
+      end
+    end
   end
 
   describe "#pause and #resume" do
@@ -337,8 +360,9 @@ describe Rdkafka::Consumer do
     end
   end
 
-  describe "#commit, #committed and #store_offset" do
-    # Make sure there's a stored offset
+
+  describe "#position, #commit, #committed and #store_offset" do
+    # Make sure there are messages to work with
     let!(:report) do
       producer.produce(
         topic:     "consume_test_topic",
@@ -356,29 +380,33 @@ describe Rdkafka::Consumer do
       )
     end
 
-    it "should only accept a topic partition list in committed" do
-      expect {
-        consumer.committed("list")
-      }.to raise_error TypeError
+    describe "#position" do
+      it "should only accept a topic partition list in position if not nil" do
+        expect {
+          consumer.position("list")
+        }.to raise_error TypeError
+      end
     end
 
-    it "should commit in sync mode" do
-      expect {
-        consumer.commit(nil, true)
-      }.not_to raise_error
-    end
+    describe "#committed" do
+      it "should only accept a topic partition list in commit if not nil" do
+        expect {
+          consumer.commit("list")
+        }.to raise_error TypeError
+      end
 
-    it "should only accept a topic partition list in commit if not nil" do
-      expect {
-        consumer.commit("list")
-      }.to raise_error TypeError
+      it "should commit in sync mode" do
+        expect {
+          consumer.commit(nil, true)
+        }.not_to raise_error
+      end
     end
 
     context "with a committed consumer" do
       before :all do
         # Make sure there are some messages.
         handles = []
-        producer = rdkafka_producer_config.producer
+        producer = rdkafka_config.producer
         10.times do
           (0..2).each do |i|
             handles << producer.produce(
@@ -422,31 +450,33 @@ describe Rdkafka::Consumer do
         }.to raise_error(Rdkafka::RdkafkaError)
       end
 
-      it "should fetch the committed offsets for the current assignment" do
-        partitions = consumer.committed.to_h["consume_test_topic"]
-        expect(partitions).not_to be_nil
-        expect(partitions[0].offset).to eq 1
-      end
+      describe "#committed" do
+        it "should fetch the committed offsets for the current assignment" do
+          partitions = consumer.committed.to_h["consume_test_topic"]
+          expect(partitions).not_to be_nil
+          expect(partitions[0].offset).to eq 1
+        end
 
-      it "should fetch the committed offsets for a specified topic partition list" do
-        list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
-          list.add_topic("consume_test_topic", [0, 1, 2])
+        it "should fetch the committed offsets for a specified topic partition list" do
+          list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
+            list.add_topic("consume_test_topic", [0, 1, 2])
+          end
+          partitions = consumer.committed(list).to_h["consume_test_topic"]
+          expect(partitions).not_to be_nil
+          expect(partitions[0].offset).to eq 1
+          expect(partitions[1].offset).to eq 1
+          expect(partitions[2].offset).to eq 1
         end
-        partitions = consumer.committed(list).to_h["consume_test_topic"]
-        expect(partitions).not_to be_nil
-        expect(partitions[0].offset).to eq 1
-        expect(partitions[1].offset).to eq 1
-        expect(partitions[2].offset).to eq 1
-      end
 
-      it "should raise an error when getting committed fails" do
-        expect(Rdkafka::Bindings).to receive(:rd_kafka_committed).and_return(20)
-        list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
-          list.add_topic("consume_test_topic", [0, 1, 2])
+        it "should raise an error when getting committed fails" do
+          expect(Rdkafka::Bindings).to receive(:rd_kafka_committed).and_return(20)
+          list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
+            list.add_topic("consume_test_topic", [0, 1, 2])
+          end
+          expect {
+            consumer.committed(list)
+          }.to raise_error Rdkafka::RdkafkaError
         end
-        expect {
-          consumer.committed(list)
-        }.to raise_error Rdkafka::RdkafkaError
       end
 
       describe "#store_offset" do
@@ -467,6 +497,8 @@ describe Rdkafka::Consumer do
           @new_consumer.store_offset(message)
           @new_consumer.commit
 
+          # TODO use position here, should be at offset
+
           list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
             list.add_topic("consume_test_topic", [0, 1, 2])
           end
@@ -481,6 +513,35 @@ describe Rdkafka::Consumer do
             @new_consumer.store_offset(message)
           }.to raise_error Rdkafka::RdkafkaError
         end
+
+        describe "#position" do
+          it "should fetch the positions for the current assignment" do
+            consumer.store_offset(message)
+
+            partitions = consumer.position.to_h["consume_test_topic"]
+            expect(partitions).not_to be_nil
+            expect(partitions[0].offset).to eq message.offset + 1
+          end
+
+          it "should fetch the positions for a specified assignment" do
+            consumer.store_offset(message)
+
+            list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
+              list.add_topic_and_partitions_with_offsets("consume_test_topic", 0 => nil, 1 => nil, 2 => nil)
+            end
+            partitions = consumer.position(list).to_h["consume_test_topic"]
+            expect(partitions).not_to be_nil
+            expect(partitions[0].offset).to eq message.offset + 1
+          end
+
+          it "should raise an error when getting the position fails" do
+            expect(Rdkafka::Bindings).to receive(:rd_kafka_position).and_return(20)
+
+            expect {
+              consumer.position
+            }.to raise_error(Rdkafka::RdkafkaError)
+          end
+        end
       end
     end
   end
@@ -1039,6 +1100,29 @@ describe Rdkafka::Consumer do
     end
   end
 
+  # Only relevant in case of a consumer with separate queues
+  describe '#events_poll' do
+    let(:stats) { [] }
+
+    before { Rdkafka::Config.statistics_callback = ->(published) { stats << published } }
+
+    after { Rdkafka::Config.statistics_callback = nil }
+
+    let(:consumer) do
+      config = rdkafka_consumer_config('statistics.interval.ms': 100)
+      config.consumer_poll_set = false
+      config.consumer
+    end
+
+    it "expect to run events_poll, operate and propagate stats on events_poll and not poll" do
+      consumer.subscribe("consume_test_topic")
+      consumer.poll(1_000)
+      expect(stats).to be_empty
+      consumer.events_poll(-1)
+      expect(stats).not_to be_empty
+    end
+  end
+
   describe "a rebalance listener" do
     let(:consumer) do
       config = rdkafka_consumer_config

data/spec/rdkafka/error_spec.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
-
 describe Rdkafka::RdkafkaError do
   it "should raise a type error for a nil response" do
     expect {

data/spec/rdkafka/metadata_spec.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
 require "securerandom"
 
 describe Rdkafka::Metadata do
@@ -31,7 +30,7 @@ describe Rdkafka::Metadata do
       it "#brokers returns our single broker" do
         expect(subject.brokers.length).to eq(1)
         expect(subject.brokers[0][:broker_id]).to eq(1)
-        expect(subject.brokers[0][:broker_name]).to eq("localhost")
+        expect(subject.brokers[0][:broker_name]).to eq("127.0.0.1")
         expect(subject.brokers[0][:broker_port]).to eq(9092)
       end
 
@@ -54,7 +53,7 @@ describe Rdkafka::Metadata do
     it "#brokers returns our single broker" do
       expect(subject.brokers.length).to eq(1)
       expect(subject.brokers[0][:broker_id]).to eq(1)
-      expect(subject.brokers[0][:broker_name]).to eq("localhost")
+      expect(subject.brokers[0][:broker_name]).to eq("127.0.0.1")
       expect(subject.brokers[0][:broker_port]).to eq(9092)
     end
 

data/spec/rdkafka/native_kafka_spec.rb

@@ -1,14 +1,13 @@
 # frozen_string_literal: true
 
-require "spec_helper"
-
 describe Rdkafka::NativeKafka do
   let(:config) { rdkafka_producer_config }
   let(:native) { config.send(:native_kafka, config.send(:native_config), :rd_kafka_producer) }
   let(:closing) { false }
   let(:thread) { double(Thread) }
+  let(:opaque) { Rdkafka::Opaque.new }
 
-  subject(:client) { described_class.new(native, run_polling_thread: true) }
+  subject(:client) { described_class.new(native, run_polling_thread: true, opaque: opaque) }
 
   before do
     allow(Thread).to receive(:new).and_return(thread)

data/spec/rdkafka/producer/delivery_handle_spec.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
-
 describe Rdkafka::Producer::DeliveryHandle do
   let(:response) { 0 }
 
@@ -44,4 +42,19 @@ describe Rdkafka::Producer::DeliveryHandle do
       end
     end
   end
+
+  describe '#create_result' do
+    let(:pending_handle) { false }
+    let(:report) { subject.create_result }
+
+    context 'when response is 0' do
+      it { expect(report.error).to eq(nil) }
+    end
+
+    context 'when response is not 0' do
+      let(:response) { 1 }
+
+      it { expect(report.error).to eq(Rdkafka::RdkafkaError.new(response)) }
+    end
+  end
 end

data/spec/rdkafka/producer/delivery_report_spec.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
-
 describe Rdkafka::Producer::DeliveryReport do
   subject { Rdkafka::Producer::DeliveryReport.new(2, 100, "topic", -1) }