rdkafka 0.13.1 → 0.14.0.rc1

data/lib/rdkafka/producer.rb

@@ -1,10 +1,15 @@
 # frozen_string_literal: true
 
-require "objspace"
-
 module Rdkafka
   # A producer for Kafka messages. To create a producer set up a {Config} and call {Config#producer producer} on that.
   class Producer
+    include Helpers::Time
+
+    # Cache partitions count for 30 seconds
+    PARTITIONS_COUNT_TTL = 30
+
+    private_constant :PARTITIONS_COUNT_TTL
+
     # @private
     # Returns the current delivery callback, by default this is nil.
     #
@@ -24,6 +29,26 @@ module Rdkafka
 
       # Makes sure, that native kafka gets closed before it gets GCed by Ruby
       ObjectSpace.define_finalizer(self, native_kafka.finalizer)
+
+      @_partitions_count_cache = Hash.new do |cache, topic|
+        topic_metadata = nil
+
+        @native_kafka.with_inner do |inner|
+          topic_metadata = ::Rdkafka::Metadata.new(inner, topic).topics&.first
+        end
+
+        cache[topic] = [
+          monotonic_now,
+          topic_metadata ? topic_metadata[:partition_count] : nil
+        ]
+      end
+    end
+
+    # @return [String] producer name
+    def name
+      @name ||= @native_kafka.with_inner do |inner|
+        ::Rdkafka::Bindings.rd_kafka_name(inner)
+      end
     end
 
     # Set a callback that will be called every time a message is successfully produced.
@@ -54,25 +79,77 @@ module Rdkafka
     # in seconds. Call this before closing a producer to ensure delivery of all messages.
     #
     # @param timeout_ms [Integer] how long should we wait for flush of all messages
+    # @return [Boolean] true if no more data and all was flushed, false in case there are still
+    #   outgoing messages after the timeout
+    #
+    # @note We raise an exception for other errors because based on the librdkafka docs, there
+    #   should be no other errors.
+    #
+    # @note For `timed_out` we do not raise an error to keep it backwards compatible
     def flush(timeout_ms=5_000)
       closed_producer_check(__method__)
 
+      code = nil
+
       @native_kafka.with_inner do |inner|
-        Rdkafka::Bindings.rd_kafka_flush(inner, timeout_ms)
+        code = Rdkafka::Bindings.rd_kafka_flush(inner, timeout_ms)
       end
+
+      # Early skip not to build the error message
+      return true if code.zero?
+
+      error = Rdkafka::RdkafkaError.new(code)
+
+      return false if error.code == :timed_out
+
+      raise(error)
+    end
+
+    # Purges the outgoing queue and releases all resources.
+    #
+    # Useful when closing the producer with outgoing messages to unstable clusters or when for
+    # any other reasons waiting cannot go on anymore. This purges both the queue and all the
+    # inflight requests + updates the delivery handles statuses so they can be materialized into
+    # `purge_queue` errors.
+    def purge
+      closed_producer_check(__method__)
+
+      code = nil
+
+      @native_kafka.with_inner do |inner|
+        code = Bindings.rd_kafka_purge(
+          inner,
+          Bindings::RD_KAFKA_PURGE_F_QUEUE | Bindings::RD_KAFKA_PURGE_F_INFLIGHT
+        )
+      end
+
+      code.zero? || raise(Rdkafka::RdkafkaError.new(code))
+
+      # Wait for the purge to affect everything
+      sleep(0.001) until flush(100)
+
+      true
     end
 
     # Partition count for a given topic.
-    # NOTE: If 'allow.auto.create.topics' is set to true in the broker, the topic will be auto-created after returning nil.
     #
     # @param topic [String] The topic name.
+    # @return [Integer] partition count for a given topic
+    #
+    # @note If 'allow.auto.create.topics' is set to true in the broker, the topic will be
+    #   auto-created after returning nil.
     #
-    # @return partition count [Integer,nil]
+    # @note We cache the partition count for a given topic for given time.
+    #   This prevents us in case someone uses `partition_key` from querying for the count with
+    #   each message. Instead we query once every 30 seconds at most
     def partition_count(topic)
       closed_producer_check(__method__)
-      @native_kafka.with_inner do |inner|
-        Rdkafka::Metadata.new(inner, topic).topics&.first[:partition_count]
+
+      @_partitions_count_cache.delete_if do |_, cached|
+        monotonic_now - cached.first > PARTITIONS_COUNT_TTL
       end
+
+      @_partitions_count_cache[topic].last
     end
 
     # Produces a message to a Kafka topic. The message is added to rdkafka's queue, call {DeliveryHandle#wait wait} on the returned delivery handle to make sure it is delivered.
@@ -193,6 +270,7 @@ module Rdkafka
     end
 
     private
+
     def closed_producer_check(method)
       raise Rdkafka::ClosedProducerError.new(method) if closed?
     end

data/lib/rdkafka/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Rdkafka
-  VERSION = "0.13.1"
-  LIBRDKAFKA_VERSION = "2.0.2"
-  LIBRDKAFKA_SOURCE_SHA256 = "f321bcb1e015a34114c83cf1aa7b99ee260236aab096b85c003170c90a47ca9d"
+  VERSION = "0.14.0.rc1"
+  LIBRDKAFKA_VERSION = "2.2.0"
+  LIBRDKAFKA_SOURCE_SHA256 = "af9a820cbecbc64115629471df7c7cecd40403b6c34bfdbb9223152677a47226"
 end

data/lib/rdkafka.rb

@@ -1,7 +1,12 @@
 # frozen_string_literal: true
 
-require "rdkafka/version"
+require "logger"
+require "objspace"
+require "ffi"
+require "json"
 
+require "rdkafka/version"
+require "rdkafka/helpers/time"
 require "rdkafka/abstract_handle"
 require "rdkafka/admin"
 require "rdkafka/admin/create_topic_handle"
@@ -22,3 +27,7 @@ require "rdkafka/native_kafka"
 require "rdkafka/producer"
 require "rdkafka/producer/delivery_handle"
 require "rdkafka/producer/delivery_report"
+
+# Main Rdkafka namespace of this gem
+module Rdkafka
+end

data/rdkafka.gemspec

@@ -4,11 +4,10 @@ require File.expand_path('lib/rdkafka/version', __dir__)
 
 Gem::Specification.new do |gem|
   gem.authors = ['Thijs Cadier']
-  gem.email = ["thijs@appsignal.com"]
+  gem.email = ["contact@karafka.io"]
   gem.description = "Modern Kafka client library for Ruby based on librdkafka"
   gem.summary = "The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka. It wraps the production-ready C client using the ffi gem and targets Kafka 1.0+ and Ruby 2.4+."
   gem.license = 'MIT'
-  gem.homepage = 'https://github.com/thijsc/rdkafka-ruby'
 
   gem.files = `git ls-files`.split($\)
   gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
@@ -16,8 +15,13 @@ Gem::Specification.new do |gem|
   gem.name = 'rdkafka'
   gem.require_paths = ['lib']
   gem.version = Rdkafka::VERSION
-  gem.required_ruby_version = '>= 2.6'
+  gem.required_ruby_version = '>= 2.7'
   gem.extensions = %w(ext/Rakefile)
+  gem.cert_chain = %w[certs/cert_chain.pem]
+
+  if $PROGRAM_NAME.end_with?('gem')
+    gem.signing_key = File.expand_path('~/.ssh/gem-private_key.pem')
+  end
 
   gem.add_dependency 'ffi', '~> 1.15'
   gem.add_dependency 'mini_portile2', '~> 2.6'
@@ -29,4 +33,14 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency 'simplecov'
   gem.add_development_dependency 'guard'
   gem.add_development_dependency 'guard-rspec'
+
+  gem.metadata = {
+    'funding_uri' => 'https://karafka.io/#become-pro',
+    'homepage_uri' => 'https://karafka.io',
+    'changelog_uri' => 'https://github.com/karafka/rdkafka-ruby/blob/main/CHANGELOG.md',
+    'bug_tracker_uri' => 'https://github.com/karafka/rdkafka-ruby/issues',
+    'source_code_uri' => 'https://github.com/karafka/rdkafka-ruby',
+    'documentation_uri' => 'https://github.com/karafka/rdkafka-ruby/blob/main/README.md',
+    'rubygems_mfa_required' => 'true'
+  }
 end

data/renovate.json

@@ -0,0 +1,6 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": [
+    "config:base"
+  ]
+}

data/spec/rdkafka/abstract_handle_spec.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
-
 describe Rdkafka::AbstractHandle do
   let(:response) { 0 }
   let(:result) { -1 }

data/spec/rdkafka/admin/create_topic_handle_spec.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
-
 describe Rdkafka::Admin::CreateTopicHandle do
   let(:response) { 0 }
 

data/spec/rdkafka/admin/create_topic_report_spec.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
-
 describe Rdkafka::Admin::CreateTopicReport do
   subject { Rdkafka::Admin::CreateTopicReport.new(
       FFI::MemoryPointer.from_string("error string"),

data/spec/rdkafka/admin/delete_topic_handle_spec.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
-
 describe Rdkafka::Admin::DeleteTopicHandle do
   let(:response) { 0 }
 

data/spec/rdkafka/admin/delete_topic_report_spec.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
-
 describe Rdkafka::Admin::DeleteTopicReport do
   subject { Rdkafka::Admin::DeleteTopicReport.new(
       FFI::MemoryPointer.from_string("error string"),

data/spec/rdkafka/admin_spec.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
 require "ostruct"
 
 describe Rdkafka::Admin do
@@ -33,7 +32,7 @@ describe Rdkafka::Admin do
           }.to raise_exception { |ex|
             expect(ex).to be_a(Rdkafka::RdkafkaError)
             expect(ex.message).to match(/Broker: Invalid topic \(topic_exception\)/)
-            expect(ex.broker_message).to match(/Topic name.*is illegal, it contains a character other than ASCII alphanumerics/)
+expect(ex.broker_message).to match(/Topic name.*is invalid: .* contains one or more characters other than ASCII alphanumerics, '.', '_' and '-'/)
           }
         end
       end

data/spec/rdkafka/bindings_spec.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
 require 'zlib'
 
 describe Rdkafka::Bindings do

data/spec/rdkafka/callbacks_spec.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
-
 describe Rdkafka::Callbacks do
 
   # The code in the call back functions is 100% covered by other specs.  Due to

data/spec/rdkafka/config_spec.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
-
 describe Rdkafka::Config do
   context "logger" do
     it "should have a default logger" do

data/spec/rdkafka/consumer/headers_spec.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
-
 describe Rdkafka::Consumer::Headers do
   let(:headers) do
     { # Note String keys!

data/spec/rdkafka/consumer/message_spec.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
-
 describe Rdkafka::Consumer::Message do
   let(:native_client) { new_native_client }
   let(:native_topic) { new_native_topic(native_client: native_client) }

data/spec/rdkafka/consumer/partition_spec.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
-
 describe Rdkafka::Consumer::Partition do
   let(:offset) { 100 }
   let(:err) { 0 }

data/spec/rdkafka/consumer/topic_partition_list_spec.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
-
 describe Rdkafka::Consumer::TopicPartitionList do
   it "should create a new list and add unassigned topics" do
     list = Rdkafka::Consumer::TopicPartitionList.new
@@ -221,5 +219,24 @@ describe Rdkafka::Consumer::TopicPartitionList do
 
       expect(list).to eq other
     end
+
+    it "should create a native list with timetamp offsets if offsets are Time" do
+      list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
+        list.add_topic_and_partitions_with_offsets("topic", 0 => Time.at(1505069646, 250_000))
+      end
+
+      tpl = list.to_native_tpl
+
+      compare_list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
+        list.add_topic_and_partitions_with_offsets(
+          "topic",
+          0 => (Time.at(1505069646, 250_000).to_f * 1000).floor
+        )
+      end
+
+      native_list = Rdkafka::Consumer::TopicPartitionList.from_native_tpl(tpl)
+
+      expect(native_list).to eq compare_list
+    end
   end
 end

data/spec/rdkafka/consumer_spec.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
 require "ostruct"
 require 'securerandom'
 
@@ -11,6 +10,10 @@ describe Rdkafka::Consumer do
   after { consumer.close }
   after { producer.close }
 
+  describe '#name' do
+    it { expect(consumer.name).to include('rdkafka#consumer-') }
+  end
+
   describe "#subscribe, #unsubscribe and #subscription" do
     it "should subscribe, unsubscribe and return the subscription" do
       expect(consumer.subscription).to be_empty
@@ -311,8 +314,9 @@ describe Rdkafka::Consumer do
     end
   end
 
-  describe "#commit, #committed and #store_offset" do
-    # Make sure there's a stored offset
+
+  describe "#position, #commit, #committed and #store_offset" do
+    # Make sure there are messages to work with
     let!(:report) do
       producer.produce(
         topic:     "consume_test_topic",
@@ -330,29 +334,33 @@ describe Rdkafka::Consumer do
       )
     end
 
-    it "should only accept a topic partition list in committed" do
-      expect {
-        consumer.committed("list")
-      }.to raise_error TypeError
+    describe "#position" do
+      it "should only accept a topic partition list in position if not nil" do
+        expect {
+          consumer.position("list")
+        }.to raise_error TypeError
+      end
     end
 
-    it "should commit in sync mode" do
-      expect {
-        consumer.commit(nil, true)
-      }.not_to raise_error
-    end
+    describe "#committed" do
+      it "should only accept a topic partition list in commit if not nil" do
+        expect {
+          consumer.commit("list")
+        }.to raise_error TypeError
+      end
 
-    it "should only accept a topic partition list in commit if not nil" do
-      expect {
-        consumer.commit("list")
-      }.to raise_error TypeError
+      it "should commit in sync mode" do
+        expect {
+          consumer.commit(nil, true)
+        }.not_to raise_error
+      end
     end
 
     context "with a committed consumer" do
       before :all do
         # Make sure there are some messages.
         handles = []
-        producer = rdkafka_producer_config.producer
+        producer = rdkafka_config.producer
         10.times do
           (0..2).each do |i|
             handles << producer.produce(
@@ -396,31 +404,33 @@ describe Rdkafka::Consumer do
         }.to raise_error(Rdkafka::RdkafkaError)
       end
 
-      it "should fetch the committed offsets for the current assignment" do
-        partitions = consumer.committed.to_h["consume_test_topic"]
-        expect(partitions).not_to be_nil
-        expect(partitions[0].offset).to eq 1
-      end
+      describe "#committed" do
+        it "should fetch the committed offsets for the current assignment" do
+          partitions = consumer.committed.to_h["consume_test_topic"]
+          expect(partitions).not_to be_nil
+          expect(partitions[0].offset).to eq 1
+        end
 
-      it "should fetch the committed offsets for a specified topic partition list" do
-        list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
-          list.add_topic("consume_test_topic", [0, 1, 2])
+        it "should fetch the committed offsets for a specified topic partition list" do
+          list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
+            list.add_topic("consume_test_topic", [0, 1, 2])
+          end
+          partitions = consumer.committed(list).to_h["consume_test_topic"]
+          expect(partitions).not_to be_nil
+          expect(partitions[0].offset).to eq 1
+          expect(partitions[1].offset).to eq 1
+          expect(partitions[2].offset).to eq 1
         end
-        partitions = consumer.committed(list).to_h["consume_test_topic"]
-        expect(partitions).not_to be_nil
-        expect(partitions[0].offset).to eq 1
-        expect(partitions[1].offset).to eq 1
-        expect(partitions[2].offset).to eq 1
-      end
 
-      it "should raise an error when getting committed fails" do
-        expect(Rdkafka::Bindings).to receive(:rd_kafka_committed).and_return(20)
-        list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
-          list.add_topic("consume_test_topic", [0, 1, 2])
+        it "should raise an error when getting committed fails" do
+          expect(Rdkafka::Bindings).to receive(:rd_kafka_committed).and_return(20)
+          list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
+            list.add_topic("consume_test_topic", [0, 1, 2])
+          end
+          expect {
+            consumer.committed(list)
+          }.to raise_error Rdkafka::RdkafkaError
         end
-        expect {
-          consumer.committed(list)
-        }.to raise_error Rdkafka::RdkafkaError
       end
 
       describe "#store_offset" do
@@ -441,6 +451,8 @@ describe Rdkafka::Consumer do
           @new_consumer.store_offset(message)
           @new_consumer.commit
 
+          # TODO use position here, should be at offset
+
           list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
             list.add_topic("consume_test_topic", [0, 1, 2])
           end
@@ -455,6 +467,35 @@ describe Rdkafka::Consumer do
             @new_consumer.store_offset(message)
           }.to raise_error Rdkafka::RdkafkaError
         end
+
+        describe "#position" do
+          it "should fetch the positions for the current assignment" do
+            consumer.store_offset(message)
+
+            partitions = consumer.position.to_h["consume_test_topic"]
+            expect(partitions).not_to be_nil
+            expect(partitions[0].offset).to eq message.offset + 1
+          end
+
+          it "should fetch the positions for a specified assignment" do
+            consumer.store_offset(message)
+
+            list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
+              list.add_topic_and_partitions_with_offsets("consume_test_topic", 0 => nil, 1 => nil, 2 => nil)
+            end
+            partitions = consumer.position(list).to_h["consume_test_topic"]
+            expect(partitions).not_to be_nil
+            expect(partitions[0].offset).to eq message.offset + 1
+          end
+
+          it "should raise an error when getting the position fails" do
+            expect(Rdkafka::Bindings).to receive(:rd_kafka_position).and_return(20)
+
+            expect {
+              consumer.position
+            }.to raise_error(Rdkafka::RdkafkaError)
+          end
+        end
       end
     end
   end
@@ -950,6 +991,69 @@ describe Rdkafka::Consumer do
     end
   end
 
+  describe "#offsets_for_times" do
+    it "should raise when not TopicPartitionList" do
+      expect { consumer.offsets_for_times([]) }.to raise_error(TypeError)
+    end
+
+    it "should raise an error when offsets_for_times fails" do
+      tpl = Rdkafka::Consumer::TopicPartitionList.new
+
+      expect(Rdkafka::Bindings).to receive(:rd_kafka_offsets_for_times).and_return(7)
+
+      expect { consumer.offsets_for_times(tpl) }.to raise_error(Rdkafka::RdkafkaError)
+    end
+
+    context "when subscribed" do
+      let(:timeout) { 1000 }
+
+      before do
+        consumer.subscribe("consume_test_topic")
+
+        # 1. partitions are assigned
+        wait_for_assignment(consumer)
+        expect(consumer.assignment).not_to be_empty
+
+        # 2. eat unrelated messages
+        while(consumer.poll(timeout)) do; end
+      end
+
+      after { consumer.unsubscribe }
+
+      def send_one_message(val)
+        producer.produce(
+          topic:     "consume_test_topic",
+          payload:   "payload #{val}",
+          key:       "key 0",
+          partition: 0
+        ).wait
+      end
+
+      it "returns a TopicParticionList with updated offsets" do
+        send_one_message("a")
+        send_one_message("b")
+        send_one_message("c")
+
+        consumer.poll(timeout)
+        message = consumer.poll(timeout)
+        consumer.poll(timeout)
+
+        tpl = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
+          list.add_topic_and_partitions_with_offsets(
+            "consume_test_topic",
+            [
+              [0, message.timestamp]
+            ]
+          )
+        end
+
+        tpl_response = consumer.offsets_for_times(tpl)
+
+        expect(tpl_response.to_h["consume_test_topic"][0].offset).to eq message.offset
+      end
+    end
+  end
+
   describe "a rebalance listener" do
     let(:consumer) do
       config = rdkafka_consumer_config
@@ -1024,7 +1128,7 @@ describe Rdkafka::Consumer do
         :assign                  => [ nil ],
         :assignment              => nil,
         :committed               => [],
-        :query_watermark_offsets => [ nil, nil ],
+        :query_watermark_offsets => [ nil, nil ]
     }.each do |method, args|
       it "raises an exception if #{method} is called" do
         expect {

data/spec/rdkafka/error_spec.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
-
 describe Rdkafka::RdkafkaError do
   it "should raise a type error for a nil response" do
     expect {

data/spec/rdkafka/metadata_spec.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
 require "securerandom"
 
 describe Rdkafka::Metadata do
@@ -31,7 +30,7 @@ describe Rdkafka::Metadata do
       it "#brokers returns our single broker" do
         expect(subject.brokers.length).to eq(1)
         expect(subject.brokers[0][:broker_id]).to eq(1)
-        expect(subject.brokers[0][:broker_name]).to eq("localhost")
+        expect(subject.brokers[0][:broker_name]).to eq("127.0.0.1")
         expect(subject.brokers[0][:broker_port]).to eq(9092)
       end
 
@@ -54,7 +53,7 @@ describe Rdkafka::Metadata do
     it "#brokers returns our single broker" do
       expect(subject.brokers.length).to eq(1)
       expect(subject.brokers[0][:broker_id]).to eq(1)
-      expect(subject.brokers[0][:broker_name]).to eq("localhost")
+      expect(subject.brokers[0][:broker_name]).to eq("127.0.0.1")
       expect(subject.brokers[0][:broker_port]).to eq(9092)
     end
 

data/spec/rdkafka/native_kafka_spec.rb

@@ -1,14 +1,13 @@
 # frozen_string_literal: true
 
-require "spec_helper"
-
 describe Rdkafka::NativeKafka do
   let(:config) { rdkafka_producer_config }
   let(:native) { config.send(:native_kafka, config.send(:native_config), :rd_kafka_producer) }
   let(:closing) { false }
   let(:thread) { double(Thread) }
+  let(:opaque) { Rdkafka::Opaque.new }
 
-  subject(:client) { described_class.new(native, run_polling_thread: true) }
+  subject(:client) { described_class.new(native, run_polling_thread: true, opaque: opaque) }
 
   before do
     allow(Thread).to receive(:new).and_return(thread)