karafka-rdkafka 0.13.0 → 0.13.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 91b097ee3e4b5df5b9deb4009297c84b9d59ffa1ec0aa0f0b0afa230418478aa
-  data.tar.gz: b0897c92e746ebe9194b735ebe0b813ae274c0a472c5381e9b097a4d059b2966
+  metadata.gz: 8d216b3535772b9039cb4f81113f75b1f1262052282c3bb159c78095cddccf3a
+  data.tar.gz: ce7e3a921cd0c4a0da1b1d00c4af672949445b0ed05e325c975a3b8533e8a134
 SHA512:
-  metadata.gz: e6fb584d20d1958e66133e45806d2d25ba885f413ccb683e1380c65611f068d65198aa459920871b2a15b0d84e434d1824ae5eb10cba04a6a118538b471cd5d8
-  data.tar.gz: ac0ff2519d7580018a7d6359f3e7bd7410f6c52aeb0e95a5b4cc749e6becc2863ddea229e11d8ef04f74865e608b4f7ee2e057730a5d837e4e6da5696817b3a8
+  metadata.gz: 5acb41af183c6a101c5b7bc88166573c22f2e84516d060b1afb78c6d958b44ffc1b13bcdb08fd8891f16c16ebd6acdabea6c51131d58c30c93e86fe7e3f714dd
+  data.tar.gz: a8daa33c897782dfb3875e9e8be078f40ed40a97c926c809f9cb349689d78d1fdeec8bfb70ca1e7a88dadade9f70dcd74ece2e647c30ed3fb855a4dfb3495aa7

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+# 0.13.2
+* Ensure operations counter decrement is fully thread-safe
+* Bump librdkafka to 2.1.1
+
+# 0.13.1
+* Add offsets_for_times method on consumer (timflapper)
+
 # 0.13.0
 * Support cooperative sticky partition assignment in the rebalance callback (methodmissing)
 * Support both string and symbol header keys (ColinDKelley)

data/lib/rdkafka/bindings.rb

@@ -193,6 +193,7 @@ module Rdkafka
     attach_function :rd_kafka_pause_partitions, [:pointer, :pointer], :int, blocking: true
     attach_function :rd_kafka_resume_partitions, [:pointer, :pointer], :int, blocking: true
     attach_function :rd_kafka_seek, [:pointer, :int32, :int64, :int], :int, blocking: true
+    attach_function :rd_kafka_offsets_for_times, [:pointer, :pointer, :int], :int, blocking: true
 
     # Headers
     attach_function :rd_kafka_header_get_all, [:pointer, :size_t, :pointer, :pointer, SizePtr], :int

data/lib/rdkafka/consumer/topic_partition_list.rb

@@ -142,11 +142,13 @@ module Rdkafka
               )
 
               if p.offset
+                offset = p.offset.is_a?(Time) ? p.offset.to_f * 1_000 : p.offset
+
                 Rdkafka::Bindings.rd_kafka_topic_partition_list_set_offset(
                   tpl,
                   topic,
                   p.partition,
-                  p.offset
+                  offset
                 )
               end
             end

data/lib/rdkafka/consumer.rb

@@ -34,9 +34,11 @@ module Rdkafka
     def close
       return if closed?
       ObjectSpace.undefine_finalizer(self)
-      @native_kafka.with_inner do |inner|
+
+      @native_kafka.synchronize do |inner|
         Rdkafka::Bindings.rd_kafka_consumer_close(inner)
       end
+
       @native_kafka.close
     end
 
@@ -429,6 +431,39 @@ module Rdkafka
       end
     end
 
+    # Lookup offset for the given partitions by timestamp.
+    #
+    # @param list [TopicPartitionList] The TopicPartitionList with timestamps instead of offsets
+    #
+    # @raise [RdKafkaError] When the OffsetForTimes lookup fails
+    #
+    # @return [TopicPartitionList]
+    def offsets_for_times(list, timeout_ms = 1000)
+      closed_consumer_check(__method__)
+
+      if !list.is_a?(TopicPartitionList)
+        raise TypeError.new("list has to be a TopicPartitionList")
+      end
+
+      tpl = list.to_native_tpl
+
+      response = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_offsets_for_times(
+          inner,
+          tpl,
+          timeout_ms # timeout
+        )
+      end
+
+      if response != 0
+        raise Rdkafka::RdkafkaError.new(response)
+      end
+
+      TopicPartitionList.from_native_tpl(tpl)
+    ensure
+      Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl) if tpl
+    end
+
     # Manually commit the current offsets of this consumer.
     #
     # To use this set `enable.auto.commit`to `false` to disable automatic triggering

data/lib/rdkafka/error.rb

@@ -92,4 +92,10 @@ module Rdkafka
       super("Illegal call to #{method.to_s} on a closed admin")
     end
   end
+
+  class ClosedInnerError < BaseError
+    def initialize
+      super("Illegal call to a closed inner librdkafka instance")
+    end
+  end
 end

data/lib/rdkafka/native_kafka.rb

@@ -10,6 +10,22 @@ module Rdkafka
       @access_mutex = Mutex.new
       # Lock around internal polling
       @poll_mutex = Mutex.new
+      # Lock around decrementing the operations in progress counter
+      # We have two mutexes - one for increment (`@access_mutex`) and one for decrement mutex
+      # because they serve different purposes:
+      #
+      #   - `@access_mutex` allows us to lock the execution and make sure that any operation within
+      #      the `#synchronize` is the only one running and that there are no other running
+      #      operations.
+      #   - `@decrement_mutex` ensures, that our decrement operation is thread-safe for any Ruby
+      #     implementation.
+      #
+      # We do not use the same mutex, because it could create a deadlock when an already
+      # incremented operation cannot decrement because `@access_lock` is now owned by a different
+      # thread in a synchronized mode and the synchronized mode is waiting on the decrement.
+      @decrement_mutex = Mutex.new
+      # counter for operations in progress using inner
+      @operations_in_progress = 0
 
       if run_polling_thread
         # Start thread to poll client for delivery callbacks,
@@ -35,9 +51,27 @@ module Rdkafka
     end
 
     def with_inner
-      return if @inner.nil?
+      if @access_mutex.owned?
+        @operations_in_progress += 1
+      else
+        @access_mutex.synchronize { @operations_in_progress += 1 }
+      end
+
+      @inner.nil? ? raise(ClosedInnerError) : yield(@inner)
+    ensure
+      @decrement_mutex.synchronize { @operations_in_progress -= 1 }
+    end
 
-      yield @inner
+    def synchronize(&block)
+      @access_mutex.synchronize do
+        # Wait for any commands using the inner to finish
+        # This can take a while on blocking operations like polling but is essential not to proceed
+        # with certain types of operations like resources destruction as it can cause the process
+        # to hang or crash
+        sleep(0.01) until @operations_in_progress.zero?
+
+        with_inner(&block)
+      end
     end
 
     def finalizer
@@ -51,31 +85,31 @@ module Rdkafka
     def close(object_id=nil)
       return if closed?
 
-      @access_mutex.lock
+      synchronize do
+        # Indicate to the outside world that we are closing
+        @closing = true
 
-      # Indicate to the outside world that we are closing
-      @closing = true
+        if @polling_thread
+          # Indicate to polling thread that we're closing
+          @polling_thread[:closing] = true
 
-      if @polling_thread
-        # Indicate to polling thread that we're closing
-        @polling_thread[:closing] = true
-
-        # Wait for the polling thread to finish up,
-        # this can be aborted in practice if this
-        # code runs from a finalizer.
-        @polling_thread.join
-      end
+          # Wait for the polling thread to finish up,
+          # this can be aborted in practice if this
+          # code runs from a finalizer.
+          @polling_thread.join
+        end
 
-      # Destroy the client after locking both mutexes
-      @poll_mutex.lock
+        # Destroy the client after locking both mutexes
+        @poll_mutex.lock
 
-      # This check prevents a race condition, where we would enter the close in two threads
-      # and after unlocking the primary one that hold the lock but finished, ours would be unlocked
-      # and would continue to run, trying to destroy inner twice
-      return unless @inner
+        # This check prevents a race condition, where we would enter the close in two threads
+        # and after unlocking the primary one that hold the lock but finished, ours would be unlocked
+        # and would continue to run, trying to destroy inner twice
+        return unless @inner
 
-      Rdkafka::Bindings.rd_kafka_destroy(@inner)
-      @inner = nil
+        Rdkafka::Bindings.rd_kafka_destroy(@inner)
+        @inner = nil
+      end
     end
   end
 end

data/lib/rdkafka/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Rdkafka
-  VERSION = "0.13.0"
-  LIBRDKAFKA_VERSION = "2.0.2"
-  LIBRDKAFKA_SOURCE_SHA256 = "f321bcb1e015a34114c83cf1aa7b99ee260236aab096b85c003170c90a47ca9d"
+  VERSION = "0.13.2"
+  LIBRDKAFKA_VERSION = "2.1.1"
+  LIBRDKAFKA_SOURCE_SHA256 = "7be1fc37ab10ebdc037d5c5a9b35b48931edafffae054b488faaff99e60e0108"
 end

data/spec/rdkafka/consumer/topic_partition_list_spec.rb

@@ -221,5 +221,24 @@ describe Rdkafka::Consumer::TopicPartitionList do
 
       expect(list).to eq other
     end
+
+    it "should create a native list with timetamp offsets if offsets are Time" do
+      list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
+        list.add_topic_and_partitions_with_offsets("topic", 0 => Time.at(1505069646, 250_000))
+      end
+
+      tpl = list.to_native_tpl
+
+      compare_list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
+        list.add_topic_and_partitions_with_offsets(
+          "topic",
+          0 => (Time.at(1505069646, 250_000).to_f * 1000).floor
+        )
+      end
+
+      native_list = Rdkafka::Consumer::TopicPartitionList.from_native_tpl(tpl)
+
+      expect(native_list).to eq compare_list
+    end
   end
 end

data/spec/rdkafka/consumer_spec.rb

@@ -312,6 +312,29 @@ describe Rdkafka::Consumer do
         consumer.poll(100)
       }.to raise_error(Rdkafka::ClosedConsumerError, /poll/)
     end
+
+    context 'when there are outgoing operations in other threads' do
+      it 'should wait and not crash' do
+        times = []
+
+        # Run a long running poll
+        thread = Thread.new do
+          times << Time.now
+          consumer.subscribe("empty_test_topic")
+          times << Time.now
+          consumer.poll(1_000)
+          times << Time.now
+        end
+
+        # Make sure it starts before we close
+        sleep(0.1)
+        consumer.close
+        close_time = Time.now
+        thread.join
+
+        times.each { |op_time| expect(op_time).to be < close_time }
+      end
+    end
   end
 
   describe "#commit, #committed and #store_offset" do
@@ -953,6 +976,69 @@ describe Rdkafka::Consumer do
     end
   end
 
+  describe "#offsets_for_times" do
+    it "should raise when not TopicPartitionList" do
+      expect { consumer.offsets_for_times([]) }.to raise_error(TypeError)
+    end
+
+    it "should raise an error when offsets_for_times fails" do
+      tpl = Rdkafka::Consumer::TopicPartitionList.new
+
+      expect(Rdkafka::Bindings).to receive(:rd_kafka_offsets_for_times).and_return(7)
+
+      expect { consumer.offsets_for_times(tpl) }.to raise_error(Rdkafka::RdkafkaError)
+    end
+
+    context "when subscribed" do
+      let(:timeout) { 1000 }
+
+      before do
+        consumer.subscribe("consume_test_topic")
+
+        # 1. partitions are assigned
+        wait_for_assignment(consumer)
+        expect(consumer.assignment).not_to be_empty
+
+        # 2. eat unrelated messages
+        while(consumer.poll(timeout)) do; end
+      end
+
+      after { consumer.unsubscribe }
+
+      def send_one_message(val)
+        producer.produce(
+          topic:     "consume_test_topic",
+          payload:   "payload #{val}",
+          key:       "key 0",
+          partition: 0
+        ).wait
+      end
+
+      it "returns a TopicParticionList with updated offsets" do
+        send_one_message("a")
+        send_one_message("b")
+        send_one_message("c")
+
+        consumer.poll(timeout)
+        message = consumer.poll(timeout)
+        consumer.poll(timeout)
+
+        tpl = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
+          list.add_topic_and_partitions_with_offsets(
+            "consume_test_topic",
+            [
+              [0, message.timestamp]
+            ]
+          )
+        end
+
+        tpl_response = consumer.offsets_for_times(tpl)
+
+        expect(tpl_response.to_h["consume_test_topic"][0].offset).to eq message.offset
+      end
+    end
+  end
+
   describe "a rebalance listener" do
     let(:consumer) do
       config = rdkafka_consumer_config

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.13.0
+  version: 0.13.2
 platform: ruby
 authors:
 - Thijs Cadier
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2023-06-19 00:00:00.000000000 Z
+date: 2023-06-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -258,7 +258,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.1
+rubygems_version: 3.3.4
 signing_key: 
 specification_version: 4
 summary: The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka.