karafka-rdkafka 0.13.2 → 0.13.9

data/lib/rdkafka/admin.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "objspace"
-
 module Rdkafka
   class Admin
     # @private
@@ -30,11 +28,12 @@ module Rdkafka
 
     # Create a topic with the given partition count and replication factor
     #
+    # @return [CreateTopicHandle] Create topic handle that can be used to wait for the result of
+    #   creating the topic
+    #
     # @raise [ConfigError] When the partition count or replication factor are out of valid range
     # @raise [RdkafkaError] When the topic name is invalid or the topic already exists
     # @raise [RdkafkaError] When the topic configuration is invalid
-    #
-    # @return [CreateTopicHandle] Create topic handle that can be used to wait for the result of creating the topic
     def create_topic(topic_name, partition_count, replication_factor, topic_config={})
       closed_admin_check(__method__)
 
@@ -107,11 +106,11 @@ module Rdkafka
       create_topic_handle
     end
 
-    # Delete the named topic
+    # Deletes the named topic
     #
+    # @return [DeleteTopicHandle] Delete topic handle that can be used to wait for the result of
+    #   deleting the topic
     # @raise [RdkafkaError] When the topic name is invalid or the topic does not exist
-    #
-    # @return [DeleteTopicHandle] Delete topic handle that can be used to wait for the result of deleting the topic
     def delete_topic(topic_name)
       closed_admin_check(__method__)
 

data/lib/rdkafka/bindings.rb

@@ -1,11 +1,16 @@
 # frozen_string_literal: true
 
-require "ffi"
-require "json"
-require "logger"
-
 module Rdkafka
   # @private
+  #
+  # @note
+  #   There are two types of responses related to errors:
+  #     - rd_kafka_error_t - a C object that we need to remap into an error or null when no error
+  #     - rd_kafka_resp_err_t - response error code (numeric) that we can use directly
+  #
+  #   It is critical to ensure, that we handle them correctly. The result type should be:
+  #     - rd_kafka_error_t - :pointer
+  #     - rd_kafka_resp_err_t - :int
   module Bindings
     extend FFI::Library
 
@@ -35,7 +40,7 @@ module Rdkafka
 
     # Polling
 
-    attach_function :rd_kafka_flush, [:pointer, :int], :void, blocking: true
+    attach_function :rd_kafka_flush, [:pointer, :int], :int, blocking: true
     attach_function :rd_kafka_poll, [:pointer, :int], :void, blocking: true
     attach_function :rd_kafka_outq_len, [:pointer], :int, blocking: true
 
@@ -96,6 +101,11 @@ module Rdkafka
 
     attach_function :rd_kafka_err2name, [:int], :string
     attach_function :rd_kafka_err2str, [:int], :string
+    attach_function :rd_kafka_error_is_fatal, [:pointer], :int
+    attach_function :rd_kafka_error_is_retriable, [:pointer], :int
+    attach_function :rd_kafka_error_txn_requires_abort, [:pointer], :int
+    attach_function :rd_kafka_error_destroy, [:pointer], :void
+    attach_function :rd_kafka_error_code, [:pointer], :int
 
     # Configuration
 
@@ -157,7 +167,7 @@ module Rdkafka
       :void, [:pointer, :int, :string, :pointer]
     ) do |_client_prr, err_code, reason, _opaque|
       if Rdkafka::Config.error_callback
-        error = Rdkafka::RdkafkaError.new(err_code, broker_message: reason)
+        error = Rdkafka::RdkafkaError.build(err_code, broker_message: reason)
         error.set_backtrace(caller)
         Rdkafka::Config.error_callback.call(error)
       end
@@ -194,6 +204,7 @@ module Rdkafka
     attach_function :rd_kafka_resume_partitions, [:pointer, :pointer], :int, blocking: true
     attach_function :rd_kafka_seek, [:pointer, :int32, :int64, :int], :int, blocking: true
     attach_function :rd_kafka_offsets_for_times, [:pointer, :pointer, :int], :int, blocking: true
+    attach_function :rd_kafka_position, [:pointer, :pointer], :int, blocking: true
 
     # Headers
     attach_function :rd_kafka_header_get_all, [:pointer, :size_t, :pointer, :pointer, SizePtr], :int
@@ -255,12 +266,19 @@ module Rdkafka
     RD_KAFKA_VTYPE_TIMESTAMP = 8
     RD_KAFKA_VTYPE_HEADER = 9
     RD_KAFKA_VTYPE_HEADERS = 10
+    RD_KAFKA_PURGE_F_QUEUE = 1
+    RD_KAFKA_PURGE_F_INFLIGHT = 2
 
     RD_KAFKA_MSG_F_COPY = 0x2
 
     attach_function :rd_kafka_producev, [:pointer, :varargs], :int, blocking: true
+    attach_function :rd_kafka_purge, [:pointer, :int], :int, blocking: true
     callback :delivery_cb, [:pointer, :pointer, :pointer], :void
     attach_function :rd_kafka_conf_set_dr_msg_cb, [:pointer, :delivery_cb], :void
+    attach_function :rd_kafka_init_transactions, [:pointer, :int], :pointer, blocking: true
+    attach_function :rd_kafka_begin_transaction, [:pointer], :pointer, blocking: true
+    attach_function :rd_kafka_abort_transaction, [:pointer, :int], :pointer, blocking: true
+    attach_function :rd_kafka_commit_transaction, [:pointer, :int], :pointer, blocking: true
 
     # Partitioner
     PARTITIONERS = %w(random consistent consistent_random murmur2 murmur2_random fnv1a fnv1a_random).each_with_object({}) do |name, hsh|

data/lib/rdkafka/config.rb

@@ -1,11 +1,9 @@
 # frozen_string_literal: true
 
-require "logger"
-
 module Rdkafka
   # Configuration for a Kafka consumer or producer. You can create an instance and use
   # the consumer and producer methods to create a client. Documentation of the available
-  # configuration options is available on https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md.
+  # configuration options is available on https://github.com/confluentinc/librdkafka/blob/master/CONFIGURATION.md.
   class Config
     # @private
     @@logger = Logger.new(STDOUT)
@@ -14,7 +12,7 @@ module Rdkafka
     # @private
     @@error_callback = nil
     # @private
-    @@opaques = {}
+    @@opaques = ObjectSpace::WeakMap.new
     # @private
     @@log_queue = Queue.new
 
@@ -53,13 +51,13 @@ module Rdkafka
 
     # Set a callback that will be called every time the underlying client emits statistics.
     # You can configure if and how often this happens using `statistics.interval.ms`.
-    # The callback is called with a hash that's documented here: https://github.com/edenhill/librdkafka/blob/master/STATISTICS.md
+    # The callback is called with a hash that's documented here: https://github.com/confluentinc/librdkafka/blob/master/STATISTICS.md
     #
     # @param callback [Proc, #call] The callback
     #
     # @return [nil]
     def self.statistics_callback=(callback)
-      raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call)
+      raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call) || callback == nil
       @@statistics_callback = callback
     end
 
@@ -114,6 +112,7 @@ module Rdkafka
     def initialize(config_hash = {})
       @config_hash = DEFAULT_CONFIG.merge(config_hash)
       @consumer_rebalance_listener = nil
+      @consumer_poll_set = true
     end
 
     # Set a config option.
@@ -142,12 +141,28 @@ module Rdkafka
       @consumer_rebalance_listener = listener
     end
 
-    # Create a consumer with this configuration.
+    # Should we use a single queue for the underlying consumer and events.
     #
-    # @raise [ConfigError] When the configuration contains invalid options
-    # @raise [ClientCreationError] When the native client cannot be created
+    # This is an advanced API that allows for more granular control of the polling process.
+    # When this value is set to `false` (`true` by defualt), there will be two queues that need to
+    # be polled:
+    #   - main librdkafka queue for events
+    #   - consumer queue with messages and rebalances
+    #
+    # It is recommended to use the defaults and only set it to `false` in advance multi-threaded
+    # and complex cases where granular events handling control is needed.
+    #
+    # @param poll_set [Boolean]
+    def consumer_poll_set=(poll_set)
+      @consumer_poll_set = poll_set
+    end
+
+    # Creates a consumer with this configuration.
     #
     # @return [Consumer] The created consumer
+    #
+    # @raise [ConfigError] When the configuration contains invalid options
+    # @raise [ClientCreationError] When the native client cannot be created
     def consumer
       opaque = Opaque.new
       config = native_config(opaque)
@@ -160,19 +175,25 @@ module Rdkafka
       # Create native client
       kafka = native_kafka(config, :rd_kafka_consumer)
 
-      # Redirect the main queue to the consumer
-      Rdkafka::Bindings.rd_kafka_poll_set_consumer(kafka)
+      # Redirect the main queue to the consumer queue
+      Rdkafka::Bindings.rd_kafka_poll_set_consumer(kafka) if @consumer_poll_set
 
       # Return consumer with Kafka client
-      Rdkafka::Consumer.new(Rdkafka::NativeKafka.new(kafka, run_polling_thread: false))
+      Rdkafka::Consumer.new(
+        Rdkafka::NativeKafka.new(
+          kafka,
+          run_polling_thread: false,
+          opaque: opaque
+        )
+      )
     end
 
     # Create a producer with this configuration.
     #
+    # @return [Producer] The created producer
+    #
     # @raise [ConfigError] When the configuration contains invalid options
     # @raise [ClientCreationError] When the native client cannot be created
-    #
-    # @return [Producer] The created producer
     def producer
       # Create opaque
       opaque = Opaque.new
@@ -182,22 +203,35 @@ module Rdkafka
       Rdkafka::Bindings.rd_kafka_conf_set_dr_msg_cb(config, Rdkafka::Callbacks::DeliveryCallbackFunction)
       # Return producer with Kafka client
       partitioner_name = self[:partitioner] || self["partitioner"]
-      Rdkafka::Producer.new(Rdkafka::NativeKafka.new(native_kafka(config, :rd_kafka_producer), run_polling_thread: true), partitioner_name).tap do |producer|
+      Rdkafka::Producer.new(
+        Rdkafka::NativeKafka.new(
+          native_kafka(config, :rd_kafka_producer),
+          run_polling_thread: true,
+          opaque: opaque
+        ),
+        partitioner_name
+      ).tap do |producer|
         opaque.producer = producer
       end
     end
 
-    # Create an admin instance with this configuration.
+    # Creates an admin instance with this configuration.
+    #
+    # @return [Admin] The created admin instance
     #
     # @raise [ConfigError] When the configuration contains invalid options
     # @raise [ClientCreationError] When the native client cannot be created
-    #
-    # @return [Admin] The created admin instance
     def admin
       opaque = Opaque.new
       config = native_config(opaque)
       Rdkafka::Bindings.rd_kafka_conf_set_background_event_cb(config, Rdkafka::Callbacks::BackgroundEventCallbackFunction)
-      Rdkafka::Admin.new(Rdkafka::NativeKafka.new(native_kafka(config, :rd_kafka_producer), run_polling_thread: true))
+      Rdkafka::Admin.new(
+        Rdkafka::NativeKafka.new(
+          native_kafka(config, :rd_kafka_producer),
+          run_polling_thread: true,
+          opaque: opaque
+        )
+      )
     end
 
     # Error that is returned by the underlying rdkafka error if an invalid configuration option is present.

data/lib/rdkafka/consumer/headers.rb

@@ -18,13 +18,11 @@ module Rdkafka
 
       # Reads a librdkafka native message's headers and returns them as a Ruby Hash
       #
-      # @param [librdkakfa message] native_message
+      # @private
       #
+      # @param [librdkakfa message] native_message
       # @return [Hash<String, String>] headers Hash for the native_message
-      #
       # @raise [Rdkafka::RdkafkaError] when fail to read headers
-      #
-      # @private
       def self.from_native(native_message)
         headers_ptrptr = FFI::MemoryPointer.new(:pointer)
         err = Rdkafka::Bindings.rd_kafka_message_headers(native_message, headers_ptrptr)