rdkafka 0.8.0 → 0.8.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8bf97d10412b4f3c0801f657796fd05c94ceede3caafe6a45719c840b534752a
-  data.tar.gz: 8cc93bffb8119cf9c97aa60cf1f6d634e66a99686e0785150ae51283a6514e8c
+  metadata.gz: d09f2751b883400c550fee28fca85ffdd3e9f8090a9ba8e7bb22eadcba5c05c6
+  data.tar.gz: 107a5b723b40ec43b10f02b1993f42ff4b30a4ba16efc21611efb860de798a3a
 SHA512:
-  metadata.gz: e8e085b3dd9d80d77003d2c7f0fbda69371c233b121eac2594556d7d62da99a4bed4aace93e9489783c789e64bafd732faaad3de2ffb5d3a5af8568c5b43676b
-  data.tar.gz: 5a267a9814e87e876544e04e6657829c298ff01b03cb5ff41a0205795ea2fb4f6a2147ee1a0949e52f7a069f5dc73d5ed2072867194032e8ea913ac5d6871e23
+  metadata.gz: 373bd71c9e1f3c1fadd4deb9e67bec7aa5821e0a7835c851f1bb64fb1ad142692145e01efa62ac357d333eebb76cef7d676ddfa53ef3a9cd08720060b4bc0937
+  data.tar.gz: ec6d8932a2987e419f6fd731ff038cb88d9d898eab75d7eff82a6e606fcb71c249dd20f6d200bda44629f4bb81550c06ef4e8083ac1b4736c57932ab2486fdbc

data/.semaphore/semaphore.yml

@@ -0,0 +1,23 @@
+version: v1.0
+name: Rdkafka Ruby
+
+agent:
+  machine:
+    type: e1-standard-2
+    os_image: ubuntu1804
+
+blocks:
+  - name: Run specs
+    task:
+      jobs:
+        - name: bundle exec rspec
+          matrix:
+            - env_var: RUBY_VERSION
+              values: [ "2.5.8", "2.6.6", "2.7.2", "jruby-9.2.13.0" ]
+          commands:
+            - sem-version ruby $RUBY_VERSION
+            - checkout
+            - bundle install --path vendor/bundl
+            - cd ext && bundle exec rake && cd ..
+            - docker-compose up -d --no-recreate
+            - bundle exec rspec

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+# 0.8.1
+* Fix topic_flag behaviour and add tests for Metadata (geoff2k)
+* Add topic admin interface (geoff2k)
+* Raise an exception if @native_kafka is nil (geoff2k)
+* Option to use zstd compression (jasonmartens)
+
 # 0.8.0
 * Upgrade librdkafka to 1.4.0
 * Integrate librdkafka metadata API and add partition_key (by Adithya-copart)

data/README.md

@@ -1,9 +1,8 @@
 # Rdkafka
 
-[![Build Status](https://travis-ci.org/appsignal/rdkafka-ruby.svg?branch=master)](https://travis-ci.org/appsignal/rdkafka-ruby)
+[![Build Status](https://appsignal.semaphoreci.com/badges/rdkafka-ruby/branches/master.svg?style=shields)](https://appsignal.semaphoreci.com/projects/rdkafka-ruby)
 [![Gem Version](https://badge.fury.io/rb/rdkafka.svg)](https://badge.fury.io/rb/rdkafka)
 [![Maintainability](https://api.codeclimate.com/v1/badges/ecb1765f81571cccdb0e/maintainability)](https://codeclimate.com/github/appsignal/rdkafka-ruby/maintainability)
-[![Test Coverage](https://api.codeclimate.com/v1/badges/ecb1765f81571cccdb0e/test_coverage)](https://codeclimate.com/github/appsignal/rdkafka-ruby/test_coverage)
 
 The `rdkafka` gem is a modern Kafka client library for Ruby based on
 [librdkafka](https://github.com/edenhill/librdkafka/).
@@ -72,6 +71,10 @@ end
 delivery_handles.each(&:wait)
 ```
 
+Note that creating a producer consumes some resources that will not be
+released until it `#close` is explicitly called, so be sure to call
+`Config#producer` only as necessary.
+
 ## Development
 
 A Docker Compose file is included to run Kafka and Zookeeper. To run

data/docker-compose.yml

@@ -1,5 +1,7 @@
+---
 
 version: '2'
+
 services:
   zookeeper:
     image: confluentinc/cp-zookeeper:latest

data/ext/Rakefile

@@ -65,7 +65,7 @@ namespace :build do
 
     recipe = MiniPortile.new("librdkafka", version)
     recipe.files << "https://github.com/edenhill/librdkafka/archive/#{ref}.tar.gz"
-    recipe.configure_options = ["--host=#{recipe.host}"]
+    recipe.configure_options = ["--host=#{recipe.host}","--enable-static", "--enable-zstd"]
     recipe.cook
 
     ext = recipe.host.include?("darwin") ? "dylib" : "so"

data/lib/rdkafka.rb

@@ -1,6 +1,13 @@
 require "rdkafka/version"
 
+require "rdkafka/abstract_handle"
+require "rdkafka/admin"
+require "rdkafka/admin/create_topic_handle"
+require "rdkafka/admin/create_topic_report"
+require "rdkafka/admin/delete_topic_handle"
+require "rdkafka/admin/delete_topic_report"
 require "rdkafka/bindings"
+require "rdkafka/callbacks"
 require "rdkafka/config"
 require "rdkafka/consumer"
 require "rdkafka/consumer/headers"

data/lib/rdkafka/abstract_handle.rb

@@ -0,0 +1,82 @@
+require "ffi"
+
+module Rdkafka
+  class AbstractHandle < FFI::Struct
+    # Subclasses must define their own layout, and the layout must start with:
+    #
+    # layout :pending, :bool,
+    #        :response, :int
+
+    REGISTRY = {}
+
+    CURRENT_TIME = -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }.freeze
+
+    private_constant :CURRENT_TIME
+
+    def self.register(handle)
+      address = handle.to_ptr.address
+      REGISTRY[address] = handle
+    end
+
+    def self.remove(address)
+      REGISTRY.delete(address)
+    end
+
+    # Whether the handle is still pending.
+    #
+    # @return [Boolean]
+    def pending?
+      self[:pending]
+    end
+
+    # Wait for the operation to complete or raise an error if this takes longer than the timeout.
+    # If there is a timeout this does not mean the operation failed, rdkafka might still be working on the operation.
+    # In this case it is possible to call wait again.
+    #
+    # @param max_wait_timeout [Numeric, nil] Amount of time to wait before timing out. If this is nil it does not time out.
+    # @param wait_timeout [Numeric] Amount of time we should wait before we recheck if the operation has completed
+    #
+    # @raise [RdkafkaError] When the operation failed
+    # @raise [WaitTimeoutError] When the timeout has been reached and the handle is still pending
+    #
+    # @return [Object] Operation-specific result
+    def wait(max_wait_timeout: 60, wait_timeout: 0.1)
+      timeout = if max_wait_timeout
+                  CURRENT_TIME.call + max_wait_timeout
+                else
+                  nil
+                end
+      loop do
+        if pending?
+          if timeout && timeout <= CURRENT_TIME.call
+            raise WaitTimeoutError.new("Waiting for #{operation_name} timed out after #{max_wait_timeout} seconds")
+          end
+          sleep wait_timeout
+        elsif self[:response] != 0
+          raise_error
+        else
+          return create_result
+        end
+      end
+    end
+
+    # @return [String] the name of the operation (e.g. "delivery")
+    def operation_name
+      raise "Must be implemented by subclass!"
+    end
+
+    # @return [Object] operation-specific result
+    def create_result
+      raise "Must be implemented by subclass!"
+    end
+
+    # Allow subclasses to override
+    def raise_error
+      raise RdkafkaError.new(self[:response])
+    end
+
+    # Error that is raised when waiting for the handle to complete
+    # takes longer than the specified timeout.
+    class WaitTimeoutError < RuntimeError; end
+  end
+end

data/lib/rdkafka/admin.rb

@@ -0,0 +1,144 @@
+module Rdkafka
+  class Admin
+    # @private
+    def initialize(native_kafka)
+      @native_kafka = native_kafka
+      @closing = false
+
+      # Start thread to poll client for callbacks
+      @polling_thread = Thread.new do
+        loop do
+          Rdkafka::Bindings.rd_kafka_poll(@native_kafka, 250)
+          # Exit thread if closing and the poll queue is empty
+          if @closing && Rdkafka::Bindings.rd_kafka_outq_len(@native_kafka) == 0
+            break
+          end
+        end
+      end
+      @polling_thread.abort_on_exception = true
+    end
+
+    # Close this admin instance
+    def close
+      return unless @native_kafka
+
+      # Indicate to polling thread that we're closing
+      @closing = true
+      # Wait for the polling thread to finish up
+      @polling_thread.join
+      Rdkafka::Bindings.rd_kafka_destroy(@native_kafka)
+      @native_kafka = nil
+    end
+
+    # Create a topic with the given partition count and replication factor
+    #
+    # @raise [ConfigError] When the partition count or replication factor are out of valid range
+    # @raise [RdkafkaError] When the topic name is invalid or the topic already exists
+    #
+    # @return [CreateTopicHandle] Create topic handle that can be used to wait for the result of creating the topic
+    def create_topic(topic_name, partition_count, replication_factor)
+
+      # Create a rd_kafka_NewTopic_t representing the new topic
+      error_buffer = FFI::MemoryPointer.from_string(" " * 256)
+      new_topic_ptr = Rdkafka::Bindings.rd_kafka_NewTopic_new(
+        FFI::MemoryPointer.from_string(topic_name),
+        partition_count,
+        replication_factor,
+        error_buffer,
+        256
+      )
+      if new_topic_ptr.null?
+        raise Rdkafka::Config::ConfigError.new(error_buffer.read_string)
+      end
+
+      # Note that rd_kafka_CreateTopics can create more than one topic at a time
+      pointer_array = [new_topic_ptr]
+      topics_array_ptr = FFI::MemoryPointer.new(:pointer)
+      topics_array_ptr.write_array_of_pointer(pointer_array)
+
+      # Get a pointer to the queue that our request will be enqueued on
+      queue_ptr = Rdkafka::Bindings.rd_kafka_queue_get_background(@native_kafka)
+      if queue_ptr.null?
+        Rdkafka::Bindings.rd_kafka_NewTopic_destroy(new_topic_ptr)
+        raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
+      end
+
+      # Create and register the handle we will return to the caller
+      create_topic_handle = CreateTopicHandle.new
+      create_topic_handle[:pending] = true
+      create_topic_handle[:response] = -1
+      CreateTopicHandle.register(create_topic_handle)
+      admin_options_ptr = Rdkafka::Bindings.rd_kafka_AdminOptions_new(@native_kafka, Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_CREATETOPICS)
+      Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, create_topic_handle.to_ptr)
+
+      begin
+        Rdkafka::Bindings.rd_kafka_CreateTopics(
+            @native_kafka,
+            topics_array_ptr,
+            1,
+            admin_options_ptr,
+            queue_ptr
+        )
+      rescue Exception => err
+        CreateTopicHandle.remove(create_topic_handle.to_ptr.address)
+        raise
+      ensure
+        Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
+        Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
+        Rdkafka::Bindings.rd_kafka_NewTopic_destroy(new_topic_ptr)
+      end
+
+      create_topic_handle
+    end
+
+    # Delete the named topic
+    #
+    # @raise [RdkafkaError] When the topic name is invalid or the topic does not exist
+    #
+    # @return [DeleteTopicHandle] Delete topic handle that can be used to wait for the result of deleting the topic
+    def delete_topic(topic_name)
+
+      # Create a rd_kafka_DeleteTopic_t representing the topic to be deleted
+      delete_topic_ptr = Rdkafka::Bindings.rd_kafka_DeleteTopic_new(FFI::MemoryPointer.from_string(topic_name))
+
+      # Note that rd_kafka_DeleteTopics can create more than one topic at a time
+      pointer_array = [delete_topic_ptr]
+      topics_array_ptr = FFI::MemoryPointer.new(:pointer)
+      topics_array_ptr.write_array_of_pointer(pointer_array)
+
+      # Get a pointer to the queue that our request will be enqueued on
+      queue_ptr = Rdkafka::Bindings.rd_kafka_queue_get_background(@native_kafka)
+      if queue_ptr.null?
+        Rdkafka::Bindings.rd_kafka_DeleteTopic_destroy(delete_topic_ptr)
+        raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
+      end
+
+      # Create and register the handle we will return to the caller
+      delete_topic_handle = DeleteTopicHandle.new
+      delete_topic_handle[:pending] = true
+      delete_topic_handle[:response] = -1
+      DeleteTopicHandle.register(delete_topic_handle)
+      admin_options_ptr = Rdkafka::Bindings.rd_kafka_AdminOptions_new(@native_kafka, Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_DELETETOPICS)
+      Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, delete_topic_handle.to_ptr)
+
+      begin
+        Rdkafka::Bindings.rd_kafka_DeleteTopics(
+            @native_kafka,
+            topics_array_ptr,
+            1,
+            admin_options_ptr,
+            queue_ptr
+        )
+      rescue Exception => err
+        DeleteTopicHandle.remove(delete_topic_handle.to_ptr.address)
+        raise
+      ensure
+        Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
+        Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
+        Rdkafka::Bindings.rd_kafka_DeleteTopic_destroy(delete_topic_ptr)
+      end
+
+      delete_topic_handle
+    end
+  end
+end

data/lib/rdkafka/admin/create_topic_handle.rb

@@ -0,0 +1,27 @@
+module Rdkafka
+  class Admin
+    class CreateTopicHandle < AbstractHandle
+      layout :pending, :bool,
+             :response, :int,
+             :error_string, :pointer,
+             :result_name, :pointer
+
+      # @return [String] the name of the operation
+      def operation_name
+        "create topic"
+      end
+
+      # @return [Boolean] whether the create topic was successful
+      def create_result
+        CreateTopicReport.new(self[:error_string], self[:result_name])
+      end
+
+      def raise_error
+        raise RdkafkaError.new(
+            self[:response],
+            broker_message: CreateTopicReport.new(self[:error_string], self[:result_name]).error_string
+        )
+      end
+    end
+  end
+end

data/lib/rdkafka/admin/create_topic_report.rb

@@ -0,0 +1,22 @@
+module Rdkafka
+  class Admin
+    class CreateTopicReport
+      # Any error message generated from the CreateTopic
+      # @return [String]
+      attr_reader :error_string
+
+      # The name of the topic created
+      # @return [String]
+      attr_reader :result_name
+
+      def initialize(error_string, result_name)
+        if error_string != FFI::Pointer::NULL
+          @error_string = error_string.read_string
+        end
+        if result_name != FFI::Pointer::NULL
+          @result_name = @result_name = result_name.read_string
+        end
+      end
+    end
+  end
+end

data/lib/rdkafka/admin/delete_topic_handle.rb

@@ -0,0 +1,27 @@
+module Rdkafka
+  class Admin
+    class DeleteTopicHandle < AbstractHandle
+      layout :pending, :bool,
+             :response, :int,
+             :error_string, :pointer,
+             :result_name, :pointer
+
+      # @return [String] the name of the operation
+      def operation_name
+        "delete topic"
+      end
+
+      # @return [Boolean] whether the delete topic was successful
+      def create_result
+        DeleteTopicReport.new(self[:error_string], self[:result_name])
+      end
+
+      def raise_error
+        raise RdkafkaError.new(
+            self[:response],
+            broker_message: DeleteTopicReport.new(self[:error_string], self[:result_name]).error_string
+        )
+      end
+    end
+  end
+end

data/lib/rdkafka/admin/delete_topic_report.rb

@@ -0,0 +1,22 @@
+module Rdkafka
+  class Admin
+    class DeleteTopicReport
+      # Any error message generated from the DeleteTopic
+      # @return [String]
+      attr_reader :error_string
+
+      # The name of the topic deleted
+      # @return [String]
+      attr_reader :result_name
+
+      def initialize(error_string, result_name)
+        if error_string != FFI::Pointer::NULL
+          @error_string = error_string.read_string
+        end
+        if result_name != FFI::Pointer::NULL
+          @result_name = @result_name = result_name.read_string
+        end
+      end
+    end
+  end
+end

data/lib/rdkafka/bindings.rb

@@ -245,22 +245,49 @@ module Rdkafka
       rd_kafka_msg_partitioner_consistent_random(nil, str_ptr, str.size, partition_count, nil, nil)
     end
 
-    DeliveryCallback = FFI::Function.new(
-      :void, [:pointer, :pointer, :pointer]
-    ) do |client_ptr, message_ptr, opaque_ptr|
-      message = Message.new(message_ptr)
-      delivery_handle_ptr_address = message[:_private].address
-      if delivery_handle = Rdkafka::Producer::DeliveryHandle.remove(delivery_handle_ptr_address)
-        # Update delivery handle
-        delivery_handle[:pending] = false
-        delivery_handle[:response] = message[:err]
-        delivery_handle[:partition] = message[:partition]
-        delivery_handle[:offset] = message[:offset]
-        # Call delivery callback on opaque
-        if opaque = Rdkafka::Config.opaques[opaque_ptr.to_i]
-          opaque.call_delivery_callback(Rdkafka::Producer::DeliveryReport.new(message[:partition], message[:offset], message[:err]))
-        end
-      end
-    end
+    # Create Topics
+
+    RD_KAFKA_ADMIN_OP_CREATETOPICS     = 1   # rd_kafka_admin_op_t
+    RD_KAFKA_EVENT_CREATETOPICS_RESULT = 100 # rd_kafka_event_type_t
+
+    attach_function :rd_kafka_CreateTopics, [:pointer, :pointer, :size_t, :pointer, :pointer], :void
+    attach_function :rd_kafka_NewTopic_new, [:pointer, :size_t, :size_t, :pointer, :size_t], :pointer
+    attach_function :rd_kafka_NewTopic_destroy, [:pointer], :void
+    attach_function :rd_kafka_event_CreateTopics_result, [:pointer], :pointer
+    attach_function :rd_kafka_CreateTopics_result_topics, [:pointer, :pointer], :pointer
+
+    # Delete Topics
+
+    RD_KAFKA_ADMIN_OP_DELETETOPICS     = 2   # rd_kafka_admin_op_t
+    RD_KAFKA_EVENT_DELETETOPICS_RESULT = 101 # rd_kafka_event_type_t
+
+    attach_function :rd_kafka_DeleteTopics, [:pointer, :pointer, :size_t, :pointer, :pointer], :int32
+    attach_function :rd_kafka_DeleteTopic_new, [:pointer], :pointer
+    attach_function :rd_kafka_DeleteTopic_destroy, [:pointer], :void
+    attach_function :rd_kafka_event_DeleteTopics_result, [:pointer], :pointer
+    attach_function :rd_kafka_DeleteTopics_result_topics, [:pointer, :pointer], :pointer
+
+    # Background Queue and Callback
+
+    attach_function :rd_kafka_queue_get_background, [:pointer], :pointer
+    attach_function :rd_kafka_conf_set_background_event_cb, [:pointer, :pointer], :void
+    attach_function :rd_kafka_queue_destroy, [:pointer], :void
+
+    # Admin Options
+
+    attach_function :rd_kafka_AdminOptions_new, [:pointer, :int32], :pointer
+    attach_function :rd_kafka_AdminOptions_set_opaque, [:pointer, :pointer], :void
+    attach_function :rd_kafka_AdminOptions_destroy, [:pointer], :void
+
+    # Extracting data from event types
+
+    attach_function :rd_kafka_event_type, [:pointer], :int32
+    attach_function :rd_kafka_event_opaque, [:pointer], :pointer
+
+    # Extracting data from topic results
+
+    attach_function :rd_kafka_topic_result_error, [:pointer], :int32
+    attach_function :rd_kafka_topic_result_error_string, [:pointer], :pointer
+    attach_function :rd_kafka_topic_result_name, [:pointer], :pointer
   end
 end