rdkafka 0.6.0 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0b248aee9a260de3d8bb56efba9255f21d3bfcb6c269a1ab15beac7ae2521b6c
-  data.tar.gz: 2d1199d5717a832674f659ac1d5e69d2a659845e9c8e5509428270a4a8390ae1
+  metadata.gz: 22bda74a815bebd487179ac5ec3faa4566752f2c87945ca5c461c427633daf1f
+  data.tar.gz: '0802ca4118ed248fa11226a2c1b2140d073d230c02e3696a03e462fe59cb129e'
 SHA512:
-  metadata.gz: 62afc350fa40f5e2117e7791927fe5ad7fcb3cb7ba13cdbba2f450dc6537d41e37668a5997fccbb260abe3fbb09e7a73c0cb5d5059b94837d5097f373c27eb75
-  data.tar.gz: 1723633bcce6569992a2368df38c2d59b217dd94af9e8afab9932ca085e9ae0fbc3661ee8fbc370d31f3985204b38b3bbe15bbdfa11ca2be61c69b4c938a40f3
+  metadata.gz: 4d3b7042fa2290386d4feaaa01c4289bc858042d66806157066257f282c84db459e80e32a2613efd3452a46c63faaff7fad9f110fb153c3385cef00fb6017f27
+  data.tar.gz: e431a320535df8293e099e1b25bf0e6da68caf9a641621faf847dd206093dd9973b41f03f95412c43a740b50a0ef70e7caad418ff20c07218f19342d1cb3826a

data/.semaphore/semaphore.yml

@@ -0,0 +1,23 @@
+version: v1.0
+name: Rdkafka Ruby
+
+agent:
+  machine:
+    type: e1-standard-4
+    os_image: ubuntu1804
+
+blocks:
+  - name: Run specs
+    task:
+      jobs:
+        - name: bundle exec rspec
+          matrix:
+            - env_var: RUBY_VERSION
+              values: [ "2.5.8", "2.6.6", "2.7.2", "3.0.0", "jruby-9.2.13.0" ]
+          commands:
+            - sem-version ruby $RUBY_VERSION
+            - checkout
+            - bundle install --path vendor/bundle
+            - cd ext && bundle exec rake && cd ..
+            - docker-compose up -d --no-recreate
+            - bundle exec rspec

data/CHANGELOG.md

@@ -1,3 +1,29 @@
+# 0.9.0
+* Fixes for Ruby 3.0
+* Allow any callable object for callbacks (gremerritt)
+* Reduce memory allocations in Rdkafka::Producer#produce (jturkel)
+* Use queue as log callback to avoid unsafe calls from trap context (breunigs)
+* Allow passing in topic configuration on create_topic (dezka)
+* Add each_batch method to consumer (mgrosso)
+
+# 0.8.1
+* Fix topic_flag behaviour and add tests for Metadata (geoff2k)
+* Add topic admin interface (geoff2k)
+* Raise an exception if @native_kafka is nil (geoff2k)
+* Option to use zstd compression (jasonmartens)
+
+# 0.8.0
+* Upgrade librdkafka to 1.4.0
+* Integrate librdkafka metadata API and add partition_key (by Adithya-copart)
+* Ruby 2.7 compatibility fix (by Geoff Thé)A
+* Add error to delivery report (by Alex Stanovsky)
+* Don't override CPPFLAGS and LDFLAGS if already set on Mac (by Hiroshi Hatake)
+* Allow use of Rake 13.x and up (by Tomasz Pajor)
+
+# 0.7.0
+* Bump librdkafka to 1.2.0 (by rob-as)
+* Allow customizing the wait time for delivery report availability (by mensfeld)
+
 # 0.6.0
 * Bump librdkafka to 1.1.0 (by Chris Gaffney)
 * Implement seek (by breunigs)
@@ -7,6 +33,7 @@
 * Add cluster and member information (by dmexe)
 * Support message headers for consumer & producer (by dmexe)
 * Add consumer rebalance listener (by dmexe)
+* Implement pause/resume partitions (by dmexe)
 
 # 0.4.2
 * Delivery callback for producer

data/README.md

@@ -1,14 +1,17 @@
 # Rdkafka
 
-[![Build Status](https://travis-ci.org/appsignal/rdkafka-ruby.svg?branch=master)](https://travis-ci.org/appsignal/rdkafka-ruby)
+[![Build Status](https://appsignal.semaphoreci.com/badges/rdkafka-ruby/branches/master.svg?style=shields)](https://appsignal.semaphoreci.com/projects/rdkafka-ruby)
 [![Gem Version](https://badge.fury.io/rb/rdkafka.svg)](https://badge.fury.io/rb/rdkafka)
 [![Maintainability](https://api.codeclimate.com/v1/badges/ecb1765f81571cccdb0e/maintainability)](https://codeclimate.com/github/appsignal/rdkafka-ruby/maintainability)
-[![Test Coverage](https://api.codeclimate.com/v1/badges/ecb1765f81571cccdb0e/test_coverage)](https://codeclimate.com/github/appsignal/rdkafka-ruby/test_coverage)
 
 The `rdkafka` gem is a modern Kafka client library for Ruby based on
 [librdkafka](https://github.com/edenhill/librdkafka/).
 It wraps the production-ready C client using the [ffi](https://github.com/ffi/ffi)
-gem and targets Kafka 1.0+ and Ruby 2.3+.
+gem and targets Kafka 1.0+ and Ruby 2.4+.
+
+`rdkafka` was written because we needed a reliable Ruby client for
+Kafka that supports modern Kafka at [AppSignal](https://appsignal.com).
+We run it in production on very high traffic systems.
 
 This gem only provides a high-level Kafka consumer. If you are running
 an older version of Kafka and/or need the legacy simple consumer we
@@ -68,12 +71,9 @@ end
 delivery_handles.each(&:wait)
 ```
 
-## Known issues
-
-When using forked process such as when using Unicorn you currently need
-to make sure that you create rdkafka instances after forking. Otherwise
-they will not work and crash your Ruby process when they are garbage
-collected. See https://github.com/appsignal/rdkafka-ruby/issues/19
+Note that creating a producer consumes some resources that will not be
+released until it `#close` is explicitly called, so be sure to call
+`Config#producer` only as necessary.
 
 ## Development
 

data/docker-compose.yml

@@ -1,18 +1,24 @@
+---
+
 version: '2'
+
 services:
   zookeeper:
-    image: wurstmeister/zookeeper
-    ports:
-      - "2181:2181"
+    image: confluentinc/cp-zookeeper:latest
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+      ZOOKEEPER_TICK_TIME: 2000
+
   kafka:
-    image: wurstmeister/kafka:1.0.1
+    image: confluentinc/cp-kafka:latest
+    depends_on:
+      - zookeeper
     ports:
-      - "9092:9092"
+      - 9092:9092
     environment:
-      KAFKA_ADVERTISED_HOST_NAME: localhost
-      KAFKA_ADVERTISED_PORT: 9092
+      KAFKA_BROKER_ID: 1
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
-      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'
-      KAFKA_CREATE_TOPICS: "consume_test_topic:3:1,empty_test_topic:3:1,load_test_topic:3:1,produce_test_topic:3:1,rake_test_topic:3:1,watermarks_test_topic:3:1"
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1

data/ext/README.md

@@ -5,19 +5,14 @@ this gem is installed.
 
 To update the `librdkafka` version follow the following steps:
 
-* Download the new version `tar.gz` from  https://github.com/edenhill/librdkafka/
-* Generate a `sha256` with (if using MacOS) `shasum -a 256 <file>`
-* Change the `sha256` in `lib/rdkafka/version.rb`
+* Go to https://github.com/edenhill/librdkafka/releases to get the new
+  version number and asset checksum for `tar.gz`.
 * Change the version in `lib/rdkafka/version.rb`
-
-## Disclaimer
-
-Currently the `librdkafka` project does not provide
-checksums of releases. The checksum provided here is generated on a best
-effort basis. If the CDN would be compromised at the time of download the
-checksum could be incorrect.
-
-Do your own verification if you rely on this behaviour.
-
-Once https://github.com/appsignal/rdkafka-ruby/issues/44 is implemented
-we will change this process.
+* Change the `sha256` in `lib/rdkafka/version.rb`
+* Run `bundle exec rake` in the `ext` directory to download and build
+  the new version
+* Run `docker-compose pull` in the main gem directory to ensure the docker
+  images used by the tests and run `docker-compose up`
+* Finally, run `bundle exec rspec` in the main gem directory to execute
+  the test suite to detect any regressions that may have been introduced
+  by the update

data/ext/Rakefile

@@ -1,6 +1,7 @@
 require File.expand_path('../../lib/rdkafka/version', __FILE__)
 require "mini_portile2"
 require "fileutils"
+require "open-uri"
 
 task :default => :clean do
   # MiniPortile#download_file_http is a monkey patch that removes the download
@@ -23,9 +24,10 @@ task :default => :clean do
   recipe = MiniPortile.new("librdkafka", Rdkafka::LIBRDKAFKA_VERSION)
 
   # Use default homebrew openssl if we're on mac and the directory exists
-  if recipe.host.include?("darwin") && Dir.exists?("/usr/local/opt/openssl")
-    ENV["CPPFLAGS"] = "-I/usr/local/opt/openssl/include"
-    ENV["LDFLAGS"] = "-L/usr/local/opt/openssl/lib"
+  # and each of flags is not empty
+  if recipe.host&.include?("darwin") && Dir.exist?("/usr/local/opt/openssl")
+    ENV["CPPFLAGS"] = "-I/usr/local/opt/openssl/include" unless ENV["CPPFLAGS"]
+    ENV["LDFLAGS"] = "-L/usr/local/opt/openssl/lib" unless ENV["LDFLAGS"]
   end
 
   recipe.files << {
@@ -55,3 +57,22 @@ task :clean do
   FileUtils.rm_rf File.join(File.dirname(__FILE__), "ports")
   FileUtils.rm_rf File.join(File.dirname(__FILE__), "tmp")
 end
+
+namespace :build do
+  desc "Build librdkafka at the given git sha or tag"
+  task :git, [:ref] do |task, args|
+    ref = args[:ref]
+    version = "git-#{ref}"
+
+    recipe = MiniPortile.new("librdkafka", version)
+    recipe.files << "https://github.com/edenhill/librdkafka/archive/#{ref}.tar.gz"
+    recipe.configure_options = ["--host=#{recipe.host}","--enable-static", "--enable-zstd"]
+    recipe.cook
+
+    ext = recipe.host.include?("darwin") ? "dylib" : "so"
+    lib = File.expand_path("ports/#{recipe.host}/librdkafka/#{version}/lib/librdkafka.#{ext}", __dir__)
+
+    # Copy will copy the content, following any symlinks
+    FileUtils.cp(lib, __dir__)
+  end
+end

data/lib/rdkafka.rb

@@ -1,6 +1,13 @@
 require "rdkafka/version"
 
+require "rdkafka/abstract_handle"
+require "rdkafka/admin"
+require "rdkafka/admin/create_topic_handle"
+require "rdkafka/admin/create_topic_report"
+require "rdkafka/admin/delete_topic_handle"
+require "rdkafka/admin/delete_topic_report"
 require "rdkafka/bindings"
+require "rdkafka/callbacks"
 require "rdkafka/config"
 require "rdkafka/consumer"
 require "rdkafka/consumer/headers"
@@ -8,6 +15,7 @@ require "rdkafka/consumer/message"
 require "rdkafka/consumer/partition"
 require "rdkafka/consumer/topic_partition_list"
 require "rdkafka/error"
+require "rdkafka/metadata"
 require "rdkafka/producer"
 require "rdkafka/producer/delivery_handle"
 require "rdkafka/producer/delivery_report"

data/lib/rdkafka/abstract_handle.rb

@@ -0,0 +1,82 @@
+require "ffi"
+
+module Rdkafka
+  class AbstractHandle < FFI::Struct
+    # Subclasses must define their own layout, and the layout must start with:
+    #
+    # layout :pending, :bool,
+    #        :response, :int
+
+    REGISTRY = {}
+
+    CURRENT_TIME = -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }.freeze
+
+    private_constant :CURRENT_TIME
+
+    def self.register(handle)
+      address = handle.to_ptr.address
+      REGISTRY[address] = handle
+    end
+
+    def self.remove(address)
+      REGISTRY.delete(address)
+    end
+
+    # Whether the handle is still pending.
+    #
+    # @return [Boolean]
+    def pending?
+      self[:pending]
+    end
+
+    # Wait for the operation to complete or raise an error if this takes longer than the timeout.
+    # If there is a timeout this does not mean the operation failed, rdkafka might still be working on the operation.
+    # In this case it is possible to call wait again.
+    #
+    # @param max_wait_timeout [Numeric, nil] Amount of time to wait before timing out. If this is nil it does not time out.
+    # @param wait_timeout [Numeric] Amount of time we should wait before we recheck if the operation has completed
+    #
+    # @raise [RdkafkaError] When the operation failed
+    # @raise [WaitTimeoutError] When the timeout has been reached and the handle is still pending
+    #
+    # @return [Object] Operation-specific result
+    def wait(max_wait_timeout: 60, wait_timeout: 0.1)
+      timeout = if max_wait_timeout
+                  CURRENT_TIME.call + max_wait_timeout
+                else
+                  nil
+                end
+      loop do
+        if pending?
+          if timeout && timeout <= CURRENT_TIME.call
+            raise WaitTimeoutError.new("Waiting for #{operation_name} timed out after #{max_wait_timeout} seconds")
+          end
+          sleep wait_timeout
+        elsif self[:response] != 0
+          raise_error
+        else
+          return create_result
+        end
+      end
+    end
+
+    # @return [String] the name of the operation (e.g. "delivery")
+    def operation_name
+      raise "Must be implemented by subclass!"
+    end
+
+    # @return [Object] operation-specific result
+    def create_result
+      raise "Must be implemented by subclass!"
+    end
+
+    # Allow subclasses to override
+    def raise_error
+      raise RdkafkaError.new(self[:response])
+    end
+
+    # Error that is raised when waiting for the handle to complete
+    # takes longer than the specified timeout.
+    class WaitTimeoutError < RuntimeError; end
+  end
+end

data/lib/rdkafka/admin.rb

@@ -0,0 +1,155 @@
+module Rdkafka
+  class Admin
+    # @private
+    def initialize(native_kafka)
+      @native_kafka = native_kafka
+      @closing = false
+
+      # Start thread to poll client for callbacks
+      @polling_thread = Thread.new do
+        loop do
+          Rdkafka::Bindings.rd_kafka_poll(@native_kafka, 250)
+          # Exit thread if closing and the poll queue is empty
+          if @closing && Rdkafka::Bindings.rd_kafka_outq_len(@native_kafka) == 0
+            break
+          end
+        end
+      end
+      @polling_thread.abort_on_exception = true
+    end
+
+    # Close this admin instance
+    def close
+      return unless @native_kafka
+
+      # Indicate to polling thread that we're closing
+      @closing = true
+      # Wait for the polling thread to finish up
+      @polling_thread.join
+      Rdkafka::Bindings.rd_kafka_destroy(@native_kafka)
+      @native_kafka = nil
+    end
+
+    # Create a topic with the given partition count and replication factor
+    #
+    # @raise [ConfigError] When the partition count or replication factor are out of valid range
+    # @raise [RdkafkaError] When the topic name is invalid or the topic already exists
+    # @raise [RdkafkaError] When the topic configuration is invalid
+    #
+    # @return [CreateTopicHandle] Create topic handle that can be used to wait for the result of creating the topic
+    def create_topic(topic_name, partition_count, replication_factor, topic_config={})
+
+      # Create a rd_kafka_NewTopic_t representing the new topic
+      error_buffer = FFI::MemoryPointer.from_string(" " * 256)
+      new_topic_ptr = Rdkafka::Bindings.rd_kafka_NewTopic_new(
+        FFI::MemoryPointer.from_string(topic_name),
+        partition_count,
+        replication_factor,
+        error_buffer,
+        256
+      )
+      if new_topic_ptr.null?
+        raise Rdkafka::Config::ConfigError.new(error_buffer.read_string)
+      end
+
+      unless topic_config.nil?
+        topic_config.each do |key, value|
+          Rdkafka::Bindings.rd_kafka_NewTopic_set_config(
+            new_topic_ptr,
+            key.to_s,
+            value.to_s
+          )
+        end
+      end
+
+      # Note that rd_kafka_CreateTopics can create more than one topic at a time
+      pointer_array = [new_topic_ptr]
+      topics_array_ptr = FFI::MemoryPointer.new(:pointer)
+      topics_array_ptr.write_array_of_pointer(pointer_array)
+
+      # Get a pointer to the queue that our request will be enqueued on
+      queue_ptr = Rdkafka::Bindings.rd_kafka_queue_get_background(@native_kafka)
+      if queue_ptr.null?
+        Rdkafka::Bindings.rd_kafka_NewTopic_destroy(new_topic_ptr)
+        raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
+      end
+
+      # Create and register the handle we will return to the caller
+      create_topic_handle = CreateTopicHandle.new
+      create_topic_handle[:pending] = true
+      create_topic_handle[:response] = -1
+      CreateTopicHandle.register(create_topic_handle)
+      admin_options_ptr = Rdkafka::Bindings.rd_kafka_AdminOptions_new(@native_kafka, Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_CREATETOPICS)
+      Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, create_topic_handle.to_ptr)
+
+      begin
+        Rdkafka::Bindings.rd_kafka_CreateTopics(
+            @native_kafka,
+            topics_array_ptr,
+            1,
+            admin_options_ptr,
+            queue_ptr
+        )
+      rescue Exception => err
+        CreateTopicHandle.remove(create_topic_handle.to_ptr.address)
+        raise
+      ensure
+        Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
+        Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
+        Rdkafka::Bindings.rd_kafka_NewTopic_destroy(new_topic_ptr)
+      end
+
+      create_topic_handle
+    end
+
+    # Delete the named topic
+    #
+    # @raise [RdkafkaError] When the topic name is invalid or the topic does not exist
+    #
+    # @return [DeleteTopicHandle] Delete topic handle that can be used to wait for the result of deleting the topic
+    def delete_topic(topic_name)
+
+      # Create a rd_kafka_DeleteTopic_t representing the topic to be deleted
+      delete_topic_ptr = Rdkafka::Bindings.rd_kafka_DeleteTopic_new(FFI::MemoryPointer.from_string(topic_name))
+
+      # Note that rd_kafka_DeleteTopics can create more than one topic at a time
+      pointer_array = [delete_topic_ptr]
+      topics_array_ptr = FFI::MemoryPointer.new(:pointer)
+      topics_array_ptr.write_array_of_pointer(pointer_array)
+
+      # Get a pointer to the queue that our request will be enqueued on
+      queue_ptr = Rdkafka::Bindings.rd_kafka_queue_get_background(@native_kafka)
+      if queue_ptr.null?
+        Rdkafka::Bindings.rd_kafka_DeleteTopic_destroy(delete_topic_ptr)
+        raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
+      end
+
+      # Create and register the handle we will return to the caller
+      delete_topic_handle = DeleteTopicHandle.new
+      delete_topic_handle[:pending] = true
+      delete_topic_handle[:response] = -1
+      DeleteTopicHandle.register(delete_topic_handle)
+      admin_options_ptr = Rdkafka::Bindings.rd_kafka_AdminOptions_new(@native_kafka, Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_DELETETOPICS)
+      Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, delete_topic_handle.to_ptr)
+
+      begin
+        Rdkafka::Bindings.rd_kafka_DeleteTopics(
+            @native_kafka,
+            topics_array_ptr,
+            1,
+            admin_options_ptr,
+            queue_ptr
+        )
+      rescue Exception => err
+        DeleteTopicHandle.remove(delete_topic_handle.to_ptr.address)
+        raise
+      ensure
+        Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
+        Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
+        Rdkafka::Bindings.rd_kafka_DeleteTopic_destroy(delete_topic_ptr)
+      end
+
+      delete_topic_handle
+    end
+  end
+end