rdkafka 0.10.0 → 0.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9f954490d858591f12e5a7fc1fed82e1b5c6b42049c5ed66d2d76bb352e03a7f
-  data.tar.gz: 0ae8cbac09963d52a3075cece9e0bf6c5895e31f87995d7997ee7f04e52a10c8
+  metadata.gz: 821523c304fc7a1fbb2c7be2b58d98d56600b645b89fdb4093f976418650035d
+  data.tar.gz: '039b8e345fd8be5f295a293d64466071dbefd77d81b01460abb0fcf343a6bed3'
 SHA512:
-  metadata.gz: 33d43862b6b7cee1a3284221f75fae0c6bdfa6df4faa4396378c96e34a8f1a2395202dd01d6a5f2baac5557010505f548e7862bdd14202de6b3f85c77ca95309
-  data.tar.gz: 22d6243a853bfd630497eab13a8b6823f1a1d4a358aee53a98e68e5a0255f5ca68932db6d261cbad34dadeca5854522d8fceb15b43004ce8719a1d5d9aa3e9bd
+  metadata.gz: 2c7ac2199a63aacd3b1420890981ed5d953ae5cdadb874886cc4e396fa1fd8f69333633319beef35a05a002d75d22335a526a126e518cc3fbbb877a1c11ef2f7
+  data.tar.gz: 5d23c6beec3759877013b040018111453e05c41238014b07a27c1a9d8b96e8af3bc037aacd1ebe89f856435cf0afb8e34a9f89443f87cf1a3682736efb79b4bd

data/.rspec

@@ -0,0 +1 @@
+--format documentation

data/.semaphore/semaphore.yml

@@ -13,7 +13,7 @@ blocks:
         - name: bundle exec rspec
           matrix:
             - env_var: RUBY_VERSION
-              values: [ "2.5.8", "2.6.6", "2.7.2", "3.0.0", "jruby-9.2.13.0" ]
+              values: [ "2.6.8", "2.7.4", "3.0.2", "jruby-9.3.1.0"]
           commands:
             - sem-version ruby $RUBY_VERSION
             - checkout

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+# 0.12.0
+* Bumps librdkafka to 1.9.0
+
+# 0.11.0
+* Upgrade librdkafka to 1.8.2
+* Bump supported minimum Ruby version to 2.6
+* Better homebrew path detection
+
 # 0.10.0
 * Upgrade librdkafka to 1.5.0
 * Add error callback config
@@ -45,7 +53,7 @@
 * Use default Homebrew openssl location if present
 * Consumer lag handles empty topics
 * End iteration in consumer when it is closed
-* Add suport for storing message offsets
+* Add support for storing message offsets
 * Add missing runtime dependency to rake
 
 # 0.4.1

data/Guardfile

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+logger level: :error
+
+guard :rspec, cmd: "bundle exec rspec --format #{ENV.fetch("FORMAT", "documentation")}" do
+  require "guard/rspec/dsl"
+  dsl = Guard::RSpec::Dsl.new(self)
+
+  # Ruby files
+  ruby = dsl.ruby
+  dsl.watch_spec_files_for(ruby.lib_files)
+  watch(%r{^lib/(.+)\.rb}) { |m| "spec/#{m[1]}_spec.rb" }
+
+  # RSpec files
+  rspec = dsl.rspec
+  watch(rspec.spec_helper) { rspec.spec_dir }
+  watch(rspec.spec_support) { rspec.spec_dir }
+  watch(rspec.spec_files)
+end

data/README.md

@@ -7,7 +7,9 @@
 The `rdkafka` gem is a modern Kafka client library for Ruby based on
 [librdkafka](https://github.com/edenhill/librdkafka/).
 It wraps the production-ready C client using the [ffi](https://github.com/ffi/ffi)
-gem and targets Kafka 1.0+ and Ruby 2.4+.
+gem and targets Kafka 1.0+ and Ruby versions that are under security or
+active maintenance. We remove Ruby version from our CI builds if they 
+become EOL.
 
 `rdkafka` was written because we needed a reliable Ruby client for
 Kafka that supports modern Kafka at [AppSignal](https://appsignal.com).

data/bin/console

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+# frozen_string_literal: true
+
+ENV["IRBRC"] = File.join(File.dirname(__FILE__), ".irbrc")
+
+require "bundler/setup"
+require "rdkafka"
+
+require "irb"
+IRB.start(__FILE__)

data/docker-compose.yml

@@ -4,13 +4,13 @@ version: '2'
 
 services:
   zookeeper:
-    image: confluentinc/cp-zookeeper:latest
+    image: confluentinc/cp-zookeeper:5.2.6
     environment:
       ZOOKEEPER_CLIENT_PORT: 2181
       ZOOKEEPER_TICK_TIME: 2000
 
   kafka:
-    image: confluentinc/cp-kafka:latest
+    image: confluentinc/cp-kafka:5.2.5-10
     depends_on:
       - zookeeper
     ports:
@@ -18,7 +18,7 @@ services:
     environment:
       KAFKA_BROKER_ID: 1
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
-      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:29092,PLAINTEXT_HOST://localhost:9092
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
       KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1

data/ext/README.md

@@ -1,6 +1,6 @@
 # Ext
 
-This gem dependes on the `librdkafka` C library. It is downloaded when
+This gem depends on the `librdkafka` C library. It is downloaded when
 this gem is installed.
 
 To update the `librdkafka` version follow the following steps:

data/ext/Rakefile

@@ -4,30 +4,14 @@ require "fileutils"
 require "open-uri"
 
 task :default => :clean do
-  # MiniPortile#download_file_http is a monkey patch that removes the download
-  # progress indicator. This indicator relies on the 'Content Length' response
-  # headers, which is not set by GitHub
-  class MiniPortile
-    def download_file_http(url, full_path, _count)
-      filename = File.basename(full_path)
-      with_tempfile(filename, full_path) do |temp_file|
-        params = { 'Accept-Encoding' => 'identity' }
-        OpenURI.open_uri(url, 'rb', params) do |io|
-          temp_file.write(io.read)
-        end
-        output
-      end
-    end
-  end
-
   # Download and compile librdkafka
   recipe = MiniPortile.new("librdkafka", Rdkafka::LIBRDKAFKA_VERSION)
 
   # Use default homebrew openssl if we're on mac and the directory exists
   # and each of flags is not empty
-  if recipe.host&.include?("darwin") && Dir.exist?("/usr/local/opt/openssl")
-    ENV["CPPFLAGS"] = "-I/usr/local/opt/openssl/include" unless ENV["CPPFLAGS"]
-    ENV["LDFLAGS"] = "-L/usr/local/opt/openssl/lib" unless ENV["LDFLAGS"]
+  if recipe.host&.include?("darwin") && system("which brew &> /dev/null") && Dir.exist?("#{homebrew_prefix = %x(brew --prefix openssl).strip}")
+    ENV["CPPFLAGS"] = "-I#{homebrew_prefix}/include" unless ENV["CPPFLAGS"]
+    ENV["LDFLAGS"] = "-L#{homebrew_prefix}/lib" unless ENV["LDFLAGS"]
   end
 
   recipe.files << {

data/lib/rdkafka/admin.rb

@@ -90,7 +90,7 @@ module Rdkafka
             admin_options_ptr,
             queue_ptr
         )
-      rescue Exception => err
+      rescue Exception
         CreateTopicHandle.remove(create_topic_handle.to_ptr.address)
         raise
       ensure
@@ -140,7 +140,7 @@ module Rdkafka
             admin_options_ptr,
             queue_ptr
         )
-      rescue Exception => err
+      rescue Exception
         DeleteTopicHandle.remove(delete_topic_handle.to_ptr.address)
         raise
       ensure

data/lib/rdkafka/bindings.rb

@@ -246,14 +246,21 @@ module Rdkafka
     attach_function :rd_kafka_conf_set_dr_msg_cb, [:pointer, :delivery_cb], :void
 
     # Partitioner
-    attach_function :rd_kafka_msg_partitioner_consistent_random, [:pointer, :pointer, :size_t, :int32, :pointer, :pointer], :int32
+    PARTITIONERS = %w(random consistent consistent_random murmur2 murmur2_random fnv1a fnv1a_random).each_with_object({}) do |name, hsh|
+      method_name = "rd_kafka_msg_partitioner_#{name}".to_sym
+      attach_function method_name, [:pointer, :pointer, :size_t, :int32, :pointer, :pointer], :int32
+      hsh[name] = method_name
+    end
 
-    def self.partitioner(str, partition_count)
+    def self.partitioner(str, partition_count, partitioner_name = "consistent_random")
       # Return RD_KAFKA_PARTITION_UA(unassigned partition) when partition count is nil/zero.
       return -1 unless partition_count&.nonzero?
 
       str_ptr = FFI::MemoryPointer.from_string(str)
-      rd_kafka_msg_partitioner_consistent_random(nil, str_ptr, str.size, partition_count, nil, nil)
+      method_name = PARTITIONERS.fetch(partitioner_name) do
+        raise Rdkafka::Config::ConfigError.new("Unknown partitioner: #{partitioner_name}")
+      end
+      public_send(method_name, nil, str_ptr, str.size, partition_count, nil, nil)
     end
 
     # Create Topics

data/lib/rdkafka/callbacks.rb

@@ -97,7 +97,7 @@ module Rdkafka
           delivery_handle[:pending] = false
           # Call delivery callback on opaque
           if opaque = Rdkafka::Config.opaques[opaque_ptr.to_i]
-            opaque.call_delivery_callback(Rdkafka::Producer::DeliveryReport.new(message[:partition], message[:offset], message[:err]))
+            opaque.call_delivery_callback(Rdkafka::Producer::DeliveryReport.new(message[:partition], message[:offset], message[:err]), delivery_handle)
           end
         end
       end

data/lib/rdkafka/config.rb

@@ -179,7 +179,7 @@ module Rdkafka
       # Set callback to receive delivery reports on config
       Rdkafka::Bindings.rd_kafka_conf_set_dr_msg_cb(config, Rdkafka::Callbacks::DeliveryCallbackFunction)
       # Return producer with Kafka client
-      Rdkafka::Producer.new(native_kafka(config, :rd_kafka_producer)).tap do |producer|
+      Rdkafka::Producer.new(Rdkafka::Producer::Client.new(native_kafka(config, :rd_kafka_producer)), self[:partitioner]).tap do |producer|
         opaque.producer = producer
       end
     end
@@ -278,8 +278,8 @@ module Rdkafka
     attr_accessor :producer
     attr_accessor :consumer_rebalance_listener
 
-    def call_delivery_callback(delivery_handle)
-      producer.call_delivery_callback(delivery_handle) if producer
+    def call_delivery_callback(delivery_report, delivery_handle)
+      producer.call_delivery_callback(delivery_report, delivery_handle) if producer
     end
 
     def call_on_partitions_assigned(consumer, list)

data/lib/rdkafka/consumer.rb

@@ -498,11 +498,11 @@ module Rdkafka
     # Exception behavior is more complicated than with `each`, in that if
     # :yield_on_error is true, and an exception is raised during the
     # poll, and messages have already been received, they will be yielded to
-    # the caller before the exception is allowed to propogate.
+    # the caller before the exception is allowed to propagate.
     #
     # If you are setting either auto.commit or auto.offset.store to false in
     # the consumer configuration, then you should let yield_on_error keep its
-    # default value of false because you are gauranteed to see these messages
+    # default value of false because you are guaranteed to see these messages
     # again. However, if both auto.commit and auto.offset.store are set to
     # true, you should set yield_on_error to true so you can process messages
     # that you may or may not see again.
@@ -518,7 +518,7 @@ module Rdkafka
     # @yield [messages, pending_exception]
     # @yieldparam messages [Array] An array of received Message
     # @yieldparam pending_exception [Exception] normally nil, or an exception
-    # which will be propogated after processing of the partial batch is complete.
+    # which will be propagated after processing of the partial batch is complete.
     #
     # @return [nil]
     def each_batch(max_items: 100, bytes_threshold: Float::INFINITY, timeout_ms: 250, yield_on_error: false, &block)

data/lib/rdkafka/producer/client.rb

@@ -0,0 +1,47 @@
+module Rdkafka
+  class Producer
+    class Client
+      def initialize(native)
+        @native = native
+
+        # Start thread to poll client for delivery callbacks
+        @polling_thread = Thread.new do
+          loop do
+            Rdkafka::Bindings.rd_kafka_poll(native, 250)
+            # Exit thread if closing and the poll queue is empty
+            if Thread.current[:closing] && Rdkafka::Bindings.rd_kafka_outq_len(native) == 0
+              break
+            end
+          end
+        end
+        @polling_thread.abort_on_exception = true
+        @polling_thread[:closing] = false
+      end
+
+      def native
+        @native
+      end
+
+      def finalizer
+        ->(_) { close }
+      end
+
+      def closed?
+        @native.nil?
+      end
+
+      def close(object_id=nil)
+        return unless @native
+
+        # Indicate to polling thread that we're closing
+        @polling_thread[:closing] = true
+        # Wait for the polling thread to finish up
+        @polling_thread.join
+
+        Rdkafka::Bindings.rd_kafka_destroy(@native)
+
+        @native = nil
+      end
+    end
+  end
+end

data/lib/rdkafka/producer/delivery_report.rb

@@ -11,7 +11,7 @@ module Rdkafka
       attr_reader :offset
 
       # Error in case happen during produce.
-      # @return [string]
+      # @return [String]
       attr_reader :error
 
       private

data/lib/rdkafka/producer.rb

@@ -1,4 +1,4 @@
-require "securerandom"
+require "objspace"
 
 module Rdkafka
   # A producer for Kafka messages. To create a producer set up a {Config} and call {Config#producer producer} on that.
@@ -10,29 +10,22 @@ module Rdkafka
     attr_reader :delivery_callback
 
     # @private
-    def initialize(native_kafka)
-      @id = SecureRandom.uuid
-      @closing = false
-      @native_kafka = native_kafka
+    # Returns the number of arguments accepted by the callback, by default this is nil.
+    #
+    # @return [Integer, nil]
+    attr_reader :delivery_callback_arity
+
+    # @private
+    def initialize(client, partitioner_name)
+      @client = client
+      @partitioner_name = partitioner_name || "consistent_random"
 
       # Makes sure, that the producer gets closed before it gets GCed by Ruby
-      ObjectSpace.define_finalizer(@id, proc { close })
-
-      # Start thread to poll client for delivery callbacks
-      @polling_thread = Thread.new do
-        loop do
-          Rdkafka::Bindings.rd_kafka_poll(@native_kafka, 250)
-          # Exit thread if closing and the poll queue is empty
-          if @closing && Rdkafka::Bindings.rd_kafka_outq_len(@native_kafka) == 0
-            break
-          end
-        end
-      end
-      @polling_thread.abort_on_exception = true
+      ObjectSpace.define_finalizer(self, client.finalizer)
     end
 
     # Set a callback that will be called every time a message is successfully produced.
-    # The callback is called with a {DeliveryReport}
+    # The callback is called with a {DeliveryReport} and {DeliveryHandle}
     #
     # @param callback [Proc, #call] The callback
     #
@@ -40,20 +33,14 @@ module Rdkafka
     def delivery_callback=(callback)
       raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call)
       @delivery_callback = callback
+      @delivery_callback_arity = arity(callback)
     end
 
     # Close this producer and wait for the internal poll queue to empty.
     def close
-      ObjectSpace.undefine_finalizer(@id)
+      ObjectSpace.undefine_finalizer(self)
 
-      return unless @native_kafka
-
-      # Indicate to polling thread that we're closing
-      @closing = true
-      # Wait for the polling thread to finish up
-      @polling_thread.join
-      Rdkafka::Bindings.rd_kafka_destroy(@native_kafka)
-      @native_kafka = nil
+      @client.close
     end
 
     # Partition count for a given topic.
@@ -65,7 +52,7 @@ module Rdkafka
     #
     def partition_count(topic)
       closed_producer_check(__method__)
-      Rdkafka::Metadata.new(@native_kafka, topic).topics&.first[:partition_count]
+      Rdkafka::Metadata.new(@client.native, topic).topics&.first[:partition_count]
     end
 
     # Produces a message to a Kafka topic. The message is added to rdkafka's queue, call {DeliveryHandle#wait wait} on the returned delivery handle to make sure it is delivered.
@@ -75,8 +62,9 @@ module Rdkafka
     #
     # @param topic [String] The topic to produce to
     # @param payload [String,nil] The message's payload
-    # @param key [String] The message's key
+    # @param key [String, nil] The message's key
     # @param partition [Integer,nil] Optional partition to produce to
+    # @param partition_key [String, nil] Optional partition key based on which partition assignment can happen
     # @param timestamp [Time,Integer,nil] Optional timestamp of this message. Integer timestamp is in milliseconds since Jan 1 1970.
     # @param headers [Hash<String,String>] Optional message headers
     #
@@ -105,7 +93,7 @@ module Rdkafka
       if partition_key
         partition_count = partition_count(topic)
         # If the topic is not present, set to -1
-        partition = Rdkafka::Bindings.partitioner(partition_key, partition_count) if partition_count
+        partition = Rdkafka::Bindings.partitioner(partition_key, partition_count, @partitioner_name) if partition_count
       end
 
       # If partition is nil, use -1 to let librdafka set the partition randomly or
@@ -156,7 +144,7 @@ module Rdkafka
 
       # Produce the message
       response = Rdkafka::Bindings.rd_kafka_producev(
-        @native_kafka,
+        @client.native,
         *args
       )
 
@@ -170,12 +158,21 @@ module Rdkafka
     end
 
     # @private
-    def call_delivery_callback(delivery_handle)
-      @delivery_callback.call(delivery_handle) if @delivery_callback
+    def call_delivery_callback(delivery_report, delivery_handle)
+      return unless @delivery_callback
+
+      args = [delivery_report, delivery_handle].take(@delivery_callback_arity)
+      @delivery_callback.call(*args)
+    end
+
+    def arity(callback)
+      return callback.arity if callback.respond_to?(:arity)
+
+      callback.method(:call).arity
     end
 
     def closed_producer_check(method)
-      raise Rdkafka::ClosedProducerError.new(method) if @native_kafka.nil?
+      raise Rdkafka::ClosedProducerError.new(method) if @client.closed?
     end
   end
 end

data/lib/rdkafka/version.rb

@@ -1,5 +1,5 @@
 module Rdkafka
-  VERSION = "0.10.0"
-  LIBRDKAFKA_VERSION = "1.5.0"
-  LIBRDKAFKA_SOURCE_SHA256 = "f7fee59fdbf1286ec23ef0b35b2dfb41031c8727c90ced6435b8cf576f23a656"
+  VERSION = "0.12.0"
+  LIBRDKAFKA_VERSION = "1.9.0"
+  LIBRDKAFKA_SOURCE_SHA256 = "59b6088b69ca6cf278c3f9de5cd6b7f3fd604212cd1c59870bc531c54147e889"
 end

data/lib/rdkafka.rb

@@ -17,5 +17,6 @@ require "rdkafka/consumer/topic_partition_list"
 require "rdkafka/error"
 require "rdkafka/metadata"
 require "rdkafka/producer"
+require "rdkafka/producer/client"
 require "rdkafka/producer/delivery_handle"
 require "rdkafka/producer/delivery_report"

data/rdkafka.gemspec

@@ -14,15 +14,17 @@ Gem::Specification.new do |gem|
   gem.name = 'rdkafka'
   gem.require_paths = ['lib']
   gem.version = Rdkafka::VERSION
-  gem.required_ruby_version = '>= 2.4'
+  gem.required_ruby_version = '>= 2.6'
   gem.extensions = %w(ext/Rakefile)
 
-  gem.add_dependency 'ffi', '~> 1.9'
-  gem.add_dependency 'mini_portile2', '~> 2.1'
-  gem.add_dependency 'rake', '>= 12.3'
+  gem.add_dependency 'ffi', '~> 1.15'
+  gem.add_dependency 'mini_portile2', '~> 2.6'
+  gem.add_dependency 'rake', '> 12'
 
-  gem.add_development_dependency 'pry', '~> 0.10'
+  gem.add_development_dependency 'pry'
   gem.add_development_dependency 'rspec', '~> 3.5'
-  gem.add_development_dependency 'rake', '~> 12.0'
-  gem.add_development_dependency 'simplecov', '~> 0.15'
+  gem.add_development_dependency 'rake'
+  gem.add_development_dependency 'simplecov'
+  gem.add_development_dependency 'guard'
+  gem.add_development_dependency 'guard-rspec'
 end

data/spec/rdkafka/abstract_handle_spec.rb

@@ -111,4 +111,3 @@ describe Rdkafka::AbstractHandle do
     end
   end
 end
-

data/spec/rdkafka/admin_spec.rb

@@ -52,7 +52,7 @@ describe Rdkafka::Admin do
       end
 
       describe "with an invalid partition count" do
-        let(:topic_partition_count) { -1 }
+        let(:topic_partition_count) { -999 }
 
         it "raises an exception" do
           expect {

data/spec/rdkafka/bindings_spec.rb

@@ -76,6 +76,13 @@ describe Rdkafka::Bindings do
       result_2 = (Zlib.crc32(partition_key) % partition_count)
       expect(result_1).to eq(result_2)
     end
+
+    it "should return the partition calculated by the specified partitioner" do
+      result_1 = Rdkafka::Bindings.partitioner(partition_key, partition_count, "murmur2")
+      ptr = FFI::MemoryPointer.from_string(partition_key)
+      result_2 = Rdkafka::Bindings.rd_kafka_msg_partitioner_murmur2(nil, ptr, partition_key.size, partition_count, nil, nil)
+      expect(result_1).to eq(result_2)
+    end
   end
 
   describe "stats callback" do

data/spec/rdkafka/config_spec.rb

@@ -108,7 +108,7 @@ describe Rdkafka::Config do
     end
 
     it "should create a consumer with valid config" do
-      consumer = rdkafka_config.consumer
+      consumer = rdkafka_consumer_config.consumer
       expect(consumer).to be_a Rdkafka::Consumer
       consumer.close
     end
@@ -136,7 +136,7 @@ describe Rdkafka::Config do
     end
 
     it "should create a producer with valid config" do
-      producer = rdkafka_config.producer
+      producer = rdkafka_consumer_config.producer
       expect(producer).to be_a Rdkafka::Producer
       producer.close
     end