rdkafka 0.7.0 → 0.8.0.beta.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 38ccc62319eff24c7def2b8cebced2a8551b28a1651e77348c868ce189a7d418
-  data.tar.gz: e23e2cad27a95e2bf51c2428e4d2246deae903f8e0d7800b086b35c5b262fc65
+  metadata.gz: 8fd9f55f469403bfcec14203f75fa0bb61ebef1a1aaad9f0a4f58b11f02b16cb
+  data.tar.gz: e0e07afcaca58f06ae35d5eaad8074ab725d90dc5be8393cb5faf37148f52f91
 SHA512:
-  metadata.gz: 5faa270613e25d64f2b5d955a637b97b8315e7e365843b0e5a10d34c6340a795ebcf8ed2cb9dba2ae3a7d6839e464271d979760d351a6a8777224c31cdb262b6
-  data.tar.gz: d209464e6fe089945638063caa4ec6ab80d7c0d171678c43e320314cdc2ff0f980806b92941d0529b8ec93d598e5a637673fe3d97df4755503d2b484cc8f670b
+  metadata.gz: 84509541f5c16afc20cc89720a0d784ae7b89d49b1b72eab160b31c600e3c9fc167bee43bc123782cca002cad06aa370fc902838514411eab9dd4b40b32a7e4f
+  data.tar.gz: 8fe627c353b195df6424c3979816eabca7e6ce1dbec2f7f1edc92d7d3f111346fb6935620cf0c2ec5815c2bcd39c5b16e2618cd80b84b346fa024fab35842cf3

data/.travis.yml

@@ -14,9 +14,21 @@ rvm:
   - 2.4
   - 2.5
   - 2.6
+  - jruby-9.2.9.0
 
 before_install:
   - gem update --system
+  - |
+    r_eng="$(ruby -e 'STDOUT.write RUBY_ENGINE')";
+    if [ "$r_eng" == "jruby" ]; then
+      sudo apt-get update && \
+      sudo apt-get install -y git && \
+      sudo apt-get install -y libpthread-stubs0-dev && \
+      sudo apt-get install -y build-essential && \
+      sudo apt-get install -y zlib1g-dev && \
+      sudo apt-get install -y libssl-dev && \
+      sudo apt-get install -y libsasl2-dev
+    fi
 
 before_script:
   - docker-compose up -d
@@ -26,7 +38,7 @@ before_script:
   - ./cc-test-reporter before-build
 
 script:
-  - bundle exec rspec
+  - bundle exec rspec --format documentation
 
 after_script:
   - docker-compose stop

data/CHANGELOG.md

@@ -1,6 +1,14 @@
-# 0.6.0
+# 0.8.0
+* Upgrade librdkafka to 1.4.0
+* Integrate librdkafka metadata API and add partition_key (by Adithya-copart)
+* Ruby 2.7 compatibility fix (by Geoff Thé)A
+* Add error to delivery report (by Alex Stanovsky)
+* Don't override CPPFLAGS and LDFLAGS if already set on Mac (by Hiroshi Hatake)
+* Allow use of Rake 13.x and up (by Tomasz Pajor)
+
+# 0.7.0
 * Bump librdkafka to 1.2.0 (by rob-as)
-* Allow customizing the wait time for delivery report availability (by mensfeld )
+* Allow customizing the wait time for delivery report availability (by mensfeld)
 
 # 0.6.0
 * Bump librdkafka to 1.1.0 (by Chris Gaffney)

data/docker-compose.yml

@@ -1,18 +1,22 @@
+
 version: '2'
 services:
   zookeeper:
-    image: wurstmeister/zookeeper
-    ports:
-      - "2181:2181"
+    image: confluentinc/cp-zookeeper:latest
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+      ZOOKEEPER_TICK_TIME: 2000
+
   kafka:
-    image: wurstmeister/kafka:1.0.1
+    image: confluentinc/cp-kafka:latest
+    depends_on:
+      - zookeeper
     ports:
-      - "9092:9092"
+      - 9092:9092
     environment:
-      KAFKA_ADVERTISED_HOST_NAME: localhost
-      KAFKA_ADVERTISED_PORT: 9092
+      KAFKA_BROKER_ID: 1
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
-      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'
-      KAFKA_CREATE_TOPICS: "consume_test_topic:3:1,empty_test_topic:3:1,load_test_topic:3:1,produce_test_topic:3:1,rake_test_topic:3:1,watermarks_test_topic:3:1"
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1

data/ext/README.md

@@ -5,19 +5,7 @@ this gem is installed.
 
 To update the `librdkafka` version follow the following steps:
 
-* Download the new version `tar.gz` from  https://github.com/edenhill/librdkafka/
-* Generate a `sha256` with (if using MacOS) `shasum -a 256 <file>`
-* Change the `sha256` in `lib/rdkafka/version.rb`
+* Go to https://github.com/edenhill/librdkafka/releases to get the new
+  version number and asset checksum for `tar.gz`.
 * Change the version in `lib/rdkafka/version.rb`
-
-## Disclaimer
-
-Currently the `librdkafka` project does not provide
-checksums of releases. The checksum provided here is generated on a best
-effort basis. If the CDN would be compromised at the time of download the
-checksum could be incorrect.
-
-Do your own verification if you rely on this behaviour.
-
-Once https://github.com/appsignal/rdkafka-ruby/issues/44 is implemented
-we will change this process.
+* Change the `sha256` in `lib/rdkafka/version.rb`

data/ext/Rakefile

@@ -23,9 +23,10 @@ task :default => :clean do
   recipe = MiniPortile.new("librdkafka", Rdkafka::LIBRDKAFKA_VERSION)
 
   # Use default homebrew openssl if we're on mac and the directory exists
-  if recipe.host.include?("darwin") && Dir.exists?("/usr/local/opt/openssl")
-    ENV["CPPFLAGS"] = "-I/usr/local/opt/openssl/include"
-    ENV["LDFLAGS"] = "-L/usr/local/opt/openssl/lib"
+  # and each of flags is not empty
+  if recipe.host&.include?("darwin") && Dir.exist?("/usr/local/opt/openssl")
+    ENV["CPPFLAGS"] = "-I/usr/local/opt/openssl/include" unless ENV["CPPFLAGS"]
+    ENV["LDFLAGS"] = "-L/usr/local/opt/openssl/lib" unless ENV["LDFLAGS"]
   end
 
   recipe.files << {
@@ -55,3 +56,22 @@ task :clean do
   FileUtils.rm_rf File.join(File.dirname(__FILE__), "ports")
   FileUtils.rm_rf File.join(File.dirname(__FILE__), "tmp")
 end
+
+namespace :build do
+  desc "Build librdkafka at the given git sha or tag"
+  task :git, [:ref] do |task, args|
+    ref = args[:ref]
+    version = "git-#{ref}"
+
+    recipe = MiniPortile.new("librdkafka", version)
+    recipe.files << "https://github.com/edenhill/librdkafka/archive/#{ref}.tar.gz"
+    recipe.configure_options = ["--host=#{recipe.host}"]
+    recipe.cook
+
+    ext = recipe.host.include?("darwin") ? "dylib" : "so"
+    lib = File.expand_path("ports/#{recipe.host}/librdkafka/#{version}/lib/librdkafka.#{ext}", __dir__)
+
+    # Copy will copy the content, following any symlinks
+    FileUtils.cp(lib, __dir__)
+  end
+end

data/lib/rdkafka.rb

@@ -8,6 +8,7 @@ require "rdkafka/consumer/message"
 require "rdkafka/consumer/partition"
 require "rdkafka/consumer/topic_partition_list"
 require "rdkafka/error"
+require "rdkafka/metadata"
 require "rdkafka/producer"
 require "rdkafka/producer/delivery_handle"
 require "rdkafka/producer/delivery_report"

data/lib/rdkafka/bindings.rb

@@ -8,7 +8,7 @@ module Rdkafka
     extend FFI::Library
 
     def self.lib_extension
-      if Gem::Platform.local.os.include?("darwin")
+      if RbConfig::CONFIG['host_os'] =~ /darwin/
         'dylib'
       else
         'so'
@@ -22,6 +22,11 @@ module Rdkafka
     RD_KAFKA_RESP_ERR__NOENT = -156
     RD_KAFKA_RESP_ERR_NO_ERROR = 0
 
+    RD_KAFKA_OFFSET_END       = -1
+    RD_KAFKA_OFFSET_BEGINNING = -2
+    RD_KAFKA_OFFSET_STORED    = -1000
+    RD_KAFKA_OFFSET_INVALID   = -1001
+
     class SizePtr < FFI::Struct
       layout :value, :size_t
     end
@@ -35,6 +40,8 @@ module Rdkafka
 
     attach_function :rd_kafka_memberid, [:pointer], :string
     attach_function :rd_kafka_clusterid, [:pointer], :string
+    attach_function :rd_kafka_metadata, [:pointer, :int, :pointer, :pointer, :int], :int
+    attach_function :rd_kafka_metadata_destroy, [:pointer], :void
 
     # Message struct
 
@@ -227,6 +234,17 @@ module Rdkafka
     callback :delivery_cb, [:pointer, :pointer, :pointer], :void
     attach_function :rd_kafka_conf_set_dr_msg_cb, [:pointer, :delivery_cb], :void
 
+    # Partitioner
+    attach_function :rd_kafka_msg_partitioner_consistent_random, [:pointer, :pointer, :size_t, :int32, :pointer, :pointer], :int32
+
+    def self.partitioner(str, partition_count)
+      # Return RD_KAFKA_PARTITION_UA(unassigned partition) when partition count is nil/zero.
+      return -1 unless partition_count&.nonzero?
+
+      str_ptr = FFI::MemoryPointer.from_string(str)
+      rd_kafka_msg_partitioner_consistent_random(nil, str_ptr, str.size, partition_count, nil, nil)
+    end
+
     DeliveryCallback = FFI::Function.new(
       :void, [:pointer, :pointer, :pointer]
     ) do |client_ptr, message_ptr, opaque_ptr|
@@ -240,7 +258,7 @@ module Rdkafka
         delivery_handle[:offset] = message[:offset]
         # Call delivery callback on opaque
         if opaque = Rdkafka::Config.opaques[opaque_ptr.to_i]
-          opaque.call_delivery_callback(Rdkafka::Producer::DeliveryReport.new(message[:partition], message[:offset]))
+          opaque.call_delivery_callback(Rdkafka::Producer::DeliveryReport.new(message[:partition], message[:offset], message[:err]))
         end
       end
     end

data/lib/rdkafka/config.rb

@@ -212,10 +212,8 @@ module Rdkafka
         Rdkafka::Bindings.rd_kafka_queue_get_main(handle)
       )
 
-      FFI::AutoPointer.new(
-        handle,
-        Rdkafka::Bindings.method(:rd_kafka_destroy)
-      )
+      # Return handle which should be closed using rd_kafka_destroy after usage.
+      handle
     end
   end
 

data/lib/rdkafka/consumer.rb

@@ -5,6 +5,9 @@ module Rdkafka
   #
   # To create a consumer set up a {Config} and call {Config#consumer consumer} on that. It is
   # mandatory to set `:"group.id"` in the configuration.
+  #
+  # Consumer implements `Enumerable`, so you can use `each` to consume messages, or for example
+  # `each_slice` to consume batches of messages.
   class Consumer
     include Enumerable
 
@@ -17,8 +20,12 @@ module Rdkafka
     # Close this consumer
     # @return [nil]
     def close
+      return unless @native_kafka
+
       @closing = true
       Rdkafka::Bindings.rd_kafka_consumer_close(@native_kafka)
+      Rdkafka::Bindings.rd_kafka_destroy(@native_kafka)
+      @native_kafka = nil
     end
 
     # Subscribe to one or more topics letting Kafka handle partition assignments.
@@ -30,20 +37,19 @@ module Rdkafka
     # @return [nil]
     def subscribe(*topics)
       # Create topic partition list with topics and no partition set
-      tpl = TopicPartitionList.new_native_tpl(topics.length)
+      tpl = Rdkafka::Bindings.rd_kafka_topic_partition_list_new(topics.length)
 
       topics.each do |topic|
-        Rdkafka::Bindings.rd_kafka_topic_partition_list_add(
-          tpl,
-          topic,
-          -1
-        )
+        Rdkafka::Bindings.rd_kafka_topic_partition_list_add(tpl, topic, -1)
       end
+
       # Subscribe to topic partition list and check this was successful
       response = Rdkafka::Bindings.rd_kafka_subscribe(@native_kafka, tpl)
       if response != 0
         raise Rdkafka::RdkafkaError.new(response, "Error subscribing to '#{topics.join(', ')}'")
       end
+    ensure
+      Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
     end
 
     # Unsubscribe from all subscribed topics.
@@ -69,12 +75,18 @@ module Rdkafka
       unless list.is_a?(TopicPartitionList)
         raise TypeError.new("list has to be a TopicPartitionList")
       end
+
       tpl = list.to_native_tpl
-      response = Rdkafka::Bindings.rd_kafka_pause_partitions(@native_kafka, tpl)
 
-      if response != 0
-        list = TopicPartitionList.from_native_tpl(tpl)
-        raise Rdkafka::RdkafkaTopicPartitionListError.new(response, list, "Error pausing '#{list.to_h}'")
+      begin
+        response = Rdkafka::Bindings.rd_kafka_pause_partitions(@native_kafka, tpl)
+
+        if response != 0
+          list = TopicPartitionList.from_native_tpl(tpl)
+          raise Rdkafka::RdkafkaTopicPartitionListError.new(response, list, "Error pausing '#{list.to_h}'")
+        end
+      ensure
+        Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
       end
     end
 
@@ -89,10 +101,16 @@ module Rdkafka
       unless list.is_a?(TopicPartitionList)
         raise TypeError.new("list has to be a TopicPartitionList")
       end
+
       tpl = list.to_native_tpl
-      response = Rdkafka::Bindings.rd_kafka_resume_partitions(@native_kafka, tpl)
-      if response != 0
-        raise Rdkafka::RdkafkaError.new(response, "Error resume '#{list.to_h}'")
+
+      begin
+        response = Rdkafka::Bindings.rd_kafka_resume_partitions(@native_kafka, tpl)
+        if response != 0
+          raise Rdkafka::RdkafkaError.new(response, "Error resume '#{list.to_h}'")
+        end
+      ensure
+        Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
       end
     end
 
@@ -102,17 +120,19 @@ module Rdkafka
     #
     # @return [TopicPartitionList]
     def subscription
-      tpl = FFI::MemoryPointer.new(:pointer)
-      response = Rdkafka::Bindings.rd_kafka_subscription(@native_kafka, tpl)
+      ptr = FFI::MemoryPointer.new(:pointer)
+      response = Rdkafka::Bindings.rd_kafka_subscription(@native_kafka, ptr)
+
       if response != 0
         raise Rdkafka::RdkafkaError.new(response)
       end
-      tpl = tpl.read(:pointer).tap { |it| it.autorelease = false }
+
+      native = ptr.read_pointer
 
       begin
-        Rdkafka::Consumer::TopicPartitionList.from_native_tpl(tpl)
+        Rdkafka::Consumer::TopicPartitionList.from_native_tpl(native)
       ensure
-        Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
+        Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(native)
       end
     end
 
@@ -125,10 +145,16 @@ module Rdkafka
       unless list.is_a?(TopicPartitionList)
         raise TypeError.new("list has to be a TopicPartitionList")
       end
+
       tpl = list.to_native_tpl
-      response = Rdkafka::Bindings.rd_kafka_assign(@native_kafka, tpl)
-      if response != 0
-        raise Rdkafka::RdkafkaError.new(response, "Error assigning '#{list.to_h}'")
+
+      begin
+        response = Rdkafka::Bindings.rd_kafka_assign(@native_kafka, tpl)
+        if response != 0
+          raise Rdkafka::RdkafkaError.new(response, "Error assigning '#{list.to_h}'")
+        end
+      ensure
+        Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
       end
     end
 
@@ -138,19 +164,23 @@ module Rdkafka
     #
     # @return [TopicPartitionList]
     def assignment
-      tpl = FFI::MemoryPointer.new(:pointer)
-      response = Rdkafka::Bindings.rd_kafka_assignment(@native_kafka, tpl)
+      ptr = FFI::MemoryPointer.new(:pointer)
+      response = Rdkafka::Bindings.rd_kafka_assignment(@native_kafka, ptr)
       if response != 0
         raise Rdkafka::RdkafkaError.new(response)
       end
 
-      tpl = tpl.read(:pointer).tap { |it| it.autorelease = false  }
+      tpl = ptr.read_pointer
 
-      begin
-        Rdkafka::Consumer::TopicPartitionList.from_native_tpl(tpl)
-      ensure
-        Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy tpl
+      if !tpl.null?
+        begin
+          Rdkafka::Consumer::TopicPartitionList.from_native_tpl(tpl)
+        ensure
+          Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy tpl
+        end
       end
+    ensure
+      ptr.free
     end
 
     # Return the current committed offset per partition for this consumer group.
@@ -168,12 +198,18 @@ module Rdkafka
       elsif !list.is_a?(TopicPartitionList)
         raise TypeError.new("list has to be nil or a TopicPartitionList")
       end
+
       tpl = list.to_native_tpl
-      response = Rdkafka::Bindings.rd_kafka_committed(@native_kafka, tpl, timeout_ms)
-      if response != 0
-        raise Rdkafka::RdkafkaError.new(response)
+
+      begin
+        response = Rdkafka::Bindings.rd_kafka_committed(@native_kafka, tpl, timeout_ms)
+        if response != 0
+          raise Rdkafka::RdkafkaError.new(response)
+        end
+        TopicPartitionList.from_native_tpl(tpl)
+      ensure
+        Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
       end
-      TopicPartitionList.from_native_tpl(tpl)
     end
 
     # Query broker for low (oldest/beginning) and high (newest/end) offsets for a partition.
@@ -195,13 +231,16 @@ module Rdkafka
         partition,
         low,
         high,
-        timeout_ms
+        timeout_ms,
       )
       if response != 0
         raise Rdkafka::RdkafkaError.new(response, "Error querying watermark offsets for partition #{partition} of #{topic}")
       end
 
-      return low.read_int64, high.read_int64
+      return low.read_array_of_int64(1).first, high.read_array_of_int64(1).first
+    ensure
+      low.free
+      high.free
     end
 
     # Calculate the consumer lag per partition for the provided topic partition list.
@@ -217,6 +256,7 @@ module Rdkafka
     # @return [Hash<String, Hash<Integer, Integer>>] A hash containing all topics with the lag per partition
     def lag(topic_partition_list, watermark_timeout_ms=100)
       out = {}
+
       topic_partition_list.to_h.each do |topic, partitions|
         # Query high watermarks for this topic's partitions
         # and compare to the offset in the list.
@@ -313,7 +353,14 @@ module Rdkafka
       end
     end
 
-    # Commit the current offsets of this consumer
+    # Manually commit the current offsets of this consumer.
+    #
+    # To use this set `enable.auto.commit`to `false` to disable automatic triggering
+    # of commits.
+    #
+    # If `enable.auto.offset.store` is set to `true` the offset of the last consumed
+    # message for every partition is used. If set to `false` you can use {store_offset} to
+    # indicate when a message has been fully processed.
     #
     # @param list [TopicPartitionList,nil] The topic with partitions to commit
     # @param async [Boolean] Whether to commit async or wait for the commit to finish
@@ -325,14 +372,16 @@ module Rdkafka
       if !list.nil? && !list.is_a?(TopicPartitionList)
         raise TypeError.new("list has to be nil or a TopicPartitionList")
       end
-      tpl = if list
-              list.to_native_tpl
-            else
-              nil
-            end
-      response = Rdkafka::Bindings.rd_kafka_commit(@native_kafka, tpl, async)
-      if response != 0
-        raise Rdkafka::RdkafkaError.new(response)
+
+      tpl = list ? list.to_native_tpl : nil
+
+      begin
+        response = Rdkafka::Bindings.rd_kafka_commit(@native_kafka, tpl, async)
+        if response != 0
+          raise Rdkafka::RdkafkaError.new(response)
+        end
+      ensure
+        Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl) if tpl
       end
     end
 
@@ -344,6 +393,8 @@ module Rdkafka
     #
     # @return [Message, nil] A message or nil if there was no new message within the timeout
     def poll(timeout_ms)
+      return unless @native_kafka
+
       message_ptr = Rdkafka::Bindings.rd_kafka_consumer_poll(@native_kafka, timeout_ms)
       if message_ptr.null?
         nil