rdkafka 0.7.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 38ccc62319eff24c7def2b8cebced2a8551b28a1651e77348c868ce189a7d418
-  data.tar.gz: e23e2cad27a95e2bf51c2428e4d2246deae903f8e0d7800b086b35c5b262fc65
+  metadata.gz: 8bf97d10412b4f3c0801f657796fd05c94ceede3caafe6a45719c840b534752a
+  data.tar.gz: 8cc93bffb8119cf9c97aa60cf1f6d634e66a99686e0785150ae51283a6514e8c
 SHA512:
-  metadata.gz: 5faa270613e25d64f2b5d955a637b97b8315e7e365843b0e5a10d34c6340a795ebcf8ed2cb9dba2ae3a7d6839e464271d979760d351a6a8777224c31cdb262b6
-  data.tar.gz: d209464e6fe089945638063caa4ec6ab80d7c0d171678c43e320314cdc2ff0f980806b92941d0529b8ec93d598e5a637673fe3d97df4755503d2b484cc8f670b
+  metadata.gz: e8e085b3dd9d80d77003d2c7f0fbda69371c233b121eac2594556d7d62da99a4bed4aace93e9489783c789e64bafd732faaad3de2ffb5d3a5af8568c5b43676b
+  data.tar.gz: 5a267a9814e87e876544e04e6657829c298ff01b03cb5ff41a0205795ea2fb4f6a2147ee1a0949e52f7a069f5dc73d5ed2072867194032e8ea913ac5d6871e23

data/.travis.yml

@@ -14,9 +14,24 @@ rvm:
   - 2.4
   - 2.5
   - 2.6
+  - 2.7.1
+  - jruby-9.2.9.0
+  - jruby-9.2.10.0
+  - jruby-9.2.11.1
 
 before_install:
   - gem update --system
+  - |
+    r_eng="$(ruby -e 'STDOUT.write RUBY_ENGINE')";
+    if [ "$r_eng" == "jruby" ]; then
+      sudo apt-get update && \
+      sudo apt-get install -y git && \
+      sudo apt-get install -y libpthread-stubs0-dev && \
+      sudo apt-get install -y build-essential && \
+      sudo apt-get install -y zlib1g-dev && \
+      sudo apt-get install -y libssl-dev && \
+      sudo apt-get install -y libsasl2-dev
+    fi
 
 before_script:
   - docker-compose up -d
@@ -26,7 +41,7 @@ before_script:
   - ./cc-test-reporter before-build
 
 script:
-  - bundle exec rspec
+  - bundle exec rspec --format documentation
 
 after_script:
   - docker-compose stop

data/CHANGELOG.md

@@ -1,6 +1,14 @@
-# 0.6.0
+# 0.8.0
+* Upgrade librdkafka to 1.4.0
+* Integrate librdkafka metadata API and add partition_key (by Adithya-copart)
+* Ruby 2.7 compatibility fix (by Geoff Thé)A
+* Add error to delivery report (by Alex Stanovsky)
+* Don't override CPPFLAGS and LDFLAGS if already set on Mac (by Hiroshi Hatake)
+* Allow use of Rake 13.x and up (by Tomasz Pajor)
+
+# 0.7.0
 * Bump librdkafka to 1.2.0 (by rob-as)
-* Allow customizing the wait time for delivery report availability (by mensfeld )
+* Allow customizing the wait time for delivery report availability (by mensfeld)
 
 # 0.6.0
 * Bump librdkafka to 1.1.0 (by Chris Gaffney)

data/docker-compose.yml

@@ -1,18 +1,22 @@
+
 version: '2'
 services:
   zookeeper:
-    image: wurstmeister/zookeeper
-    ports:
-      - "2181:2181"
+    image: confluentinc/cp-zookeeper:latest
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+      ZOOKEEPER_TICK_TIME: 2000
+
   kafka:
-    image: wurstmeister/kafka:1.0.1
+    image: confluentinc/cp-kafka:latest
+    depends_on:
+      - zookeeper
     ports:
-      - "9092:9092"
+      - 9092:9092
     environment:
-      KAFKA_ADVERTISED_HOST_NAME: localhost
-      KAFKA_ADVERTISED_PORT: 9092
+      KAFKA_BROKER_ID: 1
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
-      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'
-      KAFKA_CREATE_TOPICS: "consume_test_topic:3:1,empty_test_topic:3:1,load_test_topic:3:1,produce_test_topic:3:1,rake_test_topic:3:1,watermarks_test_topic:3:1"
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1

data/ext/README.md

@@ -5,19 +5,7 @@ this gem is installed.
 
 To update the `librdkafka` version follow the following steps:
 
-* Download the new version `tar.gz` from  https://github.com/edenhill/librdkafka/
-* Generate a `sha256` with (if using MacOS) `shasum -a 256 <file>`
-* Change the `sha256` in `lib/rdkafka/version.rb`
+* Go to https://github.com/edenhill/librdkafka/releases to get the new
+  version number and asset checksum for `tar.gz`.
 * Change the version in `lib/rdkafka/version.rb`
-
-## Disclaimer
-
-Currently the `librdkafka` project does not provide
-checksums of releases. The checksum provided here is generated on a best
-effort basis. If the CDN would be compromised at the time of download the
-checksum could be incorrect.
-
-Do your own verification if you rely on this behaviour.
-
-Once https://github.com/appsignal/rdkafka-ruby/issues/44 is implemented
-we will change this process.
+* Change the `sha256` in `lib/rdkafka/version.rb`

data/ext/Rakefile

@@ -23,9 +23,10 @@ task :default => :clean do
   recipe = MiniPortile.new("librdkafka", Rdkafka::LIBRDKAFKA_VERSION)
 
   # Use default homebrew openssl if we're on mac and the directory exists
-  if recipe.host.include?("darwin") && Dir.exists?("/usr/local/opt/openssl")
-    ENV["CPPFLAGS"] = "-I/usr/local/opt/openssl/include"
-    ENV["LDFLAGS"] = "-L/usr/local/opt/openssl/lib"
+  # and each of flags is not empty
+  if recipe.host&.include?("darwin") && Dir.exist?("/usr/local/opt/openssl")
+    ENV["CPPFLAGS"] = "-I/usr/local/opt/openssl/include" unless ENV["CPPFLAGS"]
+    ENV["LDFLAGS"] = "-L/usr/local/opt/openssl/lib" unless ENV["LDFLAGS"]
   end
 
   recipe.files << {
@@ -55,3 +56,22 @@ task :clean do
   FileUtils.rm_rf File.join(File.dirname(__FILE__), "ports")
   FileUtils.rm_rf File.join(File.dirname(__FILE__), "tmp")
 end
+
+namespace :build do
+  desc "Build librdkafka at the given git sha or tag"
+  task :git, [:ref] do |task, args|
+    ref = args[:ref]
+    version = "git-#{ref}"
+
+    recipe = MiniPortile.new("librdkafka", version)
+    recipe.files << "https://github.com/edenhill/librdkafka/archive/#{ref}.tar.gz"
+    recipe.configure_options = ["--host=#{recipe.host}"]
+    recipe.cook
+
+    ext = recipe.host.include?("darwin") ? "dylib" : "so"
+    lib = File.expand_path("ports/#{recipe.host}/librdkafka/#{version}/lib/librdkafka.#{ext}", __dir__)
+
+    # Copy will copy the content, following any symlinks
+    FileUtils.cp(lib, __dir__)
+  end
+end

data/lib/rdkafka/bindings.rb

@@ -8,7 +8,7 @@ module Rdkafka
     extend FFI::Library
 
     def self.lib_extension
-      if Gem::Platform.local.os.include?("darwin")
+      if RbConfig::CONFIG['host_os'] =~ /darwin/
         'dylib'
       else
         'so'
@@ -22,6 +22,11 @@ module Rdkafka
     RD_KAFKA_RESP_ERR__NOENT = -156
     RD_KAFKA_RESP_ERR_NO_ERROR = 0
 
+    RD_KAFKA_OFFSET_END       = -1
+    RD_KAFKA_OFFSET_BEGINNING = -2
+    RD_KAFKA_OFFSET_STORED    = -1000
+    RD_KAFKA_OFFSET_INVALID   = -1001
+
     class SizePtr < FFI::Struct
       layout :value, :size_t
     end
@@ -35,6 +40,8 @@ module Rdkafka
 
     attach_function :rd_kafka_memberid, [:pointer], :string
     attach_function :rd_kafka_clusterid, [:pointer], :string
+    attach_function :rd_kafka_metadata, [:pointer, :int, :pointer, :pointer, :int], :int
+    attach_function :rd_kafka_metadata_destroy, [:pointer], :void
 
     # Message struct
 
@@ -227,6 +234,17 @@ module Rdkafka
     callback :delivery_cb, [:pointer, :pointer, :pointer], :void
     attach_function :rd_kafka_conf_set_dr_msg_cb, [:pointer, :delivery_cb], :void
 
+    # Partitioner
+    attach_function :rd_kafka_msg_partitioner_consistent_random, [:pointer, :pointer, :size_t, :int32, :pointer, :pointer], :int32
+
+    def self.partitioner(str, partition_count)
+      # Return RD_KAFKA_PARTITION_UA(unassigned partition) when partition count is nil/zero.
+      return -1 unless partition_count&.nonzero?
+
+      str_ptr = FFI::MemoryPointer.from_string(str)
+      rd_kafka_msg_partitioner_consistent_random(nil, str_ptr, str.size, partition_count, nil, nil)
+    end
+
     DeliveryCallback = FFI::Function.new(
       :void, [:pointer, :pointer, :pointer]
     ) do |client_ptr, message_ptr, opaque_ptr|
@@ -240,7 +258,7 @@ module Rdkafka
         delivery_handle[:offset] = message[:offset]
         # Call delivery callback on opaque
         if opaque = Rdkafka::Config.opaques[opaque_ptr.to_i]
-          opaque.call_delivery_callback(Rdkafka::Producer::DeliveryReport.new(message[:partition], message[:offset]))
+          opaque.call_delivery_callback(Rdkafka::Producer::DeliveryReport.new(message[:partition], message[:offset], message[:err]))
         end
       end
     end

data/lib/rdkafka/config.rb

@@ -212,10 +212,8 @@ module Rdkafka
         Rdkafka::Bindings.rd_kafka_queue_get_main(handle)
       )
 
-      FFI::AutoPointer.new(
-        handle,
-        Rdkafka::Bindings.method(:rd_kafka_destroy)
-      )
+      # Return handle which should be closed using rd_kafka_destroy after usage.
+      handle
     end
   end
 

data/lib/rdkafka/consumer/headers.rb

@@ -19,7 +19,7 @@ module Rdkafka
           raise Rdkafka::RdkafkaError.new(err, "Error reading message headers")
         end
 
-        headers_ptr = headers_ptrptr.read(:pointer).tap { |it| it.autorelease = false }
+        headers_ptr = headers_ptrptr.read_pointer
 
         name_ptrptr = FFI::MemoryPointer.new(:pointer)
         value_ptrptr = FFI::MemoryPointer.new(:pointer)
@@ -42,12 +42,14 @@ module Rdkafka
             raise Rdkafka::RdkafkaError.new(err, "Error reading a message header at index #{idx}")
           end
 
-          name = name_ptrptr.read(:pointer).tap { |it| it.autorelease = false }
-          name = name.read_string_to_null
+          name_ptr = name_ptrptr.read_pointer
+          name = name_ptr.respond_to?(:read_string_to_null) ? name_ptr.read_string_to_null : name_ptr.read_string
 
           size = size_ptr[:value]
-          value = value_ptrptr.read(:pointer).tap { |it| it.autorelease = false }
-          value = value.read_string(size)
+
+          value_ptr = value_ptrptr.read_pointer
+
+          value = value_ptr.read_string(size)
 
           headers[name.to_sym] = value
 

data/lib/rdkafka/consumer/topic_partition_list.rb

@@ -106,7 +106,7 @@ module Rdkafka
             data[elem[:topic]] = nil
           else
             partitions = data[elem[:topic]] || []
-            offset = if elem[:offset] == -1001
+            offset = if elem[:offset] == Rdkafka::Bindings::RD_KAFKA_OFFSET_INVALID
                        nil
                      else
                        elem[:offset]
@@ -125,10 +125,10 @@ module Rdkafka
       #
       # The pointer will be cleaned by `rd_kafka_topic_partition_list_destroy` when GC releases it.
       #
-      # @return [FFI::AutoPointer]
+      # @return [FFI::Pointer]
       # @private
       def to_native_tpl
-        tpl = TopicPartitionList.new_native_tpl(count)
+        tpl = Rdkafka::Bindings.rd_kafka_topic_partition_list_new(count)
 
         @data.each do |topic, partitions|
           if partitions
@@ -138,6 +138,7 @@ module Rdkafka
                 topic,
                 p.partition
               )
+
               if p.offset
                 Rdkafka::Bindings.rd_kafka_topic_partition_list_set_offset(
                   tpl,
@@ -158,17 +159,6 @@ module Rdkafka
 
         tpl
       end
-
-      # Creates a new native tpl and wraps it into FFI::AutoPointer which in turn calls
-      # `rd_kafka_topic_partition_list_destroy` when a pointer will be cleaned by GC
-      #
-      # @param count [Integer] an initial capacity of partitions list
-      # @return [FFI::AutoPointer]
-      # @private
-      def self.new_native_tpl(count)
-        tpl = Rdkafka::Bindings.rd_kafka_topic_partition_list_new(count)
-        FFI::AutoPointer.new(tpl, Rdkafka::Bindings.method(:rd_kafka_topic_partition_list_destroy))
-      end
     end
   end
 end

data/lib/rdkafka/consumer.rb

@@ -5,6 +5,9 @@ module Rdkafka
   #
   # To create a consumer set up a {Config} and call {Config#consumer consumer} on that. It is
   # mandatory to set `:"group.id"` in the configuration.
+  #
+  # Consumer implements `Enumerable`, so you can use `each` to consume messages, or for example
+  # `each_slice` to consume batches of messages.
   class Consumer
     include Enumerable
 
@@ -17,8 +20,12 @@ module Rdkafka
     # Close this consumer
     # @return [nil]
     def close
+      return unless @native_kafka
+
       @closing = true
       Rdkafka::Bindings.rd_kafka_consumer_close(@native_kafka)
+      Rdkafka::Bindings.rd_kafka_destroy(@native_kafka)
+      @native_kafka = nil
     end
 
     # Subscribe to one or more topics letting Kafka handle partition assignments.
@@ -30,20 +37,19 @@ module Rdkafka
     # @return [nil]
     def subscribe(*topics)
       # Create topic partition list with topics and no partition set
-      tpl = TopicPartitionList.new_native_tpl(topics.length)
+      tpl = Rdkafka::Bindings.rd_kafka_topic_partition_list_new(topics.length)
 
       topics.each do |topic|
-        Rdkafka::Bindings.rd_kafka_topic_partition_list_add(
-          tpl,
-          topic,
-          -1
-        )
+        Rdkafka::Bindings.rd_kafka_topic_partition_list_add(tpl, topic, -1)
       end
+
       # Subscribe to topic partition list and check this was successful
       response = Rdkafka::Bindings.rd_kafka_subscribe(@native_kafka, tpl)
       if response != 0
         raise Rdkafka::RdkafkaError.new(response, "Error subscribing to '#{topics.join(', ')}'")
       end
+    ensure
+      Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
     end
 
     # Unsubscribe from all subscribed topics.
@@ -69,12 +75,18 @@ module Rdkafka
       unless list.is_a?(TopicPartitionList)
         raise TypeError.new("list has to be a TopicPartitionList")
       end
+
       tpl = list.to_native_tpl
-      response = Rdkafka::Bindings.rd_kafka_pause_partitions(@native_kafka, tpl)
 
-      if response != 0
-        list = TopicPartitionList.from_native_tpl(tpl)
-        raise Rdkafka::RdkafkaTopicPartitionListError.new(response, list, "Error pausing '#{list.to_h}'")
+      begin
+        response = Rdkafka::Bindings.rd_kafka_pause_partitions(@native_kafka, tpl)
+
+        if response != 0
+          list = TopicPartitionList.from_native_tpl(tpl)
+          raise Rdkafka::RdkafkaTopicPartitionListError.new(response, list, "Error pausing '#{list.to_h}'")
+        end
+      ensure
+        Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
       end
     end
 
@@ -89,10 +101,16 @@ module Rdkafka
       unless list.is_a?(TopicPartitionList)
         raise TypeError.new("list has to be a TopicPartitionList")
       end
+
       tpl = list.to_native_tpl
-      response = Rdkafka::Bindings.rd_kafka_resume_partitions(@native_kafka, tpl)
-      if response != 0
-        raise Rdkafka::RdkafkaError.new(response, "Error resume '#{list.to_h}'")
+
+      begin
+        response = Rdkafka::Bindings.rd_kafka_resume_partitions(@native_kafka, tpl)
+        if response != 0
+          raise Rdkafka::RdkafkaError.new(response, "Error resume '#{list.to_h}'")
+        end
+      ensure
+        Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
       end
     end
 
@@ -102,17 +120,19 @@ module Rdkafka
     #
     # @return [TopicPartitionList]
     def subscription
-      tpl = FFI::MemoryPointer.new(:pointer)
-      response = Rdkafka::Bindings.rd_kafka_subscription(@native_kafka, tpl)
+      ptr = FFI::MemoryPointer.new(:pointer)
+      response = Rdkafka::Bindings.rd_kafka_subscription(@native_kafka, ptr)
+
       if response != 0
         raise Rdkafka::RdkafkaError.new(response)
       end
-      tpl = tpl.read(:pointer).tap { |it| it.autorelease = false }
+
+      native = ptr.read_pointer
 
       begin
-        Rdkafka::Consumer::TopicPartitionList.from_native_tpl(tpl)
+        Rdkafka::Consumer::TopicPartitionList.from_native_tpl(native)
       ensure
-        Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
+        Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(native)
       end
     end
 
@@ -125,10 +145,16 @@ module Rdkafka
       unless list.is_a?(TopicPartitionList)
         raise TypeError.new("list has to be a TopicPartitionList")
       end
+
       tpl = list.to_native_tpl
-      response = Rdkafka::Bindings.rd_kafka_assign(@native_kafka, tpl)
-      if response != 0
-        raise Rdkafka::RdkafkaError.new(response, "Error assigning '#{list.to_h}'")
+
+      begin
+        response = Rdkafka::Bindings.rd_kafka_assign(@native_kafka, tpl)
+        if response != 0
+          raise Rdkafka::RdkafkaError.new(response, "Error assigning '#{list.to_h}'")
+        end
+      ensure
+        Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
       end
     end
 
@@ -138,19 +164,23 @@ module Rdkafka
     #
     # @return [TopicPartitionList]
     def assignment
-      tpl = FFI::MemoryPointer.new(:pointer)
-      response = Rdkafka::Bindings.rd_kafka_assignment(@native_kafka, tpl)
+      ptr = FFI::MemoryPointer.new(:pointer)
+      response = Rdkafka::Bindings.rd_kafka_assignment(@native_kafka, ptr)
       if response != 0
         raise Rdkafka::RdkafkaError.new(response)
       end
 
-      tpl = tpl.read(:pointer).tap { |it| it.autorelease = false  }
+      tpl = ptr.read_pointer
 
-      begin
-        Rdkafka::Consumer::TopicPartitionList.from_native_tpl(tpl)
-      ensure
-        Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy tpl
+      if !tpl.null?
+        begin
+          Rdkafka::Consumer::TopicPartitionList.from_native_tpl(tpl)
+        ensure
+          Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy tpl
+        end
       end
+    ensure
+      ptr.free
     end
 
     # Return the current committed offset per partition for this consumer group.
@@ -168,12 +198,18 @@ module Rdkafka
       elsif !list.is_a?(TopicPartitionList)
         raise TypeError.new("list has to be nil or a TopicPartitionList")
       end
+
       tpl = list.to_native_tpl
-      response = Rdkafka::Bindings.rd_kafka_committed(@native_kafka, tpl, timeout_ms)
-      if response != 0
-        raise Rdkafka::RdkafkaError.new(response)
+
+      begin
+        response = Rdkafka::Bindings.rd_kafka_committed(@native_kafka, tpl, timeout_ms)
+        if response != 0
+          raise Rdkafka::RdkafkaError.new(response)
+        end
+        TopicPartitionList.from_native_tpl(tpl)
+      ensure
+        Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
       end
-      TopicPartitionList.from_native_tpl(tpl)
     end
 
     # Query broker for low (oldest/beginning) and high (newest/end) offsets for a partition.
@@ -195,13 +231,16 @@ module Rdkafka
         partition,
         low,
         high,
-        timeout_ms
+        timeout_ms,
       )
       if response != 0
         raise Rdkafka::RdkafkaError.new(response, "Error querying watermark offsets for partition #{partition} of #{topic}")
       end
 
-      return low.read_int64, high.read_int64
+      return low.read_array_of_int64(1).first, high.read_array_of_int64(1).first
+    ensure
+      low.free
+      high.free
     end
 
     # Calculate the consumer lag per partition for the provided topic partition list.
@@ -217,6 +256,7 @@ module Rdkafka
     # @return [Hash<String, Hash<Integer, Integer>>] A hash containing all topics with the lag per partition
     def lag(topic_partition_list, watermark_timeout_ms=100)
       out = {}
+
       topic_partition_list.to_h.each do |topic, partitions|
         # Query high watermarks for this topic's partitions
         # and compare to the offset in the list.
@@ -313,7 +353,14 @@ module Rdkafka
       end
     end
 
-    # Commit the current offsets of this consumer
+    # Manually commit the current offsets of this consumer.
+    #
+    # To use this set `enable.auto.commit`to `false` to disable automatic triggering
+    # of commits.
+    #
+    # If `enable.auto.offset.store` is set to `true` the offset of the last consumed
+    # message for every partition is used. If set to `false` you can use {store_offset} to
+    # indicate when a message has been fully processed.
     #
     # @param list [TopicPartitionList,nil] The topic with partitions to commit
     # @param async [Boolean] Whether to commit async or wait for the commit to finish
@@ -325,14 +372,16 @@ module Rdkafka
       if !list.nil? && !list.is_a?(TopicPartitionList)
         raise TypeError.new("list has to be nil or a TopicPartitionList")
       end
-      tpl = if list
-              list.to_native_tpl
-            else
-              nil
-            end
-      response = Rdkafka::Bindings.rd_kafka_commit(@native_kafka, tpl, async)
-      if response != 0
-        raise Rdkafka::RdkafkaError.new(response)
+
+      tpl = list ? list.to_native_tpl : nil
+
+      begin
+        response = Rdkafka::Bindings.rd_kafka_commit(@native_kafka, tpl, async)
+        if response != 0
+          raise Rdkafka::RdkafkaError.new(response)
+        end
+      ensure
+        Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl) if tpl
       end
     end
 
@@ -344,6 +393,8 @@ module Rdkafka
     #
     # @return [Message, nil] A message or nil if there was no new message within the timeout
     def poll(timeout_ms)
+      return unless @native_kafka
+
       message_ptr = Rdkafka::Bindings.rd_kafka_consumer_poll(@native_kafka, timeout_ms)
       if message_ptr.null?
         nil

data/lib/rdkafka/error.rb

@@ -39,6 +39,11 @@ module Rdkafka
     def is_partition_eof?
       code == :partition_eof
     end
+
+    # Error comparison
+    def ==(another_error)
+       another_error.is_a?(self.class) && (self.to_s == another_error.to_s)
+    end
   end
 
   # Error with topic partition list returned by the underlying rdkafka library.

data/lib/rdkafka/metadata.rb

@@ -0,0 +1,91 @@
+module Rdkafka
+  class Metadata
+    attr_reader :brokers, :topics
+
+    def initialize(native_client, topic_name = nil)
+      native_topic = if topic_name
+        Rdkafka::Bindings.rd_kafka_topic_new(native_client, topic_name, nil)
+      end
+
+      ptr = FFI::MemoryPointer.new(:pointer)
+
+      # Retrieve metadata flag is 0/1 for single/multiple topics.
+      topic_flag = topic_name ? 1 : 0
+
+      # Retrieve the Metadata
+      result = Rdkafka::Bindings.rd_kafka_metadata(native_client, topic_flag, native_topic, ptr, 250)
+
+      # Error Handling
+      Rdkafka::Error.new(result) unless result.zero?
+
+      metadata_from_native(ptr.read_pointer)
+    ensure
+      Rdkafka::Bindings.rd_kafka_topic_destroy(native_topic) if topic_name
+      Rdkafka::Bindings.rd_kafka_metadata_destroy(ptr.read_pointer)
+    end
+
+    private
+
+    def metadata_from_native(ptr)
+      metadata = Metadata.new(ptr)
+      @brokers = Array.new(metadata[:brokers_count]) do |i|
+        BrokerMetadata.new(metadata[:brokers_metadata] + (i * BrokerMetadata.size)).to_h
+      end
+
+      @topics = Array.new(metadata[:topics_count]) do |i|
+        topic = TopicMetadata.new(metadata[:topics_metadata] + (i * TopicMetadata.size))
+        Rdkafka::Error.new(topic[:rd_kafka_resp_err]) unless topic[:rd_kafka_resp_err].zero?
+
+        partitions = Array.new(topic[:partition_count]) do |j|
+          partition = PartitionMetadata.new(topic[:partitions_metadata] + (j * PartitionMetadata.size))
+          Rdkafka::Error.new(partition[:rd_kafka_resp_err]) unless partition[:rd_kafka_resp_err].zero?
+          partition.to_h
+        end
+        topic.to_h.merge!(partitions: partitions)
+      end
+    end
+
+    class CustomFFIStruct < FFI::Struct
+      def to_h
+        members.each_with_object({}) do |mem, hsh|
+          val = self.[](mem)
+          next if val.is_a?(FFI::Pointer) || mem == :rd_kafka_resp_err
+
+          hsh[mem] = self.[](mem)
+        end
+      end
+    end
+
+    class Metadata < CustomFFIStruct
+      layout :brokers_count, :int,
+             :brokers_metadata, :pointer,
+             :topics_count, :int,
+             :topics_metadata, :pointer,
+             :broker_id, :int32,
+             :broker_name, :string
+    end
+
+    class BrokerMetadata < CustomFFIStruct
+      layout :broker_id, :int32,
+             :broker_name, :string,
+             :broker_port, :int
+    end
+
+    class TopicMetadata < CustomFFIStruct
+      layout :topic_name, :string,
+             :partition_count, :int,
+             :partitions_metadata, :pointer,
+             :rd_kafka_resp_err, :int
+    end
+
+    class PartitionMetadata < CustomFFIStruct
+      layout :partition_id, :int32,
+             :rd_kafka_resp_err, :int,
+             :leader, :int32,
+             :replica_count, :int,
+             :replicas, :pointer,
+             :in_sync_replica_brokers, :int,
+             :isrs, :pointer
+    end
+  end
+end