rdkafka 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 33e13eee28a6f2356635744a0490e06d236afd9d
-  data.tar.gz: 23a74b8693241447df18042c3cf0482a03ec9c82
+  metadata.gz: cd80c3cdbb5bc790349df27bdef345b86e37d341
+  data.tar.gz: 822bad9c78b508e28cc7030443841b622b4d6df2
 SHA512:
-  metadata.gz: 2c130f0bfa4a968186cf51490e7453e81ac879c2ff3fa082707332235a6d98fc94e25c6278bac249f05a62afbea2cc70534bd05fd24ee07e1754418cf909d156
-  data.tar.gz: 5c5d186816373f5cd28a5d7cc583a79cfded06522d4707da855373e06ce57ff90903c639a1232e535abfa692f27a395e699d4f6b47792a97b9777ada224c8e44
+  metadata.gz: 63f00a3e196e10359220321613b03335bcec067e342155edbe256ef8a04716b435d2444674839e694e423a08a91e240e2e7d0756fb79402a354b8c8ab49cd3d1
+  data.tar.gz: 4e112ca823c2b500187aed86edf540f3eda4fbb45cee3b22ee908a1605696545cff2a77e2b56ba2d73500e185a3c1524992638e6ccb31d8f3c14d32cb783c1bf

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+# 0.3.0
+* Move both add topic methods to one `add_topic` in `TopicPartitionList`
+* Add committed offsets to consumer
+* Add query watermark offset to consumer
+
 # 0.2.0
 * Some refactoring and add inline documentation
 

data/README.md

@@ -54,7 +54,7 @@ end
 
 ## Development
 
-Run `bundle` and `cd ext && bundle exec rake compile && cd ..`. Then
+Run `bundle` and `cd ext && bundle exec rake && cd ..`. Then
 create the topics as expected in the specs: `bundle exec rake create_topics`.
 
 You can then run `bundle exec rspec` to run the tests. To see rdkafka

data/Rakefile

@@ -7,6 +7,7 @@ task :create_topics do
                  else
                    'kafka-topics'
                  end
+  `#{kafka_topics} --create --topic=consume_test_topic --zookeeper=127.0.0.1:2181 --partitions=3 --replication-factor=1`
   `#{kafka_topics} --create --topic=produce_test_topic --zookeeper=127.0.0.1:2181 --partitions=3 --replication-factor=1`
   `#{kafka_topics} --create --topic=rake_test_topic --zookeeper=127.0.0.1:2181 --partitions=3 --replication-factor=1`
 end

data/lib/rdkafka.rb

@@ -1,10 +1,12 @@
 require "rdkafka/version"
 
+require "rdkafka/bindings"
 require "rdkafka/config"
 require "rdkafka/consumer"
 require "rdkafka/consumer/message"
+require "rdkafka/consumer/partition"
+require "rdkafka/consumer/topic_partition_list"
 require "rdkafka/error"
-require "rdkafka/ffi"
 require "rdkafka/producer"
 require "rdkafka/producer/delivery_handle"
 require "rdkafka/producer/delivery_report"

data/lib/rdkafka/{ffi.rb → bindings.rb}

@@ -3,8 +3,8 @@ require "logger"
 
 module Rdkafka
   # @private
-  module FFI
-    extend ::FFI::Library
+  module Bindings
+    extend FFI::Library
 
     def self.lib_extension
       if Gem::Platform.local.os.include?("darwin")
@@ -23,7 +23,7 @@ module Rdkafka
 
     # Message struct
 
-    class Message < ::FFI::Struct
+    class Message < FFI::Struct
       layout :err, :int,
              :rkt, :pointer,
              :partition, :int32,
@@ -42,8 +42,8 @@ module Rdkafka
 
     # TopicPartition ad TopicPartitionList structs
 
-    class TopicPartition < ::FFI::Struct
-      layout :topic, :string,
+    class TopicPartition < FFI::Struct
+     layout :topic, :string,
              :partition, :int32,
              :offset, :int64,
              :metadata, :pointer,
@@ -53,15 +53,16 @@ module Rdkafka
              :_private, :pointer
     end
 
-    class TopicPartitionList < ::FFI::Struct
+    class TopicPartitionList < FFI::Struct
       layout :cnt, :int,
              :size, :int,
-             :elems, TopicPartition.ptr
+             :elems, :pointer
     end
 
     attach_function :rd_kafka_topic_partition_list_new, [:int32], :pointer
     attach_function :rd_kafka_topic_partition_list_add, [:pointer, :string, :int32], :void
     attach_function :rd_kafka_topic_partition_list_destroy, [:pointer], :void
+    attach_function :rd_kafka_topic_partition_list_copy, [:pointer], :pointer
 
     # Errors
 
@@ -85,7 +86,7 @@ module Rdkafka
     attach_function :rd_kafka_set_log_queue, [:pointer, :pointer], :void
     attach_function :rd_kafka_queue_get_main, [:pointer], :pointer
 
-    LogCallback = ::FFI::Function.new(
+    LogCallback = FFI::Function.new(
       :void, [:pointer, :int, :string, :string]
     ) do |_client_ptr, level, _level_string, line|
       severity = case level
@@ -118,11 +119,19 @@ module Rdkafka
     # Consumer
 
     attach_function :rd_kafka_subscribe, [:pointer, :pointer], :int
+    attach_function :rd_kafka_unsubscribe, [:pointer], :int
+    attach_function :rd_kafka_subscription, [:pointer, :pointer], :int
+    attach_function :rd_kafka_assignment, [:pointer, :pointer], :int
+    attach_function :rd_kafka_committed, [:pointer, :pointer, :int], :int
     attach_function :rd_kafka_commit, [:pointer, :pointer, :bool], :int, blocking: true
     attach_function :rd_kafka_poll_set_consumer, [:pointer], :void
     attach_function :rd_kafka_consumer_poll, [:pointer, :int], :pointer, blocking: true
     attach_function :rd_kafka_consumer_close, [:pointer], :void, blocking: true
 
+    # Stats
+
+    attach_function :rd_kafka_query_watermark_offsets, [:pointer, :string, :int, :pointer, :pointer, :int], :int
+
     # Producer
 
     RD_KAFKA_VTYPE_END = 0
@@ -141,7 +150,7 @@ module Rdkafka
     callback :delivery_cb, [:pointer, :pointer, :pointer], :void
     attach_function :rd_kafka_conf_set_dr_msg_cb, [:pointer, :delivery_cb], :void
 
-    DeliveryCallback = ::FFI::Function.new(
+    DeliveryCallback = FFI::Function.new(
       :void, [:pointer, :pointer, :pointer]
     ) do |client_ptr, message_ptr, opaque_ptr|
       message = Message.new(message_ptr)

data/lib/rdkafka/config.rb

@@ -74,7 +74,7 @@ module Rdkafka
     def consumer
       kafka = native_kafka(native_config, :rd_kafka_consumer)
       # Redirect the main queue to the consumer
-      Rdkafka::FFI.rd_kafka_poll_set_consumer(kafka)
+      Rdkafka::Bindings.rd_kafka_poll_set_consumer(kafka)
       # Return consumer with Kafka client
       Rdkafka::Consumer.new(kafka)
     end
@@ -89,7 +89,7 @@ module Rdkafka
       # Create Kafka config
       config = native_config
       # Set callback to receive delivery reports on config
-      Rdkafka::FFI.rd_kafka_conf_set_dr_msg_cb(config, Rdkafka::FFI::DeliveryCallback)
+      Rdkafka::Bindings.rd_kafka_conf_set_dr_msg_cb(config, Rdkafka::Bindings::DeliveryCallback)
       # Return producer with Kafka client
       Rdkafka::Producer.new(native_kafka(config, :rd_kafka_producer))
     end
@@ -108,10 +108,10 @@ module Rdkafka
     # This method is only intented to be used to create a client,
     # using it in another way will leak memory.
     def native_config
-      Rdkafka::FFI.rd_kafka_conf_new.tap do |config|
+      Rdkafka::Bindings.rd_kafka_conf_new.tap do |config|
         @config_hash.merge(REQUIRED_CONFIG).each do |key, value|
-          error_buffer = ::FFI::MemoryPointer.from_string(" " * 256)
-          result = Rdkafka::FFI.rd_kafka_conf_set(
+          error_buffer = FFI::MemoryPointer.from_string(" " * 256)
+          result = Rdkafka::Bindings.rd_kafka_conf_set(
             config,
             key.to_s,
             value.to_s,
@@ -123,13 +123,13 @@ module Rdkafka
           end
         end
         # Set log callback
-        Rdkafka::FFI.rd_kafka_conf_set_log_cb(config, Rdkafka::FFI::LogCallback)
+        Rdkafka::Bindings.rd_kafka_conf_set_log_cb(config, Rdkafka::Bindings::LogCallback)
       end
     end
 
     def native_kafka(config, type)
-      error_buffer = ::FFI::MemoryPointer.from_string(" " * 256)
-      handle = Rdkafka::FFI.rd_kafka_new(
+      error_buffer = FFI::MemoryPointer.from_string(" " * 256)
+      handle = Rdkafka::Bindings.rd_kafka_new(
         type,
         config,
         error_buffer,
@@ -141,14 +141,14 @@ module Rdkafka
       end
 
       # Redirect log to handle's queue
-      Rdkafka::FFI.rd_kafka_set_log_queue(
+      Rdkafka::Bindings.rd_kafka_set_log_queue(
         handle,
-        Rdkafka::FFI.rd_kafka_queue_get_main(handle)
+        Rdkafka::Bindings.rd_kafka_queue_get_main(handle)
       )
 
-      ::FFI::AutoPointer.new(
+      FFI::AutoPointer.new(
         handle,
-        Rdkafka::FFI.method(:rd_kafka_destroy)
+        Rdkafka::Bindings.method(:rd_kafka_destroy)
       )
     end
   end

data/lib/rdkafka/consumer.rb

@@ -16,10 +16,10 @@ module Rdkafka
     # Close this consumer
     # @return [nil]
     def close
-      Rdkafka::FFI.rd_kafka_consumer_close(@native_kafka)
+      Rdkafka::Bindings.rd_kafka_consumer_close(@native_kafka)
     end
 
-    # Subscribe to one or more topics
+    # Subscribe to one or more topics letting Kafka handle partition assignments.
     #
     # @param topics [Array<String>] One or more topic names
     #
@@ -28,22 +28,97 @@ module Rdkafka
     # @return [nil]
     def subscribe(*topics)
       # Create topic partition list with topics and no partition set
-      tpl = Rdkafka::FFI.rd_kafka_topic_partition_list_new(topics.length)
+      tpl = Rdkafka::Bindings.rd_kafka_topic_partition_list_new(topics.length)
       topics.each do |topic|
-        Rdkafka::FFI.rd_kafka_topic_partition_list_add(
+        Rdkafka::Bindings.rd_kafka_topic_partition_list_add(
           tpl,
           topic,
           -1
         )
       end
       # Subscribe to topic partition list and check this was successful
-      response = Rdkafka::FFI.rd_kafka_subscribe(@native_kafka, tpl)
+      response = Rdkafka::Bindings.rd_kafka_subscribe(@native_kafka, tpl)
       if response != 0
         raise Rdkafka::RdkafkaError.new(response)
       end
     ensure
       # Clean up the topic partition list
-      Rdkafka::FFI.rd_kafka_topic_partition_list_destroy(tpl)
+      Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
+    end
+
+    # Unsubscribe from all subscribed topics.
+    #
+    # @raise [RdkafkaError] When unsubscribing fails
+    #
+    # @return [nil]
+    def unsubscribe
+      response = Rdkafka::Bindings.rd_kafka_unsubscribe(@native_kafka)
+      if response != 0
+        raise Rdkafka::RdkafkaError.new(response)
+      end
+    end
+
+    # Return the current subscription to topics and partitions
+    #
+    # @raise [RdkafkaError] When getting the subscription fails.
+    #
+    # @return [TopicPartitionList]
+    def subscription
+      tpl = FFI::MemoryPointer.new(:pointer)
+      response = Rdkafka::Bindings.rd_kafka_subscription(@native_kafka, tpl)
+      if response != 0
+        raise Rdkafka::RdkafkaError.new(response)
+      end
+      Rdkafka::Consumer::TopicPartitionList.new(tpl.get_pointer(0))
+    end
+
+    # Return the current committed offset per partition for this consumer group.
+    # The offset field of each requested partition will either be set to stored offset or to -1001 in case there was no stored offset for that partition.
+    #
+    # @param list [TopicPartitionList] The topic with partitions to get the offsets for.
+    # @param timeout_ms [Integer] The timeout for fetching this information.
+    #
+    # @raise [RdkafkaError] When getting the committed positions fails.
+    #
+    # @return [TopicPartitionList]
+    def committed(list, timeout_ms=200)
+      unless list.is_a?(TopicPartitionList)
+        raise TypeError.new("list has to be a TopicPartitionList")
+      end
+      tpl = list.copy_tpl
+      response = Rdkafka::Bindings.rd_kafka_committed(@native_kafka, tpl, timeout_ms)
+      if response != 0
+        raise Rdkafka::RdkafkaError.new(response)
+      end
+      Rdkafka::Consumer::TopicPartitionList.new(tpl)
+    end
+
+    # Query broker for low (oldest/beginning) and high (newest/end) offsets for a partition.
+    #
+    # @param topic [String] The topic to query
+    # @param partition [Integer] The partition to query
+    # @param timeout_ms [Integer] The timeout for querying the broker
+    #
+    # @raise [RdkafkaError] When querying the broker fails.
+    #
+    # @return [Integer] The low and high watermark
+    def query_watermark_offsets(topic, partition, timeout_ms=200)
+      low = FFI::MemoryPointer.new(:int64, 1)
+      high = FFI::MemoryPointer.new(:int64, 1)
+
+      response = Rdkafka::Bindings.rd_kafka_query_watermark_offsets(
+        @native_kafka,
+        topic,
+        partition,
+        low,
+        high,
+        timeout_ms
+      )
+      if response != 0
+        raise Rdkafka::RdkafkaError.new(response)
+      end
+
+      return low.read_int, high.read_int
     end
 
     # Commit the current offsets of this consumer
@@ -54,7 +129,7 @@ module Rdkafka
     #
     # @return [nil]
     def commit(async=false)
-      response = Rdkafka::FFI.rd_kafka_commit(@native_kafka, nil, async)
+      response = Rdkafka::Bindings.rd_kafka_commit(@native_kafka, nil, async)
       if response != 0
         raise Rdkafka::RdkafkaError.new(response)
       end
@@ -68,12 +143,12 @@ module Rdkafka
     #
     # @return [Message, nil] A message or nil if there was no new message within the timeout
     def poll(timeout_ms)
-      message_ptr = Rdkafka::FFI.rd_kafka_consumer_poll(@native_kafka, timeout_ms)
+      message_ptr = Rdkafka::Bindings.rd_kafka_consumer_poll(@native_kafka, timeout_ms)
       if message_ptr.null?
         nil
       else
         # Create struct wrapper
-        native_message = Rdkafka::FFI::Message.new(message_ptr)
+        native_message = Rdkafka::Bindings::Message.new(message_ptr)
         # Raise error if needed
         if native_message[:err] != 0
           raise Rdkafka::RdkafkaError.new(native_message[:err])
@@ -84,7 +159,7 @@ module Rdkafka
     ensure
       # Clean up rdkafka message if there is one
       unless message_ptr.null?
-        Rdkafka::FFI.rd_kafka_message_destroy(message_ptr)
+        Rdkafka::Bindings.rd_kafka_message_destroy(message_ptr)
       end
     end
 

data/lib/rdkafka/consumer/message.rb

@@ -29,7 +29,7 @@ module Rdkafka
       # @private
       def initialize(native_message)
         unless native_message[:rkt].null?
-          @topic = FFI.rd_kafka_topic_name(native_message[:rkt])
+          @topic = Rdkafka::Bindings.rd_kafka_topic_name(native_message[:rkt])
         end
         @partition = native_message[:partition]
         unless native_message[:payload].null?
@@ -39,12 +39,21 @@ module Rdkafka
           @key = native_message[:key].read_string(native_message[:key_len])
         end
         @offset = native_message[:offset]
-        @timestamp = FFI.rd_kafka_message_timestamp(native_message, nil)
+        @timestamp = Rdkafka::Bindings.rd_kafka_message_timestamp(native_message, nil)
       end
 
+      # Human readable representation of this message.
       # @return [String]
       def to_s
-        "Message in '#{topic}' with key '#{key}', payload '#{payload}', partition #{partition}, offset #{offset}, timestamp #{timestamp}"
+        "<Message in '#{topic}' with key '#{truncate(key)}', payload '#{truncate(payload)}', partition #{partition}, offset #{offset}, timestamp #{timestamp}>"
+      end
+
+      def truncate(string)
+        if string && string.length > 40
+          "#{string[0..39]}..."
+        else
+          string
+        end
       end
     end
   end

data/lib/rdkafka/consumer/partition.rb

@@ -0,0 +1,40 @@
+module Rdkafka
+  class Consumer
+    # Information about a partition, used in {TopicPartitionList}.
+    class Partition
+      # Partition number
+      # @return [Integer]
+      attr_reader :partition
+
+      # Partition's offset
+      # @return [Integer]
+      attr_reader :offset
+
+      # @private
+      def initialize(partition, offset)
+        @partition = partition
+        @offset = offset
+      end
+
+      # Human readable representation of this partition.
+      # @return [String]
+      def to_s
+        "<Partition #{partition} with offset #{offset}>"
+      end
+
+      # Human readable representation of this partition.
+      # @return [String]
+      def inspect
+        to_s
+      end
+
+      # Whether another partition is equal to this
+      # @return [Boolean]
+      def ==(other)
+        self.class == other.class &&
+          self.partition == other.partition &&
+          self.offset == other.offset
+      end
+    end
+  end
+end

data/lib/rdkafka/consumer/topic_partition_list.rb

@@ -0,0 +1,93 @@
+module Rdkafka
+  class Consumer
+    # A list of topics with their partition information
+    class TopicPartitionList
+      # Create a new topic partition list.
+      #
+      # @param pointer [FFI::Pointer, nil] Optional pointer to an existing native list
+      #
+      # @return [TopicPartitionList]
+      def initialize(pointer=nil)
+        @tpl =
+          Rdkafka::Bindings::TopicPartitionList.new(
+            FFI::AutoPointer.new(
+              pointer || Rdkafka::Bindings.rd_kafka_topic_partition_list_new(5),
+              Rdkafka::Bindings.method(:rd_kafka_topic_partition_list_destroy)
+            )
+        )
+      end
+
+      # Number of items in the list
+      # @return [Integer]
+      def count
+        @tpl[:cnt]
+      end
+
+      # Whether this list is empty
+      # @return [Boolean]
+      def empty?
+        count == 0
+      end
+
+      # Add a topic with optionally partitions to the list.
+      #
+      # @param topic [String] The topic's name
+      # @param partition [Array<Integer>] The topic's partition's
+      #
+      # @return [nil]
+      def add_topic(topic, partitions=nil)
+        if partitions.nil?
+          Rdkafka::Bindings.rd_kafka_topic_partition_list_add(
+            @tpl,
+            topic,
+            -1
+          )
+        else
+          partitions.each do |partition|
+            Rdkafka::Bindings.rd_kafka_topic_partition_list_add(
+              @tpl,
+              topic,
+              partition
+            )
+          end
+        end
+      end
+
+      # Return a `Hash` with the topics as keys and and an array of partition information as the value if present.
+      #
+      # @return [Hash<String, [Array<Partition>, nil]>]
+      def to_h
+        {}.tap do |out|
+          count.times do |i|
+            ptr = @tpl[:elems] + (i * Rdkafka::Bindings::TopicPartition.size)
+            elem = Rdkafka::Bindings::TopicPartition.new(ptr)
+            if elem[:partition] == -1
+              out[elem[:topic]] = nil
+            else
+              partitions = out[elem[:topic]] || []
+              partition = Partition.new(elem[:partition], elem[:offset])
+              partitions.push(partition)
+              out[elem[:topic]] = partitions
+            end
+          end
+        end
+      end
+
+      # Human readable representation of this list.
+      # @return [String]
+      def to_s
+        "<TopicPartitionList: #{to_h}>"
+      end
+
+      def ==(other)
+        self.to_h == other.to_h
+      end
+
+      # Return a copy of the internal native list
+      # @private
+      def copy_tpl
+        Rdkafka::Bindings.rd_kafka_topic_partition_list_copy(@tpl)
+      end
+    end
+  end
+end

data/lib/rdkafka/error.rb

@@ -14,7 +14,7 @@ module Rdkafka
     # This error's code, for example `:partition_eof`, `:msg_size_too_large`.
     # @return [Symbol]
     def code
-      code = Rdkafka::FFI.rd_kafka_err2name(@rdkafka_response).downcase
+      code = Rdkafka::Bindings.rd_kafka_err2name(@rdkafka_response).downcase
       if code[0] == "_"
         code[1..-1].to_sym
       else
@@ -25,7 +25,7 @@ module Rdkafka
     # Human readable representation of this error.
     # @return [String]
     def to_s
-      "#{Rdkafka::FFI.rd_kafka_err2str(@rdkafka_response)} (#{code})"
+      "#{Rdkafka::Bindings.rd_kafka_err2str(@rdkafka_response)} (#{code})"
     end
 
     # Whether this error indicates the partition is EOF.

data/lib/rdkafka/producer.rb

@@ -8,9 +8,9 @@ module Rdkafka
       # Start thread to poll client for delivery callbacks
       @polling_thread = Thread.new do
         loop do
-          Rdkafka::FFI.rd_kafka_poll(@native_kafka, 250)
+          Rdkafka::Bindings.rd_kafka_poll(@native_kafka, 250)
           # Exit thread if closing and the poll queue is empty
-          if @closing && Rdkafka::FFI.rd_kafka_outq_len(@native_kafka) == 0
+          if @closing && Rdkafka::Bindings.rd_kafka_outq_len(@native_kafka) == 0
             break
           end
         end
@@ -71,16 +71,16 @@ module Rdkafka
       delivery_handle[:offset] = -1
 
       # Produce the message
-      response = Rdkafka::FFI.rd_kafka_producev(
+      response = Rdkafka::Bindings.rd_kafka_producev(
         @native_kafka,
-        :int, Rdkafka::FFI::RD_KAFKA_VTYPE_TOPIC, :string, topic,
-        :int, Rdkafka::FFI::RD_KAFKA_VTYPE_MSGFLAGS, :int, Rdkafka::FFI::RD_KAFKA_MSG_F_COPY,
-        :int, Rdkafka::FFI::RD_KAFKA_VTYPE_VALUE, :buffer_in, payload, :size_t, payload_size,
-        :int, Rdkafka::FFI::RD_KAFKA_VTYPE_KEY, :buffer_in, key, :size_t, key_size,
-        :int, Rdkafka::FFI::RD_KAFKA_VTYPE_PARTITION, :int32, partition,
-        :int, Rdkafka::FFI::RD_KAFKA_VTYPE_TIMESTAMP, :int64, timestamp,
-        :int, Rdkafka::FFI::RD_KAFKA_VTYPE_OPAQUE, :pointer, delivery_handle,
-        :int, Rdkafka::FFI::RD_KAFKA_VTYPE_END
+        :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_TOPIC, :string, topic,
+        :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_MSGFLAGS, :int, Rdkafka::Bindings::RD_KAFKA_MSG_F_COPY,
+        :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_VALUE, :buffer_in, payload, :size_t, payload_size,
+        :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_KEY, :buffer_in, key, :size_t, key_size,
+        :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_PARTITION, :int32, partition,
+        :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_TIMESTAMP, :int64, timestamp,
+        :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_OPAQUE, :pointer, delivery_handle,
+        :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_END
       )
 
       # Raise error if the produce call was not successfull

data/lib/rdkafka/producer/delivery_handle.rb

@@ -2,7 +2,7 @@ module Rdkafka
   class Producer
     # Handle to wait for a delivery report which is returned when
     # producing a message.
-    class DeliveryHandle < ::FFI::Struct
+    class DeliveryHandle < FFI::Struct
       layout :pending, :bool,
              :response, :int,
              :partition, :int,

data/lib/rdkafka/version.rb

@@ -1,4 +1,4 @@
 module Rdkafka
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
   LIBRDKAFKA_VERSION = "0.11.0"
 end

data/spec/rdkafka/bindings_spec.rb

@@ -0,0 +1,13 @@
+require "spec_helper"
+
+describe Rdkafka::Bindings do
+  it "should load librdkafka" do
+    expect(Rdkafka::Bindings.ffi_libraries.map(&:name).first).to include "librdkafka"
+  end
+
+  it "should successfully call librdkafka" do
+    expect {
+      Rdkafka::Bindings.rd_kafka_conf_new
+    }.not_to raise_error
+  end
+end

data/spec/rdkafka/consumer/message_spec.rb

@@ -2,7 +2,7 @@ require "spec_helper"
 
 describe Rdkafka::Consumer::Message do
   let(:native_topic) do
-    Rdkafka::FFI.rd_kafka_topic_new(
+    Rdkafka::Bindings.rd_kafka_topic_new(
       native_client,
       "topic_name",
       nil
@@ -11,18 +11,18 @@ describe Rdkafka::Consumer::Message do
   let(:payload) { nil }
   let(:key) { nil }
   let(:native_message) do
-    Rdkafka::FFI::Message.new.tap do |message|
+    Rdkafka::Bindings::Message.new.tap do |message|
       message[:rkt] = native_topic
       message[:partition] = 3
       message[:offset] = 100
       if payload
-        ptr = ::FFI::MemoryPointer.new(:char, payload.bytesize)
+        ptr = FFI::MemoryPointer.new(:char, payload.bytesize)
         ptr.put_bytes(0, payload)
         message[:payload] = ptr
         message[:len] = payload.bytesize
       end
       if key
-        ptr = ::FFI::MemoryPointer.new(:char, key.bytesize)
+        ptr = FFI::MemoryPointer.new(:char, key.bytesize)
         ptr.put_bytes(0, key)
         message[:key] = ptr
         message[:key_len] = key.bytesize
@@ -78,4 +78,32 @@ describe Rdkafka::Consumer::Message do
       subject.timestamp
     }.not_to raise_error
   end
+
+  describe "#to_s" do
+    before do
+      allow(subject).to receive(:timestamp).and_return(1000)
+    end
+
+    it "should have a human readable representation" do
+      expect(subject.to_s).to eq "<Message in 'topic_name' with key '', payload '', partition 3, offset 100, timestamp 1000>"
+    end
+
+    context "with key and payload" do
+      let(:key) { "key" }
+      let(:payload) { "payload" }
+
+      it "should have a human readable representation" do
+        expect(subject.to_s).to eq "<Message in 'topic_name' with key 'key', payload 'payload', partition 3, offset 100, timestamp 1000>"
+      end
+    end
+
+    context "with a very long key and payload" do
+      let(:key) { "k" * 100_000 }
+      let(:payload) { "p" * 100_000 }
+
+      it "should have a human readable representation" do
+        expect(subject.to_s).to eq "<Message in 'topic_name' with key 'kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk...', payload 'pppppppppppppppppppppppppppppppppppppppp...', partition 3, offset 100, timestamp 1000>"
+      end
+    end
+  end
 end

data/spec/rdkafka/consumer/partition_spec.rb

@@ -0,0 +1,35 @@
+require "spec_helper"
+
+describe Rdkafka::Consumer::Partition do
+  subject { Rdkafka::Consumer::Partition.new(1, 100) }
+
+  it "should have a partition" do
+    expect(subject.partition).to eq 1
+  end
+
+  it "should have an offset" do
+    expect(subject.offset).to eq 100
+  end
+
+  describe "#to_s" do
+    it "should return a human readable representation" do
+      expect(subject.to_s).to eq "<Partition 1 with offset 100>"
+    end
+  end
+
+  describe "#inspect" do
+    it "should return a human readable representation" do
+      expect(subject.to_s).to eq "<Partition 1 with offset 100>"
+    end
+  end
+
+  describe "#==" do
+    it "should equal another partition with the same content" do
+      expect(subject).to eq Rdkafka::Consumer::Partition.new(1, 100)
+    end
+
+    it "should not equal another partition with different content" do
+      expect(subject).not_to eq Rdkafka::Consumer::Partition.new(2, 101)
+    end
+  end
+end

data/spec/rdkafka/consumer/topic_partition_list_spec.rb

@@ -0,0 +1,94 @@
+require "spec_helper"
+
+describe Rdkafka::Consumer::TopicPartitionList do
+  it "should create a list from an existing native list" do
+    pointer = Rdkafka::Bindings.rd_kafka_topic_partition_list_new(5)
+    Rdkafka::Bindings.rd_kafka_topic_partition_list_add(
+      pointer,
+      "topic",
+      -1
+    )
+    list = Rdkafka::Consumer::TopicPartitionList.new(pointer)
+
+    other = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
+      list.add_topic("topic")
+    end
+
+    expect(list).to eq other
+  end
+
+  it "should create a new list and add unassigned topics" do
+    list = Rdkafka::Consumer::TopicPartitionList.new
+
+    expect(list.count).to eq 0
+    expect(list.empty?).to be true
+
+    list.add_topic("topic1")
+    list.add_topic("topic2")
+
+    expect(list.count).to eq 2
+    expect(list.empty?).to be false
+
+    hash = list.to_h
+    expect(hash.count).to eq 2
+    expect(hash).to eq ({
+      "topic1" => nil,
+      "topic2" => nil
+    })
+  end
+
+  it "should create a new list and add assigned topics" do
+    list = Rdkafka::Consumer::TopicPartitionList.new
+
+    expect(list.count).to eq 0
+    expect(list.empty?).to be true
+
+    list.add_topic("topic1", [0, 1, 2])
+    list.add_topic("topic2", [0, 1])
+
+    expect(list.count).to eq 5
+    expect(list.empty?).to be false
+
+    hash = list.to_h
+    expect(hash.count).to eq 2
+    expect(hash["topic1"]).to eq([
+      Rdkafka::Consumer::Partition.new(0, -1001),
+      Rdkafka::Consumer::Partition.new(1, -1001),
+      Rdkafka::Consumer::Partition.new(2, -1001)
+    ])
+    expect(hash["topic2"]).to eq([
+      Rdkafka::Consumer::Partition.new(0, -1001),
+      Rdkafka::Consumer::Partition.new(1, -1001)
+    ])
+  end
+
+  describe "#to_s" do
+    it "should return a human readable representation" do
+      list = Rdkafka::Consumer::TopicPartitionList.new
+      list.add_topic("topic1", [0, 1])
+
+      expected = "<TopicPartitionList: {\"topic1\"=>[<Partition 0 with offset -1001>, <Partition 1 with offset -1001>]}>"
+
+      expect(list.to_s).to eq expected
+    end
+  end
+
+  describe "#==" do
+    subject do
+      Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
+        list.add_topic("topic1", [0])
+      end
+    end
+
+    it "should equal another partition with the same content" do
+      other = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
+        list.add_topic("topic1", [0])
+      end
+      expect(subject).to eq other
+    end
+
+    it "should not equal another partition with different content" do
+      expect(subject).not_to eq Rdkafka::Consumer::TopicPartitionList.new
+    end
+  end
+end

data/spec/rdkafka/consumer_spec.rb

@@ -1,5 +1,68 @@
 require "spec_helper"
 
 describe Rdkafka::Consumer do
+  let(:config) { rdkafka_config }
+  let(:consumer) { config.consumer }
+  let(:producer) { config.producer }
 
+  context "subscription" do
+    it "should subscribe" do
+      expect(consumer.subscription).to be_empty
+
+      consumer.subscribe("consume_test_topic")
+
+      expect(consumer.subscription).not_to be_empty
+      expected_subscription = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
+        list.add_topic("consume_test_topic")
+      end
+      expect(consumer.subscription).to eq expected_subscription
+
+      consumer.unsubscribe
+
+      expect(consumer.subscription).to be_empty
+    end
+  end
+
+  describe "committed" do
+    before do
+      # Make sure there's a stored offset
+      report = producer.produce(
+        topic:     "consume_test_topic",
+        payload:   "payload 1",
+        key:       "key 1",
+        partition: 0
+      ).wait
+      message = wait_for_message(
+        topic: "consume_test_topic",
+        delivery_report: report,
+        config: config
+      )
+    end
+
+    it "should fetch the committed offsets for a specified topic partition list" do
+      list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
+        list.add_topic("consume_test_topic", [0, 1, 2])
+      end
+      partitions = consumer.committed(list).to_h["consume_test_topic"]
+      expect(partitions[0].offset).to be > 0
+      expect(partitions[1].offset).to eq -1001
+      expect(partitions[2].offset).to eq -1001
+    end
+  end
+
+  describe "watermark offsets" do
+    it "should return the watermark offsets" do
+      # Make sure there's a message
+      producer.produce(
+        topic:     "consume_test_topic",
+        payload:   "payload 1",
+        key:       "key 1",
+        partition: 0
+      ).wait
+
+      low, high = consumer.query_watermark_offsets("consume_test_topic", 0, 5000)
+      expect(low).to eq 0
+      expect(high).to be > 0
+    end
+  end
 end

data/spec/spec_helper.rb

@@ -22,8 +22,9 @@ def native_client
   config.send(:native_kafka, config.send(:native_config), :rd_kafka_producer)
 end
 
-def wait_for_message(topic:, delivery_report:, timeout_in_seconds: 30)
-  consumer = rdkafka_config.consumer
+def wait_for_message(topic:, delivery_report:, timeout_in_seconds: 30, config: nil)
+  config = rdkafka_config if config.nil?
+  consumer = config.consumer
   consumer.subscribe(topic)
   timeout = Time.now.to_i + timeout_in_seconds
   loop do
@@ -38,5 +39,6 @@ def wait_for_message(topic:, delivery_report:, timeout_in_seconds: 30)
     end
   end
 ensure
+  consumer.commit
   consumer.close
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Thijs Cadier
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-10-13 00:00:00.000000000 Z
+date: 2017-10-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -98,21 +98,25 @@ files:
 - Rakefile
 - ext/Rakefile
 - lib/rdkafka.rb
+- lib/rdkafka/bindings.rb
 - lib/rdkafka/config.rb
 - lib/rdkafka/consumer.rb
 - lib/rdkafka/consumer/message.rb
+- lib/rdkafka/consumer/partition.rb
+- lib/rdkafka/consumer/topic_partition_list.rb
 - lib/rdkafka/error.rb
-- lib/rdkafka/ffi.rb
 - lib/rdkafka/producer.rb
 - lib/rdkafka/producer/delivery_handle.rb
 - lib/rdkafka/producer/delivery_report.rb
 - lib/rdkafka/version.rb
 - rdkafka.gemspec
+- spec/rdkafka/bindings_spec.rb
 - spec/rdkafka/config_spec.rb
 - spec/rdkafka/consumer/message_spec.rb
+- spec/rdkafka/consumer/partition_spec.rb
+- spec/rdkafka/consumer/topic_partition_list_spec.rb
 - spec/rdkafka/consumer_spec.rb
 - spec/rdkafka/error_spec.rb
-- spec/rdkafka/ffi_spec.rb
 - spec/rdkafka/producer/delivery_handle_spec.rb
 - spec/rdkafka/producer/delivery_report_spec.rb
 - spec/rdkafka/producer_spec.rb
@@ -143,11 +147,13 @@ specification_version: 4
 summary: Kafka client library wrapping librdkafka using the ffi gem and futures from
   concurrent-ruby for Kafka 0.10+
 test_files:
+- spec/rdkafka/bindings_spec.rb
 - spec/rdkafka/config_spec.rb
 - spec/rdkafka/consumer/message_spec.rb
+- spec/rdkafka/consumer/partition_spec.rb
+- spec/rdkafka/consumer/topic_partition_list_spec.rb
 - spec/rdkafka/consumer_spec.rb
 - spec/rdkafka/error_spec.rb
-- spec/rdkafka/ffi_spec.rb
 - spec/rdkafka/producer/delivery_handle_spec.rb
 - spec/rdkafka/producer/delivery_report_spec.rb
 - spec/rdkafka/producer_spec.rb

data/spec/rdkafka/ffi_spec.rb

@@ -1,13 +0,0 @@
-require "spec_helper"
-
-describe Rdkafka::FFI do
-  it "should load librdkafka" do
-    expect(Rdkafka::FFI.ffi_libraries.map(&:name).first).to include "librdkafka"
-  end
-
-  it "should successfully call librdkafka" do
-    expect {
-      Rdkafka::FFI.rd_kafka_conf_new
-    }.not_to raise_error
-  end
-end