rdkafka 0.3.5 → 0.4.0

data/lib/rdkafka/consumer/message.rb

@@ -23,23 +23,37 @@ module Rdkafka
       attr_reader :offset
 
       # This message's timestamp, if provided by the broker
-      # @return [Integer, nil]
+      # @return [Time, nil]
       attr_reader :timestamp
 
       # @private
       def initialize(native_message)
+        # Set topic
         unless native_message[:rkt].null?
           @topic = Rdkafka::Bindings.rd_kafka_topic_name(native_message[:rkt])
         end
+        # Set partition
         @partition = native_message[:partition]
+        # Set payload
         unless native_message[:payload].null?
           @payload = native_message[:payload].read_string(native_message[:len])
         end
+        # Set key
         unless native_message[:key].null?
           @key = native_message[:key].read_string(native_message[:key_len])
         end
+        # Set offset
         @offset = native_message[:offset]
-        @timestamp = Rdkafka::Bindings.rd_kafka_message_timestamp(native_message, nil)
+        # Set timestamp
+        raw_timestamp = Rdkafka::Bindings.rd_kafka_message_timestamp(native_message, nil)
+        @timestamp = if raw_timestamp && raw_timestamp > -1
+                       # Calculate seconds and microseconds
+                       seconds = raw_timestamp / 1000
+                       milliseconds = (raw_timestamp - seconds * 1000) * 1000
+                       Time.at(seconds, milliseconds)
+                     else
+                       nil
+                     end
       end
 
       # Human readable representation of this message.

data/lib/rdkafka/consumer/partition.rb

@@ -7,7 +7,7 @@ module Rdkafka
       attr_reader :partition
 
       # Partition's offset
-      # @return [Integer]
+      # @return [Integer, nil]
       attr_reader :offset
 
       # @private
@@ -19,7 +19,11 @@ module Rdkafka
       # Human readable representation of this partition.
       # @return [String]
       def to_s
-        "<Partition #{partition} with offset #{offset}>"
+        if offset.nil?
+          "<Partition #{partition} without offset>"
+        else
+          "<Partition #{partition} with offset #{offset}>"
+        end
       end
 
       # Human readable representation of this partition.

data/lib/rdkafka/consumer/topic_partition_list.rb

@@ -2,34 +2,37 @@ module Rdkafka
   class Consumer
     # A list of topics with their partition information
     class TopicPartitionList
-      # Create a new topic partition list.
+      # Create a topic partition list.
       #
-      # @param pointer [FFI::Pointer, nil] Optional pointer to an existing native list
+      # @param data [Hash<String => [nil,Partition]>] The topic and partion data or nil to create an empty list
       #
       # @return [TopicPartitionList]
-      def initialize(pointer=nil)
-        @tpl =
-          Rdkafka::Bindings::TopicPartitionList.new(
-            FFI::AutoPointer.new(
-              pointer || Rdkafka::Bindings.rd_kafka_topic_partition_list_new(5),
-              Rdkafka::Bindings.method(:rd_kafka_topic_partition_list_destroy)
-            )
-        )
+      def initialize(data=nil)
+        @data = data || {}
       end
 
       # Number of items in the list
       # @return [Integer]
       def count
-        @tpl[:cnt]
+        i = 0
+        @data.each do |_topic, partitions|
+          if partitions
+            i += partitions.count
+          else
+            i+= 1
+          end
+        end
+        i
       end
 
       # Whether this list is empty
       # @return [Boolean]
       def empty?
-        count == 0
+        @data.empty?
       end
 
       # Add a topic with optionally partitions to the list.
+      # Calling this method multiple times for the same topic will overwrite the previous configuraton.
       #
       # @example Add a topic with unassigned partitions
       #   tpl.add_topic("topic")
@@ -41,48 +44,36 @@ module Rdkafka
       #   tpl.add_topic("topic", 9)
       #
       # @param topic [String] The topic's name
-      # @param partition [Array<Integer>, Range<Integer>, Integer] The topic's partitions or partition count
+      # @param partitions [Array<Integer>, Range<Integer>, Integer] The topic's partitions or partition count
       #
       # @return [nil]
       def add_topic(topic, partitions=nil)
-        if partitions.is_a? Integer
-          partitions = (0..partitions - 1)
-        end
         if partitions.nil?
-          Rdkafka::Bindings.rd_kafka_topic_partition_list_add(
-            @tpl,
-            topic,
-            -1
-          )
+          @data[topic.to_s] = nil
         else
-          partitions.each do |partition|
-            Rdkafka::Bindings.rd_kafka_topic_partition_list_add(
-              @tpl,
-              topic,
-              partition
-            )
+          if partitions.is_a? Integer
+            partitions = (0..partitions - 1)
           end
+          @data[topic.to_s] = partitions.map { |p| Partition.new(p, nil) }
         end
       end
 
+      # Add a topic with partitions and offsets set to the list
+      # Calling this method multiple times for the same topic will overwrite the previous configuraton.
+      #
+      # @param topic [String] The topic's name
+      # @param partitions_with_offsets [Hash<Integer, Integer>] The topic's partitions and offsets
+      #
+      # @return [nil]
+      def add_topic_and_partitions_with_offsets(topic, partitions_with_offsets)
+        @data[topic.to_s] = partitions_with_offsets.map { |p, o| Partition.new(p, o) }
+      end
+
       # Return a `Hash` with the topics as keys and and an array of partition information as the value if present.
       #
       # @return [Hash<String, [Array<Partition>, nil]>]
       def to_h
-        {}.tap do |out|
-          count.times do |i|
-            ptr = @tpl[:elems] + (i * Rdkafka::Bindings::TopicPartition.size)
-            elem = Rdkafka::Bindings::TopicPartition.new(ptr)
-            if elem[:partition] == -1
-              out[elem[:topic]] = nil
-            else
-              partitions = out[elem[:topic]] || []
-              partition = Partition.new(elem[:partition], elem[:offset])
-              partitions.push(partition)
-              out[elem[:topic]] = partitions
-            end
-          end
-        end
+        @data
       end
 
       # Human readable representation of this list.
@@ -95,10 +86,75 @@ module Rdkafka
         self.to_h == other.to_h
       end
 
-      # Return a copy of the internal native list
+      # Create a new topic partition list based of a native one.
+      #
+      # @param pointer [FFI::Pointer] Optional pointer to an existing native list. Its contents will be copied and afterwards it will be destroyed.
+      #
+      # @return [TopicPartitionList]
+      #
+      # @private
+      def self.from_native_tpl(pointer)
+        # Data to be moved into the tpl
+        data = {}
+
+        # Create struct and copy its contents
+        native_tpl = Rdkafka::Bindings::TopicPartitionList.new(pointer)
+        native_tpl[:cnt].times do |i|
+          ptr = native_tpl[:elems] + (i * Rdkafka::Bindings::TopicPartition.size)
+          elem = Rdkafka::Bindings::TopicPartition.new(ptr)
+          if elem[:partition] == -1
+            data[elem[:topic]] = nil
+          else
+            partitions = data[elem[:topic]] || []
+            offset = if elem[:offset] == -1001
+                       nil
+                     else
+                       elem[:offset]
+                     end
+            partition = Partition.new(elem[:partition], offset)
+            partitions.push(partition)
+            data[elem[:topic]] = partitions
+          end
+        end
+
+        # Return the created object
+        TopicPartitionList.new(data)
+      ensure
+        # Destroy the tpl
+        Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(pointer)
+      end
+
+      # Create a native tpl with the contents of this object added
+      #
       # @private
-      def copy_tpl
-        Rdkafka::Bindings.rd_kafka_topic_partition_list_copy(@tpl)
+      def to_native_tpl
+        Rdkafka::Bindings.rd_kafka_topic_partition_list_new(count).tap do |tpl|
+          @data.each do |topic, partitions|
+            if partitions
+              partitions.each do |p|
+                Rdkafka::Bindings.rd_kafka_topic_partition_list_add(
+                  tpl,
+                  topic,
+                  p.partition
+                )
+                if p.offset
+                  Rdkafka::Bindings.rd_kafka_topic_partition_list_set_offset(
+                    tpl,
+                    topic,
+                    p.partition,
+                    p.offset
+                  )
+                end
+              end
+            else
+              Rdkafka::Bindings.rd_kafka_topic_partition_list_add(
+                tpl,
+                topic,
+                -1
+              )
+            end
+          end
+        end
       end
     end
   end

data/lib/rdkafka/producer.rb

@@ -32,10 +32,10 @@ module Rdkafka
     # When a timestamp is provided this is used instead of the autogenerated timestamp.
     #
     # @param topic [String] The topic to produce to
-    # @param payload [String] The message's payload
+    # @param payload [String,nil] The message's payload
     # @param key [String] The message's key
-    # @param partition [Integer] Optional partition to produce to
-    # @param timestamp [Integer] Optional timestamp of this message
+    # @param partition [Integer,nil] Optional partition to produce to
+    # @param timestamp [Time,Integer,nil] Optional timestamp of this message. Integer timestamp is in milliseconds since Jan 1 1970.
     #
     # @raise [RdkafkaError] When adding the message to rdkafka's queue failed
     #
@@ -61,8 +61,17 @@ module Rdkafka
       # on the key/randomly if there is no key
       partition = -1 if partition.nil?
 
-      # If timestamp is nil use 0 and let Kafka set one
-      timestamp = 0 if timestamp.nil?
+      # If timestamp is nil use 0 and let Kafka set one. If an integer or time
+      # use it.
+      raw_timestamp = if timestamp.nil?
+                        0
+                      elsif timestamp.is_a?(Integer)
+                        timestamp
+                      elsif timestamp.is_a?(Time)
+                        (timestamp.to_i * 1000) + (timestamp.usec / 1000)
+                      else
+                        raise TypeError.new("Timestamp has to be nil, an Integer or a Time")
+                      end
 
       delivery_handle = DeliveryHandle.new
       delivery_handle[:pending] = true
@@ -79,7 +88,7 @@ module Rdkafka
         :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_VALUE, :buffer_in, payload, :size_t, payload_size,
         :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_KEY, :buffer_in, key, :size_t, key_size,
         :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_PARTITION, :int32, partition,
-        :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_TIMESTAMP, :int64, timestamp,
+        :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_TIMESTAMP, :int64, raw_timestamp,
         :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_OPAQUE, :pointer, delivery_handle,
         :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_END
       )

data/lib/rdkafka/version.rb

@@ -1,4 +1,4 @@
 module Rdkafka
-  VERSION = "0.3.5"
-  LIBRDKAFKA_VERSION = "0.11.3"
+  VERSION = "0.4.0"
+  LIBRDKAFKA_VERSION = "0.11.5"
 end

data/spec/rdkafka/bindings_spec.rb

@@ -59,4 +59,27 @@ describe Rdkafka::Bindings do
       expect(log.string).to include "ANY -- : rdkafka: log line"
     end
   end
+
+  describe "stats callback" do
+    context "without a stats callback" do
+      it "should do nothing" do
+        expect {
+          Rdkafka::Bindings::StatsCallback.call(nil, "{}", 2, nil)
+        }.not_to raise_error
+      end
+    end
+
+    context "with a stats callback" do
+      before do
+        Rdkafka::Config.statistics_callback = lambda do |stats|
+          $received_stats = stats
+        end
+      end
+
+      it "should call the stats callback with a stats hash" do
+        Rdkafka::Bindings::StatsCallback.call(nil, "{\"received\":1}", 13, nil)
+        expect($received_stats).to eq({'received' => 1})
+      end
+    end
+  end
 end

data/spec/rdkafka/config_spec.rb

@@ -20,6 +20,22 @@ describe Rdkafka::Config do
     end
   end
 
+  context "statistics callback" do
+    it "should set the callback" do
+      expect {
+        Rdkafka::Config.statistics_callback = lambda do |stats|
+          puts stats
+        end
+      }.not_to raise_error
+    end
+
+    it "should not accept a callback that's not a proc" do
+      expect {
+        Rdkafka::Config.statistics_callback = 'a string'
+      }.to raise_error(TypeError)
+    end
+  end
+
   context "configuration" do
     it "should store configuration" do
       config = Rdkafka::Config.new

data/spec/rdkafka/consumer/message_spec.rb

@@ -65,12 +65,26 @@ describe Rdkafka::Consumer::Message do
     expect(subject.offset).to eq 100
   end
 
-  it "should have a timestamp" do
-    # There is no effective way to mock this this, just
-    # make sure it doesn't crash.
-    expect {
-      subject.timestamp
-    }.not_to raise_error
+  describe "#timestamp" do
+    context "without a timestamp" do
+      before do
+        allow(Rdkafka::Bindings).to receive(:rd_kafka_message_timestamp).and_return(-1)
+      end
+
+      it "should have a nil timestamp if not present" do
+        expect(subject.timestamp).to be_nil
+      end
+    end
+
+    context "with a timestamp" do
+      before do
+        allow(Rdkafka::Bindings).to receive(:rd_kafka_message_timestamp).and_return(1505069646250)
+      end
+
+      it "should have timestamp if present" do
+        expect(subject.timestamp).to eq Time.at(1505069646, 250_000)
+      end
+    end
   end
 
   describe "#to_s" do

data/spec/rdkafka/consumer/partition_spec.rb

@@ -1,7 +1,8 @@
 require "spec_helper"
 
 describe Rdkafka::Consumer::Partition do
-  subject { Rdkafka::Consumer::Partition.new(1, 100) }
+  let(:offset) { 100 }
+  subject { Rdkafka::Consumer::Partition.new(1, offset) }
 
   it "should have a partition" do
     expect(subject.partition).to eq 1
@@ -21,6 +22,14 @@ describe Rdkafka::Consumer::Partition do
     it "should return a human readable representation" do
       expect(subject.to_s).to eq "<Partition 1 with offset 100>"
     end
+
+    context "without offset" do
+      let(:offset) { nil }
+
+      it "should return a human readable representation" do
+        expect(subject.to_s).to eq "<Partition 1 without offset>"
+      end
+    end
   end
 
   describe "#==" do

data/spec/rdkafka/consumer/topic_partition_list_spec.rb

@@ -1,22 +1,6 @@
 require "spec_helper"
 
 describe Rdkafka::Consumer::TopicPartitionList do
-  it "should create a list from an existing native list" do
-    pointer = Rdkafka::Bindings.rd_kafka_topic_partition_list_new(5)
-    Rdkafka::Bindings.rd_kafka_topic_partition_list_add(
-      pointer,
-      "topic",
-      -1
-    )
-    list = Rdkafka::Consumer::TopicPartitionList.new(pointer)
-
-    other = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
-      list.add_topic("topic")
-    end
-
-    expect(list).to eq other
-  end
-
   it "should create a new list and add unassigned topics" do
     list = Rdkafka::Consumer::TopicPartitionList.new
 
@@ -52,13 +36,13 @@ describe Rdkafka::Consumer::TopicPartitionList do
     hash = list.to_h
     expect(hash.count).to eq 2
     expect(hash["topic1"]).to eq([
-      Rdkafka::Consumer::Partition.new(0, -1001),
-      Rdkafka::Consumer::Partition.new(1, -1001),
-      Rdkafka::Consumer::Partition.new(2, -1001)
+      Rdkafka::Consumer::Partition.new(0, nil),
+      Rdkafka::Consumer::Partition.new(1, nil),
+      Rdkafka::Consumer::Partition.new(2, nil)
     ])
     expect(hash["topic2"]).to eq([
-      Rdkafka::Consumer::Partition.new(0, -1001),
-      Rdkafka::Consumer::Partition.new(1, -1001)
+      Rdkafka::Consumer::Partition.new(0, nil),
+      Rdkafka::Consumer::Partition.new(1, nil)
     ])
   end
 
@@ -77,13 +61,13 @@ describe Rdkafka::Consumer::TopicPartitionList do
     hash = list.to_h
     expect(hash.count).to eq 2
     expect(hash["topic1"]).to eq([
-      Rdkafka::Consumer::Partition.new(0, -1001),
-      Rdkafka::Consumer::Partition.new(1, -1001),
-      Rdkafka::Consumer::Partition.new(2, -1001)
+      Rdkafka::Consumer::Partition.new(0, nil),
+      Rdkafka::Consumer::Partition.new(1, nil),
+      Rdkafka::Consumer::Partition.new(2, nil)
     ])
     expect(hash["topic2"]).to eq([
-      Rdkafka::Consumer::Partition.new(0, -1001),
-      Rdkafka::Consumer::Partition.new(1, -1001)
+      Rdkafka::Consumer::Partition.new(0, nil),
+      Rdkafka::Consumer::Partition.new(1, nil)
     ])
   end
 
@@ -102,13 +86,30 @@ describe Rdkafka::Consumer::TopicPartitionList do
     hash = list.to_h
     expect(hash.count).to eq 2
     expect(hash["topic1"]).to eq([
-      Rdkafka::Consumer::Partition.new(0, -1001),
-      Rdkafka::Consumer::Partition.new(1, -1001),
-      Rdkafka::Consumer::Partition.new(2, -1001)
+      Rdkafka::Consumer::Partition.new(0, nil),
+      Rdkafka::Consumer::Partition.new(1, nil),
+      Rdkafka::Consumer::Partition.new(2, nil)
     ])
     expect(hash["topic2"]).to eq([
-      Rdkafka::Consumer::Partition.new(0, -1001),
-      Rdkafka::Consumer::Partition.new(1, -1001)
+      Rdkafka::Consumer::Partition.new(0, nil),
+      Rdkafka::Consumer::Partition.new(1, nil)
+    ])
+  end
+
+  it "should create a new list and add topics and partitions with an offset" do
+    list = Rdkafka::Consumer::TopicPartitionList.new
+
+    expect(list.count).to eq 0
+    expect(list.empty?).to be true
+
+    list.add_topic_and_partitions_with_offsets("topic1", 0 => 5, 1 => 6, 2 => 7)
+
+    hash = list.to_h
+    expect(hash.count).to eq 1
+    expect(hash["topic1"]).to eq([
+      Rdkafka::Consumer::Partition.new(0, 5),
+      Rdkafka::Consumer::Partition.new(1, 6),
+      Rdkafka::Consumer::Partition.new(2, 7)
     ])
   end
 
@@ -117,7 +118,7 @@ describe Rdkafka::Consumer::TopicPartitionList do
       list = Rdkafka::Consumer::TopicPartitionList.new
       list.add_topic("topic1", [0, 1])
 
-      expected = "<TopicPartitionList: {\"topic1\"=>[<Partition 0 with offset -1001>, <Partition 1 with offset -1001>]}>"
+      expected = "<TopicPartitionList: {\"topic1\"=>[<Partition 0 without offset>, <Partition 1 without offset>]}>"
 
       expect(list.to_s).to eq expected
     end
@@ -141,4 +142,82 @@ describe Rdkafka::Consumer::TopicPartitionList do
       expect(subject).not_to eq Rdkafka::Consumer::TopicPartitionList.new
     end
   end
+
+  describe ".from_native_tpl" do
+    it "should create a list from an existing native list" do
+      pointer = Rdkafka::Bindings.rd_kafka_topic_partition_list_new(5)
+      Rdkafka::Bindings.rd_kafka_topic_partition_list_add(
+        pointer,
+        "topic",
+        -1
+      )
+      list = Rdkafka::Consumer::TopicPartitionList.from_native_tpl(pointer)
+
+      other = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
+        list.add_topic("topic")
+      end
+
+      expect(list).to eq other
+    end
+
+    it "should create a list from an existing native list with offsets" do
+      pointer = Rdkafka::Bindings.rd_kafka_topic_partition_list_new(5)
+      Rdkafka::Bindings.rd_kafka_topic_partition_list_add(
+        pointer,
+        "topic",
+        0
+      )
+      Rdkafka::Bindings.rd_kafka_topic_partition_list_set_offset(
+        pointer,
+        "topic",
+        0,
+        100
+      )
+      list = Rdkafka::Consumer::TopicPartitionList.from_native_tpl(pointer)
+
+      other = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
+        list.add_topic_and_partitions_with_offsets("topic", 0 => 100)
+      end
+
+      expect(list).to eq other
+    end
+  end
+
+  describe "#to_native_tpl" do
+    it "should create a native list" do
+      list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
+        list.add_topic("topic")
+      end
+
+      tpl = list.to_native_tpl
+
+      other = Rdkafka::Consumer::TopicPartitionList.from_native_tpl(tpl)
+
+      expect(list).to eq other
+    end
+
+    it "should create a native list with partitions" do
+      list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
+        list.add_topic("topic", 0..16)
+      end
+
+      tpl = list.to_native_tpl
+
+      other = Rdkafka::Consumer::TopicPartitionList.from_native_tpl(tpl)
+
+      expect(list).to eq other
+    end
+
+    it "should create a native list with offsets" do
+      list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
+        list.add_topic_and_partitions_with_offsets("topic", 0 => 100)
+      end
+
+      tpl = list.to_native_tpl
+
+      other = Rdkafka::Consumer::TopicPartitionList.from_native_tpl(tpl)
+
+      expect(list).to eq other
+    end
+  end
 end