rdkafka 0.1.9 → 0.1.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5ca27dbc2b4226938c9295a8d2a483f414fd161e
-  data.tar.gz: 2b0e7a4333d47af7b41d5d3e5da5242549740165
+  metadata.gz: 47ed8b2e877ccd9b0d4f79e6c252651f497b3aad
+  data.tar.gz: 274091ba3bf02e06444c871c7ed4af50e417be08
 SHA512:
-  metadata.gz: d97ac0211a5a6a2bd52b13d4779c8cac55c46207aaf8b0bb02d59fe3cffe50746b9be8c352015f218d2607c675c3374f3931d4aad54478f7880cb09b2947bda6
-  data.tar.gz: 4316bd794f7a83723dcd2184a6a27c0ca2b45380d1989eaac539058f0a47fe68620ba851dc3f4b93d9d610826029a55f96566326ff83128f1e5ae5849d4fa846
+  metadata.gz: ea30c37519e22b12c8ffb38fc902ccb9cb4c2fb2ef5432aa91df5aa116c5abab5965cd0787c6de76a4f664dad7d7b97753895d5ae27852374cbae1b889245d96
+  data.tar.gz: e49e57f8ccd45d62250f2eef6e2a6da3f9ea90b3393239f2b2d82222adfb3bbecd6927fce42adaae6e16bca9ee1dc25e69d7a3b25942edb6f19aa5594524b710

data/.gitignore

@@ -3,3 +3,4 @@ ext/ports
 ext/tmp
 ext/librdkafka.*
 *.gem
+.yardoc

data/README.md

@@ -1,5 +1,7 @@
 # Rdkafka
 
+[![Gem Version](https://badge.fury.io/rb/rdkafka.svg)](https://badge.fury.io/rb/rdkafka)
+
 The `rdkafka` gem is a modern Kafka client library for Ruby based on
 [librdkafka](https://github.com/edenhill/librdkafka/).
 It wraps the production-ready C client using the [ffi](https://github.com/ffi/ffi)
@@ -9,6 +11,8 @@ This gem only provides a high-level Kafka consumer. If you are running
 an older version of Kafka and/or need the legacy simple consumer we
 suggest using the [Hermann](https://github.com/reiseburo/hermann) gem.
 
+Documentation is available on [RubyDoc](http://www.rubydoc.info/github/thijsc/rdkafka-ruby/master).
+
 ## Development
 
 Run `bundle` and `cd ext && bundle exec rake compile && cd ..`. Then

data/Rakefile

@@ -1,14 +1,17 @@
 require "./lib/rdkafka"
 
 task :create_topics do
-  `kafka-topics --create --topic=produce_test_topic --zookeeper=127.0.0.1:2181 --partitions=1 --replication-factor=1`
-  `kafka-topics --create --topic=rake_test_topic --zookeeper=127.0.0.1:2181 --partitions=1 --replication-factor=1`
+  puts "Creating test topics"
+  `kafka-topics --create --topic=produce_test_topic --zookeeper=127.0.0.1:2181 --partitions=3 --replication-factor=1`
+  `kafka-topics --create --topic=rake_test_topic --zookeeper=127.0.0.1:2181 --partitions=3 --replication-factor=1`
 end
 
 task :produce_messages do
-  producer = Rdkafka::Config.new(
-    :"bootstrap.servers" => "localhost:9092"
-  ).producer
+  config = {:"bootstrap.servers" => "localhost:9092"}
+  if ENV["DEBUG"]
+    config[:debug] = "broker,topic,msg"
+  end
+  producer = Rdkafka::Config.new(config).producer
   100.times do |i|
     puts "Producing message #{i}"
     producer.produce(
@@ -20,14 +23,18 @@ task :produce_messages do
 end
 
 task :consume_messages do
-  consumer = Rdkafka::Config.new(
+  config = {
     :"bootstrap.servers" => "localhost:9092",
     :"group.id" => "rake_test",
     :"enable.partition.eof" => false,
     :"auto.offset.reset" => "earliest"
-  ).consumer
+  }
+  if ENV["DEBUG"]
+    config[:debug] = "cgrp,topic,fetch"
+  end
+  consumer = Rdkafka::Config.new(config).consumer
   consumer.subscribe("rake_test_topic")
   consumer.each do |message|
-    puts message
+    puts "Message received: #{message}"
   end
 end

data/lib/rdkafka.rb

@@ -4,4 +4,5 @@ require "rdkafka/config"
 require "rdkafka/consumer"
 require "rdkafka/error"
 require "rdkafka/ffi"
+require "rdkafka/message"
 require "rdkafka/producer"

data/lib/rdkafka/config.rb

@@ -1,6 +1,9 @@
 require "logger"
 
 module Rdkafka
+  # Configuration for a Kafka consumer or producer. You can create an instance and use
+  # the consumer and producer methods to create a client. Documentation of the available
+  # configuration options is available on https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md.
   class Config
     @@logger = Logger.new(STDOUT)
 
@@ -13,8 +16,14 @@ module Rdkafka
     end
 
     DEFAULT_CONFIG = {
+      # Request api version so advanced features work
       :"api.version.request" => true
-    }
+    }.freeze
+
+    REQUIRED_CONFIG = {
+      # Enable log queues so we get callbacks in our own threads
+      :"log.queue" => true
+    }.freeze
 
     def initialize(config_hash = {})
       @config_hash = DEFAULT_CONFIG.merge(config_hash)
@@ -30,7 +39,9 @@ module Rdkafka
 
     def consumer
       kafka = native_kafka(native_config, :rd_kafka_consumer)
+      # Redirect the main queue to the consumer
       Rdkafka::FFI.rd_kafka_poll_set_consumer(kafka)
+      # Return consumer with Kafka client
       Rdkafka::Consumer.new(kafka)
     end
 
@@ -52,7 +63,7 @@ module Rdkafka
     # using it in another way will leak memory.
     def native_config
       Rdkafka::FFI.rd_kafka_conf_new.tap do |config|
-        @config_hash.each do |key, value|
+        @config_hash.merge(REQUIRED_CONFIG).each do |key, value|
           error_buffer = ::FFI::MemoryPointer.from_string(" " * 256)
           result = Rdkafka::FFI.rd_kafka_conf_set(
             config,
@@ -65,6 +76,7 @@ module Rdkafka
             raise ConfigError.new(error_buffer.read_string)
           end
         end
+        # Set log callback
         Rdkafka::FFI.rd_kafka_conf_set_log_cb(config, Rdkafka::FFI::LogCallback)
       end
     end
@@ -82,6 +94,12 @@ module Rdkafka
         raise ClientCreationError.new(error_buffer.read_string)
       end
 
+      # Redirect log to handle's queue
+      Rdkafka::FFI.rd_kafka_set_log_queue(
+        handle,
+        Rdkafka::FFI.rd_kafka_queue_get_main(handle)
+      )
+
       ::FFI::AutoPointer.new(
         handle,
         Rdkafka::FFI.method(:rd_kafka_destroy)

data/lib/rdkafka/consumer.rb

@@ -6,6 +6,10 @@ module Rdkafka
       @native_kafka = native_kafka
     end
 
+    def close
+      Rdkafka::FFI.rd_kafka_consumer_close(@native_kafka)
+    end
+
     def subscribe(*topics)
       # Create topic partition list with topics and no partition set
       tpl = Rdkafka::FFI.rd_kafka_topic_partition_list_new(topics.length)
@@ -38,17 +42,25 @@ module Rdkafka
       if message_ptr.null?
         nil
       else
-        message = Rdkafka::FFI::Message.new(message_ptr)
-        if message.err != 0
-          raise Rdkafka::RdkafkaError.new(message.err)
+        # Create struct wrapper
+        native_message = Rdkafka::FFI::Message.new(message_ptr)
+        # Raise error if needed
+        if native_message[:err] != 0
+          raise Rdkafka::RdkafkaError.new(native_message[:err])
         end
-        message
+        # Create a message to pass out
+        Rdkafka::Message.new(native_message)
+      end
+    ensure
+      # Clean up rdkafka message if there is one
+      unless message_ptr.null?
+        Rdkafka::FFI.rd_kafka_message_destroy(message_ptr)
       end
     end
 
     def each(&block)
       loop do
-        message = poll(1000)
+        message = poll(250)
         if message
           block.call(message)
         else

data/lib/rdkafka/ffi.rb

@@ -18,10 +18,11 @@ module Rdkafka
     # Polling
 
     attach_function :rd_kafka_poll, [:pointer, :int], :void, blocking: true
+    attach_function :rd_kafka_outq_len, [:pointer], :int, blocking: true
 
     # Message struct
 
-    class Message < ::FFI::ManagedStruct
+    class Message < ::FFI::Struct
       layout :err, :int,
              :rkt, :pointer,
              :partition, :int32,
@@ -31,49 +32,10 @@ module Rdkafka
              :key_len, :size_t,
              :offset, :int64,
              :_private, :pointer
-
-      def err
-        self[:err]
-      end
-
-      def topic
-        FFI.rd_kafka_topic_name(self[:rkt])
-      end
-
-      def partition
-        self[:partition]
-      end
-
-      def payload
-        if self[:payload].null?
-          nil
-        else
-          self[:payload].read_string(self[:len])
-        end
-      end
-
-      def key
-        if self[:key].null?
-          nil
-        else
-          self[:key].read_string(self[:key_len])
-        end
-      end
-
-      def offset
-        self[:offset]
-      end
-
-      def to_s
-        "Message in '#{topic}' with key '#{key}', payload '#{payload}', partition '#{partition}', offset '#{offset}'"
-      end
-
-      def self.release(ptr)
-        rd_kafka_message_destroy(ptr)
-      end
     end
 
     attach_function :rd_kafka_message_destroy, [:pointer], :void
+    attach_function :rd_kafka_topic_new, [:pointer, :string, :pointer], :pointer
     attach_function :rd_kafka_topic_name, [:pointer], :string
 
     # TopicPartition ad TopicPartitionList structs
@@ -117,7 +79,13 @@ module Rdkafka
     callback :log_cb, [:pointer, :int, :string, :string], :void
     attach_function :rd_kafka_conf_set_log_cb, [:pointer, :log_cb], :void
 
-    LogCallback = Proc.new do |client_ptr, level, level_string, line|
+    # Log queue
+    attach_function :rd_kafka_set_log_queue, [:pointer, :pointer], :void
+    attach_function :rd_kafka_queue_get_main, [:pointer], :pointer
+
+    LogCallback = ::FFI::Function.new(
+      :void, [:pointer, :int, :string, :string]
+    ) do |_client_ptr, level, _level_string, line|
       severity = case level
                  when 0 || 1 || 2
                    Logger::FATAL
@@ -148,9 +116,10 @@ module Rdkafka
     # Consumer
 
     attach_function :rd_kafka_subscribe, [:pointer, :pointer], :int
-    attach_function :rd_kafka_commit, [:pointer, :pointer, :bool], :int
+    attach_function :rd_kafka_commit, [:pointer, :pointer, :bool], :int, blocking: true
     attach_function :rd_kafka_poll_set_consumer, [:pointer], :void
     attach_function :rd_kafka_consumer_poll, [:pointer, :int], :pointer, blocking: true
+    attach_function :rd_kafka_consumer_close, [:pointer], :void, blocking: true
 
     # Producer
 
@@ -170,7 +139,9 @@ module Rdkafka
     callback :delivery_cb, [:pointer, :pointer, :pointer], :void
     attach_function :rd_kafka_conf_set_dr_msg_cb, [:pointer, :delivery_cb], :void
 
-    DeliveryCallback = Proc.new do |client_ptr, message_ptr, opaque_ptr|
+    DeliveryCallback = ::FFI::Function.new(
+      :void, [:pointer, :pointer, :pointer]
+    ) do |client_ptr, message_ptr, opaque_ptr|
       message = Message.new(message_ptr)
       delivery_handle = Rdkafka::DeliveryHandle.new(message[:_private])
       delivery_handle[:pending] = false

data/lib/rdkafka/message.rb

@@ -0,0 +1,23 @@
+module Rdkafka
+  class Message
+    attr_reader :topic, :partition, :payload, :key, :offset
+
+    def initialize(native_message)
+      unless native_message[:rkt].null?
+        @topic = FFI.rd_kafka_topic_name(native_message[:rkt])
+      end
+      @partition = native_message[:partition]
+      unless native_message[:payload].null?
+        @payload = native_message[:payload].read_string(native_message[:len])
+      end
+      unless native_message[:key].null?
+        @key = native_message[:key].read_string(native_message[:key_len])
+      end
+      @offset = native_message[:offset]
+    end
+
+    def to_s
+      "Message in '#{topic}' with key '#{key}', payload '#{payload}', partition '#{partition}', offset '#{offset}'"
+    end
+  end
+end

data/lib/rdkafka/producer.rb

@@ -1,13 +1,26 @@
 module Rdkafka
   class Producer
     def initialize(native_kafka)
+      @closing = false
       @native_kafka = native_kafka
       # Start thread to poll client for delivery callbacks
-      @thread = Thread.new do
+      @polling_thread = Thread.new do
         loop do
-          Rdkafka::FFI.rd_kafka_poll(@native_kafka, 1000)
+          Rdkafka::FFI.rd_kafka_poll(@native_kafka, 250)
+          # Exit thread if closing and the poll queue is empty
+          if @closing && Rdkafka::FFI.rd_kafka_outq_len(@native_kafka) == 0
+            break
+          end
         end
-      end.abort_on_exception = true
+      end
+      @polling_thread.abort_on_exception = true
+    end
+
+    def close
+      # Indicate to polling thread that we're closing
+      @closing = true
+      # Wait for the polling thread to finish up
+      @polling_thread.join
     end
 
     def produce(topic:, payload: nil, key: nil, partition: nil, timestamp: nil)
@@ -75,7 +88,7 @@ module Rdkafka
     end
 
     # Wait for the delivery report
-    def wait(timeout_in_seconds=10)
+    def wait(timeout_in_seconds=60)
       timeout = if timeout_in_seconds
                   Time.now.to_i + timeout_in_seconds
                 else

data/lib/rdkafka/version.rb

@@ -1,4 +1,4 @@
 module Rdkafka
-  VERSION = "0.1.9"
+  VERSION = "0.1.10"
   LIBRDKAFKA_VERSION = "0.11.0"
 end

data/spec/rdkafka/ffi_spec.rb

@@ -1,7 +1,13 @@
 require "spec_helper"
 
 describe Rdkafka::FFI do
+  it "should load librdkafka" do
+    expect(Rdkafka::FFI.ffi_libraries.map(&:name).first).to include "librdkafka"
+  end
+
   it "should successfully call librdkafka" do
-    Rdkafka::FFI.rd_kafka_conf_new
+    expect {
+      Rdkafka::FFI.rd_kafka_conf_new
+    }.not_to raise_error
   end
 end

data/spec/rdkafka/message_spec.rb

@@ -0,0 +1,73 @@
+require "spec_helper"
+
+describe Rdkafka::Message do
+  let(:native_topic) do
+    Rdkafka::FFI.rd_kafka_topic_new(
+      native_client,
+      "topic_name",
+      nil
+    )
+  end
+  let(:payload) { nil }
+  let(:key) { nil }
+  let(:native_message) do
+    Rdkafka::FFI::Message.new.tap do |message|
+      message[:rkt] = native_topic
+      message[:partition] = 3
+      message[:offset] = 100
+      if payload
+        ptr = ::FFI::MemoryPointer.new(:char, payload.bytesize)
+        ptr.put_bytes(0, payload)
+        message[:payload] = ptr
+        message[:len] = payload.bytesize
+      end
+      if key
+        ptr = ::FFI::MemoryPointer.new(:char, key.bytesize)
+        ptr.put_bytes(0, key)
+        message[:key] = ptr
+        message[:key_len] = key.bytesize
+      end
+    end
+  end
+  subject { Rdkafka::Message.new(native_message) }
+
+  it "should have a topic" do
+    expect(subject.topic).to eq "topic_name"
+  end
+
+  it "should have a partition" do
+    expect(subject.partition).to eq 3
+  end
+
+  context "payload" do
+    it "should have a nil payload when none is present" do
+      expect(subject.payload).to be_nil
+    end
+
+    context "present payload" do
+      let(:payload) { "payload content" }
+
+      it "should have a payload" do
+        expect(subject.payload).to eq "payload content"
+      end
+    end
+  end
+
+  context "key" do
+    it "should have a nil key when none is present" do
+      expect(subject.key).to be_nil
+    end
+
+    context "present key" do
+      let(:key) { "key content" }
+
+      it "should have a key" do
+        expect(subject.key).to eq "key content"
+      end
+    end
+  end
+
+  it "should have an offset" do
+    expect(subject.offset).to eq 100
+  end
+end

data/spec/rdkafka/producer_spec.rb

@@ -2,7 +2,6 @@ require "spec_helper"
 
 describe Rdkafka::Producer do
   let(:producer) { rdkafka_config.producer }
-  let(:consumer) { rdkafka_config.consumer }
 
   it "should require a topic" do
     expect {
@@ -14,12 +13,6 @@ describe Rdkafka::Producer do
   end
 
   it "should produce a message" do
-    consumer.subscribe("produce_test_topic")
-    # Make sure the consumer is running before we produce
-    5.times do
-      consumer.poll
-    end
-
     handle = producer.produce(
       topic:   "produce_test_topic",
       payload: "payload 1",
@@ -28,20 +21,33 @@ describe Rdkafka::Producer do
     expect(handle.pending?).to be true
 
     # Check delivery handle and report
-    report = handle.wait
+    report = handle.wait(5)
     expect(handle.pending?).to be false
     expect(report).not_to be_nil
     expect(report.partition).to eq 0
     expect(report.offset).to be > 0
 
+    # Close producer
+    producer.close
+
     # Consume message and verify it's content
-    message = consumer.first
-    expect(message).not_to be_nil
+    message = wait_for_message(
+      topic: "produce_test_topic",
+      delivery_report: report
+    )
     expect(message.partition).to eq 0
-    expect(message.offset).to eq report.offset
     expect(message.payload).to eq "payload 1"
     expect(message.key).to eq "key 1"
+  end
 
-    consumer.commit
+  it "should raise a timeout error when waiting too long" do
+    handle = producer.produce(
+      topic:   "produce_test_topic",
+      payload: "payload 1",
+      key:     "key 1"
+    )
+    expect {
+      handle.wait(0)
+    }.to raise_error Rdkafka::WaitTimeoutError
   end
 end

data/spec/spec_helper.rb

@@ -6,6 +6,8 @@ def rdkafka_config
   config = {
     :"bootstrap.servers" => "localhost:9092",
     :"group.id" => "ruby_test",
+    :"client.id" => "test",
+    :"auto.offset.reset" => "earliest",
     :"enable.partition.eof" => false
   }
   if ENV["DEBUG_PRODUCER"]
@@ -15,3 +17,26 @@ def rdkafka_config
   end
   Rdkafka::Config.new(config)
 end
+
+def native_client
+  config = rdkafka_config
+  config.send(:native_kafka, config.send(:native_config), :rd_kafka_producer)
+end
+
+def wait_for_message(topic:, delivery_report:, timeout_in_seconds: 30)
+  offset = delivery_report.offset - 1
+  consumer = rdkafka_config.consumer
+  consumer.subscribe(topic)
+  timeout = Time.now.to_i + timeout_in_seconds
+  loop do
+    if timeout <= Time.now.to_i
+      raise "Timeout of #{timeout_in_seconds} seconds reached in wait_for_message"
+    end
+    message = consumer.poll(100)
+    if message && message.offset == offset
+      return message
+    end
+  end
+ensure
+  consumer.close
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.1.9
+  version: 0.1.10
 platform: ruby
 authors:
 - Thijs Cadier
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-08-31 00:00:00.000000000 Z
+date: 2017-09-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -99,6 +99,7 @@ files:
 - lib/rdkafka/consumer.rb
 - lib/rdkafka/error.rb
 - lib/rdkafka/ffi.rb
+- lib/rdkafka/message.rb
 - lib/rdkafka/producer.rb
 - lib/rdkafka/version.rb
 - rdkafka.gemspec
@@ -106,8 +107,8 @@ files:
 - spec/rdkafka/consumer_spec.rb
 - spec/rdkafka/error_spec.rb
 - spec/rdkafka/ffi_spec.rb
+- spec/rdkafka/message_spec.rb
 - spec/rdkafka/producer_spec.rb
-- spec/rdkafka_spec.rb
 - spec/spec_helper.rb
 homepage: https://github.com/thijsc/rdkafka-ruby
 licenses:
@@ -139,6 +140,6 @@ test_files:
 - spec/rdkafka/consumer_spec.rb
 - spec/rdkafka/error_spec.rb
 - spec/rdkafka/ffi_spec.rb
+- spec/rdkafka/message_spec.rb
 - spec/rdkafka/producer_spec.rb
-- spec/rdkafka_spec.rb
 - spec/spec_helper.rb

data/spec/rdkafka_spec.rb

@@ -1,4 +0,0 @@
-require "spec_helper"
-
-describe Rdkafka do
-end