rdkafka 0.0.1 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c019ec1ebf89bf81ce40340c8866e589d86826e3
-  data.tar.gz: cf1fdaadf5527542dbc1847587f43d8497d7f3d6
+  metadata.gz: 1444aae0a46b7a7fc96660980a185d659a59c1b9
+  data.tar.gz: c8b0a132119a090cd7a004c6dbe3c6b1fc06db3c
 SHA512:
-  metadata.gz: b58a95c81f0ae2b09d347acf12dbaf71057c4c26c25a6411eb5212723eee33d0b107db61fef385a39bb05079db586ba2d9406f6f1f097f5bb49e8c6852206a33
-  data.tar.gz: 3f5b7bd2277397df7acc096b3145a2e65e33df4d9ee1c7646b5d33d20992daecdd78fffb996abaccbb6b3304d131a7123a8fb94b2d7fdef4e8ad43b6403cdac5
+  metadata.gz: ba75b3de888cda846b746fdcb4b5dcbdde21a0d8efcb4143e0176bbfea0ab34dce2d8180037c6c0d6fd9dd63770b0adeb460b1beb3a5d278f423b4a8916bf3a1
+  data.tar.gz: d825345bfb72038f9b577ec7dfe5a429087ad71fe133e17fa9a65d6b7945905d22e75695dbbef8305d8b459f2f19c99dcd117746152034a492cedbf3f2bf3e38

data/README.md

@@ -4,5 +4,13 @@ Kafka client library wrapping `librdkafka` using the FFI gem for Kafka 0.10+ and
 
 ## Development
 
-Run `bundle` and `cd ext && bundle exec rake compile && cd ..`. You can then run
-`bundle exec rspec` to run the tests.
+Run `bundle` and `cd ext && bundle exec rake compile && cd ..`. Then
+create the topics as expected in the specs: `bundle exec rake create_topics`.
+
+You can then run `bundle exec rspec` to run the tests. To see rdkafka
+debug output:
+
+```
+DEBUG_PRODUCER=true bundle exec rspec
+DEBUG_CONSUMER=true bundle exec rspec
+```

data/Rakefile

@@ -0,0 +1,30 @@
+require "./lib/rdkafka"
+
+task :create_topics do
+  `kafka-topics --create --topic=produce_test_topic --zookeeper=127.0.0.1:2181 --partitions=1 --replication-factor=1`
+  `kafka-topics --create --topic=rake_test_topic --zookeeper=127.0.0.1:2181 --partitions=1 --replication-factor=1`
+end
+
+task :produce_message do
+  producer = Rdkafka::Config.new(
+    :"bootstrap.servers" => "localhost:9092"
+  ).producer
+  producer.produce(
+      topic:   "rake_test_topic",
+      payload: "payload from Rake",
+      key:     "key from Rake"
+  ).wait
+end
+
+task :consume_messages do
+  consumer = Rdkafka::Config.new(
+    :"bootstrap.servers" => "localhost:9092",
+    :"group.id" => "rake_test",
+    :"enable.partition.eof" => false,
+    :"auto.offset.reset" => "earliest"
+  ).consumer
+  consumer.subscribe("rake_test_topic")
+  consumer.each do |message|
+    puts message
+  end
+end

data/lib/rdkafka/config.rb

@@ -1,7 +1,7 @@
 module Rdkafka
   class Config
     DEFAULT_CONFIG = {
-      "api.version.request" => "true"
+      :"api.version.request" => true
     }
 
     def initialize(config_hash = {})
@@ -17,7 +17,9 @@ module Rdkafka
     end
 
     def consumer
-      Rdkafka::Consumer.new(native_kafka(native_config, :rd_kafka_consumer))
+      kafka = native_kafka(native_config, :rd_kafka_consumer)
+      Rdkafka::FFI.rd_kafka_poll_set_consumer(kafka)
+      Rdkafka::Consumer.new(kafka)
     end
 
     def producer
@@ -43,8 +45,8 @@ module Rdkafka
         error_buffer = ::FFI::MemoryPointer.from_string(" " * 256)
         result = Rdkafka::FFI.rd_kafka_conf_set(
           config,
-          key,
-          value,
+          key.to_s,
+          value.to_s,
           error_buffer,
           256
         )

data/lib/rdkafka/consumer.rb

@@ -1,7 +1,63 @@
 module Rdkafka
   class Consumer
+    include Enumerable
+
     def initialize(native_kafka)
       @native_kafka = native_kafka
     end
+
+    def subscribe(*topics)
+      # Create topic partition list with topics and no partition set
+      tpl = Rdkafka::FFI.rd_kafka_topic_partition_list_new(topics.length)
+      topics.each do |topic|
+        Rdkafka::FFI.rd_kafka_topic_partition_list_add(
+          tpl,
+          topic,
+          -1
+        )
+      end
+      # Subscribe to topic partition list and check this was successful
+      response = Rdkafka::FFI.rd_kafka_subscribe(@native_kafka, tpl)
+      if response != 0
+        raise Rdkafka::RdkafkaError.new(response)
+      end
+    ensure
+      # Clean up the topic partition list
+      Rdkafka::FFI.rd_kafka_topic_partition_list_destroy(tpl)
+    end
+
+    def commit(async=false)
+      response = Rdkafka::FFI.rd_kafka_commit(@native_kafka, nil, async)
+      if response != 0
+        raise Rdkafka::RdkafkaError.new(response)
+      end
+    end
+
+    def poll(timeout_ms=100)
+      message_ptr = Rdkafka::FFI.rd_kafka_consumer_poll(@native_kafka, timeout_ms)
+      if message_ptr.null?
+        nil
+      else
+        message = Rdkafka::FFI::Message.new(message_ptr)
+        if message.err != 0
+          raise Rdkafka::RdkafkaError.new(message.err)
+        end
+        message
+      end
+    end
+
+    def each(&block)
+      loop do
+        message = poll(10)
+        if message
+          block.call(message)
+        else
+          # Sleep here instead of using a longer poll timeout so interrupting the
+          # program works properly, MRI has a hard time interrupting FFI calls.
+          sleep 0.1
+          next
+        end
+      end
+    end
   end
 end

data/lib/rdkafka/error.rb

@@ -3,23 +3,25 @@ module Rdkafka
     attr_reader :rdkafka_response
 
     def initialize(response)
+      raise TypeError.new("Response has to be an integer") unless response.is_a? Integer
       @rdkafka_response = response
     end
 
     def code
-      if @rdkafka_response.nil?
-        :unknown_error
+      code = Rdkafka::FFI.rd_kafka_err2name(@rdkafka_response).downcase
+      if code[0] == "_"
+        code[1..-1].to_sym
       else
-        Rdkafka::FFI.rd_kafka_err2name(@rdkafka_response).downcase.to_sym
+        code.to_sym
       end
     end
 
     def to_s
-      if @rdkafka_response.nil?
-        "Unknown error: Response code is nil"
-      else
-        Rdkafka::FFI.rd_kafka_err2str(@rdkafka_response)
-      end
+      "#{Rdkafka::FFI.rd_kafka_err2str(@rdkafka_response)} (#{code})"
+    end
+
+    def is_partition_eof?
+      code == :partition_eof
     end
   end
 end

data/lib/rdkafka/ffi.rb

@@ -7,11 +7,12 @@ module Rdkafka
     ffi_lib "ext/ports/#{MiniPortile.new("librdkafka", Rdkafka::LIBRDKAFKA_VERSION).host}/librdkafka/#{Rdkafka::LIBRDKAFKA_VERSION}/lib/librdkafka.dylib"
 
     # Polling
+
     attach_function :rd_kafka_poll, [:pointer, :int], :void
 
     # Message struct
 
-    class Message < ::FFI::Struct
+    class Message < ::FFI::ManagedStruct
       layout :err, :int,
              :rkt, :pointer,
              :partition, :int32,
@@ -21,8 +22,69 @@ module Rdkafka
              :key_len, :size_t,
              :offset, :int64,
              :_private, :pointer
+
+      def err
+        self[:err]
+      end
+
+      def partition
+        self[:partition]
+      end
+
+      def payload
+        if self[:payload].null?
+          nil
+        else
+          self[:payload].read_string(self[:len])
+        end
+      end
+
+      def key
+        if self[:key].null?
+          nil
+        else
+          self[:key].read_string(self[:key_len])
+        end
+      end
+
+      def offset
+        self[:offset]
+      end
+
+      def to_s
+        "Message with key '#{key}', payload '#{payload}', partition '#{partition}', offset '#{offset}'"
+      end
+
+      def self.release(ptr)
+        rd_kafka_message_destroy(ptr)
+      end
+    end
+
+    attach_function :rd_kafka_message_destroy, [:pointer], :void
+
+    # TopicPartition ad TopicPartitionList structs
+
+    class TopicPartition < ::FFI::Struct
+      layout :topic, :string,
+             :partition, :int32,
+             :offset, :int64,
+             :metadata, :pointer,
+             :metadata_size, :size_t,
+             :opaque, :pointer,
+             :err, :int,
+             :_private, :pointer
+    end
+
+    class TopicPartitionList < ::FFI::Struct
+      layout :cnt, :int,
+             :size, :int,
+             :elems, TopicPartition.ptr
     end
 
+    attach_function :rd_kafka_topic_partition_list_new, [:int32], :pointer
+    attach_function :rd_kafka_topic_partition_list_add, [:pointer, :string, :int32], :void
+    attach_function :rd_kafka_topic_partition_list_destroy, [:pointer], :void
+
     # Errors
 
     attach_function :rd_kafka_err2name, [:int], :string
@@ -49,7 +111,14 @@ module Rdkafka
     attach_function :rd_kafka_new, [:kafka_type, :pointer, :pointer, :int], :pointer
     attach_function :rd_kafka_destroy, [:pointer], :void
 
-    # Producing
+    # Consumer
+
+    attach_function :rd_kafka_subscribe, [:pointer, :pointer], :int
+    attach_function :rd_kafka_commit, [:pointer, :pointer, :bool], :int
+    attach_function :rd_kafka_poll_set_consumer, [:pointer], :void
+    attach_function :rd_kafka_consumer_poll, [:pointer, :int], :pointer
+
+    # Producer
 
     RD_KAFKA_VTYPE_END = 0
     RD_KAFKA_VTYPE_TOPIC = 1

data/lib/rdkafka/version.rb

@@ -1,4 +1,4 @@
 module Rdkafka
-  VERSION = "0.0.1"
+  VERSION = "0.1.0"
   LIBRDKAFKA_VERSION = "0.11.0"
 end

data/rdkafka.gemspec

@@ -14,7 +14,7 @@ Gem::Specification.new do |gem|
   gem.name = 'rdkafka'
   gem.require_paths = ['lib']
   gem.version = Rdkafka::VERSION
-  gem.required_ruby_version = '>= 2.0'
+  gem.required_ruby_version = '>= 2.1'
   gem.extensions = %w(ext/Rakefile)
 
   gem.add_dependency 'ffi', '~> 1.9'

data/spec/rdkafka/error_spec.rb

@@ -1,11 +1,13 @@
 require "spec_helper"
 
 describe Rdkafka::RdkafkaError do
-  describe "#code" do
-    it "should handle a nil response" do
-      expect(Rdkafka::RdkafkaError.new(nil).code).to eq :unknown_error
-    end
+  it "should raise a type error for a nil response" do
+    expect {
+      Rdkafka::RdkafkaError.new(nil)
+    }.to raise_error TypeError
+  end
 
+  describe "#code" do
     it "should handle an invalid response" do
       expect(Rdkafka::RdkafkaError.new(933975).code).to eq :err_933975?
     end
@@ -13,19 +15,29 @@ describe Rdkafka::RdkafkaError do
     it "should return error messages from rdkafka" do
       expect(Rdkafka::RdkafkaError.new(10).code).to eq :msg_size_too_large
     end
-  end
 
-  describe "#to_s" do
-    it "should handle a nil response" do
-      expect(Rdkafka::RdkafkaError.new(nil).to_s).to eq "Unknown error: Response code is nil"
+    it "should strip a leading underscore" do
+      expect(Rdkafka::RdkafkaError.new(-191).code).to eq :partition_eof
     end
+  end
 
+  describe "#to_s" do
     it "should handle an invalid response" do
-      expect(Rdkafka::RdkafkaError.new(933975).to_s).to eq "Err-933975?"
+      expect(Rdkafka::RdkafkaError.new(933975).to_s).to eq "Err-933975? (err_933975?)"
     end
 
     it "should return error messages from rdkafka" do
-      expect(Rdkafka::RdkafkaError.new(10).to_s).to eq "Broker: Message size too large"
+      expect(Rdkafka::RdkafkaError.new(10).to_s).to eq "Broker: Message size too large (msg_size_too_large)"
+    end
+  end
+
+  describe "#is_partition_eof?" do
+    it "should be false when not partition eof" do
+      expect(Rdkafka::RdkafkaError.new(933975).is_partition_eof?).to be false
+    end
+
+    it "should be true when partition eof" do
+      expect(Rdkafka::RdkafkaError.new(-191).is_partition_eof?).to be true
     end
   end
 end

data/spec/rdkafka/producer_spec.rb

@@ -1,31 +1,47 @@
 require "spec_helper"
 
 describe Rdkafka::Producer do
-  let(:producer) do
-    rdkafka_config.producer
-  end
+  let(:producer) { rdkafka_config.producer }
+  let(:consumer) { rdkafka_config.consumer }
 
   it "should require a topic" do
     expect {
       producer.produce(
         payload: "payload",
-        key: "key"
+        key:     "key"
      )
     }.to raise_error ArgumentError, "missing keyword: topic"
   end
 
   it "should produce a message" do
+    consumer.subscribe("produce_test_topic")
+    # Make sure the consumer is running before we produce
+    5.times do
+      consumer.poll
+    end
+
     handle = producer.produce(
-      topic: "produce_test_topic",
+      topic:   "produce_test_topic",
       payload: "payload 1",
-      key: "key 1"
+      key:     "key 1"
     )
     expect(handle.pending?).to be true
 
+    # Check delivery handle and report
     report = handle.wait
     expect(handle.pending?).to be false
     expect(report).not_to be_nil
     expect(report.partition).to eq 0
     expect(report.offset).to be > 0
+
+    # Consume message and verify it's content
+    message = consumer.first
+    expect(message).not_to be_nil
+    expect(message.partition).to eq 0
+    expect(message.offset).to eq report.offset
+    expect(message.payload).to eq "payload 1"
+    expect(message.key).to eq "key 1"
+
+    consumer.commit
   end
 end

data/spec/spec_helper.rb

@@ -3,5 +3,17 @@ require "rspec"
 require "rdkafka"
 
 def rdkafka_config
-  Rdkafka::Config.new("bootstrap.servers" => "localhost:9092")
+  debug = if ENV["DEBUG_PRODUCER"]
+            "broker,topic,msg"
+          elsif ENV["DEBUG_CONSUMER"]
+            "cgrp,topic,fetch"
+          else
+            ""
+          end
+  Rdkafka::Config.new(
+    :"bootstrap.servers" => "localhost:9092",
+    :"group.id" => "ruby_test",
+    :"enable.partition.eof" => false,
+    :"debug" => debug
+  )
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.0
 platform: ruby
 authors:
 - Thijs Cadier
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-08-28 00:00:00.000000000 Z
+date: 2017-08-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -92,6 +92,7 @@ files:
 - Gemfile
 - LICENSE
 - README.md
+- Rakefile
 - ext/Rakefile
 - lib/rdkafka.rb
 - lib/rdkafka/config.rb
@@ -120,7 +121,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.0'
+      version: '2.1'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="