rdkafka 0.3.5 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f2ca53e4caa266a4e9b4cd1fd5132626259629c1835c6923b8b5b252185884c5
-  data.tar.gz: d748f67df1abc442410de913a8b67b600264aeba55cef148ac01b0a7e863a7ce
+  metadata.gz: c361e42d0a0eb4ef3c6b7baa3509baa675f35ed8b96660e5095ba8c09ef58163
+  data.tar.gz: dbf82f43bb4f69fc2bdc43f931c28009578a331eb8e4bc4bb3606b2cdeedefc8
 SHA512:
-  metadata.gz: 65900548523763e0c448d8ceb8a08fdc96d8404fcef169db0724fc8ef50336a0bd4b50d622371eaf981ecc808d432c70b44b881a044903c7fa2f20bdacf23987
-  data.tar.gz: 8d157b0d6e3e82158c4c08c1bfed99e6ccb9c1d6ef287af81e20932c33adc82566a8df239fce2a227eea42b29f991b55499770ab85e87ca61b57afcc94b686d2
+  metadata.gz: 74485766bf68d0bb42e756dc3c381da604e01500682dcdd08b0d79a9c736696fbe4845ede037a820e5eb9482626f0ad8355344bb15f47235810e29d1e6219f28
+  data.tar.gz: ae8ec9d6a81fbdc1931c3f629fd92a32d31297fb055fcecca97f322248d31a73ce23f9fdc356645215960dd34b8a979e9288afdd8d5df9b031e2cd19a428825d

data/.travis.yml

@@ -2,9 +2,13 @@ language: ruby
 
 sudo: false
 
+services:
+  - docker
+
 env:
   global:
     - CC_TEST_REPORTER_ID=9f7f740ac1b6e264e1189fa07a6687a87bcdb9f3c0f4199d4344ab3b538e187e
+    - KAFKA_HEAP_OPTS="-Xmx512m -Xms512m"
 
 rvm:
   - 2.1
@@ -15,14 +19,10 @@ rvm:
 
 before_install:
   - gem update --system
-  - wget http://www.us.apache.org/dist/kafka/1.0.0/kafka_2.12-1.0.0.tgz -O kafka.tgz
-  - mkdir -p kafka && tar xzf kafka.tgz -C kafka --strip-components 1
-  - nohup bash -c "cd kafka && bin/zookeeper-server-start.sh config/zookeeper.properties &"
-  - nohup bash -c "cd kafka && bin/kafka-server-start.sh config/server.properties &"
 
 before_script:
+  - docker-compose up -d
   - cd ext && bundle exec rake && cd ..
-  - bundle exec rake create_topics
   - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
   - chmod +x ./cc-test-reporter
   - ./cc-test-reporter before-build
@@ -31,5 +31,5 @@ script:
   - bundle exec rspec
 
 after_script:
+  - docker-compose stop
   - ./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT
-  - killall -9 java

data/CHANGELOG.md

@@ -1,3 +1,15 @@
+# 0.4.0
+* Improvements in librdkafka archive download
+* Add global statistics callback
+* Use Time for timestamps, potentially breaking change if you
+  rely on the previous behavior where it returns an integer with
+  the number of milliseconds.
+* Bump librdkafka to 0.11.5
+* Implement TopicPartitionList in Ruby so we don't have to keep
+  track of native objects.
+* Support committing a topic partition list
+* Add consumer assignment method
+
 # 0.3.5
 * Fix crash when not waiting for delivery handles
 * Run specs on Ruby 2.5

data/README.md

@@ -8,12 +8,16 @@
 The `rdkafka` gem is a modern Kafka client library for Ruby based on
 [librdkafka](https://github.com/edenhill/librdkafka/).
 It wraps the production-ready C client using the [ffi](https://github.com/ffi/ffi)
-gem and targets Kafka 0.10+ and Ruby 2.1+.
+gem and targets Kafka 1.0+ and Ruby 2.1+.
 
 This gem only provides a high-level Kafka consumer. If you are running
 an older version of Kafka and/or need the legacy simple consumer we
 suggest using the [Hermann](https://github.com/reiseburo/hermann) gem.
 
+The most important pieces of a Kafka client are implemented. We're
+working towards feature completeness, you can track that here:
+https://github.com/appsignal/rdkafka-ruby/milestone/1
+
 ## Installation
 
 This gem downloads and compiles librdkafka when it is installed. If you
@@ -25,6 +29,9 @@ See the [documentation](http://www.rubydoc.info/github/thijsc/rdkafka-ruby/maste
 
 ### Consuming messages
 
+Subscribe to a topic and get messages. Kafka will automatically spread
+the available partitions over consumers with the same group id.
+
 ```ruby
 config = {
   :"bootstrap.servers" => "localhost:9092",
@@ -40,25 +47,45 @@ end
 
 ### Producing messages
 
+Produce a number of messages, put the delivery handles in an array and
+wait for them before exiting. This way the messages will be batched and
+sent to Kafka in an efficient way.
+
 ```ruby
 config = {:"bootstrap.servers" => "localhost:9092"}
 producer = Rdkafka::Config.new(config).producer
+delivery_handles = []
 
 100.times do |i|
   puts "Producing message #{i}"
-  producer.produce(
+  delivery_handles << producer.produce(
       topic:   "ruby-test-topic",
       payload: "Payload #{i}",
       key:     "Key #{i}"
-  ).wait
+  )
 end
+
+delivery_handles.each(&:wait)
 ```
 
+## Known issues
+
+When using forked process such as when using Unicorn you currently need
+to make sure that you create rdkafka instances after forking. Otherwise
+they will not work and crash your Ruby process when they are garbage
+collected. See https://github.com/appsignal/rdkafka-ruby/issues/19
+
 ## Development
 
-For development we expect a local zookeeper and kafka instance to be
-running. Run `bundle` and `cd ext && bundle exec rake && cd ..`. Then
-create the topics as expected in the specs: `bundle exec rake create_topics`.
+A Docker Compose file is included to run Kafka and Zookeeper. To run
+that:
+
+```
+docker-compose up
+```
+
+Run `bundle` and `cd ext && bundle exec rake && cd ..` to download and
+compile `librdkafka`.
 
 You can then run `bundle exec rspec` to run the tests. To see rdkafka
 debug output:
@@ -68,6 +95,15 @@ DEBUG_PRODUCER=true bundle exec rspec
 DEBUG_CONSUMER=true bundle exec rspec
 ```
 
+After running the tests you can bring the cluster down to start with a
+clean slate:
+
+```
+docker-compose down
+```
+
+## Example
+
 To see everything working run these in separate tabs:
 
 ```

data/Rakefile

@@ -1,33 +1,24 @@
 require "./lib/rdkafka"
 
-task :create_topics do
-  puts "Creating test topics"
-  kafka_topics = if ENV['TRAVIS']
-                   'kafka/bin/kafka-topics.sh'
-                 else
-                   'kafka-topics'
-                 end
-  `#{kafka_topics} --create --topic=consume_test_topic --zookeeper=127.0.0.1:2181 --partitions=3 --replication-factor=1`
-  `#{kafka_topics} --create --topic=empty_test_topic --zookeeper=127.0.0.1:2181 --partitions=3 --replication-factor=1`
-  `#{kafka_topics} --create --topic=load_test_topic --zookeeper=127.0.0.1:2181 --partitions=3 --replication-factor=1`
-  `#{kafka_topics} --create --topic=produce_test_topic --zookeeper=127.0.0.1:2181 --partitions=3 --replication-factor=1`
-  `#{kafka_topics} --create --topic=rake_test_topic --zookeeper=127.0.0.1:2181 --partitions=3 --replication-factor=1`
-end
-
 task :produce_messages do
   config = {:"bootstrap.servers" => "localhost:9092"}
   if ENV["DEBUG"]
     config[:debug] = "broker,topic,msg"
   end
   producer = Rdkafka::Config.new(config).producer
+
+  delivery_handles = []
   100.times do |i|
     puts "Producing message #{i}"
-    producer.produce(
+    delivery_handles << producer.produce(
         topic:   "rake_test_topic",
         payload: "Payload #{i} from Rake",
         key:     "Key #{i} from Rake"
-    ).wait
+    )
   end
+  puts 'Waiting for delivery'
+  delivery_handles.each(&:wait)
+  puts 'Done'
 end
 
 task :consume_messages do
@@ -35,11 +26,16 @@ task :consume_messages do
     :"bootstrap.servers" => "localhost:9092",
     :"group.id" => "rake_test",
     :"enable.partition.eof" => false,
-    :"auto.offset.reset" => "earliest"
+    :"auto.offset.reset" => "earliest",
+    :"statistics.interval.ms" => 10_000
   }
   if ENV["DEBUG"]
     config[:debug] = "cgrp,topic,fetch"
   end
+  Rdkafka::Config.statistics_callback = lambda do |stats|
+    puts stats
+  end
+  consumer = Rdkafka::Config.new(config).consumer
   consumer = Rdkafka::Config.new(config).consumer
   consumer.subscribe("rake_test_topic")
   consumer.each do |message|

data/docker-compose.yml

@@ -0,0 +1,18 @@
+version: '2'
+services:
+  zookeeper:
+    image: wurstmeister/zookeeper
+    ports:
+      - "2181:2181"
+  kafka:
+    image: wurstmeister/kafka:1.0.1
+    ports:
+      - "9092:9092"
+    environment:
+      KAFKA_ADVERTISED_HOST_NAME: localhost
+      KAFKA_ADVERTISED_PORT: 9092
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'
+      KAFKA_CREATE_TOPICS: "consume_test_topic:3:1,empty_test_topic:3:1,load_test_topic:3:1,produce_test_topic:3:1,rake_test_topic:3:1"
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock

data/ext/Rakefile

@@ -3,9 +3,25 @@ require "mini_portile2"
 require "fileutils"
 
 task :default => :clean do
+  # MiniPortile#download_file_http is a monkey patch that removes the download
+  # progress indicator. This indicator relies on the 'Content Length' response
+  # headers, which is not set by GitHub
+  class MiniPortile
+    def download_file_http(url, full_path, _count)
+      filename = File.basename(full_path)
+      with_tempfile(filename, full_path) do |temp_file|
+        params = { 'Accept-Encoding' => 'identity' }
+        OpenURI.open_uri(url, 'rb', params) do |io|
+          temp_file.write(io.read)
+        end
+        output
+      end
+    end
+  end
+
   # Download and compile librdkafka
   recipe = MiniPortile.new("librdkafka", Rdkafka::LIBRDKAFKA_VERSION)
-  recipe.files = ["https://github.com/edenhill/librdkafka/archive/v#{Rdkafka::LIBRDKAFKA_VERSION}.tar.gz"]
+  recipe.files = ["https://codeload.github.com/edenhill/librdkafka/tar.gz/v#{Rdkafka::LIBRDKAFKA_VERSION}"]
   recipe.configure_options = ["--host=#{recipe.host}"]
   recipe.cook
   # Move dynamic library we're interested in

data/lib/rdkafka/bindings.rb

@@ -1,4 +1,5 @@
 require "ffi"
+require "json"
 require "logger"
 
 module Rdkafka
@@ -43,7 +44,7 @@ module Rdkafka
     # TopicPartition ad TopicPartitionList structs
 
     class TopicPartition < FFI::Struct
-     layout :topic, :string,
+      layout :topic, :string,
              :partition, :int32,
              :offset, :int64,
              :metadata, :pointer,
@@ -61,6 +62,7 @@ module Rdkafka
 
     attach_function :rd_kafka_topic_partition_list_new, [:int32], :pointer
     attach_function :rd_kafka_topic_partition_list_add, [:pointer, :string, :int32], :void
+    attach_function :rd_kafka_topic_partition_list_set_offset, [:pointer, :string, :int32, :int64], :void
     attach_function :rd_kafka_topic_partition_list_destroy, [:pointer], :void
     attach_function :rd_kafka_topic_partition_list_copy, [:pointer], :pointer
 
@@ -81,6 +83,8 @@ module Rdkafka
     attach_function :rd_kafka_conf_set, [:pointer, :string, :string, :pointer, :int], :kafka_config_response
     callback :log_cb, [:pointer, :int, :string, :string], :void
     attach_function :rd_kafka_conf_set_log_cb, [:pointer, :log_cb], :void
+    callback :stats_cb, [:pointer, :string, :int, :pointer], :int
+    attach_function :rd_kafka_conf_set_stats_cb, [:pointer, :stats_cb], :void
 
     # Log queue
     attach_function :rd_kafka_set_log_queue, [:pointer, :pointer], :void
@@ -106,6 +110,19 @@ module Rdkafka
       Rdkafka::Config.logger.add(severity) { "rdkafka: #{line}" }
     end
 
+    StatsCallback = FFI::Function.new(
+      :int, [:pointer, :string, :int, :pointer]
+    ) do |_client_ptr, json, _json_len, _opaque|
+      # Pass the stats hash to callback in config
+      if Rdkafka::Config.statistics_callback
+        stats = JSON.parse(json)
+        Rdkafka::Config.statistics_callback.call(stats)
+      end
+
+      # Return 0 so librdkafka frees the json string
+      0
+    end
+
     # Handle
 
     enum :kafka_type, [
@@ -121,6 +138,7 @@ module Rdkafka
     attach_function :rd_kafka_subscribe, [:pointer, :pointer], :int
     attach_function :rd_kafka_unsubscribe, [:pointer], :int
     attach_function :rd_kafka_subscription, [:pointer, :pointer], :int
+    attach_function :rd_kafka_assign, [:pointer, :pointer], :int
     attach_function :rd_kafka_assignment, [:pointer, :pointer], :int
     attach_function :rd_kafka_committed, [:pointer, :pointer, :int], :int
     attach_function :rd_kafka_commit, [:pointer, :pointer, :bool], :int, blocking: true

data/lib/rdkafka/config.rb

@@ -7,6 +7,7 @@ module Rdkafka
   class Config
     # @private
     @@logger = Logger.new(STDOUT)
+    @@statistics_callback = nil
 
     # Returns the current logger, by default this is a logger to stdout.
     #
@@ -25,6 +26,25 @@ module Rdkafka
       @@logger=logger
     end
 
+    # Set a callback that will be called every time the underlying client emits statistics.
+    # You can configure if and how often this happens using `statistics.interval.ms`.
+    # The callback is called with a hash that's documented here: https://github.com/edenhill/librdkafka/blob/master/STATISTICS.md
+    #
+    # @param callback [Proc] The callback
+    #
+    # @return [nil]
+    def self.statistics_callback=(callback)
+      raise TypeError.new("Callback has to be a proc or lambda") unless callback.is_a? Proc
+      @@statistics_callback = callback
+    end
+
+    # Returns the current statistics callback, by default this is nil.
+    #
+    # @return [Proc, nil]
+    def self.statistics_callback
+      @@statistics_callback
+    end
+
     # Default config that can be overwritten.
     DEFAULT_CONFIG = {
       # Request api version so advanced features work
@@ -122,8 +142,12 @@ module Rdkafka
             raise ConfigError.new(error_buffer.read_string)
           end
         end
+        # Set opaque pointer back to this config
+        #Rdkafka::Bindings.rd_kafka_conf_set_opaque(config, self)
         # Set log callback
         Rdkafka::Bindings.rd_kafka_conf_set_log_cb(config, Rdkafka::Bindings::LogCallback)
+        # Set stats callback
+        Rdkafka::Bindings.rd_kafka_conf_set_stats_cb(config, Rdkafka::Bindings::StatsCallback)
       end
     end
 

data/lib/rdkafka/consumer.rb

@@ -65,34 +65,68 @@ module Rdkafka
     # @return [TopicPartitionList]
     def subscription
       tpl = FFI::MemoryPointer.new(:pointer)
+      tpl.autorelease = false
       response = Rdkafka::Bindings.rd_kafka_subscription(@native_kafka, tpl)
       if response != 0
         raise Rdkafka::RdkafkaError.new(response)
       end
-      Rdkafka::Consumer::TopicPartitionList.new(tpl.get_pointer(0))
+      Rdkafka::Consumer::TopicPartitionList.from_native_tpl(tpl.get_pointer(0))
+    end
+
+    # Atomic assignment of partitions to consume
+    #
+    # @param list [TopicPartitionList] The topic with partitions to assign
+    #
+    # @raise [RdkafkaError] When assigning fails
+    def assign(list)
+      unless list.is_a?(TopicPartitionList)
+        raise TypeError.new("list has to be a TopicPartitionList")
+      end
+      tpl = list.to_native_tpl
+      response = Rdkafka::Bindings.rd_kafka_assign(@native_kafka, tpl)
+      if response != 0
+        raise Rdkafka::RdkafkaError.new(response, "Error assigning '#{list.to_h}'")
+      end
+    ensure
+      Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl) if tpl
+    end
+
+    # Returns the current partition assignment.
+    #
+    # @raise [RdkafkaError] When getting the assignment fails.
+    #
+    # @return [TopicPartitionList]
+    def assignment
+      tpl = FFI::MemoryPointer.new(:pointer)
+      tpl.autorelease = false
+      response = Rdkafka::Bindings.rd_kafka_assignment(@native_kafka, tpl)
+      if response != 0
+        raise Rdkafka::RdkafkaError.new(response)
+      end
+      Rdkafka::Consumer::TopicPartitionList.from_native_tpl(tpl.get_pointer(0))
     end
 
     # Return the current committed offset per partition for this consumer group.
     # The offset field of each requested partition will either be set to stored offset or to -1001 in case there was no stored offset for that partition.
     #
-    # TODO: This should use the subscription or assignment by default.
-    #
-    # @param list [TopicPartitionList] The topic with partitions to get the offsets for.
+    # @param list [TopicPartitionList, nil] The topic with partitions to get the offsets for or nil to use the current subscription.
     # @param timeout_ms [Integer] The timeout for fetching this information.
     #
     # @raise [RdkafkaError] When getting the committed positions fails.
     #
     # @return [TopicPartitionList]
-    def committed(list, timeout_ms=200)
-      unless list.is_a?(TopicPartitionList)
-        raise TypeError.new("list has to be a TopicPartitionList")
+    def committed(list=nil, timeout_ms=200)
+      if list.nil?
+        list = assignment
+      elsif !list.is_a?(TopicPartitionList)
+        raise TypeError.new("list has to be nil or a TopicPartitionList")
       end
-      tpl = list.copy_tpl
+      tpl = list.to_native_tpl
       response = Rdkafka::Bindings.rd_kafka_committed(@native_kafka, tpl, timeout_ms)
       if response != 0
         raise Rdkafka::RdkafkaError.new(response)
       end
-      Rdkafka::Consumer::TopicPartitionList.new(tpl)
+      TopicPartitionList.from_native_tpl(tpl)
     end
 
     # Query broker for low (oldest/beginning) and high (newest/end) offsets for a partition.
@@ -160,11 +194,21 @@ module Rdkafka
     # @raise [RdkafkaError] When comitting fails
     #
     # @return [nil]
-    def commit(async=false)
-      response = Rdkafka::Bindings.rd_kafka_commit(@native_kafka, nil, async)
+    def commit(list=nil, async=false)
+      if !list.nil? && !list.is_a?(TopicPartitionList)
+        raise TypeError.new("list has to be nil or a TopicPartitionList")
+      end
+      tpl = if list
+              list.to_native_tpl
+            else
+              nil
+            end
+      response = Rdkafka::Bindings.rd_kafka_commit(@native_kafka, tpl, async)
       if response != 0
         raise Rdkafka::RdkafkaError.new(response)
       end
+    ensure
+      Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl) if tpl
     end
 
     # Poll for the next message on one of the subscribed topics