rdkafka 0.6.0 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0b248aee9a260de3d8bb56efba9255f21d3bfcb6c269a1ab15beac7ae2521b6c
-  data.tar.gz: 2d1199d5717a832674f659ac1d5e69d2a659845e9c8e5509428270a4a8390ae1
+  metadata.gz: 38ccc62319eff24c7def2b8cebced2a8551b28a1651e77348c868ce189a7d418
+  data.tar.gz: e23e2cad27a95e2bf51c2428e4d2246deae903f8e0d7800b086b35c5b262fc65
 SHA512:
-  metadata.gz: 62afc350fa40f5e2117e7791927fe5ad7fcb3cb7ba13cdbba2f450dc6537d41e37668a5997fccbb260abe3fbb09e7a73c0cb5d5059b94837d5097f373c27eb75
-  data.tar.gz: 1723633bcce6569992a2368df38c2d59b217dd94af9e8afab9932ca085e9ae0fbc3661ee8fbc370d31f3985204b38b3bbe15bbdfa11ca2be61c69b4c938a40f3
+  metadata.gz: 5faa270613e25d64f2b5d955a637b97b8315e7e365843b0e5a10d34c6340a795ebcf8ed2cb9dba2ae3a7d6839e464271d979760d351a6a8777224c31cdb262b6
+  data.tar.gz: d209464e6fe089945638063caa4ec6ab80d7c0d171678c43e320314cdc2ff0f980806b92941d0529b8ec93d598e5a637673fe3d97df4755503d2b484cc8f670b

data/.travis.yml

@@ -11,7 +11,6 @@ env:
     - KAFKA_HEAP_OPTS="-Xmx512m -Xms512m"
 
 rvm:
-  - 2.3
   - 2.4
   - 2.5
   - 2.6

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+# 0.6.0
+* Bump librdkafka to 1.2.0 (by rob-as)
+* Allow customizing the wait time for delivery report availability (by mensfeld )
+
 # 0.6.0
 * Bump librdkafka to 1.1.0 (by Chris Gaffney)
 * Implement seek (by breunigs)
@@ -7,6 +11,7 @@
 * Add cluster and member information (by dmexe)
 * Support message headers for consumer & producer (by dmexe)
 * Add consumer rebalance listener (by dmexe)
+* Implement pause/resume partitions (by dmexe)
 
 # 0.4.2
 * Delivery callback for producer

data/README.md

@@ -8,7 +8,11 @@
 The `rdkafka` gem is a modern Kafka client library for Ruby based on
 [librdkafka](https://github.com/edenhill/librdkafka/).
 It wraps the production-ready C client using the [ffi](https://github.com/ffi/ffi)
-gem and targets Kafka 1.0+ and Ruby 2.3+.
+gem and targets Kafka 1.0+ and Ruby 2.4+.
+
+`rdkafka` was written because we needed a reliable Ruby client for
+Kafka that supports modern Kafka at [AppSignal](https://appsignal.com).
+We run it in production on very high traffic systems.
 
 This gem only provides a high-level Kafka consumer. If you are running
 an older version of Kafka and/or need the legacy simple consumer we
@@ -68,13 +72,6 @@ end
 delivery_handles.each(&:wait)
 ```
 
-## Known issues
-
-When using forked process such as when using Unicorn you currently need
-to make sure that you create rdkafka instances after forking. Otherwise
-they will not work and crash your Ruby process when they are garbage
-collected. See https://github.com/appsignal/rdkafka-ruby/issues/19
-
 ## Development
 
 A Docker Compose file is included to run Kafka and Zookeeper. To run

data/lib/rdkafka/config.rb

@@ -67,7 +67,7 @@ module Rdkafka
 
     # Returns a new config with the provided options which are merged with {DEFAULT_CONFIG}.
     #
-    # @param config_hash [Hash<String,Symbol => String>] The config options for rdkafka
+    # @param config_hash [Hash{String,Symbol => String}] The config options for rdkafka
     #
     # @return [Config]
     def initialize(config_hash = {})
@@ -155,7 +155,7 @@ module Rdkafka
 
     private
 
-    # This method is only intented to be used to create a client,
+    # This method is only intended to be used to create a client,
     # using it in another way will leak memory.
     def native_config(opaque=nil)
       Rdkafka::Bindings.rd_kafka_conf_new.tap do |config|

data/lib/rdkafka/consumer.rb

@@ -318,7 +318,7 @@ module Rdkafka
     # @param list [TopicPartitionList,nil] The topic with partitions to commit
     # @param async [Boolean] Whether to commit async or wait for the commit to finish
     #
-    # @raise [RdkafkaError] When comitting fails
+    # @raise [RdkafkaError] When committing fails
     #
     # @return [nil]
     def commit(list=nil, async=false)
@@ -367,16 +367,20 @@ module Rdkafka
     # Poll for new messages and yield for each received one. Iteration
     # will end when the consumer is closed.
     #
+    # If `enable.partition.eof` is turned on in the config this will raise an
+    # error when an eof is reached, so you probably want to disable that when
+    # using this method of iteration.
+    #
     # @raise [RdkafkaError] When polling fails
     #
     # @yieldparam message [Message] Received message
     #
     # @return [nil]
-    def each(&block)
+    def each
       loop do
         message = poll(250)
         if message
-          block.call(message)
+          yield(message)
         else
           if @closing
             break

data/lib/rdkafka/consumer/partition.rb

@@ -11,7 +11,7 @@ module Rdkafka
       attr_reader :offset
 
       # Partition's error code
-      # @retuen [Integer]
+      # @return [Integer]
       attr_reader :err
 
       # @private

data/lib/rdkafka/consumer/topic_partition_list.rb

@@ -4,7 +4,7 @@ module Rdkafka
     class TopicPartitionList
       # Create a topic partition list.
       #
-      # @param data [Hash<String => [nil,Partition]>] The topic and partion data or nil to create an empty list
+      # @param data [Hash{String => nil,Partition}] The topic and partition data or nil to create an empty list
       #
       # @return [TopicPartitionList]
       def initialize(data=nil)
@@ -71,7 +71,7 @@ module Rdkafka
 
       # Return a `Hash` with the topics as keys and and an array of partition information as the value if present.
       #
-      # @return [Hash<String, [Array<Partition>, nil]>]
+      # @return [Hash{String => Array<Partition>,nil}]
       def to_h
         @data
       end

data/lib/rdkafka/error.rb

@@ -41,7 +41,7 @@ module Rdkafka
     end
   end
 
-  # Error with potic partition list returned by the underlying rdkafka library.
+  # Error with topic partition list returned by the underlying rdkafka library.
   class RdkafkaTopicPartitionListError < RdkafkaError
     # @return [TopicPartitionList]
     attr_reader :topic_partition_list

data/lib/rdkafka/producer.rb

@@ -2,7 +2,10 @@ module Rdkafka
   # A producer for Kafka messages. To create a producer set up a {Config} and call {Config#producer producer} on that.
   class Producer
     # @private
-    @delivery_callback = nil
+    # Returns the current delivery callback, by default this is nil.
+    #
+    # @return [Proc, nil]
+    attr_reader :delivery_callback
 
     # @private
     def initialize(native_kafka)
@@ -32,13 +35,6 @@ module Rdkafka
       @delivery_callback = callback
     end
 
-    # Returns the current delivery callback, by default this is nil.
-    #
-    # @return [Proc, nil]
-    def delivery_callback
-      @delivery_callback
-    end
-
     # Close this producer and wait for the internal poll queue to empty.
     def close
       # Indicate to polling thread that we're closing
@@ -50,7 +46,7 @@ module Rdkafka
     # Produces a message to a Kafka topic. The message is added to rdkafka's queue, call {DeliveryHandle#wait wait} on the returned delivery handle to make sure it is delivered.
     #
     # When no partition is specified the underlying Kafka library picks a partition based on the key. If no key is specified, a random partition will be used.
-    # When a timestamp is provided this is used instead of the autogenerated timestamp.
+    # When a timestamp is provided this is used instead of the auto-generated timestamp.
     #
     # @param topic [String] The topic to produce to
     # @param payload [String,nil] The message's payload
@@ -133,7 +129,7 @@ module Rdkafka
         *args
       )
 
-      # Raise error if the produce call was not successfull
+      # Raise error if the produce call was not successful
       if response != 0
         DeliveryHandle.remove(delivery_handle.to_ptr.address)
         raise RdkafkaError.new(response)

data/lib/rdkafka/producer/delivery_handle.rb

@@ -10,6 +10,10 @@ module Rdkafka
 
       REGISTRY = {}
 
+      CURRENT_TIME = -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }.freeze
+
+      private_constant :CURRENT_TIME
+
       def self.register(address, handle)
         REGISTRY[address] = handle
       end
@@ -29,25 +33,25 @@ module Rdkafka
       # If there is a timeout this does not mean the message is not delivered, rdkafka might still be working on delivering the message.
       # In this case it is possible to call wait again.
       #
-      # @param timeout_in_seconds [Integer, nil] Number of seconds to wait before timing out. If this is nil it does not time out.
+      # @param max_wait_timeout [Numeric, nil] Amount of time to wait before timing out. If this is nil it does not time out.
+      # @param wait_timeout [Numeric] Amount of time we should wait before we recheck if there is a delivery report available
       #
       # @raise [RdkafkaError] When delivering the message failed
       # @raise [WaitTimeoutError] When the timeout has been reached and the handle is still pending
       #
       # @return [DeliveryReport]
-      def wait(timeout_in_seconds=60)
-        timeout = if timeout_in_seconds
-                    Time.now.to_i + timeout_in_seconds
+      def wait(max_wait_timeout: 60, wait_timeout: 0.1)
+        timeout = if max_wait_timeout
+                    CURRENT_TIME.call + max_wait_timeout
                   else
                     nil
                   end
         loop do
           if pending?
-            if timeout && timeout <= Time.now.to_i
-              raise WaitTimeoutError.new("Waiting for delivery timed out after #{timeout_in_seconds} seconds")
+            if timeout && timeout <= CURRENT_TIME.call
+              raise WaitTimeoutError.new("Waiting for delivery timed out after #{max_wait_timeout} seconds")
             end
-            sleep 0.1
-            next
+            sleep wait_timeout
           elsif self[:response] != 0
             raise RdkafkaError.new(self[:response])
           else

data/lib/rdkafka/producer/delivery_report.rb

@@ -1,6 +1,6 @@
 module Rdkafka
   class Producer
-    # Delivery report for a succesfully produced message.
+    # Delivery report for a successfully produced message.
     class DeliveryReport
       # The partition this message was produced to.
       # @return [Integer]

data/lib/rdkafka/version.rb

@@ -1,5 +1,5 @@
 module Rdkafka
-  VERSION = "0.6.0"
-  LIBRDKAFKA_VERSION = "1.1.0"
-  LIBRDKAFKA_SOURCE_SHA256 = "123b47404c16bcde194b4bd1221c21fdce832ad12912bd8074f88f64b2b86f2b"
+  VERSION = "0.7.0"
+  LIBRDKAFKA_VERSION = "1.2.0"
+  LIBRDKAFKA_SOURCE_SHA256 = "eedde1c96104e4ac2d22a4230e34f35dd60d53976ae2563e3dd7c27190a96859"
 end

data/rdkafka.gemspec

@@ -4,7 +4,7 @@ Gem::Specification.new do |gem|
   gem.authors = ['Thijs Cadier']
   gem.email = ["thijs@appsignal.com"]
   gem.description = "Modern Kafka client library for Ruby based on librdkafka"
-  gem.summary = "Kafka client library wrapping librdkafka using the ffi gem and futures from concurrent-ruby for Kafka 0.10+"
+  gem.summary = "The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka. It wraps the production-ready C client using the ffi gem and targets Kafka 1.0+ and Ruby 2.4+."
   gem.license = 'MIT'
   gem.homepage = 'https://github.com/thijsc/rdkafka-ruby'
 
@@ -14,7 +14,7 @@ Gem::Specification.new do |gem|
   gem.name = 'rdkafka'
   gem.require_paths = ['lib']
   gem.version = Rdkafka::VERSION
-  gem.required_ruby_version = '>= 2.1'
+  gem.required_ruby_version = '>= 2.4'
   gem.extensions = %w(ext/Rakefile)
 
   gem.add_dependency 'ffi', '~> 1.9'

data/spec/rdkafka/consumer_spec.rb

@@ -85,7 +85,7 @@ describe Rdkafka::Consumer do
         tpl.add_topic("consume_test_topic", (0..2))
         consumer.resume(tpl)
 
-        # 8. ensure that message is successfuly consumed
+        # 8. ensure that message is successfully consumed
         records = consumer.poll(timeout)
         expect(records).not_to be_nil
         consumer.commit

data/spec/rdkafka/producer/delivery_handle_spec.rb

@@ -46,7 +46,7 @@ describe Rdkafka::Producer::DeliveryHandle do
 
     it "should wait until the timeout and then raise an error" do
       expect {
-        subject.wait(0.1)
+        subject.wait(max_wait_timeout: 0.1)
       }.to raise_error Rdkafka::Producer::DeliveryHandle::WaitTimeoutError
     end
 
@@ -61,7 +61,7 @@ describe Rdkafka::Producer::DeliveryHandle do
       end
 
       it "should wait without a timeout" do
-        report = subject.wait(nil)
+        report = subject.wait(max_wait_timeout: nil)
 
         expect(report.partition).to eq(2)
         expect(report.offset).to eq(100)

data/spec/rdkafka/producer_spec.rb

@@ -42,7 +42,7 @@ describe Rdkafka::Producer do
       )
 
       # Wait for it to be delivered
-      handle.wait(5)
+      handle.wait(max_wait_timeout: 5)
 
       # Callback should have been called
       expect(@callback_called).to be true
@@ -70,7 +70,7 @@ describe Rdkafka::Producer do
     expect(handle.pending?).to be true
 
     # Check delivery handle and report
-    report = handle.wait(5)
+    report = handle.wait(max_wait_timeout: 5)
     expect(handle.pending?).to be false
     expect(report).not_to be_nil
     expect(report.partition).to eq 1
@@ -100,7 +100,7 @@ describe Rdkafka::Producer do
       key:       "key partition",
       partition: 1
     )
-    report = handle.wait(5)
+    report = handle.wait(max_wait_timeout: 5)
 
     # Consume message and verify it's content
     message = wait_for_message(
@@ -117,7 +117,7 @@ describe Rdkafka::Producer do
       payload: "Τη γλώσσα μου έδωσαν ελληνική",
       key:     "key utf8"
     )
-    report = handle.wait(5)
+    report = handle.wait(max_wait_timeout: 5)
 
     # Consume message and verify it's content
     message = wait_for_message(
@@ -149,7 +149,7 @@ describe Rdkafka::Producer do
         key:       "key timestamp",
         timestamp: 1505069646252
       )
-      report = handle.wait(5)
+      report = handle.wait(max_wait_timeout: 5)
 
       # Consume message and verify it's content
       message = wait_for_message(
@@ -169,7 +169,7 @@ describe Rdkafka::Producer do
         key:       "key timestamp",
         timestamp: Time.at(1505069646, 353_000)
       )
-      report = handle.wait(5)
+      report = handle.wait(max_wait_timeout: 5)
 
       # Consume message and verify it's content
       message = wait_for_message(
@@ -188,7 +188,7 @@ describe Rdkafka::Producer do
       topic:   "produce_test_topic",
       payload: "payload no key"
     )
-    report = handle.wait(5)
+    report = handle.wait(max_wait_timeout: 5)
 
     # Consume message and verify it's content
     message = wait_for_message(
@@ -205,7 +205,7 @@ describe Rdkafka::Producer do
       topic: "produce_test_topic",
       key:   "key no payload"
     )
-    report = handle.wait(5)
+    report = handle.wait(max_wait_timeout: 5)
 
     # Consume message and verify it's content
     message = wait_for_message(
@@ -224,7 +224,7 @@ describe Rdkafka::Producer do
       key:       "key headers",
       headers:   { foo: :bar, baz: :foobar }
     )
-    report = handle.wait(5)
+    report = handle.wait(max_wait_timeout: 5)
 
     # Consume message and verify it's content
     message = wait_for_message(
@@ -246,7 +246,7 @@ describe Rdkafka::Producer do
       key:       "key headers",
       headers:   {}
     )
-    report = handle.wait(5)
+    report = handle.wait(max_wait_timeout: 5)
 
     # Consume message and verify it's content
     message = wait_for_message(
@@ -280,55 +280,49 @@ describe Rdkafka::Producer do
     end
   end
 
-# TODO this spec crashes if you create and use the producer before
-# forking like so:
-#
-# @producer = producer
-#
-# This will be added as part of https://github.com/appsignal/rdkafka-ruby/issues/19
-  #it "should produce a message in a forked process" do
-  #  # Fork, produce a message, send the report of a pipe and
-  #  # wait for it in the main process.
-
-  #  reader, writer = IO.pipe
-
-  #  fork do
-  #    reader.close
-
-  #    handle = producer.produce(
-  #      topic:   "produce_test_topic",
-  #      payload: "payload",
-  #      key:     "key"
-  #    )
-
-  #    report = handle.wait(5)
-  #    producer.close
-
-  #    report_json = JSON.generate(
-  #      "partition" => report.partition,
-  #      "offset" => report.offset
-  #    )
-
-  #    writer.write(report_json)
-  #  end
-
-  #  writer.close
-
-  #  report_hash = JSON.parse(reader.read)
-  #  report = Rdkafka::Producer::DeliveryReport.new(
-  #    report_hash["partition"],
-  #    report_hash["offset"]
-  #  )
-
-  #  # Consume message and verify it's content
-  #  message = wait_for_message(
-  #    topic: "produce_test_topic",
-  #    delivery_report: report
-  #  )
-  #  expect(message.partition).to eq 1
-  #  expect(message.payload).to eq "payload"
-  #  expect(message.key).to eq "key"
-  #end
+  it "should produce a message in a forked process" do
+    # Fork, produce a message, send the report over a pipe and
+    # wait for and check the message in the main process.
+
+    reader, writer = IO.pipe
+
+    fork do
+      reader.close
+
+      handle = producer.produce(
+        topic:   "produce_test_topic",
+        payload: "payload-forked",
+        key:     "key-forked"
+      )
+
+      report = handle.wait(max_wait_timeout: 5)
+      producer.close
+
+      report_json = JSON.generate(
+        "partition" => report.partition,
+        "offset" => report.offset
+      )
+
+      writer.write(report_json)
+    end
+
+    writer.close
+
+    report_hash = JSON.parse(reader.read)
+    report = Rdkafka::Producer::DeliveryReport.new(
+      report_hash["partition"],
+      report_hash["offset"]
+    )
+
+    # Consume message and verify it's content
+    message = wait_for_message(
+      topic: "produce_test_topic",
+      delivery_report: report
+    )
+    expect(message.partition).to eq 0
+    expect(message.payload).to eq "payload-forked"
+    expect(message.key).to eq "key-forked"
+  end
 
   it "should raise an error when producing fails" do
     expect(Rdkafka::Bindings).to receive(:rd_kafka_producev).and_return(20)
@@ -348,10 +342,10 @@ describe Rdkafka::Producer do
       key:     "key timeout"
     )
     expect {
-      handle.wait(0)
+      handle.wait(max_wait_timeout: 0)
     }.to raise_error Rdkafka::Producer::DeliveryHandle::WaitTimeoutError
 
     # Waiting a second time should work
-    handle.wait(5)
+    handle.wait(max_wait_timeout: 5)
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.7.0
 platform: ruby
 authors:
 - Thijs Cadier
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-07-23 00:00:00.000000000 Z
+date: 2019-09-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -164,7 +164,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.1'
+      version: '2.4'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -172,11 +172,12 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 2.7.6.2
 signing_key: 
 specification_version: 4
-summary: Kafka client library wrapping librdkafka using the ffi gem and futures from
-  concurrent-ruby for Kafka 0.10+
+summary: The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka.
+  It wraps the production-ready C client using the ffi gem and targets Kafka 1.0+
+  and Ruby 2.4+.
 test_files:
 - spec/rdkafka/bindings_spec.rb
 - spec/rdkafka/config_spec.rb