rdkafka 0.8.0.beta.1 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. checksums.yaml +4 -4
  2. data/.semaphore/semaphore.yml +23 -0
  3. data/CHANGELOG.md +18 -0
  4. data/README.md +5 -2
  5. data/docker-compose.yml +2 -0
  6. data/ext/README.md +7 -0
  7. data/ext/Rakefile +2 -1
  8. data/lib/rdkafka/abstract_handle.rb +82 -0
  9. data/lib/rdkafka/admin/create_topic_handle.rb +27 -0
  10. data/lib/rdkafka/admin/create_topic_report.rb +22 -0
  11. data/lib/rdkafka/admin/delete_topic_handle.rb +27 -0
  12. data/lib/rdkafka/admin/delete_topic_report.rb +22 -0
  13. data/lib/rdkafka/admin.rb +155 -0
  14. data/lib/rdkafka/bindings.rb +57 -18
  15. data/lib/rdkafka/callbacks.rb +106 -0
  16. data/lib/rdkafka/config.rb +59 -3
  17. data/lib/rdkafka/consumer.rb +125 -5
  18. data/lib/rdkafka/error.rb +29 -3
  19. data/lib/rdkafka/metadata.rb +6 -5
  20. data/lib/rdkafka/producer/delivery_handle.rb +7 -53
  21. data/lib/rdkafka/producer.rb +25 -11
  22. data/lib/rdkafka/version.rb +3 -3
  23. data/lib/rdkafka.rb +7 -0
  24. data/spec/rdkafka/abstract_handle_spec.rb +114 -0
  25. data/spec/rdkafka/admin/create_topic_handle_spec.rb +52 -0
  26. data/spec/rdkafka/admin/create_topic_report_spec.rb +16 -0
  27. data/spec/rdkafka/admin/delete_topic_handle_spec.rb +52 -0
  28. data/spec/rdkafka/admin/delete_topic_report_spec.rb +16 -0
  29. data/spec/rdkafka/admin_spec.rb +203 -0
  30. data/spec/rdkafka/bindings_spec.rb +32 -8
  31. data/spec/rdkafka/callbacks_spec.rb +20 -0
  32. data/spec/rdkafka/config_spec.rb +76 -7
  33. data/spec/rdkafka/consumer_spec.rb +266 -2
  34. data/spec/rdkafka/error_spec.rb +4 -0
  35. data/spec/rdkafka/metadata_spec.rb +78 -0
  36. data/spec/rdkafka/producer/delivery_handle_spec.rb +1 -41
  37. data/spec/rdkafka/producer_spec.rb +98 -31
  38. data/spec/spec_helper.rb +28 -11
  39. metadata +32 -9
  40. data/.travis.yml +0 -45
data/lib/rdkafka/error.rb CHANGED
@@ -1,15 +1,27 @@
1
1
  module Rdkafka
2
+ # Base error class.
3
+ class BaseError < RuntimeError; end
4
+
2
5
  # Error returned by the underlying rdkafka library.
3
- class RdkafkaError < RuntimeError
6
+ class RdkafkaError < BaseError
4
7
  # The underlying raw error response
5
8
  # @return [Integer]
6
- attr_reader :rdkafka_response, :message_prefix
9
+ attr_reader :rdkafka_response
10
+
11
+ # Prefix to be used for human readable representation
12
+ # @return [String]
13
+ attr_reader :message_prefix
14
+
15
+ # Error message sent by the broker
16
+ # @return [String]
17
+ attr_reader :broker_message
7
18
 
8
19
  # @private
9
- def initialize(response, message_prefix=nil)
20
+ def initialize(response, message_prefix=nil, broker_message: nil)
10
21
  raise TypeError.new("Response has to be an integer") unless response.is_a? Integer
11
22
  @rdkafka_response = response
12
23
  @message_prefix = message_prefix
24
+ @broker_message = broker_message
13
25
  end
14
26
 
15
27
  # This error's code, for example `:partition_eof`, `:msg_size_too_large`.
@@ -57,4 +69,18 @@ module Rdkafka
57
69
  @topic_partition_list = topic_partition_list
58
70
  end
59
71
  end
72
+
73
+ # Error class for public consumer method calls on a closed consumer.
74
+ class ClosedConsumerError < BaseError
75
+ def initialize(method)
76
+ super("Illegal call to #{method.to_s} on a closed consumer")
77
+ end
78
+ end
79
+
80
+ # Error class for public producer method calls on a closed producer.
81
+ class ClosedProducerError < BaseError
82
+ def initialize(method)
83
+ super("Illegal call to #{method.to_s} on a closed producer")
84
+ end
85
+ end
60
86
  end
@@ -9,14 +9,15 @@ module Rdkafka
9
9
 
10
10
  ptr = FFI::MemoryPointer.new(:pointer)
11
11
 
12
- # Retrieve metadata flag is 0/1 for single/multiple topics.
13
- topic_flag = topic_name ? 1 : 0
12
+ # If topic_flag is 1, we request info about *all* topics in the cluster. If topic_flag is 0,
13
+ # we only request info about locally known topics (or a single topic if one is passed in).
14
+ topic_flag = topic_name.nil? ? 1 : 0
14
15
 
15
16
  # Retrieve the Metadata
16
17
  result = Rdkafka::Bindings.rd_kafka_metadata(native_client, topic_flag, native_topic, ptr, 250)
17
18
 
18
19
  # Error Handling
19
- Rdkafka::Error.new(result) unless result.zero?
20
+ raise Rdkafka::RdkafkaError.new(result) unless result.zero?
20
21
 
21
22
  metadata_from_native(ptr.read_pointer)
22
23
  ensure
@@ -34,11 +35,11 @@ module Rdkafka
34
35
 
35
36
  @topics = Array.new(metadata[:topics_count]) do |i|
36
37
  topic = TopicMetadata.new(metadata[:topics_metadata] + (i * TopicMetadata.size))
37
- Rdkafka::Error.new(topic[:rd_kafka_resp_err]) unless topic[:rd_kafka_resp_err].zero?
38
+ raise Rdkafka::RdkafkaError.new(topic[:rd_kafka_resp_err]) unless topic[:rd_kafka_resp_err].zero?
38
39
 
39
40
  partitions = Array.new(topic[:partition_count]) do |j|
40
41
  partition = PartitionMetadata.new(topic[:partitions_metadata] + (j * PartitionMetadata.size))
41
- Rdkafka::Error.new(partition[:rd_kafka_resp_err]) unless partition[:rd_kafka_resp_err].zero?
42
+ raise Rdkafka::RdkafkaError.new(partition[:rd_kafka_resp_err]) unless partition[:rd_kafka_resp_err].zero?
42
43
  partition.to_h
43
44
  end
44
45
  topic.to_h.merge!(partitions: partitions)
@@ -2,67 +2,21 @@ module Rdkafka
2
2
  class Producer
3
3
  # Handle to wait for a delivery report which is returned when
4
4
  # producing a message.
5
- class DeliveryHandle < FFI::Struct
5
+ class DeliveryHandle < Rdkafka::AbstractHandle
6
6
  layout :pending, :bool,
7
7
  :response, :int,
8
8
  :partition, :int,
9
9
  :offset, :int64
10
10
 
11
- REGISTRY = {}
12
-
13
- CURRENT_TIME = -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }.freeze
14
-
15
- private_constant :CURRENT_TIME
16
-
17
- def self.register(address, handle)
18
- REGISTRY[address] = handle
19
- end
20
-
21
- def self.remove(address)
22
- REGISTRY.delete(address)
23
- end
24
-
25
- # Whether the delivery handle is still pending.
26
- #
27
- # @return [Boolean]
28
- def pending?
29
- self[:pending]
11
+ # @return [String] the name of the operation (e.g. "delivery")
12
+ def operation_name
13
+ "delivery"
30
14
  end
31
15
 
32
- # Wait for the delivery report or raise an error if this takes longer than the timeout.
33
- # If there is a timeout this does not mean the message is not delivered, rdkafka might still be working on delivering the message.
34
- # In this case it is possible to call wait again.
35
- #
36
- # @param max_wait_timeout [Numeric, nil] Amount of time to wait before timing out. If this is nil it does not time out.
37
- # @param wait_timeout [Numeric] Amount of time we should wait before we recheck if there is a delivery report available
38
- #
39
- # @raise [RdkafkaError] When delivering the message failed
40
- # @raise [WaitTimeoutError] When the timeout has been reached and the handle is still pending
41
- #
42
- # @return [DeliveryReport]
43
- def wait(max_wait_timeout: 60, wait_timeout: 0.1)
44
- timeout = if max_wait_timeout
45
- CURRENT_TIME.call + max_wait_timeout
46
- else
47
- nil
48
- end
49
- loop do
50
- if pending?
51
- if timeout && timeout <= CURRENT_TIME.call
52
- raise WaitTimeoutError.new("Waiting for delivery timed out after #{max_wait_timeout} seconds")
53
- end
54
- sleep wait_timeout
55
- elsif self[:response] != 0
56
- raise RdkafkaError.new(self[:response])
57
- else
58
- return DeliveryReport.new(self[:partition], self[:offset])
59
- end
60
- end
16
+ # @return [DeliveryReport] a report on the delivery of the message
17
+ def create_result
18
+ DeliveryReport.new(self[:partition], self[:offset])
61
19
  end
62
-
63
- # Error that is raised when waiting for a delivery handle to complete
64
- # takes longer than the specified timeout.
65
- class WaitTimeoutError < RuntimeError; end
66
20
  end
67
21
  end
68
22
  end
@@ -1,3 +1,5 @@
1
+ require "securerandom"
2
+
1
3
  module Rdkafka
2
4
  # A producer for Kafka messages. To create a producer set up a {Config} and call {Config#producer producer} on that.
3
5
  class Producer
@@ -9,8 +11,13 @@ module Rdkafka
9
11
 
10
12
  # @private
11
13
  def initialize(native_kafka)
14
+ @id = SecureRandom.uuid
12
15
  @closing = false
13
16
  @native_kafka = native_kafka
17
+
18
+ # Makes sure, that the producer gets closed before it gets GCed by Ruby
19
+ ObjectSpace.define_finalizer(@id, proc { close })
20
+
14
21
  # Start thread to poll client for delivery callbacks
15
22
  @polling_thread = Thread.new do
16
23
  loop do
@@ -27,16 +34,18 @@ module Rdkafka
27
34
  # Set a callback that will be called every time a message is successfully produced.
28
35
  # The callback is called with a {DeliveryReport}
29
36
  #
30
- # @param callback [Proc] The callback
37
+ # @param callback [Proc, #call] The callback
31
38
  #
32
39
  # @return [nil]
33
40
  def delivery_callback=(callback)
34
- raise TypeError.new("Callback has to be a proc or lambda") unless callback.is_a? Proc
41
+ raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call)
35
42
  @delivery_callback = callback
36
43
  end
37
44
 
38
45
  # Close this producer and wait for the internal poll queue to empty.
39
46
  def close
47
+ ObjectSpace.undefine_finalizer(@id)
48
+
40
49
  return unless @native_kafka
41
50
 
42
51
  # Indicate to polling thread that we're closing
@@ -55,7 +64,8 @@ module Rdkafka
55
64
  # @return partition count [Integer,nil]
56
65
  #
57
66
  def partition_count(topic)
58
- Rdkafka::Metadata.new(@native_kafka, topic).topics&.select { |x| x[:topic_name] == topic }&.dig(0, :partition_count)
67
+ closed_producer_check(__method__)
68
+ Rdkafka::Metadata.new(@native_kafka, topic).topics&.first[:partition_count]
59
69
  end
60
70
 
61
71
  # Produces a message to a Kafka topic. The message is added to rdkafka's queue, call {DeliveryHandle#wait wait} on the returned delivery handle to make sure it is delivered.
@@ -74,6 +84,8 @@ module Rdkafka
74
84
  #
75
85
  # @return [DeliveryHandle] Delivery handle that can be used to wait for the result of producing this message
76
86
  def produce(topic:, payload: nil, key: nil, partition: nil, partition_key: nil, timestamp: nil, headers: nil)
87
+ closed_producer_check(__method__)
88
+
77
89
  # Start by checking and converting the input
78
90
 
79
91
  # Get payload length
@@ -117,7 +129,7 @@ module Rdkafka
117
129
  delivery_handle[:response] = -1
118
130
  delivery_handle[:partition] = -1
119
131
  delivery_handle[:offset] = -1
120
- DeliveryHandle.register(delivery_handle.to_ptr.address, delivery_handle)
132
+ DeliveryHandle.register(delivery_handle)
121
133
 
122
134
  args = [
123
135
  :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_TOPIC, :string, topic,
@@ -133,16 +145,14 @@ module Rdkafka
133
145
  headers.each do |key0, value0|
134
146
  key = key0.to_s
135
147
  value = value0.to_s
136
- args += [
137
- :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_HEADER,
138
- :string, key,
139
- :pointer, value,
140
- :size_t, value.bytes.size
141
- ]
148
+ args << :int << Rdkafka::Bindings::RD_KAFKA_VTYPE_HEADER
149
+ args << :string << key
150
+ args << :pointer << value
151
+ args << :size_t << value.bytes.size
142
152
  end
143
153
  end
144
154
 
145
- args += [:int, Rdkafka::Bindings::RD_KAFKA_VTYPE_END]
155
+ args << :int << Rdkafka::Bindings::RD_KAFKA_VTYPE_END
146
156
 
147
157
  # Produce the message
148
158
  response = Rdkafka::Bindings.rd_kafka_producev(
@@ -163,5 +173,9 @@ module Rdkafka
163
173
  def call_delivery_callback(delivery_handle)
164
174
  @delivery_callback.call(delivery_handle) if @delivery_callback
165
175
  end
176
+
177
+ def closed_producer_check(method)
178
+ raise Rdkafka::ClosedProducerError.new(method) if @native_kafka.nil?
179
+ end
166
180
  end
167
181
  end
@@ -1,5 +1,5 @@
1
1
  module Rdkafka
2
- VERSION = "0.8.0.beta.1"
3
- LIBRDKAFKA_VERSION = "1.4.0"
4
- LIBRDKAFKA_SOURCE_SHA256 = "ae27ea3f3d0d32d29004e7f709efbba2666c5383a107cc45b3a1949486b2eb84"
2
+ VERSION = "0.10.0"
3
+ LIBRDKAFKA_VERSION = "1.5.0"
4
+ LIBRDKAFKA_SOURCE_SHA256 = "f7fee59fdbf1286ec23ef0b35b2dfb41031c8727c90ced6435b8cf576f23a656"
5
5
  end
data/lib/rdkafka.rb CHANGED
@@ -1,6 +1,13 @@
1
1
  require "rdkafka/version"
2
2
 
3
+ require "rdkafka/abstract_handle"
4
+ require "rdkafka/admin"
5
+ require "rdkafka/admin/create_topic_handle"
6
+ require "rdkafka/admin/create_topic_report"
7
+ require "rdkafka/admin/delete_topic_handle"
8
+ require "rdkafka/admin/delete_topic_report"
3
9
  require "rdkafka/bindings"
10
+ require "rdkafka/callbacks"
4
11
  require "rdkafka/config"
5
12
  require "rdkafka/consumer"
6
13
  require "rdkafka/consumer/headers"
@@ -0,0 +1,114 @@
1
+ require "spec_helper"
2
+
3
+ describe Rdkafka::AbstractHandle do
4
+ let(:response) { 0 }
5
+ let(:result) { -1 }
6
+
7
+ context "A subclass that does not implement the required methods" do
8
+
9
+ class BadTestHandle < Rdkafka::AbstractHandle
10
+ layout :pending, :bool,
11
+ :response, :int
12
+ end
13
+
14
+ it "raises an exception if operation_name is called" do
15
+ expect {
16
+ BadTestHandle.new.operation_name
17
+ }.to raise_exception(RuntimeError, /Must be implemented by subclass!/)
18
+ end
19
+
20
+ it "raises an exception if create_result is called" do
21
+ expect {
22
+ BadTestHandle.new.create_result
23
+ }.to raise_exception(RuntimeError, /Must be implemented by subclass!/)
24
+ end
25
+ end
26
+
27
+ class TestHandle < Rdkafka::AbstractHandle
28
+ layout :pending, :bool,
29
+ :response, :int,
30
+ :result, :int
31
+
32
+ def operation_name
33
+ "test_operation"
34
+ end
35
+
36
+ def create_result
37
+ self[:result]
38
+ end
39
+ end
40
+
41
+ subject do
42
+ TestHandle.new.tap do |handle|
43
+ handle[:pending] = pending_handle
44
+ handle[:response] = response
45
+ handle[:result] = result
46
+ end
47
+ end
48
+
49
+ describe ".register and .remove" do
50
+ let(:pending_handle) { true }
51
+
52
+ it "should register and remove a delivery handle" do
53
+ Rdkafka::AbstractHandle.register(subject)
54
+ removed = Rdkafka::AbstractHandle.remove(subject.to_ptr.address)
55
+ expect(removed).to eq subject
56
+ expect(Rdkafka::AbstractHandle::REGISTRY).to be_empty
57
+ end
58
+ end
59
+
60
+ describe "#pending?" do
61
+ context "when true" do
62
+ let(:pending_handle) { true }
63
+
64
+ it "should be true" do
65
+ expect(subject.pending?).to be true
66
+ end
67
+ end
68
+
69
+ context "when not true" do
70
+ let(:pending_handle) { false }
71
+
72
+ it "should be false" do
73
+ expect(subject.pending?).to be false
74
+ end
75
+ end
76
+ end
77
+
78
+ describe "#wait" do
79
+ let(:pending_handle) { true }
80
+
81
+ it "should wait until the timeout and then raise an error" do
82
+ expect {
83
+ subject.wait(max_wait_timeout: 0.1)
84
+ }.to raise_error Rdkafka::AbstractHandle::WaitTimeoutError, /test_operation/
85
+ end
86
+
87
+ context "when not pending anymore and no error" do
88
+ let(:pending_handle) { false }
89
+ let(:result) { 1 }
90
+
91
+ it "should return a result" do
92
+ wait_result = subject.wait
93
+ expect(wait_result).to eq(result)
94
+ end
95
+
96
+ it "should wait without a timeout" do
97
+ wait_result = subject.wait(max_wait_timeout: nil)
98
+ expect(wait_result).to eq(result)
99
+ end
100
+ end
101
+
102
+ context "when not pending anymore and there was an error" do
103
+ let(:pending_handle) { false }
104
+ let(:response) { 20 }
105
+
106
+ it "should raise an rdkafka error" do
107
+ expect {
108
+ subject.wait
109
+ }.to raise_error Rdkafka::RdkafkaError
110
+ end
111
+ end
112
+ end
113
+ end
114
+
@@ -0,0 +1,52 @@
1
+ require "spec_helper"
2
+
3
+ describe Rdkafka::Admin::CreateTopicHandle do
4
+ let(:response) { 0 }
5
+
6
+ subject do
7
+ Rdkafka::Admin::CreateTopicHandle.new.tap do |handle|
8
+ handle[:pending] = pending_handle
9
+ handle[:response] = response
10
+ handle[:error_string] = FFI::Pointer::NULL
11
+ handle[:result_name] = FFI::MemoryPointer.from_string("my-test-topic")
12
+ end
13
+ end
14
+
15
+ describe "#wait" do
16
+ let(:pending_handle) { true }
17
+
18
+ it "should wait until the timeout and then raise an error" do
19
+ expect {
20
+ subject.wait(max_wait_timeout: 0.1)
21
+ }.to raise_error Rdkafka::Admin::CreateTopicHandle::WaitTimeoutError, /create topic/
22
+ end
23
+
24
+ context "when not pending anymore and no error" do
25
+ let(:pending_handle) { false }
26
+
27
+ it "should return a create topic report" do
28
+ report = subject.wait
29
+
30
+ expect(report.error_string).to eq(nil)
31
+ expect(report.result_name).to eq("my-test-topic")
32
+ end
33
+
34
+ it "should wait without a timeout" do
35
+ report = subject.wait(max_wait_timeout: nil)
36
+
37
+ expect(report.error_string).to eq(nil)
38
+ expect(report.result_name).to eq("my-test-topic")
39
+ end
40
+ end
41
+ end
42
+
43
+ describe "#raise_error" do
44
+ let(:pending_handle) { false }
45
+
46
+ it "should raise the appropriate error" do
47
+ expect {
48
+ subject.raise_error
49
+ }.to raise_exception(Rdkafka::RdkafkaError, /Success \(no_error\)/)
50
+ end
51
+ end
52
+ end