rdkafka 0.8.0.beta.1 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.semaphore/semaphore.yml +23 -0
- data/CHANGELOG.md +18 -0
- data/README.md +5 -2
- data/docker-compose.yml +2 -0
- data/ext/README.md +7 -0
- data/ext/Rakefile +2 -1
- data/lib/rdkafka/abstract_handle.rb +82 -0
- data/lib/rdkafka/admin/create_topic_handle.rb +27 -0
- data/lib/rdkafka/admin/create_topic_report.rb +22 -0
- data/lib/rdkafka/admin/delete_topic_handle.rb +27 -0
- data/lib/rdkafka/admin/delete_topic_report.rb +22 -0
- data/lib/rdkafka/admin.rb +155 -0
- data/lib/rdkafka/bindings.rb +57 -18
- data/lib/rdkafka/callbacks.rb +106 -0
- data/lib/rdkafka/config.rb +59 -3
- data/lib/rdkafka/consumer.rb +125 -5
- data/lib/rdkafka/error.rb +29 -3
- data/lib/rdkafka/metadata.rb +6 -5
- data/lib/rdkafka/producer/delivery_handle.rb +7 -53
- data/lib/rdkafka/producer.rb +25 -11
- data/lib/rdkafka/version.rb +3 -3
- data/lib/rdkafka.rb +7 -0
- data/spec/rdkafka/abstract_handle_spec.rb +114 -0
- data/spec/rdkafka/admin/create_topic_handle_spec.rb +52 -0
- data/spec/rdkafka/admin/create_topic_report_spec.rb +16 -0
- data/spec/rdkafka/admin/delete_topic_handle_spec.rb +52 -0
- data/spec/rdkafka/admin/delete_topic_report_spec.rb +16 -0
- data/spec/rdkafka/admin_spec.rb +203 -0
- data/spec/rdkafka/bindings_spec.rb +32 -8
- data/spec/rdkafka/callbacks_spec.rb +20 -0
- data/spec/rdkafka/config_spec.rb +76 -7
- data/spec/rdkafka/consumer_spec.rb +266 -2
- data/spec/rdkafka/error_spec.rb +4 -0
- data/spec/rdkafka/metadata_spec.rb +78 -0
- data/spec/rdkafka/producer/delivery_handle_spec.rb +1 -41
- data/spec/rdkafka/producer_spec.rb +98 -31
- data/spec/spec_helper.rb +28 -11
- metadata +32 -9
- data/.travis.yml +0 -45
data/lib/rdkafka/error.rb
CHANGED
@@ -1,15 +1,27 @@
|
|
1
1
|
module Rdkafka
|
2
|
+
# Base error class.
|
3
|
+
class BaseError < RuntimeError; end
|
4
|
+
|
2
5
|
# Error returned by the underlying rdkafka library.
|
3
|
-
class RdkafkaError <
|
6
|
+
class RdkafkaError < BaseError
|
4
7
|
# The underlying raw error response
|
5
8
|
# @return [Integer]
|
6
|
-
attr_reader :rdkafka_response
|
9
|
+
attr_reader :rdkafka_response
|
10
|
+
|
11
|
+
# Prefix to be used for human readable representation
|
12
|
+
# @return [String]
|
13
|
+
attr_reader :message_prefix
|
14
|
+
|
15
|
+
# Error message sent by the broker
|
16
|
+
# @return [String]
|
17
|
+
attr_reader :broker_message
|
7
18
|
|
8
19
|
# @private
|
9
|
-
def initialize(response, message_prefix=nil)
|
20
|
+
def initialize(response, message_prefix=nil, broker_message: nil)
|
10
21
|
raise TypeError.new("Response has to be an integer") unless response.is_a? Integer
|
11
22
|
@rdkafka_response = response
|
12
23
|
@message_prefix = message_prefix
|
24
|
+
@broker_message = broker_message
|
13
25
|
end
|
14
26
|
|
15
27
|
# This error's code, for example `:partition_eof`, `:msg_size_too_large`.
|
@@ -57,4 +69,18 @@ module Rdkafka
|
|
57
69
|
@topic_partition_list = topic_partition_list
|
58
70
|
end
|
59
71
|
end
|
72
|
+
|
73
|
+
# Error class for public consumer method calls on a closed consumer.
|
74
|
+
class ClosedConsumerError < BaseError
|
75
|
+
def initialize(method)
|
76
|
+
super("Illegal call to #{method.to_s} on a closed consumer")
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
# Error class for public producer method calls on a closed producer.
|
81
|
+
class ClosedProducerError < BaseError
|
82
|
+
def initialize(method)
|
83
|
+
super("Illegal call to #{method.to_s} on a closed producer")
|
84
|
+
end
|
85
|
+
end
|
60
86
|
end
|
data/lib/rdkafka/metadata.rb
CHANGED
@@ -9,14 +9,15 @@ module Rdkafka
|
|
9
9
|
|
10
10
|
ptr = FFI::MemoryPointer.new(:pointer)
|
11
11
|
|
12
|
-
#
|
13
|
-
|
12
|
+
# If topic_flag is 1, we request info about *all* topics in the cluster. If topic_flag is 0,
|
13
|
+
# we only request info about locally known topics (or a single topic if one is passed in).
|
14
|
+
topic_flag = topic_name.nil? ? 1 : 0
|
14
15
|
|
15
16
|
# Retrieve the Metadata
|
16
17
|
result = Rdkafka::Bindings.rd_kafka_metadata(native_client, topic_flag, native_topic, ptr, 250)
|
17
18
|
|
18
19
|
# Error Handling
|
19
|
-
Rdkafka::
|
20
|
+
raise Rdkafka::RdkafkaError.new(result) unless result.zero?
|
20
21
|
|
21
22
|
metadata_from_native(ptr.read_pointer)
|
22
23
|
ensure
|
@@ -34,11 +35,11 @@ module Rdkafka
|
|
34
35
|
|
35
36
|
@topics = Array.new(metadata[:topics_count]) do |i|
|
36
37
|
topic = TopicMetadata.new(metadata[:topics_metadata] + (i * TopicMetadata.size))
|
37
|
-
Rdkafka::
|
38
|
+
raise Rdkafka::RdkafkaError.new(topic[:rd_kafka_resp_err]) unless topic[:rd_kafka_resp_err].zero?
|
38
39
|
|
39
40
|
partitions = Array.new(topic[:partition_count]) do |j|
|
40
41
|
partition = PartitionMetadata.new(topic[:partitions_metadata] + (j * PartitionMetadata.size))
|
41
|
-
Rdkafka::
|
42
|
+
raise Rdkafka::RdkafkaError.new(partition[:rd_kafka_resp_err]) unless partition[:rd_kafka_resp_err].zero?
|
42
43
|
partition.to_h
|
43
44
|
end
|
44
45
|
topic.to_h.merge!(partitions: partitions)
|
@@ -2,67 +2,21 @@ module Rdkafka
|
|
2
2
|
class Producer
|
3
3
|
# Handle to wait for a delivery report which is returned when
|
4
4
|
# producing a message.
|
5
|
-
class DeliveryHandle <
|
5
|
+
class DeliveryHandle < Rdkafka::AbstractHandle
|
6
6
|
layout :pending, :bool,
|
7
7
|
:response, :int,
|
8
8
|
:partition, :int,
|
9
9
|
:offset, :int64
|
10
10
|
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
private_constant :CURRENT_TIME
|
16
|
-
|
17
|
-
def self.register(address, handle)
|
18
|
-
REGISTRY[address] = handle
|
19
|
-
end
|
20
|
-
|
21
|
-
def self.remove(address)
|
22
|
-
REGISTRY.delete(address)
|
23
|
-
end
|
24
|
-
|
25
|
-
# Whether the delivery handle is still pending.
|
26
|
-
#
|
27
|
-
# @return [Boolean]
|
28
|
-
def pending?
|
29
|
-
self[:pending]
|
11
|
+
# @return [String] the name of the operation (e.g. "delivery")
|
12
|
+
def operation_name
|
13
|
+
"delivery"
|
30
14
|
end
|
31
15
|
|
32
|
-
#
|
33
|
-
|
34
|
-
|
35
|
-
#
|
36
|
-
# @param max_wait_timeout [Numeric, nil] Amount of time to wait before timing out. If this is nil it does not time out.
|
37
|
-
# @param wait_timeout [Numeric] Amount of time we should wait before we recheck if there is a delivery report available
|
38
|
-
#
|
39
|
-
# @raise [RdkafkaError] When delivering the message failed
|
40
|
-
# @raise [WaitTimeoutError] When the timeout has been reached and the handle is still pending
|
41
|
-
#
|
42
|
-
# @return [DeliveryReport]
|
43
|
-
def wait(max_wait_timeout: 60, wait_timeout: 0.1)
|
44
|
-
timeout = if max_wait_timeout
|
45
|
-
CURRENT_TIME.call + max_wait_timeout
|
46
|
-
else
|
47
|
-
nil
|
48
|
-
end
|
49
|
-
loop do
|
50
|
-
if pending?
|
51
|
-
if timeout && timeout <= CURRENT_TIME.call
|
52
|
-
raise WaitTimeoutError.new("Waiting for delivery timed out after #{max_wait_timeout} seconds")
|
53
|
-
end
|
54
|
-
sleep wait_timeout
|
55
|
-
elsif self[:response] != 0
|
56
|
-
raise RdkafkaError.new(self[:response])
|
57
|
-
else
|
58
|
-
return DeliveryReport.new(self[:partition], self[:offset])
|
59
|
-
end
|
60
|
-
end
|
16
|
+
# @return [DeliveryReport] a report on the delivery of the message
|
17
|
+
def create_result
|
18
|
+
DeliveryReport.new(self[:partition], self[:offset])
|
61
19
|
end
|
62
|
-
|
63
|
-
# Error that is raised when waiting for a delivery handle to complete
|
64
|
-
# takes longer than the specified timeout.
|
65
|
-
class WaitTimeoutError < RuntimeError; end
|
66
20
|
end
|
67
21
|
end
|
68
22
|
end
|
data/lib/rdkafka/producer.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
require "securerandom"
|
2
|
+
|
1
3
|
module Rdkafka
|
2
4
|
# A producer for Kafka messages. To create a producer set up a {Config} and call {Config#producer producer} on that.
|
3
5
|
class Producer
|
@@ -9,8 +11,13 @@ module Rdkafka
|
|
9
11
|
|
10
12
|
# @private
|
11
13
|
def initialize(native_kafka)
|
14
|
+
@id = SecureRandom.uuid
|
12
15
|
@closing = false
|
13
16
|
@native_kafka = native_kafka
|
17
|
+
|
18
|
+
# Makes sure, that the producer gets closed before it gets GCed by Ruby
|
19
|
+
ObjectSpace.define_finalizer(@id, proc { close })
|
20
|
+
|
14
21
|
# Start thread to poll client for delivery callbacks
|
15
22
|
@polling_thread = Thread.new do
|
16
23
|
loop do
|
@@ -27,16 +34,18 @@ module Rdkafka
|
|
27
34
|
# Set a callback that will be called every time a message is successfully produced.
|
28
35
|
# The callback is called with a {DeliveryReport}
|
29
36
|
#
|
30
|
-
# @param callback [Proc] The callback
|
37
|
+
# @param callback [Proc, #call] The callback
|
31
38
|
#
|
32
39
|
# @return [nil]
|
33
40
|
def delivery_callback=(callback)
|
34
|
-
raise TypeError.new("Callback has to be
|
41
|
+
raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call)
|
35
42
|
@delivery_callback = callback
|
36
43
|
end
|
37
44
|
|
38
45
|
# Close this producer and wait for the internal poll queue to empty.
|
39
46
|
def close
|
47
|
+
ObjectSpace.undefine_finalizer(@id)
|
48
|
+
|
40
49
|
return unless @native_kafka
|
41
50
|
|
42
51
|
# Indicate to polling thread that we're closing
|
@@ -55,7 +64,8 @@ module Rdkafka
|
|
55
64
|
# @return partition count [Integer,nil]
|
56
65
|
#
|
57
66
|
def partition_count(topic)
|
58
|
-
|
67
|
+
closed_producer_check(__method__)
|
68
|
+
Rdkafka::Metadata.new(@native_kafka, topic).topics&.first[:partition_count]
|
59
69
|
end
|
60
70
|
|
61
71
|
# Produces a message to a Kafka topic. The message is added to rdkafka's queue, call {DeliveryHandle#wait wait} on the returned delivery handle to make sure it is delivered.
|
@@ -74,6 +84,8 @@ module Rdkafka
|
|
74
84
|
#
|
75
85
|
# @return [DeliveryHandle] Delivery handle that can be used to wait for the result of producing this message
|
76
86
|
def produce(topic:, payload: nil, key: nil, partition: nil, partition_key: nil, timestamp: nil, headers: nil)
|
87
|
+
closed_producer_check(__method__)
|
88
|
+
|
77
89
|
# Start by checking and converting the input
|
78
90
|
|
79
91
|
# Get payload length
|
@@ -117,7 +129,7 @@ module Rdkafka
|
|
117
129
|
delivery_handle[:response] = -1
|
118
130
|
delivery_handle[:partition] = -1
|
119
131
|
delivery_handle[:offset] = -1
|
120
|
-
DeliveryHandle.register(delivery_handle
|
132
|
+
DeliveryHandle.register(delivery_handle)
|
121
133
|
|
122
134
|
args = [
|
123
135
|
:int, Rdkafka::Bindings::RD_KAFKA_VTYPE_TOPIC, :string, topic,
|
@@ -133,16 +145,14 @@ module Rdkafka
|
|
133
145
|
headers.each do |key0, value0|
|
134
146
|
key = key0.to_s
|
135
147
|
value = value0.to_s
|
136
|
-
args
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
:size_t, value.bytes.size
|
141
|
-
]
|
148
|
+
args << :int << Rdkafka::Bindings::RD_KAFKA_VTYPE_HEADER
|
149
|
+
args << :string << key
|
150
|
+
args << :pointer << value
|
151
|
+
args << :size_t << value.bytes.size
|
142
152
|
end
|
143
153
|
end
|
144
154
|
|
145
|
-
args
|
155
|
+
args << :int << Rdkafka::Bindings::RD_KAFKA_VTYPE_END
|
146
156
|
|
147
157
|
# Produce the message
|
148
158
|
response = Rdkafka::Bindings.rd_kafka_producev(
|
@@ -163,5 +173,9 @@ module Rdkafka
|
|
163
173
|
def call_delivery_callback(delivery_handle)
|
164
174
|
@delivery_callback.call(delivery_handle) if @delivery_callback
|
165
175
|
end
|
176
|
+
|
177
|
+
def closed_producer_check(method)
|
178
|
+
raise Rdkafka::ClosedProducerError.new(method) if @native_kafka.nil?
|
179
|
+
end
|
166
180
|
end
|
167
181
|
end
|
data/lib/rdkafka/version.rb
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
module Rdkafka
|
2
|
-
VERSION = "0.
|
3
|
-
LIBRDKAFKA_VERSION = "1.
|
4
|
-
LIBRDKAFKA_SOURCE_SHA256 = "
|
2
|
+
VERSION = "0.10.0"
|
3
|
+
LIBRDKAFKA_VERSION = "1.5.0"
|
4
|
+
LIBRDKAFKA_SOURCE_SHA256 = "f7fee59fdbf1286ec23ef0b35b2dfb41031c8727c90ced6435b8cf576f23a656"
|
5
5
|
end
|
data/lib/rdkafka.rb
CHANGED
@@ -1,6 +1,13 @@
|
|
1
1
|
require "rdkafka/version"
|
2
2
|
|
3
|
+
require "rdkafka/abstract_handle"
|
4
|
+
require "rdkafka/admin"
|
5
|
+
require "rdkafka/admin/create_topic_handle"
|
6
|
+
require "rdkafka/admin/create_topic_report"
|
7
|
+
require "rdkafka/admin/delete_topic_handle"
|
8
|
+
require "rdkafka/admin/delete_topic_report"
|
3
9
|
require "rdkafka/bindings"
|
10
|
+
require "rdkafka/callbacks"
|
4
11
|
require "rdkafka/config"
|
5
12
|
require "rdkafka/consumer"
|
6
13
|
require "rdkafka/consumer/headers"
|
@@ -0,0 +1,114 @@
|
|
1
|
+
require "spec_helper"
|
2
|
+
|
3
|
+
describe Rdkafka::AbstractHandle do
|
4
|
+
let(:response) { 0 }
|
5
|
+
let(:result) { -1 }
|
6
|
+
|
7
|
+
context "A subclass that does not implement the required methods" do
|
8
|
+
|
9
|
+
class BadTestHandle < Rdkafka::AbstractHandle
|
10
|
+
layout :pending, :bool,
|
11
|
+
:response, :int
|
12
|
+
end
|
13
|
+
|
14
|
+
it "raises an exception if operation_name is called" do
|
15
|
+
expect {
|
16
|
+
BadTestHandle.new.operation_name
|
17
|
+
}.to raise_exception(RuntimeError, /Must be implemented by subclass!/)
|
18
|
+
end
|
19
|
+
|
20
|
+
it "raises an exception if create_result is called" do
|
21
|
+
expect {
|
22
|
+
BadTestHandle.new.create_result
|
23
|
+
}.to raise_exception(RuntimeError, /Must be implemented by subclass!/)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
class TestHandle < Rdkafka::AbstractHandle
|
28
|
+
layout :pending, :bool,
|
29
|
+
:response, :int,
|
30
|
+
:result, :int
|
31
|
+
|
32
|
+
def operation_name
|
33
|
+
"test_operation"
|
34
|
+
end
|
35
|
+
|
36
|
+
def create_result
|
37
|
+
self[:result]
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
subject do
|
42
|
+
TestHandle.new.tap do |handle|
|
43
|
+
handle[:pending] = pending_handle
|
44
|
+
handle[:response] = response
|
45
|
+
handle[:result] = result
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
describe ".register and .remove" do
|
50
|
+
let(:pending_handle) { true }
|
51
|
+
|
52
|
+
it "should register and remove a delivery handle" do
|
53
|
+
Rdkafka::AbstractHandle.register(subject)
|
54
|
+
removed = Rdkafka::AbstractHandle.remove(subject.to_ptr.address)
|
55
|
+
expect(removed).to eq subject
|
56
|
+
expect(Rdkafka::AbstractHandle::REGISTRY).to be_empty
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
describe "#pending?" do
|
61
|
+
context "when true" do
|
62
|
+
let(:pending_handle) { true }
|
63
|
+
|
64
|
+
it "should be true" do
|
65
|
+
expect(subject.pending?).to be true
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
context "when not true" do
|
70
|
+
let(:pending_handle) { false }
|
71
|
+
|
72
|
+
it "should be false" do
|
73
|
+
expect(subject.pending?).to be false
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
describe "#wait" do
|
79
|
+
let(:pending_handle) { true }
|
80
|
+
|
81
|
+
it "should wait until the timeout and then raise an error" do
|
82
|
+
expect {
|
83
|
+
subject.wait(max_wait_timeout: 0.1)
|
84
|
+
}.to raise_error Rdkafka::AbstractHandle::WaitTimeoutError, /test_operation/
|
85
|
+
end
|
86
|
+
|
87
|
+
context "when not pending anymore and no error" do
|
88
|
+
let(:pending_handle) { false }
|
89
|
+
let(:result) { 1 }
|
90
|
+
|
91
|
+
it "should return a result" do
|
92
|
+
wait_result = subject.wait
|
93
|
+
expect(wait_result).to eq(result)
|
94
|
+
end
|
95
|
+
|
96
|
+
it "should wait without a timeout" do
|
97
|
+
wait_result = subject.wait(max_wait_timeout: nil)
|
98
|
+
expect(wait_result).to eq(result)
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
context "when not pending anymore and there was an error" do
|
103
|
+
let(:pending_handle) { false }
|
104
|
+
let(:response) { 20 }
|
105
|
+
|
106
|
+
it "should raise an rdkafka error" do
|
107
|
+
expect {
|
108
|
+
subject.wait
|
109
|
+
}.to raise_error Rdkafka::RdkafkaError
|
110
|
+
end
|
111
|
+
end
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
@@ -0,0 +1,52 @@
|
|
1
|
+
require "spec_helper"
|
2
|
+
|
3
|
+
describe Rdkafka::Admin::CreateTopicHandle do
|
4
|
+
let(:response) { 0 }
|
5
|
+
|
6
|
+
subject do
|
7
|
+
Rdkafka::Admin::CreateTopicHandle.new.tap do |handle|
|
8
|
+
handle[:pending] = pending_handle
|
9
|
+
handle[:response] = response
|
10
|
+
handle[:error_string] = FFI::Pointer::NULL
|
11
|
+
handle[:result_name] = FFI::MemoryPointer.from_string("my-test-topic")
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
describe "#wait" do
|
16
|
+
let(:pending_handle) { true }
|
17
|
+
|
18
|
+
it "should wait until the timeout and then raise an error" do
|
19
|
+
expect {
|
20
|
+
subject.wait(max_wait_timeout: 0.1)
|
21
|
+
}.to raise_error Rdkafka::Admin::CreateTopicHandle::WaitTimeoutError, /create topic/
|
22
|
+
end
|
23
|
+
|
24
|
+
context "when not pending anymore and no error" do
|
25
|
+
let(:pending_handle) { false }
|
26
|
+
|
27
|
+
it "should return a create topic report" do
|
28
|
+
report = subject.wait
|
29
|
+
|
30
|
+
expect(report.error_string).to eq(nil)
|
31
|
+
expect(report.result_name).to eq("my-test-topic")
|
32
|
+
end
|
33
|
+
|
34
|
+
it "should wait without a timeout" do
|
35
|
+
report = subject.wait(max_wait_timeout: nil)
|
36
|
+
|
37
|
+
expect(report.error_string).to eq(nil)
|
38
|
+
expect(report.result_name).to eq("my-test-topic")
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
describe "#raise_error" do
|
44
|
+
let(:pending_handle) { false }
|
45
|
+
|
46
|
+
it "should raise the appropriate error" do
|
47
|
+
expect {
|
48
|
+
subject.raise_error
|
49
|
+
}.to raise_exception(Rdkafka::RdkafkaError, /Success \(no_error\)/)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|