rdkafka 0.12.0 → 0.13.0.beta.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.semaphore/semaphore.yml +6 -2
- data/CHANGELOG.md +12 -0
- data/Gemfile +2 -0
- data/Rakefile +2 -0
- data/ext/Rakefile +2 -0
- data/lib/rdkafka/abstract_handle.rb +2 -0
- data/lib/rdkafka/admin/create_topic_handle.rb +2 -0
- data/lib/rdkafka/admin/create_topic_report.rb +2 -0
- data/lib/rdkafka/admin/delete_topic_handle.rb +2 -0
- data/lib/rdkafka/admin/delete_topic_report.rb +2 -0
- data/lib/rdkafka/admin.rb +40 -35
- data/lib/rdkafka/bindings.rb +22 -6
- data/lib/rdkafka/callbacks.rb +7 -1
- data/lib/rdkafka/config.rb +9 -6
- data/lib/rdkafka/consumer/headers.rb +24 -7
- data/lib/rdkafka/consumer/message.rb +3 -1
- data/lib/rdkafka/consumer/partition.rb +2 -0
- data/lib/rdkafka/consumer/topic_partition_list.rb +2 -0
- data/lib/rdkafka/consumer.rb +37 -29
- data/lib/rdkafka/error.rb +9 -0
- data/lib/rdkafka/metadata.rb +2 -0
- data/lib/rdkafka/native_kafka.rb +62 -0
- data/lib/rdkafka/producer/delivery_handle.rb +5 -2
- data/lib/rdkafka/producer/delivery_report.rb +9 -2
- data/lib/rdkafka/producer.rb +23 -10
- data/lib/rdkafka/version.rb +5 -3
- data/lib/rdkafka.rb +3 -1
- data/rdkafka.gemspec +2 -0
- data/spec/rdkafka/abstract_handle_spec.rb +2 -0
- data/spec/rdkafka/admin/create_topic_handle_spec.rb +2 -0
- data/spec/rdkafka/admin/create_topic_report_spec.rb +2 -0
- data/spec/rdkafka/admin/delete_topic_handle_spec.rb +2 -0
- data/spec/rdkafka/admin/delete_topic_report_spec.rb +2 -0
- data/spec/rdkafka/admin_spec.rb +4 -3
- data/spec/rdkafka/bindings_spec.rb +2 -0
- data/spec/rdkafka/callbacks_spec.rb +2 -0
- data/spec/rdkafka/config_spec.rb +14 -0
- data/spec/rdkafka/consumer/headers_spec.rb +62 -0
- data/spec/rdkafka/consumer/message_spec.rb +3 -1
- data/spec/rdkafka/consumer/partition_spec.rb +2 -0
- data/spec/rdkafka/consumer/topic_partition_list_spec.rb +2 -0
- data/spec/rdkafka/consumer_spec.rb +84 -15
- data/spec/rdkafka/error_spec.rb +2 -0
- data/spec/rdkafka/metadata_spec.rb +3 -1
- data/spec/rdkafka/{producer/client_spec.rb → native_kafka_spec.rb} +22 -10
- data/spec/rdkafka/producer/delivery_handle_spec.rb +5 -0
- data/spec/rdkafka/producer/delivery_report_spec.rb +8 -2
- data/spec/rdkafka/producer_spec.rb +51 -19
- data/spec/spec_helper.rb +16 -0
- metadata +13 -11
- data/lib/rdkafka/producer/client.rb +0 -47
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require "spec_helper"
|
2
4
|
require "ostruct"
|
3
5
|
require 'securerandom'
|
@@ -593,7 +595,7 @@ describe Rdkafka::Consumer do
|
|
593
595
|
end
|
594
596
|
|
595
597
|
describe "#poll with headers" do
|
596
|
-
it "should return message with headers" do
|
598
|
+
it "should return message with headers using string keys (when produced with symbol keys)" do
|
597
599
|
report = producer.produce(
|
598
600
|
topic: "consume_test_topic",
|
599
601
|
key: "key headers",
|
@@ -603,7 +605,20 @@ describe Rdkafka::Consumer do
|
|
603
605
|
message = wait_for_message(topic: "consume_test_topic", consumer: consumer, delivery_report: report)
|
604
606
|
expect(message).to be
|
605
607
|
expect(message.key).to eq('key headers')
|
606
|
-
expect(message.headers).to include(foo
|
608
|
+
expect(message.headers).to include('foo' => 'bar')
|
609
|
+
end
|
610
|
+
|
611
|
+
it "should return message with headers using string keys (when produced with string keys)" do
|
612
|
+
report = producer.produce(
|
613
|
+
topic: "consume_test_topic",
|
614
|
+
key: "key headers",
|
615
|
+
headers: { 'foo' => 'bar' }
|
616
|
+
).wait
|
617
|
+
|
618
|
+
message = wait_for_message(topic: "consume_test_topic", consumer: consumer, delivery_report: report)
|
619
|
+
expect(message).to be
|
620
|
+
expect(message.key).to eq('key headers')
|
621
|
+
expect(message.headers).to include('foo' => 'bar')
|
607
622
|
end
|
608
623
|
|
609
624
|
it "should return message with no headers" do
|
@@ -698,7 +713,7 @@ describe Rdkafka::Consumer do
|
|
698
713
|
n.times do |i|
|
699
714
|
handles << producer.produce(
|
700
715
|
topic: topic_name,
|
701
|
-
payload: Time.new.to_f.to_s,
|
716
|
+
payload: i % 10 == 0 ? nil : Time.new.to_f.to_s,
|
702
717
|
key: i.to_s,
|
703
718
|
partition: 0
|
704
719
|
)
|
@@ -962,18 +977,6 @@ describe Rdkafka::Consumer do
|
|
962
977
|
expect(listener.queue).to eq([:assigned, :revoked])
|
963
978
|
end
|
964
979
|
end
|
965
|
-
|
966
|
-
def notify_listener(listener)
|
967
|
-
# 1. subscribe and poll
|
968
|
-
consumer.subscribe("consume_test_topic")
|
969
|
-
wait_for_assignment(consumer)
|
970
|
-
consumer.poll(100)
|
971
|
-
|
972
|
-
# 2. unsubscribe
|
973
|
-
consumer.unsubscribe
|
974
|
-
wait_for_unassignment(consumer)
|
975
|
-
consumer.close
|
976
|
-
end
|
977
980
|
end
|
978
981
|
|
979
982
|
context "methods that should not be called after a consumer has been closed" do
|
@@ -1005,4 +1008,70 @@ describe Rdkafka::Consumer do
|
|
1005
1008
|
end
|
1006
1009
|
end
|
1007
1010
|
end
|
1011
|
+
|
1012
|
+
it "provides a finalizer that closes the native kafka client" do
|
1013
|
+
expect(consumer.closed?).to eq(false)
|
1014
|
+
|
1015
|
+
consumer.finalizer.call("some-ignored-object-id")
|
1016
|
+
|
1017
|
+
expect(consumer.closed?).to eq(true)
|
1018
|
+
end
|
1019
|
+
|
1020
|
+
context "when the rebalance protocol is cooperative" do
|
1021
|
+
let(:consumer) do
|
1022
|
+
config = rdkafka_consumer_config(
|
1023
|
+
{
|
1024
|
+
:"partition.assignment.strategy" => "cooperative-sticky",
|
1025
|
+
:"debug" => "consumer",
|
1026
|
+
}
|
1027
|
+
)
|
1028
|
+
config.consumer_rebalance_listener = listener
|
1029
|
+
config.consumer
|
1030
|
+
end
|
1031
|
+
|
1032
|
+
let(:listener) do
|
1033
|
+
Struct.new(:queue) do
|
1034
|
+
def on_partitions_assigned(consumer, list)
|
1035
|
+
collect(:assign, list)
|
1036
|
+
end
|
1037
|
+
|
1038
|
+
def on_partitions_revoked(consumer, list)
|
1039
|
+
collect(:revoke, list)
|
1040
|
+
end
|
1041
|
+
|
1042
|
+
def collect(name, list)
|
1043
|
+
partitions = list.to_h.map { |key, values| [key, values.map(&:partition)] }.flatten
|
1044
|
+
queue << ([name] + partitions)
|
1045
|
+
end
|
1046
|
+
end.new([])
|
1047
|
+
end
|
1048
|
+
|
1049
|
+
it "should be able to assign and unassign partitions using the cooperative partition assignment APIs" do
|
1050
|
+
notify_listener(listener) do
|
1051
|
+
handles = []
|
1052
|
+
10.times do
|
1053
|
+
handles << producer.produce(
|
1054
|
+
topic: "consume_test_topic",
|
1055
|
+
payload: "payload 1",
|
1056
|
+
key: "key 1",
|
1057
|
+
partition: 0
|
1058
|
+
)
|
1059
|
+
end
|
1060
|
+
handles.each(&:wait)
|
1061
|
+
|
1062
|
+
consumer.subscribe("consume_test_topic")
|
1063
|
+
# Check the first 10 messages. Then close the consumer, which
|
1064
|
+
# should break the each loop.
|
1065
|
+
consumer.each_with_index do |message, i|
|
1066
|
+
expect(message).to be_a Rdkafka::Consumer::Message
|
1067
|
+
break if i == 10
|
1068
|
+
end
|
1069
|
+
end
|
1070
|
+
|
1071
|
+
expect(listener.queue).to eq([
|
1072
|
+
[:assign, "consume_test_topic", 0, 1, 2],
|
1073
|
+
[:revoke, "consume_test_topic", 0, 1, 2]
|
1074
|
+
])
|
1075
|
+
end
|
1076
|
+
end
|
1008
1077
|
end
|
data/spec/rdkafka/error_spec.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require "spec_helper"
|
2
4
|
require "securerandom"
|
3
5
|
|
@@ -8,7 +10,7 @@ describe Rdkafka::Metadata do
|
|
8
10
|
|
9
11
|
after do
|
10
12
|
Rdkafka::Bindings.rd_kafka_consumer_close(native_kafka)
|
11
|
-
Rdkafka::Bindings.
|
13
|
+
Rdkafka::Bindings.rd_kafka_destroy_flags(native_kafka, Rdkafka::Bindings::RD_KAFKA_DESTROY_F_IMMEDIATE)
|
12
14
|
end
|
13
15
|
|
14
16
|
context "passing in a topic name" do
|
@@ -1,17 +1,19 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require "spec_helper"
|
2
4
|
|
3
|
-
describe Rdkafka::
|
5
|
+
describe Rdkafka::NativeKafka do
|
4
6
|
let(:config) { rdkafka_producer_config }
|
5
7
|
let(:native) { config.send(:native_kafka, config.send(:native_config), :rd_kafka_producer) }
|
6
8
|
let(:closing) { false }
|
7
9
|
let(:thread) { double(Thread) }
|
8
10
|
|
9
|
-
subject(:client) { described_class.new(native) }
|
11
|
+
subject(:client) { described_class.new(native, run_polling_thread: true) }
|
10
12
|
|
11
13
|
before do
|
12
14
|
allow(Rdkafka::Bindings).to receive(:rd_kafka_poll).with(instance_of(FFI::Pointer), 250).and_call_original
|
13
15
|
allow(Rdkafka::Bindings).to receive(:rd_kafka_outq_len).with(instance_of(FFI::Pointer)).and_return(0).and_call_original
|
14
|
-
allow(Rdkafka::Bindings).to receive(:
|
16
|
+
allow(Rdkafka::Bindings).to receive(:rd_kafka_destroy_flags)
|
15
17
|
allow(Thread).to receive(:new).and_return(thread)
|
16
18
|
|
17
19
|
allow(thread).to receive(:[]=).with(:closing, anything)
|
@@ -51,6 +53,16 @@ describe Rdkafka::Producer::Client do
|
|
51
53
|
expect(Rdkafka::Bindings).to receive(:rd_kafka_outq_len).with(native).at_least(:once)
|
52
54
|
end
|
53
55
|
end
|
56
|
+
|
57
|
+
context "if not enabled" do
|
58
|
+
subject(:client) { described_class.new(native, run_polling_thread: false) }
|
59
|
+
|
60
|
+
it "is not created" do
|
61
|
+
expect(Thread).not_to receive(:new)
|
62
|
+
|
63
|
+
client
|
64
|
+
end
|
65
|
+
end
|
54
66
|
end
|
55
67
|
|
56
68
|
def polling_loop_expects(&block)
|
@@ -63,8 +75,8 @@ describe Rdkafka::Producer::Client do
|
|
63
75
|
client
|
64
76
|
end
|
65
77
|
|
66
|
-
it "exposes
|
67
|
-
expect(client.
|
78
|
+
it "exposes inner client" do
|
79
|
+
expect(client.inner).to eq(native)
|
68
80
|
end
|
69
81
|
|
70
82
|
context "when client was not yet closed (`nil`)" do
|
@@ -74,7 +86,7 @@ describe Rdkafka::Producer::Client do
|
|
74
86
|
|
75
87
|
context "and attempt to close" do
|
76
88
|
it "calls the `destroy` binding" do
|
77
|
-
expect(Rdkafka::Bindings).to receive(:
|
89
|
+
expect(Rdkafka::Bindings).to receive(:rd_kafka_destroy_flags).with(native, Rdkafka::Bindings::RD_KAFKA_DESTROY_F_IMMEDIATE)
|
78
90
|
|
79
91
|
client.close
|
80
92
|
end
|
@@ -94,7 +106,7 @@ describe Rdkafka::Producer::Client do
|
|
94
106
|
it "closes and unassign the native client" do
|
95
107
|
client.close
|
96
108
|
|
97
|
-
expect(client.
|
109
|
+
expect(client.inner).to eq(nil)
|
98
110
|
expect(client.closed?).to eq(true)
|
99
111
|
end
|
100
112
|
end
|
@@ -109,7 +121,7 @@ describe Rdkafka::Producer::Client do
|
|
109
121
|
|
110
122
|
context "and attempt to close again" do
|
111
123
|
it "does not call the `destroy` binding" do
|
112
|
-
expect(Rdkafka::Bindings).not_to receive(:
|
124
|
+
expect(Rdkafka::Bindings).not_to receive(:rd_kafka_destroy_flags)
|
113
125
|
|
114
126
|
client.close
|
115
127
|
end
|
@@ -129,13 +141,13 @@ describe Rdkafka::Producer::Client do
|
|
129
141
|
it "does not close and unassign the native client again" do
|
130
142
|
client.close
|
131
143
|
|
132
|
-
expect(client.
|
144
|
+
expect(client.inner).to eq(nil)
|
133
145
|
expect(client.closed?).to eq(true)
|
134
146
|
end
|
135
147
|
end
|
136
148
|
end
|
137
149
|
|
138
|
-
it "
|
150
|
+
it "provides a finalizer that closes the native kafka client" do
|
139
151
|
expect(client.closed?).to eq(false)
|
140
152
|
|
141
153
|
client.finalizer.call("some-ignored-object-id")
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require "spec_helper"
|
2
4
|
|
3
5
|
describe Rdkafka::Producer::DeliveryHandle do
|
@@ -9,6 +11,7 @@ describe Rdkafka::Producer::DeliveryHandle do
|
|
9
11
|
handle[:response] = response
|
10
12
|
handle[:partition] = 2
|
11
13
|
handle[:offset] = 100
|
14
|
+
handle[:topic_name] = FFI::MemoryPointer.from_string("produce_test_topic")
|
12
15
|
end
|
13
16
|
end
|
14
17
|
|
@@ -29,6 +32,7 @@ describe Rdkafka::Producer::DeliveryHandle do
|
|
29
32
|
|
30
33
|
expect(report.partition).to eq(2)
|
31
34
|
expect(report.offset).to eq(100)
|
35
|
+
expect(report.topic_name).to eq("produce_test_topic")
|
32
36
|
end
|
33
37
|
|
34
38
|
it "should wait without a timeout" do
|
@@ -36,6 +40,7 @@ describe Rdkafka::Producer::DeliveryHandle do
|
|
36
40
|
|
37
41
|
expect(report.partition).to eq(2)
|
38
42
|
expect(report.offset).to eq(100)
|
43
|
+
expect(report.topic_name).to eq("produce_test_topic")
|
39
44
|
end
|
40
45
|
end
|
41
46
|
end
|
@@ -1,7 +1,9 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require "spec_helper"
|
2
4
|
|
3
5
|
describe Rdkafka::Producer::DeliveryReport do
|
4
|
-
subject { Rdkafka::Producer::DeliveryReport.new(2, 100, "
|
6
|
+
subject { Rdkafka::Producer::DeliveryReport.new(2, 100, "topic", -1) }
|
5
7
|
|
6
8
|
it "should get the partition" do
|
7
9
|
expect(subject.partition).to eq 2
|
@@ -11,7 +13,11 @@ describe Rdkafka::Producer::DeliveryReport do
|
|
11
13
|
expect(subject.offset).to eq 100
|
12
14
|
end
|
13
15
|
|
16
|
+
it "should get the topic_name" do
|
17
|
+
expect(subject.topic_name).to eq "topic"
|
18
|
+
end
|
19
|
+
|
14
20
|
it "should get the error" do
|
15
|
-
expect(subject.error).to eq
|
21
|
+
expect(subject.error).to eq -1
|
16
22
|
end
|
17
23
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require "spec_helper"
|
2
4
|
require "zlib"
|
3
5
|
|
@@ -7,7 +9,8 @@ describe Rdkafka::Producer do
|
|
7
9
|
|
8
10
|
after do
|
9
11
|
# Registry should always end up being empty
|
10
|
-
|
12
|
+
registry = Rdkafka::Producer::DeliveryHandle::REGISTRY
|
13
|
+
expect(registry).to be_empty, registry.inspect
|
11
14
|
producer.close
|
12
15
|
consumer.close
|
13
16
|
end
|
@@ -30,6 +33,7 @@ describe Rdkafka::Producer do
|
|
30
33
|
expect(report).not_to be_nil
|
31
34
|
expect(report.partition).to eq 1
|
32
35
|
expect(report.offset).to be >= 0
|
36
|
+
expect(report.topic_name).to eq "produce_test_topic"
|
33
37
|
@callback_called = true
|
34
38
|
end
|
35
39
|
|
@@ -113,6 +117,7 @@ describe Rdkafka::Producer do
|
|
113
117
|
expect(called_report.first).not_to be_nil
|
114
118
|
expect(called_report.first.partition).to eq 1
|
115
119
|
expect(called_report.first.offset).to be >= 0
|
120
|
+
expect(called_report.first.topic_name).to eq "produce_test_topic"
|
116
121
|
end
|
117
122
|
|
118
123
|
it "should provide handle" do
|
@@ -180,10 +185,11 @@ describe Rdkafka::Producer do
|
|
180
185
|
expect(report.partition).to eq 1
|
181
186
|
expect(report.offset).to be >= 0
|
182
187
|
|
183
|
-
#
|
188
|
+
# Flush and close producer
|
189
|
+
producer.flush
|
184
190
|
producer.close
|
185
191
|
|
186
|
-
# Consume message and verify
|
192
|
+
# Consume message and verify its content
|
187
193
|
message = wait_for_message(
|
188
194
|
topic: "produce_test_topic",
|
189
195
|
delivery_report: report,
|
@@ -207,7 +213,7 @@ describe Rdkafka::Producer do
|
|
207
213
|
)
|
208
214
|
report = handle.wait(max_wait_timeout: 5)
|
209
215
|
|
210
|
-
# Consume message and verify
|
216
|
+
# Consume message and verify its content
|
211
217
|
message = wait_for_message(
|
212
218
|
topic: "produce_test_topic",
|
213
219
|
delivery_report: report,
|
@@ -251,6 +257,28 @@ describe Rdkafka::Producer do
|
|
251
257
|
expect(messages[2].key).to eq key
|
252
258
|
end
|
253
259
|
|
260
|
+
it "should produce a message with empty string without crashing" do
|
261
|
+
messages = [{key: 'a', partition_key: ''}]
|
262
|
+
|
263
|
+
messages = messages.map do |m|
|
264
|
+
handle = producer.produce(
|
265
|
+
topic: "partitioner_test_topic",
|
266
|
+
payload: "payload partition",
|
267
|
+
key: m[:key],
|
268
|
+
partition_key: m[:partition_key]
|
269
|
+
)
|
270
|
+
report = handle.wait(max_wait_timeout: 5)
|
271
|
+
|
272
|
+
wait_for_message(
|
273
|
+
topic: "partitioner_test_topic",
|
274
|
+
delivery_report: report,
|
275
|
+
)
|
276
|
+
end
|
277
|
+
|
278
|
+
expect(messages[0].partition).to eq 0
|
279
|
+
expect(messages[0].key).to eq 'a'
|
280
|
+
end
|
281
|
+
|
254
282
|
it "should produce a message with utf-8 encoding" do
|
255
283
|
handle = producer.produce(
|
256
284
|
topic: "produce_test_topic",
|
@@ -259,7 +287,7 @@ describe Rdkafka::Producer do
|
|
259
287
|
)
|
260
288
|
report = handle.wait(max_wait_timeout: 5)
|
261
289
|
|
262
|
-
# Consume message and verify
|
290
|
+
# Consume message and verify its content
|
263
291
|
message = wait_for_message(
|
264
292
|
topic: "produce_test_topic",
|
265
293
|
delivery_report: report,
|
@@ -292,7 +320,7 @@ describe Rdkafka::Producer do
|
|
292
320
|
)
|
293
321
|
report = handle.wait(max_wait_timeout: 5)
|
294
322
|
|
295
|
-
# Consume message and verify
|
323
|
+
# Consume message and verify its content
|
296
324
|
message = wait_for_message(
|
297
325
|
topic: "produce_test_topic",
|
298
326
|
delivery_report: report,
|
@@ -313,7 +341,7 @@ describe Rdkafka::Producer do
|
|
313
341
|
)
|
314
342
|
report = handle.wait(max_wait_timeout: 5)
|
315
343
|
|
316
|
-
# Consume message and verify
|
344
|
+
# Consume message and verify its content
|
317
345
|
message = wait_for_message(
|
318
346
|
topic: "produce_test_topic",
|
319
347
|
delivery_report: report,
|
@@ -333,7 +361,7 @@ describe Rdkafka::Producer do
|
|
333
361
|
)
|
334
362
|
report = handle.wait(max_wait_timeout: 5)
|
335
363
|
|
336
|
-
# Consume message and verify
|
364
|
+
# Consume message and verify its content
|
337
365
|
message = wait_for_message(
|
338
366
|
topic: "produce_test_topic",
|
339
367
|
delivery_report: report,
|
@@ -351,7 +379,7 @@ describe Rdkafka::Producer do
|
|
351
379
|
)
|
352
380
|
report = handle.wait(max_wait_timeout: 5)
|
353
381
|
|
354
|
-
# Consume message and verify
|
382
|
+
# Consume message and verify its content
|
355
383
|
message = wait_for_message(
|
356
384
|
topic: "produce_test_topic",
|
357
385
|
delivery_report: report,
|
@@ -371,7 +399,7 @@ describe Rdkafka::Producer do
|
|
371
399
|
)
|
372
400
|
report = handle.wait(max_wait_timeout: 5)
|
373
401
|
|
374
|
-
# Consume message and verify
|
402
|
+
# Consume message and verify its content
|
375
403
|
message = wait_for_message(
|
376
404
|
topic: "produce_test_topic",
|
377
405
|
delivery_report: report,
|
@@ -380,9 +408,9 @@ describe Rdkafka::Producer do
|
|
380
408
|
|
381
409
|
expect(message.payload).to eq "payload headers"
|
382
410
|
expect(message.key).to eq "key headers"
|
383
|
-
expect(message.headers[
|
384
|
-
expect(message.headers[
|
385
|
-
expect(message.headers[
|
411
|
+
expect(message.headers["foo"]).to eq "bar"
|
412
|
+
expect(message.headers["baz"]).to eq "foobar"
|
413
|
+
expect(message.headers["foobar"]).to be_nil
|
386
414
|
end
|
387
415
|
|
388
416
|
it "should produce a message with empty headers" do
|
@@ -394,7 +422,7 @@ describe Rdkafka::Producer do
|
|
394
422
|
)
|
395
423
|
report = handle.wait(max_wait_timeout: 5)
|
396
424
|
|
397
|
-
# Consume message and verify
|
425
|
+
# Consume message and verify its content
|
398
426
|
message = wait_for_message(
|
399
427
|
topic: "produce_test_topic",
|
400
428
|
delivery_report: report,
|
@@ -432,10 +460,10 @@ describe Rdkafka::Producer do
|
|
432
460
|
# wait for and check the message in the main process.
|
433
461
|
reader, writer = IO.pipe
|
434
462
|
|
435
|
-
fork do
|
463
|
+
pid = fork do
|
436
464
|
reader.close
|
437
465
|
|
438
|
-
#
|
466
|
+
# Avoid sharing the client between processes.
|
439
467
|
producer = rdkafka_producer_config.producer
|
440
468
|
|
441
469
|
handle = producer.produce(
|
@@ -448,24 +476,28 @@ describe Rdkafka::Producer do
|
|
448
476
|
|
449
477
|
report_json = JSON.generate(
|
450
478
|
"partition" => report.partition,
|
451
|
-
"offset" => report.offset
|
479
|
+
"offset" => report.offset,
|
480
|
+
"topic_name" => report.topic_name
|
452
481
|
)
|
453
482
|
|
454
483
|
writer.write(report_json)
|
455
484
|
writer.close
|
485
|
+
producer.flush
|
456
486
|
producer.close
|
457
487
|
end
|
488
|
+
Process.wait(pid)
|
458
489
|
|
459
490
|
writer.close
|
460
491
|
report_hash = JSON.parse(reader.read)
|
461
492
|
report = Rdkafka::Producer::DeliveryReport.new(
|
462
493
|
report_hash["partition"],
|
463
|
-
report_hash["offset"]
|
494
|
+
report_hash["offset"],
|
495
|
+
report_hash["topic_name"]
|
464
496
|
)
|
465
497
|
|
466
498
|
reader.close
|
467
499
|
|
468
|
-
# Consume message and verify
|
500
|
+
# Consume message and verify its content
|
469
501
|
message = wait_for_message(
|
470
502
|
topic: "produce_test_topic",
|
471
503
|
delivery_report: report,
|
data/spec/spec_helper.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
unless ENV["CI"] == "true"
|
2
4
|
require "simplecov"
|
3
5
|
SimpleCov.start do
|
@@ -104,6 +106,20 @@ def wait_for_unassignment(consumer)
|
|
104
106
|
end
|
105
107
|
end
|
106
108
|
|
109
|
+
def notify_listener(listener, &block)
|
110
|
+
# 1. subscribe and poll
|
111
|
+
consumer.subscribe("consume_test_topic")
|
112
|
+
wait_for_assignment(consumer)
|
113
|
+
consumer.poll(100)
|
114
|
+
|
115
|
+
block.call if block
|
116
|
+
|
117
|
+
# 2. unsubscribe
|
118
|
+
consumer.unsubscribe
|
119
|
+
wait_for_unassignment(consumer)
|
120
|
+
consumer.close
|
121
|
+
end
|
122
|
+
|
107
123
|
RSpec.configure do |config|
|
108
124
|
config.filter_run focus: true
|
109
125
|
config.run_all_when_everything_filtered = true
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rdkafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.13.0.beta.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Thijs Cadier
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-
|
11
|
+
date: 2022-10-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: ffi
|
@@ -176,8 +176,8 @@ files:
|
|
176
176
|
- lib/rdkafka/consumer/topic_partition_list.rb
|
177
177
|
- lib/rdkafka/error.rb
|
178
178
|
- lib/rdkafka/metadata.rb
|
179
|
+
- lib/rdkafka/native_kafka.rb
|
179
180
|
- lib/rdkafka/producer.rb
|
180
|
-
- lib/rdkafka/producer/client.rb
|
181
181
|
- lib/rdkafka/producer/delivery_handle.rb
|
182
182
|
- lib/rdkafka/producer/delivery_report.rb
|
183
183
|
- lib/rdkafka/version.rb
|
@@ -191,13 +191,14 @@ files:
|
|
191
191
|
- spec/rdkafka/bindings_spec.rb
|
192
192
|
- spec/rdkafka/callbacks_spec.rb
|
193
193
|
- spec/rdkafka/config_spec.rb
|
194
|
+
- spec/rdkafka/consumer/headers_spec.rb
|
194
195
|
- spec/rdkafka/consumer/message_spec.rb
|
195
196
|
- spec/rdkafka/consumer/partition_spec.rb
|
196
197
|
- spec/rdkafka/consumer/topic_partition_list_spec.rb
|
197
198
|
- spec/rdkafka/consumer_spec.rb
|
198
199
|
- spec/rdkafka/error_spec.rb
|
199
200
|
- spec/rdkafka/metadata_spec.rb
|
200
|
-
- spec/rdkafka/
|
201
|
+
- spec/rdkafka/native_kafka_spec.rb
|
201
202
|
- spec/rdkafka/producer/delivery_handle_spec.rb
|
202
203
|
- spec/rdkafka/producer/delivery_report_spec.rb
|
203
204
|
- spec/rdkafka/producer_spec.rb
|
@@ -206,7 +207,7 @@ homepage: https://github.com/thijsc/rdkafka-ruby
|
|
206
207
|
licenses:
|
207
208
|
- MIT
|
208
209
|
metadata: {}
|
209
|
-
post_install_message:
|
210
|
+
post_install_message:
|
210
211
|
rdoc_options: []
|
211
212
|
require_paths:
|
212
213
|
- lib
|
@@ -217,12 +218,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
217
218
|
version: '2.6'
|
218
219
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
219
220
|
requirements:
|
220
|
-
- - "
|
221
|
+
- - ">"
|
221
222
|
- !ruby/object:Gem::Version
|
222
|
-
version:
|
223
|
+
version: 1.3.1
|
223
224
|
requirements: []
|
224
|
-
rubygems_version: 3.
|
225
|
-
signing_key:
|
225
|
+
rubygems_version: 3.3.13
|
226
|
+
signing_key:
|
226
227
|
specification_version: 4
|
227
228
|
summary: The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka.
|
228
229
|
It wraps the production-ready C client using the ffi gem and targets Kafka 1.0+
|
@@ -237,13 +238,14 @@ test_files:
|
|
237
238
|
- spec/rdkafka/bindings_spec.rb
|
238
239
|
- spec/rdkafka/callbacks_spec.rb
|
239
240
|
- spec/rdkafka/config_spec.rb
|
241
|
+
- spec/rdkafka/consumer/headers_spec.rb
|
240
242
|
- spec/rdkafka/consumer/message_spec.rb
|
241
243
|
- spec/rdkafka/consumer/partition_spec.rb
|
242
244
|
- spec/rdkafka/consumer/topic_partition_list_spec.rb
|
243
245
|
- spec/rdkafka/consumer_spec.rb
|
244
246
|
- spec/rdkafka/error_spec.rb
|
245
247
|
- spec/rdkafka/metadata_spec.rb
|
246
|
-
- spec/rdkafka/
|
248
|
+
- spec/rdkafka/native_kafka_spec.rb
|
247
249
|
- spec/rdkafka/producer/delivery_handle_spec.rb
|
248
250
|
- spec/rdkafka/producer/delivery_report_spec.rb
|
249
251
|
- spec/rdkafka/producer_spec.rb
|
@@ -1,47 +0,0 @@
|
|
1
|
-
module Rdkafka
|
2
|
-
class Producer
|
3
|
-
class Client
|
4
|
-
def initialize(native)
|
5
|
-
@native = native
|
6
|
-
|
7
|
-
# Start thread to poll client for delivery callbacks
|
8
|
-
@polling_thread = Thread.new do
|
9
|
-
loop do
|
10
|
-
Rdkafka::Bindings.rd_kafka_poll(native, 250)
|
11
|
-
# Exit thread if closing and the poll queue is empty
|
12
|
-
if Thread.current[:closing] && Rdkafka::Bindings.rd_kafka_outq_len(native) == 0
|
13
|
-
break
|
14
|
-
end
|
15
|
-
end
|
16
|
-
end
|
17
|
-
@polling_thread.abort_on_exception = true
|
18
|
-
@polling_thread[:closing] = false
|
19
|
-
end
|
20
|
-
|
21
|
-
def native
|
22
|
-
@native
|
23
|
-
end
|
24
|
-
|
25
|
-
def finalizer
|
26
|
-
->(_) { close }
|
27
|
-
end
|
28
|
-
|
29
|
-
def closed?
|
30
|
-
@native.nil?
|
31
|
-
end
|
32
|
-
|
33
|
-
def close(object_id=nil)
|
34
|
-
return unless @native
|
35
|
-
|
36
|
-
# Indicate to polling thread that we're closing
|
37
|
-
@polling_thread[:closing] = true
|
38
|
-
# Wait for the polling thread to finish up
|
39
|
-
@polling_thread.join
|
40
|
-
|
41
|
-
Rdkafka::Bindings.rd_kafka_destroy(@native)
|
42
|
-
|
43
|
-
@native = nil
|
44
|
-
end
|
45
|
-
end
|
46
|
-
end
|
47
|
-
end
|