rdkafka 0.11.0 → 0.12.0.beta.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/rdkafka/admin.rb +2 -2
- data/lib/rdkafka/config.rb +1 -1
- data/lib/rdkafka/producer/client.rb +47 -0
- data/lib/rdkafka/producer.rb +9 -30
- data/lib/rdkafka/version.rb +3 -3
- data/lib/rdkafka.rb +1 -0
- data/rdkafka.gemspec +1 -1
- data/spec/rdkafka/consumer_spec.rb +3 -5
- data/spec/rdkafka/producer/client_spec.rb +145 -0
- data/spec/rdkafka/producer_spec.rb +1 -5
- metadata +13 -10
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b7009092099e6d6f23e2e4c3ed22799e8e8473ac4265a7c72a81b55d7f4d3df6
|
4
|
+
data.tar.gz: 847c47119591237cca59f178e7a0c438f098ed26039bf96de04d01df7d03362c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 49b59362a354331726158d9d4f3c34b882ac87495d688861089a316873d7a25d4cdea16f37d347cd59db04f2b79410b4885d15edf336d75c4b8b9a4675326af6
|
7
|
+
data.tar.gz: f85edc6bc9937a65169314ce9c068caac55dbd9ddd9d6b2e25c31f97323ac3b2584e9ff8f2d65342cd296278cb524802814e06c806c88b01b940bc13153f5418
|
data/lib/rdkafka/admin.rb
CHANGED
@@ -90,7 +90,7 @@ module Rdkafka
|
|
90
90
|
admin_options_ptr,
|
91
91
|
queue_ptr
|
92
92
|
)
|
93
|
-
rescue Exception
|
93
|
+
rescue Exception
|
94
94
|
CreateTopicHandle.remove(create_topic_handle.to_ptr.address)
|
95
95
|
raise
|
96
96
|
ensure
|
@@ -140,7 +140,7 @@ module Rdkafka
|
|
140
140
|
admin_options_ptr,
|
141
141
|
queue_ptr
|
142
142
|
)
|
143
|
-
rescue Exception
|
143
|
+
rescue Exception
|
144
144
|
DeleteTopicHandle.remove(delete_topic_handle.to_ptr.address)
|
145
145
|
raise
|
146
146
|
ensure
|
data/lib/rdkafka/config.rb
CHANGED
@@ -179,7 +179,7 @@ module Rdkafka
|
|
179
179
|
# Set callback to receive delivery reports on config
|
180
180
|
Rdkafka::Bindings.rd_kafka_conf_set_dr_msg_cb(config, Rdkafka::Callbacks::DeliveryCallbackFunction)
|
181
181
|
# Return producer with Kafka client
|
182
|
-
Rdkafka::Producer.new(native_kafka(config, :rd_kafka_producer)).tap do |producer|
|
182
|
+
Rdkafka::Producer.new(Rdkafka::Producer::Client.new(native_kafka(config, :rd_kafka_producer))).tap do |producer|
|
183
183
|
opaque.producer = producer
|
184
184
|
end
|
185
185
|
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
module Rdkafka
|
2
|
+
class Producer
|
3
|
+
class Client
|
4
|
+
def initialize(native)
|
5
|
+
@native = native
|
6
|
+
|
7
|
+
# Start thread to poll client for delivery callbacks
|
8
|
+
@polling_thread = Thread.new do
|
9
|
+
loop do
|
10
|
+
Rdkafka::Bindings.rd_kafka_poll(native, 250)
|
11
|
+
# Exit thread if closing and the poll queue is empty
|
12
|
+
if Thread.current[:closing] && Rdkafka::Bindings.rd_kafka_outq_len(native) == 0
|
13
|
+
break
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
@polling_thread.abort_on_exception = true
|
18
|
+
@polling_thread[:closing] = false
|
19
|
+
end
|
20
|
+
|
21
|
+
def native
|
22
|
+
@native
|
23
|
+
end
|
24
|
+
|
25
|
+
def finalizer
|
26
|
+
->(_) { close }
|
27
|
+
end
|
28
|
+
|
29
|
+
def closed?
|
30
|
+
@native.nil?
|
31
|
+
end
|
32
|
+
|
33
|
+
def close(object_id=nil)
|
34
|
+
return unless @native
|
35
|
+
|
36
|
+
# Indicate to polling thread that we're closing
|
37
|
+
@polling_thread[:closing] = true
|
38
|
+
# Wait for the polling thread to finish up
|
39
|
+
@polling_thread.join
|
40
|
+
|
41
|
+
Rdkafka::Bindings.rd_kafka_destroy(@native)
|
42
|
+
|
43
|
+
@native = nil
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
data/lib/rdkafka/producer.rb
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
require "
|
1
|
+
require "objspace"
|
2
2
|
|
3
3
|
module Rdkafka
|
4
4
|
# A producer for Kafka messages. To create a producer set up a {Config} and call {Config#producer producer} on that.
|
@@ -10,25 +10,11 @@ module Rdkafka
|
|
10
10
|
attr_reader :delivery_callback
|
11
11
|
|
12
12
|
# @private
|
13
|
-
def initialize(
|
14
|
-
@
|
15
|
-
@closing = false
|
16
|
-
@native_kafka = native_kafka
|
13
|
+
def initialize(client)
|
14
|
+
@client = client
|
17
15
|
|
18
16
|
# Makes sure, that the producer gets closed before it gets GCed by Ruby
|
19
|
-
ObjectSpace.define_finalizer(
|
20
|
-
|
21
|
-
# Start thread to poll client for delivery callbacks
|
22
|
-
@polling_thread = Thread.new do
|
23
|
-
loop do
|
24
|
-
Rdkafka::Bindings.rd_kafka_poll(@native_kafka, 250)
|
25
|
-
# Exit thread if closing and the poll queue is empty
|
26
|
-
if @closing && Rdkafka::Bindings.rd_kafka_outq_len(@native_kafka) == 0
|
27
|
-
break
|
28
|
-
end
|
29
|
-
end
|
30
|
-
end
|
31
|
-
@polling_thread.abort_on_exception = true
|
17
|
+
ObjectSpace.define_finalizer(self, client.finalizer)
|
32
18
|
end
|
33
19
|
|
34
20
|
# Set a callback that will be called every time a message is successfully produced.
|
@@ -44,16 +30,9 @@ module Rdkafka
|
|
44
30
|
|
45
31
|
# Close this producer and wait for the internal poll queue to empty.
|
46
32
|
def close
|
47
|
-
ObjectSpace.undefine_finalizer(
|
48
|
-
|
49
|
-
return unless @native_kafka
|
33
|
+
ObjectSpace.undefine_finalizer(self)
|
50
34
|
|
51
|
-
|
52
|
-
@closing = true
|
53
|
-
# Wait for the polling thread to finish up
|
54
|
-
@polling_thread.join
|
55
|
-
Rdkafka::Bindings.rd_kafka_destroy(@native_kafka)
|
56
|
-
@native_kafka = nil
|
35
|
+
@client.close
|
57
36
|
end
|
58
37
|
|
59
38
|
# Partition count for a given topic.
|
@@ -65,7 +44,7 @@ module Rdkafka
|
|
65
44
|
#
|
66
45
|
def partition_count(topic)
|
67
46
|
closed_producer_check(__method__)
|
68
|
-
Rdkafka::Metadata.new(@
|
47
|
+
Rdkafka::Metadata.new(@client.native, topic).topics&.first[:partition_count]
|
69
48
|
end
|
70
49
|
|
71
50
|
# Produces a message to a Kafka topic. The message is added to rdkafka's queue, call {DeliveryHandle#wait wait} on the returned delivery handle to make sure it is delivered.
|
@@ -157,7 +136,7 @@ module Rdkafka
|
|
157
136
|
|
158
137
|
# Produce the message
|
159
138
|
response = Rdkafka::Bindings.rd_kafka_producev(
|
160
|
-
@
|
139
|
+
@client.native,
|
161
140
|
*args
|
162
141
|
)
|
163
142
|
|
@@ -176,7 +155,7 @@ module Rdkafka
|
|
176
155
|
end
|
177
156
|
|
178
157
|
def closed_producer_check(method)
|
179
|
-
raise Rdkafka::ClosedProducerError.new(method) if @
|
158
|
+
raise Rdkafka::ClosedProducerError.new(method) if @client.closed?
|
180
159
|
end
|
181
160
|
end
|
182
161
|
end
|
data/lib/rdkafka/version.rb
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
module Rdkafka
|
2
|
-
VERSION = "0.
|
3
|
-
LIBRDKAFKA_VERSION = "1.
|
4
|
-
LIBRDKAFKA_SOURCE_SHA256 = "
|
2
|
+
VERSION = "0.12.0.beta.1"
|
3
|
+
LIBRDKAFKA_VERSION = "1.9.0-RC1"
|
4
|
+
LIBRDKAFKA_SOURCE_SHA256 = "c6aefe22ee14e8d036304bd61a5e5a61e100f0554df911fe107af1375c9f01f5"
|
5
5
|
end
|
data/lib/rdkafka.rb
CHANGED
data/rdkafka.gemspec
CHANGED
@@ -18,7 +18,7 @@ Gem::Specification.new do |gem|
|
|
18
18
|
gem.extensions = %w(ext/Rakefile)
|
19
19
|
|
20
20
|
gem.add_dependency 'ffi', '~> 1.15'
|
21
|
-
gem.add_dependency 'mini_portile2', '~> 2.
|
21
|
+
gem.add_dependency 'mini_portile2', '~> 2.6'
|
22
22
|
gem.add_dependency 'rake', '> 12'
|
23
23
|
|
24
24
|
gem.add_development_dependency 'pry'
|
@@ -241,7 +241,7 @@ describe Rdkafka::Consumer do
|
|
241
241
|
|
242
242
|
it "should return the assignment when subscribed" do
|
243
243
|
# Make sure there's a message
|
244
|
-
|
244
|
+
producer.produce(
|
245
245
|
topic: "consume_test_topic",
|
246
246
|
payload: "payload 1",
|
247
247
|
key: "key 1",
|
@@ -272,7 +272,7 @@ describe Rdkafka::Consumer do
|
|
272
272
|
it "should close a consumer" do
|
273
273
|
consumer.subscribe("consume_test_topic")
|
274
274
|
100.times do |i|
|
275
|
-
|
275
|
+
producer.produce(
|
276
276
|
topic: "consume_test_topic",
|
277
277
|
payload: "payload #{i}",
|
278
278
|
key: "key #{i}",
|
@@ -289,7 +289,7 @@ describe Rdkafka::Consumer do
|
|
289
289
|
describe "#commit, #committed and #store_offset" do
|
290
290
|
# Make sure there's a stored offset
|
291
291
|
let!(:report) do
|
292
|
-
|
292
|
+
producer.produce(
|
293
293
|
topic: "consume_test_topic",
|
294
294
|
payload: "payload 1",
|
295
295
|
key: "key 1",
|
@@ -831,7 +831,6 @@ describe Rdkafka::Consumer do
|
|
831
831
|
)
|
832
832
|
consumer = config.consumer
|
833
833
|
consumer.subscribe(topic_name)
|
834
|
-
loop_count = 0
|
835
834
|
batches_yielded = []
|
836
835
|
exceptions_yielded = []
|
837
836
|
each_batch_iterations = 0
|
@@ -875,7 +874,6 @@ describe Rdkafka::Consumer do
|
|
875
874
|
)
|
876
875
|
consumer = config.consumer
|
877
876
|
consumer.subscribe(topic_name)
|
878
|
-
loop_count = 0
|
879
877
|
batches_yielded = []
|
880
878
|
exceptions_yielded = []
|
881
879
|
each_batch_iterations = 0
|
@@ -0,0 +1,145 @@
|
|
1
|
+
require "spec_helper"
|
2
|
+
|
3
|
+
describe Rdkafka::Producer::Client do
|
4
|
+
let(:config) { rdkafka_producer_config }
|
5
|
+
let(:native) { config.send(:native_kafka, config.send(:native_config), :rd_kafka_producer) }
|
6
|
+
let(:closing) { false }
|
7
|
+
let(:thread) { double(Thread) }
|
8
|
+
|
9
|
+
subject(:client) { described_class.new(native) }
|
10
|
+
|
11
|
+
before do
|
12
|
+
allow(Rdkafka::Bindings).to receive(:rd_kafka_poll).with(instance_of(FFI::Pointer), 250).and_call_original
|
13
|
+
allow(Rdkafka::Bindings).to receive(:rd_kafka_outq_len).with(instance_of(FFI::Pointer)).and_return(0).and_call_original
|
14
|
+
allow(Rdkafka::Bindings).to receive(:rd_kafka_destroy)
|
15
|
+
allow(Thread).to receive(:new).and_return(thread)
|
16
|
+
|
17
|
+
allow(thread).to receive(:[]=).with(:closing, anything)
|
18
|
+
allow(thread).to receive(:join)
|
19
|
+
allow(thread).to receive(:abort_on_exception=).with(anything)
|
20
|
+
end
|
21
|
+
|
22
|
+
context "defaults" do
|
23
|
+
it "sets the thread to abort on exception" do
|
24
|
+
expect(thread).to receive(:abort_on_exception=).with(true)
|
25
|
+
|
26
|
+
client
|
27
|
+
end
|
28
|
+
|
29
|
+
it "sets the thread `closing` flag to false" do
|
30
|
+
expect(thread).to receive(:[]=).with(:closing, false)
|
31
|
+
|
32
|
+
client
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
context "the polling thread" do
|
37
|
+
it "is created" do
|
38
|
+
expect(Thread).to receive(:new)
|
39
|
+
|
40
|
+
client
|
41
|
+
end
|
42
|
+
|
43
|
+
it "polls the native with default 250ms timeout" do
|
44
|
+
polling_loop_expects do
|
45
|
+
expect(Rdkafka::Bindings).to receive(:rd_kafka_poll).with(instance_of(FFI::Pointer), 250)
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
it "check the out queue of native client" do
|
50
|
+
polling_loop_expects do
|
51
|
+
expect(Rdkafka::Bindings).to receive(:rd_kafka_outq_len).with(native)
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
def polling_loop_expects(&block)
|
57
|
+
Thread.current[:closing] = true # this forces the loop break with line #12
|
58
|
+
|
59
|
+
allow(Thread).to receive(:new).and_yield do |_|
|
60
|
+
block.call
|
61
|
+
end.and_return(thread)
|
62
|
+
|
63
|
+
client
|
64
|
+
end
|
65
|
+
|
66
|
+
it "exposes `native` client" do
|
67
|
+
expect(client.native).to eq(native)
|
68
|
+
end
|
69
|
+
|
70
|
+
context "when client was not yet closed (`nil`)" do
|
71
|
+
it "is not closed" do
|
72
|
+
expect(client.closed?).to eq(false)
|
73
|
+
end
|
74
|
+
|
75
|
+
context "and attempt to close" do
|
76
|
+
it "calls the `destroy` binding" do
|
77
|
+
expect(Rdkafka::Bindings).to receive(:rd_kafka_destroy).with(native)
|
78
|
+
|
79
|
+
client.close
|
80
|
+
end
|
81
|
+
|
82
|
+
it "indicates to the polling thread that it is closing" do
|
83
|
+
expect(thread).to receive(:[]=).with(:closing, true)
|
84
|
+
|
85
|
+
client.close
|
86
|
+
end
|
87
|
+
|
88
|
+
it "joins the polling thread" do
|
89
|
+
expect(thread).to receive(:join)
|
90
|
+
|
91
|
+
client.close
|
92
|
+
end
|
93
|
+
|
94
|
+
it "closes and unassign the native client" do
|
95
|
+
client.close
|
96
|
+
|
97
|
+
expect(client.native).to eq(nil)
|
98
|
+
expect(client.closed?).to eq(true)
|
99
|
+
end
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
context "when client was already closed" do
|
104
|
+
before { client.close }
|
105
|
+
|
106
|
+
it "is closed" do
|
107
|
+
expect(client.closed?).to eq(true)
|
108
|
+
end
|
109
|
+
|
110
|
+
context "and attempt to close again" do
|
111
|
+
it "does not call the `destroy` binding" do
|
112
|
+
expect(Rdkafka::Bindings).not_to receive(:rd_kafka_destroy)
|
113
|
+
|
114
|
+
client.close
|
115
|
+
end
|
116
|
+
|
117
|
+
it "does not indicate to the polling thread that it is closing" do
|
118
|
+
expect(thread).not_to receive(:[]=).with(:closing, true)
|
119
|
+
|
120
|
+
client.close
|
121
|
+
end
|
122
|
+
|
123
|
+
it "does not join the polling thread" do
|
124
|
+
expect(thread).not_to receive(:join)
|
125
|
+
|
126
|
+
client.close
|
127
|
+
end
|
128
|
+
|
129
|
+
it "does not close and unassign the native client again" do
|
130
|
+
client.close
|
131
|
+
|
132
|
+
expect(client.native).to eq(nil)
|
133
|
+
expect(client.closed?).to eq(true)
|
134
|
+
end
|
135
|
+
end
|
136
|
+
end
|
137
|
+
|
138
|
+
it "provide a finalizer Proc that closes the `native` client" do
|
139
|
+
expect(client.closed?).to eq(false)
|
140
|
+
|
141
|
+
client.finalizer.call("some-ignored-object-id")
|
142
|
+
|
143
|
+
expect(client.closed?).to eq(true)
|
144
|
+
end
|
145
|
+
end
|
@@ -376,13 +376,9 @@ describe Rdkafka::Producer do
|
|
376
376
|
end
|
377
377
|
end
|
378
378
|
|
379
|
-
it "should produce a message in a forked process" do
|
379
|
+
it "should produce a message in a forked process", skip: defined?(JRUBY_VERSION) && "Kernel#fork is not available" do
|
380
380
|
# Fork, produce a message, send the report over a pipe and
|
381
381
|
# wait for and check the message in the main process.
|
382
|
-
|
383
|
-
# Kernel#fork is not available in JRuby
|
384
|
-
skip if defined?(JRUBY_VERSION)
|
385
|
-
|
386
382
|
reader, writer = IO.pipe
|
387
383
|
|
388
384
|
fork do
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rdkafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.12.0.beta.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Thijs Cadier
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2022-03-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: ffi
|
@@ -30,14 +30,14 @@ dependencies:
|
|
30
30
|
requirements:
|
31
31
|
- - "~>"
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version: '2.
|
33
|
+
version: '2.6'
|
34
34
|
type: :runtime
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
38
|
- - "~>"
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version: '2.
|
40
|
+
version: '2.6'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: rake
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
@@ -177,6 +177,7 @@ files:
|
|
177
177
|
- lib/rdkafka/error.rb
|
178
178
|
- lib/rdkafka/metadata.rb
|
179
179
|
- lib/rdkafka/producer.rb
|
180
|
+
- lib/rdkafka/producer/client.rb
|
180
181
|
- lib/rdkafka/producer/delivery_handle.rb
|
181
182
|
- lib/rdkafka/producer/delivery_report.rb
|
182
183
|
- lib/rdkafka/version.rb
|
@@ -196,6 +197,7 @@ files:
|
|
196
197
|
- spec/rdkafka/consumer_spec.rb
|
197
198
|
- spec/rdkafka/error_spec.rb
|
198
199
|
- spec/rdkafka/metadata_spec.rb
|
200
|
+
- spec/rdkafka/producer/client_spec.rb
|
199
201
|
- spec/rdkafka/producer/delivery_handle_spec.rb
|
200
202
|
- spec/rdkafka/producer/delivery_report_spec.rb
|
201
203
|
- spec/rdkafka/producer_spec.rb
|
@@ -204,7 +206,7 @@ homepage: https://github.com/thijsc/rdkafka-ruby
|
|
204
206
|
licenses:
|
205
207
|
- MIT
|
206
208
|
metadata: {}
|
207
|
-
post_install_message:
|
209
|
+
post_install_message:
|
208
210
|
rdoc_options: []
|
209
211
|
require_paths:
|
210
212
|
- lib
|
@@ -215,12 +217,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
215
217
|
version: '2.6'
|
216
218
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
217
219
|
requirements:
|
218
|
-
- - "
|
220
|
+
- - ">"
|
219
221
|
- !ruby/object:Gem::Version
|
220
|
-
version:
|
222
|
+
version: 1.3.1
|
221
223
|
requirements: []
|
222
|
-
rubygems_version: 3.
|
223
|
-
signing_key:
|
224
|
+
rubygems_version: 3.0.3
|
225
|
+
signing_key:
|
224
226
|
specification_version: 4
|
225
227
|
summary: The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka.
|
226
228
|
It wraps the production-ready C client using the ffi gem and targets Kafka 1.0+
|
@@ -241,6 +243,7 @@ test_files:
|
|
241
243
|
- spec/rdkafka/consumer_spec.rb
|
242
244
|
- spec/rdkafka/error_spec.rb
|
243
245
|
- spec/rdkafka/metadata_spec.rb
|
246
|
+
- spec/rdkafka/producer/client_spec.rb
|
244
247
|
- spec/rdkafka/producer/delivery_handle_spec.rb
|
245
248
|
- spec/rdkafka/producer/delivery_report_spec.rb
|
246
249
|
- spec/rdkafka/producer_spec.rb
|