karafka-rdkafka 0.16.0 → 0.16.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +4 -0
- data/README.md +1 -0
- data/dist/librdkafka_2.4.0.tar.gz +0 -0
- data/ext/Rakefile +3 -1
- data/lib/rdkafka/consumer.rb +16 -3
- data/lib/rdkafka/version.rb +1 -1
- data/spec/rdkafka/consumer_spec.rb +89 -0
- data.tar.gz.sig +0 -0
- metadata +3 -2
- metadata.gz.sig +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f8685868c1ad3f222639845933ba5aec660ec3923465c4cf91c3235ece03eec8
|
4
|
+
data.tar.gz: 96d7c33291d166418e696f05cce117b89861dd7fbbf53b587dbf1a90a3f4d2b2
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 30de272e1cac622957ecc061426bf7314791528769dc6afc3f445f5fe6b3241a5b51a88931c97619fcdf1447bd0d6b7913c959ccb3b5d384e32d5ce89ef5d915
|
7
|
+
data.tar.gz: f912ec91dd3983a89807879b272f8bbcf777f705bd305aba77148998afcf2ba4372ae070d2510a19110a424ad9231177c64a25742b5de47b22c426769da5c858
|
checksums.yaml.gz.sig
CHANGED
Binary file
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,9 @@
|
|
1
1
|
# Rdkafka Changelog
|
2
2
|
|
3
|
+
## 0.16.1 (2024-07-10)
|
4
|
+
- [Feature] Add `#seek_by` to be able to seek for a message by topic, partition and offset (zinahia)
|
5
|
+
- [Fix] Switch to local release of librdkafka to mitigate its unavailability.
|
6
|
+
|
3
7
|
## 0.16.0 (2024-06-17)
|
4
8
|
- **[Breaking]** Messages without headers returned by `#poll` contain frozen empty hash.
|
5
9
|
- **[Breaking]** `HashWithSymbolKeysTreatedLikeStrings` has been removed so headers are regular hashes with string keys.
|
data/README.md
CHANGED
@@ -163,6 +163,7 @@ bundle exec rake produce_messages
|
|
163
163
|
|
164
164
|
| rdkafka-ruby | librdkafka |
|
165
165
|
|-|-|
|
166
|
+
| 0.17.0 (Unreleased) | 2.4.0 (2024-05-07) |
|
166
167
|
| 0.16.0 (2024-06-13) | 2.3.0 (2023-10-25) |
|
167
168
|
| 0.15.0 (2023-12-03) | 2.3.0 (2023-10-25) |
|
168
169
|
| 0.14.0 (2023-11-21) | 2.2.0 (2023-07-12) |
|
Binary file
|
data/ext/Rakefile
CHANGED
@@ -22,8 +22,10 @@ task :default => :clean do
|
|
22
22
|
ENV["LDFLAGS"] = "-L#{homebrew_prefix}/lib" unless ENV["LDFLAGS"]
|
23
23
|
end
|
24
24
|
|
25
|
+
releases = File.expand_path(File.join(File.dirname(__FILE__), '../dist'))
|
26
|
+
|
25
27
|
recipe.files << {
|
26
|
-
:url => "
|
28
|
+
:url => "file://#{releases}/librdkafka_#{Rdkafka::LIBRDKAFKA_VERSION}.tar.gz",
|
27
29
|
:sha256 => Rdkafka::LIBRDKAFKA_SOURCE_SHA256
|
28
30
|
}
|
29
31
|
recipe.configure_options = ["--host=#{recipe.host}"]
|
data/lib/rdkafka/consumer.rb
CHANGED
@@ -435,6 +435,19 @@ module Rdkafka
|
|
435
435
|
# @return [nil]
|
436
436
|
# @raise [RdkafkaError] When seeking fails
|
437
437
|
def seek(message)
|
438
|
+
seek_by(message.topic, message.partition, message.offset)
|
439
|
+
end
|
440
|
+
|
441
|
+
# Seek to a particular message by providing the topic, partition and offset.
|
442
|
+
# The next poll on the topic/partition will return the
|
443
|
+
# message at the given offset.
|
444
|
+
#
|
445
|
+
# @param topic [String] The topic in which to seek
|
446
|
+
# @param partition [Integer] The partition number to seek
|
447
|
+
# @param offset [Integer] The partition offset to seek
|
448
|
+
# @return [nil]
|
449
|
+
# @raise [RdkafkaError] When seeking fails
|
450
|
+
def seek_by(topic, partition, offset)
|
438
451
|
closed_consumer_check(__method__)
|
439
452
|
|
440
453
|
# rd_kafka_offset_store is one of the few calls that does not support
|
@@ -442,14 +455,14 @@ module Rdkafka
|
|
442
455
|
native_topic = @native_kafka.with_inner do |inner|
|
443
456
|
Rdkafka::Bindings.rd_kafka_topic_new(
|
444
457
|
inner,
|
445
|
-
|
458
|
+
topic,
|
446
459
|
nil
|
447
460
|
)
|
448
461
|
end
|
449
462
|
response = Rdkafka::Bindings.rd_kafka_seek(
|
450
463
|
native_topic,
|
451
|
-
|
452
|
-
|
464
|
+
partition,
|
465
|
+
offset,
|
453
466
|
0 # timeout
|
454
467
|
)
|
455
468
|
Rdkafka::RdkafkaError.validate!(response)
|
data/lib/rdkafka/version.rb
CHANGED
@@ -258,6 +258,95 @@ describe Rdkafka::Consumer do
|
|
258
258
|
end
|
259
259
|
end
|
260
260
|
|
261
|
+
describe "#seek_by" do
|
262
|
+
let(:topic) { "consume_test_topic" }
|
263
|
+
let(:partition) { 0 }
|
264
|
+
let(:offset) { 0 }
|
265
|
+
|
266
|
+
it "should raise an error when seeking fails" do
|
267
|
+
expect(Rdkafka::Bindings).to receive(:rd_kafka_seek).and_return(20)
|
268
|
+
expect {
|
269
|
+
consumer.seek_by(topic, partition, offset)
|
270
|
+
}.to raise_error Rdkafka::RdkafkaError
|
271
|
+
end
|
272
|
+
|
273
|
+
context "subscription" do
|
274
|
+
let(:timeout) { 1000 }
|
275
|
+
|
276
|
+
before do
|
277
|
+
consumer.subscribe(topic)
|
278
|
+
|
279
|
+
# 1. partitions are assigned
|
280
|
+
wait_for_assignment(consumer)
|
281
|
+
expect(consumer.assignment).not_to be_empty
|
282
|
+
|
283
|
+
# 2. eat unrelated messages
|
284
|
+
while(consumer.poll(timeout)) do; end
|
285
|
+
end
|
286
|
+
after { consumer.unsubscribe }
|
287
|
+
|
288
|
+
def send_one_message(val)
|
289
|
+
producer.produce(
|
290
|
+
topic: topic,
|
291
|
+
payload: "payload #{val}",
|
292
|
+
key: "key 1",
|
293
|
+
partition: 0
|
294
|
+
).wait
|
295
|
+
end
|
296
|
+
|
297
|
+
it "works when a partition is paused" do
|
298
|
+
# 3. get reference message
|
299
|
+
send_one_message(:a)
|
300
|
+
message1 = consumer.poll(timeout)
|
301
|
+
expect(message1&.payload).to eq "payload a"
|
302
|
+
|
303
|
+
# 4. pause the subscription
|
304
|
+
tpl = Rdkafka::Consumer::TopicPartitionList.new
|
305
|
+
tpl.add_topic(topic, 1)
|
306
|
+
consumer.pause(tpl)
|
307
|
+
|
308
|
+
# 5. seek by the previous message fields
|
309
|
+
consumer.seek_by(message1.topic, message1.partition, message1.offset)
|
310
|
+
|
311
|
+
# 6. resume the subscription
|
312
|
+
tpl = Rdkafka::Consumer::TopicPartitionList.new
|
313
|
+
tpl.add_topic(topic, 1)
|
314
|
+
consumer.resume(tpl)
|
315
|
+
|
316
|
+
# 7. ensure same message is read again
|
317
|
+
message2 = consumer.poll(timeout)
|
318
|
+
|
319
|
+
# This is needed because `enable.auto.offset.store` is true but when running in CI that
|
320
|
+
# is overloaded, offset store lags
|
321
|
+
sleep(2)
|
322
|
+
|
323
|
+
consumer.commit
|
324
|
+
expect(message1.offset).to eq message2.offset
|
325
|
+
expect(message1.payload).to eq message2.payload
|
326
|
+
end
|
327
|
+
|
328
|
+
it "allows skipping messages" do
|
329
|
+
# 3. send messages
|
330
|
+
send_one_message(:a)
|
331
|
+
send_one_message(:b)
|
332
|
+
send_one_message(:c)
|
333
|
+
|
334
|
+
# 4. get reference message
|
335
|
+
message = consumer.poll(timeout)
|
336
|
+
expect(message&.payload).to eq "payload a"
|
337
|
+
|
338
|
+
# 5. seek over one message
|
339
|
+
consumer.seek_by(message.topic, message.partition, message.offset + 2)
|
340
|
+
|
341
|
+
# 6. ensure that only one message is available
|
342
|
+
records = consumer.poll(timeout)
|
343
|
+
expect(records&.payload).to eq "payload c"
|
344
|
+
records = consumer.poll(timeout)
|
345
|
+
expect(records).to be_nil
|
346
|
+
end
|
347
|
+
end
|
348
|
+
end
|
349
|
+
|
261
350
|
describe "#assign and #assignment" do
|
262
351
|
it "should return an empty assignment if nothing is assigned" do
|
263
352
|
expect(consumer.assignment).to be_empty
|
data.tar.gz.sig
CHANGED
Binary file
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: karafka-rdkafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.16.
|
4
|
+
version: 0.16.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Thijs Cadier
|
@@ -36,7 +36,7 @@ cert_chain:
|
|
36
36
|
AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
|
37
37
|
msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
|
38
38
|
-----END CERTIFICATE-----
|
39
|
-
date: 2024-
|
39
|
+
date: 2024-07-10 00:00:00.000000000 Z
|
40
40
|
dependencies:
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: ffi
|
@@ -186,6 +186,7 @@ files:
|
|
186
186
|
- README.md
|
187
187
|
- Rakefile
|
188
188
|
- certs/cert_chain.pem
|
189
|
+
- dist/librdkafka_2.4.0.tar.gz
|
189
190
|
- docker-compose.yml
|
190
191
|
- ext/README.md
|
191
192
|
- ext/Rakefile
|
metadata.gz.sig
CHANGED
Binary file
|