karafka-rdkafka 0.16.0 → 0.16.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +4 -0
- data/README.md +1 -0
- data/dist/librdkafka_2.4.0.tar.gz +0 -0
- data/ext/Rakefile +3 -1
- data/lib/rdkafka/consumer.rb +16 -3
- data/lib/rdkafka/version.rb +1 -1
- data/spec/rdkafka/consumer_spec.rb +89 -0
- data.tar.gz.sig +0 -0
- metadata +3 -2
- metadata.gz.sig +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f8685868c1ad3f222639845933ba5aec660ec3923465c4cf91c3235ece03eec8
|
4
|
+
data.tar.gz: 96d7c33291d166418e696f05cce117b89861dd7fbbf53b587dbf1a90a3f4d2b2
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 30de272e1cac622957ecc061426bf7314791528769dc6afc3f445f5fe6b3241a5b51a88931c97619fcdf1447bd0d6b7913c959ccb3b5d384e32d5ce89ef5d915
|
7
|
+
data.tar.gz: f912ec91dd3983a89807879b272f8bbcf777f705bd305aba77148998afcf2ba4372ae070d2510a19110a424ad9231177c64a25742b5de47b22c426769da5c858
|
checksums.yaml.gz.sig
CHANGED
Binary file
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,9 @@
|
|
1
1
|
# Rdkafka Changelog
|
2
2
|
|
3
|
+
## 0.16.1 (2024-07-10)
|
4
|
+
- [Feature] Add `#seek_by` to be able to seek for a message by topic, partition and offset (zinahia)
|
5
|
+
- [Fix] Switch to local release of librdkafka to mitigate its unavailability.
|
6
|
+
|
3
7
|
## 0.16.0 (2024-06-17)
|
4
8
|
- **[Breaking]** Messages without headers returned by `#poll` contain frozen empty hash.
|
5
9
|
- **[Breaking]** `HashWithSymbolKeysTreatedLikeStrings` has been removed so headers are regular hashes with string keys.
|
data/README.md
CHANGED
@@ -163,6 +163,7 @@ bundle exec rake produce_messages
|
|
163
163
|
|
164
164
|
| rdkafka-ruby | librdkafka |
|
165
165
|
|-|-|
|
166
|
+
| 0.17.0 (Unreleased) | 2.4.0 (2024-05-07) |
|
166
167
|
| 0.16.0 (2024-06-13) | 2.3.0 (2023-10-25) |
|
167
168
|
| 0.15.0 (2023-12-03) | 2.3.0 (2023-10-25) |
|
168
169
|
| 0.14.0 (2023-11-21) | 2.2.0 (2023-07-12) |
|
Binary file
|
data/ext/Rakefile
CHANGED
@@ -22,8 +22,10 @@ task :default => :clean do
|
|
22
22
|
ENV["LDFLAGS"] = "-L#{homebrew_prefix}/lib" unless ENV["LDFLAGS"]
|
23
23
|
end
|
24
24
|
|
25
|
+
releases = File.expand_path(File.join(File.dirname(__FILE__), '../dist'))
|
26
|
+
|
25
27
|
recipe.files << {
|
26
|
-
:url => "
|
28
|
+
:url => "file://#{releases}/librdkafka_#{Rdkafka::LIBRDKAFKA_VERSION}.tar.gz",
|
27
29
|
:sha256 => Rdkafka::LIBRDKAFKA_SOURCE_SHA256
|
28
30
|
}
|
29
31
|
recipe.configure_options = ["--host=#{recipe.host}"]
|
data/lib/rdkafka/consumer.rb
CHANGED
@@ -435,6 +435,19 @@ module Rdkafka
|
|
435
435
|
# @return [nil]
|
436
436
|
# @raise [RdkafkaError] When seeking fails
|
437
437
|
def seek(message)
|
438
|
+
seek_by(message.topic, message.partition, message.offset)
|
439
|
+
end
|
440
|
+
|
441
|
+
# Seek to a particular message by providing the topic, partition and offset.
|
442
|
+
# The next poll on the topic/partition will return the
|
443
|
+
# message at the given offset.
|
444
|
+
#
|
445
|
+
# @param topic [String] The topic in which to seek
|
446
|
+
# @param partition [Integer] The partition number to seek
|
447
|
+
# @param offset [Integer] The partition offset to seek
|
448
|
+
# @return [nil]
|
449
|
+
# @raise [RdkafkaError] When seeking fails
|
450
|
+
def seek_by(topic, partition, offset)
|
438
451
|
closed_consumer_check(__method__)
|
439
452
|
|
440
453
|
# rd_kafka_offset_store is one of the few calls that does not support
|
@@ -442,14 +455,14 @@ module Rdkafka
|
|
442
455
|
native_topic = @native_kafka.with_inner do |inner|
|
443
456
|
Rdkafka::Bindings.rd_kafka_topic_new(
|
444
457
|
inner,
|
445
|
-
|
458
|
+
topic,
|
446
459
|
nil
|
447
460
|
)
|
448
461
|
end
|
449
462
|
response = Rdkafka::Bindings.rd_kafka_seek(
|
450
463
|
native_topic,
|
451
|
-
|
452
|
-
|
464
|
+
partition,
|
465
|
+
offset,
|
453
466
|
0 # timeout
|
454
467
|
)
|
455
468
|
Rdkafka::RdkafkaError.validate!(response)
|
data/lib/rdkafka/version.rb
CHANGED
@@ -258,6 +258,95 @@ describe Rdkafka::Consumer do
|
|
258
258
|
end
|
259
259
|
end
|
260
260
|
|
261
|
+
describe "#seek_by" do
|
262
|
+
let(:topic) { "consume_test_topic" }
|
263
|
+
let(:partition) { 0 }
|
264
|
+
let(:offset) { 0 }
|
265
|
+
|
266
|
+
it "should raise an error when seeking fails" do
|
267
|
+
expect(Rdkafka::Bindings).to receive(:rd_kafka_seek).and_return(20)
|
268
|
+
expect {
|
269
|
+
consumer.seek_by(topic, partition, offset)
|
270
|
+
}.to raise_error Rdkafka::RdkafkaError
|
271
|
+
end
|
272
|
+
|
273
|
+
context "subscription" do
|
274
|
+
let(:timeout) { 1000 }
|
275
|
+
|
276
|
+
before do
|
277
|
+
consumer.subscribe(topic)
|
278
|
+
|
279
|
+
# 1. partitions are assigned
|
280
|
+
wait_for_assignment(consumer)
|
281
|
+
expect(consumer.assignment).not_to be_empty
|
282
|
+
|
283
|
+
# 2. eat unrelated messages
|
284
|
+
while(consumer.poll(timeout)) do; end
|
285
|
+
end
|
286
|
+
after { consumer.unsubscribe }
|
287
|
+
|
288
|
+
def send_one_message(val)
|
289
|
+
producer.produce(
|
290
|
+
topic: topic,
|
291
|
+
payload: "payload #{val}",
|
292
|
+
key: "key 1",
|
293
|
+
partition: 0
|
294
|
+
).wait
|
295
|
+
end
|
296
|
+
|
297
|
+
it "works when a partition is paused" do
|
298
|
+
# 3. get reference message
|
299
|
+
send_one_message(:a)
|
300
|
+
message1 = consumer.poll(timeout)
|
301
|
+
expect(message1&.payload).to eq "payload a"
|
302
|
+
|
303
|
+
# 4. pause the subscription
|
304
|
+
tpl = Rdkafka::Consumer::TopicPartitionList.new
|
305
|
+
tpl.add_topic(topic, 1)
|
306
|
+
consumer.pause(tpl)
|
307
|
+
|
308
|
+
# 5. seek by the previous message fields
|
309
|
+
consumer.seek_by(message1.topic, message1.partition, message1.offset)
|
310
|
+
|
311
|
+
# 6. resume the subscription
|
312
|
+
tpl = Rdkafka::Consumer::TopicPartitionList.new
|
313
|
+
tpl.add_topic(topic, 1)
|
314
|
+
consumer.resume(tpl)
|
315
|
+
|
316
|
+
# 7. ensure same message is read again
|
317
|
+
message2 = consumer.poll(timeout)
|
318
|
+
|
319
|
+
# This is needed because `enable.auto.offset.store` is true but when running in CI that
|
320
|
+
# is overloaded, offset store lags
|
321
|
+
sleep(2)
|
322
|
+
|
323
|
+
consumer.commit
|
324
|
+
expect(message1.offset).to eq message2.offset
|
325
|
+
expect(message1.payload).to eq message2.payload
|
326
|
+
end
|
327
|
+
|
328
|
+
it "allows skipping messages" do
|
329
|
+
# 3. send messages
|
330
|
+
send_one_message(:a)
|
331
|
+
send_one_message(:b)
|
332
|
+
send_one_message(:c)
|
333
|
+
|
334
|
+
# 4. get reference message
|
335
|
+
message = consumer.poll(timeout)
|
336
|
+
expect(message&.payload).to eq "payload a"
|
337
|
+
|
338
|
+
# 5. seek over one message
|
339
|
+
consumer.seek_by(message.topic, message.partition, message.offset + 2)
|
340
|
+
|
341
|
+
# 6. ensure that only one message is available
|
342
|
+
records = consumer.poll(timeout)
|
343
|
+
expect(records&.payload).to eq "payload c"
|
344
|
+
records = consumer.poll(timeout)
|
345
|
+
expect(records).to be_nil
|
346
|
+
end
|
347
|
+
end
|
348
|
+
end
|
349
|
+
|
261
350
|
describe "#assign and #assignment" do
|
262
351
|
it "should return an empty assignment if nothing is assigned" do
|
263
352
|
expect(consumer.assignment).to be_empty
|
data.tar.gz.sig
CHANGED
Binary file
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: karafka-rdkafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.16.
|
4
|
+
version: 0.16.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Thijs Cadier
|
@@ -36,7 +36,7 @@ cert_chain:
|
|
36
36
|
AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
|
37
37
|
msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
|
38
38
|
-----END CERTIFICATE-----
|
39
|
-
date: 2024-
|
39
|
+
date: 2024-07-10 00:00:00.000000000 Z
|
40
40
|
dependencies:
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: ffi
|
@@ -186,6 +186,7 @@ files:
|
|
186
186
|
- README.md
|
187
187
|
- Rakefile
|
188
188
|
- certs/cert_chain.pem
|
189
|
+
- dist/librdkafka_2.4.0.tar.gz
|
189
190
|
- docker-compose.yml
|
190
191
|
- ext/README.md
|
191
192
|
- ext/Rakefile
|
metadata.gz.sig
CHANGED
Binary file
|