karafka-rdkafka 0.16.0 → 0.16.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 78fe68d4a566b63ffa94633d2752cc95c535c020236749aa0452192d10cd2c43
4
- data.tar.gz: 61a9fe426703153762c94508a535a780b9ee1917cacf93ccbe37d1c4eb974d6c
3
+ metadata.gz: f8685868c1ad3f222639845933ba5aec660ec3923465c4cf91c3235ece03eec8
4
+ data.tar.gz: 96d7c33291d166418e696f05cce117b89861dd7fbbf53b587dbf1a90a3f4d2b2
5
5
  SHA512:
6
- metadata.gz: 46e2862b087c375042b93682d10378901d23b238f696cfd17048a880c3b28170243250e5cceecfd42c74c98791273df9577ed1371b300629a60727df8df2f9c4
7
- data.tar.gz: c76d0666e617aa0879ffa99c1888b7fad0ebbd1f1195f14285b130b82ba34c339c3f92c9718f0e8180d2d1cccd1e99f23ea765ea314fb0570713ce4ba23b8071
6
+ metadata.gz: 30de272e1cac622957ecc061426bf7314791528769dc6afc3f445f5fe6b3241a5b51a88931c97619fcdf1447bd0d6b7913c959ccb3b5d384e32d5ce89ef5d915
7
+ data.tar.gz: f912ec91dd3983a89807879b272f8bbcf777f705bd305aba77148998afcf2ba4372ae070d2510a19110a424ad9231177c64a25742b5de47b22c426769da5c858
checksums.yaml.gz.sig CHANGED
Binary file
data/CHANGELOG.md CHANGED
@@ -1,5 +1,9 @@
1
1
  # Rdkafka Changelog
2
2
 
3
+ ## 0.16.1 (2024-07-10)
4
+ - [Feature] Add `#seek_by` to be able to seek for a message by topic, partition and offset (zinahia)
5
+ - [Fix] Switch to local release of librdkafka to mitigate its unavailability.
6
+
3
7
  ## 0.16.0 (2024-06-17)
4
8
  - **[Breaking]** Messages without headers returned by `#poll` contain frozen empty hash.
5
9
  - **[Breaking]** `HashWithSymbolKeysTreatedLikeStrings` has been removed so headers are regular hashes with string keys.
data/README.md CHANGED
@@ -163,6 +163,7 @@ bundle exec rake produce_messages
163
163
 
164
164
  | rdkafka-ruby | librdkafka |
165
165
  |-|-|
166
+ | 0.17.0 (Unreleased) | 2.4.0 (2024-05-07) |
166
167
  | 0.16.0 (2024-06-13) | 2.3.0 (2023-10-25) |
167
168
  | 0.15.0 (2023-12-03) | 2.3.0 (2023-10-25) |
168
169
  | 0.14.0 (2023-11-21) | 2.2.0 (2023-07-12) |
Binary file
data/ext/Rakefile CHANGED
@@ -22,8 +22,10 @@ task :default => :clean do
22
22
  ENV["LDFLAGS"] = "-L#{homebrew_prefix}/lib" unless ENV["LDFLAGS"]
23
23
  end
24
24
 
25
+ releases = File.expand_path(File.join(File.dirname(__FILE__), '../dist'))
26
+
25
27
  recipe.files << {
26
- :url => "https://codeload.github.com/edenhill/librdkafka/tar.gz/v#{Rdkafka::LIBRDKAFKA_VERSION}",
28
+ :url => "file://#{releases}/librdkafka_#{Rdkafka::LIBRDKAFKA_VERSION}.tar.gz",
27
29
  :sha256 => Rdkafka::LIBRDKAFKA_SOURCE_SHA256
28
30
  }
29
31
  recipe.configure_options = ["--host=#{recipe.host}"]
@@ -435,6 +435,19 @@ module Rdkafka
435
435
  # @return [nil]
436
436
  # @raise [RdkafkaError] When seeking fails
437
437
  def seek(message)
438
+ seek_by(message.topic, message.partition, message.offset)
439
+ end
440
+
441
+ # Seek to a particular message by providing the topic, partition and offset.
442
+ # The next poll on the topic/partition will return the
443
+ # message at the given offset.
444
+ #
445
+ # @param topic [String] The topic in which to seek
446
+ # @param partition [Integer] The partition number to seek
447
+ # @param offset [Integer] The partition offset to seek
448
+ # @return [nil]
449
+ # @raise [RdkafkaError] When seeking fails
450
+ def seek_by(topic, partition, offset)
438
451
  closed_consumer_check(__method__)
439
452
 
440
453
  # rd_kafka_offset_store is one of the few calls that does not support
@@ -442,14 +455,14 @@ module Rdkafka
442
455
  native_topic = @native_kafka.with_inner do |inner|
443
456
  Rdkafka::Bindings.rd_kafka_topic_new(
444
457
  inner,
445
- message.topic,
458
+ topic,
446
459
  nil
447
460
  )
448
461
  end
449
462
  response = Rdkafka::Bindings.rd_kafka_seek(
450
463
  native_topic,
451
- message.partition,
452
- message.offset,
464
+ partition,
465
+ offset,
453
466
  0 # timeout
454
467
  )
455
468
  Rdkafka::RdkafkaError.validate!(response)
@@ -1,7 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Rdkafka
4
- VERSION = "0.16.0"
4
+ VERSION = "0.16.1"
5
5
  LIBRDKAFKA_VERSION = "2.4.0"
6
6
  LIBRDKAFKA_SOURCE_SHA256 = "d645e47d961db47f1ead29652606a502bdd2a880c85c1e060e94eea040f1a19a"
7
7
  end
@@ -258,6 +258,95 @@ describe Rdkafka::Consumer do
258
258
  end
259
259
  end
260
260
 
261
+ describe "#seek_by" do
262
+ let(:topic) { "consume_test_topic" }
263
+ let(:partition) { 0 }
264
+ let(:offset) { 0 }
265
+
266
+ it "should raise an error when seeking fails" do
267
+ expect(Rdkafka::Bindings).to receive(:rd_kafka_seek).and_return(20)
268
+ expect {
269
+ consumer.seek_by(topic, partition, offset)
270
+ }.to raise_error Rdkafka::RdkafkaError
271
+ end
272
+
273
+ context "subscription" do
274
+ let(:timeout) { 1000 }
275
+
276
+ before do
277
+ consumer.subscribe(topic)
278
+
279
+ # 1. partitions are assigned
280
+ wait_for_assignment(consumer)
281
+ expect(consumer.assignment).not_to be_empty
282
+
283
+ # 2. eat unrelated messages
284
+ while(consumer.poll(timeout)) do; end
285
+ end
286
+ after { consumer.unsubscribe }
287
+
288
+ def send_one_message(val)
289
+ producer.produce(
290
+ topic: topic,
291
+ payload: "payload #{val}",
292
+ key: "key 1",
293
+ partition: 0
294
+ ).wait
295
+ end
296
+
297
+ it "works when a partition is paused" do
298
+ # 3. get reference message
299
+ send_one_message(:a)
300
+ message1 = consumer.poll(timeout)
301
+ expect(message1&.payload).to eq "payload a"
302
+
303
+ # 4. pause the subscription
304
+ tpl = Rdkafka::Consumer::TopicPartitionList.new
305
+ tpl.add_topic(topic, 1)
306
+ consumer.pause(tpl)
307
+
308
+ # 5. seek by the previous message fields
309
+ consumer.seek_by(message1.topic, message1.partition, message1.offset)
310
+
311
+ # 6. resume the subscription
312
+ tpl = Rdkafka::Consumer::TopicPartitionList.new
313
+ tpl.add_topic(topic, 1)
314
+ consumer.resume(tpl)
315
+
316
+ # 7. ensure same message is read again
317
+ message2 = consumer.poll(timeout)
318
+
319
+ # This is needed because `enable.auto.offset.store` is true but when running in CI that
320
+ # is overloaded, offset store lags
321
+ sleep(2)
322
+
323
+ consumer.commit
324
+ expect(message1.offset).to eq message2.offset
325
+ expect(message1.payload).to eq message2.payload
326
+ end
327
+
328
+ it "allows skipping messages" do
329
+ # 3. send messages
330
+ send_one_message(:a)
331
+ send_one_message(:b)
332
+ send_one_message(:c)
333
+
334
+ # 4. get reference message
335
+ message = consumer.poll(timeout)
336
+ expect(message&.payload).to eq "payload a"
337
+
338
+ # 5. seek over one message
339
+ consumer.seek_by(message.topic, message.partition, message.offset + 2)
340
+
341
+ # 6. ensure that only one message is available
342
+ records = consumer.poll(timeout)
343
+ expect(records&.payload).to eq "payload c"
344
+ records = consumer.poll(timeout)
345
+ expect(records).to be_nil
346
+ end
347
+ end
348
+ end
349
+
261
350
  describe "#assign and #assignment" do
262
351
  it "should return an empty assignment if nothing is assigned" do
263
352
  expect(consumer.assignment).to be_empty
data.tar.gz.sig CHANGED
Binary file
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: karafka-rdkafka
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.16.0
4
+ version: 0.16.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Thijs Cadier
@@ -36,7 +36,7 @@ cert_chain:
36
36
  AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
37
37
  msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
38
38
  -----END CERTIFICATE-----
39
- date: 2024-06-17 00:00:00.000000000 Z
39
+ date: 2024-07-10 00:00:00.000000000 Z
40
40
  dependencies:
41
41
  - !ruby/object:Gem::Dependency
42
42
  name: ffi
@@ -186,6 +186,7 @@ files:
186
186
  - README.md
187
187
  - Rakefile
188
188
  - certs/cert_chain.pem
189
+ - dist/librdkafka_2.4.0.tar.gz
189
190
  - docker-compose.yml
190
191
  - ext/README.md
191
192
  - ext/Rakefile
metadata.gz.sig CHANGED
Binary file