rdkafka 0.12.0 → 0.13.0.beta.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. checksums.yaml +4 -4
  2. data/.semaphore/semaphore.yml +7 -3
  3. data/CHANGELOG.md +18 -0
  4. data/Gemfile +2 -0
  5. data/README.md +26 -0
  6. data/Rakefile +2 -0
  7. data/ext/Rakefile +2 -0
  8. data/lib/rdkafka/abstract_handle.rb +2 -0
  9. data/lib/rdkafka/admin/create_topic_handle.rb +2 -0
  10. data/lib/rdkafka/admin/create_topic_report.rb +2 -0
  11. data/lib/rdkafka/admin/delete_topic_handle.rb +2 -0
  12. data/lib/rdkafka/admin/delete_topic_report.rb +2 -0
  13. data/lib/rdkafka/admin.rb +48 -31
  14. data/lib/rdkafka/bindings.rb +50 -37
  15. data/lib/rdkafka/callbacks.rb +7 -1
  16. data/lib/rdkafka/config.rb +13 -10
  17. data/lib/rdkafka/consumer/headers.rb +24 -7
  18. data/lib/rdkafka/consumer/message.rb +3 -1
  19. data/lib/rdkafka/consumer/partition.rb +2 -0
  20. data/lib/rdkafka/consumer/topic_partition_list.rb +2 -0
  21. data/lib/rdkafka/consumer.rb +84 -44
  22. data/lib/rdkafka/error.rb +9 -0
  23. data/lib/rdkafka/metadata.rb +4 -2
  24. data/lib/rdkafka/native_kafka.rb +83 -0
  25. data/lib/rdkafka/producer/delivery_handle.rb +5 -2
  26. data/lib/rdkafka/producer/delivery_report.rb +9 -2
  27. data/lib/rdkafka/producer.rb +35 -13
  28. data/lib/rdkafka/version.rb +5 -3
  29. data/lib/rdkafka.rb +3 -1
  30. data/rdkafka.gemspec +2 -0
  31. data/spec/rdkafka/abstract_handle_spec.rb +2 -0
  32. data/spec/rdkafka/admin/create_topic_handle_spec.rb +2 -0
  33. data/spec/rdkafka/admin/create_topic_report_spec.rb +2 -0
  34. data/spec/rdkafka/admin/delete_topic_handle_spec.rb +2 -0
  35. data/spec/rdkafka/admin/delete_topic_report_spec.rb +2 -0
  36. data/spec/rdkafka/admin_spec.rb +4 -3
  37. data/spec/rdkafka/bindings_spec.rb +2 -0
  38. data/spec/rdkafka/callbacks_spec.rb +2 -0
  39. data/spec/rdkafka/config_spec.rb +17 -2
  40. data/spec/rdkafka/consumer/headers_spec.rb +62 -0
  41. data/spec/rdkafka/consumer/message_spec.rb +2 -0
  42. data/spec/rdkafka/consumer/partition_spec.rb +2 -0
  43. data/spec/rdkafka/consumer/topic_partition_list_spec.rb +2 -0
  44. data/spec/rdkafka/consumer_spec.rb +97 -22
  45. data/spec/rdkafka/error_spec.rb +2 -0
  46. data/spec/rdkafka/metadata_spec.rb +2 -0
  47. data/spec/rdkafka/{producer/client_spec.rb → native_kafka_spec.rb} +13 -34
  48. data/spec/rdkafka/producer/delivery_handle_spec.rb +5 -0
  49. data/spec/rdkafka/producer/delivery_report_spec.rb +8 -2
  50. data/spec/rdkafka/producer_spec.rb +51 -19
  51. data/spec/spec_helper.rb +17 -1
  52. metadata +14 -14
  53. data/bin/console +0 -11
  54. data/lib/rdkafka/producer/client.rb +0 -47
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require "spec_helper"
2
4
  require "zlib"
3
5
 
@@ -7,7 +9,8 @@ describe Rdkafka::Producer do
7
9
 
8
10
  after do
9
11
  # Registry should always end up being empty
10
- expect(Rdkafka::Producer::DeliveryHandle::REGISTRY).to be_empty
12
+ registry = Rdkafka::Producer::DeliveryHandle::REGISTRY
13
+ expect(registry).to be_empty, registry.inspect
11
14
  producer.close
12
15
  consumer.close
13
16
  end
@@ -30,6 +33,7 @@ describe Rdkafka::Producer do
30
33
  expect(report).not_to be_nil
31
34
  expect(report.partition).to eq 1
32
35
  expect(report.offset).to be >= 0
36
+ expect(report.topic_name).to eq "produce_test_topic"
33
37
  @callback_called = true
34
38
  end
35
39
 
@@ -113,6 +117,7 @@ describe Rdkafka::Producer do
113
117
  expect(called_report.first).not_to be_nil
114
118
  expect(called_report.first.partition).to eq 1
115
119
  expect(called_report.first.offset).to be >= 0
120
+ expect(called_report.first.topic_name).to eq "produce_test_topic"
116
121
  end
117
122
 
118
123
  it "should provide handle" do
@@ -180,10 +185,11 @@ describe Rdkafka::Producer do
180
185
  expect(report.partition).to eq 1
181
186
  expect(report.offset).to be >= 0
182
187
 
183
- # Close producer
188
+ # Flush and close producer
189
+ producer.flush
184
190
  producer.close
185
191
 
186
- # Consume message and verify it's content
192
+ # Consume message and verify its content
187
193
  message = wait_for_message(
188
194
  topic: "produce_test_topic",
189
195
  delivery_report: report,
@@ -207,7 +213,7 @@ describe Rdkafka::Producer do
207
213
  )
208
214
  report = handle.wait(max_wait_timeout: 5)
209
215
 
210
- # Consume message and verify it's content
216
+ # Consume message and verify its content
211
217
  message = wait_for_message(
212
218
  topic: "produce_test_topic",
213
219
  delivery_report: report,
@@ -251,6 +257,28 @@ describe Rdkafka::Producer do
251
257
  expect(messages[2].key).to eq key
252
258
  end
253
259
 
260
+ it "should produce a message with empty string without crashing" do
261
+ messages = [{key: 'a', partition_key: ''}]
262
+
263
+ messages = messages.map do |m|
264
+ handle = producer.produce(
265
+ topic: "partitioner_test_topic",
266
+ payload: "payload partition",
267
+ key: m[:key],
268
+ partition_key: m[:partition_key]
269
+ )
270
+ report = handle.wait(max_wait_timeout: 5)
271
+
272
+ wait_for_message(
273
+ topic: "partitioner_test_topic",
274
+ delivery_report: report,
275
+ )
276
+ end
277
+
278
+ expect(messages[0].partition).to eq 0
279
+ expect(messages[0].key).to eq 'a'
280
+ end
281
+
254
282
  it "should produce a message with utf-8 encoding" do
255
283
  handle = producer.produce(
256
284
  topic: "produce_test_topic",
@@ -259,7 +287,7 @@ describe Rdkafka::Producer do
259
287
  )
260
288
  report = handle.wait(max_wait_timeout: 5)
261
289
 
262
- # Consume message and verify it's content
290
+ # Consume message and verify its content
263
291
  message = wait_for_message(
264
292
  topic: "produce_test_topic",
265
293
  delivery_report: report,
@@ -292,7 +320,7 @@ describe Rdkafka::Producer do
292
320
  )
293
321
  report = handle.wait(max_wait_timeout: 5)
294
322
 
295
- # Consume message and verify it's content
323
+ # Consume message and verify its content
296
324
  message = wait_for_message(
297
325
  topic: "produce_test_topic",
298
326
  delivery_report: report,
@@ -313,7 +341,7 @@ describe Rdkafka::Producer do
313
341
  )
314
342
  report = handle.wait(max_wait_timeout: 5)
315
343
 
316
- # Consume message and verify it's content
344
+ # Consume message and verify its content
317
345
  message = wait_for_message(
318
346
  topic: "produce_test_topic",
319
347
  delivery_report: report,
@@ -333,7 +361,7 @@ describe Rdkafka::Producer do
333
361
  )
334
362
  report = handle.wait(max_wait_timeout: 5)
335
363
 
336
- # Consume message and verify it's content
364
+ # Consume message and verify its content
337
365
  message = wait_for_message(
338
366
  topic: "produce_test_topic",
339
367
  delivery_report: report,
@@ -351,7 +379,7 @@ describe Rdkafka::Producer do
351
379
  )
352
380
  report = handle.wait(max_wait_timeout: 5)
353
381
 
354
- # Consume message and verify it's content
382
+ # Consume message and verify its content
355
383
  message = wait_for_message(
356
384
  topic: "produce_test_topic",
357
385
  delivery_report: report,
@@ -371,7 +399,7 @@ describe Rdkafka::Producer do
371
399
  )
372
400
  report = handle.wait(max_wait_timeout: 5)
373
401
 
374
- # Consume message and verify it's content
402
+ # Consume message and verify its content
375
403
  message = wait_for_message(
376
404
  topic: "produce_test_topic",
377
405
  delivery_report: report,
@@ -380,9 +408,9 @@ describe Rdkafka::Producer do
380
408
 
381
409
  expect(message.payload).to eq "payload headers"
382
410
  expect(message.key).to eq "key headers"
383
- expect(message.headers[:foo]).to eq "bar"
384
- expect(message.headers[:baz]).to eq "foobar"
385
- expect(message.headers[:foobar]).to be_nil
411
+ expect(message.headers["foo"]).to eq "bar"
412
+ expect(message.headers["baz"]).to eq "foobar"
413
+ expect(message.headers["foobar"]).to be_nil
386
414
  end
387
415
 
388
416
  it "should produce a message with empty headers" do
@@ -394,7 +422,7 @@ describe Rdkafka::Producer do
394
422
  )
395
423
  report = handle.wait(max_wait_timeout: 5)
396
424
 
397
- # Consume message and verify it's content
425
+ # Consume message and verify its content
398
426
  message = wait_for_message(
399
427
  topic: "produce_test_topic",
400
428
  delivery_report: report,
@@ -432,10 +460,10 @@ describe Rdkafka::Producer do
432
460
  # wait for and check the message in the main process.
433
461
  reader, writer = IO.pipe
434
462
 
435
- fork do
463
+ pid = fork do
436
464
  reader.close
437
465
 
438
- # Avoids sharing the socket between processes.
466
+ # Avoid sharing the client between processes.
439
467
  producer = rdkafka_producer_config.producer
440
468
 
441
469
  handle = producer.produce(
@@ -448,24 +476,28 @@ describe Rdkafka::Producer do
448
476
 
449
477
  report_json = JSON.generate(
450
478
  "partition" => report.partition,
451
- "offset" => report.offset
479
+ "offset" => report.offset,
480
+ "topic_name" => report.topic_name
452
481
  )
453
482
 
454
483
  writer.write(report_json)
455
484
  writer.close
485
+ producer.flush
456
486
  producer.close
457
487
  end
488
+ Process.wait(pid)
458
489
 
459
490
  writer.close
460
491
  report_hash = JSON.parse(reader.read)
461
492
  report = Rdkafka::Producer::DeliveryReport.new(
462
493
  report_hash["partition"],
463
- report_hash["offset"]
494
+ report_hash["offset"],
495
+ report_hash["topic_name"]
464
496
  )
465
497
 
466
498
  reader.close
467
499
 
468
- # Consume message and verify it's content
500
+ # Consume message and verify its content
469
501
  message = wait_for_message(
470
502
  topic: "produce_test_topic",
471
503
  delivery_report: report,
data/spec/spec_helper.rb CHANGED
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  unless ENV["CI"] == "true"
2
4
  require "simplecov"
3
5
  SimpleCov.start do
@@ -71,7 +73,7 @@ def new_native_topic(topic_name="topic_name", native_client: )
71
73
  end
72
74
 
73
75
  def wait_for_message(topic:, delivery_report:, timeout_in_seconds: 30, consumer: nil)
74
- new_consumer = !!consumer
76
+ new_consumer = consumer.nil?
75
77
  consumer ||= rdkafka_consumer_config.consumer
76
78
  consumer.subscribe(topic)
77
79
  timeout = Time.now.to_i + timeout_in_seconds
@@ -104,6 +106,20 @@ def wait_for_unassignment(consumer)
104
106
  end
105
107
  end
106
108
 
109
+ def notify_listener(listener, &block)
110
+ # 1. subscribe and poll
111
+ consumer.subscribe("consume_test_topic")
112
+ wait_for_assignment(consumer)
113
+ consumer.poll(100)
114
+
115
+ block.call if block
116
+
117
+ # 2. unsubscribe
118
+ consumer.unsubscribe
119
+ wait_for_unassignment(consumer)
120
+ consumer.close
121
+ end
122
+
107
123
  RSpec.configure do |config|
108
124
  config.filter_run focus: true
109
125
  config.run_all_when_everything_filtered = true
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: rdkafka
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.12.0
4
+ version: 0.13.0.beta.7
5
5
  platform: ruby
6
6
  authors:
7
7
  - Thijs Cadier
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2022-06-17 00:00:00.000000000 Z
11
+ date: 2023-06-12 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: ffi
@@ -139,8 +139,7 @@ dependencies:
139
139
  description: Modern Kafka client library for Ruby based on librdkafka
140
140
  email:
141
141
  - thijs@appsignal.com
142
- executables:
143
- - console
142
+ executables: []
144
143
  extensions:
145
144
  - ext/Rakefile
146
145
  extra_rdoc_files: []
@@ -155,7 +154,6 @@ files:
155
154
  - LICENSE
156
155
  - README.md
157
156
  - Rakefile
158
- - bin/console
159
157
  - docker-compose.yml
160
158
  - ext/README.md
161
159
  - ext/Rakefile
@@ -176,8 +174,8 @@ files:
176
174
  - lib/rdkafka/consumer/topic_partition_list.rb
177
175
  - lib/rdkafka/error.rb
178
176
  - lib/rdkafka/metadata.rb
177
+ - lib/rdkafka/native_kafka.rb
179
178
  - lib/rdkafka/producer.rb
180
- - lib/rdkafka/producer/client.rb
181
179
  - lib/rdkafka/producer/delivery_handle.rb
182
180
  - lib/rdkafka/producer/delivery_report.rb
183
181
  - lib/rdkafka/version.rb
@@ -191,13 +189,14 @@ files:
191
189
  - spec/rdkafka/bindings_spec.rb
192
190
  - spec/rdkafka/callbacks_spec.rb
193
191
  - spec/rdkafka/config_spec.rb
192
+ - spec/rdkafka/consumer/headers_spec.rb
194
193
  - spec/rdkafka/consumer/message_spec.rb
195
194
  - spec/rdkafka/consumer/partition_spec.rb
196
195
  - spec/rdkafka/consumer/topic_partition_list_spec.rb
197
196
  - spec/rdkafka/consumer_spec.rb
198
197
  - spec/rdkafka/error_spec.rb
199
198
  - spec/rdkafka/metadata_spec.rb
200
- - spec/rdkafka/producer/client_spec.rb
199
+ - spec/rdkafka/native_kafka_spec.rb
201
200
  - spec/rdkafka/producer/delivery_handle_spec.rb
202
201
  - spec/rdkafka/producer/delivery_report_spec.rb
203
202
  - spec/rdkafka/producer_spec.rb
@@ -206,7 +205,7 @@ homepage: https://github.com/thijsc/rdkafka-ruby
206
205
  licenses:
207
206
  - MIT
208
207
  metadata: {}
209
- post_install_message:
208
+ post_install_message:
210
209
  rdoc_options: []
211
210
  require_paths:
212
211
  - lib
@@ -217,12 +216,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
217
216
  version: '2.6'
218
217
  required_rubygems_version: !ruby/object:Gem::Requirement
219
218
  requirements:
220
- - - ">="
219
+ - - ">"
221
220
  - !ruby/object:Gem::Version
222
- version: '0'
221
+ version: 1.3.1
223
222
  requirements: []
224
- rubygems_version: 3.0.3
225
- signing_key:
223
+ rubygems_version: 3.4.1
224
+ signing_key:
226
225
  specification_version: 4
227
226
  summary: The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka.
228
227
  It wraps the production-ready C client using the ffi gem and targets Kafka 1.0+
@@ -237,13 +236,14 @@ test_files:
237
236
  - spec/rdkafka/bindings_spec.rb
238
237
  - spec/rdkafka/callbacks_spec.rb
239
238
  - spec/rdkafka/config_spec.rb
239
+ - spec/rdkafka/consumer/headers_spec.rb
240
240
  - spec/rdkafka/consumer/message_spec.rb
241
241
  - spec/rdkafka/consumer/partition_spec.rb
242
242
  - spec/rdkafka/consumer/topic_partition_list_spec.rb
243
243
  - spec/rdkafka/consumer_spec.rb
244
244
  - spec/rdkafka/error_spec.rb
245
245
  - spec/rdkafka/metadata_spec.rb
246
- - spec/rdkafka/producer/client_spec.rb
246
+ - spec/rdkafka/native_kafka_spec.rb
247
247
  - spec/rdkafka/producer/delivery_handle_spec.rb
248
248
  - spec/rdkafka/producer/delivery_report_spec.rb
249
249
  - spec/rdkafka/producer_spec.rb
data/bin/console DELETED
@@ -1,11 +0,0 @@
1
- #!/usr/bin/env ruby
2
-
3
- # frozen_string_literal: true
4
-
5
- ENV["IRBRC"] = File.join(File.dirname(__FILE__), ".irbrc")
6
-
7
- require "bundler/setup"
8
- require "rdkafka"
9
-
10
- require "irb"
11
- IRB.start(__FILE__)
@@ -1,47 +0,0 @@
1
- module Rdkafka
2
- class Producer
3
- class Client
4
- def initialize(native)
5
- @native = native
6
-
7
- # Start thread to poll client for delivery callbacks
8
- @polling_thread = Thread.new do
9
- loop do
10
- Rdkafka::Bindings.rd_kafka_poll(native, 250)
11
- # Exit thread if closing and the poll queue is empty
12
- if Thread.current[:closing] && Rdkafka::Bindings.rd_kafka_outq_len(native) == 0
13
- break
14
- end
15
- end
16
- end
17
- @polling_thread.abort_on_exception = true
18
- @polling_thread[:closing] = false
19
- end
20
-
21
- def native
22
- @native
23
- end
24
-
25
- def finalizer
26
- ->(_) { close }
27
- end
28
-
29
- def closed?
30
- @native.nil?
31
- end
32
-
33
- def close(object_id=nil)
34
- return unless @native
35
-
36
- # Indicate to polling thread that we're closing
37
- @polling_thread[:closing] = true
38
- # Wait for the polling thread to finish up
39
- @polling_thread.join
40
-
41
- Rdkafka::Bindings.rd_kafka_destroy(@native)
42
-
43
- @native = nil
44
- end
45
- end
46
- end
47
- end