rdkafka 0.12.0 → 0.13.0.beta.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.semaphore/semaphore.yml +7 -3
- data/CHANGELOG.md +18 -0
- data/Gemfile +2 -0
- data/README.md +26 -0
- data/Rakefile +2 -0
- data/ext/Rakefile +2 -0
- data/lib/rdkafka/abstract_handle.rb +2 -0
- data/lib/rdkafka/admin/create_topic_handle.rb +2 -0
- data/lib/rdkafka/admin/create_topic_report.rb +2 -0
- data/lib/rdkafka/admin/delete_topic_handle.rb +2 -0
- data/lib/rdkafka/admin/delete_topic_report.rb +2 -0
- data/lib/rdkafka/admin.rb +48 -31
- data/lib/rdkafka/bindings.rb +50 -37
- data/lib/rdkafka/callbacks.rb +7 -1
- data/lib/rdkafka/config.rb +13 -10
- data/lib/rdkafka/consumer/headers.rb +24 -7
- data/lib/rdkafka/consumer/message.rb +3 -1
- data/lib/rdkafka/consumer/partition.rb +2 -0
- data/lib/rdkafka/consumer/topic_partition_list.rb +2 -0
- data/lib/rdkafka/consumer.rb +84 -44
- data/lib/rdkafka/error.rb +9 -0
- data/lib/rdkafka/metadata.rb +4 -2
- data/lib/rdkafka/native_kafka.rb +83 -0
- data/lib/rdkafka/producer/delivery_handle.rb +5 -2
- data/lib/rdkafka/producer/delivery_report.rb +9 -2
- data/lib/rdkafka/producer.rb +35 -13
- data/lib/rdkafka/version.rb +5 -3
- data/lib/rdkafka.rb +3 -1
- data/rdkafka.gemspec +2 -0
- data/spec/rdkafka/abstract_handle_spec.rb +2 -0
- data/spec/rdkafka/admin/create_topic_handle_spec.rb +2 -0
- data/spec/rdkafka/admin/create_topic_report_spec.rb +2 -0
- data/spec/rdkafka/admin/delete_topic_handle_spec.rb +2 -0
- data/spec/rdkafka/admin/delete_topic_report_spec.rb +2 -0
- data/spec/rdkafka/admin_spec.rb +4 -3
- data/spec/rdkafka/bindings_spec.rb +2 -0
- data/spec/rdkafka/callbacks_spec.rb +2 -0
- data/spec/rdkafka/config_spec.rb +17 -2
- data/spec/rdkafka/consumer/headers_spec.rb +62 -0
- data/spec/rdkafka/consumer/message_spec.rb +2 -0
- data/spec/rdkafka/consumer/partition_spec.rb +2 -0
- data/spec/rdkafka/consumer/topic_partition_list_spec.rb +2 -0
- data/spec/rdkafka/consumer_spec.rb +97 -22
- data/spec/rdkafka/error_spec.rb +2 -0
- data/spec/rdkafka/metadata_spec.rb +2 -0
- data/spec/rdkafka/{producer/client_spec.rb → native_kafka_spec.rb} +13 -34
- data/spec/rdkafka/producer/delivery_handle_spec.rb +5 -0
- data/spec/rdkafka/producer/delivery_report_spec.rb +8 -2
- data/spec/rdkafka/producer_spec.rb +51 -19
- data/spec/spec_helper.rb +17 -1
- metadata +14 -14
- data/bin/console +0 -11
- data/lib/rdkafka/producer/client.rb +0 -47
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Rdkafka
|
2
4
|
class Consumer
|
3
5
|
# A message that was consumed from a topic.
|
@@ -18,7 +20,7 @@ module Rdkafka
|
|
18
20
|
# @return [String, nil]
|
19
21
|
attr_reader :key
|
20
22
|
|
21
|
-
# This message's offset in
|
23
|
+
# This message's offset in its partition
|
22
24
|
# @return [Integer]
|
23
25
|
attr_reader :offset
|
24
26
|
|
data/lib/rdkafka/consumer.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Rdkafka
|
2
4
|
# A consumer of Kafka messages. It uses the high-level consumer approach where the Kafka
|
3
5
|
# brokers automatically assign partitions and load balance partitions over consumers that
|
@@ -14,18 +16,26 @@ module Rdkafka
|
|
14
16
|
# @private
|
15
17
|
def initialize(native_kafka)
|
16
18
|
@native_kafka = native_kafka
|
17
|
-
|
19
|
+
end
|
20
|
+
|
21
|
+
def finalizer
|
22
|
+
->(_) { close }
|
18
23
|
end
|
19
24
|
|
20
25
|
# Close this consumer
|
21
26
|
# @return [nil]
|
22
27
|
def close
|
23
|
-
return
|
28
|
+
return if closed?
|
29
|
+
ObjectSpace.undefine_finalizer(self)
|
30
|
+
@native_kafka.with_inner do |inner|
|
31
|
+
Rdkafka::Bindings.rd_kafka_consumer_close(inner)
|
32
|
+
end
|
33
|
+
@native_kafka.close
|
34
|
+
end
|
24
35
|
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
@native_kafka = nil
|
36
|
+
# Whether this consumer has closed
|
37
|
+
def closed?
|
38
|
+
@native_kafka.closed?
|
29
39
|
end
|
30
40
|
|
31
41
|
# Subscribe to one or more topics letting Kafka handle partition assignments.
|
@@ -46,7 +56,9 @@ module Rdkafka
|
|
46
56
|
end
|
47
57
|
|
48
58
|
# Subscribe to topic partition list and check this was successful
|
49
|
-
response =
|
59
|
+
response = @native_kafka.with_inner do |inner|
|
60
|
+
Rdkafka::Bindings.rd_kafka_subscribe(inner, tpl)
|
61
|
+
end
|
50
62
|
if response != 0
|
51
63
|
raise Rdkafka::RdkafkaError.new(response, "Error subscribing to '#{topics.join(', ')}'")
|
52
64
|
end
|
@@ -62,7 +74,9 @@ module Rdkafka
|
|
62
74
|
def unsubscribe
|
63
75
|
closed_consumer_check(__method__)
|
64
76
|
|
65
|
-
response =
|
77
|
+
response = @native_kafka.with_inner do |inner|
|
78
|
+
Rdkafka::Bindings.rd_kafka_unsubscribe(inner)
|
79
|
+
end
|
66
80
|
if response != 0
|
67
81
|
raise Rdkafka::RdkafkaError.new(response)
|
68
82
|
end
|
@@ -85,7 +99,9 @@ module Rdkafka
|
|
85
99
|
tpl = list.to_native_tpl
|
86
100
|
|
87
101
|
begin
|
88
|
-
response =
|
102
|
+
response = @native_kafka.with_inner do |inner|
|
103
|
+
Rdkafka::Bindings.rd_kafka_pause_partitions(inner, tpl)
|
104
|
+
end
|
89
105
|
|
90
106
|
if response != 0
|
91
107
|
list = TopicPartitionList.from_native_tpl(tpl)
|
@@ -113,7 +129,9 @@ module Rdkafka
|
|
113
129
|
tpl = list.to_native_tpl
|
114
130
|
|
115
131
|
begin
|
116
|
-
response =
|
132
|
+
response = @native_kafka.with_inner do |inner|
|
133
|
+
Rdkafka::Bindings.rd_kafka_resume_partitions(inner, tpl)
|
134
|
+
end
|
117
135
|
if response != 0
|
118
136
|
raise Rdkafka::RdkafkaError.new(response, "Error resume '#{list.to_h}'")
|
119
137
|
end
|
@@ -131,7 +149,9 @@ module Rdkafka
|
|
131
149
|
closed_consumer_check(__method__)
|
132
150
|
|
133
151
|
ptr = FFI::MemoryPointer.new(:pointer)
|
134
|
-
response =
|
152
|
+
response = @native_kafka.with_inner do |inner|
|
153
|
+
Rdkafka::Bindings.rd_kafka_subscription(inner, ptr)
|
154
|
+
end
|
135
155
|
|
136
156
|
if response != 0
|
137
157
|
raise Rdkafka::RdkafkaError.new(response)
|
@@ -161,7 +181,9 @@ module Rdkafka
|
|
161
181
|
tpl = list.to_native_tpl
|
162
182
|
|
163
183
|
begin
|
164
|
-
response =
|
184
|
+
response = @native_kafka.with_inner do |inner|
|
185
|
+
Rdkafka::Bindings.rd_kafka_assign(inner, tpl)
|
186
|
+
end
|
165
187
|
if response != 0
|
166
188
|
raise Rdkafka::RdkafkaError.new(response, "Error assigning '#{list.to_h}'")
|
167
189
|
end
|
@@ -179,7 +201,9 @@ module Rdkafka
|
|
179
201
|
closed_consumer_check(__method__)
|
180
202
|
|
181
203
|
ptr = FFI::MemoryPointer.new(:pointer)
|
182
|
-
response =
|
204
|
+
response = @native_kafka.with_inner do |inner|
|
205
|
+
Rdkafka::Bindings.rd_kafka_assignment(inner, ptr)
|
206
|
+
end
|
183
207
|
if response != 0
|
184
208
|
raise Rdkafka::RdkafkaError.new(response)
|
185
209
|
end
|
@@ -218,7 +242,9 @@ module Rdkafka
|
|
218
242
|
tpl = list.to_native_tpl
|
219
243
|
|
220
244
|
begin
|
221
|
-
response =
|
245
|
+
response = @native_kafka.with_inner do |inner|
|
246
|
+
Rdkafka::Bindings.rd_kafka_committed(inner, tpl, timeout_ms)
|
247
|
+
end
|
222
248
|
if response != 0
|
223
249
|
raise Rdkafka::RdkafkaError.new(response)
|
224
250
|
end
|
@@ -243,14 +269,16 @@ module Rdkafka
|
|
243
269
|
low = FFI::MemoryPointer.new(:int64, 1)
|
244
270
|
high = FFI::MemoryPointer.new(:int64, 1)
|
245
271
|
|
246
|
-
response =
|
247
|
-
|
248
|
-
|
249
|
-
|
250
|
-
|
251
|
-
|
252
|
-
|
253
|
-
|
272
|
+
response = @native_kafka.with_inner do |inner|
|
273
|
+
Rdkafka::Bindings.rd_kafka_query_watermark_offsets(
|
274
|
+
inner,
|
275
|
+
topic,
|
276
|
+
partition,
|
277
|
+
low,
|
278
|
+
high,
|
279
|
+
timeout_ms,
|
280
|
+
)
|
281
|
+
end
|
254
282
|
if response != 0
|
255
283
|
raise Rdkafka::RdkafkaError.new(response, "Error querying watermark offsets for partition #{partition} of #{topic}")
|
256
284
|
end
|
@@ -298,7 +326,9 @@ module Rdkafka
|
|
298
326
|
# @return [String, nil]
|
299
327
|
def cluster_id
|
300
328
|
closed_consumer_check(__method__)
|
301
|
-
|
329
|
+
@native_kafka.with_inner do |inner|
|
330
|
+
Rdkafka::Bindings.rd_kafka_clusterid(inner)
|
331
|
+
end
|
302
332
|
end
|
303
333
|
|
304
334
|
# Returns this client's broker-assigned group member id
|
@@ -308,7 +338,9 @@ module Rdkafka
|
|
308
338
|
# @return [String, nil]
|
309
339
|
def member_id
|
310
340
|
closed_consumer_check(__method__)
|
311
|
-
|
341
|
+
@native_kafka.with_inner do |inner|
|
342
|
+
Rdkafka::Bindings.rd_kafka_memberid(inner)
|
343
|
+
end
|
312
344
|
end
|
313
345
|
|
314
346
|
# Store offset of a message to be used in the next commit of this consumer
|
@@ -325,11 +357,13 @@ module Rdkafka
|
|
325
357
|
|
326
358
|
# rd_kafka_offset_store is one of the few calls that does not support
|
327
359
|
# a string as the topic, so create a native topic for it.
|
328
|
-
native_topic =
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
|
360
|
+
native_topic = @native_kafka.with_inner do |inner|
|
361
|
+
Rdkafka::Bindings.rd_kafka_topic_new(
|
362
|
+
inner,
|
363
|
+
message.topic,
|
364
|
+
nil
|
365
|
+
)
|
366
|
+
end
|
333
367
|
response = Rdkafka::Bindings.rd_kafka_offset_store(
|
334
368
|
native_topic,
|
335
369
|
message.partition,
|
@@ -357,11 +391,13 @@ module Rdkafka
|
|
357
391
|
|
358
392
|
# rd_kafka_offset_store is one of the few calls that does not support
|
359
393
|
# a string as the topic, so create a native topic for it.
|
360
|
-
native_topic =
|
361
|
-
|
362
|
-
|
363
|
-
|
364
|
-
|
394
|
+
native_topic = @native_kafka.with_inner do |inner|
|
395
|
+
Rdkafka::Bindings.rd_kafka_topic_new(
|
396
|
+
inner,
|
397
|
+
message.topic,
|
398
|
+
nil
|
399
|
+
)
|
400
|
+
end
|
365
401
|
response = Rdkafka::Bindings.rd_kafka_seek(
|
366
402
|
native_topic,
|
367
403
|
message.partition,
|
@@ -402,7 +438,9 @@ module Rdkafka
|
|
402
438
|
tpl = list ? list.to_native_tpl : nil
|
403
439
|
|
404
440
|
begin
|
405
|
-
response =
|
441
|
+
response = @native_kafka.with_inner do |inner|
|
442
|
+
Rdkafka::Bindings.rd_kafka_commit(inner, tpl, async)
|
443
|
+
end
|
406
444
|
if response != 0
|
407
445
|
raise Rdkafka::RdkafkaError.new(response)
|
408
446
|
end
|
@@ -421,7 +459,9 @@ module Rdkafka
|
|
421
459
|
def poll(timeout_ms)
|
422
460
|
closed_consumer_check(__method__)
|
423
461
|
|
424
|
-
message_ptr =
|
462
|
+
message_ptr = @native_kafka.with_inner do |inner|
|
463
|
+
Rdkafka::Bindings.rd_kafka_consumer_poll(inner, timeout_ms)
|
464
|
+
end
|
425
465
|
if message_ptr.null?
|
426
466
|
nil
|
427
467
|
else
|
@@ -436,7 +476,7 @@ module Rdkafka
|
|
436
476
|
end
|
437
477
|
ensure
|
438
478
|
# Clean up rdkafka message if there is one
|
439
|
-
if
|
479
|
+
if message_ptr && !message_ptr.null?
|
440
480
|
Rdkafka::Bindings.rd_kafka_message_destroy(message_ptr)
|
441
481
|
end
|
442
482
|
end
|
@@ -459,7 +499,7 @@ module Rdkafka
|
|
459
499
|
if message
|
460
500
|
yield(message)
|
461
501
|
else
|
462
|
-
if
|
502
|
+
if closed?
|
463
503
|
break
|
464
504
|
else
|
465
505
|
next
|
@@ -468,10 +508,6 @@ module Rdkafka
|
|
468
508
|
end
|
469
509
|
end
|
470
510
|
|
471
|
-
def closed_consumer_check(method)
|
472
|
-
raise Rdkafka::ClosedConsumerError.new(method) if @native_kafka.nil?
|
473
|
-
end
|
474
|
-
|
475
511
|
# Poll for new messages and yield them in batches that may contain
|
476
512
|
# messages from more than one partition.
|
477
513
|
#
|
@@ -527,7 +563,7 @@ module Rdkafka
|
|
527
563
|
bytes = 0
|
528
564
|
end_time = monotonic_now + timeout_ms / 1000.0
|
529
565
|
loop do
|
530
|
-
break if
|
566
|
+
break if closed?
|
531
567
|
max_wait = end_time - monotonic_now
|
532
568
|
max_wait_ms = if max_wait <= 0
|
533
569
|
0 # should not block, but may retrieve a message
|
@@ -545,7 +581,7 @@ module Rdkafka
|
|
545
581
|
end
|
546
582
|
if message
|
547
583
|
slice << message
|
548
|
-
bytes += message.payload.bytesize
|
584
|
+
bytes += message.payload.bytesize if message.payload
|
549
585
|
end
|
550
586
|
if slice.size == max_items || bytes >= bytes_threshold || monotonic_now >= end_time - 0.001
|
551
587
|
yield slice.dup, nil
|
@@ -561,5 +597,9 @@ module Rdkafka
|
|
561
597
|
# needed because Time.now can go backwards
|
562
598
|
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
563
599
|
end
|
600
|
+
|
601
|
+
def closed_consumer_check(method)
|
602
|
+
raise Rdkafka::ClosedConsumerError.new(method) if closed?
|
603
|
+
end
|
564
604
|
end
|
565
605
|
end
|
data/lib/rdkafka/error.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Rdkafka
|
2
4
|
# Base error class.
|
3
5
|
class BaseError < RuntimeError; end
|
@@ -83,4 +85,11 @@ module Rdkafka
|
|
83
85
|
super("Illegal call to #{method.to_s} on a closed producer")
|
84
86
|
end
|
85
87
|
end
|
88
|
+
|
89
|
+
# Error class for public consumer method calls on a closed admin.
|
90
|
+
class ClosedAdminError < BaseError
|
91
|
+
def initialize(method)
|
92
|
+
super("Illegal call to #{method.to_s} on a closed admin")
|
93
|
+
end
|
94
|
+
end
|
86
95
|
end
|
data/lib/rdkafka/metadata.rb
CHANGED
@@ -1,8 +1,10 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Rdkafka
|
2
4
|
class Metadata
|
3
5
|
attr_reader :brokers, :topics
|
4
6
|
|
5
|
-
def initialize(native_client, topic_name = nil)
|
7
|
+
def initialize(native_client, topic_name = nil, timeout_ms = 250)
|
6
8
|
native_topic = if topic_name
|
7
9
|
Rdkafka::Bindings.rd_kafka_topic_new(native_client, topic_name, nil)
|
8
10
|
end
|
@@ -14,7 +16,7 @@ module Rdkafka
|
|
14
16
|
topic_flag = topic_name.nil? ? 1 : 0
|
15
17
|
|
16
18
|
# Retrieve the Metadata
|
17
|
-
result = Rdkafka::Bindings.rd_kafka_metadata(native_client, topic_flag, native_topic, ptr,
|
19
|
+
result = Rdkafka::Bindings.rd_kafka_metadata(native_client, topic_flag, native_topic, ptr, timeout_ms)
|
18
20
|
|
19
21
|
# Error Handling
|
20
22
|
raise Rdkafka::RdkafkaError.new(result) unless result.zero?
|
@@ -0,0 +1,83 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Rdkafka
|
4
|
+
# @private
|
5
|
+
# A wrapper around a native kafka that polls and cleanly exits
|
6
|
+
class NativeKafka
|
7
|
+
def initialize(inner, run_polling_thread:)
|
8
|
+
@inner = inner
|
9
|
+
# Lock around external access
|
10
|
+
@access_mutex = Mutex.new
|
11
|
+
# Lock around internal polling
|
12
|
+
@poll_mutex = Mutex.new
|
13
|
+
|
14
|
+
if run_polling_thread
|
15
|
+
# Start thread to poll client for delivery callbacks,
|
16
|
+
# not used in consumer.
|
17
|
+
@polling_thread = Thread.new do
|
18
|
+
loop do
|
19
|
+
@poll_mutex.synchronize do
|
20
|
+
Rdkafka::Bindings.rd_kafka_poll(inner, 100)
|
21
|
+
end
|
22
|
+
|
23
|
+
# Exit thread if closing and the poll queue is empty
|
24
|
+
if Thread.current[:closing] && Rdkafka::Bindings.rd_kafka_outq_len(inner) == 0
|
25
|
+
break
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
@polling_thread.abort_on_exception = true
|
31
|
+
@polling_thread[:closing] = false
|
32
|
+
end
|
33
|
+
|
34
|
+
@closing = false
|
35
|
+
end
|
36
|
+
|
37
|
+
def with_inner
|
38
|
+
return if @inner.nil?
|
39
|
+
|
40
|
+
@access_mutex.synchronize do
|
41
|
+
yield @inner
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def finalizer
|
46
|
+
->(_) { close }
|
47
|
+
end
|
48
|
+
|
49
|
+
def closed?
|
50
|
+
@closing || @inner.nil?
|
51
|
+
end
|
52
|
+
|
53
|
+
def close(object_id=nil)
|
54
|
+
return if closed?
|
55
|
+
|
56
|
+
@access_mutex.lock
|
57
|
+
|
58
|
+
# Indicate to the outside world that we are closing
|
59
|
+
@closing = true
|
60
|
+
|
61
|
+
if @polling_thread
|
62
|
+
# Indicate to polling thread that we're closing
|
63
|
+
@polling_thread[:closing] = true
|
64
|
+
|
65
|
+
# Wait for the polling thread to finish up,
|
66
|
+
# this can be aborted in practice if this
|
67
|
+
# code runs from a finalizer.
|
68
|
+
@polling_thread.join
|
69
|
+
end
|
70
|
+
|
71
|
+
# Destroy the client after locking both mutexes
|
72
|
+
@poll_mutex.lock
|
73
|
+
|
74
|
+
# This check prevents a race condition, where we would enter the close in two threads
|
75
|
+
# and after unlocking the primary one that hold the lock but finished, ours would be unlocked
|
76
|
+
# and would continue to run, trying to destroy inner twice
|
77
|
+
return unless @inner
|
78
|
+
|
79
|
+
Rdkafka::Bindings.rd_kafka_destroy(@inner)
|
80
|
+
@inner = nil
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Rdkafka
|
2
4
|
class Producer
|
3
5
|
# Handle to wait for a delivery report which is returned when
|
@@ -6,7 +8,8 @@ module Rdkafka
|
|
6
8
|
layout :pending, :bool,
|
7
9
|
:response, :int,
|
8
10
|
:partition, :int,
|
9
|
-
:offset, :int64
|
11
|
+
:offset, :int64,
|
12
|
+
:topic_name, :pointer
|
10
13
|
|
11
14
|
# @return [String] the name of the operation (e.g. "delivery")
|
12
15
|
def operation_name
|
@@ -15,7 +18,7 @@ module Rdkafka
|
|
15
18
|
|
16
19
|
# @return [DeliveryReport] a report on the delivery of the message
|
17
20
|
def create_result
|
18
|
-
DeliveryReport.new(self[:partition], self[:offset])
|
21
|
+
DeliveryReport.new(self[:partition], self[:offset], self[:topic_name].read_string)
|
19
22
|
end
|
20
23
|
end
|
21
24
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Rdkafka
|
2
4
|
class Producer
|
3
5
|
# Delivery report for a successfully produced message.
|
@@ -10,15 +12,20 @@ module Rdkafka
|
|
10
12
|
# @return [Integer]
|
11
13
|
attr_reader :offset
|
12
14
|
|
13
|
-
#
|
15
|
+
# The name of the topic this message was produced to.
|
14
16
|
# @return [String]
|
17
|
+
attr_reader :topic_name
|
18
|
+
|
19
|
+
# Error in case happen during produce.
|
20
|
+
# @return [Integer]
|
15
21
|
attr_reader :error
|
16
22
|
|
17
23
|
private
|
18
24
|
|
19
|
-
def initialize(partition, offset, error = nil)
|
25
|
+
def initialize(partition, offset, topic_name = nil, error = nil)
|
20
26
|
@partition = partition
|
21
27
|
@offset = offset
|
28
|
+
@topic_name = topic_name
|
22
29
|
@error = error
|
23
30
|
end
|
24
31
|
end
|
data/lib/rdkafka/producer.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require "objspace"
|
2
4
|
|
3
5
|
module Rdkafka
|
@@ -16,12 +18,12 @@ module Rdkafka
|
|
16
18
|
attr_reader :delivery_callback_arity
|
17
19
|
|
18
20
|
# @private
|
19
|
-
def initialize(
|
20
|
-
@
|
21
|
+
def initialize(native_kafka, partitioner_name)
|
22
|
+
@native_kafka = native_kafka
|
21
23
|
@partitioner_name = partitioner_name || "consistent_random"
|
22
24
|
|
23
|
-
# Makes sure, that
|
24
|
-
ObjectSpace.define_finalizer(self,
|
25
|
+
# Makes sure, that native kafka gets closed before it gets GCed by Ruby
|
26
|
+
ObjectSpace.define_finalizer(self, native_kafka.finalizer)
|
25
27
|
end
|
26
28
|
|
27
29
|
# Set a callback that will be called every time a message is successfully produced.
|
@@ -38,9 +40,26 @@ module Rdkafka
|
|
38
40
|
|
39
41
|
# Close this producer and wait for the internal poll queue to empty.
|
40
42
|
def close
|
43
|
+
return if closed?
|
41
44
|
ObjectSpace.undefine_finalizer(self)
|
45
|
+
@native_kafka.close
|
46
|
+
end
|
47
|
+
|
48
|
+
# Whether this producer has closed
|
49
|
+
def closed?
|
50
|
+
@native_kafka.closed?
|
51
|
+
end
|
52
|
+
|
53
|
+
# Wait until all outstanding producer requests are completed, with the given timeout
|
54
|
+
# in seconds. Call this before closing a producer to ensure delivery of all messages.
|
55
|
+
#
|
56
|
+
# @param timeout_ms [Integer] how long should we wait for flush of all messages
|
57
|
+
def flush(timeout_ms=5_000)
|
58
|
+
closed_producer_check(__method__)
|
42
59
|
|
43
|
-
@
|
60
|
+
@native_kafka.with_inner do |inner|
|
61
|
+
Rdkafka::Bindings.rd_kafka_flush(inner, timeout_ms)
|
62
|
+
end
|
44
63
|
end
|
45
64
|
|
46
65
|
# Partition count for a given topic.
|
@@ -49,10 +68,11 @@ module Rdkafka
|
|
49
68
|
# @param topic [String] The topic name.
|
50
69
|
#
|
51
70
|
# @return partition count [Integer,nil]
|
52
|
-
#
|
53
71
|
def partition_count(topic)
|
54
72
|
closed_producer_check(__method__)
|
55
|
-
|
73
|
+
@native_kafka.with_inner do |inner|
|
74
|
+
Rdkafka::Metadata.new(inner, topic).topics&.first[:partition_count]
|
75
|
+
end
|
56
76
|
end
|
57
77
|
|
58
78
|
# Produces a message to a Kafka topic. The message is added to rdkafka's queue, call {DeliveryHandle#wait wait} on the returned delivery handle to make sure it is delivered.
|
@@ -143,10 +163,12 @@ module Rdkafka
|
|
143
163
|
args << :int << Rdkafka::Bindings::RD_KAFKA_VTYPE_END
|
144
164
|
|
145
165
|
# Produce the message
|
146
|
-
response =
|
147
|
-
|
148
|
-
|
149
|
-
|
166
|
+
response = @native_kafka.with_inner do |inner|
|
167
|
+
Rdkafka::Bindings.rd_kafka_producev(
|
168
|
+
inner,
|
169
|
+
*args
|
170
|
+
)
|
171
|
+
end
|
150
172
|
|
151
173
|
# Raise error if the produce call was not successful
|
152
174
|
if response != 0
|
@@ -157,7 +179,6 @@ module Rdkafka
|
|
157
179
|
delivery_handle
|
158
180
|
end
|
159
181
|
|
160
|
-
# @private
|
161
182
|
def call_delivery_callback(delivery_report, delivery_handle)
|
162
183
|
return unless @delivery_callback
|
163
184
|
|
@@ -171,8 +192,9 @@ module Rdkafka
|
|
171
192
|
callback.method(:call).arity
|
172
193
|
end
|
173
194
|
|
195
|
+
private
|
174
196
|
def closed_producer_check(method)
|
175
|
-
raise Rdkafka::ClosedProducerError.new(method) if
|
197
|
+
raise Rdkafka::ClosedProducerError.new(method) if closed?
|
176
198
|
end
|
177
199
|
end
|
178
200
|
end
|
data/lib/rdkafka/version.rb
CHANGED
@@ -1,5 +1,7 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Rdkafka
|
2
|
-
VERSION = "0.
|
3
|
-
LIBRDKAFKA_VERSION = "
|
4
|
-
LIBRDKAFKA_SOURCE_SHA256 = "
|
4
|
+
VERSION = "0.13.0.beta.7"
|
5
|
+
LIBRDKAFKA_VERSION = "2.0.2"
|
6
|
+
LIBRDKAFKA_SOURCE_SHA256 = "f321bcb1e015a34114c83cf1aa7b99ee260236aab096b85c003170c90a47ca9d"
|
5
7
|
end
|
data/lib/rdkafka.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require "rdkafka/version"
|
2
4
|
|
3
5
|
require "rdkafka/abstract_handle"
|
@@ -16,7 +18,7 @@ require "rdkafka/consumer/partition"
|
|
16
18
|
require "rdkafka/consumer/topic_partition_list"
|
17
19
|
require "rdkafka/error"
|
18
20
|
require "rdkafka/metadata"
|
21
|
+
require "rdkafka/native_kafka"
|
19
22
|
require "rdkafka/producer"
|
20
|
-
require "rdkafka/producer/client"
|
21
23
|
require "rdkafka/producer/delivery_handle"
|
22
24
|
require "rdkafka/producer/delivery_report"
|
data/rdkafka.gemspec
CHANGED
data/spec/rdkafka/admin_spec.rb
CHANGED
@@ -1,9 +1,11 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require "spec_helper"
|
2
4
|
require "ostruct"
|
3
5
|
|
4
6
|
describe Rdkafka::Admin do
|
5
|
-
let(:config)
|
6
|
-
let(:admin)
|
7
|
+
let(:config) { rdkafka_config }
|
8
|
+
let(:admin) { config.admin }
|
7
9
|
|
8
10
|
after do
|
9
11
|
# Registry should always end up being empty
|
@@ -174,7 +176,6 @@ describe Rdkafka::Admin do
|
|
174
176
|
end
|
175
177
|
end
|
176
178
|
|
177
|
-
|
178
179
|
it "deletes a topic that was newly created" do
|
179
180
|
create_topic_handle = admin.create_topic(topic_name, topic_partition_count, topic_replication_factor)
|
180
181
|
create_topic_report = create_topic_handle.wait(max_wait_timeout: 15.0)
|