rdkafka 0.29.0 → 0.29.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +41 -24
- data/README.md +2 -2
- data/bin/verify_kafka_warnings +5 -0
- data/docker-compose-ssl.yml +3 -0
- data/docker-compose.yml +3 -0
- data/lib/rdkafka/admin/create_acl_report.rb +2 -1
- data/lib/rdkafka/admin/delete_records_handle.rb +31 -0
- data/lib/rdkafka/admin/delete_records_report.rb +25 -0
- data/lib/rdkafka/admin/describe_acl_report.rb +2 -1
- data/lib/rdkafka/admin/list_consumer_groups_handle.rb +31 -0
- data/lib/rdkafka/admin/list_consumer_groups_report.rb +83 -0
- data/lib/rdkafka/admin/list_offsets_report.rb +5 -2
- data/lib/rdkafka/admin.rb +318 -199
- data/lib/rdkafka/bindings.rb +53 -12
- data/lib/rdkafka/callbacks/delete_records_handler.rb +38 -0
- data/lib/rdkafka/callbacks/list_consumer_groups_handler.rb +39 -0
- data/lib/rdkafka/callbacks.rb +4 -2
- data/lib/rdkafka/clients.rb +114 -0
- data/lib/rdkafka/config.rb +108 -56
- data/lib/rdkafka/consumer/allocator.rb +38 -0
- data/lib/rdkafka/consumer/topic_partition_list.rb +15 -10
- data/lib/rdkafka/consumer.rb +147 -74
- data/lib/rdkafka/defaults.rb +5 -7
- data/lib/rdkafka/helpers/list_offsets.rb +127 -0
- data/lib/rdkafka/helpers/oauth.rb +10 -5
- data/lib/rdkafka/native_kafka.rb +20 -7
- data/lib/rdkafka/producer/delivery_handle.rb +1 -2
- data/lib/rdkafka/producer.rb +25 -17
- data/lib/rdkafka/version.rb +1 -1
- data/lib/rdkafka.rb +11 -0
- data/package-lock.json +6 -6
- data/renovate.json +1 -1
- metadata +11 -2
|
@@ -2,8 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
module Rdkafka
|
|
4
4
|
class Producer
|
|
5
|
-
# Handle to wait for a delivery report which is returned when
|
|
6
|
-
# producing a message.
|
|
5
|
+
# Handle to wait for a delivery report which is returned when producing a message.
|
|
7
6
|
class DeliveryHandle < Rdkafka::AbstractHandle
|
|
8
7
|
layout :pending, :bool,
|
|
9
8
|
:response, :int,
|
data/lib/rdkafka/producer.rb
CHANGED
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
module Rdkafka
|
|
4
|
-
# A producer for Kafka messages. To create a producer set up a {Config} and call
|
|
4
|
+
# A producer for Kafka messages. To create a producer set up a {Config} and call
|
|
5
|
+
# {Config#producer producer} on that.
|
|
5
6
|
class Producer
|
|
6
7
|
include Helpers::Time
|
|
7
8
|
include Helpers::OAuth
|
|
8
9
|
include Helpers::Metadata
|
|
9
10
|
|
|
10
|
-
#
|
|
11
|
+
# @!visibility private
|
|
11
12
|
@@partitions_count_cache = PartitionsCountCache.new
|
|
12
13
|
|
|
13
14
|
# Global (process wide) partitions cache. We use it to store number of topics partitions,
|
|
@@ -251,10 +252,9 @@ module Rdkafka
|
|
|
251
252
|
# @return [Integer] the number of messages in the queue
|
|
252
253
|
# @raise [Rdkafka::ClosedProducerError] if called on a closed producer
|
|
253
254
|
#
|
|
254
|
-
# @note This method is thread-safe as it uses the @native_kafka.with_inner synchronization
|
|
255
|
-
#
|
|
256
255
|
# @example
|
|
257
256
|
# producer.queue_size #=> 42
|
|
257
|
+
# @note This method is thread-safe as it uses the @native_kafka.with_inner synchronization
|
|
258
258
|
def queue_size
|
|
259
259
|
closed_producer_check(__method__)
|
|
260
260
|
|
|
@@ -280,10 +280,6 @@ module Rdkafka
|
|
|
280
280
|
# @return [nil]
|
|
281
281
|
# @raise [Rdkafka::ClosedProducerError] if called on a closed producer
|
|
282
282
|
#
|
|
283
|
-
# @note This method holds the inner lock until the queue is empty or `:stop` is returned.
|
|
284
|
-
# Other producer operations (produce, close, etc.) will wait until this method returns.
|
|
285
|
-
# @note This method is thread-safe as it uses @native_kafka.with_inner synchronization
|
|
286
|
-
#
|
|
287
283
|
# @example Drain all pending callbacks
|
|
288
284
|
# producer.events_poll_nb_each { |_count| }
|
|
289
285
|
#
|
|
@@ -292,6 +288,9 @@ module Rdkafka
|
|
|
292
288
|
# producer.events_poll_nb_each do |_count|
|
|
293
289
|
# :stop if monotonic_now >= deadline
|
|
294
290
|
# end
|
|
291
|
+
# @note This method holds the inner lock until the queue is empty or `:stop` is returned.
|
|
292
|
+
# Other producer operations (produce, close, etc.) will wait until this method returns.
|
|
293
|
+
# @note This method is thread-safe as it uses @native_kafka.with_inner synchronization
|
|
295
294
|
def events_poll_nb_each
|
|
296
295
|
closed_producer_check(__method__)
|
|
297
296
|
|
|
@@ -340,23 +339,32 @@ module Rdkafka
|
|
|
340
339
|
end
|
|
341
340
|
end
|
|
342
341
|
|
|
343
|
-
# Produces a message to a Kafka topic. The message is added to rdkafka's queue, call
|
|
342
|
+
# Produces a message to a Kafka topic. The message is added to rdkafka's queue, call
|
|
343
|
+
# {DeliveryHandle#wait wait} on the returned delivery handle to make sure it is delivered.
|
|
344
344
|
#
|
|
345
|
-
# When no partition is specified the underlying Kafka library picks a partition based on the
|
|
346
|
-
#
|
|
345
|
+
# When no partition is specified the underlying Kafka library picks a partition based on the
|
|
346
|
+
# key. If no key is specified, a random partition will be used. When a timestamp is provided
|
|
347
|
+
# this is used instead of the auto-generated timestamp.
|
|
347
348
|
#
|
|
348
349
|
# @param topic [String] The topic to produce to
|
|
349
350
|
# @param payload [String, nil]
|
|
350
351
|
# @param key [String, nil]
|
|
351
352
|
# @param partition [Integer, nil] Optional partition to produce to
|
|
352
|
-
# @param partition_key [String, nil] Optional partition key based on which partition
|
|
353
|
-
#
|
|
354
|
-
# @param
|
|
355
|
-
#
|
|
356
|
-
# @param
|
|
353
|
+
# @param partition_key [String, nil] Optional partition key based on which partition
|
|
354
|
+
# assignment can happen
|
|
355
|
+
# @param timestamp [Time, Integer, nil] Optional timestamp of this message. Integer
|
|
356
|
+
# timestamp is in milliseconds since Jan 1 1970.
|
|
357
|
+
# @param headers [Hash{String => String, Array<String>}] Optional message headers. Values
|
|
358
|
+
# can be either a single string or an array of strings to support duplicate headers per
|
|
359
|
+
# KIP-82
|
|
360
|
+
# @param label [Object, nil] a label that can be assigned when producing a message that
|
|
361
|
+
# will be part of the delivery handle and the delivery report
|
|
362
|
+
# @param topic_config [Hash] topic config for given message dispatch. Allows to send
|
|
363
|
+
# messages to topics with different configuration
|
|
357
364
|
# @param partitioner [String] name of the partitioner to use
|
|
358
365
|
#
|
|
359
|
-
# @return [DeliveryHandle] Delivery handle that can be used to wait for the result of
|
|
366
|
+
# @return [DeliveryHandle] Delivery handle that can be used to wait for the result of
|
|
367
|
+
# producing this message
|
|
360
368
|
#
|
|
361
369
|
# @raise [RdkafkaError] When adding the message to rdkafka's queue failed
|
|
362
370
|
def produce(
|
data/lib/rdkafka/version.rb
CHANGED
data/lib/rdkafka.rb
CHANGED
|
@@ -9,6 +9,7 @@ require "rdkafka/version"
|
|
|
9
9
|
require "rdkafka/helpers/time"
|
|
10
10
|
require "rdkafka/helpers/oauth"
|
|
11
11
|
require "rdkafka/helpers/metadata"
|
|
12
|
+
require "rdkafka/helpers/list_offsets"
|
|
12
13
|
require "rdkafka/defaults"
|
|
13
14
|
require "rdkafka/abstract_handle"
|
|
14
15
|
require "rdkafka/admin"
|
|
@@ -16,6 +17,8 @@ require "rdkafka/admin/create_topic_handle"
|
|
|
16
17
|
require "rdkafka/admin/create_topic_report"
|
|
17
18
|
require "rdkafka/admin/delete_groups_handle"
|
|
18
19
|
require "rdkafka/admin/delete_groups_report"
|
|
20
|
+
require "rdkafka/admin/delete_records_handle"
|
|
21
|
+
require "rdkafka/admin/delete_records_report"
|
|
19
22
|
require "rdkafka/admin/delete_topic_handle"
|
|
20
23
|
require "rdkafka/admin/delete_topic_report"
|
|
21
24
|
require "rdkafka/admin/create_partitions_handle"
|
|
@@ -32,6 +35,8 @@ require "rdkafka/admin/incremental_alter_configs_handle"
|
|
|
32
35
|
require "rdkafka/admin/incremental_alter_configs_report"
|
|
33
36
|
require "rdkafka/admin/list_offsets_handle"
|
|
34
37
|
require "rdkafka/admin/list_offsets_report"
|
|
38
|
+
require "rdkafka/admin/list_consumer_groups_handle"
|
|
39
|
+
require "rdkafka/admin/list_consumer_groups_report"
|
|
35
40
|
require "rdkafka/admin/acl_binding_result"
|
|
36
41
|
require "rdkafka/admin/config_binding_result"
|
|
37
42
|
require "rdkafka/admin/config_resource_binding_result"
|
|
@@ -42,17 +47,20 @@ require "rdkafka/callbacks/create_topic_handler"
|
|
|
42
47
|
require "rdkafka/callbacks/delete_topic_handler"
|
|
43
48
|
require "rdkafka/callbacks/create_partitions_handler"
|
|
44
49
|
require "rdkafka/callbacks/delete_groups_handler"
|
|
50
|
+
require "rdkafka/callbacks/delete_records_handler"
|
|
45
51
|
require "rdkafka/callbacks/create_acl_handler"
|
|
46
52
|
require "rdkafka/callbacks/delete_acl_handler"
|
|
47
53
|
require "rdkafka/callbacks/describe_acl_handler"
|
|
48
54
|
require "rdkafka/callbacks/describe_configs_handler"
|
|
49
55
|
require "rdkafka/callbacks/incremental_alter_configs_handler"
|
|
50
56
|
require "rdkafka/callbacks/list_offsets_handler"
|
|
57
|
+
require "rdkafka/callbacks/list_consumer_groups_handler"
|
|
51
58
|
require "rdkafka/config"
|
|
52
59
|
require "rdkafka/consumer"
|
|
53
60
|
require "rdkafka/consumer/headers"
|
|
54
61
|
require "rdkafka/consumer/message"
|
|
55
62
|
require "rdkafka/consumer/partition"
|
|
63
|
+
require "rdkafka/consumer/allocator"
|
|
56
64
|
require "rdkafka/consumer/topic_partition_list"
|
|
57
65
|
require "rdkafka/error"
|
|
58
66
|
require "rdkafka/metadata"
|
|
@@ -61,9 +69,12 @@ require "rdkafka/producer/partitions_count_cache"
|
|
|
61
69
|
require "rdkafka/producer"
|
|
62
70
|
require "rdkafka/producer/delivery_handle"
|
|
63
71
|
require "rdkafka/producer/delivery_report"
|
|
72
|
+
require "rdkafka/clients"
|
|
64
73
|
|
|
65
74
|
# Main Rdkafka namespace of this gem
|
|
66
75
|
module Rdkafka
|
|
67
76
|
end
|
|
68
77
|
|
|
69
78
|
Rdkafka::Bindings.rd_kafka_global_init
|
|
79
|
+
|
|
80
|
+
at_exit { Rdkafka::Clients.close_all }
|
data/package-lock.json
CHANGED
|
@@ -103,9 +103,9 @@
|
|
|
103
103
|
}
|
|
104
104
|
},
|
|
105
105
|
"node_modules/fastq": {
|
|
106
|
-
"version": "1.20.
|
|
107
|
-
"resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.
|
|
108
|
-
"integrity": "sha512-
|
|
106
|
+
"version": "1.20.2",
|
|
107
|
+
"resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.2.tgz",
|
|
108
|
+
"integrity": "sha512-UpGiiODyCGprM8EPP6JodP6jC9Rws6TCuiDOD+nn0CJhR8guI3g/ozo4ugL0vJ+Yz1UtJuuRPqvQuybVOF1VQA==",
|
|
109
109
|
"dev": true,
|
|
110
110
|
"license": "ISC",
|
|
111
111
|
"dependencies": {
|
|
@@ -286,9 +286,9 @@
|
|
|
286
286
|
}
|
|
287
287
|
},
|
|
288
288
|
"node_modules/smol-toml": {
|
|
289
|
-
"version": "1.
|
|
290
|
-
"resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.
|
|
291
|
-
"integrity": "sha512-
|
|
289
|
+
"version": "1.8.0",
|
|
290
|
+
"resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.8.0.tgz",
|
|
291
|
+
"integrity": "sha512-kCZr2V3ch9i00x8zXRhjUNVcjG9ijES5dDudkXvUVCT5QlJNQWElSJdZqyPemffHoLNUYwOcou0Fy+ojN0uHSQ==",
|
|
292
292
|
"dev": true,
|
|
293
293
|
"license": "BSD-3-Clause",
|
|
294
294
|
"engines": {
|
data/renovate.json
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: rdkafka
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.29.
|
|
4
|
+
version: 0.29.2
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Thijs Cadier
|
|
@@ -114,6 +114,8 @@ files:
|
|
|
114
114
|
- lib/rdkafka/admin/delete_acl_report.rb
|
|
115
115
|
- lib/rdkafka/admin/delete_groups_handle.rb
|
|
116
116
|
- lib/rdkafka/admin/delete_groups_report.rb
|
|
117
|
+
- lib/rdkafka/admin/delete_records_handle.rb
|
|
118
|
+
- lib/rdkafka/admin/delete_records_report.rb
|
|
117
119
|
- lib/rdkafka/admin/delete_topic_handle.rb
|
|
118
120
|
- lib/rdkafka/admin/delete_topic_report.rb
|
|
119
121
|
- lib/rdkafka/admin/describe_acl_handle.rb
|
|
@@ -122,6 +124,8 @@ files:
|
|
|
122
124
|
- lib/rdkafka/admin/describe_configs_report.rb
|
|
123
125
|
- lib/rdkafka/admin/incremental_alter_configs_handle.rb
|
|
124
126
|
- lib/rdkafka/admin/incremental_alter_configs_report.rb
|
|
127
|
+
- lib/rdkafka/admin/list_consumer_groups_handle.rb
|
|
128
|
+
- lib/rdkafka/admin/list_consumer_groups_report.rb
|
|
125
129
|
- lib/rdkafka/admin/list_offsets_handle.rb
|
|
126
130
|
- lib/rdkafka/admin/list_offsets_report.rb
|
|
127
131
|
- lib/rdkafka/bindings.rb
|
|
@@ -132,19 +136,24 @@ files:
|
|
|
132
136
|
- lib/rdkafka/callbacks/create_topic_handler.rb
|
|
133
137
|
- lib/rdkafka/callbacks/delete_acl_handler.rb
|
|
134
138
|
- lib/rdkafka/callbacks/delete_groups_handler.rb
|
|
139
|
+
- lib/rdkafka/callbacks/delete_records_handler.rb
|
|
135
140
|
- lib/rdkafka/callbacks/delete_topic_handler.rb
|
|
136
141
|
- lib/rdkafka/callbacks/describe_acl_handler.rb
|
|
137
142
|
- lib/rdkafka/callbacks/describe_configs_handler.rb
|
|
138
143
|
- lib/rdkafka/callbacks/incremental_alter_configs_handler.rb
|
|
144
|
+
- lib/rdkafka/callbacks/list_consumer_groups_handler.rb
|
|
139
145
|
- lib/rdkafka/callbacks/list_offsets_handler.rb
|
|
146
|
+
- lib/rdkafka/clients.rb
|
|
140
147
|
- lib/rdkafka/config.rb
|
|
141
148
|
- lib/rdkafka/consumer.rb
|
|
149
|
+
- lib/rdkafka/consumer/allocator.rb
|
|
142
150
|
- lib/rdkafka/consumer/headers.rb
|
|
143
151
|
- lib/rdkafka/consumer/message.rb
|
|
144
152
|
- lib/rdkafka/consumer/partition.rb
|
|
145
153
|
- lib/rdkafka/consumer/topic_partition_list.rb
|
|
146
154
|
- lib/rdkafka/defaults.rb
|
|
147
155
|
- lib/rdkafka/error.rb
|
|
156
|
+
- lib/rdkafka/helpers/list_offsets.rb
|
|
148
157
|
- lib/rdkafka/helpers/metadata.rb
|
|
149
158
|
- lib/rdkafka/helpers/oauth.rb
|
|
150
159
|
- lib/rdkafka/helpers/time.rb
|
|
@@ -183,7 +192,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
183
192
|
- !ruby/object:Gem::Version
|
|
184
193
|
version: '0'
|
|
185
194
|
requirements: []
|
|
186
|
-
rubygems_version: 4.0.
|
|
195
|
+
rubygems_version: 4.0.16
|
|
187
196
|
specification_version: 4
|
|
188
197
|
summary: The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka.
|
|
189
198
|
It wraps the production-ready C client using the ffi gem and targets Kafka 1.0+
|