rdkafka 0.22.2 → 0.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +63 -3
- data/Gemfile +8 -0
- data/Gemfile.lint +14 -0
- data/Gemfile.lint.lock +123 -0
- data/README.md +19 -14
- data/Rakefile +21 -21
- data/bin/verify_kafka_warnings +39 -0
- data/dist/{librdkafka-2.8.0.tar.gz → librdkafka-2.14.0.tar.gz} +0 -0
- data/docker-compose-ssl.yml +35 -0
- data/docker-compose.yml +2 -2
- data/ext/Rakefile +27 -27
- data/lib/rdkafka/abstract_handle.rb +23 -5
- data/lib/rdkafka/admin/acl_binding_result.rb +5 -5
- data/lib/rdkafka/admin/config_resource_binding_result.rb +1 -0
- data/lib/rdkafka/admin/create_acl_handle.rb +7 -4
- data/lib/rdkafka/admin/create_acl_report.rb +3 -2
- data/lib/rdkafka/admin/create_partitions_handle.rb +8 -5
- data/lib/rdkafka/admin/create_partitions_report.rb +1 -0
- data/lib/rdkafka/admin/create_topic_handle.rb +8 -5
- data/lib/rdkafka/admin/create_topic_report.rb +3 -0
- data/lib/rdkafka/admin/delete_acl_handle.rb +9 -6
- data/lib/rdkafka/admin/delete_acl_report.rb +5 -3
- data/lib/rdkafka/admin/delete_groups_handle.rb +10 -5
- data/lib/rdkafka/admin/delete_groups_report.rb +3 -0
- data/lib/rdkafka/admin/delete_topic_handle.rb +8 -5
- data/lib/rdkafka/admin/delete_topic_report.rb +3 -0
- data/lib/rdkafka/admin/describe_acl_handle.rb +9 -6
- data/lib/rdkafka/admin/describe_acl_report.rb +5 -3
- data/lib/rdkafka/admin/describe_configs_handle.rb +7 -4
- data/lib/rdkafka/admin/describe_configs_report.rb +7 -1
- data/lib/rdkafka/admin/incremental_alter_configs_handle.rb +7 -4
- data/lib/rdkafka/admin/incremental_alter_configs_report.rb +7 -1
- data/lib/rdkafka/admin/list_offsets_handle.rb +36 -0
- data/lib/rdkafka/admin/list_offsets_report.rb +51 -0
- data/lib/rdkafka/admin.rb +301 -135
- data/lib/rdkafka/bindings.rb +199 -110
- data/lib/rdkafka/callbacks.rb +124 -21
- data/lib/rdkafka/config.rb +81 -33
- data/lib/rdkafka/consumer/headers.rb +3 -2
- data/lib/rdkafka/consumer/message.rb +12 -11
- data/lib/rdkafka/consumer/partition.rb +8 -4
- data/lib/rdkafka/consumer/topic_partition_list.rb +21 -17
- data/lib/rdkafka/consumer.rb +397 -45
- data/lib/rdkafka/defaults.rb +106 -0
- data/lib/rdkafka/error.rb +40 -14
- data/lib/rdkafka/helpers/oauth.rb +45 -13
- data/lib/rdkafka/helpers/time.rb +5 -0
- data/lib/rdkafka/metadata.rb +45 -21
- data/lib/rdkafka/native_kafka.rb +89 -4
- data/lib/rdkafka/producer/delivery_handle.rb +5 -5
- data/lib/rdkafka/producer/delivery_report.rb +10 -6
- data/lib/rdkafka/producer/partitions_count_cache.rb +29 -19
- data/lib/rdkafka/producer.rb +168 -82
- data/lib/rdkafka/version.rb +6 -3
- data/lib/rdkafka.rb +3 -0
- data/package-lock.json +331 -0
- data/package.json +9 -0
- data/rdkafka.gemspec +57 -36
- data/renovate.json +29 -24
- metadata +29 -124
- data/.github/CODEOWNERS +0 -3
- data/.github/FUNDING.yml +0 -1
- data/.github/workflows/ci_linux_x86_64_gnu.yml +0 -271
- data/.github/workflows/ci_linux_x86_64_musl.yml +0 -194
- data/.github/workflows/ci_macos_arm64.yml +0 -284
- data/.github/workflows/push_linux_x86_64_gnu.yml +0 -65
- data/.github/workflows/push_linux_x86_64_musl.yml +0 -79
- data/.github/workflows/push_macos_arm64.yml +0 -54
- data/.github/workflows/push_ruby.yml +0 -37
- data/.github/workflows/verify-action-pins.yml +0 -16
- data/.gitignore +0 -14
- data/.rspec +0 -2
- data/.ruby-gemset +0 -1
- data/.ruby-version +0 -1
- data/.yardopts +0 -2
- data/ext/README.md +0 -19
- data/ext/build_common.sh +0 -361
- data/ext/build_linux_x86_64_gnu.sh +0 -306
- data/ext/build_linux_x86_64_musl.sh +0 -763
- data/ext/build_macos_arm64.sh +0 -550
- data/spec/rdkafka/abstract_handle_spec.rb +0 -117
- data/spec/rdkafka/admin/create_acl_handle_spec.rb +0 -56
- data/spec/rdkafka/admin/create_acl_report_spec.rb +0 -18
- data/spec/rdkafka/admin/create_topic_handle_spec.rb +0 -52
- data/spec/rdkafka/admin/create_topic_report_spec.rb +0 -16
- data/spec/rdkafka/admin/delete_acl_handle_spec.rb +0 -85
- data/spec/rdkafka/admin/delete_acl_report_spec.rb +0 -72
- data/spec/rdkafka/admin/delete_topic_handle_spec.rb +0 -52
- data/spec/rdkafka/admin/delete_topic_report_spec.rb +0 -16
- data/spec/rdkafka/admin/describe_acl_handle_spec.rb +0 -85
- data/spec/rdkafka/admin/describe_acl_report_spec.rb +0 -73
- data/spec/rdkafka/admin_spec.rb +0 -971
- data/spec/rdkafka/bindings_spec.rb +0 -199
- data/spec/rdkafka/callbacks_spec.rb +0 -20
- data/spec/rdkafka/config_spec.rb +0 -258
- data/spec/rdkafka/consumer/headers_spec.rb +0 -73
- data/spec/rdkafka/consumer/message_spec.rb +0 -139
- data/spec/rdkafka/consumer/partition_spec.rb +0 -57
- data/spec/rdkafka/consumer/topic_partition_list_spec.rb +0 -248
- data/spec/rdkafka/consumer_spec.rb +0 -1274
- data/spec/rdkafka/error_spec.rb +0 -89
- data/spec/rdkafka/metadata_spec.rb +0 -79
- data/spec/rdkafka/native_kafka_spec.rb +0 -130
- data/spec/rdkafka/producer/delivery_handle_spec.rb +0 -45
- data/spec/rdkafka/producer/delivery_report_spec.rb +0 -25
- data/spec/rdkafka/producer/partitions_count_cache_spec.rb +0 -359
- data/spec/rdkafka/producer_spec.rb +0 -1345
- data/spec/spec_helper.rb +0 -195
|
@@ -28,10 +28,11 @@ module Rdkafka
|
|
|
28
28
|
# @return [Time, nil]
|
|
29
29
|
attr_reader :timestamp
|
|
30
30
|
|
|
31
|
-
# @return [Hash
|
|
31
|
+
# @return [Hash{String => String}] message headers
|
|
32
32
|
attr_reader :headers
|
|
33
33
|
|
|
34
34
|
# @private
|
|
35
|
+
# @param native_message [Rdkafka::Bindings::Message] native message struct from librdkafka
|
|
35
36
|
def initialize(native_message)
|
|
36
37
|
# Set topic
|
|
37
38
|
unless native_message[:rkt].null?
|
|
@@ -52,13 +53,11 @@ module Rdkafka
|
|
|
52
53
|
# Set timestamp
|
|
53
54
|
raw_timestamp = Rdkafka::Bindings.rd_kafka_message_timestamp(native_message, nil)
|
|
54
55
|
@timestamp = if raw_timestamp && raw_timestamp > -1
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
nil
|
|
61
|
-
end
|
|
56
|
+
# Calculate seconds and microseconds
|
|
57
|
+
seconds = raw_timestamp / 1000
|
|
58
|
+
milliseconds = (raw_timestamp - seconds * 1000) * 1000
|
|
59
|
+
Time.at(seconds, milliseconds)
|
|
60
|
+
end
|
|
62
61
|
|
|
63
62
|
@headers = Headers.from_native(native_message)
|
|
64
63
|
end
|
|
@@ -71,6 +70,11 @@ module Rdkafka
|
|
|
71
70
|
"<Message in '#{topic}' with key '#{truncate(key)}', payload '#{truncate(payload)}', partition #{partition}, offset #{offset}, timestamp #{timestamp}#{is_headers}>"
|
|
72
71
|
end
|
|
73
72
|
|
|
73
|
+
private
|
|
74
|
+
|
|
75
|
+
# Truncates a string for display purposes
|
|
76
|
+
# @param string [String, nil] the string to truncate
|
|
77
|
+
# @return [String, nil] truncated string or nil
|
|
74
78
|
def truncate(string)
|
|
75
79
|
if string && string.length > 40
|
|
76
80
|
"#{string[0..39]}..."
|
|
@@ -78,9 +82,6 @@ module Rdkafka
|
|
|
78
82
|
string
|
|
79
83
|
end
|
|
80
84
|
end
|
|
81
|
-
|
|
82
|
-
private
|
|
83
|
-
|
|
84
85
|
end
|
|
85
86
|
end
|
|
86
87
|
end
|
|
@@ -17,7 +17,10 @@ module Rdkafka
|
|
|
17
17
|
attr_reader :err
|
|
18
18
|
|
|
19
19
|
# @private
|
|
20
|
-
|
|
20
|
+
# @param partition [Integer] partition number
|
|
21
|
+
# @param offset [Integer, nil] partition offset
|
|
22
|
+
# @param err [Integer] error code from librdkafka
|
|
23
|
+
def initialize(partition, offset, err = Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR)
|
|
21
24
|
@partition = partition
|
|
22
25
|
@offset = offset
|
|
23
26
|
@err = err
|
|
@@ -28,7 +31,7 @@ module Rdkafka
|
|
|
28
31
|
def to_s
|
|
29
32
|
message = "<Partition #{partition}"
|
|
30
33
|
message += " offset=#{offset}" if offset
|
|
31
|
-
message += " err=#{err}" if err !=
|
|
34
|
+
message += " err=#{err}" if err != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
|
|
32
35
|
message += ">"
|
|
33
36
|
message
|
|
34
37
|
end
|
|
@@ -40,11 +43,12 @@ module Rdkafka
|
|
|
40
43
|
end
|
|
41
44
|
|
|
42
45
|
# Whether another partition is equal to this
|
|
46
|
+
# @param other [Object] object to compare with
|
|
43
47
|
# @return [Boolean]
|
|
44
48
|
def ==(other)
|
|
45
49
|
self.class == other.class &&
|
|
46
|
-
|
|
47
|
-
|
|
50
|
+
partition == other.partition &&
|
|
51
|
+
offset == other.offset
|
|
48
52
|
end
|
|
49
53
|
end
|
|
50
54
|
end
|
|
@@ -9,7 +9,7 @@ module Rdkafka
|
|
|
9
9
|
# @param data [Hash{String => nil,Partition}] The topic and partition data or nil to create an empty list
|
|
10
10
|
#
|
|
11
11
|
# @return [TopicPartitionList]
|
|
12
|
-
def initialize(data=nil)
|
|
12
|
+
def initialize(data = nil)
|
|
13
13
|
@data = data || {}
|
|
14
14
|
end
|
|
15
15
|
|
|
@@ -18,10 +18,10 @@ module Rdkafka
|
|
|
18
18
|
def count
|
|
19
19
|
i = 0
|
|
20
20
|
@data.each do |_topic, partitions|
|
|
21
|
-
if partitions
|
|
22
|
-
|
|
21
|
+
i += if partitions
|
|
22
|
+
partitions.count
|
|
23
23
|
else
|
|
24
|
-
|
|
24
|
+
1
|
|
25
25
|
end
|
|
26
26
|
end
|
|
27
27
|
i
|
|
@@ -49,15 +49,14 @@ module Rdkafka
|
|
|
49
49
|
#
|
|
50
50
|
# @example Add a topic with all topics up to a count
|
|
51
51
|
# tpl.add_topic("topic", 9)
|
|
52
|
-
|
|
53
|
-
def add_topic(topic, partitions=nil)
|
|
52
|
+
def add_topic(topic, partitions = nil)
|
|
54
53
|
if partitions.nil?
|
|
55
54
|
@data[topic.to_s] = nil
|
|
56
55
|
else
|
|
57
56
|
if partitions.is_a? Integer
|
|
58
57
|
partitions = (0..partitions - 1)
|
|
59
58
|
end
|
|
60
|
-
@data[topic.to_s] = partitions.map { |p| Partition.new(p, nil,
|
|
59
|
+
@data[topic.to_s] = partitions.map { |p| Partition.new(p, nil, Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR) }
|
|
61
60
|
end
|
|
62
61
|
end
|
|
63
62
|
|
|
@@ -65,11 +64,13 @@ module Rdkafka
|
|
|
65
64
|
# Calling this method multiple times for the same topic will overwrite the previous configuraton.
|
|
66
65
|
#
|
|
67
66
|
# @param topic [String] The topic's name
|
|
68
|
-
# @param partitions_with_offsets [Hash
|
|
69
|
-
#
|
|
67
|
+
# @param partitions_with_offsets [Hash{Integer => Integer}, Array<Consumer::Partition>] The topic's
|
|
68
|
+
# partitions and offsets (Hash) or partitions with offsets and metadata (Array)
|
|
70
69
|
# @return [nil]
|
|
71
70
|
def add_topic_and_partitions_with_offsets(topic, partitions_with_offsets)
|
|
72
|
-
@data[topic.to_s] = partitions_with_offsets.map
|
|
71
|
+
@data[topic.to_s] = partitions_with_offsets.map do |p, o|
|
|
72
|
+
p.is_a?(Partition) ? p : Partition.new(p, o)
|
|
73
|
+
end
|
|
73
74
|
end
|
|
74
75
|
|
|
75
76
|
# Return a `Hash` with the topics as keys and and an array of partition information as the value if present.
|
|
@@ -85,8 +86,11 @@ module Rdkafka
|
|
|
85
86
|
"<TopicPartitionList: #{to_h}>"
|
|
86
87
|
end
|
|
87
88
|
|
|
89
|
+
# Check equality with another TopicPartitionList
|
|
90
|
+
# @param other [TopicPartitionList] object to compare with
|
|
91
|
+
# @return [Boolean]
|
|
88
92
|
def ==(other)
|
|
89
|
-
|
|
93
|
+
to_h == other.to_h
|
|
90
94
|
end
|
|
91
95
|
|
|
92
96
|
# Create a new topic partition list based of a native one.
|
|
@@ -105,15 +109,15 @@ module Rdkafka
|
|
|
105
109
|
native_tpl[:cnt].times do |i|
|
|
106
110
|
ptr = native_tpl[:elems] + (i * Rdkafka::Bindings::TopicPartition.size)
|
|
107
111
|
elem = Rdkafka::Bindings::TopicPartition.new(ptr)
|
|
108
|
-
if elem[:partition] ==
|
|
112
|
+
if elem[:partition] == Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
|
|
109
113
|
data[elem[:topic]] = nil
|
|
110
114
|
else
|
|
111
115
|
partitions = data[elem[:topic]] || []
|
|
112
116
|
offset = if elem[:offset] == Rdkafka::Bindings::RD_KAFKA_OFFSET_INVALID
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
+
nil
|
|
118
|
+
else
|
|
119
|
+
elem[:offset]
|
|
120
|
+
end
|
|
117
121
|
partition = Partition.new(elem[:partition], offset, elem[:err])
|
|
118
122
|
partitions.push(partition)
|
|
119
123
|
data[elem[:topic]] = partitions
|
|
@@ -157,7 +161,7 @@ module Rdkafka
|
|
|
157
161
|
Rdkafka::Bindings.rd_kafka_topic_partition_list_add(
|
|
158
162
|
tpl,
|
|
159
163
|
topic,
|
|
160
|
-
|
|
164
|
+
Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
|
|
161
165
|
)
|
|
162
166
|
end
|
|
163
167
|
end
|