karafka-rdkafka 0.14.8 → 0.14.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +7 -0
- data/karafka-rdkafka.gemspec +2 -2
- data/lib/rdkafka/bindings.rb +2 -0
- data/lib/rdkafka/config.rb +24 -7
- data/lib/rdkafka/producer.rb +14 -7
- data/lib/rdkafka/version.rb +1 -1
- data/spec/rdkafka/config_spec.rb +20 -0
- data/spec/rdkafka/consumer_spec.rb +5 -0
- data.tar.gz.sig +0 -0
- metadata +4 -3
- metadata.gz.sig +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 92d61e2b492453bf19ead6abf1c9377d5222aeba36c19c25caff5641a2c8fb1b
|
4
|
+
data.tar.gz: 4170931c8ced8d09813d22359c36d36da09b3f39d2429daad6a342dd3df2982c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ceae2da64aad6589779b60160fc28b5db6bc305c289f76bbf7f0aa034afdf88277b3aa75f4c14ec8b441250e81bca7bf7afededea22c2eb0568753cf6968c2b3
|
7
|
+
data.tar.gz: 61a3c8a40bca6d598782b093f6a2575fcdbe247d6a31c624f95e236015d713e3d2ace38e2fae43b622b9e4c6b37be57ef0f2d508a0a57310d3eeb79404146436
|
checksums.yaml.gz.sig
CHANGED
Binary file
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,12 @@
|
|
1
1
|
# Rdkafka Changelog
|
2
2
|
|
3
|
+
## 0.14.10 (2024-02-08)
|
4
|
+
- [Fix] Background logger stops working after forking causing memory leaks (mensfeld).
|
5
|
+
|
6
|
+
## 0.14.9 (2024-01-29)
|
7
|
+
- [Fix] Partition cache caches invalid `nil` result for `PARTITIONS_COUNT_TTL`.
|
8
|
+
- [Enhancement] Report `-1` instead of `nil` in case `partition_count` failure.
|
9
|
+
|
3
10
|
## 0.14.8 (2024-01-24)
|
4
11
|
- [Enhancement] Provide support for Nix OS (alexandriainfantino)
|
5
12
|
- [Enhancement] Skip intermediate array creation on delivery report callback execution (one per message) (mensfeld)
|
data/karafka-rdkafka.gemspec
CHANGED
@@ -3,10 +3,10 @@
|
|
3
3
|
require File.expand_path('lib/rdkafka/version', __dir__)
|
4
4
|
|
5
5
|
Gem::Specification.new do |gem|
|
6
|
-
gem.authors = ['Thijs Cadier']
|
6
|
+
gem.authors = ['Thijs Cadier', 'Maciej Mensfeld']
|
7
7
|
gem.email = ["contact@karafka.io"]
|
8
8
|
gem.description = "Modern Kafka client library for Ruby based on librdkafka"
|
9
|
-
gem.summary = "The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka. It wraps the production-ready C client using the ffi gem and targets Kafka 1.0+ and Ruby 2.
|
9
|
+
gem.summary = "The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka. It wraps the production-ready C client using the ffi gem and targets Kafka 1.0+ and Ruby 2.7+."
|
10
10
|
gem.license = 'MIT'
|
11
11
|
|
12
12
|
gem.files = `git ls-files`.split($\)
|
data/lib/rdkafka/bindings.rb
CHANGED
data/lib/rdkafka/config.rb
CHANGED
@@ -15,13 +15,12 @@ module Rdkafka
|
|
15
15
|
@@opaques = ObjectSpace::WeakMap.new
|
16
16
|
# @private
|
17
17
|
@@log_queue = Queue.new
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
end
|
18
|
+
# @private
|
19
|
+
# We memoize thread on the first log flush
|
20
|
+
# This allows us also to restart logger thread on forks
|
21
|
+
@@log_thread = nil
|
22
|
+
# @private
|
23
|
+
@@log_mutex = Mutex.new
|
25
24
|
|
26
25
|
# Returns the current logger, by default this is a logger to stdout.
|
27
26
|
#
|
@@ -30,6 +29,24 @@ module Rdkafka
|
|
30
29
|
@@logger
|
31
30
|
end
|
32
31
|
|
32
|
+
# Makes sure that there is a thread for consuming logs
|
33
|
+
# We do not spawn thread immediately and we need to check if it operates to support forking
|
34
|
+
def self.ensure_log_thread
|
35
|
+
return if @@log_thread && @@log_thread.alive?
|
36
|
+
|
37
|
+
@@log_mutex.synchronize do
|
38
|
+
# Restart if dead (fork, crash)
|
39
|
+
@@log_thread = nil if @@log_thread && !@@log_thread.alive?
|
40
|
+
|
41
|
+
@@log_thread ||= Thread.start do
|
42
|
+
loop do
|
43
|
+
severity, msg = @@log_queue.pop
|
44
|
+
@@logger.add(severity, msg)
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
33
50
|
# Returns a queue whose contents will be passed to the configured logger. Each entry
|
34
51
|
# should follow the format [Logger::Severity, String]. The benefit over calling the
|
35
52
|
# logger directly is that this is safe to use from trap contexts.
|
data/lib/rdkafka/producer.rb
CHANGED
@@ -40,10 +40,16 @@ module Rdkafka
|
|
40
40
|
topic_metadata = ::Rdkafka::Metadata.new(inner, topic).topics&.first
|
41
41
|
end
|
42
42
|
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
43
|
+
partition_count = topic_metadata ? topic_metadata[:partition_count] : -1
|
44
|
+
|
45
|
+
# This approach caches the failure to fetch only for 1 second. This will make sure, that
|
46
|
+
# we do not cache the failure for too long but also "buys" us a bit of time in case there
|
47
|
+
# would be issues in the cluster so we won't overaload it with consecutive requests
|
48
|
+
cache[topic] = if partition_count.positive?
|
49
|
+
[monotonic_now, partition_count]
|
50
|
+
else
|
51
|
+
[monotonic_now - PARTITIONS_COUNT_TTL + 5, partition_count]
|
52
|
+
end
|
47
53
|
end
|
48
54
|
end
|
49
55
|
|
@@ -199,14 +205,15 @@ module Rdkafka
|
|
199
205
|
# Partition count for a given topic.
|
200
206
|
#
|
201
207
|
# @param topic [String] The topic name.
|
202
|
-
# @return [Integer] partition count for a given topic
|
208
|
+
# @return [Integer] partition count for a given topic or `-1` if it could not be obtained.
|
203
209
|
#
|
204
210
|
# @note If 'allow.auto.create.topics' is set to true in the broker, the topic will be
|
205
211
|
# auto-created after returning nil.
|
206
212
|
#
|
207
213
|
# @note We cache the partition count for a given topic for given time.
|
208
214
|
# This prevents us in case someone uses `partition_key` from querying for the count with
|
209
|
-
# each message. Instead we query once every 30 seconds at most
|
215
|
+
# each message. Instead we query once every 30 seconds at most if we have a valid partition
|
216
|
+
# count or every 5 seconds in case we were not able to obtain number of partitions
|
210
217
|
def partition_count(topic)
|
211
218
|
closed_producer_check(__method__)
|
212
219
|
|
@@ -256,7 +263,7 @@ module Rdkafka
|
|
256
263
|
if partition_key
|
257
264
|
partition_count = partition_count(topic)
|
258
265
|
# If the topic is not present, set to -1
|
259
|
-
partition = Rdkafka::Bindings.partitioner(partition_key, partition_count, @partitioner_name) if partition_count
|
266
|
+
partition = Rdkafka::Bindings.partitioner(partition_key, partition_count, @partitioner_name) if partition_count.positive?
|
260
267
|
end
|
261
268
|
|
262
269
|
# If partition is nil, use -1 to let librdafka set the partition randomly or
|
data/lib/rdkafka/version.rb
CHANGED
data/spec/rdkafka/config_spec.rb
CHANGED
@@ -22,6 +22,7 @@ describe Rdkafka::Config do
|
|
22
22
|
it "supports logging queue" do
|
23
23
|
log = StringIO.new
|
24
24
|
Rdkafka::Config.logger = Logger.new(log)
|
25
|
+
Rdkafka::Config.ensure_log_thread
|
25
26
|
|
26
27
|
Rdkafka::Config.log_queue << [Logger::FATAL, "I love testing"]
|
27
28
|
20.times do
|
@@ -31,6 +32,25 @@ describe Rdkafka::Config do
|
|
31
32
|
|
32
33
|
expect(log.string).to include "FATAL -- : I love testing"
|
33
34
|
end
|
35
|
+
|
36
|
+
it "expect to start new logger thread after fork and work" do
|
37
|
+
reader, writer = IO.pipe
|
38
|
+
|
39
|
+
pid = fork do
|
40
|
+
$stdout.reopen(writer)
|
41
|
+
Rdkafka::Config.logger = Logger.new($stdout)
|
42
|
+
reader.close
|
43
|
+
producer = rdkafka_producer_config(debug: 'all').producer
|
44
|
+
producer.close
|
45
|
+
writer.close
|
46
|
+
sleep(1)
|
47
|
+
end
|
48
|
+
|
49
|
+
writer.close
|
50
|
+
Process.wait(pid)
|
51
|
+
output = reader.read
|
52
|
+
expect(output.split("\n").size).to be >= 20
|
53
|
+
end
|
34
54
|
end
|
35
55
|
|
36
56
|
context "statistics callback" do
|
@@ -211,6 +211,11 @@ describe Rdkafka::Consumer do
|
|
211
211
|
|
212
212
|
# 7. ensure same message is read again
|
213
213
|
message2 = consumer.poll(timeout)
|
214
|
+
|
215
|
+
# This is needed because `enable.auto.offset.store` is true but when running in CI that
|
216
|
+
# is overloaded, offset store lags
|
217
|
+
sleep(1)
|
218
|
+
|
214
219
|
consumer.commit
|
215
220
|
expect(message1.offset).to eq message2.offset
|
216
221
|
expect(message1.payload).to eq message2.payload
|
data.tar.gz.sig
CHANGED
Binary file
|
metadata
CHANGED
@@ -1,10 +1,11 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: karafka-rdkafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.14.
|
4
|
+
version: 0.14.10
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Thijs Cadier
|
8
|
+
- Maciej Mensfeld
|
8
9
|
autorequire:
|
9
10
|
bindir: bin
|
10
11
|
cert_chain:
|
@@ -35,7 +36,7 @@ cert_chain:
|
|
35
36
|
AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
|
36
37
|
msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
|
37
38
|
-----END CERTIFICATE-----
|
38
|
-
date: 2024-
|
39
|
+
date: 2024-02-08 00:00:00.000000000 Z
|
39
40
|
dependencies:
|
40
41
|
- !ruby/object:Gem::Dependency
|
41
42
|
name: ffi
|
@@ -282,7 +283,7 @@ signing_key:
|
|
282
283
|
specification_version: 4
|
283
284
|
summary: The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka.
|
284
285
|
It wraps the production-ready C client using the ffi gem and targets Kafka 1.0+
|
285
|
-
and Ruby 2.
|
286
|
+
and Ruby 2.7+.
|
286
287
|
test_files:
|
287
288
|
- spec/rdkafka/abstract_handle_spec.rb
|
288
289
|
- spec/rdkafka/admin/create_acl_handle_spec.rb
|
metadata.gz.sig
CHANGED
Binary file
|