karafka-rdkafka 0.14.9 → 0.14.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 8a3805cc154b0e1ca0d71fe06a852013c3e8d8dfd8148d4de8902aed66a2774d
4
- data.tar.gz: 908236ad1ab0be52549f01ddf90ed37355a3b6e9e3373b867eeb83fa05fdf39c
3
+ metadata.gz: 92d61e2b492453bf19ead6abf1c9377d5222aeba36c19c25caff5641a2c8fb1b
4
+ data.tar.gz: 4170931c8ced8d09813d22359c36d36da09b3f39d2429daad6a342dd3df2982c
5
5
  SHA512:
6
- metadata.gz: 6460881076e6c45ae5feb38a3cfd4da3160f13a092f6a7acdbbe268e297b10e27ad792dab32a8c193824a5ef44fc9c2f40c5685ee5c489370056f1c408297f83
7
- data.tar.gz: c2f42212e4df244d36f2312506dd69e52e8064c3485dc1fd0692d223ac9f2512a8b8102b7f473766386eb997dc7bb490fc4fc3c4a2815c528d6c1a8cddee7fcd
6
+ metadata.gz: ceae2da64aad6589779b60160fc28b5db6bc305c289f76bbf7f0aa034afdf88277b3aa75f4c14ec8b441250e81bca7bf7afededea22c2eb0568753cf6968c2b3
7
+ data.tar.gz: 61a3c8a40bca6d598782b093f6a2575fcdbe247d6a31c624f95e236015d713e3d2ace38e2fae43b622b9e4c6b37be57ef0f2d508a0a57310d3eeb79404146436
checksums.yaml.gz.sig CHANGED
Binary file
data/CHANGELOG.md CHANGED
@@ -1,5 +1,8 @@
1
1
  # Rdkafka Changelog
2
2
 
3
+ ## 0.14.10 (2024-02-08)
4
+ - [Fix] Background logger stops working after forking causing memory leaks (mensfeld).
5
+
3
6
  ## 0.14.9 (2024-01-29)
4
7
  - [Fix] Partition cache caches invalid `nil` result for `PARTITIONS_COUNT_TTL`.
5
8
  - [Enhancement] Report `-1` instead of `nil` in case `partition_count` failure.
@@ -3,10 +3,10 @@
3
3
  require File.expand_path('lib/rdkafka/version', __dir__)
4
4
 
5
5
  Gem::Specification.new do |gem|
6
- gem.authors = ['Thijs Cadier']
6
+ gem.authors = ['Thijs Cadier', 'Maciej Mensfeld']
7
7
  gem.email = ["contact@karafka.io"]
8
8
  gem.description = "Modern Kafka client library for Ruby based on librdkafka"
9
- gem.summary = "The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka. It wraps the production-ready C client using the ffi gem and targets Kafka 1.0+ and Ruby 2.4+."
9
+ gem.summary = "The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka. It wraps the production-ready C client using the ffi gem and targets Kafka 1.0+ and Ruby 2.7+."
10
10
  gem.license = 'MIT'
11
11
 
12
12
  gem.files = `git ls-files`.split($\)
@@ -147,6 +147,8 @@ module Rdkafka
147
147
  else
148
148
  Logger::UNKNOWN
149
149
  end
150
+
151
+ Rdkafka::Config.ensure_log_thread
150
152
  Rdkafka::Config.log_queue << [severity, "rdkafka: #{line}"]
151
153
  end
152
154
 
@@ -15,13 +15,12 @@ module Rdkafka
15
15
  @@opaques = ObjectSpace::WeakMap.new
16
16
  # @private
17
17
  @@log_queue = Queue.new
18
-
19
- Thread.start do
20
- loop do
21
- severity, msg = @@log_queue.pop
22
- @@logger.add(severity, msg)
23
- end
24
- end
18
+ # @private
19
+ # We memoize thread on the first log flush
20
+ # This allows us also to restart logger thread on forks
21
+ @@log_thread = nil
22
+ # @private
23
+ @@log_mutex = Mutex.new
25
24
 
26
25
  # Returns the current logger, by default this is a logger to stdout.
27
26
  #
@@ -30,6 +29,24 @@ module Rdkafka
30
29
  @@logger
31
30
  end
32
31
 
32
+ # Makes sure that there is a thread for consuming logs
33
+ # We do not spawn thread immediately and we need to check if it operates to support forking
34
+ def self.ensure_log_thread
35
+ return if @@log_thread && @@log_thread.alive?
36
+
37
+ @@log_mutex.synchronize do
38
+ # Restart if dead (fork, crash)
39
+ @@log_thread = nil if @@log_thread && !@@log_thread.alive?
40
+
41
+ @@log_thread ||= Thread.start do
42
+ loop do
43
+ severity, msg = @@log_queue.pop
44
+ @@logger.add(severity, msg)
45
+ end
46
+ end
47
+ end
48
+ end
49
+
33
50
  # Returns a queue whose contents will be passed to the configured logger. Each entry
34
51
  # should follow the format [Logger::Severity, String]. The benefit over calling the
35
52
  # logger directly is that this is safe to use from trap contexts.
@@ -1,7 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Rdkafka
4
- VERSION = "0.14.9"
4
+ VERSION = "0.14.10"
5
5
  LIBRDKAFKA_VERSION = "2.3.0"
6
6
  LIBRDKAFKA_SOURCE_SHA256 = "2d49c35c77eeb3d42fa61c43757fcbb6a206daa560247154e60642bcdcc14d12"
7
7
  end
@@ -22,6 +22,7 @@ describe Rdkafka::Config do
22
22
  it "supports logging queue" do
23
23
  log = StringIO.new
24
24
  Rdkafka::Config.logger = Logger.new(log)
25
+ Rdkafka::Config.ensure_log_thread
25
26
 
26
27
  Rdkafka::Config.log_queue << [Logger::FATAL, "I love testing"]
27
28
  20.times do
@@ -31,6 +32,25 @@ describe Rdkafka::Config do
31
32
 
32
33
  expect(log.string).to include "FATAL -- : I love testing"
33
34
  end
35
+
36
+ it "expect to start new logger thread after fork and work" do
37
+ reader, writer = IO.pipe
38
+
39
+ pid = fork do
40
+ $stdout.reopen(writer)
41
+ Rdkafka::Config.logger = Logger.new($stdout)
42
+ reader.close
43
+ producer = rdkafka_producer_config(debug: 'all').producer
44
+ producer.close
45
+ writer.close
46
+ sleep(1)
47
+ end
48
+
49
+ writer.close
50
+ Process.wait(pid)
51
+ output = reader.read
52
+ expect(output.split("\n").size).to be >= 20
53
+ end
34
54
  end
35
55
 
36
56
  context "statistics callback" do
@@ -211,6 +211,11 @@ describe Rdkafka::Consumer do
211
211
 
212
212
  # 7. ensure same message is read again
213
213
  message2 = consumer.poll(timeout)
214
+
215
+ # This is needed because `enable.auto.offset.store` is true but when running in CI that
216
+ # is overloaded, offset store lags
217
+ sleep(1)
218
+
214
219
  consumer.commit
215
220
  expect(message1.offset).to eq message2.offset
216
221
  expect(message1.payload).to eq message2.payload
data.tar.gz.sig CHANGED
Binary file
metadata CHANGED
@@ -1,10 +1,11 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: karafka-rdkafka
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.14.9
4
+ version: 0.14.10
5
5
  platform: ruby
6
6
  authors:
7
7
  - Thijs Cadier
8
+ - Maciej Mensfeld
8
9
  autorequire:
9
10
  bindir: bin
10
11
  cert_chain:
@@ -35,7 +36,7 @@ cert_chain:
35
36
  AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
36
37
  msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
37
38
  -----END CERTIFICATE-----
38
- date: 2024-01-29 00:00:00.000000000 Z
39
+ date: 2024-02-08 00:00:00.000000000 Z
39
40
  dependencies:
40
41
  - !ruby/object:Gem::Dependency
41
42
  name: ffi
@@ -282,7 +283,7 @@ signing_key:
282
283
  specification_version: 4
283
284
  summary: The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka.
284
285
  It wraps the production-ready C client using the ffi gem and targets Kafka 1.0+
285
- and Ruby 2.4+.
286
+ and Ruby 2.7+.
286
287
  test_files:
287
288
  - spec/rdkafka/abstract_handle_spec.rb
288
289
  - spec/rdkafka/admin/create_acl_handle_spec.rb
metadata.gz.sig CHANGED
@@ -1,4 +1,3 @@
1
- v�f).c̀K� ���^�K�aY��7�+�ju�E� e�:9��Qmi�V����R-�����Rm��.Ю0w��d��� -B�VR0+h��
2
- +��-��Go�ބ㵀,�Nl4u�c�8��\�|�n/�h�R�55��U4Ї�E rj���Qdz8� Y���ޘv���f���i^�����x����(볬�|�%SY�N����r񟊯���d"���4I1�6S���a��5l�4�k]�-��S��Jgz�A��I��l3�� X�p/����z
3
- y�(c�s�dA���,�
4
- F(���
1
+
2
+ zt���h Jv1]e�8��
3
+