karafka 2.0.0.rc4 → 2.0.0.rc5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 526402b906d00f844c5b25925a854bc45f5127dc8c06646f1cd1432f265dfb81
-  data.tar.gz: c58f6be491c4c4e82237307d37874100588c97241439788325600bea77d4254f
+  metadata.gz: 52fde84aac9ffef63ecbc1d367e3bfdc7209f6c2904f2a7142f51b1bd0613002
+  data.tar.gz: 4b4e7838e19505469ec20a654a3d7bb9c2588f351f9ea64fdb05e224b476d562
 SHA512:
-  metadata.gz: '08922eef9890af84f1b444329061283f7987258200a9e338b892944a6ca061a60abed273e02acf44f91728f0ed219ec494e96c05a1155db2321e941e24c8c328'
-  data.tar.gz: 5b3de319f1887af51eeda3ffc5b42a86601a382a55ab30e8cc7abce0526d4471ff274b3627250d47ec9a70de763fd337a7c8eff79add2b9409c935133bbe3610
+  metadata.gz: 0ebdda184eca4c7dcfe39821125ba7a8645336607300395de792a7c8808cde07f54f5382278946fab58af0e95f3a85ab05be9ed6e7b292fc7692922cb7890747
+  data.tar.gz: 416cb01ed1b6dac0681969ce72505d19855a041e7ceb29e369ed9fc80675755ad1f478a20ee2d1f2c0e925036ba6f08adda727f8ecd3f4f07f39f0be347ae4a1

data/CHANGELOG.md

@@ -1,8 +1,18 @@
 # Karafka framework changelog
 
+## 2.0.0.rc5 (2022-08-01)
+- Improve specs stability
+- Improve forceful shutdown
+- Add support for debug `TTIN` backtrace printing
+- Fix a case where logger listener would not intercept `warn` level
+- Require `rdkafka` >= `0.12`
+- Replace statistics decorator with the one from `karafka-core`
+
 ## 2.0.0.rc4 (2022-07-28)
 - Remove `dry-monitor`
 - Use `karafka-core`
+- Improve forceful shutdown resources finalization
+- Cache consumer client name
 
 ## 2.0.0.rc3 (2022-07-26)
 - Fix Pro partitioner hash function may not utilize all the threads (#907).
@@ -69,8 +79,8 @@
 - Fix a case where consecutive CTRL+C (non-stop) would case an exception during forced shutdown
 - Add missing `consumer.prepared.error` into `LoggerListener`
 - Delegate partition resuming from the consumers to listeners threads.
-- Add support for Long Running Jobs (LRJ) for ActiveJob [PRO]
-- Add support for Long Running Jobs for consumers [PRO]
+- Add support for Long-Running Jobs (LRJ) for ActiveJob [PRO]
+- Add support for Long-Running Jobs for consumers [PRO]
 - Allow `active_job_topic` to accept a block for extra topic related settings
 - Remove no longer needed logger threads
 - Auto-adapt number of processes for integration specs based on the number of CPUs

data/Gemfile.lock

@@ -1,11 +1,11 @@
 PATH
   remote: .
   specs:
-    karafka (2.0.0.rc4)
-      karafka-core (>= 2.0.0, < 3.0.0)
-      rdkafka (>= 0.10)
+    karafka (2.0.0.rc5)
+      karafka-core (>= 2.0.2, < 3.0.0)
+      rdkafka (>= 0.12)
       thor (>= 0.20)
-      waterdrop (>= 2.4.0, < 3.0.0)
+      waterdrop (>= 2.4.1, < 3.0.0)
       zeitwerk (~> 2.3)
 
 GEM
@@ -30,7 +30,7 @@ GEM
       activesupport (>= 5.0)
     i18n (1.12.0)
       concurrent-ruby (~> 1.0)
-    karafka-core (2.0.0)
+    karafka-core (2.0.2)
       concurrent-ruby (>= 1.1)
     mini_portile2 (2.8.0)
     minitest (5.16.2)
@@ -61,8 +61,8 @@ GEM
     thor (1.2.1)
     tzinfo (2.0.5)
       concurrent-ruby (~> 1.0)
-    waterdrop (2.4.0)
-      karafka-core (~> 2.0)
+    waterdrop (2.4.1)
+      karafka-core (>= 2.0.2, < 3.0.0)
       rdkafka (>= 0.10)
       zeitwerk (~> 2.3)
     zeitwerk (2.6.0)

data/README.md

@@ -4,7 +4,7 @@
 [![Gem Version](https://badge.fury.io/rb/karafka.svg)](http://badge.fury.io/rb/karafka)
 [![Join the chat at https://slack.karafka.io](https://raw.githubusercontent.com/karafka/misc/master/slack.svg)](https://slack.karafka.io)
 
-**Note**: All of the documentation here refers to Karafka `2.0.0.rc3` or higher. If you are looking for the documentation for Karafka `1.4`, please click [here](https://github.com/karafka/wiki/tree/1.4).
+**Note**: All of the documentation here refers to Karafka `2.0.0.rc4` or higher. If you are looking for the documentation for Karafka `1.4`, please click [here](https://github.com/karafka/wiki/tree/1.4).
 
 ## About Karafka
 
@@ -55,7 +55,7 @@ We also maintain many [integration specs](https://github.com/karafka/karafka/tre
 1. Add and install Karafka:
 
 ```bash
-bundle add karafka -v 2.0.0.rc3
+bundle add karafka -v 2.0.0.rc4
 
 bundle exec karafka install
 ```

data/bin/integrations

@@ -37,7 +37,7 @@ class Scenario
     'consumption/worker_critical_error_behaviour.rb' => [0, 2].freeze,
     'shutdown/on_hanging_jobs_and_a_shutdown.rb' => [2].freeze,
     'shutdown/on_hanging_on_shutdown_job_and_a_shutdown.rb' => [2].freeze,
-    'shutdown/on_hanging_poll_and_shutdown.rb' => [2].freeze
+    'shutdown/on_hanging_listener_and_shutdown.rb' => [2].freeze
   }.freeze
 
   private_constant :MAX_RUN_TIME, :EXIT_CODES

data/karafka.gemspec

@@ -16,10 +16,10 @@ Gem::Specification.new do |spec|
   spec.description = 'Framework used to simplify Apache Kafka based Ruby applications development'
   spec.licenses    = ['LGPL-3.0', 'Commercial']
 
-  spec.add_dependency 'karafka-core', '>= 2.0.0', '< 3.0.0'
-  spec.add_dependency 'rdkafka', '>= 0.10'
+  spec.add_dependency 'karafka-core', '>= 2.0.2', '< 3.0.0'
+  spec.add_dependency 'rdkafka', '>= 0.12'
   spec.add_dependency 'thor', '>= 0.20'
-  spec.add_dependency 'waterdrop', '>= 2.4.0', '< 3.0.0'
+  spec.add_dependency 'waterdrop', '>= 2.4.1', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 
   spec.required_ruby_version = '>= 2.7.0'

data/lib/karafka/connection/client.rb

@@ -275,6 +275,11 @@ module Karafka
 
       # Commits the stored offsets in a sync way and closes the consumer.
       def close
+        # Once client is closed, we should not close it again
+        # This could only happen in case of a race-condition when forceful shutdown happens
+        # and triggers this from a different thread
+        return if @closed
+
         @mutex.synchronize do
           internal_commit_offsets(async: false)
 

data/lib/karafka/connection/listener.rb

@@ -34,6 +34,8 @@ module Karafka
         # We can do this that way because we always first schedule jobs using messages before we
         # fetch another batch.
         @messages_buffer = MessagesBuffer.new(subscription_group)
+        @mutex = Mutex.new
+        @stopped = false
       end
 
       # Runs the main listener fetch loop.
@@ -51,6 +53,25 @@ module Karafka
         fetch_loop
       end
 
+      # Stops the jobs queue, triggers shutdown on all the executors (sync), commits offsets and
+      # stops kafka client.
+      #
+      # @note This method is not private despite being part of the fetch loop because in case of
+      #   a forceful shutdown, it may be invoked from a separate thread
+      #
+      # @note We wrap it with a mutex exactly because of the above case of forceful shutdown
+      def shutdown
+        return if @stopped
+
+        @mutex.synchronize do
+          @stopped = true
+          @executors.clear
+          @coordinators.reset
+          @client.commit_offsets!
+          @client.stop
+        end
+      end
+
       private
 
       # Fetches the data and adds it to the jobs queue.
@@ -239,13 +260,6 @@ module Karafka
         @client.batch_poll until @jobs_queue.empty?(@subscription_group.id)
       end
 
-      # Stops the jobs queue, triggers shutdown on all the executors (sync), commits offsets and
-      # stops kafka client.
-      def shutdown
-        @client.commit_offsets!
-        @client.stop
-      end
-
       # We can stop client without a problem, as it will reinitialize itself when running the
       # `#fetch_loop` again. We just need to remember to also reset the runner as it is a long
       # running one, so with a new connection to Kafka, we need to initialize the state of the

data/lib/karafka/instrumentation/callbacks/statistics.rb

@@ -16,8 +16,7 @@ module Karafka
           @consumer_group_id = consumer_group_id
           @client_name = client_name
           @monitor = monitor
-          # We decorate both Karafka and WaterDrop statistics the same way
-          @statistics_decorator = ::WaterDrop::Instrumentation::Callbacks::StatisticsDecorator.new
+          @statistics_decorator = ::Karafka::Core::Monitoring::StatisticsDecorator.new
         end
 
         # Emits decorated statistics to the monitor

data/lib/karafka/instrumentation/logger_listener.rb

@@ -9,6 +9,7 @@ module Karafka
       USED_LOG_LEVELS = %i[
         debug
         info
+        warn
         error
         fatal
       ].freeze
@@ -60,11 +61,28 @@ module Karafka
         info "[#{job.id}] #{job_type} job for #{consumer} on #{topic} finished in #{time}ms"
       end
 
-      # Logs info about system signals that Karafka received.
+      # Logs info about system signals that Karafka received and prints backtrace for threads in
+      # case of ttin
       #
       # @param event [Dry::Events::Event] event details including payload
       def on_process_notice_signal(event)
         info "Received #{event[:signal]} system signal"
+
+        # We print backtrace only for ttin
+        return unless event[:signal] == :SIGTTIN
+
+        # Inspired by Sidekiq
+        Thread.list.each do |thread|
+          tid = (thread.object_id ^ ::Process.pid).to_s(36)
+
+          warn "Thread TID-#{tid} #{thread['label']}"
+
+          if thread.backtrace
+            warn thread.backtrace.join("\n")
+          else
+            warn '<no backtrace available>'
+          end
+        end
       end
 
       # Logs info that we're initializing Karafka app.

data/lib/karafka/patches/rdkafka/consumer.rb

@@ -12,7 +12,7 @@ module Karafka
         # @note We need this to make sure that we allocate proper dispatched events only to
         #   callback listeners that should publish them
         def name
-          ::Rdkafka::Bindings.rd_kafka_name(@native_kafka)
+          @name ||= ::Rdkafka::Bindings.rd_kafka_name(@native_kafka)
         end
       end
     end

data/lib/karafka/pro/base_consumer.rb

@@ -36,7 +36,7 @@ module Karafka
         end
       end
 
-      # Runs extra logic after consumption that is related to handling long running jobs
+      # Runs extra logic after consumption that is related to handling long-running jobs
       # @note This overwrites the '#on_after_consume' from the base consumer
       def on_after_consume
         coordinator.on_finished do |first_group_message, last_group_message|
@@ -59,7 +59,7 @@ module Karafka
           # Mark as consumed only if manual offset management is not on
           mark_as_consumed(last_message) unless topic.manual_offset_management? || revoked?
 
-          # If this is not a long running job there is nothing for us to do here
+          # If this is not a long-running job there is nothing for us to do here
           return unless topic.long_running_job?
 
           # Once processing is done, we move to the new offset based on commits

data/lib/karafka/pro/routing/topic_extensions.rb

@@ -28,7 +28,7 @@ module Karafka
           virtual_partitioner != nil
         end
 
-        # @return [Boolean] is a given job on a topic a long running one
+        # @return [Boolean] is a given job on a topic a long-running one
         def long_running_job?
           @long_running_job || false
         end

data/lib/karafka/process.rb

@@ -9,6 +9,7 @@ module Karafka
       SIGINT
       SIGQUIT
       SIGTERM
+      SIGTTIN
     ].freeze
 
     HANDLED_SIGNALS.each do |signal|

data/lib/karafka/server.rb

@@ -104,6 +104,9 @@ module Karafka
         # We're done waiting, lets kill them!
         workers.each(&:terminate)
         listeners.each(&:terminate)
+        # We always need to shutdown clients to make sure we do not force the GC to close consumer.
+        # This can cause memory leaks and crashes.
+        listeners.each(&:shutdown)
 
         Karafka::App.producer.close
 

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.0.rc4'
+  VERSION = '2.0.0.rc5'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.0.0.rc4
+  version: 2.0.0.rc5
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -34,7 +34,7 @@ cert_chain:
   R2P11bWoCtr70BsccVrN8jEhzwXngMyI2gVt750Y+dbTu1KgRqZKp/ECe7ZzPzXj
   pIy9vHxTANKYVyI4qj8OrFdEM5BQNu8oQpL0iQ==
   -----END CERTIFICATE-----
-date: 2022-07-28 00:00:00.000000000 Z
+date: 2022-08-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -42,7 +42,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: 2.0.2
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -52,7 +52,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: 2.0.2
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -62,14 +62,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0.10'
+        version: '0.12'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0.10'
+        version: '0.12'
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
@@ -90,7 +90,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.4.0
+        version: 2.4.1
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -100,7 +100,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.4.0
+        version: 2.4.1
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0