karafka 2.0.13 → 2.0.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0a38f7b176a40f09dc07c58b8abc8b13863f66fc2f3bff4d61fcd889c0bbb213
-  data.tar.gz: 9962f21cb52e566843e4b6994da8ac182752523a937cffa18bd735eefadc409f
+  metadata.gz: 5d21c47bcabdc50e520f7eb81f17fd31f00916a902aba6e596a2a70e1586a927
+  data.tar.gz: eff93034c3d6275d067c5b29b1b7b8d7ad2fb8cca73b08bd64398917338077af
 SHA512:
-  metadata.gz: 5b722f8b0baced05f35e907dadb523c129fe12f64066f5efe9bd81fdc0634ccfb824d463baa8f34b7918410c3f6e8a451ceaf5645bb08a8adeaaccff99ec29f1
-  data.tar.gz: d2649dda560f9c171f804da107df33c1a7fdf81c21bb92d33b7b967aee950a042868c566b63fb32cd495d870e4164246f8ee15aca7c169c66fb82f0994b321ff
+  metadata.gz: adc869f03b9f3774f9c5f4351980be5f12f36606242da7062f8abe2886d85fb8576fa10c8f2615f4e0e30a3329abd27b943ae4bbcfe842607e32d26b988ef58a
+  data.tar.gz: a7412443e62cb84dcba452aedd230769860c1a8ecb84ede8c34d20b47995117fae22cb60598190436347f50f38811b0a05d55e4219b4b41c3dc382a0f5117511

data/.github/workflows/ci.yml

@@ -18,7 +18,7 @@ jobs:
     strategy:
       fail-fast: false
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           fetch-depth: 0
 
@@ -39,7 +39,7 @@ jobs:
     strategy:
       fail-fast: false
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           fetch-depth: 0
       - name: Run Coditsu
@@ -64,7 +64,7 @@ jobs:
           - ruby: '3.1'
             coverage: 'true'
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Install package dependencies
         run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
 
@@ -97,7 +97,7 @@ jobs:
           - ruby: '3.1'
             coverage: 'true'
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Install package dependencies
         run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
 

data/.rspec

@@ -1 +1,3 @@
 --require spec_helper
+# Integration specs run with their one framework, not via RSpec
+--exclude-pattern "spec/integrations/**/*_spec.rb"

data/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Karafka framework changelog
 
+## 2.0.14 (2022-10-16)
+- Prevent consecutive stop signals from starting multiple supervision shutdowns.
+- Provide `Karafka::Embedded` to simplify the start/stop process when running Karafka from within other process (Puma, Sidekiq, etc).
+- Fix a race condition when un-pausing a long-running-job exactly upon listener resuming would crash the listener loop (#1072).
+
 ## 2.0.13 (2022-10-14)
 - Early exit upon attempts to commit current or earlier offset twice.
 - Add more integration specs covering edge cases.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.0.13)
+    karafka (2.0.14)
       karafka-core (>= 2.0.2, < 3.0.0)
       rdkafka (>= 0.12)
       thor (>= 0.20)

data/bin/integrations

@@ -24,7 +24,7 @@ ROOT_PATH = Pathname.new(File.expand_path(File.join(File.dirname(__FILE__), '../
 # When the value is high, there's a problem with thread allocation on Github CI, tht is why
 # we limit it. Locally we can run a lot of those, as many of them have sleeps and do not use a lot
 # of CPU
-CONCURRENCY = ENV.key?('CI') ? 3 : Etc.nprocessors * 2
+CONCURRENCY = ENV.key?('CI') ? 4 : Etc.nprocessors * 2
 
 # How may bytes do we want to keep from the stdout in the buffer for when we need to print it
 MAX_BUFFER_OUTPUT = 51_200
@@ -39,10 +39,10 @@ class Scenario
   # This includes exactly those
   EXIT_CODES = {
     default: [0],
-    'consumption/worker_critical_error_behaviour.rb' => [0, 2].freeze,
-    'shutdown/on_hanging_jobs_and_a_shutdown.rb' => [2].freeze,
-    'shutdown/on_hanging_on_shutdown_job_and_a_shutdown.rb' => [2].freeze,
-    'shutdown/on_hanging_listener_and_shutdown.rb' => [2].freeze
+    'consumption/worker_critical_error_behaviour_spec.rb' => [0, 2].freeze,
+    'shutdown/on_hanging_jobs_and_a_shutdown_spec.rb' => [2].freeze,
+    'shutdown/on_hanging_on_shutdown_job_and_a_shutdown_spec.rb' => [2].freeze,
+    'shutdown/on_hanging_listener_and_shutdown_spec.rb' => [2].freeze
   }.freeze
 
   private_constant :MAX_RUN_TIME, :EXIT_CODES
@@ -202,7 +202,7 @@ class Scenario
 end
 
 # Load all the specs
-specs = Dir[ROOT_PATH.join('spec/integrations/**/*.rb')]
+specs = Dir[ROOT_PATH.join('spec/integrations/**/*_spec.rb')]
 
 # If filters is provided, apply
 # Allows to provide several filters one after another and applies all of them
@@ -210,6 +210,7 @@ ARGV.each do |filter|
   specs.delete_if { |name| !name.include?(filter) }
 end
 
+
 raise ArgumentError, "No integration specs with filters: #{ARGV.join(', ')}" if specs.empty?
 
 # Randomize order

data/lib/karafka/embedded.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Allows to start and stop Karafka as part of a different process
+  module Embedded
+    class << self
+      # Starts Karafka without supervision and without ownership of signals in a background thread
+      # so it won't interrupt other things running
+      def start
+        Thread.new { Karafka::Server.start }
+      end
+
+      # Stops Karafka upon any event
+      #
+      # @note This method is blocking because we want to wait until Karafka is stopped with final
+      #   process shutdown
+      def stop
+        # Stop needs to be blocking to wait for all the things to finalize
+        Karafka::Server.stop
+      end
+    end
+  end
+end

data/lib/karafka/process.rb

@@ -29,12 +29,19 @@ module Karafka
     # Creates an instance of process and creates empty hash for callbacks
     def initialize
       @callbacks = Hash.new { |hsh, key| hsh[key] = [] }
+      @supervised = false
     end
 
     # Method catches all HANDLED_SIGNALS and performs appropriate callbacks (if defined)
     # @note If there are no callbacks, this method will just ignore a given signal that was sent
     def supervise
       HANDLED_SIGNALS.each { |signal| trap_signal(signal) }
+      @supervised = true
+    end
+
+    # Is the current process supervised and are trap signals installed
+    def supervised?
+      @supervised
     end
 
     private

data/lib/karafka/server.rb

@@ -31,6 +31,7 @@ module Karafka
         process.on_sigint { Thread.new { stop } }
         process.on_sigquit { Thread.new { stop } }
         process.on_sigterm { Thread.new { stop } }
+        process.supervise
 
         # Start is blocking until stop is called and when we stop, it will wait until
         # all of the things are ready to stop
@@ -61,7 +62,6 @@ module Karafka
       # @note We don't need to sleep because Karafka::Fetcher is locking and waiting to
       # finish loop (and it won't happen until we explicitly want to stop)
       def start
-        process.supervise
         Karafka::App.run!
         Karafka::Runner.new.call
       end
@@ -73,6 +73,9 @@ module Karafka
       #   lock them forever. If you need to run Karafka shutdown from within workers threads,
       #   please start a separate thread to do so.
       def stop
+        # Initialize the stopping process only if Karafka was running
+        return unless Karafka::App.running?
+
         Karafka::App.stop!
 
         timeout = Karafka::App.config.shutdown_timeout
@@ -110,8 +113,12 @@ module Karafka
 
         Karafka::App.producer.close
 
+        # We also do not forcefully terminate everything when running in the embedded mode,
+        # otherwise we would overwrite the shutdown process of the process that started Karafka
+        return unless process.supervised?
+
         # exit! is not within the instrumentation as it would not trigger due to exit
-        Kernel.exit! FORCEFUL_EXIT_CODE
+        Kernel.exit!(FORCEFUL_EXIT_CODE)
       ensure
         Karafka::App.stopped!
       end

data/lib/karafka/status.rb

@@ -25,6 +25,9 @@ module Karafka
       end
 
       define_method transition do
+        # Do nothing if the state change would change nothing (same state)
+        return if @status == state
+
         @status = state
 
         # Skip on creation (initializing)

data/lib/karafka/time_trackers/pause.rb

@@ -3,6 +3,12 @@
 module Karafka
   module TimeTrackers
     # Handles Kafka topic partition pausing and resuming with exponential back-offs.
+    # Since expiring and pausing can happen from both consumer and listener, this needs to be
+    # thread-safe.
+    #
+    # @note We do not have to worry about performance implications of a mutex wrapping most of the
+    #   code here, as this is not a frequently used tracker. It is active only once per batch in
+    #   case of long-running-jobs and upon errors.
     class Pause < Base
       attr_reader :count
 
@@ -36,6 +42,7 @@ module Karafka
         @timeout = timeout
         @max_timeout = max_timeout
         @exponential_backoff = exponential_backoff
+        @mutex = Mutex.new
         super()
       end
 
@@ -45,35 +52,47 @@ module Karafka
       # @note Providing this value can be useful when we explicitly want to pause for a certain
       #   period of time, outside of any regular pausing logic
       def pause(timeout = backoff_interval)
-        @started_at = now
-        @ends_at = @started_at + timeout
-        @count += 1
+        @mutex.synchronize do
+          @started_at = now
+          @ends_at = @started_at + timeout
+          @count += 1
+        end
       end
 
       # Marks the pause as resumed.
       def resume
-        @started_at = nil
-        @ends_at = nil
+        @mutex.synchronize do
+          @started_at = nil
+          @ends_at = nil
+        end
       end
 
       # Expires the pause, so it can be considered expired
       def expire
-        @ends_at = nil
+        @mutex.synchronize do
+          @ends_at = nil
+        end
       end
 
       # @return [Boolean] are we paused from processing
       def paused?
-        !@started_at.nil?
+        @mutex.synchronize do
+          !@started_at.nil?
+        end
       end
 
       # @return [Boolean] did the pause expire
       def expired?
-        @ends_at ? now >= @ends_at : true
+        @mutex.synchronize do
+          @ends_at ? now >= @ends_at : true
+        end
       end
 
       # Resets the pause counter.
       def reset
-        @count = 0
+        @mutex.synchronize do
+          @count = 0
+        end
       end
 
       private

data/lib/karafka/time_trackers/poll.rb

@@ -2,7 +2,8 @@
 
 module Karafka
   module TimeTrackers
-    # Object used to keep track of time we've used running certain operations.
+    # Object used to keep track of time we've used running certain operations. Polling is
+    # running in a single thread, thus we do not have to worry about this being thread-safe.
     #
     # @example Keep track of sleeping and stop after 3 seconds of 0.1 sleep intervals
     #   time_poll = Poll.new(3000)

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.13'
+  VERSION = '2.0.14'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.0.13
+  version: 2.0.14
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2022-10-14 00:00:00.000000000 Z
+date: 2022-10-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -188,6 +188,7 @@ files:
 - lib/karafka/contracts/consumer_group.rb
 - lib/karafka/contracts/consumer_group_topic.rb
 - lib/karafka/contracts/server_cli_options.rb
+- lib/karafka/embedded.rb
 - lib/karafka/env.rb
 - lib/karafka/errors.rb
 - lib/karafka/helpers/async.rb