karafka 2.0.0 → 2.1.0

data/lib/karafka/time_trackers/pause.rb

@@ -3,8 +3,14 @@
 module Karafka
   module TimeTrackers
     # Handles Kafka topic partition pausing and resuming with exponential back-offs.
+    # Since expiring and pausing can happen from both consumer and listener, this needs to be
+    # thread-safe.
+    #
+    # @note We do not have to worry about performance implications of a mutex wrapping most of the
+    #   code here, as this is not a frequently used tracker. It is active only once per batch in
+    #   case of long-running-jobs and upon errors.
     class Pause < Base
-      attr_reader :count
+      attr_reader :attempt, :current_timeout
 
       # @param timeout [Integer] how long should we wait when anything went wrong (in ms)
       # @param max_timeout [Integer, nil] if exponential is on, what is the max value we can reach
@@ -13,67 +19,91 @@ module Karafka
       #   timeout value
       # @return [Karafka::TimeTrackers::Pause]
       # @example
-      #   pause = Karafka::TimeTrackers::Pause.new(timeout: 1000)
+      #   options = { timeout: 1000, max_timeout: 1000, exponential_backoff: false }
+      #   pause = Karafka::TimeTrackers::Pause.new(**options)
       #   pause.expired? #=> true
       #   pause.paused? #=> false
       #   pause.pause
+      #   pause.increment
       #   sleep(1.1)
       #   pause.paused? #=> true
       #   pause.expired? #=> true
-      #   pause.count #=> 1
+      #   pause.attempt #=> 1
       #   pause.pause
-      #   pause.count #=> 1
+      #   pause.increment
+      #   pause.attempt #=> 2
       #   pause.paused? #=> true
       #   pause.expired? #=> false
       #   pause.resume
-      #   pause.count #=> 2
+      #   pause.attempt #=> 2
       #   pause.paused? #=> false
       #   pause.reset
-      #   pause.count #=> 0
+      #   pause.attempt #=> 0
       def initialize(timeout:, max_timeout:, exponential_backoff:)
         @started_at = nil
-        @count = 0
+        @attempt = 0
         @timeout = timeout
+        @current_timeout = timeout
         @max_timeout = max_timeout
         @exponential_backoff = exponential_backoff
+        @mutex = Mutex.new
         super()
       end
 
       # Pauses the processing from now till the end of the interval (backoff or non-backoff)
-      # and records the count.
+      # and records the attempt.
       # @param timeout [Integer] timeout value in milliseconds that overwrites the default timeout
       # @note Providing this value can be useful when we explicitly want to pause for a certain
       #   period of time, outside of any regular pausing logic
       def pause(timeout = backoff_interval)
-        @started_at = now
-        @ends_at = @started_at + timeout
-        @count += 1
+        @mutex.synchronize do
+          @current_timeout = timeout
+          @started_at = monotonic_now
+          @ends_at = @started_at + timeout
+        end
+      end
+
+      # Increments the number of attempt by 1
+      def increment
+        @mutex.synchronize do
+          @attempt += 1
+        end
       end
 
       # Marks the pause as resumed.
       def resume
-        @started_at = nil
-        @ends_at = nil
+        @mutex.synchronize do
+          @started_at = nil
+          @ends_at = nil
+        end
       end
 
       # Expires the pause, so it can be considered expired
       def expire
-        @ends_at = nil
+        @mutex.synchronize do
+          @ends_at = nil
+        end
       end
 
       # @return [Boolean] are we paused from processing
       def paused?
-        !@started_at.nil?
+        @mutex.synchronize do
+          !@started_at.nil?
+        end
       end
 
       # @return [Boolean] did the pause expire
       def expired?
-        @ends_at ? now >= @ends_at : true
+        @mutex.synchronize do
+          @ends_at ? monotonic_now >= @ends_at : true
+        end
       end
 
-      # Resets the pause counter.
+      # Resets the pause attempt count.
       def reset
-        @count = 0
+        @mutex.synchronize do
+          @attempt = 0
+        end
       end
 
       private
@@ -81,7 +111,7 @@ module Karafka
       # Computers the exponential backoff
       # @return [Integer] backoff in milliseconds
       def backoff_interval
-        backoff_factor = @exponential_backoff ? 2**@count : 1
+        backoff_factor = @exponential_backoff ? 2**@attempt : 1
 
         timeout = backoff_factor * @timeout
 

data/lib/karafka/time_trackers/poll.rb

@@ -2,7 +2,8 @@
 
 module Karafka
   module TimeTrackers
-    # Object used to keep track of time we've used running certain operations.
+    # Object used to keep track of time we've used running certain operations. Polling is
+    # running in a single thread, thus we do not have to worry about this being thread-safe.
     #
     # @example Keep track of sleeping and stop after 3 seconds of 0.1 sleep intervals
     #   time_poll = Poll.new(3000)
@@ -33,12 +34,12 @@ module Karafka
       # Starts time tracking.
       def start
         @attempts += 1
-        @started_at = now
+        @started_at = monotonic_now
       end
 
       # Stops time tracking of a given piece of code and updates the remaining time.
       def checkpoint
-        @remaining -= (now - @started_at)
+        @remaining -= (monotonic_now - @started_at)
       end
 
       # @return [Boolean] If anything went wrong, can we retry after a backoff period or not
@@ -50,6 +51,9 @@ module Karafka
       # Sleeps for amount of time matching attempt, so we sleep more with each attempt in case of
       #   a retry.
       def backoff
+        # backoff should not be included in the remaining time computation, otherwise it runs
+        # shortly, never back-offing beyond a small number because of the sleep
+        @remaining += backoff_interval
         # Sleep requires seconds not ms
         sleep(backoff_interval / 1_000.0)
       end

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.0'
+  VERSION = '2.1.0'
 end

data/lib/karafka.rb

@@ -13,7 +13,10 @@
   openssl
   base64
   date
+  singleton
+  digest
   zeitwerk
+  concurrent/atomic/atomic_fixnum
 ].each(&method(:require))
 
 # Karafka framework main namespace
@@ -67,6 +70,19 @@ module Karafka
       App.config.license.token != false
     end
 
+    # @return [Boolean] Do we run within/with Rails. We use this to initialize Railtie and proxy
+    #   the console invocation to Rails
+    def rails?
+      return @rails if instance_variable_defined?('@rails')
+
+      # Do not load Rails again if already loaded
+      Object.const_defined?('Rails::Railtie') || require('rails')
+
+      @rails = true
+    rescue LoadError
+      @rails = false
+    end
+
     # @return [String] path to a default file that contains booting procedure etc
     # @note By default it is a file called 'karafka.rb' but it can be specified as you wish if you
     #   have Karafka that is merged into a Sinatra/Rails app and karafka.rb is taken.
@@ -79,15 +95,52 @@ module Karafka
     def boot_file
       Pathname.new(ENV['KARAFKA_BOOT_FILE'] || File.join(Karafka.root, 'karafka.rb'))
     end
+
+    # We need to be able to overwrite both monitor and logger after the configuration in case they
+    # would be changed because those two (with defaults) can be used prior to the setup and their
+    # state change should be reflected in the updated setup
+    #
+    # This method refreshes the things that might have been altered by the configuration
+    def refresh!
+      config = ::Karafka::App.config
+
+      @logger = config.logger
+      @producer = config.producer
+      @monitor = config.monitor
+    end
   end
 end
 
 loader = Zeitwerk::Loader.for_gem
 # Do not load Rails extensions by default, this will be handled by Railtie if they are needed
 loader.ignore(Karafka.gem_root.join('lib/active_job'))
-# Do not load pro components, this will be handled by license manager
-loader.ignore(Karafka.gem_root.join('lib/karafka/pro'))
+# Do not load CurrentAttributes components as they will be loaded if needed
+# @note We have to exclude both the .rb file as well as the whole directory so users can require
+# current attributes only when needed
+loader.ignore(Karafka.gem_root.join('lib/karafka/active_job/current_attributes'))
+loader.ignore(Karafka.gem_root.join('lib/karafka/active_job/current_attributes.rb'))
+# Do not load Railtie. It will load if after everything is ready, so we don't have to load any
+# Karafka components when we require this railtie. Railtie needs to be loaded last.
+loader.ignore(Karafka.gem_root.join('lib/karafka/railtie'))
+# Do not load pro components as they will be loaded if needed and allowed
+loader.ignore(Karafka.core_root.join('pro/'))
 # Do not load vendors instrumentation components. Those need to be required manually if needed
-loader.ignore(Karafka.gem_root.join('lib/karafka/instrumentation/vendors'))
+loader.ignore(Karafka.core_root.join('instrumentation/vendors'))
 loader.setup
 loader.eager_load
+
+# This will load features but since Pro are not loaded automatically, they will not be visible
+# nor included here
+::Karafka::Routing::Features::Base.load_all
+
+# We need to detect and require (not setup) Pro components during the gem load, because we need
+# to make pro components available in case anyone wants to use them as a base to their own
+# custom components. Otherwise inheritance would not work.
+Karafka::Licenser.detect do
+  require 'karafka/pro/loader'
+
+  Karafka::Pro::Loader.require_all
+end
+
+# Load railtie after everything else is ready so we know we can rely on it.
+require 'karafka/railtie'

data/renovate.json

@@ -0,0 +1,6 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": [
+    "config:base"
+  ]
+}