karafka 2.0.23 → 2.0.26

data/lib/karafka/runner.rb

@@ -19,6 +19,7 @@ module Karafka
       # We aggregate threads here for a supervised shutdown process
       Karafka::Server.workers = workers
       Karafka::Server.listeners = listeners
+      Karafka::Server.jobs_queue = jobs_queue
 
       # All the listener threads need to finish
       listeners.each(&:join)

data/lib/karafka/server.rb

@@ -20,6 +20,9 @@ module Karafka
       # Set of workers
       attr_accessor :workers
 
+      # Jobs queue
+      attr_accessor :jobs_queue
+
       # Method which runs app
       def run
         self.listeners = []
@@ -45,9 +48,9 @@ module Karafka
 
         # We always need to wait for Karafka to stop here since we should wait for the stop running
         # in a separate thread (or trap context) to indicate everything is closed
-        # Since `#start` is blocking, we were get here only after the runner is done. This will
+        # Since `#start` is blocking, we will get here only after the runner is done. This will
         # not add any performance degradation because of that.
-        Thread.pass until Karafka::App.stopped?
+        sleep(0.1) until Karafka::App.terminated?
       # Try its best to shutdown underlying components before re-raising
       # rubocop:disable Lint/RescueException
       rescue Exception => e
@@ -75,6 +78,7 @@ module Karafka
         # Initialize the stopping process only if Karafka was running
         return if Karafka::App.stopping?
         return if Karafka::App.stopped?
+        return if Karafka::App.terminated?
 
         Karafka::App.stop!
 
@@ -84,13 +88,7 @@ module Karafka
         # their work and if so, we can just return and normal shutdown process will take place
         # We divide it by 1000 because we use time in ms.
         ((timeout / 1_000) * SUPERVISION_CHECK_FACTOR).to_i.times do
-          if listeners.count(&:alive?).zero? &&
-             workers.count(&:alive?).zero?
-
-            Karafka::App.producer.close
-
-            return
-          end
+          return if listeners.count(&:alive?).zero? && workers.count(&:alive?).zero?
 
           sleep SUPERVISION_SLEEP
         end
@@ -111,8 +109,6 @@ module Karafka
         # This can cause memory leaks and crashes.
         listeners.each(&:shutdown)
 
-        Karafka::App.producer.close
-
         # We also do not forcefully terminate everything when running in the embedded mode,
         # otherwise we would overwrite the shutdown process of the process that started Karafka
         return unless process.supervised?
@@ -122,19 +118,32 @@ module Karafka
       ensure
         # We need to check if it wasn't an early exit to make sure that only on stop invocation
         # can change the status after everything is closed
-        Karafka::App.stopped! if timeout
+        if timeout
+          Karafka::App.stopped!
+
+          # We close producer as the last thing as it can be used in the notification pipeline
+          # to dispatch state changes, etc
+          Karafka::App.producer.close
+
+          Karafka::App.terminate!
+        end
       end
 
       # Quiets the Karafka server.
-      # Karafka will stop processing but won't quiet to consumer group, so no rebalance will be
+      #
+      # Karafka will stop processing but won't quit the consumer group, so no rebalance will be
       # triggered until final shutdown.
       def quiet
-        # If we are already quieting or in the stop procedures, we should not do it again.
-        return if Karafka::App.quieting?
-        return if Karafka::App.stopping?
-        return if Karafka::App.stopped?
-
+        # We don't have to safe-guard it with check states as the state transitions work only
+        # in one direction
         Karafka::App.quiet!
+
+        # We need one more thread to monitor the process and move to quieted once everything
+        # is quiet and no processing is happening anymore
+        Thread.new do
+          sleep(0.1) until listeners.coordinators.all?(&:finished?)
+          Karafka::App.quieted!
+        end
       end
 
       private

data/lib/karafka/setup/config.rb

@@ -16,7 +16,10 @@ module Karafka
 
       # Defaults for kafka settings, that will be overwritten only if not present already
       KAFKA_DEFAULTS = {
-        'client.id': 'karafka'
+        'client.id': 'karafka',
+        # We emit the statistics by default, so all the instrumentation and web-ui work out of
+        # the box, without requiring users to take any extra actions aside from enabling.
+        'statistics.interval.ms': 5_000
       }.freeze
 
       # Contains settings that should not be used in production but make life easier in dev
@@ -130,6 +133,12 @@ module Karafka
           setting :strategy_selector, default: Processing::StrategySelector.new
         end
 
+        # Things related to operating on messages
+        setting :messages do
+          # Parser is used to convert raw payload prior to deserialization
+          setting :parser, default: Messages::Parser.new
+        end
+
         # Karafka components for ActiveJob
         setting :active_job do
           # option dispatcher [Karafka::ActiveJob::Dispatcher] default dispatcher for ActiveJob
@@ -155,7 +164,7 @@ module Karafka
           # Will configure all the pro components
           # This needs to happen before end user configuration as the end user may overwrite some
           # of the pro defaults with custom components
-          Pro::Loader.setup(config) if Karafka.pro?
+          Pro::Loader.pre_setup(config) if Karafka.pro?
 
           configure(&block)
           merge_kafka_defaults!(config)
@@ -164,6 +173,10 @@ module Karafka
 
           configure_components
 
+          # Runs things that need to be executed after config is defined and all the components
+          # are also configured
+          Pro::Loader.post_setup(config) if Karafka.pro?
+
           Karafka::App.initialized!
         end
 

data/lib/karafka/status.rb

@@ -8,34 +8,58 @@ module Karafka
       initializing: :initialize!,
       initialized: :initialized!,
       running: :run!,
+      # will no longer pickup any work, but current work will be finished
       quieting: :quiet!,
+      # no work is happening but we keep process with the assignments running
+      quiet: :quieted!,
+      # shutdown started
       stopping: :stop!,
-      stopped: :stopped!
+      # all things are done and most of the things except critical are closed
+      stopped: :stopped!,
+      # immediately after this process exists
+      terminated: :terminate!
     }.freeze
 
-    private_constant :STATES
+    # Mutex to ensure that state transitions are thread-safe
+    MUTEX = Mutex.new
+
+    private_constant :MUTEX
 
     # By default we are in the initializing state
     def initialize
       initialize!
     end
 
+    # @return [String] stringified current app status
+    def to_s
+      @status.to_s
+    end
+
+    # Resets the status state
+    # This is used mostly in the integration suite
+    def reset!
+      @status = :initializing
+    end
+
     STATES.each do |state, transition|
       define_method :"#{state}?" do
         @status == state
       end
 
       define_method transition do
-        # Do nothing if the state change would change nothing (same state)
-        return if @status == state
+        MUTEX.synchronize do
+          # Do not allow reverse state transitions (we always go one way) or transition to the same
+          # state as currently
+          return if @status && STATES.keys.index(state) <= STATES.keys.index(@status)
 
-        @status = state
+          @status = state
 
-        # Skip on creation (initializing)
-        # We skip as during this state we do not have yet a monitor
-        return if initializing?
+          # Skip on creation (initializing)
+          # We skip as during this state we do not have yet a monitor
+          return if initializing?
 
-        Karafka.monitor.instrument("app.#{state}")
+          Karafka.monitor.instrument("app.#{state}")
+        end
       end
     end
   end

data/lib/karafka/templates/karafka.rb.erb

@@ -5,8 +5,7 @@
 # If by any chance you've wanted a setup for Rails app, either run the `karafka:install`
 # command again or refer to the install templates available in the source codes
 
-ENV['RACK_ENV'] ||= 'development'
-ENV['KARAFKA_ENV'] ||= ENV['RACK_ENV']
+ENV['KARAFKA_ENV'] ||= 'development'
 Bundler.require(:default, ENV['KARAFKA_ENV'])
 
 # Zeitwerk custom loader for loading the app components before the whole

data/lib/karafka/time_trackers/base.rb

@@ -8,12 +8,7 @@ module Karafka
   module TimeTrackers
     # Base class for all the time-trackers.
     class Base
-      private
-
-      # @return [Float] current time in milliseconds
-      def now
-        ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) * 1000
-      end
+      include ::Karafka::Core::Helpers::Time
     end
   end
 end

data/lib/karafka/time_trackers/pause.rb

@@ -10,7 +10,7 @@ module Karafka
     #   code here, as this is not a frequently used tracker. It is active only once per batch in
     #   case of long-running-jobs and upon errors.
     class Pause < Base
-      attr_reader :attempt
+      attr_reader :attempt, :current_timeout
 
       # @param timeout [Integer] how long should we wait when anything went wrong (in ms)
       # @param max_timeout [Integer, nil] if exponential is on, what is the max value we can reach
@@ -43,6 +43,7 @@ module Karafka
         @started_at = nil
         @attempt = 0
         @timeout = timeout
+        @current_timeout = timeout
         @max_timeout = max_timeout
         @exponential_backoff = exponential_backoff
         @mutex = Mutex.new
@@ -56,7 +57,8 @@ module Karafka
       #   period of time, outside of any regular pausing logic
       def pause(timeout = backoff_interval)
         @mutex.synchronize do
-          @started_at = now
+          @current_timeout = timeout
+          @started_at = monotonic_now
           @ends_at = @started_at + timeout
         end
       end
@@ -93,7 +95,7 @@ module Karafka
       # @return [Boolean] did the pause expire
       def expired?
         @mutex.synchronize do
-          @ends_at ? now >= @ends_at : true
+          @ends_at ? monotonic_now >= @ends_at : true
         end
       end
 

data/lib/karafka/time_trackers/poll.rb

@@ -34,12 +34,12 @@ module Karafka
       # Starts time tracking.
       def start
         @attempts += 1
-        @started_at = now
+        @started_at = monotonic_now
       end
 
       # Stops time tracking of a given piece of code and updates the remaining time.
       def checkpoint
-        @remaining -= (now - @started_at)
+        @remaining -= (monotonic_now - @started_at)
       end
 
       # @return [Boolean] If anything went wrong, can we retry after a backoff period or not

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.23'
+  VERSION = '2.0.26'
 end

data/lib/karafka.rb

@@ -14,7 +14,9 @@
   base64
   date
   singleton
+  digest
   zeitwerk
+  concurrent/atomic/atomic_fixnum
 ].each(&method(:require))
 
 # Karafka framework main namespace

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.0.23
+  version: 2.0.26
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2022-12-07 00:00:00.000000000 Z
+date: 2023-01-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -43,7 +43,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.6
+        version: 2.0.8
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -53,7 +53,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.6
+        version: 2.0.8
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -77,7 +77,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.4.3
+        version: 2.4.7
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -87,7 +87,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.4.3
+        version: 2.4.7
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -145,9 +145,11 @@ files:
 - bin/scenario
 - bin/stress_many
 - bin/stress_one
+- bin/verify_license_integrity
 - certs/cert_chain.pem
 - certs/karafka-pro.pem
-- config/errors.yml
+- config/locales/errors.yml
+- config/locales/pro_errors.yml
 - docker-compose.yml
 - karafka.gemspec
 - lib/active_job/karafka.rb
@@ -204,6 +206,7 @@ files:
 - lib/karafka/messages/message.rb
 - lib/karafka/messages/messages.rb
 - lib/karafka/messages/metadata.rb
+- lib/karafka/messages/parser.rb
 - lib/karafka/messages/seek.rb
 - lib/karafka/patches/rdkafka/bindings.rb
 - lib/karafka/patches/rdkafka/consumer.rb
@@ -212,6 +215,13 @@ files:
 - lib/karafka/pro/active_job/dispatcher.rb
 - lib/karafka/pro/active_job/job_options_contract.rb
 - lib/karafka/pro/base_consumer.rb
+- lib/karafka/pro/encryption.rb
+- lib/karafka/pro/encryption/cipher.rb
+- lib/karafka/pro/encryption/contracts/config.rb
+- lib/karafka/pro/encryption/errors.rb
+- lib/karafka/pro/encryption/messages/middleware.rb
+- lib/karafka/pro/encryption/messages/parser.rb
+- lib/karafka/pro/encryption/setup/config.rb
 - lib/karafka/pro/loader.rb
 - lib/karafka/pro/performance_tracker.rb
 - lib/karafka/pro/processing/coordinator.rb
@@ -340,7 +350,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.26
+rubygems_version: 3.4.1
 signing_key: 
 specification_version: 4
 summary: Karafka is Ruby and Rails efficient Kafka processing framework.