karafka 2.3.1 → 2.3.3

data/lib/karafka/swarm/supervisor.rb

@@ -0,0 +1,187 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Swarm
+    # Supervisor that starts forks and uses monitor to monitor them. Also handles shutdown of
+    # all the processes including itself.
+    #
+    # In case any node dies, it will be restarted.
+    #
+    # @note Technically speaking supervisor is never in the running state because we do not want
+    #   to have any sockets or anything else on it that could break under forking.
+    #   It has its own "supervising" state from which it can go to the final shutdown.
+    class Supervisor
+      include Karafka::Core::Helpers::Time
+      include Helpers::ConfigImporter.new(
+        monitor: %i[monitor],
+        swarm: %i[internal swarm],
+        manager: %i[internal swarm manager],
+        supervision_interval: %i[internal swarm supervision_interval],
+        shutdown_timeout: %i[shutdown_timeout],
+        supervision_sleep: %i[internal supervision_sleep],
+        forceful_exit_code: %i[internal forceful_exit_code],
+        process: %i[internal process]
+      )
+
+      def initialize
+        @mutex = Mutex.new
+        @queue = Processing::TimedQueue.new
+      end
+
+      # Creates needed number of forks, installs signals and starts supervision
+      def run
+        # Close producer just in case. While it should not be used, we do not want even a
+        # theoretical case since librdkafka is not thread-safe.
+        # We close it prior to forking just to make sure, there is no issue with initialized
+        # producer (should not be initialized but just in case)
+        Karafka.producer.close
+
+        Karafka::App.warmup
+
+        manager.start
+
+        process.on_sigint { stop }
+        process.on_sigquit { stop }
+        process.on_sigterm { stop }
+        process.on_sigtstp { quiet }
+        process.on_sigttin { signal('TTIN') }
+        # Needed to be registered as we want to unlock on child changes
+        process.on_sigchld {}
+        process.on_any_active { unlock }
+        process.supervise
+
+        Karafka::App.supervise!
+
+        loop do
+          return if Karafka::App.terminated?
+
+          lock
+          control
+        end
+      # If anything went wrong, signal this and die
+      # Supervisor is meant to be thin and not cause any issues. If you encounter this case
+      # please report it as it should be considered critical
+      rescue StandardError => e
+        monitor.instrument(
+          'error.occurred',
+          caller: self,
+          error: e,
+          manager: manager,
+          type: 'swarm.supervisor.error'
+        )
+
+        @nodes.terminate
+      end
+
+      private
+
+      # Keeps the lock on the queue so we control nodes only when it is needed
+      # @note We convert to seconds since the queue timeout requires seconds
+      def lock
+        @queue.pop(timeout: supervision_interval / 1_000.0)
+      end
+
+      # Frees the lock on events that could require nodes control
+      def unlock
+        @queue << true
+      end
+
+      # Stops all the nodes and supervisor once all nodes are dead.
+      # It will forcefully stop all nodes if they exit the shutdown timeout. While in theory each
+      # of the nodes anyhow has its own supervisor, this is a last resort to stop everything.
+      def stop
+        # Ensure that the stopping procedure is initialized only once
+        @mutex.synchronize do
+          return if @stopping
+
+          @stopping = true
+        end
+
+        initialized = true
+        Karafka::App.stop!
+
+        manager.stop
+
+        # We check from time to time (for the timeout period) if all the threads finished
+        # their work and if so, we can just return and normal shutdown process will take place
+        # We divide it by 1000 because we use time in ms.
+        ((shutdown_timeout / 1_000) * (1 / supervision_sleep)).to_i.times do
+          if manager.stopped?
+            manager.cleanup
+            return
+          end
+
+          sleep(supervision_sleep)
+        end
+
+        raise Errors::ForcefulShutdownError
+      rescue Errors::ForcefulShutdownError => e
+        monitor.instrument(
+          'error.occurred',
+          caller: self,
+          error: e,
+          manager: manager,
+          type: 'app.stopping.error'
+        )
+
+        # Run forceful kill
+        manager.terminate
+        # And wait until linux kills them
+        # This prevents us from existing forcefully with any dead child process still existing
+        # Since we have sent the `KILL` signal, it must die, so we can wait until all dead
+        sleep(supervision_sleep) until manager.stopped?
+
+        # Cleanup the process table
+        manager.cleanup
+
+        # We do not use `exit!` here similar to regular server because we do not have to worry
+        # about any librdkafka related hanging connections, etc
+        Kernel.exit(forceful_exit_code)
+      ensure
+        if initialized
+          Karafka::App.stopped!
+          Karafka::App.terminate!
+        end
+      end
+
+      # Moves all the nodes and itself to the quiet state
+      def quiet
+        @mutex.synchronize do
+          return if @quieting
+
+          @quieting = true
+
+          Karafka::App.quiet!
+          manager.quiet
+          Karafka::App.quieted!
+        end
+      end
+
+      # Checks on the children nodes and takes appropriate actions.
+      # - If node is dead, will cleanup
+      # - If node is no longer reporting as healthy will start a graceful shutdown
+      # - If node does not want to close itself gracefully, will kill it
+      # - If node was dead, new node will be started as a recovery means
+      def control
+        @mutex.synchronize do
+          # If we are in quieting or stopping we should no longer control children
+          # Those states aim to finally shutdown nodes and we should not forcefully do anything
+          # to them. This especially applies to the quieting mode where any complex lifecycle
+          # reporting listeners may no longer report correctly
+          return if @quieting
+          return if @stopping
+
+          manager.control
+        end
+      end
+
+      # Sends desired signal to each node
+      # @param signal [String]
+      def signal(signal)
+        @mutex.synchronize do
+          manager.signal(signal)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/swarm.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Namespace for the Swarm capabilities.
+  #
+  # Karafka in the swarm mode will fork additional processes and use the parent process as a
+  # supervisor. This capability allows to run multiple processes alongside but saves some memory
+  # due to CoW.
+  module Swarm
+    class << self
+      # Raises an error if swarm is not supported on a given platform
+      def ensure_supported!
+        return if supported?
+
+        raise(
+          Errors::UnsupportedOptionError,
+          'Swarm mode not supported on this platform'
+        )
+      end
+
+      # @return [Boolean] true if fork API and pidfd OS API are available, otherwise false
+      def supported?
+        ::Process.respond_to?(:fork) && Swarm::Pidfd.supported?
+      end
+    end
+  end
+end

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.3.1'
+  VERSION = '2.3.3'
 end

data/lib/karafka.rb

@@ -41,7 +41,7 @@ module Karafka
 
     # @return [WaterDrop::Producer] waterdrop messages producer
     def producer
-      @producer ||= App.config.producer
+      App.config.producer
     end
 
     # @return [::Karafka::Monitor] monitor that we want to use

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.3.1
+  version: 2.3.3
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2024-02-09 00:00:00.000000000 Z
+date: 2024-02-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -162,6 +162,7 @@ files:
 - lib/karafka/cli/info.rb
 - lib/karafka/cli/install.rb
 - lib/karafka/cli/server.rb
+- lib/karafka/cli/swarm.rb
 - lib/karafka/cli/topics.rb
 - lib/karafka/connection/client.rb
 - lib/karafka/connection/conductor.rb
@@ -186,6 +187,7 @@ files:
 - lib/karafka/errors.rb
 - lib/karafka/helpers/async.rb
 - lib/karafka/helpers/colorize.rb
+- lib/karafka/helpers/config_importer.rb
 - lib/karafka/helpers/interval_runner.rb
 - lib/karafka/helpers/multi_delegator.rb
 - lib/karafka/instrumentation/assignments_tracker.rb
@@ -205,7 +207,9 @@ files:
 - lib/karafka/instrumentation/vendors/datadog/dashboard.json
 - lib/karafka/instrumentation/vendors/datadog/logger_listener.rb
 - lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb
+- lib/karafka/instrumentation/vendors/kubernetes/base_listener.rb
 - lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb
+- lib/karafka/instrumentation/vendors/kubernetes/swarm_liveness_listener.rb
 - lib/karafka/licenser.rb
 - lib/karafka/messages/batch_metadata.rb
 - lib/karafka/messages/builders/batch_metadata.rb
@@ -245,6 +249,9 @@ files:
 - lib/karafka/pro/loader.rb
 - lib/karafka/pro/processing/collapser.rb
 - lib/karafka/pro/processing/coordinator.rb
+- lib/karafka/pro/processing/coordinators/errors_tracker.rb
+- lib/karafka/pro/processing/coordinators/filters_applier.rb
+- lib/karafka/pro/processing/coordinators/virtual_offset_manager.rb
 - lib/karafka/pro/processing/executor.rb
 - lib/karafka/pro/processing/expansions_selector.rb
 - lib/karafka/pro/processing/filters/base.rb
@@ -253,7 +260,6 @@ files:
 - lib/karafka/pro/processing/filters/inline_insights_delayer.rb
 - lib/karafka/pro/processing/filters/throttler.rb
 - lib/karafka/pro/processing/filters/virtual_limiter.rb
-- lib/karafka/pro/processing/filters_applier.rb
 - lib/karafka/pro/processing/jobs/consume_non_blocking.rb
 - lib/karafka/pro/processing/jobs/periodic.rb
 - lib/karafka/pro/processing/jobs/periodic_non_blocking.rb
@@ -317,12 +323,12 @@ files:
 - lib/karafka/pro/processing/strategies/mom/vp.rb
 - lib/karafka/pro/processing/strategies/vp/default.rb
 - lib/karafka/pro/processing/strategy_selector.rb
-- lib/karafka/pro/processing/virtual_offset_manager.rb
 - lib/karafka/pro/routing/features/active_job.rb
 - lib/karafka/pro/routing/features/active_job/builder.rb
 - lib/karafka/pro/routing/features/base.rb
 - lib/karafka/pro/routing/features/dead_letter_queue.rb
 - lib/karafka/pro/routing/features/dead_letter_queue/contracts/topic.rb
+- lib/karafka/pro/routing/features/dead_letter_queue/topic.rb
 - lib/karafka/pro/routing/features/delaying.rb
 - lib/karafka/pro/routing/features/delaying/config.rb
 - lib/karafka/pro/routing/features/delaying/contracts/topic.rb
@@ -375,6 +381,10 @@ files:
 - lib/karafka/pro/routing/features/periodic_job/config.rb
 - lib/karafka/pro/routing/features/periodic_job/contracts/topic.rb
 - lib/karafka/pro/routing/features/periodic_job/topic.rb
+- lib/karafka/pro/routing/features/swarm.rb
+- lib/karafka/pro/routing/features/swarm/config.rb
+- lib/karafka/pro/routing/features/swarm/contracts/topic.rb
+- lib/karafka/pro/routing/features/swarm/topic.rb
 - lib/karafka/pro/routing/features/throttling.rb
 - lib/karafka/pro/routing/features/throttling/config.rb
 - lib/karafka/pro/routing/features/throttling/contracts/topic.rb
@@ -383,6 +393,7 @@ files:
 - lib/karafka/pro/routing/features/virtual_partitions/config.rb
 - lib/karafka/pro/routing/features/virtual_partitions/contracts/topic.rb
 - lib/karafka/pro/routing/features/virtual_partitions/topic.rb
+- lib/karafka/pro/swarm/liveness_listener.rb
 - lib/karafka/process.rb
 - lib/karafka/processing/coordinator.rb
 - lib/karafka/processing/coordinators_buffer.rb
@@ -455,6 +466,12 @@ files:
 - lib/karafka/setup/config.rb
 - lib/karafka/setup/dsl.rb
 - lib/karafka/status.rb
+- lib/karafka/swarm.rb
+- lib/karafka/swarm/liveness_listener.rb
+- lib/karafka/swarm/manager.rb
+- lib/karafka/swarm/node.rb
+- lib/karafka/swarm/pidfd.rb
+- lib/karafka/swarm/supervisor.rb
 - lib/karafka/templates/application_consumer.rb.erb
 - lib/karafka/templates/example_consumer.rb.erb
 - lib/karafka/templates/karafka.rb.erb

data/lib/karafka/pro/processing/filters_applier.rb

@@ -1,105 +0,0 @@
-# frozen_string_literal: true
-
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
-
-module Karafka
-  module Pro
-    module Processing
-      # Applier for all filters we want to have. Whether related to limiting messages based
-      # on the payload or any other things.
-      #
-      # From the outside world perspective, this encapsulates all the filters.
-      # This means that this is the API we expose as a single filter, allowing us to control
-      # the filtering via many filters easily.
-      class FiltersApplier
-        # @return [Array] registered filters array. Useful if we want to inject internal context
-        #   aware filters.
-        attr_reader :filters
-
-        # @param coordinator [Pro::Coordinator] pro coordinator
-        def initialize(coordinator)
-          # Builds filters out of their factories
-          # We build it that way (providing topic and partition) because there may be a case where
-          # someone wants to have a specific logic that is per topic or partition. Like for example
-          # a case where there is a cache bypassing revocations for topic partition.
-          #
-          # We provide full Karafka routing topic here and not the name only, in case the filter
-          # would be customized based on other topic settings (like VPs, etc)
-          #
-          # This setup allows for biggest flexibility also because topic object holds the reference
-          # to the subscription group and consumer group
-          @filters = coordinator.topic.filtering.factories.map do |factory|
-            factory.call(coordinator.topic, coordinator.partition)
-          end
-        end
-
-        # @param messages [Array<Karafka::Messages::Message>] array with messages from the
-        #   partition
-        def apply!(messages)
-          return unless active?
-
-          @filters.each { |filter| filter.apply!(messages) }
-        end
-
-        # @return [Boolean] did we filter out any messages during filtering run
-        def applied?
-          return false unless active?
-
-          !applied.empty?
-        end
-
-        # @return [Symbol] consumer post-filtering action that should be taken
-        def action
-          return :skip unless applied?
-
-          # The highest priority is on a potential backoff from any of the filters because it is
-          # the less risky (delay and continue later)
-          return :pause if applied.any? { |filter| filter.action == :pause }
-
-          # If none of the filters wanted to pause, we can check for any that would want to seek
-          # and if there is any, we can go with this strategy
-          return :seek if applied.any? { |filter| filter.action == :seek }
-
-          :skip
-        end
-
-        # @return [Integer] minimum timeout we need to pause. This is the minimum for all the
-        #   filters to satisfy all of them.
-        def timeout
-          applied.map(&:timeout).compact.min || 0
-        end
-
-        # The first message we do need to get next time we poll. We use the minimum not to jump
-        # accidentally by over any.
-        # @return [Karafka::Messages::Message, nil] cursor message or nil if none
-        # @note Cursor message can also return the offset in the time format
-        def cursor
-          return nil unless active?
-
-          applied.map(&:cursor).compact.min_by(&:offset)
-        end
-
-        private
-
-        # @return [Boolean] is filtering active
-        def active?
-          !@filters.empty?
-        end
-
-        # @return [Array<Object>] filters that applied any sort of messages limiting
-        def applied
-          @filters.select(&:applied?)
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/virtual_offset_manager.rb

@@ -1,177 +0,0 @@
-# frozen_string_literal: true
-
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
-
-module Karafka
-  module Pro
-    module Processing
-      # Manager that keeps track of our offsets with the virtualization layer that are local
-      # to given partition assignment. It allows for easier offset management for virtual
-      # virtual partition cases as it provides us ability to mark as consumed and move the
-      # real offset behind as expected.
-      #
-      # @note We still use the regular coordinator "real" offset management as we want to have
-      #   them as separated as possible because the real seek offset management is also used for
-      #   pausing, filtering and others and should not be impacted by the virtual one
-      #
-      # @note This manager is **not** thread-safe by itself. It should operate from coordinator
-      #   locked locations.
-      class VirtualOffsetManager
-        attr_reader :groups
-
-        # @param topic [String]
-        # @param partition [Integer]
-        # @param offset_metadata_strategy [Symbol] what metadata should we select. That is, should
-        #   we use the most recent or one picked from the offset that is going to be committed
-        #
-        # @note We need topic and partition because we use a seek message (virtual) for real offset
-        #   management. We could keep real message reference but this can be memory consuming
-        #   and not worth it.
-        def initialize(topic, partition, offset_metadata_strategy)
-          @topic = topic
-          @partition = partition
-          @groups = []
-          @marked = {}
-          @offsets_metadata = {}
-          @real_offset = -1
-          @offset_metadata_strategy = offset_metadata_strategy
-          @current_offset_metadata = nil
-        end
-
-        # Clears the manager for a next collective operation
-        def clear
-          @groups.clear
-          @offsets_metadata.clear
-          @current_offset_metadata = nil
-          @marked.clear
-          @real_offset = -1
-        end
-
-        # Registers an offset group coming from one virtual consumer. In order to move the real
-        # underlying offset accordingly, we need to make sure to track the virtual consumers
-        # offsets groups independently and only materialize the end result.
-        #
-        # @param offsets_group [Array<Integer>] offsets from one virtual consumer
-        def register(offsets_group)
-          @groups << offsets_group
-
-          offsets_group.each { |offset| @marked[offset] = false }
-        end
-
-        # Marks given message as marked (virtually consumed).
-        # We mark given message offset and other earlier offsets from the same group as done
-        # and we can refresh our real offset representation based on that as it might have changed
-        # to a newer real offset.
-        # @param message [Karafka::Messages::Message] message coming from VP we want to mark
-        # @param offset_metadata [String, nil] offset metadata. `nil` if none
-        def mark(message, offset_metadata)
-          offset = message.offset
-
-          # Store metadata when we materialize the most stable offset
-          @offsets_metadata[offset] = offset_metadata
-          @current_offset_metadata = offset_metadata
-
-          group = @groups.find { |reg_group| reg_group.include?(offset) }
-
-          # This case can happen when someone uses MoM and wants to mark message from a previous
-          # batch as consumed. We can add it, since the real offset refresh will point to it
-          unless group
-            group = [offset]
-            @groups << group
-          end
-
-          position = group.index(offset)
-
-          # Mark all previous messages from the same group also as virtually consumed
-          group[0..position].each do |markable_offset|
-            # Set previous messages metadata offset as the offset of higher one for overwrites
-            # unless a different metadata were set explicitely
-            @offsets_metadata[markable_offset] ||= offset_metadata
-            @marked[markable_offset] = true
-          end
-
-          # Recompute the real offset representation
-          materialize_real_offset
-        end
-
-        # Mark all from all groups including the `message`.
-        # Useful when operating in a collapsed state for marking
-        # @param message [Karafka::Messages::Message]
-        # @param offset_metadata [String, nil]
-        def mark_until(message, offset_metadata)
-          mark(message, offset_metadata)
-
-          @groups.each do |group|
-            group.each do |offset|
-              next if offset > message.offset
-
-              @offsets_metadata[offset] = offset_metadata
-              @marked[offset] = true
-            end
-          end
-
-          materialize_real_offset
-        end
-
-        # @return [Array<Integer>] Offsets of messages already marked as consumed virtually
-        def marked
-          @marked.select { |_, status| status }.map(&:first).sort
-        end
-
-        # Is there a real offset we can mark as consumed
-        # @return [Boolean]
-        def markable?
-          !@real_offset.negative?
-        end
-
-        # @return [Array<Messages::Seek, String>] markable message for real offset marking and
-        #   its associated metadata
-        def markable
-          raise Errors::InvalidRealOffsetUsageError unless markable?
-
-          offset_metadata = case @offset_metadata_strategy
-                            when :exact
-                              @offsets_metadata.fetch(@real_offset)
-                            when :current
-                              @current_offset_metadata
-                            else
-                              raise Errors::UnsupportedCaseError, @offset_metadata_strategy
-                            end
-
-          [
-            Messages::Seek.new(
-              @topic,
-              @partition,
-              @real_offset
-            ),
-            offset_metadata
-          ]
-        end
-
-        private
-
-        # Recomputes the biggest possible real offset we can have.
-        # It picks the the biggest offset that has uninterrupted stream of virtually marked as
-        # consumed because this will be the collective offset.
-        def materialize_real_offset
-          @marked.to_a.sort_by(&:first).each do |offset, marked|
-            break unless marked
-
-            @real_offset = offset
-          end
-
-          @real_offset = (@marked.keys.min - 1) if @real_offset.negative?
-        end
-      end
-    end
-  end
-end