karafka 2.3.1 → 2.3.2

data/lib/karafka/swarm.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Namespace for the Swarm capabilities.
+  #
+  # Karafka in the swarm mode will fork additional processes and use the parent process as a
+  # supervisor. This capability allows to run multiple processes alongside but saves some memory
+  # due to CoW.
+  module Swarm
+    class << self
+      # Raises an error if swarm is not supported on a given platform
+      def ensure_supported!
+        return if supported?
+
+        raise(
+          Errors::UnsupportedOptionError,
+          'Swarm mode not supported on this platform'
+        )
+      end
+
+      # @return [Boolean] true if fork API and pidfd OS API are available, otherwise false
+      def supported?
+        ::Process.respond_to?(:fork) && Swarm::Pidfd.supported?
+      end
+    end
+  end
+end

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.3.1'
+  VERSION = '2.3.2'
 end

data/lib/karafka.rb

@@ -41,7 +41,7 @@ module Karafka
 
     # @return [WaterDrop::Producer] waterdrop messages producer
     def producer
-      @producer ||= App.config.producer
+      App.config.producer
     end
 
     # @return [::Karafka::Monitor] monitor that we want to use

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.3.1
+  version: 2.3.2
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2024-02-09 00:00:00.000000000 Z
+date: 2024-02-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -162,6 +162,7 @@ files:
 - lib/karafka/cli/info.rb
 - lib/karafka/cli/install.rb
 - lib/karafka/cli/server.rb
+- lib/karafka/cli/swarm.rb
 - lib/karafka/cli/topics.rb
 - lib/karafka/connection/client.rb
 - lib/karafka/connection/conductor.rb
@@ -186,6 +187,7 @@ files:
 - lib/karafka/errors.rb
 - lib/karafka/helpers/async.rb
 - lib/karafka/helpers/colorize.rb
+- lib/karafka/helpers/config_importer.rb
 - lib/karafka/helpers/interval_runner.rb
 - lib/karafka/helpers/multi_delegator.rb
 - lib/karafka/instrumentation/assignments_tracker.rb
@@ -205,7 +207,9 @@ files:
 - lib/karafka/instrumentation/vendors/datadog/dashboard.json
 - lib/karafka/instrumentation/vendors/datadog/logger_listener.rb
 - lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb
+- lib/karafka/instrumentation/vendors/kubernetes/base_listener.rb
 - lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb
+- lib/karafka/instrumentation/vendors/kubernetes/swarm_liveness_listener.rb
 - lib/karafka/licenser.rb
 - lib/karafka/messages/batch_metadata.rb
 - lib/karafka/messages/builders/batch_metadata.rb
@@ -245,6 +249,9 @@ files:
 - lib/karafka/pro/loader.rb
 - lib/karafka/pro/processing/collapser.rb
 - lib/karafka/pro/processing/coordinator.rb
+- lib/karafka/pro/processing/coordinators/errors_tracker.rb
+- lib/karafka/pro/processing/coordinators/filters_applier.rb
+- lib/karafka/pro/processing/coordinators/virtual_offset_manager.rb
 - lib/karafka/pro/processing/executor.rb
 - lib/karafka/pro/processing/expansions_selector.rb
 - lib/karafka/pro/processing/filters/base.rb
@@ -253,7 +260,6 @@ files:
 - lib/karafka/pro/processing/filters/inline_insights_delayer.rb
 - lib/karafka/pro/processing/filters/throttler.rb
 - lib/karafka/pro/processing/filters/virtual_limiter.rb
-- lib/karafka/pro/processing/filters_applier.rb
 - lib/karafka/pro/processing/jobs/consume_non_blocking.rb
 - lib/karafka/pro/processing/jobs/periodic.rb
 - lib/karafka/pro/processing/jobs/periodic_non_blocking.rb
@@ -317,12 +323,12 @@ files:
 - lib/karafka/pro/processing/strategies/mom/vp.rb
 - lib/karafka/pro/processing/strategies/vp/default.rb
 - lib/karafka/pro/processing/strategy_selector.rb
-- lib/karafka/pro/processing/virtual_offset_manager.rb
 - lib/karafka/pro/routing/features/active_job.rb
 - lib/karafka/pro/routing/features/active_job/builder.rb
 - lib/karafka/pro/routing/features/base.rb
 - lib/karafka/pro/routing/features/dead_letter_queue.rb
 - lib/karafka/pro/routing/features/dead_letter_queue/contracts/topic.rb
+- lib/karafka/pro/routing/features/dead_letter_queue/topic.rb
 - lib/karafka/pro/routing/features/delaying.rb
 - lib/karafka/pro/routing/features/delaying/config.rb
 - lib/karafka/pro/routing/features/delaying/contracts/topic.rb
@@ -383,6 +389,7 @@ files:
 - lib/karafka/pro/routing/features/virtual_partitions/config.rb
 - lib/karafka/pro/routing/features/virtual_partitions/contracts/topic.rb
 - lib/karafka/pro/routing/features/virtual_partitions/topic.rb
+- lib/karafka/pro/swarm/liveness_listener.rb
 - lib/karafka/process.rb
 - lib/karafka/processing/coordinator.rb
 - lib/karafka/processing/coordinators_buffer.rb
@@ -455,6 +462,12 @@ files:
 - lib/karafka/setup/config.rb
 - lib/karafka/setup/dsl.rb
 - lib/karafka/status.rb
+- lib/karafka/swarm.rb
+- lib/karafka/swarm/liveness_listener.rb
+- lib/karafka/swarm/manager.rb
+- lib/karafka/swarm/node.rb
+- lib/karafka/swarm/pidfd.rb
+- lib/karafka/swarm/supervisor.rb
 - lib/karafka/templates/application_consumer.rb.erb
 - lib/karafka/templates/example_consumer.rb.erb
 - lib/karafka/templates/karafka.rb.erb

data/lib/karafka/pro/processing/filters_applier.rb

@@ -1,105 +0,0 @@
-# frozen_string_literal: true
-
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
-
-module Karafka
-  module Pro
-    module Processing
-      # Applier for all filters we want to have. Whether related to limiting messages based
-      # on the payload or any other things.
-      #
-      # From the outside world perspective, this encapsulates all the filters.
-      # This means that this is the API we expose as a single filter, allowing us to control
-      # the filtering via many filters easily.
-      class FiltersApplier
-        # @return [Array] registered filters array. Useful if we want to inject internal context
-        #   aware filters.
-        attr_reader :filters
-
-        # @param coordinator [Pro::Coordinator] pro coordinator
-        def initialize(coordinator)
-          # Builds filters out of their factories
-          # We build it that way (providing topic and partition) because there may be a case where
-          # someone wants to have a specific logic that is per topic or partition. Like for example
-          # a case where there is a cache bypassing revocations for topic partition.
-          #
-          # We provide full Karafka routing topic here and not the name only, in case the filter
-          # would be customized based on other topic settings (like VPs, etc)
-          #
-          # This setup allows for biggest flexibility also because topic object holds the reference
-          # to the subscription group and consumer group
-          @filters = coordinator.topic.filtering.factories.map do |factory|
-            factory.call(coordinator.topic, coordinator.partition)
-          end
-        end
-
-        # @param messages [Array<Karafka::Messages::Message>] array with messages from the
-        #   partition
-        def apply!(messages)
-          return unless active?
-
-          @filters.each { |filter| filter.apply!(messages) }
-        end
-
-        # @return [Boolean] did we filter out any messages during filtering run
-        def applied?
-          return false unless active?
-
-          !applied.empty?
-        end
-
-        # @return [Symbol] consumer post-filtering action that should be taken
-        def action
-          return :skip unless applied?
-
-          # The highest priority is on a potential backoff from any of the filters because it is
-          # the less risky (delay and continue later)
-          return :pause if applied.any? { |filter| filter.action == :pause }
-
-          # If none of the filters wanted to pause, we can check for any that would want to seek
-          # and if there is any, we can go with this strategy
-          return :seek if applied.any? { |filter| filter.action == :seek }
-
-          :skip
-        end
-
-        # @return [Integer] minimum timeout we need to pause. This is the minimum for all the
-        #   filters to satisfy all of them.
-        def timeout
-          applied.map(&:timeout).compact.min || 0
-        end
-
-        # The first message we do need to get next time we poll. We use the minimum not to jump
-        # accidentally by over any.
-        # @return [Karafka::Messages::Message, nil] cursor message or nil if none
-        # @note Cursor message can also return the offset in the time format
-        def cursor
-          return nil unless active?
-
-          applied.map(&:cursor).compact.min_by(&:offset)
-        end
-
-        private
-
-        # @return [Boolean] is filtering active
-        def active?
-          !@filters.empty?
-        end
-
-        # @return [Array<Object>] filters that applied any sort of messages limiting
-        def applied
-          @filters.select(&:applied?)
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/virtual_offset_manager.rb

@@ -1,177 +0,0 @@
-# frozen_string_literal: true
-
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
-
-module Karafka
-  module Pro
-    module Processing
-      # Manager that keeps track of our offsets with the virtualization layer that are local
-      # to given partition assignment. It allows for easier offset management for virtual
-      # virtual partition cases as it provides us ability to mark as consumed and move the
-      # real offset behind as expected.
-      #
-      # @note We still use the regular coordinator "real" offset management as we want to have
-      #   them as separated as possible because the real seek offset management is also used for
-      #   pausing, filtering and others and should not be impacted by the virtual one
-      #
-      # @note This manager is **not** thread-safe by itself. It should operate from coordinator
-      #   locked locations.
-      class VirtualOffsetManager
-        attr_reader :groups
-
-        # @param topic [String]
-        # @param partition [Integer]
-        # @param offset_metadata_strategy [Symbol] what metadata should we select. That is, should
-        #   we use the most recent or one picked from the offset that is going to be committed
-        #
-        # @note We need topic and partition because we use a seek message (virtual) for real offset
-        #   management. We could keep real message reference but this can be memory consuming
-        #   and not worth it.
-        def initialize(topic, partition, offset_metadata_strategy)
-          @topic = topic
-          @partition = partition
-          @groups = []
-          @marked = {}
-          @offsets_metadata = {}
-          @real_offset = -1
-          @offset_metadata_strategy = offset_metadata_strategy
-          @current_offset_metadata = nil
-        end
-
-        # Clears the manager for a next collective operation
-        def clear
-          @groups.clear
-          @offsets_metadata.clear
-          @current_offset_metadata = nil
-          @marked.clear
-          @real_offset = -1
-        end
-
-        # Registers an offset group coming from one virtual consumer. In order to move the real
-        # underlying offset accordingly, we need to make sure to track the virtual consumers
-        # offsets groups independently and only materialize the end result.
-        #
-        # @param offsets_group [Array<Integer>] offsets from one virtual consumer
-        def register(offsets_group)
-          @groups << offsets_group
-
-          offsets_group.each { |offset| @marked[offset] = false }
-        end
-
-        # Marks given message as marked (virtually consumed).
-        # We mark given message offset and other earlier offsets from the same group as done
-        # and we can refresh our real offset representation based on that as it might have changed
-        # to a newer real offset.
-        # @param message [Karafka::Messages::Message] message coming from VP we want to mark
-        # @param offset_metadata [String, nil] offset metadata. `nil` if none
-        def mark(message, offset_metadata)
-          offset = message.offset
-
-          # Store metadata when we materialize the most stable offset
-          @offsets_metadata[offset] = offset_metadata
-          @current_offset_metadata = offset_metadata
-
-          group = @groups.find { |reg_group| reg_group.include?(offset) }
-
-          # This case can happen when someone uses MoM and wants to mark message from a previous
-          # batch as consumed. We can add it, since the real offset refresh will point to it
-          unless group
-            group = [offset]
-            @groups << group
-          end
-
-          position = group.index(offset)
-
-          # Mark all previous messages from the same group also as virtually consumed
-          group[0..position].each do |markable_offset|
-            # Set previous messages metadata offset as the offset of higher one for overwrites
-            # unless a different metadata were set explicitely
-            @offsets_metadata[markable_offset] ||= offset_metadata
-            @marked[markable_offset] = true
-          end
-
-          # Recompute the real offset representation
-          materialize_real_offset
-        end
-
-        # Mark all from all groups including the `message`.
-        # Useful when operating in a collapsed state for marking
-        # @param message [Karafka::Messages::Message]
-        # @param offset_metadata [String, nil]
-        def mark_until(message, offset_metadata)
-          mark(message, offset_metadata)
-
-          @groups.each do |group|
-            group.each do |offset|
-              next if offset > message.offset
-
-              @offsets_metadata[offset] = offset_metadata
-              @marked[offset] = true
-            end
-          end
-
-          materialize_real_offset
-        end
-
-        # @return [Array<Integer>] Offsets of messages already marked as consumed virtually
-        def marked
-          @marked.select { |_, status| status }.map(&:first).sort
-        end
-
-        # Is there a real offset we can mark as consumed
-        # @return [Boolean]
-        def markable?
-          !@real_offset.negative?
-        end
-
-        # @return [Array<Messages::Seek, String>] markable message for real offset marking and
-        #   its associated metadata
-        def markable
-          raise Errors::InvalidRealOffsetUsageError unless markable?
-
-          offset_metadata = case @offset_metadata_strategy
-                            when :exact
-                              @offsets_metadata.fetch(@real_offset)
-                            when :current
-                              @current_offset_metadata
-                            else
-                              raise Errors::UnsupportedCaseError, @offset_metadata_strategy
-                            end
-
-          [
-            Messages::Seek.new(
-              @topic,
-              @partition,
-              @real_offset
-            ),
-            offset_metadata
-          ]
-        end
-
-        private
-
-        # Recomputes the biggest possible real offset we can have.
-        # It picks the the biggest offset that has uninterrupted stream of virtually marked as
-        # consumed because this will be the collective offset.
-        def materialize_real_offset
-          @marked.to_a.sort_by(&:first).each do |offset, marked|
-            break unless marked
-
-            @real_offset = offset
-          end
-
-          @real_offset = (@marked.keys.min - 1) if @real_offset.negative?
-        end
-      end
-    end
-  end
-end