karafka 1.4.12 → 2.2.10

data/lib/karafka/pro/routing/features/pausing.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        # Feature allowing for a per-route reconfiguration of the pausing strategy
+        # It can be useful when different topics should have different backoff policies
+        class Pausing < Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/throttling/config.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Throttling < Base
+          # Throttling feature configuration
+          Config = Struct.new(
+            :active,
+            :limit,
+            :interval,
+            keyword_init: true
+          ) do
+            alias_method :active?, :active
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/throttling/contracts/topic.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Throttling < Base
+          # Namespace for throttling contracts
+          module Contracts
+            # Rules around throttling settings
+            class Topic < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('topic')
+              end
+
+              nested(:throttling) do
+                required(:active) { |val| [true, false].include?(val) }
+                required(:interval) { |val| val.is_a?(Integer) && val.positive? }
+                required(:limit) do |val|
+                  (val.is_a?(Integer) || val == Float::INFINITY) && val.positive?
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/throttling/topic.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Throttling < Base
+          # Topic throttling API extensions
+          module Topic
+            # @param limit [Integer] max messages to process in an time interval
+            # @param interval [Integer] time interval for processing
+            def throttling(
+              limit: Float::INFINITY,
+              interval: 60_000
+            )
+              # Those settings are used for validation
+              @throttling ||= begin
+                config = Config.new(
+                  active: limit != Float::INFINITY,
+                  limit: limit,
+                  interval: interval
+                )
+
+                # If someone defined throttling setup, we need to create appropriate filter for it
+                # and inject it via filtering feature
+                if config.active?
+                  factory = ->(*) { Pro::Processing::Filters::Throttler.new(limit, interval) }
+                  filter(factory)
+                end
+
+                config
+              end
+            end
+
+            # Just an alias for nice API
+            #
+            # @param args [Array] Anything `#throttling` accepts
+            def throttle(**args)
+              throttling(**args)
+            end
+
+            # @return [Boolean] is a given job throttled
+            def throttling?
+              throttling.active?
+            end
+
+            # @return [Hash] topic with all its native configuration options plus throttling
+            def to_h
+              super.merge(
+                throttling: throttling.to_h
+              ).freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/throttling.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        # Ability to throttle ingestion of data per topic partition
+        # Useful when we have fixed limit of things we can process in a given time period without
+        # getting into trouble. It can be used for example to:
+        #   - make sure we do not insert things to DB too fast
+        #   - make sure we do not dispatch HTTP requests to external resources too fast
+        #
+        # This feature is virtual. It materializes itself via the `Filtering` feature.
+        class Throttling < Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/virtual_partitions/config.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class VirtualPartitions < Base
+          # Config for virtual partitions
+          Config = Struct.new(
+            :active,
+            :partitioner,
+            :max_partitions,
+            keyword_init: true
+          ) { alias_method :active?, :active }
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/virtual_partitions/contracts/topic.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class VirtualPartitions < Base
+          # Namespace for VP contracts
+          module Contracts
+            # Rules around virtual partitions
+            class Topic < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('topic')
+              end
+
+              nested(:virtual_partitions) do
+                required(:active) { |val| [true, false].include?(val) }
+                required(:partitioner) { |val| val.nil? || val.respond_to?(:call) }
+                required(:max_partitions) { |val| val.is_a?(Integer) && val >= 1 }
+              end
+
+              # When virtual partitions are defined, partitioner needs to respond to `#call` and it
+              # cannot be nil
+              virtual do |data, errors|
+                next unless errors.empty?
+
+                virtual_partitions = data[:virtual_partitions]
+
+                next unless virtual_partitions[:active]
+                next if virtual_partitions[:partitioner].respond_to?(:call)
+
+                [[%i[virtual_partitions partitioner], :respond_to_call]]
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/virtual_partitions/topic.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class VirtualPartitions < Base
+          # Topic extensions to be able to manage virtual partitions feature
+          module Topic
+            # @param max_partitions [Integer] max number of virtual partitions that can come out of
+            #   the single distribution flow. When set to more than the Karafka threading, will
+            #   create more work than workers. When less, can ensure we have spare resources to
+            #   process other things in parallel.
+            # @param partitioner [nil, #call] nil or callable partitioner
+            # @return [VirtualPartitions] method that allows to set the virtual partitions details
+            #   during the routing configuration and then allows to retrieve it
+            def virtual_partitions(
+              max_partitions: Karafka::App.config.concurrency,
+              partitioner: nil
+            )
+              @virtual_partitions ||= Config.new(
+                active: !partitioner.nil?,
+                max_partitions: max_partitions,
+                partitioner: partitioner
+              )
+            end
+
+            # @return [Boolean] are virtual partitions enabled for given topic
+            def virtual_partitions?
+              virtual_partitions.active?
+            end
+
+            # @return [Hash] topic with all its native configuration options plus manual offset
+            #   management namespace settings
+            def to_h
+              super.merge(
+                virtual_partitions: virtual_partitions.to_h
+              ).freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/virtual_partitions.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        # Virtual Partitions feature config and DSL namespace.
+        #
+        # Virtual Partitions allow you to parallelize the processing of data from a single
+        # partition. This can drastically increase throughput when IO operations are involved.
+        class VirtualPartitions < Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this repository
+# and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+module Karafka
+  # Namespace for pro components, licensed under the commercial license agreement.
+  module Pro
+  end
+end

data/lib/karafka/process.rb

@@ -4,11 +4,16 @@ module Karafka
   # Class used to catch signals from ruby Signal class in order to manage Karafka stop
   # @note There might be only one process - this class is a singleton
   class Process
+    # Allow for process tagging for instrumentation
+    extend ::Karafka::Core::Taggable
+
     # Signal types that we handle
     HANDLED_SIGNALS = %i[
       SIGINT
       SIGQUIT
       SIGTERM
+      SIGTTIN
+      SIGTSTP
     ].freeze
 
     HANDLED_SIGNALS.each do |signal|
@@ -28,33 +33,44 @@ module Karafka
     # Creates an instance of process and creates empty hash for callbacks
     def initialize
       @callbacks = Hash.new { |hsh, key| hsh[key] = [] }
+      @supervised = false
     end
 
     # Method catches all HANDLED_SIGNALS and performs appropriate callbacks (if defined)
     # @note If there are no callbacks, this method will just ignore a given signal that was sent
     def supervise
       HANDLED_SIGNALS.each { |signal| trap_signal(signal) }
+      @supervised = true
+    end
+
+    # Is the current process supervised and are trap signals installed
+    def supervised?
+      @supervised
     end
 
     private
 
     # Traps a single signal and performs callbacks (if any) or just ignores this signal
     # @param [Symbol] signal type that we want to catch
+    # @note Since we do a lot of threading and queuing, we don't want to handle signals from the
+    # trap context s some things may not work there as expected, that is why we spawn a separate
+    # thread to handle the signals process
     def trap_signal(signal)
-      trap(signal) do
-        notice_signal(signal)
-        (@callbacks[signal] || []).each(&:call)
+      previous_handler = ::Signal.trap(signal) do
+        Thread.new do
+          notice_signal(signal)
+
+          (@callbacks[signal] || []).each(&:call)
+        end
+
+        previous_handler.call if previous_handler.respond_to?(:call)
       end
     end
 
     # Informs monitoring about trapped signal
     # @param [Symbol] signal type that we received
-    # @note We cannot perform logging from trap context, that's why
-    #   we have to spin up a new thread to do this
     def notice_signal(signal)
-      Thread.new do
-        Karafka.monitor.instrument('process.notice_signal', caller: self, signal: signal)
-      end
+      Karafka.monitor.instrument('process.notice_signal', caller: self, signal: signal)
     end
   end
 end

data/lib/karafka/processing/coordinator.rb

@@ -0,0 +1,181 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Basic coordinator that allows us to provide coordination objects into consumers.
+    #
+    # This is a wrapping layer to simplify management of work to be handled around consumption.
+    #
+    # @note This coordinator needs to be thread safe. Some operations are performed only in the
+    #   listener thread, but we go with thread-safe by default for all not to worry about potential
+    #   future mistakes.
+    class Coordinator
+      attr_reader :pause_tracker, :seek_offset, :topic, :partition
+
+      # @param topic [Karafka::Routing::Topic]
+      # @param partition [Integer]
+      # @param pause_tracker [Karafka::TimeTrackers::Pause] pause tracker for given topic partition
+      def initialize(topic, partition, pause_tracker)
+        @topic = topic
+        @partition = partition
+        @pause_tracker = pause_tracker
+        @revoked = false
+        @consumptions = {}
+        @running_jobs = 0
+        @manual_pause = false
+        @manual_seek = false
+        @mutex = Mutex.new
+        @marked = false
+        @failure = false
+      end
+
+      # Starts the coordinator for given consumption jobs
+      # @param messages [Array<Karafka::Messages::Message>] batch of message for which we are
+      #   going to coordinate work. Not used with regular coordinator.
+      def start(messages)
+        @failure = false
+        @running_jobs = 0
+        # We need to clear the consumption results hash here, otherwise we could end up storing
+        # consumption results of consumer instances we no longer control
+        @consumptions.clear
+
+        # When starting to run, no pause is expected and no manual pause as well
+        @manual_pause = false
+
+        # No user invoked seeks on a new run
+        @manual_seek = false
+
+        # We set it on the first encounter and never again, because then the offset setting
+        # should be up to the consumers logic (our or the end user)
+        # Seek offset needs to be always initialized as for case where manual offset management
+        # is turned on, we need to have reference to the first offset even in case of running
+        # multiple batches without marking any messages as consumed. Rollback needs to happen to
+        # the last place we know of or the last message + 1 that was marked
+        #
+        # It is however worth keeping in mind, that this may need to be used with `#marked?` to
+        # make sure that the first offset is an offset that has been marked.
+        @seek_offset ||= messages.first.offset
+      end
+
+      # @param offset [Integer] message offset
+      def seek_offset=(offset)
+        synchronize do
+          @marked = true
+          @seek_offset = offset
+        end
+      end
+
+      # Increases number of jobs that we handle with this coordinator
+      def increment
+        synchronize { @running_jobs += 1 }
+      end
+
+      # Decrements number of jobs we handle at the moment
+      def decrement
+        synchronize do
+          @running_jobs -= 1
+
+          return @running_jobs unless @running_jobs.negative?
+
+          # This should never happen. If it does, something is heavily out of sync. Please reach
+          # out to us if you encounter this
+          raise Karafka::Errors::InvalidCoordinatorStateError, 'Was zero before decrementation'
+        end
+      end
+
+      # Is all the consumption done and finished successfully for this coordinator
+      # We do not say we're successful until all work is done, because running work may still
+      # crash.
+      def success?
+        synchronize do
+          @running_jobs.zero? && @consumptions.values.all?(&:success?)
+        end
+      end
+
+      # Mark given consumption on consumer as successful
+      # @param consumer [Karafka::BaseConsumer] consumer that finished successfully
+      def success!(consumer)
+        synchronize do
+          consumption(consumer).success!
+        end
+      end
+
+      # Mark given consumption on consumer as failed
+      # @param consumer [Karafka::BaseConsumer] consumer that failed
+      # @param error [StandardError] error that occurred
+      def failure!(consumer, error)
+        synchronize do
+          @failure = true
+          consumption(consumer).failure!(error)
+        end
+      end
+
+      # @return [Boolean] true if any of work we were running failed
+      def failure?
+        @failure
+      end
+
+      # Marks given coordinator for processing group as revoked
+      #
+      # This is invoked in two places:
+      #   - from the main listener loop when we detect revoked partitions
+      #   - from the consumer in case checkpointing fails
+      #
+      # This means, we can end up having consumer being aware that it was revoked prior to the
+      # listener loop dispatching the revocation job. It is ok, as effectively nothing will be
+      # processed until revocation jobs are done.
+      def revoke
+        synchronize { @revoked = true }
+      end
+
+      # @return [Boolean] is the partition we are processing revoked or not
+      def revoked?
+        @revoked
+      end
+
+      # @return [Boolean] was the new seek offset assigned at least once. This is needed because
+      #   by default we assign seek offset of a first message ever, however this is insufficient
+      #   for DLQ in a scenario where the first message would be broken. We would never move
+      #   out of it and would end up in an endless loop.
+      def marked?
+        @marked
+      end
+
+      # Store in the coordinator info, that this pause was done manually by the end user and not
+      # by the system itself
+      def manual_pause
+        @manual_pause = true
+      end
+
+      # @return [Boolean] are we in a pause that was initiated by the user
+      def manual_pause?
+        @pause_tracker.paused? && @manual_pause
+      end
+
+      # Marks seek as manual for coordination purposes
+      def manual_seek
+        @manual_seek = true
+      end
+
+      # @return [Boolean] did a user invoke seek in the current operations scope
+      def manual_seek?
+        @manual_seek
+      end
+
+      # Allows to run synchronized (locked) code that can operate in between virtual partitions
+      # @param block [Proc] code we want to run in the synchronized mode
+      def synchronize(&block)
+        @mutex.synchronize(&block)
+      end
+
+      private
+
+      # @param consumer [Object] karafka consumer (normal or pro)
+      # @return [Karafka::Processing::Result] result object which we can use to indicate
+      #   consumption processing state.
+      def consumption(consumer)
+        @consumptions[consumer] ||= Processing::Result.new
+      end
+    end
+  end
+end