karafka 2.0.0 → 2.1.0

data/lib/karafka/pro/routing/features/filtering/topic.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Filtering < Base
+          # Filtering feature topic extensions
+          module Topic
+            # @param factory [#call, nil] Callable that can produce new filters instances per
+            #   assigned topic partition. nil as default so this feature is disabled
+            def filter(factory = nil)
+              @filtering ||= Config.new(factories: [])
+              @filtering.factories << factory if factory
+              @filtering
+            end
+
+            # @param args [Array] Anything `#filter` accepts
+            # @return [Filtering::Config] alias to match the naming API for features
+            def filtering(*args)
+              filter(*args)
+            end
+
+            # @return [Boolean] is a given job throttled
+            def filtering?
+              filtering.active?
+            end
+
+            # @return [Hash] topic with all its native configuration options plus throttling
+            def to_h
+              super.merge(
+                filtering: filtering.to_h
+              ).freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/filtering.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        # Filtering provides a generic API allowing you to pre-filter messages before they are
+        # dispatched to jobs and processed.
+        #
+        # It allows for throttling, delayed jobs and other filtering implementations.
+        class Filtering < Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/long_running_job/config.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class LongRunningJob < Base
+          # Long-Running Jobs configuration
+          Config = Struct.new(
+            :active,
+            keyword_init: true
+          ) { alias_method :active?, :active }
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/long_running_job/contract.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class LongRunningJob < Base
+          # Rules around long-running job settings
+          class Contract < Contracts::Base
+            configure do |config|
+              config.error_messages = YAML.safe_load(
+                File.read(
+                  File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                )
+              ).fetch('en').fetch('validations').fetch('topic')
+            end
+
+            nested(:long_running_job) do
+              required(:active) { |val| [true, false].include?(val) }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/long_running_job/topic.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class LongRunningJob < Base
+          # Long-Running Jobs topic API extensions
+          module Topic
+            # @param active [Boolean] do we want to enable long-running job feature for this topic
+            def long_running_job(active = false)
+              @long_running_job ||= Config.new(active: active)
+            end
+
+            # @return [Boolean] is a given job on a topic a long-running one
+            def long_running_job?
+              long_running_job.active?
+            end
+
+            # @return [Hash] topic with all its native configuration options plus lrj
+            def to_h
+              super.merge(
+                long_running_job: long_running_job.to_h
+              ).freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/long_running_job.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Namespace for Pro routing enhancements
+    module Routing
+      # Namespace for additional Pro features
+      module Features
+        # Long-Running Jobs feature config and DSL namespace.
+        #
+        # Long-Running Jobs allow you to run Karafka jobs beyond `max.poll.interval.ms`
+        class LongRunningJob < Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/throttling/config.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Throttling < Base
+          # Throttling feature configuration
+          Config = Struct.new(
+            :active,
+            :limit,
+            :interval,
+            keyword_init: true
+          ) do
+            alias_method :active?, :active
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/throttling/contract.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Throttling < Base
+          # Rules around throttling settings
+          class Contract < Contracts::Base
+            configure do |config|
+              config.error_messages = YAML.safe_load(
+                File.read(
+                  File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                )
+              ).fetch('en').fetch('validations').fetch('topic')
+            end
+
+            nested(:throttling) do
+              required(:active) { |val| [true, false].include?(val) }
+              required(:interval) { |val| val.is_a?(Integer) && val.positive? }
+              required(:limit) do |val|
+                (val.is_a?(Integer) || val == Float::INFINITY) && val.positive?
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/throttling/topic.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Throttling < Base
+          # Topic throttling API extensions
+          module Topic
+            # @param limit [Integer] max messages to process in an time interval
+            # @param interval [Integer] time interval for processing
+            def throttling(
+              limit: Float::INFINITY,
+              interval: 60_000
+            )
+              # Those settings are used for validation
+              @throttling ||= begin
+                config = Config.new(
+                  active: limit != Float::INFINITY,
+                  limit: limit,
+                  interval: interval
+                )
+
+                # If someone defined throttling setup, we need to create appropriate filter for it
+                # and inject it via filtering feature
+                if config.active?
+                  factory = ->(*) { Pro::Processing::Filters::Throttler.new(limit, interval) }
+                  filter(factory)
+                end
+
+                config
+              end
+            end
+
+            # Just an alias for nice API
+            #
+            # @param args [Array] Anything `#throttling` accepts
+            def throttle(**args)
+              throttling(**args)
+            end
+
+            # @return [Boolean] is a given job throttled
+            def throttling?
+              throttling.active?
+            end
+
+            # @return [Hash] topic with all its native configuration options plus throttling
+            def to_h
+              super.merge(
+                throttling: throttling.to_h
+              ).freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/throttling.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        # Ability to throttle ingestion of data per topic partition
+        # Useful when we have fixed limit of things we can process in a given time period without
+        # getting into trouble. It can be used for example to:
+        #   - make sure we do not insert things to DB too fast
+        #   - make sure we do not dispatch HTTP requests to external resources too fast
+        #
+        # This feature is virtual. It materializes itself via the `Filtering` feature.
+        class Throttling < Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/virtual_partitions/config.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class VirtualPartitions < Base
+          # Config for virtual partitions
+          Config = Struct.new(
+            :active,
+            :partitioner,
+            :max_partitions,
+            keyword_init: true
+          ) { alias_method :active?, :active }
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/virtual_partitions/contract.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class VirtualPartitions < Base
+          # Rules around virtual partitions
+          class Contract < Contracts::Base
+            configure do |config|
+              config.error_messages = YAML.safe_load(
+                File.read(
+                  File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                )
+              ).fetch('en').fetch('validations').fetch('topic')
+            end
+
+            nested(:virtual_partitions) do
+              required(:active) { |val| [true, false].include?(val) }
+              required(:partitioner) { |val| val.nil? || val.respond_to?(:call) }
+              required(:max_partitions) { |val| val.is_a?(Integer) && val >= 1 }
+            end
+
+            # When virtual partitions are defined, partitioner needs to respond to `#call` and it
+            # cannot be nil
+            virtual do |data, errors|
+              next unless errors.empty?
+
+              virtual_partitions = data[:virtual_partitions]
+
+              next unless virtual_partitions[:active]
+              next if virtual_partitions[:partitioner].respond_to?(:call)
+
+              [[%i[virtual_partitions partitioner], :respond_to_call]]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/virtual_partitions/topic.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class VirtualPartitions < Base
+          # Topic extensions to be able to manage virtual partitions feature
+          module Topic
+            # @param max_partitions [Integer] max number of virtual partitions that can come out of
+            #   the single distribution flow. When set to more than the Karafka threading, will
+            #   create more work than workers. When less, can ensure we have spare resources to
+            #   process other things in parallel.
+            # @param partitioner [nil, #call] nil or callable partitioner
+            # @return [VirtualPartitions] method that allows to set the virtual partitions details
+            #   during the routing configuration and then allows to retrieve it
+            def virtual_partitions(
+              max_partitions: Karafka::App.config.concurrency,
+              partitioner: nil
+            )
+              @virtual_partitions ||= Config.new(
+                active: !partitioner.nil?,
+                max_partitions: max_partitions,
+                partitioner: partitioner
+              )
+            end
+
+            # @return [Boolean] are virtual partitions enabled for given topic
+            def virtual_partitions?
+              virtual_partitions.active?
+            end
+
+            # @return [Hash] topic with all its native configuration options plus manual offset
+            #   management namespace settings
+            def to_h
+              super.merge(
+                virtual_partitions: virtual_partitions.to_h
+              ).freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/virtual_partitions.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        # Virtual Partitions feature config and DSL namespace.
+        #
+        # Virtual Partitions allow you to parallelize the processing of data from a single
+        # partition. This can drastically increase throughput when IO operations are involved.
+        class VirtualPartitions < Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/process.rb

@@ -4,12 +4,16 @@ module Karafka
   # Class used to catch signals from ruby Signal class in order to manage Karafka stop
   # @note There might be only one process - this class is a singleton
   class Process
+    # Allow for process tagging for instrumentation
+    extend ::Karafka::Core::Taggable
+
     # Signal types that we handle
     HANDLED_SIGNALS = %i[
       SIGINT
       SIGQUIT
       SIGTERM
       SIGTTIN
+      SIGTSTP
     ].freeze
 
     HANDLED_SIGNALS.each do |signal|
@@ -29,33 +33,44 @@ module Karafka
     # Creates an instance of process and creates empty hash for callbacks
     def initialize
       @callbacks = Hash.new { |hsh, key| hsh[key] = [] }
+      @supervised = false
     end
 
     # Method catches all HANDLED_SIGNALS and performs appropriate callbacks (if defined)
     # @note If there are no callbacks, this method will just ignore a given signal that was sent
     def supervise
       HANDLED_SIGNALS.each { |signal| trap_signal(signal) }
+      @supervised = true
+    end
+
+    # Is the current process supervised and are trap signals installed
+    def supervised?
+      @supervised
     end
 
     private
 
     # Traps a single signal and performs callbacks (if any) or just ignores this signal
     # @param [Symbol] signal type that we want to catch
+    # @note Since we do a lot of threading and queuing, we don't want to handle signals from the
+    # trap context s some things may not work there as expected, that is why we spawn a separate
+    # thread to handle the signals process
     def trap_signal(signal)
-      trap(signal) do
-        notice_signal(signal)
-        (@callbacks[signal] || []).each(&:call)
+      previous_handler = ::Signal.trap(signal) do
+        Thread.new do
+          notice_signal(signal)
+
+          (@callbacks[signal] || []).each(&:call)
+        end
+
+        previous_handler.call if previous_handler.respond_to?(:call)
       end
     end
 
     # Informs monitoring about trapped signal
     # @param [Symbol] signal type that we received
-    # @note We cannot perform logging from trap context, that's why
-    #   we have to spin up a new thread to do this
     def notice_signal(signal)
-      Thread.new do
-        Karafka.monitor.instrument('process.notice_signal', caller: self, signal: signal)
-      end
+      Karafka.monitor.instrument('process.notice_signal', caller: self, signal: signal)
     end
   end
 end