karafka 1.4.12 → 2.2.10

data/lib/karafka/pro/routing/features/expiring/topic.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Expiring < Base
+          # Topic expiring API extensions
+          module Topic
+            # @param ttl [Integer, nil] maximum time in ms a message is considered alive
+            def expiring(ttl = nil)
+              # Those settings are used for validation
+              @expiring ||= begin
+                config = Config.new(active: !ttl.nil?, ttl: ttl)
+
+                if config.active?
+                  factory = ->(*) { Pro::Processing::Filters::Expirer.new(ttl) }
+                  filter(factory)
+                end
+
+                config
+              end
+            end
+
+            # Just an alias for nice API
+            #
+            # @param args [Array] Anything `#expiring` accepts
+            def expire_in(*args)
+              expiring(*args)
+            end
+
+            # @return [Boolean] is a given job expiring
+            def expiring?
+              expiring.active?
+            end
+
+            # @return [Hash] topic with all its native configuration options plus expiring
+            def to_h
+              super.merge(
+                expiring: expiring.to_h
+              ).freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/expiring.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        # Expiring allows us to filter out messages that are older than our expectation.
+        # This can also be done in a consumer, but applying the filtering prior to the jobs
+        # enqueuing allows us to improve operations with virtual partitions and limit the number of
+        # not necessary messages being ever seen
+        class Expiring < Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/filtering/config.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Filtering < Base
+          # Filtering feature configuration
+          Config = Struct.new(:factories, keyword_init: true) do
+            # @return [Boolean] is this feature in use. Are any filters defined
+            def active?
+              !factories.empty?
+            end
+
+            # @return [Array<Object>] array of filters applicable to a topic partition
+            def filters
+              factories.map(&:call)
+            end
+
+            # @return [Hash] this config hash
+            def to_h
+              super.merge(active: active?)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/filtering/contracts/topic.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Filtering < Base
+          # Namespace for filtering feature contracts
+          module Contracts
+            # Contract to validate configuration of the filtering feature
+            class Topic < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('topic')
+              end
+
+              nested(:filtering) do
+                required(:active) { |val| [true, false].include?(val) }
+
+                required(:factories) do |val|
+                  val.is_a?(Array) && val.all? { |factory| factory.respond_to?(:call) }
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/filtering/topic.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Filtering < Base
+          # Filtering feature topic extensions
+          module Topic
+            # @param factory [#call, nil] Callable that can produce new filters instances per
+            #   assigned topic partition. nil as default so this feature is disabled
+            def filter(factory = nil)
+              @filtering ||= Config.new(factories: [])
+              @filtering.factories << factory if factory
+              @filtering
+            end
+
+            # @param args [Array] Anything `#filter` accepts
+            # @return [Filtering::Config] alias to match the naming API for features
+            def filtering(*args)
+              filter(*args)
+            end
+
+            # @return [Boolean] is a given job throttled
+            def filtering?
+              filtering.active?
+            end
+
+            # @return [Hash] topic with all its native configuration options plus throttling
+            def to_h
+              super.merge(
+                filtering: filtering.to_h
+              ).freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/filtering.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        # Filtering provides a generic API allowing you to pre-filter messages before they are
+        # dispatched to jobs and processed.
+        #
+        # It allows for throttling, delayed jobs and other filtering implementations.
+        class Filtering < Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/inline_insights/config.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class InlineInsights < Base
+          # Config of this feature
+          Config = Struct.new(
+            :active,
+            :required,
+            keyword_init: true
+          ) do
+            alias_method :active?, :active
+            alias_method :required?, :required
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/inline_insights/contracts/topic.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class InlineInsights < Base
+          # Inline Insights related contracts namespace
+          module Contracts
+            # Contract for inline insights topic setup
+            class Topic < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('topic')
+              end
+
+              nested :inline_insights do
+                required(:active) { |val| [true, false].include?(val) }
+                required(:required) { |val| [true, false].include?(val) }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/inline_insights/topic.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class InlineInsights < Base
+          # Routing topic inline insights API
+          module Topic
+            # @param active [Boolean] should inline insights be activated
+            # @param required [Boolean] are the insights required to operate
+            def inline_insights(active = -1, required: -1)
+              # This weird style of checking allows us to activate inline insights in few ways:
+              #   - inline_insights(true)
+              #   - inline_insights(required: true)
+              #   - inline_insights(required: false)
+              #
+              # In each of those cases inline insights will become active
+              @inline_insights ||= begin
+                config = Config.new(
+                  active: active == true || (active == -1 && required != -1),
+                  required: required == true
+                )
+
+                if config.active? && config.required?
+                  factory = lambda do |topic, partition|
+                    Pro::Processing::Filters::InlineInsightsDelayer.new(topic, partition)
+                  end
+
+                  filter(factory)
+                end
+
+                config
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/inline_insights.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        # Enhanced inline insights
+        # Allows you to set up an automatic filter that will ensure, that metrics are always
+        # available when processing starts.
+        class InlineInsights < Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/long_running_job/config.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class LongRunningJob < Base
+          # Long-Running Jobs configuration
+          Config = Struct.new(
+            :active,
+            keyword_init: true
+          ) { alias_method :active?, :active }
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/long_running_job/contracts/topic.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class LongRunningJob < Base
+          # Namespace for LRJ contracts
+          module Contracts
+            # Rules around long-running job settings
+            class Topic < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('topic')
+              end
+
+              nested(:long_running_job) do
+                required(:active) { |val| [true, false].include?(val) }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/long_running_job/topic.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class LongRunningJob < Base
+          # Long-Running Jobs topic API extensions
+          module Topic
+            # @param active [Boolean] do we want to enable long-running job feature for this topic
+            def long_running_job(active = false)
+              @long_running_job ||= Config.new(active: active)
+            end
+
+            # @return [Boolean] is a given job on a topic a long-running one
+            def long_running_job?
+              long_running_job.active?
+            end
+
+            # @return [Hash] topic with all its native configuration options plus lrj
+            def to_h
+              super.merge(
+                long_running_job: long_running_job.to_h
+              ).freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/long_running_job.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Namespace for Pro routing enhancements
+    module Routing
+      # Namespace for additional Pro features
+      module Features
+        # Long-Running Jobs feature config and DSL namespace.
+        #
+        # Long-Running Jobs allow you to run Karafka jobs beyond `max.poll.interval.ms`
+        class LongRunningJob < Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/patterns/builder.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Patterns < Base
+          # Expansions for the routing builder
+          module Builder
+            # Allows us to define the simple routing pattern matching
+            #
+            # @param regexp_or_name [Symbol, String, Regexp] name of the pattern or regexp for
+            #   automatic-based named patterns
+            # @param regexp [Regexp, nil] nil if we use auto-generated name based on the regexp or
+            #   the regexp if we used named patterns
+            # @param block [Proc]
+            def pattern(regexp_or_name, regexp = nil, &block)
+              consumer_group('app') do
+                pattern(regexp_or_name, regexp, &block)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/patterns/config.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        # Holds pattern info reference
+        # Type is set to:
+        #   `:regular` - in case patterns are not used and topic is just a regular existing topic
+        #                matched directly based on the name
+        #   `:discovered` - in case it is a real topic on which we started to listed
+        #   `:matcher` - represents a regular expression used by librdkafka
+        class Patterns < Base
+          # Config for pattern based topic
+          # Only pattern related topics are active in this context
+          Config = Struct.new(
+            :active,
+            :type,
+            :pattern,
+            keyword_init: true
+          ) do
+            alias_method :active?, :active
+
+            # @return [Boolean] is this a matcher topic
+            def matcher?
+              type == :matcher
+            end
+
+            # @return [Boolean] is this a discovered topic
+            def discovered?
+              type == :discovered
+            end
+
+            # @return [Boolean] is this a regular topic
+            def regular?
+              type == :regular
+            end
+          end
+        end
+      end
+    end
+  end
+end