karafka 2.5.9 → 2.6.0.beta1

data/lib/karafka/pro/routing/features/consumer_groups/periodic_job.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          # Feature allowing to run consumer operations even when no data is present on periodic
+          # interval.
+          # This allows for advanced window-based operations regardless of income of new data and
+          # other advanced cases where the consumer is needed even when no data is coming
+          class PeriodicJob < Base
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/consumer_groups/recurring_tasks/builder.rb

@@ -0,0 +1,150 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          class RecurringTasks < Base
+            # Routing extensions for recurring tasks
+            module Builder
+              # Enabled recurring tasks operations and adds needed topics and other stuff.
+              #
+              # @param active [Boolean] should recurring tasks be active. We use a boolean flag to
+              #   have API consistency in the system, so it matches other routing related APIs.
+              # @param block [Proc] optional reconfiguration of the topics definitions.
+              def recurring_tasks(active = false, &block)
+                return unless active
+
+                # We only require zlib when we decide to run recurring tasks because it is not needed
+                # otherwise.
+                require "zlib"
+                ensure_fugit_availability!
+
+                tasks_cfg = App.config.recurring_tasks
+                topics_cfg = tasks_cfg.topics
+
+                consumer_group tasks_cfg.group_id do
+                  # Registers the primary topic that we use to control schedules execution. This is
+                  # the one that we use to trigger recurring tasks.
+                  schedules_topic = topic(topics_cfg.schedules.name) do
+                    consumer tasks_cfg.consumer_class
+                    deserializer tasks_cfg.deserializer
+                    # Because the topic method name as well as builder proxy method name is the same
+                    # we need to reference it via target directly
+                    target.recurring_tasks(true)
+
+                    # We manage offsets directly because messages can have both schedules and
+                    # commands and we need to apply them only when we need to
+                    manual_offset_management(true)
+
+                    # We use multi-batch operations and in-memory state for schedules. This needs to
+                    # always operate without re-creation.
+                    consumer_persistence(true)
+
+                    # This needs to be enabled for the eof to work correctly
+                    kafka("enable.partition.eof": true, inherit: true)
+                    eofed(true)
+
+                    # Favour latency. This is a low traffic topic that only accepts user initiated
+                    # low-frequency commands
+                    max_messages(1)
+                    # Since we always react on the received message, we can block for longer periods
+                    # of time
+                    max_wait_time(10_000)
+
+                    # Since the execution of particular tasks is isolated and guarded, it should not
+                    # leak. This means, that this is to handle errors like schedule version
+                    # incompatibility and other errors that will not go away without a redeployment
+                    pause_timeout(60 * 1_000)
+                    pause_max_timeout(60 * 1_000)
+
+                    # Keep older data for a day and compact to the last state available
+                    config(
+                      "cleanup.policy": "compact,delete",
+                      "retention.ms": 86_400_000
+                    )
+
+                    # This is the core of execution. Since we're producers of states, we need a way
+                    # to tick without having new data
+                    periodic(
+                      interval: App.config.recurring_tasks.interval,
+                      during_pause: false,
+                      during_retry: false
+                    )
+
+                    # If this is the direct schedules redefinition style, we run it
+                    # The second one (see end of this method) allows for linear reconfiguration of
+                    # both the topics
+                    instance_eval(&block) if block && block.arity.zero?
+                  end
+
+                  # This topic is to store logs that we can then inspect either from the admin or via
+                  # the Web UI
+                  logs_topic = topic(topics_cfg.logs.name) do
+                    active(false)
+                    deserializer tasks_cfg.deserializer
+                    target.recurring_tasks(true)
+
+                    # Keep cron logs of executions for a week and after that remove. Week should be
+                    # enough and should not produce too much data.
+                    config(
+                      "cleanup.policy": "delete",
+                      "retention.ms": 604_800_000
+                    )
+                  end
+
+                  yield(schedules_topic, logs_topic) if block && block.arity.positive?
+                end
+              end
+
+              # Checks if fugit is present. If not, will try to require it as it might not have
+              # been required but is available. If fails, will crash.
+              def ensure_fugit_availability!
+                return if Object.const_defined?(:Fugit)
+
+                require "fugit"
+              rescue LoadError
+                raise(
+                  Karafka::Errors::DependencyConstraintsError,
+                  <<~ERROR_MSG
+                    Failed to require fugit gem.
+                    Add it to your Gemfile, as it is required for the recurring tasks to work.
+                  ERROR_MSG
+                )
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/{swarm → consumer_groups/recurring_tasks}/config.rb

@@ -32,14 +32,13 @@ module Karafka
   module Pro
     module Routing
       module Features
-        class Swarm < Base
-          # Swarm feature configuration
-          Config = Struct.new(
-            :active,
-            :nodes,
-            keyword_init: true
-          ) do
-            alias_method :active?, :active
+        module ConsumerGroups
+          class RecurringTasks < Base
+            # Recurring tasks configuration
+            Config = Struct.new(
+              :active,
+              keyword_init: true
+            ) { alias_method :active?, :active }
           end
         end
       end

data/lib/karafka/pro/routing/features/{inline_insights → consumer_groups/recurring_tasks}/contracts/topic.rb

@@ -32,20 +32,21 @@ module Karafka
   module Pro
     module Routing
       module Features
-        class InlineInsights < Base
-          # Inline Insights related contracts namespace
-          module Contracts
-            # Contract for inline insights topic setup
-            class Topic < Karafka::Contracts::Base
-              configure do |config|
-                config.error_messages = YAML.safe_load_file(
-                  File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
-                ).fetch("en").fetch("validations").fetch("routing").fetch("topic")
-              end
+        module ConsumerGroups
+          class RecurringTasks < Base
+            # Namespace for recurring tasks contracts
+            module Contracts
+              # Rules around recurring tasks settings
+              class Topic < Karafka::Contracts::Base
+                configure do |config|
+                  config.error_messages = YAML.safe_load_file(
+                    File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
+                  ).fetch("en").fetch("validations").fetch("routing").fetch("topic")
+                end
 
-              nested :inline_insights do
-                required(:active) { |val| [true, false].include?(val) }
-                required(:required) { |val| [true, false].include?(val) }
+                nested(:recurring_tasks) do
+                  required(:active) { |val| [true, false].include?(val) }
+                end
               end
             end
           end

data/lib/karafka/pro/routing/features/{scheduled_messages → consumer_groups/recurring_tasks}/proxy.rb

@@ -32,10 +32,12 @@ module Karafka
   module Pro
     module Routing
       module Features
-        class ScheduledMessages < Base
-          # Routing proxy extensions for scheduled messages
-          module Proxy
-            include Builder
+        module ConsumerGroups
+          class RecurringTasks < Base
+            # Routing proxy extensions for recurring tasks
+            module Proxy
+              include Builder
+            end
           end
         end
       end

data/lib/karafka/pro/routing/features/consumer_groups/recurring_tasks/topic.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          class RecurringTasks < Base
+            # Topic extensions to be able to check if given topic is a recurring tasks topic
+            # Please note, that this applies to both the schedules topics and reports topics
+            module Topic
+              # This method sets up the extra instance variable to nil before calling
+              # the parent class initializer. The explicit initialization
+              # to nil is included as an optimization for Ruby's object shapes system,
+              # which improves memory layout and access performance.
+              def initialize(...)
+                @recurring_tasks = nil
+                super
+              end
+
+              # @param active [Boolean] should this topic be considered related to recurring tasks
+              def recurring_tasks(active = false)
+                @recurring_tasks ||= Config.new(active: active)
+              end
+
+              # @return [Boolean] is this an ActiveJob topic
+              def recurring_tasks?
+                recurring_tasks.active?
+              end
+
+              # @return [Hash] topic with all its native configuration options plus active job
+              #   namespace settings
+              def to_h
+                super.merge(
+                  recurring_tasks: recurring_tasks.to_h
+                ).freeze
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/consumer_groups/recurring_tasks.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          # This feature allows you to run recurring tasks aka cron jobs from within Karafka
+          # It uses two special topics for tracking the schedule and reporting executions
+          class RecurringTasks < Base
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/consumer_groups/scheduled_messages/builder.rb

@@ -0,0 +1,154 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          class ScheduledMessages < Base
+            # Routing extensions for scheduled messages
+            module Builder
+              # Enabled scheduled messages operations and adds needed topics and other stuff.
+              #
+              # @param topic_name [String, false] name for scheduled messages topic that is also used
+              #   as a group identifier. Users can have multiple schedule topics flows to prevent key
+              #   collisions, prioritize and do other stuff. `false` if not active.
+              # @param block [Proc] optional reconfiguration of the topics definitions.
+              # @note Namespace for topics should include the divider as it is not automatically
+              #   added.
+              def scheduled_messages(topic_name = false, &block)
+                return unless topic_name
+
+                # Load zlib only if user enables scheduled messages
+                require "zlib"
+
+                # We set it to 5 so we have enough space to handle more events. All related topics
+                # should have same partition count.
+                default_partitions = 5
+                msg_cfg = App.config.scheduled_messages
+
+                consumer_group msg_cfg.group_id do
+                  # Registers the primary topic that we use to control schedules execution. This is
+                  # the one that we use to trigger scheduled messages.
+                  messages_topic = topic(topic_name) do
+                    instance_eval(&block) if block && block.arity.zero?
+
+                    consumer msg_cfg.consumer_class
+
+                    deserializers(
+                      headers: msg_cfg.deserializers.headers
+                    )
+
+                    # Because the topic method name as well as builder proxy method name is the same
+                    # we need to reference it via target directly
+                    target.scheduled_messages(true)
+
+                    # We manage offsets directly because messages can have both schedules and
+                    # commands and we need to apply them only when we need to
+                    manual_offset_management(true)
+
+                    # We use multi-batch operations and in-memory state for schedules. This needs to
+                    # always operate without re-creation.
+                    consumer_persistence(true)
+
+                    # This needs to be enabled for the eof to work correctly
+                    kafka(
+                      "enable.partition.eof": true,
+                      "auto.offset.reset": "earliest",
+                      inherit: true
+                    )
+                    eofed(true)
+
+                    # Since this is a topic that gets replayed because of schedule management, we do
+                    # want to get more data faster during recovery
+                    max_messages(10_000)
+
+                    max_wait_time(1_000)
+
+                    # This is a setup that should allow messages to be compacted fairly fast. Since
+                    # each dispatched message should be removed via tombstone, they do not have to
+                    # be present in the topic for too long.
+                    config(
+                      partitions: default_partitions,
+                      # Will ensure, that after tombstone is present, given scheduled message, that
+                      # has been dispatched is removed by Kafka
+                      "cleanup.policy": "compact",
+                      # When 10% or more dispatches are done, compact data
+                      "min.cleanable.dirty.ratio": 0.1,
+                      # Frequent segment rotation to support intense compaction
+                      "segment.ms": 3_600_000,
+                      "delete.retention.ms": 3_600_000,
+                      "segment.bytes": 52_428_800
+                    )
+
+                    # This is the core of execution. Since we dispatch data in time intervals, we
+                    # need to be able to do this even when no new data is coming
+                    periodic(
+                      interval: msg_cfg.interval,
+                      during_pause: false,
+                      during_retry: false
+                    )
+
+                    # If this is the direct schedules redefinition style, we run it
+                    # The second one (see end of this method) allows for linear reconfiguration of
+                    # both the topics
+                    instance_eval(&block) if block && block.arity.zero?
+                  end
+
+                  # Holds states of scheduler per each of the partitions since they tick
+                  # independently. We only hold future statistics not to have to deal with
+                  # any type of state restoration
+                  states_topic = topic("#{topic_name}#{msg_cfg.states_postfix}") do
+                    active(false)
+                    target.scheduled_messages(true)
+                    config(
+                      partitions: default_partitions,
+                      "cleanup.policy": "compact",
+                      "min.cleanable.dirty.ratio": 0.1,
+                      "segment.ms": 3_600_000,
+                      "delete.retention.ms": 3_600_000,
+                      "segment.bytes": 52_428_800
+                    )
+                    deserializers(
+                      payload: msg_cfg.deserializers.payload
+                    )
+                  end
+
+                  yield(messages_topic, states_topic) if block && block.arity.positive?
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/consumer_groups/scheduled_messages/config.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          class ScheduledMessages < Base
+            # Scheduled messages configuration
+            Config = Struct.new(
+              :active,
+              keyword_init: true
+            ) { alias_method :active?, :active }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/{long_running_job → consumer_groups/scheduled_messages}/contracts/topic.rb

@@ -32,19 +32,21 @@ module Karafka
   module Pro
     module Routing
       module Features
-        class LongRunningJob < Base
-          # Namespace for LRJ contracts
-          module Contracts
-            # Rules around long-running job settings
-            class Topic < Karafka::Contracts::Base
-              configure do |config|
-                config.error_messages = YAML.safe_load_file(
-                  File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
-                ).fetch("en").fetch("validations").fetch("routing").fetch("topic")
-              end
+        module ConsumerGroups
+          class ScheduledMessages < Base
+            # Namespace for scheduled messages contracts
+            module Contracts
+              # Rules around scheduled messages settings
+              class Topic < Karafka::Contracts::Base
+                configure do |config|
+                  config.error_messages = YAML.safe_load_file(
+                    File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
+                  ).fetch("en").fetch("validations").fetch("routing").fetch("topic")
+                end
 
-              nested(:long_running_job) do
-                required(:active) { |val| [true, false].include?(val) }
+                nested(:scheduled_messages) do
+                  required(:active) { |val| [true, false].include?(val) }
+                end
               end
             end
           end

data/lib/karafka/pro/routing/features/{delaying/config.rb → consumer_groups/scheduled_messages/proxy.rb}

@@ -32,10 +32,12 @@ module Karafka
   module Pro
     module Routing
       module Features
-        class Delaying < Base
-          # Delaying feature configuration
-          Config = Struct.new(:active, :delay, keyword_init: true) do
-            alias_method :active?, :active
+        module ConsumerGroups
+          class ScheduledMessages < Base
+            # Routing proxy extensions for scheduled messages
+            module Proxy
+              include Builder
+            end
           end
         end
       end