karafka 2.5.9 → 2.6.0.beta1

data/lib/karafka/pro/routing/features/periodic_job/contracts/topic.rb

@@ -1,59 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Routing
-      module Features
-        class PeriodicJob < Base
-          # Namespace for periodics messages contracts
-          module Contracts
-            # Contract to validate configuration of the periodics feature
-            class Topic < Karafka::Contracts::Base
-              configure do |config|
-                config.error_messages = YAML.safe_load_file(
-                  File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
-                ).fetch("en").fetch("validations").fetch("routing").fetch("topic")
-              end
-
-              nested(:periodic_job) do
-                required(:active) { |val| [true, false].include?(val) }
-                required(:interval) { |val| val.is_a?(Integer) && val >= 100 }
-                required(:during_pause) { |val| [true, false].include?(val) }
-                required(:during_retry) { |val| [true, false].include?(val) }
-                required(:materialized) { |val| [true, false].include?(val) }
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/routing/features/periodic_job/topic.rb

@@ -1,120 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Routing
-      module Features
-        class PeriodicJob < Base
-          # Periodic topic action flows extensions
-          module Topic
-            # This method calls the parent class initializer and then sets up the
-            # extra instance variable to nil. The explicit initialization
-            # to nil is included as an optimization for Ruby's object shapes system,
-            # which improves memory layout and access performance.
-            def initialize(...)
-              super
-              @periodic_job = nil
-            end
-
-            # Defines topic as periodic. Periodic topics consumers will invoke `#tick` with each
-            # poll where messages were not received.
-            # @param active [Boolean] should ticking happen for this topic assignments.
-            # @param interval [Integer] minimum interval to run periodic jobs on given topic.
-            # @param during_pause [Boolean, nil] Should periodic jobs run when partition is paused.
-            #   It is set to `nil` by default allowing for detection when this value is not
-            #   configured but should be built dynamically based on LRJ status.
-            # @param during_retry [Boolean, nil] Should we run when there was an error and we are
-            #   in a retry flow. Please note that for this to work, `during_pause` also needs to be
-            #   set to true as errors retry happens after pause.
-            def periodic_job(
-              active = false,
-              interval: nil,
-              during_pause: nil,
-              during_retry: nil
-            )
-              @periodic_job ||= begin
-                # Set to active if any of the values was configured
-                active = true unless interval.nil?
-                active = true unless during_pause.nil?
-                active = true unless during_retry.nil?
-                # Default is not to retry during retry flow
-                during_retry = false if during_retry.nil?
-
-                # If no interval, use default
-                interval ||= Karafka::App.config.internal.tick_interval
-
-                Config.new(
-                  active: active,
-                  interval: interval,
-                  during_pause: during_pause,
-                  during_retry: during_retry,
-                  # This is internal setting for state management, not part of the configuration
-                  # Do not overwrite.
-                  # If `during_pause` is explicit, we do not select it based on LRJ setup and we
-                  # consider if fully ready out of the box
-                  materialized: !during_pause.nil?
-                )
-              end
-
-              return @periodic_job if @periodic_job.materialized?
-              return @periodic_job unless @long_running_job
-
-              # If not configured in any way, we want not to process during pause for LRJ.
-              # LRJ pauses by default when processing and during this time we do not want to
-              # tick at all. This prevents us from running periodic jobs while LRJ jobs are
-              # running. This of course has a side effect of not running when paused for any
-              # other reason but it is a compromise in the default settings
-              @periodic_job.during_pause = !long_running_job?
-              @periodic_job.materialized = true
-
-              @periodic_job
-            end
-
-            alias_method :periodic, :periodic_job
-
-            # @return [Boolean] is periodics active
-            def periodic_job?
-              periodic_job.active?
-            end
-
-            # @return [Hash] topic with all its native configuration options plus periodics flows
-            #   settings
-            def to_h
-              super.merge(
-                periodic_job: periodic_job.to_h
-              ).freeze
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/routing/features/periodic_job.rb

@@ -1,44 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Routing
-      module Features
-        # Feature allowing to run consumer operations even when no data is present on periodic
-        # interval.
-        # This allows for advanced window-based operations regardless of income of new data and
-        # other advanced cases where the consumer is needed even when no data is coming
-        class PeriodicJob < Base
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/routing/features/recurring_tasks/builder.rb

@@ -1,148 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Routing
-      module Features
-        class RecurringTasks < Base
-          # Routing extensions for recurring tasks
-          module Builder
-            # Enabled recurring tasks operations and adds needed topics and other stuff.
-            #
-            # @param active [Boolean] should recurring tasks be active. We use a boolean flag to
-            #   have API consistency in the system, so it matches other routing related APIs.
-            # @param block [Proc] optional reconfiguration of the topics definitions.
-            def recurring_tasks(active = false, &block)
-              return unless active
-
-              # We only require zlib when we decide to run recurring tasks because it is not needed
-              # otherwise.
-              require "zlib"
-              ensure_fugit_availability!
-
-              tasks_cfg = App.config.recurring_tasks
-              topics_cfg = tasks_cfg.topics
-
-              consumer_group tasks_cfg.group_id do
-                # Registers the primary topic that we use to control schedules execution. This is
-                # the one that we use to trigger recurring tasks.
-                schedules_topic = topic(topics_cfg.schedules.name) do
-                  consumer tasks_cfg.consumer_class
-                  deserializer tasks_cfg.deserializer
-                  # Because the topic method name as well as builder proxy method name is the same
-                  # we need to reference it via target directly
-                  target.recurring_tasks(true)
-
-                  # We manage offsets directly because messages can have both schedules and
-                  # commands and we need to apply them only when we need to
-                  manual_offset_management(true)
-
-                  # We use multi-batch operations and in-memory state for schedules. This needs to
-                  # always operate without re-creation.
-                  consumer_persistence(true)
-
-                  # This needs to be enabled for the eof to work correctly
-                  kafka("enable.partition.eof": true, inherit: true)
-                  eofed(true)
-
-                  # Favour latency. This is a low traffic topic that only accepts user initiated
-                  # low-frequency commands
-                  max_messages(1)
-                  # Since we always react on the received message, we can block for longer periods
-                  # of time
-                  max_wait_time(10_000)
-
-                  # Since the execution of particular tasks is isolated and guarded, it should not
-                  # leak. This means, that this is to handle errors like schedule version
-                  # incompatibility and other errors that will not go away without a redeployment
-                  pause_timeout(60 * 1_000)
-                  pause_max_timeout(60 * 1_000)
-
-                  # Keep older data for a day and compact to the last state available
-                  config(
-                    "cleanup.policy": "compact,delete",
-                    "retention.ms": 86_400_000
-                  )
-
-                  # This is the core of execution. Since we're producers of states, we need a way
-                  # to tick without having new data
-                  periodic(
-                    interval: App.config.recurring_tasks.interval,
-                    during_pause: false,
-                    during_retry: false
-                  )
-
-                  # If this is the direct schedules redefinition style, we run it
-                  # The second one (see end of this method) allows for linear reconfiguration of
-                  # both the topics
-                  instance_eval(&block) if block && block.arity.zero?
-                end
-
-                # This topic is to store logs that we can then inspect either from the admin or via
-                # the Web UI
-                logs_topic = topic(topics_cfg.logs.name) do
-                  active(false)
-                  deserializer tasks_cfg.deserializer
-                  target.recurring_tasks(true)
-
-                  # Keep cron logs of executions for a week and after that remove. Week should be
-                  # enough and should not produce too much data.
-                  config(
-                    "cleanup.policy": "delete",
-                    "retention.ms": 604_800_000
-                  )
-                end
-
-                yield(schedules_topic, logs_topic) if block && block.arity.positive?
-              end
-            end
-
-            # Checks if fugit is present. If not, will try to require it as it might not have
-            # been required but is available. If fails, will crash.
-            def ensure_fugit_availability!
-              return if Object.const_defined?(:Fugit)
-
-              require "fugit"
-            rescue LoadError
-              raise(
-                Karafka::Errors::DependencyConstraintsError,
-                <<~ERROR_MSG
-                  Failed to require fugit gem.
-                  Add it to your Gemfile, as it is required for the recurring tasks to work.
-                ERROR_MSG
-              )
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/routing/features/recurring_tasks/config.rb

@@ -1,45 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Routing
-      module Features
-        class RecurringTasks < Base
-          # Recurring tasks configuration
-          Config = Struct.new(
-            :active,
-            keyword_init: true
-          ) { alias_method :active?, :active }
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/routing/features/recurring_tasks/topic.rb

@@ -1,70 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Routing
-      module Features
-        class RecurringTasks < Base
-          # Topic extensions to be able to check if given topic is a recurring tasks topic
-          # Please note, that this applies to both the schedules topics and reports topics
-          module Topic
-            # This method calls the parent class initializer and then sets up the
-            # extra instance variable to nil. The explicit initialization
-            # to nil is included as an optimization for Ruby's object shapes system,
-            # which improves memory layout and access performance.
-            def initialize(...)
-              super
-              @recurring_tasks = nil
-            end
-
-            # @param active [Boolean] should this topic be considered related to recurring tasks
-            def recurring_tasks(active = false)
-              @recurring_tasks ||= Config.new(active: active)
-            end
-
-            # @return [Boolean] is this an ActiveJob topic
-            def recurring_tasks?
-              recurring_tasks.active?
-            end
-
-            # @return [Hash] topic with all its native configuration options plus active job
-            #   namespace settings
-            def to_h
-              super.merge(
-                recurring_tasks: recurring_tasks.to_h
-              ).freeze
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/routing/features/recurring_tasks.rb

@@ -1,42 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Routing
-      module Features
-        # This feature allows you to run recurring tasks aka cron jobs from within Karafka
-        # It uses two special topics for tracking the schedule and reporting executions
-        class RecurringTasks < Base
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/routing/features/scheduled_messages/builder.rb

@@ -1,152 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Routing
-      module Features
-        class ScheduledMessages < Base
-          # Routing extensions for scheduled messages
-          module Builder
-            # Enabled scheduled messages operations and adds needed topics and other stuff.
-            #
-            # @param topic_name [String, false] name for scheduled messages topic that is also used
-            #   as a group identifier. Users can have multiple schedule topics flows to prevent key
-            #   collisions, prioritize and do other stuff. `false` if not active.
-            # @param block [Proc] optional reconfiguration of the topics definitions.
-            # @note Namespace for topics should include the divider as it is not automatically
-            #   added.
-            def scheduled_messages(topic_name = false, &block)
-              return unless topic_name
-
-              # Load zlib only if user enables scheduled messages
-              require "zlib"
-
-              # We set it to 5 so we have enough space to handle more events. All related topics
-              # should have same partition count.
-              default_partitions = 5
-              msg_cfg = App.config.scheduled_messages
-
-              consumer_group msg_cfg.group_id do
-                # Registers the primary topic that we use to control schedules execution. This is
-                # the one that we use to trigger scheduled messages.
-                messages_topic = topic(topic_name) do
-                  instance_eval(&block) if block && block.arity.zero?
-
-                  consumer msg_cfg.consumer_class
-
-                  deserializers(
-                    headers: msg_cfg.deserializers.headers
-                  )
-
-                  # Because the topic method name as well as builder proxy method name is the same
-                  # we need to reference it via target directly
-                  target.scheduled_messages(true)
-
-                  # We manage offsets directly because messages can have both schedules and
-                  # commands and we need to apply them only when we need to
-                  manual_offset_management(true)
-
-                  # We use multi-batch operations and in-memory state for schedules. This needs to
-                  # always operate without re-creation.
-                  consumer_persistence(true)
-
-                  # This needs to be enabled for the eof to work correctly
-                  kafka(
-                    "enable.partition.eof": true,
-                    "auto.offset.reset": "earliest",
-                    inherit: true
-                  )
-                  eofed(true)
-
-                  # Since this is a topic that gets replayed because of schedule management, we do
-                  # want to get more data faster during recovery
-                  max_messages(10_000)
-
-                  max_wait_time(1_000)
-
-                  # This is a setup that should allow messages to be compacted fairly fast. Since
-                  # each dispatched message should be removed via tombstone, they do not have to
-                  # be present in the topic for too long.
-                  config(
-                    partitions: default_partitions,
-                    # Will ensure, that after tombstone is present, given scheduled message, that
-                    # has been dispatched is removed by Kafka
-                    "cleanup.policy": "compact",
-                    # When 10% or more dispatches are done, compact data
-                    "min.cleanable.dirty.ratio": 0.1,
-                    # Frequent segment rotation to support intense compaction
-                    "segment.ms": 3_600_000,
-                    "delete.retention.ms": 3_600_000,
-                    "segment.bytes": 52_428_800
-                  )
-
-                  # This is the core of execution. Since we dispatch data in time intervals, we
-                  # need to be able to do this even when no new data is coming
-                  periodic(
-                    interval: msg_cfg.interval,
-                    during_pause: false,
-                    during_retry: false
-                  )
-
-                  # If this is the direct schedules redefinition style, we run it
-                  # The second one (see end of this method) allows for linear reconfiguration of
-                  # both the topics
-                  instance_eval(&block) if block && block.arity.zero?
-                end
-
-                # Holds states of scheduler per each of the partitions since they tick
-                # independently. We only hold future statistics not to have to deal with
-                # any type of state restoration
-                states_topic = topic("#{topic_name}#{msg_cfg.states_postfix}") do
-                  active(false)
-                  target.scheduled_messages(true)
-                  config(
-                    partitions: default_partitions,
-                    "cleanup.policy": "compact",
-                    "min.cleanable.dirty.ratio": 0.1,
-                    "segment.ms": 3_600_000,
-                    "delete.retention.ms": 3_600_000,
-                    "segment.bytes": 52_428_800
-                  )
-                  deserializers(
-                    payload: msg_cfg.deserializers.payload
-                  )
-                end
-
-                yield(messages_topic, states_topic) if block && block.arity.positive?
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end