karafka 2.4.0 → 2.4.18

data/lib/karafka/pro/recurring_tasks/setup/config.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module RecurringTasks
+      # Setup and config related recurring tasks components
+      module Setup
+        # Config for recurring tasks
+        class Config
+          extend ::Karafka::Core::Configurable
+
+          setting(:consumer_class, default: Consumer)
+          setting(:deserializer, default: Deserializer.new)
+          setting(:group_id, default: 'karafka_recurring_tasks')
+          # By default we will run the scheduling every 15 seconds since we provide a minute-based
+          # precision
+          setting(:interval, default: 15_000)
+          # Should we log the executions. If true (default) with each cron execution, there will
+          # be a special message published. Useful for debugging.
+          setting(:logging, default: true)
+
+          # Producer to be used by the recurring tasks.
+          # By default it is a `Karafka.producer`, however it may be overwritten if we want to use
+          # a separate instance in case of heavy usage of the  transactional producer, etc.
+          setting(
+            :producer,
+            constructor: -> { ::Karafka.producer },
+            lazy: true
+          )
+
+          setting(:topics) do
+            setting(:schedules, default: 'karafka_recurring_tasks_schedules')
+            setting(:logs, default: 'karafka_recurring_tasks_logs')
+          end
+
+          configure
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/recurring_tasks/task.rb

@@ -0,0 +1,143 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module RecurringTasks
+      # Represents a single recurring task that can be executed when the time comes.
+      # Tasks should be lightweight. Anything heavy should be executed by scheduling appropriate
+      # jobs here.
+      class Task
+        include Helpers::ConfigImporter.new(
+          monitor: %i[monitor]
+        )
+
+        # @return [String] this task id
+        attr_reader :id
+
+        # @return [Fugit::Cron] cron from parsing the raw cron expression
+        attr_reader :cron
+
+        # Allows for update of previous time when restoring the materialized state
+        attr_accessor :previous_time
+
+        # @param id [String] unique id. If re-used between versions, will replace older occurrence.
+        # @param cron [String] cron expression matching this task expected execution times.
+        # @param previous_time [Integer, Time] previous time this task was executed. 0 if never.
+        # @param enabled [Boolean] should this task be enabled. Users may disable given task
+        #   temporarily, this is why we need to know that.
+        # @param block [Proc] code to execute.
+        def initialize(id:, cron:, previous_time: 0, enabled: true, &block)
+          @id = id
+          @cron = ::Fugit::Cron.do_parse(cron)
+          @previous_time = previous_time
+          @start_time = Time.now
+          @executable = block
+          @enabled = enabled
+          @trigger = false
+          @changed = false
+        end
+
+        # @return [Boolean] true if anything in the task has changed and we should persist it
+        def changed?
+          @changed
+        end
+
+        # Disables this task execution indefinitely
+        def disable
+          touch
+          @enabled = false
+        end
+
+        # Enables back this task
+        def enable
+          touch
+          @enabled = true
+        end
+
+        # @return [Boolean] is this an executable task
+        def enabled?
+          @enabled
+        end
+
+        # Triggers the execution of this task at the earliest opportunity
+        def trigger
+          touch
+          @trigger = true
+        end
+
+        # @return [EtOrbi::EoTime] next execution time
+        def next_time
+          @cron.next_time(@previous_time.to_i.zero? ? @start_time : @previous_time)
+        end
+
+        # @return [Boolean] should we execute this task at this moment in time
+        def call?
+          return true if @trigger
+          return false unless enabled?
+
+          # Ensure the job is only due if current_time is strictly after the next_time
+          # Please note that we can only compare eorbi against time and not the other way around
+          next_time <= Time.now
+        end
+
+        # Executes the given task and publishes appropriate notification bus events.
+        def call
+          monitor.instrument(
+            'recurring_tasks.task.executed',
+            task: self
+          ) do
+            # We check for presence of the `@executable` because user can define cron schedule
+            # without the code block
+            return unless @executable
+
+            execute
+          end
+        rescue StandardError => e
+          monitor.instrument(
+            'error.occurred',
+            caller: self,
+            error: e,
+            task: self,
+            type: 'recurring_tasks.task.execute.error'
+          )
+        ensure
+          @trigger = false
+          @previous_time = Time.now
+        end
+
+        # Runs the executable block without any instrumentation or error handling. Useful for
+        # debugging and testing
+        def execute
+          @executable.call
+        end
+
+        # Removes the changes indicator flag
+        def clear
+          @changed = false
+        end
+
+        # @return [Hash] hash version of the task. Used for contract validation.
+        def to_h
+          {
+            id: id,
+            cron: @cron.original,
+            previous_time: previous_time,
+            next_time: next_time,
+            changed: changed?,
+            enabled: enabled?
+          }
+        end
+
+        private
+
+        # Marks the task as changed
+        def touch
+          @changed = true
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/recurring_tasks.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    # Recurring tasks functionality
+    module RecurringTasks
+      class << self
+        # @return [Schedule, nil] current defined schedule or nil if not defined
+        def schedule
+          @schedule || define('0.0.0') {}
+        end
+
+        # Simplified API for schedules definitions and validates the tasks data
+        #
+        # @param version [String]
+        # @param block [Proc]
+        #
+        # @example
+        #   Karafka::Pro::RecurringTasks.define('1.0.1') do
+        #     schedule(id: 'mailer', cron: '* * * * *') do
+        #       MailingJob.perform_async
+        #     end
+        #   end
+        def define(version = '1.0.0', &block)
+          @schedule = Schedule.new(version: version)
+          @schedule.instance_exec(&block)
+
+          @schedule.each do |task|
+            Contracts::Task.new.validate!(task.to_h)
+          end
+
+          @schedule
+        end
+
+        # Defines nice command methods to dispatch cron requests
+        Executor::COMMANDS.each do |command_name|
+          class_eval <<~RUBY, __FILE__, __LINE__ + 1
+            # @param task_id [String] task to which we want to dispatch command or '*' if to all
+            def #{command_name}(task_id)
+              Dispatcher.command('#{command_name}', task_id)
+            end
+          RUBY
+        end
+
+        # Below are private APIs
+
+        # Sets up additional config scope, validations and other things
+        #
+        # @param config [Karafka::Core::Configurable::Node] root node config
+        def pre_setup(config)
+          # Expand the config with this feature specific stuff
+          config.instance_eval do
+            setting(:recurring_tasks, default: Setup::Config.config)
+          end
+        end
+
+        # @param config [Karafka::Core::Configurable::Node] root node config
+        def post_setup(config)
+          RecurringTasks::Contracts::Config.new.validate!(config.to_h)
+
+          # Published after task is successfully executed
+          Karafka.monitor.notifications_bus.register_event('recurring_tasks.task.executed')
+
+          # Initialize empty dummy schedule, so we always have one and so we do not have to
+          # deal with a case where there is no schedule
+          RecurringTasks.schedule
+
+          # User can disable logging of executions, in which case we don't track them
+          return unless Karafka::App.config.recurring_tasks.logging
+
+          Karafka.monitor.subscribe(Listener.new)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/active_job/builder.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/routing/features/active_job.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/routing/features/adaptive_iterator/config.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class AdaptiveIterator < Base
+          # Adaptive Iterator configuration
+          Config = Struct.new(
+            :active,
+            :safety_margin,
+            :marking_method,
+            :clean_after_yielding,
+            keyword_init: true
+          ) do
+            alias_method :active?, :active
+            alias_method :clean_after_yielding?, :clean_after_yielding
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/adaptive_iterator/contracts/topic.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class AdaptiveIterator < Base
+          # Namespace for adaptive iterator contracts
+          module Contracts
+            # Contract to validate configuration of the adaptive iterator feature
+            class Topic < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('topic')
+              end
+
+              nested(:adaptive_iterator) do
+                required(:active) { |val| [true, false].include?(val) }
+                required(:safety_margin) { |val| val.is_a?(Integer) && val.positive? && val < 100 }
+                required(:clean_after_yielding) { |val| [true, false].include?(val) }
+
+                required(:marking_method) do |val|
+                  %i[mark_as_consumed mark_as_consumed!].include?(val)
+                end
+              end
+
+              # Since adaptive iterator uses `#seek` and can break processing in the middle, we
+              # cannot use it with virtual partitions that can process data in a distributed
+              # manner
+              virtual do |data, errors|
+                next unless errors.empty?
+
+                adaptive_iterator = data[:adaptive_iterator]
+                virtual_partitions = data[:virtual_partitions]
+
+                next unless adaptive_iterator[:active]
+                next unless virtual_partitions[:active]
+
+                [[%i[adaptive_iterator], :with_virtual_partitions]]
+              end
+
+              # There is no point of using the adaptive iterator with LRJ because of how LRJ works
+              virtual do |data, errors|
+                next unless errors.empty?
+
+                adaptive_iterator = data[:adaptive_iterator]
+                long_running_jobs = data[:long_running_job]
+
+                next unless adaptive_iterator[:active]
+                next unless long_running_jobs[:active]
+
+                [[%i[adaptive_iterator], :with_long_running_job]]
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/adaptive_iterator/topic.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class AdaptiveIterator < Base
+          # Topic extension allowing us to enable and configure adaptive iterator
+          module Topic
+            # @param active [Boolean] should we use the automatic adaptive iterator
+            # @param safety_margin [Integer]
+            #   How big of a margin we leave ourselves so we can safely communicate back with
+            #   Kafka, etc. We stop and seek back when we've burned 85% of the time by default.
+            #   We leave 15% of time for post-processing operations so we have space before we
+            #   hit max.poll.interval.ms.
+            # @param marking_method [Symbol] If we should, how should we mark
+            # @param clean_after_yielding [Boolean]  Should we clean post-yielding via the
+            #   cleaner API
+            def adaptive_iterator(
+              active: false,
+              safety_margin: 10,
+              marking_method: :mark_as_consumed,
+              clean_after_yielding: true
+            )
+              @adaptive_iterator ||= Config.new(
+                active: active,
+                safety_margin: safety_margin,
+                marking_method: marking_method,
+                clean_after_yielding: clean_after_yielding
+              )
+            end
+
+            # @return [Boolean] Is adaptive iterator active. It is always `true`, since we use it
+            #   via explicit messages batch wrapper
+            def adaptive_iterator?
+              adaptive_iterator.active?
+            end
+
+            # @return [Hash] topic with all its native configuration options plus poll guarding
+            #   setup configuration.
+            def to_h
+              super.merge(
+                adaptive_iterator: adaptive_iterator.to_h
+              ).freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/adaptive_iterator.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        # Feature that pro-actively monitors remaining time until max poll interval ms and
+        # cost of processing of each message in a batch. When there is no more time to process
+        # more messages from the batch, it will seek back so we do not reach max poll interval.
+        # It can be useful when we reach this once in a while. For a constant long-running jobs,
+        # please use the Long-Running Jobs feature instead.
+        #
+        # It also provides some wrapping over typical operations users do, like stopping if
+        # revoked, auto-marking, etc
+        class AdaptiveIterator < Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/base.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/routing/features/dead_letter_queue/contracts/topic.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/routing/features/dead_letter_queue/topic.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/routing/features/dead_letter_queue.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/routing/features/delaying/config.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/routing/features/delaying/contracts/topic.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/routing/features/delaying/topic.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/routing/features/delaying.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/routing/features/direct_assignments/config.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/routing/features/direct_assignments/contracts/consumer_group.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro