karafka 2.4.8 → 2.4.9

data/lib/karafka/pro/recurring_tasks/task.rb

@@ -0,0 +1,151 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module RecurringTasks
+      # Represents a single recurring task that can be executed when the time comes.
+      # Tasks should be lightweight. Anything heavy should be executed by scheduling appropriate
+      # jobs here.
+      class Task
+        include Helpers::ConfigImporter.new(
+          monitor: %i[monitor]
+        )
+
+        # @return [String] this task id
+        attr_reader :id
+
+        # @return [Fugit::Cron] cron from parsing the raw cron expression
+        attr_reader :cron
+
+        # Allows for update of previous time when restoring the materialized state
+        attr_accessor :previous_time
+
+        # @param id [String] unique id. If re-used between versions, will replace older occurrence.
+        # @param cron [String] cron expression matching this task expected execution times.
+        # @param previous_time [Integer, Time] previous time this task was executed. 0 if never.
+        # @param enabled [Boolean] should this task be enabled. Users may disable given task
+        #   temporarily, this is why we need to know that.
+        # @param block [Proc] code to execute.
+        def initialize(id:, cron:, previous_time: 0, enabled: true, &block)
+          @id = id
+          @cron = ::Fugit::Cron.do_parse(cron)
+          @previous_time = previous_time
+          @start_time = Time.now
+          @executable = block
+          @enabled = enabled
+          @trigger = false
+          @changed = false
+        end
+
+        # @return [Boolean] true if anything in the task has changed and we should persist it
+        def changed?
+          @changed
+        end
+
+        # Disables this task execution indefinitely
+        def disable
+          touch
+          @enabled = false
+        end
+
+        # Enables back this task
+        def enable
+          touch
+          @enabled = true
+        end
+
+        # @return [Boolean] is this an executable task
+        def enabled?
+          @enabled
+        end
+
+        # Triggers the execution of this task at the earliest opportunity
+        def trigger
+          touch
+          @trigger = true
+        end
+
+        # @return [EtOrbi::EoTime] next execution time
+        def next_time
+          @cron.next_time(@previous_time.to_i.zero? ? @start_time : @previous_time)
+        end
+
+        # @return [Boolean] should we execute this task at this moment in time
+        def call?
+          return true if @trigger
+          return false unless enabled?
+
+          # Ensure the job is only due if current_time is strictly after the next_time
+          # Please note that we can only compare eorbi against time and not the other way around
+          next_time <= Time.now
+        end
+
+        # Executes the given task and publishes appropriate notification bus events.
+        def call
+          monitor.instrument(
+            'recurring_tasks.task.executed',
+            task: self
+          ) do
+            # We check for presence of the `@executable` because user can define cron schedule
+            # without the code block
+            return unless @executable
+
+            execute
+          end
+        rescue StandardError => e
+          monitor.instrument(
+            'error.occurred',
+            caller: self,
+            error: e,
+            task: self,
+            type: 'recurring_tasks.task.execute.error'
+          )
+        ensure
+          @trigger = false
+          @previous_time = Time.now
+        end
+
+        # Runs the executable block without any instrumentation or error handling. Useful for
+        # debugging and testing
+        def execute
+          @executable.call
+        end
+
+        # Removes the changes indicator flag
+        def clear
+          @changed = false
+        end
+
+        # @return [Hash] hash version of the task. Used for contract validation.
+        def to_h
+          {
+            id: id,
+            cron: @cron.original,
+            previous_time: previous_time,
+            next_time: next_time,
+            changed: changed?,
+            enabled: enabled?
+          }
+        end
+
+        private
+
+        # Marks the task as changed
+        def touch
+          @changed = true
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/recurring_tasks.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Recurring tasks functionality
+    module RecurringTasks
+      class << self
+        # @return [Schedule, nil] current defined schedule or nil if not defined
+        def schedule
+          @schedule || define('0.0.0') {}
+        end
+
+        # Simplified API for schedules definitions and validates the tasks data
+        #
+        # @param version [String]
+        # @param block [Proc]
+        #
+        # @example
+        #   Karafka::Pro::RecurringTasks.define('1.0.1') do
+        #     schedule(id: 'mailer', cron: '* * * * *') do
+        #       MailingJob.perform_async
+        #     end
+        #   end
+        def define(version = '1.0.0', &block)
+          @schedule = Schedule.new(version: version)
+          @schedule.instance_exec(&block)
+
+          @schedule.each do |task|
+            Contracts::Task.new.validate!(task.to_h)
+          end
+
+          @schedule
+        end
+
+        # Defines nice command methods to dispatch cron requests
+        Executor::COMMANDS.each do |command_name|
+          class_eval <<~RUBY, __FILE__, __LINE__ + 1
+            # @param task_id [String] task to which we want to dispatch command or '*' if to all
+            def #{command_name}(task_id)
+              Dispatcher.command('#{command_name}', task_id)
+            end
+          RUBY
+        end
+
+        # Below are private APIs
+
+        # Sets up additional config scope, validations and other things
+        #
+        # @param config [Karafka::Core::Configurable::Node] root node config
+        def pre_setup(config)
+          # Expand the config with this feature specific stuff
+          config.instance_eval do
+            setting(:recurring_tasks, default: Setup::Config.config)
+          end
+        end
+
+        # @param config [Karafka::Core::Configurable::Node] root node config
+        def post_setup(config)
+          RecurringTasks::Contracts::Config.new.validate!(config.to_h)
+
+          # Published after task is successfully executed
+          Karafka.monitor.notifications_bus.register_event('recurring_tasks.task.executed')
+
+          # Initialize empty dummy schedule, so we always have one and so we do not have to
+          # deal with a case where there is no schedule
+          RecurringTasks.schedule
+
+          # User can disable logging of executions, in which case we don't track them
+          return unless Karafka::App.config.recurring_tasks.logging
+
+          Karafka.monitor.subscribe(Listener.new)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/recurring_tasks/builder.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class RecurringTasks < Base
+          # Routing extensions for recurring tasks
+          module Builder
+            # Enabled recurring tasks operations and adds needed topics and other stuff.
+            #
+            # @param active [Boolean] should recurring tasks be active. We use a boolean flag to
+            #   have API consistency in the system, so it matches other routing related APIs.
+            # @param block [Proc] optional reconfiguration of the tasks topic definitions.
+            # @note Since we cannot provide two blocks, reconfiguration of logs topic can be only
+            #   done if user explicitly redefines it in the routing.
+            def recurring_tasks(active = false, &block)
+              return unless active
+
+              # We only require zlib when we decide to run recurring tasks because it is not needed
+              # otherwise.
+              require 'zlib'
+              ensure_fugit_availability!
+
+              tasks_cfg = App.config.recurring_tasks
+              topics_cfg = tasks_cfg.topics
+
+              consumer_group tasks_cfg.group_id do
+                # Registers the primary topic that we use to control schedules execution. This is
+                # the one that we use to trigger recurring tasks.
+                topic(topics_cfg.schedules) do
+                  consumer tasks_cfg.consumer_class
+                  deserializer tasks_cfg.deserializer
+                  # Because the topic method name as well as builder proxy method name is the same
+                  # we need to reference it via target directly
+                  target.recurring_tasks(true)
+
+                  # We manage offsets directly because messages can have both schedules and
+                  # commands and we need to apply them only when we need to
+                  manual_offset_management(true)
+
+                  # We use multi-batch operations and in-memory state for schedules. This needs to
+                  # always operate without re-creation.
+                  consumer_persistence(true)
+
+                  # This needs to be enabled for the eof to work correctly
+                  kafka('enable.partition.eof': true, inherit: true)
+                  eofed(true)
+
+                  # Favour latency. This is a low traffic topic that only accepts user initiated
+                  # low-frequency commands
+                  max_messages(1)
+                  # Since we always react on the received message, we can block for longer periods
+                  # of time
+                  max_wait_time(10_000)
+
+                  # Since the execution of particular tasks is isolated and guarded, it should not
+                  # leak. This means, that this is to handle errors like schedule version
+                  # incompatibility and other errors that will not go away without a redeployment
+                  pause_timeout(60 * 1_000)
+                  pause_max_timeout(60 * 1_000)
+
+                  # Keep older data for a day and compact to the last state available
+                  config(
+                    'cleanup.policy': 'compact,delete',
+                    'retention.ms': 86_400_000
+                  )
+
+                  # This is the core of execution. Since we're producers of states, we need a way
+                  # to tick without having new data
+                  periodic(
+                    interval: App.config.recurring_tasks.interval,
+                    during_pause: false,
+                    during_retry: false
+                  )
+
+                  next unless block
+
+                  instance_eval(&block)
+                end
+
+                # This topic is to store logs that we can then inspect either from the admin or via
+                # the Web UI
+                topic(topics_cfg.logs) do
+                  active(false)
+                  deserializer tasks_cfg.deserializer
+                  target.recurring_tasks(true)
+
+                  # Keep cron logs of executions for a week and after that remove. Week should be
+                  # enough and should not produce too much data.
+                  config(
+                    'cleanup.policy': 'delete',
+                    'retention.ms': 604_800_000
+                  )
+                end
+              end
+            end
+
+            # Checks if fugit is present. If not, will try to require it as it might not have
+            # been required but is available. If fails, will crash.
+            def ensure_fugit_availability!
+              return if Object.const_defined?(:Fugit)
+
+              require 'fugit'
+            rescue LoadError
+              raise(
+                ::Karafka::Errors::DependencyConstraintsError,
+                <<~ERROR_MSG
+                  Failed to require fugit gem.
+                  Add it to your Gemfile, as it is required for the recurring tasks to work.
+                ERROR_MSG
+              )
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/recurring_tasks/config.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class RecurringTasks < Base
+          # Recurring tasks configuration
+          Config = Struct.new(
+            :active,
+            keyword_init: true
+          ) { alias_method :active?, :active }
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/recurring_tasks/contracts/topic.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class RecurringTasks < Base
+          # Namespace for recurring tasks contracts
+          module Contracts
+            # Rules around recurring tasks settings
+            class Topic < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('topic')
+              end
+
+              nested(:recurring_tasks) do
+                required(:active) { |val| [true, false].include?(val) }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/recurring_tasks/proxy.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class RecurringTasks < Base
+          # Routing proxy extensions for recurring tasks
+          module Proxy
+            include Builder
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/recurring_tasks/topic.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class RecurringTasks < Base
+          # Topic extensions to be able to check if given topic is a recurring tasks topic
+          # Please note, that this applies to both the schedules topics and reports topics
+          module Topic
+            # @param active [Boolean] should this topic be considered related to recurring tasks
+            def recurring_tasks(active = false)
+              @recurring_tasks ||= Config.new(active: active)
+            end
+
+            # @return [Boolean] is this an ActiveJob topic
+            def recurring_tasks?
+              recurring_tasks.active?
+            end
+
+            # @return [Hash] topic with all its native configuration options plus active job
+            #   namespace settings
+            def to_h
+              super.merge(
+                recurring_tasks: recurring_tasks.to_h
+              ).freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/recurring_tasks.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        # This feature allows you to run recurring tasks aka cron jobs from within Karafka
+        # It uses two special topics for tracking the schedule and reporting executions
+        class RecurringTasks < Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/strategies/dlq.rb

@@ -76,9 +76,13 @@ module Karafka
             dispatch_to_dlq(skippable_message)
 
             # We mark the broken message as consumed and move on
-            mark_dispatched_to_dlq(skippable_message)
+            if mark_after_dispatch?
+              mark_dispatched_to_dlq(skippable_message)
 
-            return if revoked?
+              return if revoked?
+            else
+              coordinator.seek_offset = skippable_message.offset + 1
+            end
 
             # We pause to backoff once just in case.
             pause(coordinator.seek_offset, nil, false)
@@ -120,6 +124,16 @@ module Karafka
           )
         end
 
+        # @return [Boolean] should we mark given message as consumed after dispatch. For default
+        #  non MOM strategies if user did not explicitly tell us not to, we mark it. Default is
+        #  `nil`, which means `true` in this case. If user provided alternative value, we go
+        #  with it.
+        def mark_after_dispatch?
+          return true if topic.dead_letter_queue.mark_after_dispatch.nil?
+
+          topic.dead_letter_queue.mark_after_dispatch
+        end
+
         # Marks message that went to DLQ (if applicable) based on the requested method
         # @param skippable_message [Karafka::Messages::Message]
         def mark_dispatched_to_dlq(skippable_message)

data/lib/karafka/processing/strategies/dlq_mom.rb

@@ -33,16 +33,35 @@ module Karafka
 
             dispatch_to_dlq(skippable_message)
 
-            # Save the next offset we want to go with after moving given message to DLQ
-            # Without this, we would not be able to move forward and we would end up
-            # in an infinite loop trying to un-pause from the message we've already processed
-            # Of course, since it's a MoM a rebalance or kill, will move it back as no
-            # offsets are being committed
-            coordinator.seek_offset = skippable_message.offset + 1
+            # We mark the broken message as consumed and move on
+            if mark_after_dispatch?
+              mark_dispatched_to_dlq(skippable_message)
+
+              return if revoked?
+            else
+              # Save the next offset we want to go with after moving given message to DLQ
+              # Without this, we would not be able to move forward and we would end up
+              # in an infinite loop trying to un-pause from the message we've already processed
+              # Of course, since it's a MoM a rebalance or kill, will move it back as no
+              # offsets are being committed
+              coordinator.seek_offset = skippable_message.offset + 1
+            end
 
             pause(coordinator.seek_offset, nil, false)
           end
         end
+
+        # @return [Boolean] should we mark given message as consumed after dispatch. For
+        #  MOM strategies if user did not explicitly tell us to mark, we do not mark. Default is
+        #  `nil`, which means `false` in this case. If user provided alternative value, we go
+        #  with it.
+        #
+        # @note Please note, this is the opposite behavior than in case of AOM strategies.
+        def mark_after_dispatch?
+          return false if topic.dead_letter_queue.mark_after_dispatch.nil?
+
+          topic.dead_letter_queue.mark_after_dispatch
+        end
       end
     end
   end

data/lib/karafka/processing/worker.rb

@@ -18,6 +18,9 @@ module Karafka
     #                not user code but need to run after user code base is executed.
     class Worker
       include Helpers::Async
+      include Helpers::ConfigImporter.new(
+        worker_job_call_wrapper: %i[internal processing worker_job_call_wrapper]
+      )
 
       # @return [String] id of this worker
       attr_reader :id
@@ -27,6 +30,7 @@ module Karafka
       def initialize(jobs_queue)
         @id = SecureRandom.hex(6)
         @jobs_queue = jobs_queue
+        @non_wrapped_flow = worker_job_call_wrapper == false
       end
 
       private
@@ -60,7 +64,13 @@ module Karafka
             # pass it until done.
             @jobs_queue.tick(job.group_id) if job.non_blocking?
 
-            job.call
+            if @non_wrapped_flow
+              job.call
+            else
+              worker_job_call_wrapper.wrap do
+                job.call
+              end
+            end
 
             job.after_call