karafka 2.4.8 → 2.4.10

data/lib/karafka/pro/recurring_tasks/schedule.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module RecurringTasks
+      # Represents the current code-context schedule with defined tasks and their cron execution
+      # details. Single schedule includes all the information about all the tasks that we have
+      # defined and to be executed in a given moment in time.
+      class Schedule
+        # @return [String]
+        attr_reader :version
+
+        # @return [Hash<String, Task>]
+        attr_reader :tasks
+
+        # @param version [String] schedule version. In case of usage of versioning it is used to
+        #   ensure, that older still active processes do not intercept the assignment to run older
+        #   version of the scheduler. It is important to make sure, that this string is comparable.
+        def initialize(version:)
+          @version = version
+          @tasks = {}
+        end
+
+        # Adds task into the tasks accumulator
+        # @param task [Task]
+        # @note In case of multiple tasks with the same id, it will overwrite
+        def <<(task)
+          @tasks[task.id] = task
+        end
+
+        # Iterates over tasks yielding them one after another
+        # @param block [Proc] block that will be executed with each task
+        def each(&block)
+          @tasks.each_value(&block)
+        end
+
+        # @param id [String] id of a particular recurring task
+        # @return [Task, nil] task with a given id or nil if not found
+        def find(id)
+          @tasks[id]
+        end
+
+        # Allows us to have a nice DSL for defining schedules
+        # @param args [Array] attributes accepted by the task initializer
+        # @param block [Proc] block to execute
+        def schedule(**args, &block)
+          self << Task.new(**args, &block)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/recurring_tasks/serializer.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module RecurringTasks
+      # Converts schedule command and log details into data we can dispatch to Kafka.
+      class Serializer
+        # Current recurring tasks related schema structure
+        SCHEMA_VERSION = '1.0'
+
+        # @param schedule [Karafka::Pro::RecurringTasks::Schedule] schedule to serialize
+        # @return [String] serialized and compressed current schedule data with its tasks and their
+        #   current state.
+        def schedule(schedule)
+          tasks = {}
+
+          schedule.each do |task|
+            tasks[task.id] = {
+              id: task.id,
+              cron: task.cron.original,
+              previous_time: task.previous_time.to_i,
+              next_time: task.next_time.to_i,
+              enabled: task.enabled?
+            }
+          end
+
+          data = {
+            schema_version: SCHEMA_VERSION,
+            schedule_version: schedule.version,
+            dispatched_at: Time.now.to_f,
+            type: 'schedule',
+            tasks: tasks
+          }
+
+          compress(
+            serialize(data)
+          )
+        end
+
+        # @param command_name [String] command name
+        # @param task_id [String] task id or '*' to match all.
+        # @return [String] serialized and compressed command data
+        def command(command_name, task_id)
+          data = {
+            schema_version: SCHEMA_VERSION,
+            schedule_version: ::Karafka::Pro::RecurringTasks.schedule.version,
+            dispatched_at: Time.now.to_f,
+            type: 'command',
+            command: {
+              name: command_name
+            },
+            task: {
+              id: task_id
+            }
+          }
+
+          compress(
+            serialize(data)
+          )
+        end
+
+        # @param event [Karafka::Core::Monitoring::Event] recurring task dispatch event
+        # @return [String] serialized and compressed event log data
+        def log(event)
+          task = event[:task]
+
+          data = {
+            schema_version: SCHEMA_VERSION,
+            schedule_version: ::Karafka::Pro::RecurringTasks.schedule.version,
+            dispatched_at: Time.now.to_f,
+            type: 'log',
+            task: {
+              id: task.id,
+              time_taken: event.payload[:time] || -1,
+              result: event.payload.key?(:error) ? 'failure' : 'success'
+            }
+          }
+
+          compress(
+            serialize(data)
+          )
+        end
+
+        private
+
+        # @param hash [Hash] hash to cast to json
+        # @return [String] json hash
+        def serialize(hash)
+          hash.to_json
+        end
+
+        # Compresses the provided data
+        #
+        # @param data [String] data to compress
+        # @return [String] compressed data
+        def compress(data)
+          Zlib::Deflate.deflate(data)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/recurring_tasks/setup/config.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module RecurringTasks
+      # Setup and config related recurring tasks components
+      module Setup
+        # Config for recurring tasks
+        class Config
+          extend ::Karafka::Core::Configurable
+
+          setting(:consumer_class, default: Consumer)
+          setting(:deserializer, default: Deserializer.new)
+          setting(:group_id, default: 'karafka_recurring_tasks')
+          # By default we will run the scheduling every 15 seconds since we provide a minute-based
+          # precision
+          setting(:interval, default: 15_000)
+          # Should we log the executions. If true (default) with each cron execution, there will
+          # be a special message published. Useful for debugging.
+          setting(:logging, default: true)
+
+          # Producer to be used by the recurring tasks.
+          # By default it is a `Karafka.producer`, however it may be overwritten if we want to use
+          # a separate instance in case of heavy usage of the  transactional producer, etc.
+          setting(
+            :producer,
+            constructor: -> { ::Karafka.producer },
+            lazy: true
+          )
+
+          setting(:topics) do
+            setting(:schedules, default: 'karafka_recurring_tasks_schedules')
+            setting(:logs, default: 'karafka_recurring_tasks_logs')
+          end
+
+          configure
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/recurring_tasks/task.rb

@@ -0,0 +1,151 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module RecurringTasks
+      # Represents a single recurring task that can be executed when the time comes.
+      # Tasks should be lightweight. Anything heavy should be executed by scheduling appropriate
+      # jobs here.
+      class Task
+        include Helpers::ConfigImporter.new(
+          monitor: %i[monitor]
+        )
+
+        # @return [String] this task id
+        attr_reader :id
+
+        # @return [Fugit::Cron] cron from parsing the raw cron expression
+        attr_reader :cron
+
+        # Allows for update of previous time when restoring the materialized state
+        attr_accessor :previous_time
+
+        # @param id [String] unique id. If re-used between versions, will replace older occurrence.
+        # @param cron [String] cron expression matching this task expected execution times.
+        # @param previous_time [Integer, Time] previous time this task was executed. 0 if never.
+        # @param enabled [Boolean] should this task be enabled. Users may disable given task
+        #   temporarily, this is why we need to know that.
+        # @param block [Proc] code to execute.
+        def initialize(id:, cron:, previous_time: 0, enabled: true, &block)
+          @id = id
+          @cron = ::Fugit::Cron.do_parse(cron)
+          @previous_time = previous_time
+          @start_time = Time.now
+          @executable = block
+          @enabled = enabled
+          @trigger = false
+          @changed = false
+        end
+
+        # @return [Boolean] true if anything in the task has changed and we should persist it
+        def changed?
+          @changed
+        end
+
+        # Disables this task execution indefinitely
+        def disable
+          touch
+          @enabled = false
+        end
+
+        # Enables back this task
+        def enable
+          touch
+          @enabled = true
+        end
+
+        # @return [Boolean] is this an executable task
+        def enabled?
+          @enabled
+        end
+
+        # Triggers the execution of this task at the earliest opportunity
+        def trigger
+          touch
+          @trigger = true
+        end
+
+        # @return [EtOrbi::EoTime] next execution time
+        def next_time
+          @cron.next_time(@previous_time.to_i.zero? ? @start_time : @previous_time)
+        end
+
+        # @return [Boolean] should we execute this task at this moment in time
+        def call?
+          return true if @trigger
+          return false unless enabled?
+
+          # Ensure the job is only due if current_time is strictly after the next_time
+          # Please note that we can only compare eorbi against time and not the other way around
+          next_time <= Time.now
+        end
+
+        # Executes the given task and publishes appropriate notification bus events.
+        def call
+          monitor.instrument(
+            'recurring_tasks.task.executed',
+            task: self
+          ) do
+            # We check for presence of the `@executable` because user can define cron schedule
+            # without the code block
+            return unless @executable
+
+            execute
+          end
+        rescue StandardError => e
+          monitor.instrument(
+            'error.occurred',
+            caller: self,
+            error: e,
+            task: self,
+            type: 'recurring_tasks.task.execute.error'
+          )
+        ensure
+          @trigger = false
+          @previous_time = Time.now
+        end
+
+        # Runs the executable block without any instrumentation or error handling. Useful for
+        # debugging and testing
+        def execute
+          @executable.call
+        end
+
+        # Removes the changes indicator flag
+        def clear
+          @changed = false
+        end
+
+        # @return [Hash] hash version of the task. Used for contract validation.
+        def to_h
+          {
+            id: id,
+            cron: @cron.original,
+            previous_time: previous_time,
+            next_time: next_time,
+            changed: changed?,
+            enabled: enabled?
+          }
+        end
+
+        private
+
+        # Marks the task as changed
+        def touch
+          @changed = true
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/recurring_tasks.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Recurring tasks functionality
+    module RecurringTasks
+      class << self
+        # @return [Schedule, nil] current defined schedule or nil if not defined
+        def schedule
+          @schedule || define('0.0.0') {}
+        end
+
+        # Simplified API for schedules definitions and validates the tasks data
+        #
+        # @param version [String]
+        # @param block [Proc]
+        #
+        # @example
+        #   Karafka::Pro::RecurringTasks.define('1.0.1') do
+        #     schedule(id: 'mailer', cron: '* * * * *') do
+        #       MailingJob.perform_async
+        #     end
+        #   end
+        def define(version = '1.0.0', &block)
+          @schedule = Schedule.new(version: version)
+          @schedule.instance_exec(&block)
+
+          @schedule.each do |task|
+            Contracts::Task.new.validate!(task.to_h)
+          end
+
+          @schedule
+        end
+
+        # Defines nice command methods to dispatch cron requests
+        Executor::COMMANDS.each do |command_name|
+          class_eval <<~RUBY, __FILE__, __LINE__ + 1
+            # @param task_id [String] task to which we want to dispatch command or '*' if to all
+            def #{command_name}(task_id)
+              Dispatcher.command('#{command_name}', task_id)
+            end
+          RUBY
+        end
+
+        # Below are private APIs
+
+        # Sets up additional config scope, validations and other things
+        #
+        # @param config [Karafka::Core::Configurable::Node] root node config
+        def pre_setup(config)
+          # Expand the config with this feature specific stuff
+          config.instance_eval do
+            setting(:recurring_tasks, default: Setup::Config.config)
+          end
+        end
+
+        # @param config [Karafka::Core::Configurable::Node] root node config
+        def post_setup(config)
+          RecurringTasks::Contracts::Config.new.validate!(config.to_h)
+
+          # Published after task is successfully executed
+          Karafka.monitor.notifications_bus.register_event('recurring_tasks.task.executed')
+
+          # Initialize empty dummy schedule, so we always have one and so we do not have to
+          # deal with a case where there is no schedule
+          RecurringTasks.schedule
+
+          # User can disable logging of executions, in which case we don't track them
+          return unless Karafka::App.config.recurring_tasks.logging
+
+          Karafka.monitor.subscribe(Listener.new)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/recurring_tasks/builder.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class RecurringTasks < Base
+          # Routing extensions for recurring tasks
+          module Builder
+            # Enabled recurring tasks operations and adds needed topics and other stuff.
+            #
+            # @param active [Boolean] should recurring tasks be active. We use a boolean flag to
+            #   have API consistency in the system, so it matches other routing related APIs.
+            # @param block [Proc] optional reconfiguration of the topics definitions.
+            def recurring_tasks(active = false, &block)
+              return unless active
+
+              # We only require zlib when we decide to run recurring tasks because it is not needed
+              # otherwise.
+              require 'zlib'
+              ensure_fugit_availability!
+
+              tasks_cfg = App.config.recurring_tasks
+              topics_cfg = tasks_cfg.topics
+
+              consumer_group tasks_cfg.group_id do
+                # Registers the primary topic that we use to control schedules execution. This is
+                # the one that we use to trigger recurring tasks.
+                schedules_topic = topic(topics_cfg.schedules) do
+                  consumer tasks_cfg.consumer_class
+                  deserializer tasks_cfg.deserializer
+                  # Because the topic method name as well as builder proxy method name is the same
+                  # we need to reference it via target directly
+                  target.recurring_tasks(true)
+
+                  # We manage offsets directly because messages can have both schedules and
+                  # commands and we need to apply them only when we need to
+                  manual_offset_management(true)
+
+                  # We use multi-batch operations and in-memory state for schedules. This needs to
+                  # always operate without re-creation.
+                  consumer_persistence(true)
+
+                  # This needs to be enabled for the eof to work correctly
+                  kafka('enable.partition.eof': true, inherit: true)
+                  eofed(true)
+
+                  # Favour latency. This is a low traffic topic that only accepts user initiated
+                  # low-frequency commands
+                  max_messages(1)
+                  # Since we always react on the received message, we can block for longer periods
+                  # of time
+                  max_wait_time(10_000)
+
+                  # Since the execution of particular tasks is isolated and guarded, it should not
+                  # leak. This means, that this is to handle errors like schedule version
+                  # incompatibility and other errors that will not go away without a redeployment
+                  pause_timeout(60 * 1_000)
+                  pause_max_timeout(60 * 1_000)
+
+                  # Keep older data for a day and compact to the last state available
+                  config(
+                    'cleanup.policy': 'compact,delete',
+                    'retention.ms': 86_400_000
+                  )
+
+                  # This is the core of execution. Since we're producers of states, we need a way
+                  # to tick without having new data
+                  periodic(
+                    interval: App.config.recurring_tasks.interval,
+                    during_pause: false,
+                    during_retry: false
+                  )
+
+                  # If this is the direct schedules redefinition style, we run it
+                  # The second one (see end of this method) allows for linear reconfiguration of
+                  # both the topics
+                  instance_eval(&block) if block && block.arity.zero?
+                end
+
+                # This topic is to store logs that we can then inspect either from the admin or via
+                # the Web UI
+                logs_topic = topic(topics_cfg.logs) do
+                  active(false)
+                  deserializer tasks_cfg.deserializer
+                  target.recurring_tasks(true)
+
+                  # Keep cron logs of executions for a week and after that remove. Week should be
+                  # enough and should not produce too much data.
+                  config(
+                    'cleanup.policy': 'delete',
+                    'retention.ms': 604_800_000
+                  )
+                end
+
+                yield(schedules_topic, logs_topic) if block && block.arity.positive?
+              end
+            end
+
+            # Checks if fugit is present. If not, will try to require it as it might not have
+            # been required but is available. If fails, will crash.
+            def ensure_fugit_availability!
+              return if Object.const_defined?(:Fugit)
+
+              require 'fugit'
+            rescue LoadError
+              raise(
+                ::Karafka::Errors::DependencyConstraintsError,
+                <<~ERROR_MSG
+                  Failed to require fugit gem.
+                  Add it to your Gemfile, as it is required for the recurring tasks to work.
+                ERROR_MSG
+              )
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/recurring_tasks/config.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class RecurringTasks < Base
+          # Recurring tasks configuration
+          Config = Struct.new(
+            :active,
+            keyword_init: true
+          ) { alias_method :active?, :active }
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/recurring_tasks/contracts/topic.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class RecurringTasks < Base
+          # Namespace for recurring tasks contracts
+          module Contracts
+            # Rules around recurring tasks settings
+            class Topic < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('topic')
+              end
+
+              nested(:recurring_tasks) do
+                required(:active) { |val| [true, false].include?(val) }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/recurring_tasks/proxy.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class RecurringTasks < Base
+          # Routing proxy extensions for recurring tasks
+          module Proxy
+            include Builder
+          end
+        end
+      end
+    end
+  end
+end