karafka 2.4.8 → 2.4.9

data/lib/karafka/pro/recurring_tasks/contracts/task.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module RecurringTasks
+      # Recurring Tasks related contracts
+      module Contracts
+        # Ensures that task details are as expected
+        class Task < ::Karafka::Contracts::Base
+          configure do |config|
+            config.error_messages = YAML.safe_load(
+              File.read(
+                File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+              )
+            ).fetch('en').fetch('validations').fetch('recurring_tasks')
+          end
+
+          # Regexp to ensure all tasks ids are URL safe
+          ID_REGEXP = /\A[a-zA-Z0-9_-]{1,}\z/
+
+          required(:id) { |val| val.is_a?(String) && val.match?(ID_REGEXP) }
+          required(:cron) { |val| val.is_a?(String) && val.length.positive? }
+          required(:enabled) { |val| [true, false].include?(val) }
+          required(:changed) { |val| [true, false].include?(val) }
+          required(:previous_time) { |val| val.is_a?(Numeric) || val.is_a?(Time) }
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/recurring_tasks/deserializer.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module RecurringTasks
+      # Recurring Tasks data deserializer. We compress data ourselves because we cannot rely on
+      # any external optional features like certain compression types, etc. By doing this that way
+      # we can ensure we have full control over the compression.
+      #
+      # @note We use `symbolize_names` because we want to use the same convention of hash building
+      #   for producing, consuming and displaying related data as in other places.
+      class Deserializer
+        # @param message [::Karafka::Messages::Message]
+        # @return [Hash] deserialized data
+        def call(message)
+          ::JSON.parse(
+            Zlib::Inflate.inflate(message.raw_payload),
+            symbolize_names: true
+          )
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/recurring_tasks/dispatcher.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module RecurringTasks
+      # Dispatches appropriate recurring tasks related messages to expected topics
+      class Dispatcher
+        class << self
+          # Snapshots to Kafka current schedule state
+          def schedule
+            produce(
+              topics.schedules,
+              'state:schedule',
+              serializer.schedule(::Karafka::Pro::RecurringTasks.schedule)
+            )
+          end
+
+          # Dispatches the command request
+          #
+          # @param name [String, Symbol] name of the command we want to deal with in the process
+          # @param task_id [String] id of the process. We use name instead of id only
+          #   because in the web ui we work with the full name and it is easier. Since
+          def command(name, task_id)
+            produce(
+              topics.schedules,
+              "command:#{name}:#{task_id}",
+              serializer.command(name, task_id)
+            )
+          end
+
+          # Dispatches the task execution log record
+          # @param event [Karafka::Core::Monitoring::Event]
+          def log(event)
+            produce(
+              topics.logs,
+              event[:task].id,
+              serializer.log(event)
+            )
+          end
+
+          private
+
+          # @return [::WaterDrop::Producer] web ui producer
+          def producer
+            ::Karafka::App.config.recurring_tasks.producer
+          end
+
+          # @return [String] consumers commands topic
+          def topics
+            ::Karafka::App.config.recurring_tasks.topics
+          end
+
+          # @return [Serializer]
+          def serializer
+            Serializer.new
+          end
+
+          # Converts payload to json, compresses it and dispatches to Kafka
+          #
+          # @param topic [String] target topic
+          # @param key [String]
+          # @param payload [Hash] hash with payload
+          def produce(topic, key, payload)
+            producer.produce_async(
+              topic: topic,
+              key: key,
+              partition: 0,
+              payload: payload,
+              headers: { 'zlib' => 'true' }
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/recurring_tasks/errors.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module RecurringTasks
+      # Recurring Tasks related errors
+      module Errors
+        # Base for all the recurring tasks errors
+        BaseError = Class.new(::Karafka::Errors::BaseError)
+
+        # Raised when older cron manager version is trying to work with newer schema
+        IncompatibleSchemaError = Class.new(BaseError)
+
+        # Raised if you use versioned schedule and an existing schedule from Kafka was picked up
+        # by a process with older schedule version.
+        # This is a safe-guard to protect against a cases where there would be a temporary
+        # reassignment of newer schedule data into older process during deployment. It should go
+        # away once all processes are rolled.
+        IncompatibleScheduleError = Class.new(BaseError)
+      end
+    end
+  end
+end

data/lib/karafka/pro/recurring_tasks/executor.rb

@@ -0,0 +1,152 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module RecurringTasks
+      # Executor is responsible for management of the state of recurring tasks schedule and
+      # is the heart of recurring tasks. It coordinates the replaying process as well as
+      # tracking of data on changes.
+      class Executor
+        # Task commands we support and that can be triggered on tasks (if matched)
+        COMMANDS = %w[
+          disable
+          enable
+          trigger
+        ].freeze
+
+        def initialize
+          @replaying = true
+          @incompatible = false
+          @catchup_commands = []
+          @catchup_schedule = nil
+          @matcher = Matcher.new
+        end
+
+        # @return [Boolean] are we in the replaying phase or not (false means, regular operations)
+        def replaying?
+          @replaying
+        end
+
+        # @return [Boolean] Is the current process schedule incompatible (older) than the one
+        #   that we have in memory
+        def incompatible?
+          @incompatible
+        end
+
+        # Applies given command to task (or many tasks) by running the command on tasks that match
+        # @param command_hash [Hash] deserialized command data
+        def apply_command(command_hash)
+          cmd_name = command_hash[:command][:name]
+
+          raise(Karafka::Errors::UnsupportedCaseError, cmd_name) unless COMMANDS.include?(cmd_name)
+
+          schedule.each do |task|
+            next unless @matcher.matches?(task, command_hash)
+
+            task.public_send(cmd_name)
+          end
+        end
+
+        # Updates the catchup state
+        # @param schedule_hash [Hash] deserialized schedule hash hash
+        def update_state(schedule_hash)
+          @catchup_schedule = schedule_hash
+        end
+
+        # Once all previous data is accumulated runs the catchup process to establish current
+        # state of the recurring tasks schedule execution.
+        #
+        # It includes applying any requested commands as well as synchronizing execution details
+        # for existing schedule and making sure all is loaded correctly.
+        def replay
+          # Ensure replaying happens only once
+          return unless @replaying
+
+          @replaying = false
+
+          # When there is nothing to replay and synchronize, we can just save the state and
+          # proceed
+          if @catchup_commands.empty? && @catchup_schedule.nil?
+            snapshot
+
+            return
+          end
+
+          # If the schedule version we have in Kafka is higher than ours, we cannot proceed
+          # This prevents us from applying older changes to a new schedule
+          if @catchup_schedule[:schedule_version] > schedule.version
+            @incompatible = true
+
+            return
+          end
+
+          # Now we can synchronize the in-memory state based on the last state stored in Kafka
+          schedule.each do |task|
+            stored_task = @catchup_schedule[:tasks][task.id.to_sym]
+
+            next unless stored_task
+
+            stored_previous_time = stored_task[:previous_time]
+            task.previous_time = stored_previous_time.zero? ? 0 : Time.at(stored_previous_time)
+
+            stored_task[:enabled] ? task.enable : task.disable
+          end
+
+          @catchup_commands.each do |cmd|
+            apply_command(cmd)
+          end
+
+          # We make sure to save in Kafka once more once everything is up to date
+          snapshot
+
+          @catchup_schedule = nil
+          @catchup_commands = []
+        end
+
+        # Run all tasks that should run at a given time and if any tasks were changed in any way
+        # or executed, stores the most recent state back to Kafka
+        def call
+          changed = false
+
+          schedule.each do |task|
+            changed = true if task.changed?
+
+            unless task.call?
+              task.clear
+
+              next
+            end
+
+            changed = true
+            task.call
+          end
+
+          snapshot if changed
+        end
+
+        private
+
+        # @return [Karafka::Pro::RecurringTasks::Schedule] current in-memory schedule
+        def schedule
+          ::Karafka::Pro::RecurringTasks.schedule
+        end
+
+        # Dispatches the current schedule state to Kafka
+        def snapshot
+          Dispatcher.schedule
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/recurring_tasks/listener.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module RecurringTasks
+      # Listener we use to track execution of recurring tasks and publish those events into the
+      # recurring tasks log table
+      class Listener
+        # @param event [Karafka::Core::Monitoring::Event] task execution event
+        def on_recurring_tasks_task_executed(event)
+          Dispatcher.log(event)
+        end
+
+        # @param event [Karafka::Core::Monitoring::Event] error that occurred
+        # @note We do nothing with other errors. We only want to log and dispatch information about
+        #   the recurring tasks errors. The general Web UI error tracking may also work but those
+        #   are independent. It is not to replace the Web UI tracking but to just log failed
+        #   executions in the same way as successful but just with the failure as an outcome.
+        def on_error_occurred(event)
+          return unless event[:type] == 'recurring_tasks.task.execute.error'
+
+          Dispatcher.log(event)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/recurring_tasks/matcher.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module RecurringTasks
+      # Matcher used to check if given command can be applied to a given task.
+      class Matcher
+        # @param task [Karafka::Pro::RecurringTasks::Task]
+        # @param payload [Hash] command message payload
+        # @return [Boolean] is this message dedicated to current process and is actionable
+        def matches?(task, payload)
+          # We only match commands
+          return false unless payload[:type] == 'command'
+
+          # * is a wildcard to match all for batch commands
+          return false unless payload[:task][:id] == '*' || payload[:task][:id] == task.id
+
+          # Ignore messages that have different schema. This can happen in the middle of
+          # upgrades of the framework. We ignore this not to risk compatibility issues
+          return false unless payload[:schema_version] == Serializer::SCHEMA_VERSION
+
+          true
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/recurring_tasks/schedule.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module RecurringTasks
+      # Represents the current code-context schedule with defined tasks and their cron execution
+      # details. Single schedule includes all the information about all the tasks that we have
+      # defined and to be executed in a given moment in time.
+      class Schedule
+        # @return [String]
+        attr_reader :version
+
+        # @return [Hash<String, Task>]
+        attr_reader :tasks
+
+        # @param version [String] schedule version. In case of usage of versioning it is used to
+        #   ensure, that older still active processes do not intercept the assignment to run older
+        #   version of the scheduler. It is important to make sure, that this string is comparable.
+        def initialize(version:)
+          @version = version
+          @tasks = {}
+        end
+
+        # Adds task into the tasks accumulator
+        # @param task [Task]
+        # @note In case of multiple tasks with the same id, it will overwrite
+        def <<(task)
+          @tasks[task.id] = task
+        end
+
+        # Iterates over tasks yielding them one after another
+        # @param block [Proc] block that will be executed with each task
+        def each(&block)
+          @tasks.each_value(&block)
+        end
+
+        # @param id [String] id of a particular recurring task
+        # @return [Task, nil] task with a given id or nil if not found
+        def find(id)
+          @tasks[id]
+        end
+
+        # Allows us to have a nice DSL for defining schedules
+        # @param args [Array] attributes accepted by the task initializer
+        # @param block [Proc] block to execute
+        def schedule(**args, &block)
+          self << Task.new(**args, &block)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/recurring_tasks/serializer.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module RecurringTasks
+      # Converts schedule command and log details into data we can dispatch to Kafka.
+      class Serializer
+        # Current recurring tasks related schema structure
+        SCHEMA_VERSION = '1.0'
+
+        # @param schedule [Karafka::Pro::RecurringTasks::Schedule] schedule to serialize
+        # @return [String] serialized and compressed current schedule data with its tasks and their
+        #   current state.
+        def schedule(schedule)
+          tasks = {}
+
+          schedule.each do |task|
+            tasks[task.id] = {
+              id: task.id,
+              cron: task.cron.original,
+              previous_time: task.previous_time.to_i,
+              next_time: task.next_time.to_i,
+              enabled: task.enabled?
+            }
+          end
+
+          data = {
+            schema_version: SCHEMA_VERSION,
+            schedule_version: schedule.version,
+            dispatched_at: Time.now.to_f,
+            type: 'schedule',
+            tasks: tasks
+          }
+
+          compress(
+            serialize(data)
+          )
+        end
+
+        # @param command_name [String] command name
+        # @param task_id [String] task id or '*' to match all.
+        # @return [String] serialized and compressed command data
+        def command(command_name, task_id)
+          data = {
+            schema_version: SCHEMA_VERSION,
+            schedule_version: ::Karafka::Pro::RecurringTasks.schedule.version,
+            dispatched_at: Time.now.to_f,
+            type: 'command',
+            command: {
+              name: command_name
+            },
+            task: {
+              id: task_id
+            }
+          }
+
+          compress(
+            serialize(data)
+          )
+        end
+
+        # @param event [Karafka::Core::Monitoring::Event] recurring task dispatch event
+        # @return [String] serialized and compressed event log data
+        def log(event)
+          task = event[:task]
+
+          data = {
+            schema_version: SCHEMA_VERSION,
+            schedule_version: ::Karafka::Pro::RecurringTasks.schedule.version,
+            dispatched_at: Time.now.to_f,
+            type: 'log',
+            task: {
+              id: task.id,
+              time_taken: event.payload[:time] || -1,
+              result: event.payload.key?(:error) ? 'failure' : 'success'
+            }
+          }
+
+          compress(
+            serialize(data)
+          )
+        end
+
+        private
+
+        # @param hash [Hash] hash to cast to json
+        # @return [String] json hash
+        def serialize(hash)
+          hash.to_json
+        end
+
+        # Compresses the provided data
+        #
+        # @param data [String] data to compress
+        # @return [String] compressed data
+        def compress(data)
+          Zlib::Deflate.deflate(data)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/recurring_tasks/setup/config.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module RecurringTasks
+      # Setup and config related recurring tasks components
+      module Setup
+        # Config for recurring tasks
+        class Config
+          extend ::Karafka::Core::Configurable
+
+          setting(:consumer_class, default: Consumer)
+          setting(:deserializer, default: Deserializer.new)
+          setting(:group_id, default: 'karafka_recurring_tasks')
+          # By default we will run the scheduling every 15 seconds since we provide a minute-based
+          # precision
+          setting(:interval, default: 15_000)
+          # Should we log the executions. If true (default) with each cron execution, there will
+          # be a special message published. Useful for debugging.
+          setting(:logging, default: true)
+
+          # Producer to be used by the recurring tasks.
+          # By default it is a `Karafka.producer`, however it may be overwritten if we want to use
+          # a separate instance in case of heavy usage of the  transactional producer, etc.
+          setting(
+            :producer,
+            constructor: -> { ::Karafka.producer },
+            lazy: true
+          )
+
+          setting(:topics) do
+            setting(:schedules, default: 'karafka_recurring_tasks_schedules')
+            setting(:logs, default: 'karafka_recurring_tasks_logs')
+          end
+
+          configure
+        end
+      end
+    end
+  end
+end