karafka 2.4.0 → 2.4.18

data/lib/karafka/pro/recurring_tasks/consumer.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module RecurringTasks
+      # Consumer responsible for management of the recurring tasks and their execution
+      # There are some assumptions made here that always need to be satisfied:
+      #   - we only run schedules that are of same or newer version
+      #   - we always mark as consumed in such a way, that the first message received after
+      #     assignment (if any) is a state
+      class Consumer < ::Karafka::BaseConsumer
+        # @param args [Array] all arguments accepted by the consumer
+        def initialize(*args)
+          super
+          @executor = Executor.new
+        end
+
+        def consume
+          # There is nothing we can do if we operate on a newer schedule. In such cases we should
+          # just wait and re-raise error hoping someone will notice or that this will be
+          # reassigned to a process with newer schedule
+          raise(Errors::IncompatibleScheduleError) if @executor.incompatible?
+
+          messages.each do |message|
+            payload = message.payload
+            type = payload[:type]
+
+            case type
+            when 'schedule'
+              # If we're replaying data, we need to record the most recent stored state, so we
+              # can use this data to fully initialize the scheduler
+              @executor.update_state(payload) if @executor.replaying?
+
+              # If this is first message we cannot mark it on the previous offset
+              next if message.offset.zero?
+
+              # We always mark as consumed in such a way, that when replaying, we start from a
+              # schedule state message. This makes it easier to recover.
+              mark_as_consumed Karafka::Messages::Seek.new(
+                topic.name,
+                partition,
+                message.offset - 1
+              )
+            when 'command'
+              @executor.apply_command(payload)
+
+              next if @executor.replaying?
+
+              # Execute on each incoming command to have nice latency but only after replaying
+              # During replaying we should not execute because there may be more state changes
+              # that collectively have a different outcome
+              @executor.call
+            else
+              raise ::Karafka::Errors::UnsupportedCaseError, type
+            end
+          end
+
+          eofed if eofed?
+        end
+
+        # Starts the final replay process if we reached eof during replaying
+        def eofed
+          # We only mark as replayed if we were replaying in the first place
+          # If already replayed, nothing to do
+          return unless @executor.replaying?
+
+          @executor.replay
+        end
+
+        # Runs the cron execution if all good.
+        def tick
+          # Do nothing until we fully recover the correct state
+          return if @executor.replaying?
+
+          # If the state is incompatible, we can only raise an error.
+          # We do it here and in the `#consume` so the pause-retry kicks in basically reporting
+          # on this issue once every minute. That way user should not miss this.
+          # We use seek to move so we can achieve a pause of 60 seconds in between consecutive
+          # errors instead of on each tick because it is much more frequent.
+          if @executor.incompatible?
+            if messages.empty?
+              raise Errors::IncompatibleScheduleError
+            else
+              return seek(messages.last.offset - 1)
+            end
+          end
+
+          # If all good and compatible we can execute the recurring tasks
+          @executor.call
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/recurring_tasks/contracts/config.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module RecurringTasks
+      # Recurring Tasks related contracts
+      module Contracts
+        # Makes sure, all the expected config is defined as it should be
+        class Config < ::Karafka::Contracts::Base
+          configure do |config|
+            config.error_messages = YAML.safe_load(
+              File.read(
+                File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+              )
+            ).fetch('en').fetch('validations').fetch('config')
+          end
+
+          nested(:recurring_tasks) do
+            required(:consumer_class) { |val| val < ::Karafka::BaseConsumer }
+            required(:deserializer) { |val| !val.nil? }
+            required(:logging) { |val| [true, false].include?(val) }
+            # Do not allow to run more often than every 5 seconds
+            required(:interval) { |val| val.is_a?(Integer) && val >= 1_000 }
+            required(:group_id) do |val|
+              val.is_a?(String) && Karafka::Contracts::TOPIC_REGEXP.match?(val)
+            end
+
+            nested(:topics) do
+              required(:schedules) do |val|
+                val.is_a?(String) && Karafka::Contracts::TOPIC_REGEXP.match?(val)
+              end
+
+              required(:logs) do |val|
+                val.is_a?(String) && Karafka::Contracts::TOPIC_REGEXP.match?(val)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/recurring_tasks/contracts/task.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module RecurringTasks
+      # Recurring Tasks related contracts
+      module Contracts
+        # Ensures that task details are as expected
+        class Task < ::Karafka::Contracts::Base
+          configure do |config|
+            config.error_messages = YAML.safe_load(
+              File.read(
+                File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+              )
+            ).fetch('en').fetch('validations').fetch('recurring_tasks')
+          end
+
+          # Regexp to ensure all tasks ids are URL safe
+          ID_REGEXP = /\A[a-zA-Z0-9_-]{1,}\z/
+
+          required(:id) { |val| val.is_a?(String) && val.match?(ID_REGEXP) }
+          required(:cron) { |val| val.is_a?(String) && val.length.positive? }
+          required(:enabled) { |val| [true, false].include?(val) }
+          required(:changed) { |val| [true, false].include?(val) }
+          required(:previous_time) { |val| val.is_a?(Numeric) || val.is_a?(Time) }
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/recurring_tasks/deserializer.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module RecurringTasks
+      # Recurring Tasks data deserializer. We compress data ourselves because we cannot rely on
+      # any external optional features like certain compression types, etc. By doing this that way
+      # we can ensure we have full control over the compression.
+      #
+      # @note We use `symbolize_names` because we want to use the same convention of hash building
+      #   for producing, consuming and displaying related data as in other places.
+      class Deserializer
+        # @param message [::Karafka::Messages::Message]
+        # @return [Hash] deserialized data
+        def call(message)
+          ::JSON.parse(
+            Zlib::Inflate.inflate(message.raw_payload),
+            symbolize_names: true
+          )
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/recurring_tasks/dispatcher.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module RecurringTasks
+      # Dispatches appropriate recurring tasks related messages to expected topics
+      class Dispatcher
+        class << self
+          # Snapshots to Kafka current schedule state
+          def schedule
+            produce(
+              topics.schedules,
+              'state:schedule',
+              serializer.schedule(::Karafka::Pro::RecurringTasks.schedule)
+            )
+          end
+
+          # Dispatches the command request
+          #
+          # @param name [String, Symbol] name of the command we want to deal with in the process
+          # @param task_id [String] id of the process. We use name instead of id only
+          #   because in the web ui we work with the full name and it is easier. Since
+          def command(name, task_id)
+            produce(
+              topics.schedules,
+              "command:#{name}:#{task_id}",
+              serializer.command(name, task_id)
+            )
+          end
+
+          # Dispatches the task execution log record
+          # @param event [Karafka::Core::Monitoring::Event]
+          def log(event)
+            produce(
+              topics.logs,
+              event[:task].id,
+              serializer.log(event)
+            )
+          end
+
+          private
+
+          # @return [::WaterDrop::Producer] web ui producer
+          def producer
+            ::Karafka::App.config.recurring_tasks.producer
+          end
+
+          # @return [String] consumers commands topic
+          def topics
+            ::Karafka::App.config.recurring_tasks.topics
+          end
+
+          # @return [Serializer]
+          def serializer
+            Serializer.new
+          end
+
+          # Converts payload to json, compresses it and dispatches to Kafka
+          #
+          # @param topic [String] target topic
+          # @param key [String]
+          # @param payload [Hash] hash with payload
+          def produce(topic, key, payload)
+            producer.produce_async(
+              topic: topic,
+              key: key,
+              partition: 0,
+              payload: payload,
+              headers: { 'zlib' => 'true' }
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/recurring_tasks/errors.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module RecurringTasks
+      # Recurring Tasks related errors
+      module Errors
+        # Base for all the recurring tasks errors
+        BaseError = Class.new(::Karafka::Errors::BaseError)
+
+        # Raised when older cron manager version is trying to work with newer schema
+        IncompatibleSchemaError = Class.new(BaseError)
+
+        # Raised if you use versioned schedule and an existing schedule from Kafka was picked up
+        # by a process with older schedule version.
+        # This is a safe-guard to protect against a cases where there would be a temporary
+        # reassignment of newer schedule data into older process during deployment. It should go
+        # away once all processes are rolled.
+        IncompatibleScheduleError = Class.new(BaseError)
+      end
+    end
+  end
+end

data/lib/karafka/pro/recurring_tasks/executor.rb

@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module RecurringTasks
+      # Executor is responsible for management of the state of recurring tasks schedule and
+      # is the heart of recurring tasks. It coordinates the replaying process as well as
+      # tracking of data on changes.
+      class Executor
+        # Task commands we support and that can be triggered on tasks (if matched)
+        COMMANDS = %w[
+          disable
+          enable
+          trigger
+        ].freeze
+
+        def initialize
+          @replaying = true
+          @incompatible = false
+          @catchup_commands = []
+          @catchup_schedule = nil
+          @matcher = Matcher.new
+        end
+
+        # @return [Boolean] are we in the replaying phase or not (false means, regular operations)
+        def replaying?
+          @replaying
+        end
+
+        # @return [Boolean] Is the current process schedule incompatible (older) than the one
+        #   that we have in memory
+        def incompatible?
+          @incompatible
+        end
+
+        # Applies given command to task (or many tasks) by running the command on tasks that match
+        # @param command_hash [Hash] deserialized command data
+        def apply_command(command_hash)
+          cmd_name = command_hash[:command][:name]
+
+          raise(Karafka::Errors::UnsupportedCaseError, cmd_name) unless COMMANDS.include?(cmd_name)
+
+          schedule.each do |task|
+            next unless @matcher.matches?(task, command_hash)
+
+            task.public_send(cmd_name)
+          end
+        end
+
+        # Updates the catchup state
+        # @param schedule_hash [Hash] deserialized schedule hash hash
+        def update_state(schedule_hash)
+          @catchup_schedule = schedule_hash
+        end
+
+        # Once all previous data is accumulated runs the catchup process to establish current
+        # state of the recurring tasks schedule execution.
+        #
+        # It includes applying any requested commands as well as synchronizing execution details
+        # for existing schedule and making sure all is loaded correctly.
+        def replay
+          # Ensure replaying happens only once
+          return unless @replaying
+
+          @replaying = false
+
+          # When there is nothing to replay and synchronize, we can just save the state and
+          # proceed
+          if @catchup_commands.empty? && @catchup_schedule.nil?
+            snapshot
+
+            return
+          end
+
+          # If the schedule version we have in Kafka is higher than ours, we cannot proceed
+          # This prevents us from applying older changes to a new schedule
+          if @catchup_schedule[:schedule_version] > schedule.version
+            @incompatible = true
+
+            return
+          end
+
+          # Now we can synchronize the in-memory state based on the last state stored in Kafka
+          schedule.each do |task|
+            stored_task = @catchup_schedule[:tasks][task.id.to_sym]
+
+            next unless stored_task
+
+            stored_previous_time = stored_task[:previous_time]
+            task.previous_time = stored_previous_time.zero? ? 0 : Time.at(stored_previous_time)
+
+            stored_task[:enabled] ? task.enable : task.disable
+          end
+
+          @catchup_commands.each do |cmd|
+            apply_command(cmd)
+          end
+
+          # We make sure to save in Kafka once more once everything is up to date
+          snapshot
+
+          @catchup_schedule = nil
+          @catchup_commands = []
+        end
+
+        # Run all tasks that should run at a given time and if any tasks were changed in any way
+        # or executed, stores the most recent state back to Kafka
+        def call
+          changed = false
+
+          schedule.each do |task|
+            changed = true if task.changed?
+
+            unless task.call?
+              task.clear
+
+              next
+            end
+
+            changed = true
+            task.call
+          end
+
+          snapshot if changed
+        end
+
+        private
+
+        # @return [Karafka::Pro::RecurringTasks::Schedule] current in-memory schedule
+        def schedule
+          ::Karafka::Pro::RecurringTasks.schedule
+        end
+
+        # Dispatches the current schedule state to Kafka
+        def snapshot
+          Dispatcher.schedule
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/recurring_tasks/listener.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module RecurringTasks
+      # Listener we use to track execution of recurring tasks and publish those events into the
+      # recurring tasks log table
+      class Listener
+        # @param event [Karafka::Core::Monitoring::Event] task execution event
+        def on_recurring_tasks_task_executed(event)
+          Dispatcher.log(event)
+        end
+
+        # @param event [Karafka::Core::Monitoring::Event] error that occurred
+        # @note We do nothing with other errors. We only want to log and dispatch information about
+        #   the recurring tasks errors. The general Web UI error tracking may also work but those
+        #   are independent. It is not to replace the Web UI tracking but to just log failed
+        #   executions in the same way as successful but just with the failure as an outcome.
+        def on_error_occurred(event)
+          return unless event[:type] == 'recurring_tasks.task.execute.error'
+
+          Dispatcher.log(event)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/recurring_tasks/matcher.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module RecurringTasks
+      # Matcher used to check if given command can be applied to a given task.
+      class Matcher
+        # @param task [Karafka::Pro::RecurringTasks::Task]
+        # @param payload [Hash] command message payload
+        # @return [Boolean] is this message dedicated to current process and is actionable
+        def matches?(task, payload)
+          # We only match commands
+          return false unless payload[:type] == 'command'
+
+          # * is a wildcard to match all for batch commands
+          return false unless payload[:task][:id] == '*' || payload[:task][:id] == task.id
+
+          # Ignore messages that have different schema. This can happen in the middle of
+          # upgrades of the framework. We ignore this not to risk compatibility issues
+          return false unless payload[:schema_version] == Serializer::SCHEMA_VERSION
+
+          true
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/recurring_tasks/schedule.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module RecurringTasks
+      # Represents the current code-context schedule with defined tasks and their cron execution
+      # details. Single schedule includes all the information about all the tasks that we have
+      # defined and to be executed in a given moment in time.
+      class Schedule
+        # @return [String]
+        attr_reader :version
+
+        # @return [Hash<String, Task>]
+        attr_reader :tasks
+
+        # @param version [String] schedule version. In case of usage of versioning it is used to
+        #   ensure, that older still active processes do not intercept the assignment to run older
+        #   version of the scheduler. It is important to make sure, that this string is comparable.
+        def initialize(version:)
+          @version = version
+          @tasks = {}
+        end
+
+        # Adds task into the tasks accumulator
+        # @param task [Task]
+        # @note In case of multiple tasks with the same id, it will overwrite
+        def <<(task)
+          @tasks[task.id] = task
+        end
+
+        # Iterates over tasks yielding them one after another
+        # @param block [Proc] block that will be executed with each task
+        def each(&block)
+          @tasks.each_value(&block)
+        end
+
+        # @param id [String] id of a particular recurring task
+        # @return [Task, nil] task with a given id or nil if not found
+        def find(id)
+          @tasks[id]
+        end
+
+        # Allows us to have a nice DSL for defining schedules
+        # @param args [Array] attributes accepted by the task initializer
+        # @param block [Proc] block to execute
+        def schedule(**args, &block)
+          self << Task.new(**args, &block)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/recurring_tasks/serializer.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module RecurringTasks
+      # Converts schedule command and log details into data we can dispatch to Kafka.
+      class Serializer
+        # Current recurring tasks related schema structure
+        SCHEMA_VERSION = '1.0'
+
+        # @param schedule [Karafka::Pro::RecurringTasks::Schedule] schedule to serialize
+        # @return [String] serialized and compressed current schedule data with its tasks and their
+        #   current state.
+        def schedule(schedule)
+          tasks = {}
+
+          schedule.each do |task|
+            tasks[task.id] = {
+              id: task.id,
+              cron: task.cron.original,
+              previous_time: task.previous_time.to_i,
+              next_time: task.next_time.to_i,
+              enabled: task.enabled?
+            }
+          end
+
+          data = {
+            schema_version: SCHEMA_VERSION,
+            schedule_version: schedule.version,
+            dispatched_at: Time.now.to_f,
+            type: 'schedule',
+            tasks: tasks
+          }
+
+          compress(
+            serialize(data)
+          )
+        end
+
+        # @param command_name [String] command name
+        # @param task_id [String] task id or '*' to match all.
+        # @return [String] serialized and compressed command data
+        def command(command_name, task_id)
+          data = {
+            schema_version: SCHEMA_VERSION,
+            schedule_version: ::Karafka::Pro::RecurringTasks.schedule.version,
+            dispatched_at: Time.now.to_f,
+            type: 'command',
+            command: {
+              name: command_name
+            },
+            task: {
+              id: task_id
+            }
+          }
+
+          compress(
+            serialize(data)
+          )
+        end
+
+        # @param event [Karafka::Core::Monitoring::Event] recurring task dispatch event
+        # @return [String] serialized and compressed event log data
+        def log(event)
+          task = event[:task]
+
+          data = {
+            schema_version: SCHEMA_VERSION,
+            schedule_version: ::Karafka::Pro::RecurringTasks.schedule.version,
+            dispatched_at: Time.now.to_f,
+            type: 'log',
+            task: {
+              id: task.id,
+              time_taken: event.payload[:time] || -1,
+              result: event.payload.key?(:error) ? 'failure' : 'success'
+            }
+          }
+
+          compress(
+            serialize(data)
+          )
+        end
+
+        private
+
+        # @param hash [Hash] hash to cast to json
+        # @return [String] json hash
+        def serialize(hash)
+          hash.to_json
+        end
+
+        # Compresses the provided data
+        #
+        # @param data [String] data to compress
+        # @return [String] compressed data
+        def compress(data)
+          Zlib::Deflate.deflate(data)
+        end
+      end
+    end
+  end
+end