kaal 0.3.0 → 0.5.0

data/lib/kaal/definition/database_engine.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+# Copyright Codevedas Inc. 2025-present
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+require 'json'
+require 'kaal/definition/registry'
+require 'kaal/definition/persistence_helpers'
+require 'kaal/persistence/database'
+
+module Kaal
+  module Definition
+    # Sequel-backed definition registry persisted in kaal_definitions.
+    class DatabaseEngine < Registry
+      def initialize(database:)
+        super()
+        @database = Kaal::Persistence::Database.new(database)
+      end
+
+      def upsert_definition(key:, cron:, enabled: true, source: 'code', metadata: {})
+        rows = dataset.where(key: key)
+        existing = rows.first
+        now = Time.now.utc
+        payload = {
+          key: key,
+          cron: cron,
+          enabled: enabled,
+          source: source,
+          metadata: JSON.generate(metadata || {}),
+          created_at: existing ? existing[:created_at] : now,
+          updated_at: now,
+          disabled_at: PersistenceHelpers.disabled_at_for(existing, enabled, now)
+        }
+
+        if existing
+          rows.update(payload)
+        else
+          dataset.insert(payload)
+        end
+
+        find_definition(key)
+      end
+
+      def remove_definition(key)
+        rows = dataset.where(key: key)
+        row = rows.first
+        return nil unless row
+
+        rows.delete
+        self.class.normalize_row(row)
+      end
+
+      def find_definition(key)
+        self.class.normalize_row(dataset.where(key: key).first)
+      end
+
+      def all_definitions
+        dataset.order(:key).all.map { |row| self.class.normalize_row(row) }
+      end
+
+      def enabled_definitions
+        dataset.where(enabled: true).order(:key).all.map { |row| self.class.normalize_row(row) }
+      end
+
+      def self.normalize_row(row)
+        return nil unless row
+
+        {
+          key: row[:key],
+          cron: row[:cron],
+          enabled: row[:enabled] ? true : false,
+          source: row[:source],
+          metadata: PersistenceHelpers.parse_metadata(row[:metadata]),
+          created_at: row[:created_at],
+          updated_at: row[:updated_at],
+          disabled_at: row[:disabled_at]
+        }
+      end
+
+      private
+
+      def dataset
+        @database.definitions_dataset
+      end
+    end
+  end
+end

data/lib/kaal/delayed_job/database_engine.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+# Copyright Codevedas Inc. 2025-present
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+require 'json'
+require 'kaal/delayed_job/registry'
+require 'kaal/persistence/database'
+
+module Kaal
+  module DelayedJob
+    # Sequel-backed delayed-job store persisted in kaal_delayed_jobs.
+    class DatabaseEngine < Registry
+      def initialize(database:, use_skip_locked: false)
+        super()
+        @database = Kaal::Persistence::Database.new(database)
+        @use_skip_locked = use_skip_locked
+      end
+
+      def enqueue(job_id:, run_at:, job_class:, args:, queue: nil, connection: nil)
+        now = Time.now.utc
+        payload = {
+          job_id: job_id,
+          run_at: run_at,
+          job_class: job_class,
+          args: JSON.generate(args),
+          queue: queue,
+          created_at: now
+        }
+
+        dataset_for(connection).insert(payload)
+        self.class.normalize_row(payload)
+      rescue ::Sequel::UniqueConstraintViolation
+        raise DuplicateJobError, "Delayed job #{job_id.inspect} already exists"
+      end
+
+      def pop_due(now:, limit:)
+        return pop_due_with_skip_locked(now:, limit:) if @use_skip_locked
+
+        pop_due_with_delete_confirmation(now:, limit:)
+      end
+
+      def find_job(job_id, connection: @database.connection)
+        self.class.normalize_row(connection[:kaal_delayed_jobs].where(job_id: job_id).first)
+      end
+
+      def all_jobs
+        connection[:kaal_delayed_jobs].order(:run_at, :job_id).filter_map { |row| self.class.normalize_row(row) }
+      end
+
+      def claim_strategy
+        @use_skip_locked ? :skip_locked : :delete_confirmation
+      end
+
+      def self.normalize_row(row)
+        return nil unless row
+
+        {
+          job_id: row[:job_id],
+          run_at: row[:run_at],
+          job_class: row[:job_class],
+          args: parse_args(row[:args]),
+          queue: row[:queue],
+          created_at: row[:created_at]
+        }
+      rescue JSON::ParserError
+        nil
+      end
+
+      private
+
+      def pop_due_with_skip_locked(now:, limit:)
+        connection.transaction do
+          delayed_jobs_dataset = connection[:kaal_delayed_jobs]
+          due_rows = delayed_jobs_dataset.where { run_at <= now }.order(:run_at, :job_id).for_update.skip_locked.limit(limit).all
+          job_ids = due_rows.map { |row| row[:job_id] }
+          normalized_jobs = due_rows.filter_map { |row| self.class.normalize_row(row) }
+          delayed_jobs_dataset.where(job_id: job_ids).delete unless job_ids.empty?
+          normalized_jobs
+        end
+      end
+
+      def pop_due_with_delete_confirmation(now:, limit:)
+        connection.transaction do
+          delayed_jobs_dataset = connection[:kaal_delayed_jobs]
+          due_rows = delayed_jobs_dataset.where { run_at <= now }.order(:run_at, :job_id).limit(limit).all
+          due_rows.each_with_object([]) do |row, jobs|
+            deleted = delayed_jobs_dataset.where(job_id: row[:job_id]).delete
+            normalized_job = self.class.normalize_row(row)
+            jobs << normalized_job if deleted.positive? && normalized_job
+          end
+        end
+      end
+
+      def self.parse_args(args_payload)
+        JSON.parse(args_payload || '[]')
+      end
+      private_class_method :parse_args
+
+      def dataset_for(connection)
+        return dataset unless connection
+
+        Kaal::Persistence::Database.new(connection).delayed_jobs_dataset
+      end
+
+      def dataset
+        @database.delayed_jobs_dataset
+      end
+
+      def connection
+        @database.connection
+      end
+    end
+  end
+end

data/lib/kaal/delayed_job/dispatch_failure_logger.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+# Copyright Codevedas Inc. 2025-present
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+module Kaal
+  module DelayedJob
+    # Shared delayed-dispatch failure logging for at-most-once dispatches.
+    module DispatchFailureLogger
+      module_function
+
+      def log_claimed_dispatch_failure(logger:, job:, error:)
+        return unless logger
+
+        message = "Delayed job #{job.fetch(:job_id)} dispatch failed after claim; " \
+                  "job_class=#{job.fetch(:job_class).inspect} " \
+                  "queue=#{job[:queue].inspect} " \
+                  "run_at=#{job.fetch(:run_at)} " \
+                  'job was already claimed and will not be retried: ' \
+                  "#{error.class}: #{error.message}"
+
+        if logger.respond_to?(:fatal)
+          logger.fatal(message)
+        else
+          logger.error(message)
+        end
+      end
+    end
+  end
+end

data/lib/kaal/delayed_job/memory_engine.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+# Copyright Codevedas Inc. 2025-present
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+require 'kaal/delayed_job/registry'
+require 'kaal/support/hash_tools'
+
+module Kaal
+  module DelayedJob
+    # In-memory delayed job store for single-process development and tests.
+    class MemoryEngine < Registry
+      include Kaal::Support::HashTools
+
+      def initialize
+        super
+        @jobs = {}
+        @mutex = Mutex.new
+      end
+
+      def enqueue(job_id:, run_at:, job_class:, args:, queue: nil, **)
+        @mutex.synchronize do
+          raise DuplicateJobError, "Delayed job #{job_id.inspect} already exists" if @jobs.key?(job_id)
+
+          job = build_job(job_id:, run_at:, job_class:, args:, queue:)
+          @jobs[job_id] = job
+          deep_dup(job)
+        end
+      end
+
+      def pop_due(now:, limit:)
+        @mutex.synchronize do
+          due_jobs = @jobs.values
+                          .select { |job| job[:run_at] <= now }
+                          .sort_by { |job| job.values_at(:run_at, :job_id) }
+                          .first(limit)
+
+          due_jobs.each do |job|
+            job_id = job.fetch(:job_id)
+            @jobs.delete(job_id)
+          end
+          due_jobs.map { |job| deep_dup(job) }
+        end
+      end
+
+      def find_job(job_id)
+        @mutex.synchronize { deep_dup(@jobs[job_id]) }
+      end
+
+      def all_jobs
+        @mutex.synchronize do
+          @jobs.values.sort_by { |job| [job[:run_at], job[:job_id]] }.map { |job| deep_dup(job) }
+        end
+      end
+
+      def clear
+        @mutex.synchronize { @jobs.clear }
+      end
+
+      def claim_strategy
+        :atomic_pop
+      end
+
+      private
+
+      def build_job(job_id:, run_at:, job_class:, args:, queue:)
+        {
+          job_id: job_id,
+          run_at: run_at,
+          job_class: job_class,
+          args: deep_dup(args),
+          queue: queue,
+          created_at: Time.now.utc
+        }
+      end
+    end
+  end
+end

data/lib/kaal/delayed_job/mysql_version_support.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+# Copyright Codevedas Inc. 2025-present
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+module Kaal
+  module DelayedJob
+    # MySQL version helper for delayed-job claim strategy selection.
+    module MySQLVersionSupport
+      MINIMUM_SKIP_LOCKED_VERSION = 800_00
+
+      module_function
+
+      def skip_locked_supported?(version_string)
+        version_number(version_string) >= MINIMUM_SKIP_LOCKED_VERSION
+      end
+
+      def version_number(version_string)
+        major, minor, patch = version_components(version_string)
+        return 0 unless major && minor && patch
+
+        (major * 10_000) + (minor * 100) + patch
+      end
+
+      def version_components(version_string)
+        major, minor, patch = version_string.to_s.split('.', 3)
+        [
+          integer_prefix(major),
+          integer_prefix(minor),
+          integer_prefix(patch)
+        ]
+      end
+
+      def integer_prefix(value)
+        digits = value.to_s.each_char.take_while { |character| character.between?('0', '9') }.join
+        return nil if digits.empty?
+
+        digits.to_i
+      end
+    end
+  end
+end

data/lib/kaal/delayed_job/redis_engine.rb

@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+# Copyright Codevedas Inc. 2025-present
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+require 'json'
+require 'time'
+require 'kaal/delayed_job/registry'
+
+module Kaal
+  module DelayedJob
+    # Redis-backed delayed-job store using a sorted set plus payload hash.
+    class RedisEngine < Registry
+      def initialize(redis, namespace: 'kaal')
+        super()
+        @redis = redis
+        @namespace = namespace
+      end
+
+      def enqueue(job_id:, run_at:, job_class:, args:, queue: nil, **)
+        payload = JSON.generate(
+          job_id: job_id,
+          run_at: run_at.iso8601,
+          job_class: job_class,
+          args: args,
+          queue: queue,
+          created_at: Time.now.utc.iso8601
+        )
+
+        script = <<~LUA
+          if redis.call('HEXISTS', KEYS[1], ARGV[1]) == 1 then
+            return 0
+          end
+
+          redis.call('HSET', KEYS[1], ARGV[1], ARGV[2])
+          redis.call('ZADD', KEYS[2], ARGV[3], ARGV[1])
+          return 1
+        LUA
+
+        result = @redis.eval(script, keys: [payloads_key, schedule_key], argv: [job_id, payload, run_at.to_f])
+        raise DuplicateJobError, "Delayed job #{job_id.inspect} already exists" unless [1, '1', true].include?(result)
+
+        find_job(job_id)
+      end
+
+      def pop_due(now:, limit:)
+        script = <<~LUA
+          local job_ids = redis.call('ZRANGEBYSCORE', KEYS[2], '-inf', ARGV[1], 'LIMIT', 0, ARGV[2])
+          local payloads = {}
+
+          for _, job_id in ipairs(job_ids) do
+            if redis.call('ZREM', KEYS[2], job_id) == 1 then
+              local payload = redis.call('HGET', KEYS[1], job_id)
+              if payload then
+                redis.call('HDEL', KEYS[1], job_id)
+                table.insert(payloads, payload)
+              end
+            end
+          end
+
+          return payloads
+        LUA
+
+        Array(@redis.eval(script, keys: [payloads_key, schedule_key], argv: [now.to_f, limit])).filter_map do |raw|
+          self.class.deserialize(raw)
+        end
+      end
+
+      def find_job(job_id)
+        self.class.deserialize(@redis.hget(payloads_key, job_id))
+      end
+
+      def all_jobs
+        Array(@redis.zrange(schedule_key, 0, -1)).filter_map { |job_id| find_job(job_id) }
+      end
+
+      def claim_strategy
+        :atomic_pop
+      end
+
+      def self.deserialize(raw)
+        return nil unless raw
+
+        parsed = JSON.parse(raw)
+        run_at = parse_time(parsed['run_at'])
+        created_at = parse_time(parsed['created_at'])
+        return nil unless run_at && created_at
+
+        {
+          job_id: parsed['job_id'],
+          run_at: run_at,
+          job_class: parsed['job_class'],
+          args: parsed['args'] || [],
+          queue: parsed['queue'],
+          created_at: created_at
+        }
+      rescue JSON::ParserError
+        nil
+      end
+
+      def self.parse_time(value)
+        Time.iso8601(value.to_s)
+      rescue ArgumentError
+        nil
+      end
+
+      private
+
+      def payloads_key
+        "#{@namespace}:delayed_jobs:payloads"
+      end
+
+      def schedule_key
+        "#{@namespace}:delayed_jobs:schedule"
+      end
+    end
+  end
+end

data/lib/kaal/delayed_job/registry.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+# Copyright Codevedas Inc. 2025-present
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+module Kaal
+  module DelayedJob
+    # Raised when a delayed job with the same identifier already exists.
+    class DuplicateJobError < StandardError; end
+
+    # Base abstraction for delayed-job persistence.
+    class Registry
+      def enqueue(**)
+        raise NotImplementedError, "#{self.class.name} must implement #enqueue"
+      end
+
+      def pop_due(**)
+        raise NotImplementedError, "#{self.class.name} must implement #pop_due"
+      end
+
+      def find_job(_job_id)
+        raise NotImplementedError, "#{self.class.name} must implement #find_job"
+      end
+
+      def all_jobs
+        raise NotImplementedError, "#{self.class.name} must implement #all_jobs"
+      end
+
+      def claim_strategy
+        raise NotImplementedError, "#{self.class.name} must implement #claim_strategy"
+      end
+
+      def requires_dispatch_lock?
+        false
+      end
+    end
+  end
+end

data/lib/kaal/dispatch/database_engine.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+# Copyright Codevedas Inc. 2025-present
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+require 'kaal/dispatch/registry'
+require 'kaal/persistence/database'
+
+module Kaal
+  module Dispatch
+    # Sequel-backed dispatch registry stored in kaal_dispatches.
+    class DatabaseEngine < Registry
+      def initialize(database:, namespace: nil)
+        super()
+        @database = Kaal::Persistence::Database.new(database)
+        @namespace = namespace
+      end
+
+      def log_dispatch(key, fire_time, node_id, status = 'dispatched')
+        now = Time.now.utc
+        storage_key = namespaced_key(key)
+        attributes = {
+          key: storage_key,
+          fire_time: fire_time,
+          dispatched_at: now,
+          node_id: node_id,
+          status: status
+        }
+        dispatches_dataset = dataset
+        update_values = { dispatched_at: now, node_id: node_id, status: status }
+        begin
+          dispatches_dataset.insert_conflict(
+            target: %i[key fire_time],
+            update: update_values
+          ).insert(attributes)
+        rescue NoMethodError => e
+          raise unless e.name == :insert_conflict
+
+          begin
+            dispatches_dataset.insert(attributes)
+          rescue ::Sequel::UniqueConstraintViolation
+            dispatches_dataset.where(key: storage_key, fire_time: fire_time).update(update_values)
+          end
+        end
+
+        find_dispatch(key, fire_time)
+      end
+
+      def find_dispatch(key, fire_time)
+        self.class.normalize_row(dataset.where(key: namespaced_key(key), fire_time: fire_time).first, namespace: @namespace)
+      end
+
+      def find_by_key(key)
+        query(key: namespaced_key(key))
+      end
+
+      def find_by_node(node_id)
+        query(node_id: node_id)
+      end
+
+      def find_by_status(status)
+        query(status: status)
+      end
+
+      def cleanup(recovery_window: 86_400)
+        cutoff_time = Time.now.utc - recovery_window
+        cleanup_dataset.where { fire_time < cutoff_time }.delete
+      end
+
+      def self.normalize_row(row, namespace: nil)
+        return nil unless row
+
+        {
+          key: strip_namespace(row[:key], namespace:),
+          fire_time: row[:fire_time],
+          dispatched_at: row[:dispatched_at],
+          node_id: row[:node_id],
+          status: row[:status]
+        }
+      end
+
+      def self.strip_namespace(key, namespace:)
+        return key if namespace.to_s.empty?
+
+        prefix = "#{namespace}:"
+        key.start_with?(prefix) ? key.delete_prefix(prefix) : key
+      end
+
+      private
+
+      def dataset
+        @database.dispatches_dataset
+      end
+
+      def namespaced_key(key)
+        return key if @namespace.to_s.empty?
+
+        "#{@namespace}:#{key}"
+      end
+
+      def query(filters)
+        query_dataset(filters).reverse_order(:fire_time).all.map { |row| self.class.normalize_row(row, namespace: @namespace) }
+      end
+
+      def query_dataset(filters)
+        relation = dataset.where(filters)
+        return relation if @namespace.to_s.empty? || filters.key?(:key)
+
+        relation.where(::Sequel.lit('key LIKE ?', "#{@namespace}:%"))
+      end
+
+      def cleanup_dataset
+        return dataset if @namespace.to_s.empty?
+
+        dataset.where(::Sequel.lit('key LIKE ?', "#{@namespace}:%"))
+      end
+    end
+  end
+end

data/lib/kaal/internal/active_record/base_record.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+# Copyright Codevedas Inc. 2025-present
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+module Kaal
+  module Internal
+    module ActiveRecord
+      # Shared abstract Active Record base class for Kaal tables.
+      class BaseRecord < ::ActiveRecord::Base
+        self.abstract_class = true
+      end
+    end
+  end
+end