kaal 0.3.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 54a0eb1cebfc4adc18c4b4ed105d47bb26a84d3512fbce891243a356bca5715a
-  data.tar.gz: dd1fb18c3060c4688ea0f96c5628cd40001fb318b7745d411c46108fa06b45b3
+  metadata.gz: c304a5d511f122e3b14d0a6dfa88153d49fbf8bf2ddec699340545831c38e0af
+  data.tar.gz: c03c7fa46c7f301d100ed38bca9e9d96e3451bd69d18b801b654f8fb97bee9a8
 SHA512:
-  metadata.gz: fda7173306897889b750707be46778bfc0b4893ce87759f30f060a37879212eab2731316a31c875acdcf0e7aad9db1caddbc11e7c0b0b16e3707edbfd1b1eec3
-  data.tar.gz: a8d64f8ca728665167e8be7ad5e1e6a417066f6016ff2b838aa47cde5aa3369045b1ea2c3aaf326ce4b00cc94d0f519ebe464978660ad13cc29f6db90cf3c834
+  metadata.gz: b65e18daf9353b7bdd1bf4709aa8175eb1be0e7e76b68b55f2163be96e9db800adc87c075512a7c0b0a44e34f85b56055f2f9f4db165e3187ca1067a17195775
+  data.tar.gz: 6b148d15f5f3092af565d47b8120c7bf52b28ba198fbbc14a5fcb2aad516aaf6de6d4814e5f583e494c00b1b7bdb283d7f2a8eb7dc8879eaf6bd1005c9584383

data/README.md

@@ -2,7 +2,7 @@
 
 Distributed cron scheduling for plain Ruby.
 
-`kaal` is the core engine gem. It owns scheduler/runtime behavior, the registry APIs, and the plain Ruby CLI. SQL persistence lives in adapter gems such as `kaal-sequel` and `kaal-activerecord`.
+`kaal` is the core engine gem. It owns scheduler/runtime behavior, the registry APIs, the plain Ruby CLI, delayed-job dispatch, and the optional SQL backend surfaces.
 
 ## Installation
 
@@ -29,11 +29,7 @@ Supported backends:
 - `memory`
 - `redis`
 
-If you want SQL persistence instead, add one of:
-
-- `kaal-sequel` for Sequel-backed SQL in plain Ruby
-- `kaal-activerecord` for Active Record-backed SQL in plain Ruby
-- `kaal-rails` for Rails
+If you want SQL persistence instead, add the runtime libraries your app uses, such as `sequel`, `activerecord`, `sqlite3`, `pg`, or `mysql2`, then configure one of the explicit `Kaal::Backend::*` SQL backends.
 
 ## Configuration
 
@@ -114,6 +110,8 @@ REDIS_URL=redis://127.0.0.1:6379/0 bin/rspec-e2e redis
 
 ## Runtime API
 
+Recurring jobs:
+
 ```ruby
 Kaal.register(
   key: 'reports:daily',
@@ -126,10 +124,44 @@ Kaal.register(
 Kaal.start!
 ```
 
-## Adapter Gems
+Delayed jobs:
+
+```ruby
+Kaal.enqueue_at(
+  at: Time.now.utc + 300,
+  job_class: "InvoiceReminderJob",
+  args: [123],
+  queue: "mailers",
+  job_id: "invoice-reminder:123"
+)
+```
+
+Rules shared by the runtime surface:
+
+- delayed jobs use `job_id` as their identity and require it to be unique while pending
+- delayed-job `args` are positional only
+- recurring and delayed jobs share the same job-class dispatch rules
+- string job classes are constantized and class or module values are used directly
+
+To restrict delayed-job class names:
+
+```ruby
+Kaal.configure do |config|
+  config.delayed_job_allowed_class_prefixes = ["Reports::", "Billing::"]
+end
+```
+
+An empty `delayed_job_allowed_class_prefixes` list leaves delayed-job class resolution unrestricted. That is reasonable for local or trusted deployments. On shared Redis or SQL backends in production, set a restrictive prefix list.
+
+## SQL Backends
+
+Use the explicit SQL backends when you want persisted registries:
+
+- `Kaal::Backend::SQLite`
+- `Kaal::Backend::Postgres`
+- `Kaal::Backend::MySQL`
+- `kaal-rails` for Rails-native install and auto-wiring
 
-Use adapter gems when you want persisted SQL registries:
+For SQL-backed deployments, run the generated migrations so `kaal_delayed_jobs` exists alongside the recurring scheduler tables.
 
-- `kaal-sequel` for Sequel-backed persistence
-- `kaal-activerecord` for Active Record-backed persistence
-- `kaal-rails` for Rails plugin integration over `kaal-activerecord`
+Postgres and supported MySQL versions claim due delayed jobs with `SKIP LOCKED`. Older SQL paths still preserve correctness with delete confirmation, and Kaal adds a small pre-claim jitter there to reduce multi-node contention.

data/lib/kaal/active_record_support.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+# Copyright Codevedas Inc. 2025-present
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+require 'fileutils'
+
+module Kaal
+  # Active Record migration/install support for SQL-backed Kaal backends.
+  module ActiveRecord
+    module_function
+
+    def install_postgres_migration(target_dir:, migration_name: 'Create Kaal Postgres Backend')
+      install_migrations(target_dir:, backend: 'postgres', migration_name:)
+    end
+
+    def install_mysql_migration(target_dir:, migration_name: 'Create Kaal MySQL Backend')
+      install_migrations(target_dir:, backend: 'mysql', migration_name:)
+    end
+
+    def install_sqlite_migration(target_dir:, migration_name: 'Create Kaal SQLite Backend')
+      install_migrations(target_dir:, backend: 'sqlite', migration_name:)
+    end
+
+    def install_migrations(target_dir:, backend:, migration_name: nil, time_source: -> { Time.now.utc })
+      class_name = normalize_migration_name(migration_name, fallback: default_migration_class_for(backend))
+      base_path = File.expand_path(target_dir)
+      FileUtils.mkdir_p(base_path)
+      templates = Kaal::Internal::ActiveRecord::MigrationTemplates.for_backend(backend)
+
+      templates.map.with_index do |(_name, contents), index|
+        suffix = underscore(class_name)
+        suffix = "#{suffix}_#{migration_suffixes_for(backend).fetch(index)}" if templates.length > 1
+        path = File.expand_path("#{(time_source.call + index).strftime('%Y%m%d%H%M%S')}_#{suffix}.rb", base_path)
+        File.write(path, contents)
+        path
+      end
+    end
+
+    def require_activerecord!
+      require 'active_record'
+      require 'active_support/inflector'
+    rescue LoadError => e
+      raise LoadError,
+            "#{e.message}. Add `gem 'activerecord'` to your Gemfile to use Active Record-backed Kaal SQL support.",
+            cause: e
+    end
+
+    def normalize_migration_name(name, fallback:)
+      normalized = name.to_s.each_char.with_object(+'') do |char, buffer|
+        if alphanumeric?(char)
+          buffer << char
+        elsif !buffer.empty? && !buffer.end_with?(' ')
+          buffer << ' '
+        end
+      end.split.map!(&:capitalize).join
+      normalized.empty? ? fallback : normalized
+    end
+
+    def underscore(value)
+      require_activerecord!
+      ::ActiveSupport::Inflector.underscore(value)
+    end
+
+    def default_migration_class_for(backend)
+      "CreateKaal#{backend.capitalize}Backend"
+    end
+
+    def migration_suffixes_for(backend)
+      return %w[dispatches locks definitions delayed_jobs] if backend.to_s == 'sqlite'
+
+      %w[dispatches definitions delayed_jobs]
+    end
+
+    def alphanumeric?(char)
+      char.between?('a', 'z') ||
+        char.between?('A', 'Z') ||
+        char.between?('0', '9')
+    end
+  end
+end

data/lib/kaal/backend/adapter.rb

@@ -94,6 +94,10 @@ module Kaal
       def definition_registry
         nil
       end
+
+      def delayed_store
+        nil
+      end
     end
 
     ##

data/lib/kaal/backend/memory_adapter.rb

@@ -6,6 +6,7 @@
 # LICENSE file in the root directory of this source tree.
 require_relative 'dispatch_logging'
 require_relative '../definition/memory_engine'
+require_relative '../delayed_job/memory_engine'
 
 module Kaal
   module Backend
@@ -50,6 +51,10 @@ module Kaal
         @definition_registry ||= Kaal::Definition::MemoryEngine.new
       end
 
+      def delayed_store
+        @delayed_store ||= Kaal::DelayedJob::MemoryEngine.new
+      end
+
       ##
       # Attempt to acquire a lock in memory.
       #

data/lib/kaal/backend/mysql.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+# Copyright Codevedas Inc. 2025-present
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+module Kaal
+  module Backend
+    # MySQL-backed backend for either Sequel or Active Record persistence.
+    class MySQL < Adapter
+      UNSET_SKIP_LOCKED_SUPPORT = Object.new.freeze
+
+      def initialize(database: nil, connection: nil, namespace: nil,
+                     use_skip_locked: UNSET_SKIP_LOCKED_SUPPORT)
+        super()
+        backend_class = self.class
+        @engine = if database
+                    Kaal::Sequel.require_sequel!
+                    require 'kaal/internal/sequel'
+                    backend_class.send(:build_sequel_backend, database, namespace, use_skip_locked)
+                  else
+                    Kaal::ActiveRecord.require_activerecord!
+                    require 'kaal/internal/active_record'
+                    backend_class.send(:build_active_record_backend, connection, namespace, use_skip_locked)
+                  end
+      end
+
+      def dispatch_registry
+        @engine.dispatch_registry
+      end
+
+      def definition_registry
+        @engine.definition_registry
+      end
+
+      def delayed_store
+        @engine.delayed_store
+      end
+
+      def acquire(key, ttl)
+        @engine.acquire(key, ttl)
+      end
+
+      def release(key)
+        @engine.release(key)
+      end
+
+      def self.build_sequel_backend(database, namespace, use_skip_locked)
+        return Kaal::Internal::Sequel::MySQLBackend.new(database, namespace:) if use_skip_locked.equal?(UNSET_SKIP_LOCKED_SUPPORT)
+
+        Kaal::Internal::Sequel::MySQLBackend.new(database, namespace:, use_skip_locked:)
+      end
+      private_class_method :build_sequel_backend
+
+      def self.build_active_record_backend(connection, namespace, use_skip_locked)
+        return Kaal::Internal::ActiveRecord::MySQLBackend.new(connection, namespace:) if use_skip_locked.equal?(UNSET_SKIP_LOCKED_SUPPORT)
+
+        Kaal::Internal::ActiveRecord::MySQLBackend.new(connection, namespace:, use_skip_locked:)
+      end
+      private_class_method :build_active_record_backend
+    end
+  end
+end

data/lib/kaal/backend/postgres.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+# Copyright Codevedas Inc. 2025-present
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+module Kaal
+  module Backend
+    # PostgreSQL-backed backend for either Sequel or Active Record persistence.
+    class Postgres < Adapter
+      def initialize(database: nil, connection: nil, namespace: nil)
+        super()
+        @engine = if database
+                    Kaal::Sequel.require_sequel!
+                    require 'kaal/internal/sequel'
+                    Kaal::Internal::Sequel::PostgresBackend.new(database, namespace:)
+                  else
+                    Kaal::ActiveRecord.require_activerecord!
+                    require 'kaal/internal/active_record'
+                    Kaal::Internal::ActiveRecord::PostgresBackend.new(connection, namespace:)
+                  end
+      end
+
+      def dispatch_registry
+        @engine.dispatch_registry
+      end
+
+      def definition_registry
+        @engine.definition_registry
+      end
+
+      def delayed_store
+        @engine.delayed_store
+      end
+
+      def acquire(key, ttl)
+        @engine.acquire(key, ttl)
+      end
+
+      def release(key)
+        @engine.release(key)
+      end
+    end
+  end
+end

data/lib/kaal/backend/redis_adapter.rb

@@ -7,6 +7,7 @@
 require 'securerandom'
 require_relative 'dispatch_logging'
 require_relative '../definition/redis_engine'
+require_relative '../delayed_job/redis_engine'
 
 module Kaal
   module Backend
@@ -65,6 +66,10 @@ module Kaal
         @definition_registry ||= Kaal::Definition::RedisEngine.new(@redis, namespace: @namespace)
       end
 
+      def delayed_store
+        @delayed_store ||= Kaal::DelayedJob::RedisEngine.new(@redis, namespace: @namespace)
+      end
+
       ##
       # Attempt to acquire a distributed lock in Redis.
       #

data/lib/kaal/backend/sqlite.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+# Copyright Codevedas Inc. 2025-present
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+module Kaal
+  module Backend
+    # SQLite-backed backend for either Sequel or Active Record persistence.
+    class SQLite < Adapter
+      def initialize(database: nil, connection: nil, namespace: nil, **)
+        super()
+        @engine = if database
+                    Kaal::Sequel.require_sequel!
+                    require 'kaal/internal/sequel'
+                    Kaal::Internal::Sequel::DatabaseBackend.new(database, namespace:)
+                  else
+                    Kaal::ActiveRecord.require_activerecord!
+                    require 'kaal/internal/active_record'
+                    Kaal::Internal::ActiveRecord::DatabaseBackend.new(connection, namespace:, **)
+                  end
+      end
+
+      def dispatch_registry
+        @engine.dispatch_registry
+      end
+
+      def definition_registry
+        @engine.definition_registry
+      end
+
+      def delayed_store
+        @engine.delayed_store
+      end
+
+      def acquire(key, ttl)
+        @engine.acquire(key, ttl)
+      end
+
+      def release(key)
+        @engine.release(key)
+      end
+    end
+  end
+end

data/lib/kaal/cli.rb

@@ -20,6 +20,7 @@ module Kaal
         Kaal.reset_configuration!
         Kaal.reset_registry!
         load config_path
+        Kaal.warn_on_risky_configuration!
         runtime_context = RuntimeContext.default(root_path: root_path)
         Kaal.load_scheduler_file!(runtime_context: runtime_context) if File.exist?(scheduler_path)
       end

data/lib/kaal/config/configuration.rb

@@ -31,7 +31,8 @@ module Kaal
       recovery_startup_jitter: 5, # max random delay in seconds
       scheduler_config_path: 'config/scheduler.yml',
       scheduler_conflict_policy: :error,
-      scheduler_missing_file_policy: :warn
+      scheduler_missing_file_policy: :warn,
+      delayed_job_allowed_class_prefixes: []
     }.freeze
 
     ##
@@ -69,6 +70,15 @@ module Kaal
       validation_errors
     end
 
+    # Non-fatal configuration warnings.
+    #
+    # @return [Array<String>] warning messages
+    def validation_warnings
+      warnings = []
+      add_delayed_job_security_warning(warnings)
+      warnings
+    end
+
     ##
     # Validate the configuration settings.
     # Raises errors if required settings are invalid.
@@ -79,6 +89,10 @@ module Kaal
       errors = validation_errors
       raise ConfigurationError, errors.join('; ') if errors.any?
 
+      validation_warnings.each do |warning|
+        @values[:logger]&.warn(warning)
+      end
+
       self
     end
 
@@ -105,7 +119,8 @@ module Kaal
         recovery_startup_jitter: @values[:recovery_startup_jitter],
         scheduler_config_path: @values[:scheduler_config_path],
         scheduler_conflict_policy: @values[:scheduler_conflict_policy],
-        scheduler_missing_file_policy: @values[:scheduler_missing_file_policy]
+        scheduler_missing_file_policy: @values[:scheduler_missing_file_policy],
+        delayed_job_allowed_class_prefixes: @values[:delayed_job_allowed_class_prefixes]
       }
     end
 
@@ -191,6 +206,13 @@ module Kaal
       errors << 'scheduler_missing_file_policy must be :warn or :error'
     end
 
+    def add_delayed_job_security_warning(warnings)
+      warning = Kaal::Config::DelayedJobSecurityPolicy.warning_for(self)
+      return unless warning
+
+      warnings << warning
+    end
+
     def handle_known_key(method_name)
       name = method_name.to_s
       setter = name.end_with?('=')
@@ -218,10 +240,19 @@ module Kaal
         value ? true : false
       when :scheduler_conflict_policy, :scheduler_missing_file_policy
         value&.to_sym
+      when :delayed_job_allowed_class_prefixes
+        normalize_delayed_job_allowed_class_prefixes(value)
       else
         value
       end
     end
+
+    def normalize_delayed_job_allowed_class_prefixes(value)
+      Array(value).filter_map do |entry|
+        normalized_entry = entry.to_s.strip
+        normalized_entry unless normalized_entry.empty?
+      end
+    end
   end
 
   ##

data/lib/kaal/config/delayed_job_security_policy.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+# Copyright Codevedas Inc. 2025-present
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+module Kaal
+  module Config
+    # Evaluates whether delayed-job class resolution is too open for the
+    # current deployment shape and returns the matching warning message.
+    module DelayedJobSecurityPolicy
+      NON_SHARED_BACKEND_CLASS_NAMES = ['NilClass', 'Kaal::Backend::MemoryAdapter', 'Kaal::Backend::NullAdapter'].freeze
+      WARNING_MESSAGE = 'Delayed jobs resolve stored job_class values at dispatch time. ' \
+                        'delayed_job_allowed_class_prefixes is empty, so class resolution is unrestricted on this shared backend. ' \
+                        'Configure a restrictive delayed_job_allowed_class_prefixes list for production deployments.'
+
+      module_function
+
+      def warning_for(configuration)
+        return unless production_like_environment?
+        return unless shared_delayed_job_backend?(configuration.backend)
+        return unless Array(configuration.delayed_job_allowed_class_prefixes).empty?
+
+        WARNING_MESSAGE
+      end
+
+      def production_like_environment?(env: ENV, rails: current_rails)
+        rails_env = rails_environment(rails)
+        return rails_env.production? if rails_env
+
+        %w[RACK_ENV HANAMI_ENV APP_ENV RAILS_ENV RUBY_ENV].any? do |key|
+          env.fetch(key, nil).to_s.strip == 'production'
+        end
+      rescue StandardError
+        false
+      end
+
+      def shared_delayed_job_backend?(backend)
+        backend_class = backend.class
+        return false if NON_SHARED_BACKEND_CLASS_NAMES.include?(backend_class.name)
+
+        backend_class.instance_method(:delayed_store).owner.name != 'Kaal::Backend::Adapter'
+      rescue StandardError
+        false
+      end
+
+      def current_rails
+        return unless defined?(::Rails)
+
+        ::Rails
+      end
+
+      def rails_environment(rails)
+        rails.env
+      rescue StandardError
+        nil
+      end
+    end
+  end
+end

data/lib/kaal/config.rb

@@ -5,6 +5,7 @@
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory of this source tree.
 require 'kaal/config/configuration'
+require 'kaal/config/delayed_job_security_policy'
 require 'kaal/config/scheduler_config_error'
 require 'kaal/config/scheduler_time_zone_resolver'
 

data/lib/kaal/core/coordinator.rb

@@ -16,21 +16,12 @@ module Kaal
   #
   # The coordinator:
   # 1. Runs a background thread on tick_interval
-  # 2. For each registered cron, calculates due fire times within the window
-  # 3. Attempts to acquire a distributed lease for each due time
-  # 4. Calls the enqueue callback if the lease is acquired
-  # 5. Supports graceful shutdown and re-entrancy for testing
-  #
-  # @example Start the coordinator
-  #   coordinator = Kaal::Coordinator.new
-  #   coordinator.start!
-  #
-  # @example Manual tick execution (for testing)
-  #   coordinator.tick!
-  #
-  # @example Stop the coordinator
-  #   coordinator.stop!
+  # 2. Calculates due cron fire times and acquires distributed leases for them
+  # 3. Dispatches claimed work and supports graceful shutdown and test re-entrancy
   class Coordinator
+    DELAYED_JOB_BATCH_SIZE = 100
+    DELAYED_JOB_MAX_BATCHES_PER_TICK = 10
+    DELAYED_JOB_DELETE_CONFIRMATION_JITTER_MAX = 0.05
     ##
     # Initialize a new Coordinator instance.
     #
@@ -174,6 +165,7 @@ module Kaal
       each_enabled_entry do |entry|
         calculate_and_dispatch_due_times(entry)
       end
+      dispatch_due_delayed_jobs
     rescue ConfigurationError => e
       log_configuration_error('Kaal coordinator tick failed', e)
       raise
@@ -377,15 +369,72 @@ module Kaal
       logger&.error("Work dispatch failed for #{cron_key}: #{e.message}")
     end
 
-    def generate_idempotency_key(cron_key, fire_time)
-      Kaal::IdempotencyKeyGenerator.call(cron_key, fire_time, configuration: @configuration)
+    def dispatch_due_delayed_jobs
+      delayed_store = delayed_store_for_tick
+      return unless delayed_store
+
+      DELAYED_JOB_MAX_BATCHES_PER_TICK.times do
+        break if stop_delayed_dispatch?
+
+        apply_delayed_job_claim_jitter_if_needed(delayed_store)
+        due_jobs = delayed_store.pop_due(now: Time.now.utc, limit: DELAYED_JOB_BATCH_SIZE)
+        break if due_jobs.empty?
+
+        due_jobs.each do |job|
+          break if stop_delayed_dispatch?
+
+          dispatch_delayed_job(job, delayed_store)
+        end
+      end
+    rescue StandardError => e
+      @configuration.logger&.error("Delayed job dispatch failed: #{e.message}")
+    end
+
+    def dispatch_delayed_job(job, delayed_store)
+      if delayed_store.requires_dispatch_lock?
+        lock_key = generate_delayed_lock_key(job.fetch(:job_id))
+        return unless acquire_lock(lock_key)
+      end
+
+      job_class = Kaal::JobDispatcher.resolve_job_class(
+        job_class_name: job.fetch(:job_class),
+        key: job.fetch(:job_id),
+        queue: job[:queue]
+      )
+      Kaal::JobDispatcher.dispatch(job_class:, queue: job[:queue], args: job.fetch(:args))
+      @configuration.logger&.debug("Dispatched delayed job #{job.fetch(:job_id)} for #{job.fetch(:run_at)}")
+      true
+    rescue StandardError => e
+      Kaal::DelayedJob::DispatchFailureLogger.log_claimed_dispatch_failure(
+        logger: @configuration.logger,
+        job:,
+        error: e
+      )
+      nil
+    end
+
+    def delayed_store_for_tick
+      backend = @configuration.backend
+      backend.respond_to?(:delayed_store) ? backend.delayed_store : nil
     end
 
-    def generate_lock_key(cron_key, fire_time)
-      namespace = @configuration.namespace || 'kaal'
-      "#{namespace}:dispatch:#{cron_key}:#{fire_time.to_i}"
+    def stop_delayed_dispatch?
+      stop_requested?
     end
 
+    def apply_delayed_job_claim_jitter_if_needed(delayed_store)
+      return unless delayed_store.claim_strategy == :delete_confirmation
+
+      jitter = rand * DELAYED_JOB_DELETE_CONFIRMATION_JITTER_MAX
+      sleep(jitter) if jitter.positive?
+    end
+
+    def generate_idempotency_key(cron_key, fire_time) = Kaal::IdempotencyKeyGenerator.call(cron_key, fire_time, configuration: @configuration)
+
+    def generate_lock_key(cron_key, fire_time) = "#{@configuration.namespace || 'kaal'}:dispatch:#{cron_key}:#{fire_time.to_i}"
+
+    def generate_delayed_lock_key(job_id) = "#{@configuration.namespace || 'kaal'}:delayed_dispatch:#{job_id}"
+
     def sleep_until_next_tick
       @mutex.synchronize do
         @tick_cv.wait(@mutex, @configuration.tick_interval)