kaal 0.2.0 → 0.3.0

data/lib/kaal/scheduler_file/job_applier.rb

@@ -0,0 +1,242 @@
+# frozen_string_literal: true
+
+# Copyright Codevedas Inc. 2025-present
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+require 'kaal/support/hash_tools'
+
+module Kaal
+  class SchedulerFileLoader
+    # Applies normalized scheduler jobs and rolls them back on failure.
+    class JobApplier
+      include Kaal::Support::HashTools
+
+      def initialize(configuration:, definition_registry:, registry:, logger:, helper_bundle:)
+        @configuration = configuration
+        @definition_registry = definition_registry
+        @registry = registry
+        @logger = logger
+        @helper_bundle = helper_bundle
+      end
+
+      def apply(job)
+        key = job.fetch(:key)
+        cron = job.fetch(:cron)
+        job_class_name = job.fetch(:job_class_name)
+        queue = job.fetch(:queue)
+        existing_definition = @definition_registry.find_definition(key)
+        existing_registry_entry = @registry.find(key)
+        return nil if conflict?(key:, existing_definition:)
+
+        job_class = resolved_job_class(job_class_name:, key:, queue:)
+        callback = callback_for(
+          key: key,
+          job_class_name: job_class_name,
+          queue: queue,
+          args_template: job.fetch(:args),
+          kwargs_template: job.fetch(:kwargs)
+        )
+        persisted_metadata = persisted_metadata(job, job_class)
+
+        @definition_registry.upsert_definition(
+          key: key,
+          cron: cron,
+          enabled: job.fetch(:enabled),
+          source: 'file',
+          metadata: persisted_metadata
+        )
+
+        begin
+          @registry.upsert(key: key, cron: cron, enqueue: callback)
+        rescue StandardError
+          rollback_job(key:, existing_definition:, existing_registry_entry:)
+          raise
+        end
+
+        { key: key, existing_definition: existing_definition, existing_registry_entry: existing_registry_entry }
+      end
+
+      def rollback_jobs(applied_job_contexts)
+        applied_job_contexts.reverse_each do |applied_job_context|
+          rollback_job(**applied_job_context)
+        end
+      end
+
+      def callback_for(key:, job_class_name:, queue:, args_template:, kwargs_template:)
+        job_class = resolved_job_class(job_class_name:, key:, queue:)
+        build_callback(
+          {
+            key: key,
+            queue: queue,
+            args: args_template,
+            kwargs: kwargs_template
+          },
+          job_class
+        )
+      end
+
+      def resolved_job_class(job_class_name:, key:, queue: nil)
+        resolve_job_class(job_class_name:, key:, queue:)
+      end
+
+      def conflict?(key:, existing_definition:)
+        existing_source = existing_definition&.[](:source)
+        return false unless existing_source && existing_source.to_s != 'file'
+
+        policy = @configuration.scheduler_conflict_policy
+        case policy
+        when :error
+          raise SchedulerConfigError, "Scheduler key conflict for '#{key}' with existing source '#{existing_source}'"
+        when :code_wins
+          @logger&.warn("Skipping scheduler file job '#{key}' because scheduler_conflict_policy is :code_wins")
+          true
+        when :file_wins
+          false
+        else
+          raise SchedulerConfigError, "Unsupported scheduler_conflict_policy '#{policy}'"
+        end
+      end
+
+      def rollback_job(key:, existing_definition:, existing_registry_entry:)
+        if existing_definition
+          @definition_registry.upsert_definition(
+            **Definition::AttributeHelpers.definition_attributes(existing_definition), enabled: existing_definition[:enabled]
+          )
+        else
+          @definition_registry.remove_definition(key)
+        end
+
+        @registry.remove(key) if @registry.registered?(key)
+
+        return unless existing_registry_entry
+
+        @registry.upsert(
+          key: existing_registry_entry.key,
+          cron: existing_registry_entry.cron,
+          enqueue: existing_registry_entry.enqueue
+        )
+      rescue StandardError => e
+        @logger&.error("Failed to rollback scheduler file application for #{key}: #{e.message}")
+      end
+
+      private
+
+      def persisted_metadata(job, job_class)
+        metadata, job_class_name, queue, args, kwargs =
+          job.values_at(:metadata, :job_class_name, :queue, :args, :kwargs)
+        normalized_metadata = @helper_bundle.stringify_keys(deep_dup(metadata || {}))
+        Kaal::Support::HashTools.deep_merge(
+          normalized_metadata,
+          'execution' => {
+            'target' => active_job_dispatch?(job_class, queue) ? 'active_job' : 'ruby',
+            'job_class' => job_class_name,
+            'queue' => queue,
+            'args' => args,
+            'kwargs' => kwargs
+          }
+        )
+      end
+
+      def build_callback(job, job_class)
+        key = job.fetch(:key)
+        queue = job.fetch(:queue)
+        args_template = job.fetch(:args)
+        kwargs_template = job.fetch(:kwargs)
+
+        lambda do |fire_time:, idempotency_key:|
+          context = {
+            fire_time: fire_time,
+            idempotency_key: idempotency_key,
+            key: key
+          }
+          resolved_args = @helper_bundle.resolve_placeholders(deep_dup(args_template), context)
+          raw_kwargs = @helper_bundle.resolve_placeholders(deep_dup(kwargs_template), context) || {}
+          raise SchedulerConfigError, "kwargs for scheduler job '#{key}' must be a mapping, got #{raw_kwargs.class}" unless raw_kwargs.is_a?(Hash)
+
+          validate_keyword_keys(raw_kwargs, key)
+
+          resolved_kwargs = raw_kwargs.transform_keys(&:to_sym)
+          dispatch_job(job_class, queue, resolved_args, resolved_kwargs)
+        end
+      end
+
+      def validate_keyword_keys(raw_kwargs, key)
+        keys = raw_kwargs.keys
+        index = 0
+        while index < keys.length
+          kwargs_key = keys[index]
+          if kwargs_key.is_a?(String) || kwargs_key.is_a?(Symbol)
+            index += 1
+            next
+          end
+
+          raise SchedulerConfigError,
+                "Invalid keyword argument key #{kwargs_key.inspect} (#{kwargs_key.class}) for scheduler job '#{key}'"
+        end
+
+        nil
+      end
+
+      def resolve_job_class(job_class_name:, key:, queue: nil)
+        normalized_job_class_name = job_class_name.to_s.strip
+        raise SchedulerConfigError, "Job class cannot be blank for key '#{key}'" if normalized_job_class_name.empty?
+
+        error_message = "Unknown job_class #{normalized_job_class_name.inspect} for key '#{key}'"
+        job_class = begin
+          Kaal::Support::HashTools.constantize(normalized_job_class_name)
+        rescue NameError
+          nil
+        end
+
+        return validate_dispatch_interface(job_class, key, queue) if job_class
+
+        raise_unknown_job_class(error_message)
+      end
+
+      private :build_callback, :resolve_job_class
+
+      def dispatch_job(job_class, queue, args, kwargs)
+        job_class_name = job_class.name
+
+        if queue && !job_class.respond_to?(:set)
+          raise SchedulerConfigError,
+                "job_class '#{job_class_name}' must respond to .set to use queue #{queue.inspect}"
+        end
+
+        if queue
+          job_class.set(queue: queue).perform_later(*args, **kwargs)
+        elsif job_class.respond_to?(:perform_later)
+          job_class.perform_later(*args, **kwargs)
+        elsif job_class.respond_to?(:perform)
+          job_class.perform(*args, **kwargs)
+        else
+          raise SchedulerConfigError,
+                "job_class '#{job_class_name}' must respond to .perform, .perform_later, or .set(...).perform_later"
+        end
+      end
+
+      def raise_unknown_job_class(error_message)
+        raise SchedulerConfigError, error_message
+      end
+
+      def validate_dispatch_interface(job_class, key, queue)
+        queue_present = !queue.nil?
+        supports_set = job_class.respond_to?(:set)
+        supports_perform_later = job_class.respond_to?(:perform_later)
+        supports_perform = job_class.respond_to?(:perform)
+
+        return job_class if queue_present && supports_set
+        return job_class if !queue_present && supports_perform_later
+        return job_class if !queue_present && supports_perform
+
+        raise SchedulerConfigError,
+              "job_class '#{job_class.name}' for key '#{key}' must respond to .perform, .perform_later, or .set(...).perform_later"
+      end
+
+      def active_job_dispatch?(job_class, queue)
+        (queue && job_class.respond_to?(:set)) || job_class.respond_to?(:perform_later)
+      end
+    end
+  end
+end

data/lib/kaal/scheduler_file/job_normalizer.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+# Copyright Codevedas Inc. 2025-present
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+require 'kaal/support/hash_tools'
+
+module Kaal
+  class SchedulerFileLoader
+    # Normalizes scheduler job payloads into application-ready hashes.
+    class JobNormalizer
+      include Kaal::Support::HashTools
+
+      def initialize(hash_transform:, placeholder_support:, cron_validator:)
+        @hash_transform = hash_transform
+        @placeholder_support = placeholder_support
+        @cron_validator = cron_validator
+      end
+
+      def call(job_payload)
+        payload = @hash_transform.stringify_keys(job_payload)
+        key = payload.fetch('key', '').to_s.strip
+        raise SchedulerConfigError, 'Job key cannot be blank' if key.empty?
+
+        cron = required_string(payload, field: 'cron', error_prefix: "Job cron cannot be blank for key '#{key}'")
+        job_class_name = required_string(payload, field: 'job_class', error_prefix: "Job class cannot be blank for key '#{key}'")
+        validate_cron(key:, cron:)
+        options = extract_job_options(payload, key:)
+
+        {
+          key: key,
+          cron: cron,
+          job_class_name: job_class_name,
+          **options
+        }
+      end
+
+      private
+
+      def required_string(payload, field:, error_prefix:)
+        value = payload.fetch(field, '').to_s.strip
+        raise SchedulerConfigError, error_prefix if value.empty?
+
+        value
+      end
+
+      def validate_cron(key:, cron:)
+        return if @cron_validator.call(cron)
+
+        raise SchedulerConfigError, "Invalid cron expression '#{cron}' for key '#{key}'"
+      end
+
+      def extract_job_options(payload, key:)
+        metadata, args, kwargs, queue, enabled_value = payload.values_at('metadata', 'args', 'kwargs', 'queue', 'enabled')
+        args ||= []
+        kwargs ||= {}
+        enabled = true
+        if payload.key?('enabled')
+          raise SchedulerConfigError, "enabled must be a boolean for key '#{key}'" unless enabled_value.is_a?(TrueClass) || enabled_value.is_a?(FalseClass)
+
+          enabled = enabled_value
+        end
+
+        raise SchedulerConfigError, "metadata must be a mapping for key '#{key}'" if metadata && !metadata.is_a?(Hash)
+
+        validate_job_option_types(key:, args:, kwargs:, queue:)
+        @placeholder_support.validate_placeholders(args, key:)
+        @placeholder_support.validate_placeholders(kwargs, key:)
+
+        {
+          queue: queue,
+          args: deep_dup(args),
+          kwargs: deep_dup(kwargs),
+          enabled: enabled,
+          metadata: metadata ? deep_dup(metadata) : {}
+        }
+      end
+
+      def validate_job_option_types(key:, args:, kwargs:, queue:)
+        raise SchedulerConfigError, "args must be an array for key '#{key}'" unless args.is_a?(Array)
+        raise SchedulerConfigError, "kwargs must be a mapping for key '#{key}'" unless kwargs.is_a?(Hash)
+        raise SchedulerConfigError, "queue must be a string for key '#{key}'" if queue && !queue.is_a?(String)
+        return if kwargs.keys.all? { |kwargs_key| kwargs_key.is_a?(String) || kwargs_key.is_a?(Symbol) }
+
+        raise SchedulerConfigError, "kwargs keys must be strings or symbols for key '#{key}'"
+      end
+    end
+  end
+end

data/lib/kaal/scheduler_file/loader.rb

@@ -0,0 +1,152 @@
+# frozen_string_literal: true
+
+# Copyright Codevedas Inc. 2025-present
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+require 'kaal/runtime/runtime_context'
+require 'kaal/scheduler_file/hash_transform'
+require 'kaal/scheduler_file/placeholder_support'
+require 'kaal/support/hash_tools'
+require_relative 'helper_bundle'
+require_relative 'payload_loader'
+require_relative 'job_normalizer'
+require_relative 'job_applier'
+
+module Kaal
+  # Loads scheduler definitions from config/scheduler.yml and registers them.
+  class SchedulerFileLoader
+    include SchedulerHashTransform
+    include SchedulerPlaceholderSupport
+    include Kaal::Support::HashTools
+
+    PLACEHOLDER_PATTERN = /\{\{\s*([a-zA-Z0-9_.]+)\s*\}\}/
+    ALLOWED_PLACEHOLDERS = {
+      'fire_time.iso8601' => ->(ctx) { ctx.fetch(:fire_time).iso8601 },
+      'fire_time.unix' => ->(ctx) { ctx.fetch(:fire_time).to_i },
+      'idempotency_key' => ->(ctx) { ctx.fetch(:idempotency_key) },
+      'key' => ->(ctx) { ctx.fetch(:key) }
+    }.freeze
+
+    def initialize(
+      configuration:,
+      definition_registry:,
+      registry:,
+      logger:,
+      runtime_context: RuntimeContext.default
+    )
+      @configuration = configuration
+      @definition_registry = definition_registry
+      @registry = registry
+      @logger = logger
+      @runtime_context = runtime_context
+      @placeholder_resolvers = ALLOWED_PLACEHOLDERS
+    end
+
+    def load
+      applied_job_contexts = []
+      path, payload = payload_loader.load
+      return handle_missing_file(path) unless payload
+
+      jobs = extract_jobs(payload)
+      validate_unique_keys(jobs)
+      normalized_jobs = jobs.map { |job_payload| normalize_job(job_payload) }
+      applied_jobs = []
+      normalized_jobs.each do |job|
+        applied_job_context = apply_job(job)
+        next unless applied_job_context
+
+        applied_jobs << job
+        applied_job_contexts << applied_job_context
+      end
+
+      applied_jobs
+    rescue StandardError
+      rollback_applied_jobs(applied_job_contexts)
+      raise
+    end
+
+    private
+
+    def handle_missing_file(path)
+      payload_loader.handle_missing_file(path)
+    end
+
+    def extract_jobs(payload)
+      payload_loader.extract_jobs(payload)
+    end
+
+    def validate_unique_keys(jobs)
+      payload_loader.validate_unique_keys(jobs)
+    end
+
+    def normalize_job(job_payload)
+      job_normalizer.call(job_payload)
+    end
+
+    def extract_job_options(payload, key:)
+      job_normalizer.send(:extract_job_options, payload, key:)
+    end
+
+    def apply_job(job)
+      job_applier.apply(job)
+    end
+
+    def rollback_applied_jobs(applied_job_contexts = [])
+      job_applier.rollback_jobs(applied_job_contexts)
+    end
+
+    def rollback_applied_job(key:, existing_definition:, existing_registry_entry:)
+      job_applier.rollback_job(key:, existing_definition:, existing_registry_entry:)
+    end
+
+    def skip_due_to_conflict?(key:, existing_definition:)
+      job_applier.conflict?(key:, existing_definition:)
+    end
+
+    def build_callback(key:, job_class_name:, queue:, args_template:, kwargs_template:)
+      job_applier.callback_for(
+        key: key,
+        job_class_name: job_class_name,
+        queue: queue,
+        args_template: args_template,
+        kwargs_template: kwargs_template
+      )
+    end
+
+    def resolve_job_class(job_class_name:, key:, queue: nil)
+      job_applier.resolved_job_class(job_class_name:, key:, queue:)
+    end
+
+    def payload_loader
+      @payload_loader ||= PayloadLoader.new(
+        configuration: @configuration,
+        runtime_context: @runtime_context,
+        logger: @logger,
+        hash_transform: helper_bundle
+      )
+    end
+
+    def job_normalizer
+      @job_normalizer ||= JobNormalizer.new(
+        hash_transform: helper_bundle,
+        placeholder_support: helper_bundle,
+        cron_validator: ->(cron) { Kaal.valid?(cron) }
+      )
+    end
+
+    def job_applier
+      @job_applier ||= JobApplier.new(
+        configuration: @configuration,
+        definition_registry: @definition_registry,
+        registry: @registry,
+        logger: @logger,
+        helper_bundle: helper_bundle
+      )
+    end
+
+    def helper_bundle
+      @helper_bundle ||= HelperBundle.new(loader: self)
+    end
+  end
+end

data/lib/kaal/scheduler_file/payload_loader.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+# Copyright Codevedas Inc. 2025-present
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+require 'erb'
+require 'yaml'
+
+module Kaal
+  class SchedulerFileLoader
+    # Loads and validates scheduler YAML payloads from disk.
+    class PayloadLoader
+      def initialize(configuration:, runtime_context:, logger:, hash_transform:)
+        @configuration = configuration
+        @runtime_context = runtime_context
+        @logger = logger
+        @hash_transform = hash_transform
+      end
+
+      def load
+        path = scheduler_file_path
+        return [path, nil] unless File.exist?(path)
+
+        [path, parse_yaml(path)]
+      end
+
+      def handle_missing_file(path)
+        message = "Scheduler file not found at #{path}"
+        raise SchedulerConfigError, message if @configuration.scheduler_missing_file_policy == :error
+
+        @logger&.warn(message)
+        []
+      end
+
+      def extract_jobs(payload)
+        environment_name = @runtime_context.environment_name
+        defaults = fetch_hash(payload, 'defaults')
+        env_payload = fetch_hash(payload, environment_name)
+        default_jobs = defaults.fetch('jobs', [])
+        env_jobs = env_payload.fetch('jobs', [])
+        raise SchedulerConfigError, "Expected 'defaults.jobs' to be an array" unless default_jobs.is_a?(Array)
+        raise SchedulerConfigError, "Expected '#{environment_name}.jobs' to be an array" unless env_jobs.is_a?(Array)
+
+        default_jobs + env_jobs
+      end
+
+      def validate_unique_keys(jobs)
+        keys = jobs.map do |job_payload|
+          raise SchedulerConfigError, "Each jobs entry must be a mapping, got #{job_payload.class}" unless job_payload.is_a?(Hash)
+
+          @hash_transform.stringify_keys(job_payload)['key'].to_s.strip
+        end
+        duplicates = keys.group_by(&:itself).select { |key, arr| !key.empty? && arr.size > 1 }.keys
+        return if duplicates.empty?
+
+        raise SchedulerConfigError, "Duplicate job keys in scheduler file: #{duplicates.join(', ')}"
+      end
+
+      private
+
+      def scheduler_file_path
+        configured_path = @configuration.scheduler_config_path.to_s.strip
+        raise SchedulerConfigError, 'scheduler_config_path cannot be blank' if configured_path.empty?
+
+        @runtime_context.resolve_path(configured_path)
+      end
+
+      def parse_yaml(path)
+        rendered = render_yaml_erb(path)
+        parsed = YAML.safe_load(rendered) || {}
+        raise SchedulerConfigError, "Expected scheduler YAML root to be a mapping in #{path}" unless parsed.is_a?(Hash)
+
+        @hash_transform.stringify_keys(parsed)
+      rescue Psych::Exception => e
+        raise SchedulerConfigError, "Failed to parse scheduler YAML at #{path}: #{e.message}"
+      end
+
+      def render_yaml_erb(path)
+        ERB.new(File.read(path), trim_mode: '-').result
+      rescue StandardError, SyntaxError => e
+        raise SchedulerConfigError, "Failed to evaluate scheduler ERB at #{path}: #{e.message}"
+      end
+
+      def fetch_hash(payload, key)
+        section = payload.fetch(key)
+        raise SchedulerConfigError, "Expected '#{key}' section to be a mapping" unless section.is_a?(Hash)
+
+        section
+      rescue KeyError
+        {}
+      end
+    end
+  end
+end

data/lib/kaal/{scheduler_placeholder_support.rb → scheduler_file/placeholder_support.rb}

@@ -1,5 +1,9 @@
 # frozen_string_literal: true
 
+# Copyright Codevedas Inc. 2025-present
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
 module Kaal
   # Placeholder parsing/resolution for scheduler args and kwargs.
   module SchedulerPlaceholderSupport

data/lib/kaal/scheduler_file.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+# Copyright Codevedas Inc. 2025-present
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+require 'kaal/scheduler_file/loader'
+require 'kaal/scheduler_file/hash_transform'
+require 'kaal/scheduler_file/placeholder_support'
+
+module Kaal
+  # Scheduler file loading and payload helpers.
+  module SchedulerFile
+    Loader = ::Kaal::SchedulerFileLoader
+    HashTransform = ::Kaal::SchedulerHashTransform
+    PlaceholderSupport = ::Kaal::SchedulerPlaceholderSupport
+  end
+end

data/lib/kaal/support/hash_tools.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+# Copyright Codevedas Inc. 2025-present
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+module Kaal
+  module Support
+    # Small deep-copy and key-normalization helpers used across config and scheduler loading.
+    module HashTools
+      module_function
+
+      def deep_dup(value)
+        case value
+        when Hash
+          value.each_with_object({}) do |(key, child), memo|
+            duplicated_pair = [deep_dup(key), deep_dup(child)]
+            memo[duplicated_pair[0]] = duplicated_pair[1]
+          end
+        when Array
+          value.map { |child| duplicate_child(child) }
+        else
+          duplicable?(value) ? value.dup : value
+        end
+      end
+
+      def stringify_keys(value)
+        transform_keys(value, &:to_s)
+      end
+
+      def symbolize_keys(value)
+        transform_keys(value) { |key| key.to_s.to_sym }
+      end
+
+      def deep_merge(left, right)
+        left.merge(right) do |_key, left_value, right_value|
+          if left_value.is_a?(Hash) && right_value.is_a?(Hash)
+            deep_merge(left_value, right_value)
+          else
+            deep_dup(right_value)
+          end
+        end
+      end
+
+      def constantize(name)
+        name.to_s.split('::').reject(&:empty?).reduce(Object) { |scope, part| scope.const_get(part) }
+      end
+
+      def duplicable?(value)
+        !value.is_a?(NilClass) &&
+          !value.is_a?(FalseClass) &&
+          !value.is_a?(TrueClass) &&
+          !value.is_a?(Symbol) &&
+          !value.is_a?(Numeric) &&
+          !value.is_a?(Method) &&
+          !value.is_a?(Proc)
+      end
+
+      def transform_keys(value, &)
+        case value
+        when Hash
+          transform_hash_keys(value, &)
+        when Array
+          transform_array_keys(value, &)
+        else
+          value
+        end
+      end
+
+      def duplicate_child(child)
+        deep_dup(child)
+      end
+
+      def transform_child_keys(child, &)
+        transform_keys(child, &)
+      end
+
+      def transform_hash_keys(value, &)
+        value.each_with_object({}) do |(key, child), memo|
+          transformed_pair = [yield(key), transform_child_keys(child, &)]
+          memo[transformed_pair[0]] = transformed_pair[1]
+        end
+      end
+
+      def transform_array_keys(value, &)
+        value.map { |child| transform_child_keys(child, &) }
+      end
+
+      private_class_method :duplicate_child, :transform_child_keys, :transform_hash_keys, :transform_array_keys
+      private_class_method :transform_keys
+    end
+  end
+end