job-workflow 0.1.3

data/lib/job_workflow/queue_adapters/abstract.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module JobWorkflow
+  module QueueAdapters
+    # rubocop:disable Naming/PredicateMethod
+    class Abstract
+      #:  () -> void
+      def initialize_adapter!; end
+
+      #:  () -> bool
+      def semaphore_available?
+        raise NotImplementedError, "#{self.class}#semaphore_available? must be implemented"
+      end
+
+      #:  (Semaphore) -> bool
+      def semaphore_wait(semaphore)
+        raise NotImplementedError, "#{self.class}#semaphore_wait must be implemented"
+      end
+
+      #:  (Semaphore) -> bool
+      def semaphore_signal(semaphore)
+        raise NotImplementedError, "#{self.class}#semaphore_signal must be implemented"
+      end
+
+      #:  (Array[String]) -> Hash[String, untyped]
+      def fetch_job_statuses(job_ids)
+        raise NotImplementedError, "#{self.class}#fetch_job_statuses must be implemented"
+      end
+
+      #:  (untyped) -> Symbol
+      def job_status(job)
+        raise NotImplementedError, "#{self.class}#job_status must be implemented"
+      end
+
+      #:  () -> bool
+      def supports_concurrency_limits?
+        raise NotImplementedError, "#{self.class}#supports_concurrency_limits? must be implemented"
+      end
+
+      #:  (String) -> bool
+      def pause_queue(_queue_name)
+        raise NotImplementedError, "#{self.class}#pause_queue must be implemented"
+      end
+
+      #:  (String) -> bool
+      def resume_queue(_queue_name)
+        raise NotImplementedError, "#{self.class}#resume_queue must be implemented"
+      end
+
+      #:  (String) -> bool
+      def queue_paused?(_queue_name)
+        raise NotImplementedError, "#{self.class}#queue_paused? must be implemented"
+      end
+
+      #:  () -> Array[String]
+      def paused_queues
+        raise NotImplementedError, "#{self.class}#paused_queues must be implemented"
+      end
+
+      #:  (String) -> Integer?
+      def queue_latency(_queue_name)
+        raise NotImplementedError, "#{self.class}#queue_latency must be implemented"
+      end
+
+      #:  (String) -> Integer
+      def queue_size(_queue_name)
+        raise NotImplementedError, "#{self.class}#queue_size must be implemented"
+      end
+
+      #:  (String) -> bool
+      def clear_queue(_queue_name)
+        raise NotImplementedError, "#{self.class}#clear_queue must be implemented"
+      end
+
+      #:  (String) -> Hash[String, untyped]?
+      def find_job(_job_id)
+        raise NotImplementedError, "#{self.class}#find_job must be implemented"
+      end
+
+      #:  (DSL, Numeric) -> bool
+      def reschedule_job(_job, _wait)
+        false
+      end
+    end
+    # rubocop:enable Naming/PredicateMethod
+  end
+end

data/lib/job_workflow/queue_adapters/null_adapter.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+module JobWorkflow
+  module QueueAdapters
+    # rubocop:disable Naming/PredicateMethod
+    class NullAdapter < Abstract
+      #:  () -> void
+      def initialize_adapter!; end
+
+      def initialize # rubocop:disable Lint/MissingSuper
+        @paused_queues = Set.new #: Set[String]
+        @queue_jobs = {} #: Hash[String, Array[untyped]]
+        @stored_jobs = {} #: Hash[String, Hash[String, untyped]]
+      end
+
+      #:  () -> bool
+      def semaphore_available?
+        false
+      end
+
+      #:  (Semaphore) -> bool
+      def semaphore_wait(_semaphore)
+        true
+      end
+
+      #:  (Semaphore) -> bool
+      def semaphore_signal(_semaphore)
+        true
+      end
+
+      #:  (Array[String]) -> Hash[String, untyped]
+      def fetch_job_statuses(_job_ids)
+        {}
+      end
+
+      #:  (untyped) -> Symbol
+      def job_status(_job)
+        :pending
+      end
+
+      #:  () -> bool
+      def supports_concurrency_limits?
+        false
+      end
+
+      #:  (String) -> bool
+      def pause_queue(queue_name)
+        @paused_queues.add(queue_name)
+        true
+      end
+
+      #:  (String) -> bool
+      def resume_queue(queue_name)
+        @paused_queues.delete(queue_name)
+        true
+      end
+
+      #:  (String) -> bool
+      def queue_paused?(queue_name)
+        @paused_queues.include?(queue_name)
+      end
+
+      #:  () -> Array[String]
+      def paused_queues
+        @paused_queues.to_a
+      end
+
+      #:  (String) -> Integer?
+      def queue_latency(_queue_name)
+        nil
+      end
+
+      #:  (String) -> Integer
+      def queue_size(queue_name)
+        jobs = @queue_jobs[queue_name]
+        return 0 if jobs.nil?
+
+        jobs.size
+      end
+
+      #:  (String) -> bool
+      def clear_queue(queue_name)
+        empty_jobs = [] #: Array[untyped]
+        @queue_jobs[queue_name] = empty_jobs
+        true
+      end
+
+      #:  (String) -> Hash[String, untyped]?
+      def find_job(job_id)
+        @stored_jobs[job_id]
+      end
+
+      #:  (DSL, Numeric) -> bool
+      def reschedule_job(_job, _wait)
+        false
+      end
+
+      # @note Test helpers
+      #
+      #:  (String, untyped) -> void
+      def enqueue_test_job(queue_name, job)
+        unless @queue_jobs.key?(queue_name)
+          empty_jobs = [] #: Array[untyped]
+          @queue_jobs[queue_name] = empty_jobs
+        end
+        @queue_jobs[queue_name] << job
+      end
+
+      # @note Test helpers - stores job data for find_job
+      #
+      #:  (String, Hash[String, untyped]) -> void
+      def store_job(job_id, job_data)
+        @stored_jobs[job_id] = job_data
+      end
+
+      # @note Test helpers
+      #
+      #:  () -> void
+      def reset!
+        @paused_queues.clear
+        @queue_jobs.clear
+        @stored_jobs.clear
+      end
+    end
+    # rubocop:enable Naming/PredicateMethod
+  end
+end

data/lib/job_workflow/queue_adapters/solid_queue_adapter.rb

@@ -0,0 +1,224 @@
+# frozen_string_literal: true
+
+module JobWorkflow
+  module QueueAdapters
+    # rubocop:disable Naming/PredicateMethod
+    class SolidQueueAdapter < Abstract
+      # @note
+      #   - Registry scope: @semaphore_registry is process-scoped (shared across fibers/threads
+      #     in the same process) and lives for the lifetime of the worker process. It is not
+      #     serialized to persistent storage; semaphores are transient per worker instance.
+      #   - Cleanup: The adapter relies on SolidQueue::Worker lifecycle hooks to clean up
+      #     active semaphores when the worker stops. If a worker crashes, semaphores will
+      #     leak until the underlying database records expire or are manually cleaned.
+      #
+      #:  () -> void
+      def initialize
+        @semaphore_registry = {} #: Hash[Object, ^(SolidQueue::Worker) -> void]
+        super
+      end
+
+      #:  () -> void
+      def initialize_adapter!
+        SolidQueue::Configuration.prepend(SchedulingPatch) if defined?(SolidQueue::Configuration)
+        SolidQueue::ClaimedExecution.prepend(ClaimedExecutionPatch) if defined?(SolidQueue::ClaimedExecution)
+      end
+
+      #:  () -> bool
+      def semaphore_available?
+        defined?(SolidQueue::Semaphore) ? true : false
+      end
+
+      # @note
+      #   - Thread safety: @semaphore_registry is a non-thread-safe Hash. In multi-threaded workers,
+      #     concurrent calls to semaphore_wait or semaphore_signal may cause race conditions.
+      #     Mitigation: SolidQueue workers typically run in single-threaded Fiber mode; verify
+      #     worker configuration does not enable raw multithreading.
+      #   - Double-wait behavior: If semaphore_wait is called twice for the same Semaphore
+      #     (e.g., due to retry or requeue), the second call returns false and does not
+      #     re-register the hook. This is a fail-fast contract: the semaphore is already
+      #     being waited and will signal the registered hook.
+      #
+      #:  (Semaphore) -> bool
+      def semaphore_wait(semaphore)
+        return true unless semaphore_available?
+        return false if semaphore_registry.key?(semaphore)
+        return false unless SolidQueue::Semaphore.wait(semaphore)
+
+        hook = ->(_) { SolidQueue::Semaphore.signal(semaphore) }
+        semaphore_registry[semaphore] = hook
+        SolidQueue::Worker.on_stop(&hook)
+        true
+      end
+
+      # @note
+      #   - Lifecycle management: The adapter is responsible for removing the hook from
+      #     SolidQueue::Worker.lifecycle_hooks[:stop] before calling signal. The hook must
+      #     be deleted from the registry and the global lifecycle_hooks to prevent redundant
+      #     signal calls after the semaphore has already been signaled.
+      #   - Hook deletion order: The hook is deleted before calling signal to ensure the
+      #     hook lambda is no longer invoked even if the signal triggers a worker stop.
+      #
+      #:  (Semaphore) -> bool
+      def semaphore_signal(semaphore)
+        return true unless semaphore_available?
+        return true unless semaphore_registry.key?(semaphore)
+
+        hook = semaphore_registry[semaphore]
+        SolidQueue::Worker.lifecycle_hooks[:stop].delete(hook)
+        semaphore_registry.delete(semaphore)
+        SolidQueue::Semaphore.signal(semaphore)
+      end
+
+      #:  (Array[String]) -> Hash[String, untyped]
+      def fetch_job_statuses(job_ids)
+        return {} unless defined?(SolidQueue::Job)
+
+        SolidQueue::Job.where(active_job_id: job_ids).index_by(&:active_job_id)
+      end
+
+      #:  (untyped) -> Symbol
+      def job_status(job)
+        return :failed if job.failed?
+        return :succeeded if job.finished?
+        return :running if job.claimed?
+
+        :pending
+      end
+
+      #:  () -> bool
+      def supports_concurrency_limits?
+        defined?(SolidQueue) ? true : false
+      end
+
+      #:  (String) -> bool
+      def pause_queue(queue_name)
+        return false unless defined?(SolidQueue::Queue)
+
+        SolidQueue::Queue.find_by_name(queue_name).pause
+        true
+      rescue ActiveRecord::RecordNotUnique
+        true
+      end
+
+      #:  (String) -> bool
+      def resume_queue(queue_name)
+        return false unless defined?(SolidQueue::Queue)
+
+        SolidQueue::Queue.find_by_name(queue_name).resume
+        true
+      end
+
+      #:  (String) -> bool
+      def queue_paused?(queue_name)
+        return false unless defined?(SolidQueue::Queue)
+
+        SolidQueue::Queue.find_by_name(queue_name).paused?
+      end
+
+      #:  () -> Array[String]
+      def paused_queues
+        return [] unless defined?(SolidQueue::Pause)
+
+        SolidQueue::Pause.pluck(:queue_name)
+      end
+
+      #:  (String) -> Integer?
+      def queue_latency(queue_name)
+        return nil unless defined?(SolidQueue::Queue)
+
+        SolidQueue::Queue.find_by_name(queue_name).latency
+      end
+
+      #:  (String) -> Integer
+      def queue_size(queue_name)
+        return 0 unless defined?(SolidQueue::Queue)
+
+        SolidQueue::Queue.find_by_name(queue_name).size
+      end
+
+      #:  (String) -> bool
+      def clear_queue(queue_name)
+        return false unless defined?(SolidQueue::Queue)
+
+        SolidQueue::Queue.find_by_name(queue_name).clear
+        true
+      end
+
+      # @note
+      #   - SolidQueue stores the full ActiveJob serialization in job.arguments
+      #   - We need to extract the actual arguments array for consistency
+      #
+      #:  (String) -> Hash[String, untyped]?
+      def find_job(job_id)
+        return unless defined?(SolidQueue::Job)
+
+        job = SolidQueue::Job.find_by(active_job_id: job_id)
+        return if job.nil?
+
+        {
+          "job_id" => job.active_job_id,
+          "class_name" => job.class_name,
+          "queue_name" => job.queue_name,
+          "arguments" => job.arguments.is_a?(Hash) ? job.arguments["arguments"] : job.arguments,
+          "status" => job_status(job)
+        }
+      end
+
+      #:  (DSL, Numeric) -> bool
+      def reschedule_job(job, wait)
+        return false unless defined?(SolidQueue::Job)
+
+        solid_queue_job = SolidQueue::Job.find_by(active_job_id: job.job_id)
+        return false unless solid_queue_job&.claimed?
+
+        reschedule_solid_queue_job(solid_queue_job, job, wait)
+      rescue ActiveRecord::RecordNotFound
+        false
+      end
+
+      private
+
+      attr_reader :semaphore_registry #: Hash[Object, ^(SolidQueue::Worker) -> void]
+
+      #:  (SolidQueue::Job, DSL, Numeric) -> bool
+      def reschedule_solid_queue_job(solid_queue_job, active_job, wait)
+        solid_queue_job.with_lock do
+          solid_queue_job.claimed_execution&.destroy!
+          solid_queue_job.update!(
+            scheduled_at: wait.seconds.from_now,
+            arguments: active_job.serialize.deep_stringify_keys["arguments"]
+          )
+          solid_queue_job.prepare_for_execution
+        end
+        true
+      end
+
+      # @rbs module-self SolidQueue::ClaimedExecution
+      module ClaimedExecutionPatch
+        private
+
+        #:  () -> SolidQueue::ClaimedExecution
+        def finished
+          return self unless self.class.exists?(id)
+
+          super
+        end
+      end
+
+      module SchedulingPatch
+        private
+
+        #:  () -> Hash[Symbol, Hash[Symbol, untyped]]
+        def recurring_tasks_config
+          super.merge!(
+            DSL._included_classes.to_a.reduce(
+              {} #: Hash[Symbol, Hash[Symbol, untyped]]
+            ) { |acc, job_class| acc.merge(job_class._workflow.build_schedules_hash) }
+          )
+        end
+      end
+    end
+    # rubocop:enable Naming/PredicateMethod
+  end
+end

data/lib/job_workflow/runner.rb

@@ -0,0 +1,173 @@
+# frozen_string_literal: true
+
+module JobWorkflow
+  class Runner # rubocop:disable Metrics/ClassLength
+    attr_reader :context #: Context
+
+    #:  (context: Context) -> void
+    def initialize(context:)
+      @context = context
+      @job = context._job || (raise "current job is not set in context")
+    end
+
+    #:  () -> void
+    def run
+      task = context._task_context.task
+      return run_task(task) if !task.nil? && context.sub_job?
+
+      catch(:rescheduled) { run_workflow }
+    end
+
+    private
+
+    attr_reader :job #: DSL
+
+    #:  () -> Workflow
+    def workflow
+      job.class._workflow
+    end
+
+    #:  () -> Array[Task]
+    def tasks
+      workflow.tasks
+    end
+
+    #:  () -> HookRegistry
+    def hooks
+      workflow.hooks
+    end
+
+    #:  () -> void
+    def run_workflow
+      Instrumentation.instrument_workflow(job) do
+        tasks.each do |task|
+          next if skip_task?(task)
+
+          job.step(task.task_name) do |step|
+            wait_for_dependent_tasks(task, step)
+            task.enqueue.should_enqueue?(context) ? enqueue_task(task) : run_task(task)
+          end
+        end
+      end
+    end
+
+    #:  (Task) -> bool
+    def skip_task?(task)
+      result = !task.condition.call(context)
+      Instrumentation.notify_task_skip(job, task, "condition_not_met") if result
+      result
+    end
+
+    #:  (Task) -> void
+    def run_task(task)
+      context._load_parent_task_output
+      context._with_each_value(task).each do |ctx|
+        run_each_task(task, ctx)
+      rescue StandardError => e
+        run_error_hooks(task, ctx, e)
+        raise
+      end
+    end
+
+    #:  (Task, Context) -> void
+    def run_each_task(task, ctx)
+      Instrumentation.instrument_task(job, task, ctx) do
+        ctx._with_task_throttle do
+          run_hooks(task, ctx) do
+            data = task.block.call(ctx)
+            add_task_output(ctx:, task:, each_index: ctx._task_context.index, data:)
+          end
+        end
+      end
+    end
+
+    #:  (Task, Context) { () -> void } -> void
+    def run_hooks(task, ctx, &)
+      hooks.before_hooks_for(task.task_name).each { |hook| hook.block.call(ctx) }
+      run_around_hooks(task, ctx, hooks.around_hooks_for(task.task_name), &)
+      hooks.after_hooks_for(task.task_name).each { |hook| hook.block.call(ctx) }
+    end
+
+    #:  (Task, Context, Array[Hook]) { () -> void } -> void
+    def run_around_hooks(task, ctx, around_hooks, &)
+      return yield if around_hooks.empty?
+
+      hook = around_hooks.first
+      remaining = around_hooks[1..] || []
+      callable = TaskCallable.new { run_around_hooks(task, ctx, remaining, &) }
+      hook.block.call(ctx, callable)
+      raise TaskCallable::NotCalledError, task.task_name unless callable.called?
+    end
+
+    #:  (Task, Context, StandardError) -> void
+    def run_error_hooks(task, ctx, error)
+      hooks.error_hooks_for(task.task_name).each { |hook| hook.block.call(ctx, error, task) }
+    end
+
+    #:  (Task) -> void
+    def enqueue_task(task)
+      sub_jobs = context._with_each_value(task).map { |ctx| job.class.from_context(ctx) }
+      ActiveJob.perform_all_later(sub_jobs)
+      context.job_status.update_task_job_statuses_from_jobs(task_name: task.task_name, jobs: sub_jobs)
+      Instrumentation.notify_task_enqueue(job, task, sub_jobs.size)
+    end
+
+    #:  (Task, ActiveJob::Continuation::Step) -> void
+    def wait_for_dependent_tasks(waiting_task, step)
+      waiting_task.depends_on.each do |dependent_task_name|
+        dependent_task = workflow.fetch_task(dependent_task_name)
+        next if dependent_task.nil? || context.job_status.needs_waiting?(dependent_task.task_name)
+
+        Instrumentation.instrument_dependent_wait(job, dependent_task) do
+          poll_until_complete_or_reschedule(waiting_task, dependent_task, step)
+        end
+
+        update_task_outputs(dependent_task)
+      end
+    end
+
+    #:  (Task, Task, ActiveJob::Continuation::Step) -> void
+    def poll_until_complete_or_reschedule(waiting_task, dependent_task, step)
+      poll_state = { count: 0, started_at: Time.current }
+      dependency_wait = waiting_task.dependency_wait
+
+      loop do
+        step.checkpoint!
+        context.job_status.update_task_job_statuses_from_db(dependent_task.task_name)
+        break if context.job_status.needs_waiting?(dependent_task.task_name)
+
+        poll_state[:count] += 1
+        reschedule_if_needed(dependent_task, dependency_wait, poll_state)
+        sleep dependency_wait.poll_interval
+      end
+    end
+
+    #:  (Task, TaskDependencyWait, Hash[Symbol, untyped]) -> void
+    def reschedule_if_needed(dependent_task, dependency_wait, poll_state)
+      return if dependency_wait.polling_only?
+      return if dependency_wait.polling_keep?(poll_state[:started_at])
+      return unless QueueAdapter.current.reschedule_job(job, dependency_wait.reschedule_delay)
+
+      Instrumentation.notify_dependent_reschedule(
+        job,
+        dependent_task,
+        dependency_wait.reschedule_delay,
+        poll_state[:count]
+      )
+      throw :rescheduled
+    end
+
+    #:  (ctx: Context, task: Task, each_index: Integer, data: untyped) -> void
+    def add_task_output(ctx:, task:, data:, each_index:)
+      return if task.output.empty?
+
+      ctx._add_task_output(TaskOutput.from_task(task:, each_index:, data:))
+    end
+
+    #:  (Task) -> void
+    def update_task_outputs(task)
+      finished_job_ids = context.job_status.finished_job_ids(task_name: task.task_name)
+      context.output.update_task_outputs_from_db(finished_job_ids, context.workflow)
+    end
+  end
+end

data/lib/job_workflow/schedule.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module JobWorkflow
+  class Schedule
+    attr_reader :key #: Symbol
+    attr_reader :class_name #: String
+    attr_reader :expression #: String
+    attr_reader :queue #: String?
+    attr_reader :priority #: Integer?
+    attr_reader :args #: Hash[Symbol, untyped]
+    attr_reader :description #: String?
+
+    # rubocop:disable Metrics/ParameterLists
+    #:  (
+    #     expression: String,
+    #     class_name: String,
+    #     ?key: (String | Symbol)?,
+    #     ?queue: String?,
+    #     ?priority: Integer?,
+    #     ?args: Hash[Symbol, untyped],
+    #     ?description: String?
+    #   ) -> void
+    def initialize(expression:, class_name:, key: nil, queue: nil, priority: nil, args: {}, description: nil)
+      @expression = expression #: String
+      @class_name = class_name #: String
+      @key = (key || class_name).to_sym #: Symbol
+      @queue = queue #: String?
+      @priority = priority #: Integer?
+      @args = args #: Hash[Symbol, untyped]
+      @description = description #: String?
+    end
+    # rubocop:enable Metrics/ParameterLists
+
+    #:  () -> Hash[Symbol, untyped]
+    def to_config
+      {
+        class: class_name,
+        schedule: expression,
+        queue: queue,
+        priority: priority,
+        args: args.empty? ? nil : [args],
+        description: description
+      }.compact
+    end
+  end
+end