job-workflow 0.1.3

data/lib/job_workflow/instrumentation.rb

@@ -0,0 +1,257 @@
+# frozen_string_literal: true
+
+module JobWorkflow
+  # Instrumentation provides ActiveSupport::Notifications-based event instrumentation for JobWorkflow workflows and tasks.
+  #
+  # @example Subscribing to events
+  #   ```ruby
+  #   ActiveSupport::Notifications.subscribe("task.start.job_workflow") do |name, start, finish, id, payload|
+  #     puts "Task #{payload[:task_name]} started"
+  #   end
+  #   ```
+  module Instrumentation # rubocop:disable Metrics/ModuleLength
+    NAMESPACE = "job_workflow"
+
+    module Events
+      WORKFLOW = "workflow.#{NAMESPACE}".freeze
+      WORKFLOW_START = "workflow.start.#{NAMESPACE}".freeze
+      WORKFLOW_COMPLETE = "workflow.complete.#{NAMESPACE}".freeze
+      TASK = "task.#{NAMESPACE}".freeze
+      TASK_START = "task.start.#{NAMESPACE}".freeze
+      TASK_COMPLETE = "task.complete.#{NAMESPACE}".freeze
+      TASK_ERROR = "task.error.#{NAMESPACE}".freeze
+      TASK_SKIP = "task.skip.#{NAMESPACE}".freeze
+      TASK_ENQUEUE = "task.enqueue.#{NAMESPACE}".freeze
+      TASK_RETRY = "task.retry.#{NAMESPACE}".freeze
+      THROTTLE_ACQUIRE = "throttle.acquire.#{NAMESPACE}".freeze
+      THROTTLE_ACQUIRE_START = "throttle.acquire.start.#{NAMESPACE}".freeze
+      THROTTLE_ACQUIRE_COMPLETE = "throttle.acquire.complete.#{NAMESPACE}".freeze
+      THROTTLE_RELEASE = "throttle.release.#{NAMESPACE}".freeze
+      DEPENDENT_WAIT = "dependent.wait.#{NAMESPACE}".freeze
+      DEPENDENT_WAIT_START = "dependent.wait.start.#{NAMESPACE}".freeze
+      DEPENDENT_WAIT_COMPLETE = "dependent.wait.complete.#{NAMESPACE}".freeze
+      DEPENDENT_RESCHEDULE = "dependent.reschedule.#{NAMESPACE}".freeze
+      QUEUE_PAUSE = "queue.pause.#{NAMESPACE}".freeze
+      QUEUE_RESUME = "queue.resume.#{NAMESPACE}".freeze
+      CUSTOM = "custom.#{NAMESPACE}".freeze
+      DRY_RUN = "dry_run.#{NAMESPACE}".freeze
+      DRY_RUN_SKIP = "dry_run.skip.#{NAMESPACE}".freeze
+      DRY_RUN_EXECUTE = "dry_run.execute.#{NAMESPACE}".freeze
+    end
+
+    class << self
+      #:  (DSL) { () -> untyped } -> untyped
+      def instrument_workflow(job, &)
+        payload = build_workflow_payload(job)
+        instrument(Events::WORKFLOW_START, payload)
+        instrument(Events::WORKFLOW, payload, &)
+      ensure
+        instrument(Events::WORKFLOW_COMPLETE, payload)
+      end
+
+      #:  (DSL, Task, Context) { () -> untyped } -> untyped
+      def instrument_task(job, task, ctx, &)
+        payload = build_task_payload(job, task, ctx)
+        instrument(Events::TASK_START, payload)
+        instrument(Events::TASK, payload, &)
+      ensure
+        instrument(Events::TASK_COMPLETE, payload)
+      end
+
+      #:  (DSL, Task, String) -> void
+      def notify_task_skip(job, task, reason)
+        instrument(Events::TASK_SKIP, build_task_skip_payload(job, task, reason))
+      end
+
+      #:  (DSL, Task, Integer) -> void
+      def notify_task_enqueue(job, task, sub_job_count)
+        instrument(Events::TASK_ENQUEUE, build_task_enqueue_payload(job, task, sub_job_count))
+      end
+
+      #:  (Task, Context, String, Integer, Float, StandardError) -> void
+      def notify_task_retry(task, ctx, job_id, attempt, delay, error) # rubocop:disable Metrics/ParameterLists
+        instrument(Events::TASK_RETRY, build_task_retry_payload(task, ctx, job_id, attempt, delay, error))
+      end
+
+      #:  (DSL, Task) { () -> untyped } -> untyped
+      def instrument_dependent_wait(job, task, &)
+        payload = build_dependent_payload(job, task)
+        instrument(Events::DEPENDENT_WAIT_START, payload)
+        instrument(Events::DEPENDENT_WAIT, payload, &)
+      ensure
+        instrument(Events::DEPENDENT_WAIT_COMPLETE, payload)
+      end
+
+      #:  (DSL, Task, Numeric, Integer) -> void
+      def notify_dependent_reschedule(job, task, reschedule_delay, poll_count)
+        instrument(
+          Events::DEPENDENT_RESCHEDULE,
+          build_dependent_reschedule_payload(job, task, reschedule_delay, poll_count)
+        )
+      end
+
+      #:  (Semaphore) { () -> untyped } -> untyped
+      def instrument_throttle(semaphore, &)
+        payload = build_throttle_payload(semaphore)
+        instrument(Events::THROTTLE_ACQUIRE_START, payload)
+        instrument(Events::THROTTLE_ACQUIRE, payload, &)
+      ensure
+        instrument(Events::THROTTLE_ACQUIRE_COMPLETE, payload)
+      end
+
+      #:  (Semaphore) -> void
+      def notify_throttle_release(semaphore)
+        instrument(Events::THROTTLE_RELEASE, build_throttle_payload(semaphore))
+      end
+
+      #:  (String) -> void
+      def notify_queue_pause(queue_name)
+        instrument(Events::QUEUE_PAUSE, build_queue_payload(queue_name))
+      end
+
+      #:  (String) -> void
+      def notify_queue_resume(queue_name)
+        instrument(Events::QUEUE_RESUME, build_queue_payload(queue_name))
+      end
+
+      #:  (String, Hash[Symbol, untyped]) { () -> untyped } -> untyped
+      def instrument_custom(operation, payload = {}, &)
+        event_name = "#{operation}.#{NAMESPACE}"
+        instrument(event_name, payload, &)
+      end
+
+      #:  (DSL, Context, Symbol?, Integer, bool) { () -> untyped } -> untyped
+      def instrument_dry_run(job, ctx, dry_run_name, skip_in_dry_run_index, dry_run, &)
+        start_event = dry_run ? Events::DRY_RUN_SKIP : Events::DRY_RUN_EXECUTE
+        payload = build_skip_in_dry_run_payload(job, ctx, dry_run_name, skip_in_dry_run_index, dry_run)
+        instrument(start_event, payload)
+        instrument(Events::DRY_RUN, payload, &)
+      end
+
+      private
+
+      #:  (String, Hash[Symbol, untyped]) ?{ () -> untyped } -> untyped
+      def instrument(event_name, payload = {}, &)
+        ActiveSupport::Notifications.instrument(event_name, payload, &)
+      end
+
+      #:  (DSL) -> Hash[Symbol, untyped]
+      def build_workflow_payload(job)
+        {
+          job:,
+          job_id: job.job_id,
+          job_name: job.class.name
+        }
+      end
+
+      #:  (DSL, Task, Context) -> Hash[Symbol, untyped]
+      def build_task_payload(job, task, ctx)
+        task_ctx = ctx._task_context
+        {
+          job:,
+          job_id: job.job_id,
+          job_name: job.class.name,
+          task:,
+          task_name: task.task_name,
+          context: ctx,
+          each_index: task_ctx.index,
+          retry_count: task_ctx.retry_count
+        }
+      end
+
+      #:  (DSL, Task, String) -> Hash[Symbol, untyped]
+      def build_task_skip_payload(job, task, reason)
+        {
+          job:,
+          job_id: job.job_id,
+          job_name: job.class.name,
+          task:,
+          task_name: task.task_name,
+          reason:
+        }
+      end
+
+      #:  (DSL, Task, Integer) -> Hash[Symbol, untyped]
+      def build_task_enqueue_payload(job, task, sub_job_count)
+        {
+          job:,
+          job_id: job.job_id,
+          job_name: job.class.name,
+          task:,
+          task_name: task.task_name,
+          sub_job_count:
+        }
+      end
+
+      #:  (Task, Context, String, Integer, Float, StandardError) -> Hash[Symbol, untyped]
+      def build_task_retry_payload(task, ctx, job_id, attempt, delay, error) # rubocop:disable Metrics/ParameterLists
+        task_ctx = ctx._task_context
+        {
+          task:,
+          task_name: task.task_name,
+          job_id:,
+          each_index: task_ctx.index,
+          attempt:,
+          max_attempts: task.task_retry.count,
+          delay_seconds: delay.round(3),
+          error:,
+          error_class: error.class.name,
+          error_message: error.message
+        }
+      end
+
+      #:  (DSL, Task) -> Hash[Symbol, untyped]
+      def build_dependent_payload(job, task)
+        {
+          job:,
+          job_id: job.job_id,
+          job_name: job.class.name,
+          task:,
+          dependent_task_name: task.task_name
+        }
+      end
+
+      #:  (DSL, Task, Numeric, Integer) -> Hash[Symbol, untyped]
+      def build_dependent_reschedule_payload(job, task, reschedule_delay, poll_count)
+        {
+          job:,
+          job_id: job.job_id,
+          job_name: job.class.name,
+          task:,
+          dependent_task_name: task.task_name,
+          reschedule_delay:,
+          poll_count:
+        }
+      end
+
+      #:  (Semaphore) -> Hash[Symbol, untyped]
+      def build_throttle_payload(semaphore)
+        {
+          semaphore:,
+          concurrency_key: semaphore.concurrency_key,
+          concurrency_limit: semaphore.concurrency_limit
+        }
+      end
+
+      #:  (String) -> Hash[Symbol, untyped]
+      def build_queue_payload(queue_name)
+        {
+          queue_name:
+        }
+      end
+
+      #:  (DSL, Context, Symbol?, Integer, bool) -> Hash[Symbol, untyped]
+      def build_skip_in_dry_run_payload(job, ctx, dry_run_name, dry_run_index, dry_run)
+        {
+          job_id: job.job_id,
+          job_name: job.class.name,
+          task_name: ctx._task_context.task&.task_name,
+          each_index: ctx._task_context.index,
+          dry_run_name:,
+          dry_run_index:,
+          dry_run:
+        }
+      end
+    end
+  end
+end

data/lib/job_workflow/job_status.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module JobWorkflow
+  class JobStatus
+    class << self
+      #:  (Array[Hash[untyped, untyped]]) -> JobStatus
+      def from_hash_array(array)
+        new(task_job_statuses: array.map { |hash| TaskJobStatus.from_hash(hash) })
+      end
+
+      #:  (Hash[String, untyped]) -> JobStatus
+      def deserialize(hash)
+        new(task_job_statuses: hash.fetch("task_job_statuses", []).map { |shash| TaskJobStatus.deserialize(shash) })
+      end
+    end
+
+    #:  (?task_job_statuses: Array[TaskJobStatus]) -> void
+    def initialize(task_job_statuses: [])
+      self.task_job_statuses = {}
+      task_job_statuses.each { |task_job_status| update_task_job_status(task_job_status) }
+    end
+
+    #:  (task_name: Symbol) -> Array[TaskJobStatus]
+    def fetch_all(task_name:)
+      task_job_statuses.fetch(task_name, []).compact
+    end
+
+    #:  (task_name: Symbol, index: Integer) -> TaskJobStatus?
+    def fetch(task_name:, index:)
+      task_job_statuses.fetch(task_name, [])[index]
+    end
+
+    #:  (task_name: Symbol) -> Array[String]
+    def finished_job_ids(task_name:)
+      fetch_all(task_name:).filter(&:finished?).map(&:job_id)
+    end
+
+    #:  () -> Array[TaskJobStatus]
+    def flat_task_job_statuses
+      task_job_statuses.values.flatten
+    end
+
+    # @note
+    #   - If the array is empty, the task is not enqueued and is considered completed.
+    #   - If we add a task existence check in the future, we'll check here.
+    #
+    #:  (Symbol) -> bool
+    def needs_waiting?(task_name)
+      task_job_statuses.fetch(task_name, []).all?(&:finished?)
+    end
+
+    #:  (TaskJobStatus) -> void
+    def update_task_job_status(task_job_status)
+      task_job_statuses[task_job_status.task_name] ||= []
+      task_job_statuses[task_job_status.task_name][task_job_status.each_index] = task_job_status
+    end
+
+    #:  (task_name: Symbol, jobs: Array[DSL]) -> void
+    def update_task_job_statuses_from_jobs(task_name:, jobs:)
+      jobs.each.with_index do |job, index|
+        update_task_job_status(
+          TaskJobStatus.new(
+            task_name:,
+            job_id: job.job_id,
+            each_index: index,
+            status: :pending
+          )
+        )
+      end
+    end
+
+    #:  (Symbol) -> void
+    def update_task_job_statuses_from_db(task_name)
+      statuses = task_job_statuses.fetch(task_name, []).reject(&:finished?).index_by(&:job_id)
+      return if statuses.empty?
+
+      task_jobs = QueueAdapter.current.fetch_job_statuses(statuses.keys)
+
+      statuses.each do |job_id, task_job_status|
+        task_job = task_jobs[job_id]
+        next unless task_job
+
+        task_job_status.update_status(QueueAdapter.current.job_status(task_job))
+        update_task_job_status(task_job_status)
+      end
+    end
+
+    private
+
+    attr_accessor :task_job_statuses #: Hash[Symbol, Array[TaskJobStatus]]
+  end
+end

data/lib/job_workflow/logger.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+module JobWorkflow
+  # Logger provides structured JSON logging for JobWorkflow workflows.
+  #
+  # @example Basic usage
+  #   ```ruby
+  #   JobWorkflow.logger = ActiveSupport::Logger.new($stdout)
+  #   JobWorkflow.logger.formatter = JobWorkflow::Logger::JsonFormatter.new
+  #   ```
+  #
+  # @example With custom log tags
+  #   ```ruby
+  #   JobWorkflow.logger.formatter = JobWorkflow::Logger::JsonFormatter.new(log_tags: [:request_id])
+  #   ```
+  module Logger
+    #:  (ActiveSupport::Logger) -> void
+    attr_writer :logger
+
+    #:  () -> ActiveSupport::Logger
+    def logger
+      @logger ||= build_default_logger
+    end
+
+    private
+
+    #:  () -> ActiveSupport::Logger
+    def build_default_logger
+      logger = ActiveSupport::Logger.new($stdout)
+      logger.formatter = Logger::JsonFormatter.new
+      logger
+    end
+
+    # JSON formatter for structured logging output.
+    # @rbs inherits ::Logger::Formatter
+    class JsonFormatter < ::Logger::Formatter
+      include ActiveSupport::TaggedLogging::Formatter
+
+      #:  (?log_tags: Array[Symbol]) -> void
+      def initialize(log_tags: [])
+        @log_tags = log_tags
+        super()
+      end
+
+      #:  (String, Time, String, String | Hash[untyped, untyped]) -> String
+      def call(severity, time, progname, msg)
+        base_hash = build_base_hash(severity, time, progname)
+        tags_hash = build_tags_hash
+        msg_hash = build_msg_hash(msg)
+        "#{JSON.generate({ **base_hash, **tags_hash, **msg_hash })}\n"
+      end
+
+      private
+
+      attr_reader :log_tags #: Array[Symbol]
+
+      #:  (String, Time, String) -> Hash[Symbol, untyped]
+      def build_base_hash(severity, time, progname)
+        time_in_zone = time.in_time_zone(Time.zone || "UTC")
+        { time: time_in_zone.iso8601(6), level: severity, progname: progname }
+      end
+
+      #:  () -> Hash[Symbol, untyped]
+      def build_tags_hash
+        log_tags.zip(current_tags).to_h
+      end
+
+      #:  (String | Hash[untyped, untyped]) -> Hash[Symbol, untyped]
+      def build_msg_hash(msg)
+        case msg
+        when Hash
+          msg.symbolize_keys
+        else
+          parse_json_or_message(msg.to_s)
+        end
+      end
+
+      #:  (String) -> Hash[Symbol, untyped]
+      def parse_json_or_message(msg)
+        JSON.parse(msg, symbolize_names: true)
+      rescue JSON::ParserError
+        { message: msg }
+      end
+    end
+  end
+end

data/lib/job_workflow/namespace.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module JobWorkflow
+  class Namespace
+    attr_reader :name #: Symbol
+    attr_reader :parent #: Namespace?
+
+    class << self
+      #:  () -> Namespace
+      def default
+        new(name: :"")
+      end
+    end
+
+    #:  (name: Symbol, ?parent: Namespace?) -> void
+    def initialize(name:, parent: nil)
+      @name = name #: Symbol
+      @parent = parent #: Namespace?
+    end
+
+    #:  () -> bool
+    def default?
+      name.empty?
+    end
+
+    #:  (Namespace) -> Namespace
+    def update_parent(parent)
+      self.class.new(name:, parent:)
+    end
+
+    #:  () -> Symbol
+    def full_name
+      [parent&.full_name, name.to_s].compact.reject(&:empty?).join(":").to_sym
+    end
+  end
+end

data/lib/job_workflow/output.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module JobWorkflow
+  class Output
+    class << self
+      #:  (Array[Hash[untyped, untyped]]) -> Output
+      def from_hash_array(array)
+        task_outputs = array.map do |hash|
+          normalized_hash = hash.transform_keys(&:to_sym)
+          task_name = normalized_hash[:task_name]
+          each_index = normalized_hash[:each_index]
+          data = normalized_hash[:data]
+          TaskOutput.new(task_name:, each_index:, data:)
+        end
+        new(task_outputs:)
+      end
+
+      #:  (Hash[String, untyped]) -> Output
+      def deserialize(hash)
+        new(task_outputs: hash.fetch("task_outputs", []).map { |shash| TaskOutput.deserialize(shash) })
+      end
+    end
+
+    #:  (?task_outputs: Array[TaskOutput]) -> void
+    def initialize(task_outputs: [])
+      self.task_outputs = {}
+      task_outputs.each { |task_output| add_task_output(task_output) }
+    end
+
+    #:  (task_name: Symbol?) -> Array[TaskOutput]
+    def fetch_all(task_name:)
+      fixed_type_task_name = task_name #: Symbol
+      task_outputs.fetch(fixed_type_task_name, []).compact
+    end
+
+    #:  (task_name: Symbol?, each_index: Integer) -> TaskOutput?
+    def fetch(task_name:, each_index:)
+      fixed_type_task_name = task_name #: Symbol
+      task_outputs.fetch(fixed_type_task_name, [])[each_index]
+    end
+
+    #:  (Symbol | String) -> Array[TaskOutput?]
+    def [](task_name)
+      task_outputs.fetch(task_name.to_sym, [])
+    end
+
+    #:  (TaskOutput) -> void
+    def add_task_output(task_output)
+      task_outputs[task_output.task_name] ||= []
+      task_outputs[task_output.task_name][task_output.each_index] = task_output
+    end
+
+    #:  (Array[String], Workflow) -> void
+    def update_task_outputs_from_db(job_ids, workflow)
+      jobs = SolidQueue::Job.where(active_job_id: job_ids)
+      return if jobs.empty?
+
+      update_task_outputs_from_jobs(jobs.to_a, workflow)
+    end
+
+    #:  (Array[SolidQueue::Job], Workflow) -> void
+    def update_task_outputs_from_jobs(jobs, workflow)
+      jobs.each do |job|
+        context = Context.deserialize(job.arguments["job_workflow_context"].merge("workflow" => workflow))
+        task_output = context.each_task_output
+        next if task_output.nil?
+
+        add_task_output(task_output)
+      end
+    end
+
+    #:  () -> Array[TaskOutput]
+    def flat_task_outputs
+      task_outputs.values.flatten.compact
+    end
+
+    private
+
+    attr_accessor :task_outputs #: Hash[Symbol, Array[TaskOutput]]
+  end
+end

data/lib/job_workflow/output_def.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module JobWorkflow
+  class OutputDef
+    attr_reader :name #: Symbol
+    attr_reader :type #: String
+
+    #:  (name: Symbol, type: String) -> void
+    def initialize(name:, type:)
+      @name = name
+      @type = type
+    end
+  end
+end

data/lib/job_workflow/queue.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module JobWorkflow
+  # Queue provides a unified interface for queue operations across different queue adapters.
+  #
+  # @example Pausing and resuming a queue
+  #   ```ruby
+  #   JobWorkflow::Queue.pause(:import_workflow)
+  #   JobWorkflow::Queue.paused?(:import_workflow)  # => true
+  #   JobWorkflow::Queue.resume(:import_workflow)
+  #   JobWorkflow::Queue.paused?(:import_workflow)  # => false
+  #   ```
+  #
+  # @example Getting queue metrics
+  #   ```ruby
+  #   JobWorkflow::Queue.latency(:import_workflow)  # => 120 (seconds)
+  #   JobWorkflow::Queue.size(:import_workflow)     # => 42 (pending jobs)
+  #   ```
+  #
+  # @example Listing workflows associated with a queue
+  #   ```ruby
+  #   JobWorkflow::Queue.workflows(:import_workflow)  # => [ImportJob, DataSyncJob]
+  #   ```
+  class Queue
+    class << self
+      #:  (String | Symbol) -> bool
+      def pause(queue_name)
+        queue_name_str = queue_name.to_s
+        result = QueueAdapter.current.pause_queue(queue_name_str)
+        Instrumentation.notify_queue_pause(queue_name_str) if result
+        result
+      end
+
+      #:  (String | Symbol) -> bool
+      def resume(queue_name)
+        queue_name_str = queue_name.to_s
+        result = QueueAdapter.current.resume_queue(queue_name_str)
+        Instrumentation.notify_queue_resume(queue_name_str) if result
+        result
+      end
+
+      #:  (String | Symbol) -> bool
+      def paused?(queue_name)
+        QueueAdapter.current.queue_paused?(queue_name.to_s)
+      end
+
+      #:  () -> Array[String]
+      def paused_queues
+        QueueAdapter.current.paused_queues
+      end
+
+      #:  (String | Symbol) -> Integer?
+      def latency(queue_name)
+        QueueAdapter.current.queue_latency(queue_name.to_s)
+      end
+
+      #:  (String | Symbol) -> Integer
+      def size(queue_name)
+        QueueAdapter.current.queue_size(queue_name.to_s)
+      end
+
+      #:  (String | Symbol) -> bool
+      def clear(queue_name)
+        QueueAdapter.current.clear_queue(queue_name.to_s)
+      end
+
+      #:  (String | Symbol) -> Array[singleton(DSL)]
+      def workflows(queue_name)
+        queue_name_str = queue_name.to_s
+        DSL._included_classes.filter { |job_class| job_class.queue_name == queue_name_str }.to_a
+      end
+    end
+  end
+end

data/lib/job_workflow/queue_adapter.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require_relative "queue_adapters/abstract"
+require_relative "queue_adapters/null_adapter"
+require_relative "queue_adapters/solid_queue_adapter"
+
+module JobWorkflow
+  module QueueAdapter
+    # @rbs!
+    #   def self._current: () -> QueueAdapters::Abstract
+    #   def self._current=: (QueueAdapters::Abstract?) -> void
+
+    mattr_accessor :_current
+
+    class << self
+      #:  () -> QueueAdapters::Abstract
+      def current
+        self._current ||= detect_adapter
+      end
+
+      #:  () -> void
+      def reset!
+        self._current = nil
+      end
+
+      private
+
+      #:  () -> QueueAdapters::Abstract
+      def detect_adapter
+        if defined?(SolidQueue)
+          QueueAdapters::SolidQueueAdapter.new
+        else
+          QueueAdapters::NullAdapter.new
+        end
+      end
+    end
+  end
+end