job-workflow 0.5.0 → 0.6.1

data/lib/job_workflow/monitoring/execution_view_model.rb

@@ -0,0 +1,258 @@
+# frozen_string_literal: true
+
+module JobWorkflow
+  module Monitoring
+    class ExecutionViewModel
+      attr_reader :job_id #: String
+      attr_reader :queue_name #: String?
+      attr_reader :status #: WorkflowStatus
+
+      #:  (job_id: String, queue_name: String?, status: WorkflowStatus) -> void
+      def initialize(job_id:, queue_name:, status:)
+        @job_id = job_id
+        @queue_name = queue_name
+        @status = status
+      end
+
+      #:  () -> String
+      def job_class_name
+        status.job_class_name
+      end
+
+      #:  () -> Symbol
+      def workflow_status
+        status.status
+      end
+
+      #:  () -> Symbol?
+      def current_task_name
+        status.current_task_name
+      end
+
+      #:  () -> Arguments
+      def arguments
+        status.arguments
+      end
+
+      #:  () -> Hash[untyped, untyped]
+      def filtered_arguments
+        ParameterFilter.filter(arguments.to_h)
+      end
+
+      #:  () -> Array[Hash[Symbol, untyped]]
+      def tasks
+        @tasks ||= status.context.workflow.tasks.map { |task| task_view_model(task) }
+      end
+
+      #:  () -> Symbol?
+      def failed_task_name
+        @failed_task_name ||= begin
+          failed_task = tasks.find { |task| task[:status] == :failed }
+          failed_task&.fetch(:name)
+        end
+      end
+
+      #:  () -> String?
+      def mission_control_job_path
+        JobWorkflow::Monitoring.mission_control_job_path(job_id, status: workflow_status)
+      end
+
+      #:  () -> Hash[Symbol, untyped]
+      def dag_layout
+        @dag_layout ||= DagLayout.new(tasks:).to_h
+      end
+
+      #:  () -> bool
+      def running?
+        workflow_status == :running
+      end
+
+      #:  () -> Hash[Symbol, untyped]
+      def to_h
+        {
+          job_id:,
+          queue_name:,
+          job_class_name:,
+          status: workflow_status,
+          current_task_name:,
+          failed_task_name:,
+          arguments: filtered_arguments,
+          tasks:,
+          mission_control_job_path:
+        }
+      end
+
+      private
+
+      #:  (Symbol, Array[TaskOutput], Array[TaskJobStatus]) -> Symbol
+      def task_status(task_name, task_outputs, task_job_statuses)
+        return :failed if task_job_statuses.any?(&:failed?)
+        return :succeeded if completed_task?(task_name, task_outputs, task_job_statuses)
+        return :running if task_running?(task_name, task_job_statuses)
+
+        :pending
+      end
+
+      #:  (Symbol, Array[TaskOutput], Array[TaskJobStatus]) -> bool
+      def completed_task?(task_name, task_outputs, task_job_statuses)
+        return true if !running? && task_outputs.any?
+        return task_job_statuses.all?(&:succeeded?) if task_job_statuses.any?
+        return true if status.completed_task_names.include?(task_name)
+
+        task_outputs.any?
+      end
+
+      #:  (Symbol, Array[TaskJobStatus]) -> bool
+      def task_running?(task_name, task_job_statuses)
+        current_task_running?(task_name) ||
+          (!task_job_statuses.empty? && task_job_statuses.any? { |task_status| !task_status.finished? })
+      end
+
+      #:  (Symbol) -> bool
+      def current_task_running?(task_name) = running? && current_task_name == task_name
+
+      #:  (Task) -> Hash[Symbol, untyped]
+      def task_view_model(task)
+        task_name = task.task_name
+        task_outputs, task_job_statuses = task_state(task_name)
+
+        task_configuration_view(task).merge(
+          task_runtime_view(task_name, task_outputs, task_job_statuses)
+        )
+      end
+
+      #:  (Task) -> Hash[Symbol, untyped]
+      def task_configuration_view(task)
+        {
+          name: task.task_name,
+          depends_on: task.depends_on,
+          each: task.each?,
+          configuration: task_configuration(task)
+        }
+      end
+
+      #:  (Symbol, Array[TaskOutput], Array[TaskJobStatus]) -> Hash[Symbol, untyped]
+      def task_runtime_view(task_name, task_outputs, task_job_statuses)
+        {
+          status: task_status(task_name, task_outputs, task_job_statuses),
+          each_progress: each_progress(task_outputs, task_job_statuses),
+          outputs: task_outputs_view(task_outputs),
+          sub_task_jobs: sub_task_jobs_view(task_job_statuses)
+        }
+      end
+
+      #:  (Symbol) -> [Array[TaskOutput], Array[TaskJobStatus]]
+      def task_state(task_name) = [status.output.fetch_all(task_name:), status.job_status.fetch_all(task_name:)]
+
+      #:  (Task) -> Hash[Symbol, untyped]
+      def task_configuration(task)
+        {
+          job_name: task.job_name,
+          each: callable_summary(task.each),
+          condition: callable_summary(task.condition),
+          enqueue: enqueue_configuration(task),
+          outputs: output_configuration(task),
+          retry: retry_configuration(task),
+          throttle: throttle_configuration(task),
+          timeout: task.timeout,
+          dependency_wait: dependency_wait_configuration(task),
+          dry_run: callable_summary(task.dry_run_config.evaluator)
+        }
+      end
+
+      #:  (Task) -> Hash[Symbol, untyped]
+      def enqueue_configuration(task)
+        {
+          enabled: primitive_summary(task.enqueue.condition),
+          queue: task.enqueue.queue
+        }
+      end
+
+      #:  (Task) -> Array[Hash[Symbol, untyped]]
+      def output_configuration(task)
+        task.output.map { |output| { name: output.name, type: output.type } }
+      end
+
+      #:  (Task) -> Hash[Symbol, untyped]
+      def retry_configuration(task)
+        {
+          count: task.task_retry.count,
+          strategy: task.task_retry.strategy,
+          base_delay: task.task_retry.base_delay,
+          jitter: task.task_retry.jitter
+        }
+      end
+
+      #:  (Task) -> Hash[Symbol, untyped]
+      def throttle_configuration(task)
+        {
+          key: task.throttle.key,
+          limit: task.throttle.limit,
+          ttl: task.throttle.ttl
+        }
+      end
+
+      #:  (Task) -> Hash[Symbol, untyped]
+      def dependency_wait_configuration(task)
+        {
+          poll_timeout: task.dependency_wait.poll_timeout,
+          poll_interval: task.dependency_wait.poll_interval,
+          reschedule_delay: task.dependency_wait.reschedule_delay,
+          polling_only: task.dependency_wait.polling_only?
+        }
+      end
+
+      #:  (untyped) -> untyped
+      def callable_summary(value)
+        case value
+        when nil
+          nil
+        when Proc
+          "proc"
+        else
+          value
+        end
+      end
+
+      #:  (untyped) -> untyped
+      def primitive_summary(value)
+        value.is_a?(Proc) ? "proc" : value
+      end
+
+      #:  (Array[TaskOutput], Array[TaskJobStatus]) -> Hash[Symbol, Integer]
+      def each_progress(task_outputs, task_job_statuses)
+        {
+          total: [task_outputs.size, task_job_statuses.size].max,
+          succeeded: task_job_statuses.count(&:succeeded?),
+          failed: task_job_statuses.count(&:failed?),
+          pending: task_job_statuses.count { |task_status| task_status.status == :pending },
+          running: task_job_statuses.count { |task_status| task_status.status == :running }
+        }
+      end
+
+      #:  (Array[TaskOutput]) -> Array[Hash[Symbol, untyped]]
+      def task_outputs_view(task_outputs)
+        task_outputs.map do |output|
+          { each_index: output.each_index, data: ParameterFilter.filter(output.data) }
+        end
+      end
+
+      #:  (Array[TaskJobStatus]) -> Array[Hash[Symbol, untyped]]
+      def sub_task_jobs_view(task_job_statuses)
+        Array(task_job_statuses).map do |task_job_status|
+          {
+            job_id: task_job_status.job_id,
+            each_index: task_job_status.each_index,
+            status: task_job_status.status,
+            mission_control_job_path: mission_control_job_path_for(task_job_status.job_id, task_job_status.status)
+          }
+        end
+      end
+
+      #:  (String?, Symbol?) -> String?
+      def mission_control_job_path_for(job_id, status = nil)
+        JobWorkflow::Monitoring.mission_control_job_path(job_id, status:)
+      end
+    end
+  end
+end

data/lib/job_workflow/monitoring/parameter_filter.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require "active_support/parameter_filter"
+
+module JobWorkflow
+  module Monitoring
+    class ParameterFilter
+      class << self
+        #: (untyped) -> untyped
+        def filter(value)
+          case value
+          when Hash
+            parameter_filter.filter(value)
+          when Array
+            value.map { |item| filter(item) }
+          else
+            value
+          end
+        end
+
+        private
+
+        #: () -> ActiveSupport::ParameterFilter
+        def parameter_filter
+          ActiveSupport::ParameterFilter.new(filters)
+        end
+
+        #: () -> Array[untyped]
+        def filters
+          return [] unless defined?(Rails.application) && Rails.application.config.respond_to?(:filter_parameters)
+
+          Rails.application.config.filter_parameters
+        end
+      end
+    end
+  end
+end

data/lib/job_workflow/monitoring/workflow_definition.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module JobWorkflow
+  module Monitoring
+    class WorkflowDefinition
+      attr_reader :job_class #: singleton(DSL)
+
+      #:  (job_class: singleton(DSL)) -> void
+      def initialize(job_class:)
+        @job_class = job_class
+      end
+
+      #:  () -> String
+      def job_class_name
+        job_class.name
+      end
+
+      #:  () -> Integer
+      def task_count
+        job_class._workflow.tasks.size
+      end
+    end
+  end
+end

data/lib/job_workflow/monitoring/workflow_registry.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module JobWorkflow
+  module Monitoring
+    class WorkflowRegistry
+      class << self
+        #:  () -> Array[singleton(DSL)]
+        def all
+          DSL._included_classes.to_a
+             .reverse
+             .select(&:name)
+             .uniq(&:name)
+             .reverse
+             .sort_by(&:name)
+        end
+
+        #:  (String) -> singleton(DSL)?
+        def find(job_class_name)
+          all.find { |job_class| job_class.name == job_class_name }
+        end
+      end
+    end
+  end
+end

data/lib/job_workflow/monitoring.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+require_relative "monitoring/workflow_registry"
+require_relative "monitoring/workflow_definition"
+require_relative "monitoring/execution_page"
+require_relative "monitoring/dag_layout"
+require_relative "monitoring/parameter_filter"
+require_relative "monitoring/execution_view_model"
+require_relative "monitoring/execution_registry"
+
+module JobWorkflow
+  module Monitoring
+    mattr_accessor :base_controller_class
+
+    class << self
+      #:  () -> Array[WorkflowDefinition]
+      def workflows
+        WorkflowRegistry.all.map { |job_class| WorkflowDefinition.new(job_class:) }
+      end
+
+      #:  (String?, Symbol?) -> String?
+      def mission_control_job_path(job_id, status: nil)
+        return if job_id.nil?
+
+        path_template = mission_control_job_route_path
+        return if path_template.nil?
+
+        path = path_template.sub(":id", job_id.to_s)
+        path += "#error" if status == :failed
+        path
+      end
+
+      #:  () -> String
+      def resolved_base_controller_class
+        return base_controller_class if base_controller_class.present?
+
+        mission_control_base_controller_class = defined?(MissionControl::Jobs) &&
+                                                MissionControl::Jobs.base_controller_class
+        return mission_control_base_controller_class if mission_control_base_controller_class.present?
+
+        "::ApplicationController"
+      end
+
+      #:  (untyped config) -> void
+      def configure_engine_config(config)
+        config.job_workflow = ActiveSupport::OrderedOptions.new unless config.try(:job_workflow)
+        config.job_workflow.monitoring ||= ActiveSupport::OrderedOptions.new
+
+        config.before_initialize do
+          config.job_workflow.monitoring.each do |key, value|
+            JobWorkflow::Monitoring.public_send("#{key}=", value)
+          end
+        end
+      end
+
+      private
+
+      #:  () -> String?
+      def mission_control_job_route_path
+        application_id = mission_control_application_id
+        route_name = application_id ? "application_job" : "job"
+        mount_path = mission_control_mount_path
+        return if mount_path.nil?
+
+        path = engine_route_path(route_name)
+        return if path.nil?
+
+        combined_path = "#{mount_path.chomp("/")}#{path}"
+        application_id ? combined_path.sub(":application_id", application_id.to_s) : combined_path
+      end
+
+      #:  () -> String?
+      def mission_control_application_id
+        return unless defined?(MissionControl::Jobs)
+
+        MissionControl::Jobs.applications.first&.id
+      end
+
+      #:  () -> String?
+      def mission_control_mount_path
+        path = rails_routes&.find { |candidate| candidate.name == "mission_control_jobs" }&.path
+        normalized_route_path(path)
+      end
+
+      #:  (String) -> String?
+      def engine_route_path(route_name)
+        path = mission_control_routes&.find { |candidate| candidate.name == route_name }&.path
+        normalized_route_path(path)
+      end
+
+      #:  (untyped) -> String?
+      def normalized_route_path(path)
+        return unless path.respond_to?(:spec)
+
+        path_spec = path.spec.to_s
+        return unless path_spec.is_a?(String) && !path_spec.empty?
+
+        path_spec.delete_suffix("(.:format)")
+      end
+
+      #:  () -> untyped
+      def mission_control_routes
+        return unless defined?(MissionControl::Jobs::Engine)
+
+        MissionControl::Jobs::Engine.routes.routes
+      end
+
+      #:  () -> untyped
+      def rails_routes
+        return unless defined?(Rails.application)
+        return unless Rails.application.respond_to?(:routes)
+
+        routes = Rails.application.routes
+        return unless routes.respond_to?(:routes)
+
+        routes.routes
+      end
+    end
+  end
+end

data/lib/job_workflow/queue_adapters/abstract.rb

@@ -77,17 +77,22 @@ module JobWorkflow
         raise NotImplementedError, "#{self.class}#find_job must be implemented"
       end
 
+      #:  (job_class_name: String, limit: Integer, cursor: String?) -> Hash[Symbol, untyped]
+      def fetch_root_workflow_job_page(job_class_name:, limit:, cursor:)
+        raise NotImplementedError, "#{self.class}#fetch_root_workflow_job_page must be implemented"
+      end
+
       #:  (Array[String]) -> Array[Hash[String, untyped]]
       def fetch_job_contexts(_job_ids)
         raise NotImplementedError, "#{self.class}#fetch_job_contexts must be implemented"
       end
 
-      #:  (DSL, Numeric) -> bool
+      #:  (_JobInterface, Numeric) -> bool
       def reschedule_job(_job, _wait)
         false
       end
 
-      #:  (DSL) -> void
+      #:  (_JobInterface) -> void
       def persist_job_context(_job); end
     end
     # rubocop:enable Naming/PredicateMethod

data/lib/job_workflow/queue_adapters/null_adapter.rb

@@ -90,12 +90,23 @@ module JobWorkflow
         @stored_jobs[job_id]
       end
 
+      #:  (job_class_name: String, limit: Integer, cursor: String?) -> Hash[Symbol, untyped]
+      def fetch_root_workflow_job_page(job_class_name:, limit:, cursor:)
+        jobs = @stored_jobs.values.select { |job| job["class_name"] == job_class_name }.reverse
+        offset = cursor.nil? ? 0 : cursor.to_i
+        page_jobs = jobs.slice(offset, limit + 1) || []
+        {
+          jobs: page_jobs.first(limit),
+          next_cursor: page_jobs.size > limit ? (offset + limit).to_s : nil
+        }
+      end
+
       #:  (Array[String]) -> Array[Hash[String, untyped]]
       def fetch_job_contexts(_job_ids)
         []
       end
 
-      #:  (DSL, Numeric) -> bool
+      #:  (_JobInterface, Numeric) -> bool
       def reschedule_job(_job, _wait)
         false
       end

data/lib/job_workflow/queue_adapters/solid_queue_adapter.rb

@@ -160,15 +160,17 @@ module JobWorkflow
         job = without_query_cache { SolidQueue::Job.find_by(active_job_id: job_id) }
         return if job.nil?
 
-        args = job.arguments
-        {
-          "job_id" => job.active_job_id,
-          "class_name" => job.class_name,
-          "queue_name" => job.queue_name,
-          "arguments" => args.is_a?(Hash) ? args["arguments"] : args,
-          "job_workflow_context" => args.is_a?(Hash) ? args["job_workflow_context"] : nil,
-          "status" => job_status(job)
-        }
+        normalized_job_data(job)
+      end
+
+      #:  (job_class_name: String, limit: Integer, cursor: String?) -> Hash[Symbol, untyped]
+      def fetch_root_workflow_job_page(job_class_name:, limit:, cursor:)
+        return { jobs: [], next_cursor: nil } unless defined?(SolidQueue::Job)
+
+        without_query_cache do
+          jobs = root_jobs_relation(job_class_name:, cursor:).limit(limit + 1).to_a
+          build_page(jobs, limit:)
+        end
       end
 
       # @note
@@ -186,7 +188,7 @@ module JobWorkflow
         end
       end
 
-      #:  (DSL, Numeric) -> bool
+      #:  (_JobInterface, Numeric) -> bool
       def reschedule_job(job, wait)
         return false unless defined?(SolidQueue::Job)
 
@@ -204,7 +206,7 @@ module JobWorkflow
       #     outputs computed during job execution would be lost because
       #     SolidQueue does not re-serialize job arguments after perform.
       #
-      #:  (DSL) -> void
+      #:  (_JobInterface) -> void
       def persist_job_context(job)
         return unless defined?(SolidQueue::Job)
 
@@ -229,7 +231,35 @@ module JobWorkflow
         defined?(SolidQueue::Job) ? SolidQueue::Job.uncached(&) : yield
       end
 
-      #:  (SolidQueue::Job, DSL, Numeric) -> bool
+      #:  (SolidQueue::Job) -> Hash[String, untyped]
+      def normalized_job_data(job)
+        args = job.arguments
+        {
+          "job_id" => job.active_job_id,
+          "class_name" => job.class_name,
+          "queue_name" => job.queue_name,
+          "arguments" => args.is_a?(Hash) ? args["arguments"] : args,
+          "job_workflow_context" => args.is_a?(Hash) ? args["job_workflow_context"] : nil,
+          "status" => job_status(job)
+        }
+      end
+
+      #:  (job_class_name: String, cursor: String?) -> untyped
+      def root_jobs_relation(job_class_name:, cursor:)
+        relation = SolidQueue::Job.where(class_name: job_class_name)
+        relation = relation.where("id < ?", cursor.to_i) unless cursor.nil?
+        relation.order(id: :desc)
+      end
+
+      #:  (Array[SolidQueue::Job], limit: Integer) -> Hash[Symbol, untyped]
+      def build_page(jobs, limit:)
+        {
+          jobs: jobs.first(limit).map { |job| normalized_job_data(job) },
+          next_cursor: jobs.size > limit ? jobs[limit - 1].id.to_s : nil
+        }
+      end
+
+      #:  (SolidQueue::Job, _JobInterface, Numeric) -> bool
       def reschedule_solid_queue_job(solid_queue_job, active_job, wait)
         solid_queue_job.with_lock do
           solid_queue_job.claimed_execution&.destroy!

data/lib/job_workflow/railtie.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative "monitoring/engine"
+
 module JobWorkflow
   class Railtie < Rails::Railtie
     config.after_initialize do

data/lib/job_workflow/runner.rb

@@ -25,11 +25,11 @@ module JobWorkflow
 
     private
 
-    attr_reader :job #: DSL
+    attr_reader :job #: _JobInterface
 
     #:  () -> Workflow
     def workflow
-      job.class._workflow
+      context.workflow
     end
 
     #:  () -> Array[Task]
@@ -133,7 +133,9 @@ module JobWorkflow
 
     #:  (Task) -> void
     def enqueue_task(task)
-      sub_jobs = context._with_each_value(task).map { |ctx| job.class.from_context(ctx) }
+      sub_jobs = context._with_each_value(task).map do |ctx|
+        SubTaskJob.from_parent_context(context: ctx)
+      end
       persist_current_job_context
       ActiveJob.perform_all_later(sub_jobs)
       context.job_status.update_task_job_statuses_from_jobs(task_name: task.task_name, jobs: sub_jobs)