job-workflow 0.4.0 → 0.6.0

data/lib/job_workflow/monitoring/parameter_filter.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require "active_support/parameter_filter"
+
+module JobWorkflow
+  module Monitoring
+    class ParameterFilter
+      class << self
+        #: (untyped) -> untyped
+        def filter(value)
+          case value
+          when Hash
+            parameter_filter.filter(value)
+          when Array
+            value.map { |item| filter(item) }
+          else
+            value
+          end
+        end
+
+        private
+
+        #: () -> ActiveSupport::ParameterFilter
+        def parameter_filter
+          ActiveSupport::ParameterFilter.new(filters)
+        end
+
+        #: () -> Array[untyped]
+        def filters
+          return [] unless defined?(Rails.application) && Rails.application.config.respond_to?(:filter_parameters)
+
+          Rails.application.config.filter_parameters
+        end
+      end
+    end
+  end
+end

data/lib/job_workflow/monitoring/workflow_definition.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module JobWorkflow
+  module Monitoring
+    class WorkflowDefinition
+      attr_reader :job_class #: singleton(DSL)
+
+      #:  (job_class: singleton(DSL)) -> void
+      def initialize(job_class:)
+        @job_class = job_class
+      end
+
+      #:  () -> String
+      def job_class_name
+        job_class.name
+      end
+
+      #:  () -> Integer
+      def task_count
+        job_class._workflow.tasks.size
+      end
+    end
+  end
+end

data/lib/job_workflow/monitoring/workflow_registry.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module JobWorkflow
+  module Monitoring
+    class WorkflowRegistry
+      class << self
+        #:  () -> Array[singleton(DSL)]
+        def all
+          DSL._included_classes.to_a
+             .reverse
+             .select(&:name)
+             .uniq(&:name)
+             .reverse
+             .sort_by(&:name)
+        end
+
+        #:  (String) -> singleton(DSL)?
+        def find(job_class_name)
+          all.find { |job_class| job_class.name == job_class_name }
+        end
+      end
+    end
+  end
+end

data/lib/job_workflow/monitoring.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+require_relative "monitoring/workflow_registry"
+require_relative "monitoring/workflow_definition"
+require_relative "monitoring/execution_page"
+require_relative "monitoring/dag_layout"
+require_relative "monitoring/parameter_filter"
+require_relative "monitoring/execution_view_model"
+require_relative "monitoring/execution_registry"
+
+module JobWorkflow
+  module Monitoring
+    mattr_accessor :base_controller_class
+
+    class << self
+      #:  () -> Array[WorkflowDefinition]
+      def workflows
+        WorkflowRegistry.all.map { |job_class| WorkflowDefinition.new(job_class:) }
+      end
+
+      #:  (String?, Symbol?) -> String?
+      def mission_control_job_path(job_id, status: nil)
+        return if job_id.nil?
+
+        path_template = mission_control_job_route_path
+        return if path_template.nil?
+
+        path = path_template.sub(":id", job_id.to_s)
+        path += "#error" if status == :failed
+        path
+      end
+
+      #:  () -> String
+      def resolved_base_controller_class
+        return base_controller_class if base_controller_class.present?
+
+        mission_control_base_controller_class = defined?(MissionControl::Jobs) &&
+                                                MissionControl::Jobs.base_controller_class
+        return mission_control_base_controller_class if mission_control_base_controller_class.present?
+
+        "::ApplicationController"
+      end
+
+      #:  (untyped config) -> void
+      def configure_engine_config(config)
+        config.job_workflow = ActiveSupport::OrderedOptions.new unless config.try(:job_workflow)
+        config.job_workflow.monitoring ||= ActiveSupport::OrderedOptions.new
+
+        config.before_initialize do
+          config.job_workflow.monitoring.each do |key, value|
+            JobWorkflow::Monitoring.public_send("#{key}=", value)
+          end
+        end
+      end
+
+      private
+
+      #:  () -> String?
+      def mission_control_job_route_path
+        application_id = mission_control_application_id
+        route_name = application_id ? "application_job" : "job"
+        mount_path = mission_control_mount_path
+        return if mount_path.nil?
+
+        path = engine_route_path(route_name)
+        return if path.nil?
+
+        combined_path = "#{mount_path.chomp("/")}#{path}"
+        application_id ? combined_path.sub(":application_id", application_id.to_s) : combined_path
+      end
+
+      #:  () -> String?
+      def mission_control_application_id
+        return unless defined?(MissionControl::Jobs)
+
+        MissionControl::Jobs.applications.first&.id
+      end
+
+      #:  () -> String?
+      def mission_control_mount_path
+        path = rails_routes&.find { |candidate| candidate.name == "mission_control_jobs" }&.path
+        normalized_route_path(path)
+      end
+
+      #:  (String) -> String?
+      def engine_route_path(route_name)
+        path = mission_control_routes&.find { |candidate| candidate.name == route_name }&.path
+        normalized_route_path(path)
+      end
+
+      #:  (untyped) -> String?
+      def normalized_route_path(path)
+        return unless path.respond_to?(:spec)
+
+        path_spec = path.spec.to_s
+        return unless path_spec.is_a?(String) && !path_spec.empty?
+
+        path_spec.delete_suffix("(.:format)")
+      end
+
+      #:  () -> untyped
+      def mission_control_routes
+        return unless defined?(MissionControl::Jobs::Engine)
+
+        MissionControl::Jobs::Engine.routes.routes
+      end
+
+      #:  () -> untyped
+      def rails_routes
+        return unless defined?(Rails.application)
+        return unless Rails.application.respond_to?(:routes)
+
+        routes = Rails.application.routes
+        return unless routes.respond_to?(:routes)
+
+        routes.routes
+      end
+    end
+  end
+end

data/lib/job_workflow/queue_adapters/abstract.rb

@@ -77,17 +77,22 @@ module JobWorkflow
         raise NotImplementedError, "#{self.class}#find_job must be implemented"
       end
 
+      #:  (job_class_name: String, limit: Integer, cursor: String?) -> Hash[Symbol, untyped]
+      def fetch_root_workflow_job_page(job_class_name:, limit:, cursor:)
+        raise NotImplementedError, "#{self.class}#fetch_root_workflow_job_page must be implemented"
+      end
+
       #:  (Array[String]) -> Array[Hash[String, untyped]]
       def fetch_job_contexts(_job_ids)
         raise NotImplementedError, "#{self.class}#fetch_job_contexts must be implemented"
       end
 
-      #:  (DSL, Numeric) -> bool
+      #:  (_JobInterface, Numeric) -> bool
       def reschedule_job(_job, _wait)
         false
       end
 
-      #:  (DSL) -> void
+      #:  (_JobInterface) -> void
       def persist_job_context(_job); end
     end
     # rubocop:enable Naming/PredicateMethod

data/lib/job_workflow/queue_adapters/null_adapter.rb

@@ -90,12 +90,23 @@ module JobWorkflow
         @stored_jobs[job_id]
       end
 
+      #:  (job_class_name: String, limit: Integer, cursor: String?) -> Hash[Symbol, untyped]
+      def fetch_root_workflow_job_page(job_class_name:, limit:, cursor:)
+        jobs = @stored_jobs.values.select { |job| job["class_name"] == job_class_name }.reverse
+        offset = cursor.nil? ? 0 : cursor.to_i
+        page_jobs = jobs.slice(offset, limit + 1) || []
+        {
+          jobs: page_jobs.first(limit),
+          next_cursor: page_jobs.size > limit ? (offset + limit).to_s : nil
+        }
+      end
+
       #:  (Array[String]) -> Array[Hash[String, untyped]]
       def fetch_job_contexts(_job_ids)
         []
       end
 
-      #:  (DSL, Numeric) -> bool
+      #:  (_JobInterface, Numeric) -> bool
       def reschedule_job(_job, _wait)
         false
       end

data/lib/job_workflow/queue_adapters/solid_queue_adapter.rb

@@ -160,15 +160,17 @@ module JobWorkflow
         job = without_query_cache { SolidQueue::Job.find_by(active_job_id: job_id) }
         return if job.nil?
 
-        args = job.arguments
-        {
-          "job_id" => job.active_job_id,
-          "class_name" => job.class_name,
-          "queue_name" => job.queue_name,
-          "arguments" => args.is_a?(Hash) ? args["arguments"] : args,
-          "job_workflow_context" => args.is_a?(Hash) ? args["job_workflow_context"] : nil,
-          "status" => job_status(job)
-        }
+        normalized_job_data(job)
+      end
+
+      #:  (job_class_name: String, limit: Integer, cursor: String?) -> Hash[Symbol, untyped]
+      def fetch_root_workflow_job_page(job_class_name:, limit:, cursor:)
+        return { jobs: [], next_cursor: nil } unless defined?(SolidQueue::Job)
+
+        without_query_cache do
+          jobs = root_jobs_relation(job_class_name:, cursor:).limit(limit + 1).to_a
+          build_page(jobs, limit:)
+        end
       end
 
       # @note
@@ -186,7 +188,7 @@ module JobWorkflow
         end
       end
 
-      #:  (DSL, Numeric) -> bool
+      #:  (_JobInterface, Numeric) -> bool
       def reschedule_job(job, wait)
         return false unless defined?(SolidQueue::Job)
 
@@ -204,7 +206,7 @@ module JobWorkflow
       #     outputs computed during job execution would be lost because
       #     SolidQueue does not re-serialize job arguments after perform.
       #
-      #:  (DSL) -> void
+      #:  (_JobInterface) -> void
       def persist_job_context(job)
         return unless defined?(SolidQueue::Job)
 
@@ -229,7 +231,35 @@ module JobWorkflow
         defined?(SolidQueue::Job) ? SolidQueue::Job.uncached(&) : yield
       end
 
-      #:  (SolidQueue::Job, DSL, Numeric) -> bool
+      #:  (SolidQueue::Job) -> Hash[String, untyped]
+      def normalized_job_data(job)
+        args = job.arguments
+        {
+          "job_id" => job.active_job_id,
+          "class_name" => job.class_name,
+          "queue_name" => job.queue_name,
+          "arguments" => args.is_a?(Hash) ? args["arguments"] : args,
+          "job_workflow_context" => args.is_a?(Hash) ? args["job_workflow_context"] : nil,
+          "status" => job_status(job)
+        }
+      end
+
+      #:  (job_class_name: String, cursor: String?) -> untyped
+      def root_jobs_relation(job_class_name:, cursor:)
+        relation = SolidQueue::Job.where(class_name: job_class_name)
+        relation = relation.where("id < ?", cursor.to_i) unless cursor.nil?
+        relation.order(id: :desc)
+      end
+
+      #:  (Array[SolidQueue::Job], limit: Integer) -> Hash[Symbol, untyped]
+      def build_page(jobs, limit:)
+        {
+          jobs: jobs.first(limit).map { |job| normalized_job_data(job) },
+          next_cursor: jobs.size > limit ? jobs[limit - 1].id.to_s : nil
+        }
+      end
+
+      #:  (SolidQueue::Job, _JobInterface, Numeric) -> bool
       def reschedule_solid_queue_job(solid_queue_job, active_job, wait)
         solid_queue_job.with_lock do
           solid_queue_job.claimed_execution&.destroy!

data/lib/job_workflow/railtie.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require_relative "monitoring/engine"
+
+module JobWorkflow
+  class Railtie < Rails::Railtie
+    config.after_initialize do
+      JobWorkflow::QueueAdapter.reset!
+      JobWorkflow::QueueAdapter.current.initialize_adapter!
+    end
+  end
+end

data/lib/job_workflow/runner.rb

@@ -14,7 +14,7 @@ module JobWorkflow
     def run
       task = context._task_context.task
       if !task.nil? && context.sub_job?
-        run_task(task)
+        job.step(task.task_name) { |step| run_task(task, step:) }
         persist_current_job_context
         return
       end
@@ -25,11 +25,11 @@ module JobWorkflow
 
     private
 
-    attr_reader :job #: DSL
+    attr_reader :job #: _JobInterface
 
     #:  () -> Workflow
     def workflow
-      job.class._workflow
+      context.workflow
     end
 
     #:  () -> Array[Task]
@@ -63,25 +63,46 @@ module JobWorkflow
       result
     end
 
-    #:  (Task, ?step: ActiveJob::Continuation::Step?) -> void
-    def run_task(task, step: nil)
+    #:  (Task, step: ActiveJob::Continuation::Step) -> void
+    def run_task(task, step:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
       context._load_parent_task_output
-      context._with_each_value(task, start_index: step&.cursor).each do |ctx|
-        run_each_task(task, ctx)
-        step&.set!(ctx._task_context.index + 1)
+      start_index, task_cursor = decode_task_cursor(task, step.cursor)
+
+      context._with_each_value(task, start_index:).each do |ctx|
+        iteration_cursor = task_cursor
+        iteration_cursor = nil if task.each? && (task_cursor.nil? || start_index != ctx._task_context.index)
+
+        run_each_task(task, ctx, step:, cursor: iteration_cursor)
+        step.set!(ctx._task_context.index + 1) if task.each?
       rescue StandardError => e
         run_error_hooks(task, ctx, e)
         raise
       end
     end
 
-    #:  (Task, Context) -> void
-    def run_each_task(task, ctx)
+    #:  (Task, untyped) -> [Integer?, untyped]
+    def decode_task_cursor(task, task_cursor)
+      return [nil, task_cursor] unless task.each?
+      return [task_cursor, nil] if task_cursor.is_a?(Integer)
+
+      if task_cursor.is_a?(Hash) && task_cursor[Context::EACH_TASK_CURSOR_MARKER]
+        return [task_cursor.fetch("index"), task_cursor.fetch("cursor")]
+      end
+
+      raise "invalid each task cursor: #{task_cursor.inspect}" unless task_cursor.nil?
+
+      [nil, nil]
+    end
+
+    #:  (Task, Context, step: ActiveJob::Continuation::Step, ?cursor: untyped) -> void
+    def run_each_task(task, ctx, step:, cursor: nil)
       Instrumentation.instrument_task(job, task, ctx) do
-        ctx._with_task_throttle do
-          run_hooks(task, ctx) do
-            data = task.block.call(ctx)
-            add_task_output(ctx:, task:, each_index: ctx._task_context.index, data:)
+        ctx._with_current_step(step, cursor:) do
+          ctx._with_task_throttle do
+            run_hooks(task, ctx) do
+              data = task.block.call(ctx)
+              add_task_output(ctx:, task:, each_index: ctx._task_context.index, data:)
+            end
           end
         end
       end
@@ -112,7 +133,9 @@ module JobWorkflow
 
     #:  (Task) -> void
     def enqueue_task(task)
-      sub_jobs = context._with_each_value(task).map { |ctx| job.class.from_context(ctx) }
+      sub_jobs = context._with_each_value(task).map do |ctx|
+        SubTaskJob.from_parent_context(context: ctx)
+      end
       persist_current_job_context
       ActiveJob.perform_all_later(sub_jobs)
       context.job_status.update_task_job_statuses_from_jobs(task_name: task.task_name, jobs: sub_jobs)

data/lib/job_workflow/sub_task_job.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+module JobWorkflow
+  class SubTaskJob < ActiveJob::Base
+    include ActiveJob::Continuable
+
+    class << self
+      #:  (context: Context) -> SubTaskJob
+      def from_parent_context(context:)
+        validate_sub_task_context!(context)
+
+        new_context = context.dup
+        job = new(context.arguments.to_h)
+        new_context._job = job
+        job._context = new_context
+        task = new_context._task_context.task || (raise "task is not set")
+        return job if task.enqueue.queue.nil?
+
+        job.set(queue: task.enqueue.queue)
+      end
+
+      private
+
+      #:  (Context) -> void
+      def validate_sub_task_context!(context)
+        task_context = context._task_context
+        raise ArgumentError, "task_context.task is required" if task_context.task.nil?
+        raise ArgumentError, "task_context.parent_job_id is required" if task_context.parent_job_id.nil?
+      end
+    end
+
+    #:  (Hash[untyped, untyped]) -> void
+    def perform(arguments)
+      payload = arguments.symbolize_keys
+      self._context = build_context(payload)
+      Runner.new(context: _context).run
+    end
+
+    #:  () -> Output
+    def output
+      context = _context
+      raise "context is not set." if context.nil?
+
+      context.output
+    end
+
+    attr_accessor :_context #: Context?
+
+    #:  () -> Hash[String, untyped]
+    def serialize
+      super.merge({ "job_workflow_context" => _context&.serialize || serialized_job_workflow_context }.compact)
+    end
+
+    #:  (Hash[String, untyped]) -> void
+    def deserialize(job_data)
+      super
+      self.serialized_job_workflow_context = job_data["job_workflow_context"]
+    end
+
+    private
+
+    attr_accessor :serialized_job_workflow_context #: Hash[String, untyped]?
+
+    #:  (Hash[Symbol, untyped]) -> Context
+    def build_context(payload)
+      context_data = extract_context_data(payload)
+      parent_job_id = context_data.fetch("task_context").fetch("parent_job_id")
+      parent_job_data = QueueAdapter.current.find_job(parent_job_id)
+      raise WorkflowStatus::NotFoundError, "Workflow with job_id '#{parent_job_id}' not found" if parent_job_data.nil?
+
+      workflow = resolve_workflow(parent_job_data.fetch("class_name"))
+      Context.deserialize(context_data.merge("job" => self, "workflow" => workflow))
+             ._update_arguments(payload.except(:job_workflow_context))
+    end
+
+    #:  (job_class_name: String) -> Workflow
+    def resolve_workflow(job_class_name)
+      job_class = JobWorkflow::DSL._included_classes.to_a.reverse.find { |klass| klass.name == job_class_name }
+      job_class ||= job_class_name.safe_constantize
+      raise NameError, "uninitialized constant #{job_class_name}" if job_class.nil?
+
+      job_class._workflow
+    end
+
+    #:  (Hash[Symbol, untyped]) -> Hash[String, untyped]
+    def extract_context_data(payload)
+      context_data = serialized_job_workflow_context || payload[:job_workflow_context]
+      raise "job_workflow_context is not set." if context_data.nil?
+
+      context_data.deep_stringify_keys
+    end
+  end
+end

data/lib/job_workflow/task.rb

@@ -2,6 +2,8 @@
 
 module JobWorkflow
   class Task
+    DEFAULT_EACH = ->(_ctx) { [nil] }
+
     attr_reader :job_name #: String
     attr_reader :block #: ^(untyped) -> void
     attr_reader :each #: ^(Context) -> untyped
@@ -72,6 +74,11 @@ module JobWorkflow
       "#{job_name}:#{task_name}"
     end
 
+    #:  () -> bool
+    def each?
+      !each.nil? && !each.equal?(DEFAULT_EACH)
+    end
+
     private
 
     attr_reader :name #: Symbol

data/lib/job_workflow/task_enqueue.rb

@@ -4,7 +4,6 @@ module JobWorkflow
   class TaskEnqueue
     attr_reader :condition #: true | false | ^(Context) -> bool
     attr_reader :queue #: String?
-    attr_reader :concurrency #: Integer?
 
     class << self
       #:  (true | false | ^(Context) -> bool | Hash[Symbol, untyped] | nil) -> TaskEnqueue
@@ -13,26 +12,39 @@ module JobWorkflow
         when TrueClass, FalseClass, Proc
           new(condition: value)
         when Hash
+          validate_hash_keys!(value)
+
           new(
             condition: value.fetch(:condition, !value.empty?),
-            queue: value[:queue],
-            concurrency: value[:concurrency]
+            queue: value[:queue]
           )
         else
           new
         end
       end
+
+      private
+
+      #: (Hash[Symbol, untyped]) -> void
+      def validate_hash_keys!(value)
+        unsupported_keys = value.keys - %i[condition queue]
+        return if unsupported_keys.empty?
+
+        if unsupported_keys == %i[concurrency]
+          raise ArgumentError, "enqueue does not support :concurrency; use throttle instead"
+        end
+
+        raise ArgumentError, "enqueue supports only :condition and :queue"
+      end
     end
 
     #:  (
     #     ?condition: true | false | ^(Context) -> bool,
-    #     ?queue: String?,
-    #     ?concurrency: Integer?
+    #     ?queue: String?
     #   ) -> void
-    def initialize(condition: false, queue: nil, concurrency: nil)
+    def initialize(condition: false, queue: nil)
       @condition = condition
       @queue = queue
-      @concurrency = concurrency
     end
 
     #:  (Context) -> bool
@@ -41,10 +53,5 @@ module JobWorkflow
 
       !!condition
     end
-
-    #:  () -> bool
-    def should_limits_concurrency?
-      !!condition && !concurrency.nil? && QueueAdapter.current.supports_concurrency_limits?
-    end
   end
 end

data/lib/job_workflow/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module JobWorkflow
-  VERSION = "0.4.0" # : String
+  VERSION = "0.6.0" # : String
 end

data/lib/job_workflow/workflow_status.rb

@@ -24,6 +24,7 @@ module JobWorkflow
       def find_by(job_id:)
         data = QueueAdapter.current.find_job(job_id)
         return if data.nil?
+        return if data["class_name"] == JobWorkflow::SubTaskJob.name
 
         WorkflowStatus.from_job_data(data)
       end
@@ -33,15 +34,33 @@ module JobWorkflow
         job_class_name = data["class_name"]
         job_class = job_class_name.constantize
         workflow = job_class._workflow
+        context = context_from_job_data(data, workflow)
 
+        new(context:, job_class_name:, status: data["status"])
+      end
+
+      private
+
+      #:  (Hash[String, untyped], Workflow) -> Context
+      def context_from_job_data(data, workflow)
         context_data = data["job_workflow_context"] || data["arguments"]&.first&.dig("job_workflow_context")
         context = if context_data
                     Context.deserialize(context_data.merge("workflow" => workflow))
                   else
                     Context.from_hash({ workflow: })
                   end
+        serialized_arguments = workflow_arguments_data(data)
+        return context if serialized_arguments.nil?
 
-        new(context:, job_class_name:, status: data["status"])
+        context._update_arguments(ActiveJob::Arguments.deserialize([serialized_arguments]).first)
+      end
+
+      #:  (Hash[String, untyped]) -> Hash[String, untyped]?
+      def workflow_arguments_data(data)
+        serialized_arguments = data["arguments"]&.first
+        return if serialized_arguments.nil? || serialized_arguments.key?("job_workflow_context")
+
+        serialized_arguments
       end
     end