good_pipeline 0.3.0 → 0.4.0

data/demo/test/integration/test_halt_execution.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class TestHaltExecution < ActiveSupport::TestCase
+  def test_halt_pipeline_marks_step_halted
+    pipeline_class = Class.new(GoodPipeline::Pipeline) do
+      failure_strategy :halt
+      define_method(:configure) do |**|
+        run :halt_step, HaltExecutionJob
+        run :after_step, DownloadJob, after: :halt_step
+      end
+    end
+    Object.const_set(:HaltSucceededPipeline, pipeline_class) unless defined?(::HaltSucceededPipeline)
+
+    chain = HaltSucceededPipeline.run
+    result = run_pipeline_to_completion(chain)
+
+    halt_step = result.steps.find_by(key: "halt_step")
+    assert_equal "halted", halt_step.coordination_status
+    assert halt_step.halt_requested?, "halt_requested should be true"
+  end
+
+  def test_halt_pipeline_skips_remaining_steps
+    pipeline_class = Class.new(GoodPipeline::Pipeline) do
+      failure_strategy :halt
+      define_method(:configure) do |**|
+        run :halt_step, HaltExecutionJob
+        run :after_step, DownloadJob, after: :halt_step
+      end
+    end
+    Object.const_set(:HaltSkipsPipeline, pipeline_class) unless defined?(::HaltSkipsPipeline)
+
+    chain = HaltSkipsPipeline.run
+    result = run_pipeline_to_completion(chain)
+
+    after_step = result.steps.find_by(key: "after_step")
+    assert_equal "skipped", after_step.coordination_status
+  end
+
+  def test_halt_pipeline_pipeline_succeeds
+    pipeline_class = Class.new(GoodPipeline::Pipeline) do
+      failure_strategy :halt
+      define_method(:configure) do |**|
+        run :halt_step, HaltExecutionJob
+        run :after_step, DownloadJob, after: :halt_step
+      end
+    end
+    Object.const_set(:HaltSucceedsPipeline, pipeline_class) unless defined?(::HaltSucceedsPipeline)
+
+    chain = HaltSucceedsPipeline.run
+    result = run_pipeline_to_completion(chain)
+
+    assert_equal "succeeded", result.status
+  end
+
+  def test_halt_pipeline_job_succeeds_in_good_job
+    pipeline_class = Class.new(GoodPipeline::Pipeline) do
+      failure_strategy :halt
+      define_method(:configure) do |**|
+        run :halt_step, HaltExecutionJob
+      end
+    end
+    Object.const_set(:HaltJobSucceedsPipeline, pipeline_class) unless defined?(::HaltJobSucceedsPipeline)
+
+    chain = HaltJobSucceedsPipeline.run
+    run_pipeline_to_completion(chain)
+
+    halt_step = chain.steps.find_by(key: "halt_step")
+    good_job = GoodJob::Job.find(halt_step.good_job_id)
+
+    assert_equal 1, good_job.executions_count
+    assert_nil good_job.error, "GoodJob record should have no error"
+    assert_not_nil good_job.finished_at
+  end
+
+  def test_halt_pipeline_with_parallel_steps
+    pipeline_class = Class.new(GoodPipeline::Pipeline) do
+      failure_strategy :continue
+      define_method(:configure) do |**|
+        run :halt_step, HaltExecutionJob
+        run :normal_step, DownloadJob
+        run :after_both, CleanupJob, after: %i[halt_step normal_step]
+      end
+    end
+    Object.const_set(:HaltParallelPipeline, pipeline_class) unless defined?(::HaltParallelPipeline)
+
+    chain = HaltParallelPipeline.run
+    result = run_pipeline_to_completion(chain)
+
+    halt_step = result.steps.find_by(key: "halt_step")
+    normal_step = result.steps.find_by(key: "normal_step")
+    after_both = result.steps.find_by(key: "after_both")
+
+    assert_equal "halted", halt_step.coordination_status
+    assert_equal "succeeded", normal_step.coordination_status
+    assert_equal "skipped", after_both.coordination_status
+    assert_equal "succeeded", result.status
+  end
+end

data/demo/test/integration/test_queue_configuration.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class TestQueueConfigurationEndToEnd < ActiveSupport::TestCase
+  teardown do
+    GoodPipeline.coordination_queue_name = nil
+    GoodPipeline.callback_queue_name = nil
+  end
+
+  def test_full_pipeline_with_custom_queues
+    klass = Class.new(GoodPipeline::Pipeline) do
+      failure_strategy :halt
+      coordination_queue_name "e2e_coordination"
+      callback_queue_name "e2e_callbacks"
+
+      define_method(:configure) do |**_kwargs|
+        run :step_a, DownloadJob
+        run :step_b, TranscodeJob
+        run :step_c, PublishJob, after: %i[step_a step_b]
+      end
+    end
+    klass.define_singleton_method(:name) { "QueueE2ETestPipeline" }
+    Object.const_set(:QueueE2ETestPipeline, klass) unless defined?(::QueueE2ETestPipeline)
+
+    pipeline_record = QueueE2ETestPipeline.run
+
+    # Verify step batch queue names
+    pipeline_record.steps.each do |step|
+      next unless step.good_job_batch_id
+
+      batch_record = GoodJob::BatchRecord.find(step.good_job_batch_id)
+
+      assert_equal "e2e_coordination", batch_record.callback_queue_name,
+                   "Step #{step.key} batch should use coordination queue"
+    end
+
+    # Verify pipeline batch queue name
+    actual_record = GoodPipeline::PipelineRecord.find(pipeline_record.id)
+    pipeline_batch = GoodJob::BatchRecord.find(actual_record.good_job_batch_id)
+
+    assert_equal "e2e_coordination", pipeline_batch.callback_queue_name
+
+    # Run to completion and verify callback job queue
+    result = run_pipeline_to_completion(pipeline_record)
+
+    assert_equal "succeeded", result.status
+
+    callback_job = GoodJob::Job.where(job_class: "GoodPipeline::PipelineCallbackJob").last
+
+    assert_equal "e2e_callbacks", callback_job.queue_name
+  end
+
+  def test_global_config_applies_when_no_dsl
+    GoodPipeline.coordination_queue_name = "global_coord"
+    GoodPipeline.callback_queue_name = "global_cb"
+
+    klass = Class.new(GoodPipeline::Pipeline) do
+      failure_strategy :halt
+      define_method(:configure) do |**_kwargs|
+        run :step_a, DownloadJob
+      end
+    end
+    klass.define_singleton_method(:name) { "GlobalQueueTestPipeline" }
+    Object.const_set(:GlobalQueueTestPipeline, klass) unless defined?(::GlobalQueueTestPipeline)
+
+    pipeline_record = GlobalQueueTestPipeline.run
+
+    step = pipeline_record.steps.first
+    step_batch = GoodJob::BatchRecord.find(step.good_job_batch_id)
+
+    assert_equal "global_coord", step_batch.callback_queue_name
+
+    result = run_pipeline_to_completion(pipeline_record)
+
+    assert_equal "succeeded", result.status
+
+    callback_job = GoodJob::Job.where(job_class: "GoodPipeline::PipelineCallbackJob").last
+
+    assert_equal "global_cb", callback_job.queue_name
+  end
+end

data/demo/test/test_helper.rb

@@ -33,6 +33,21 @@ module ActiveSupport
       GoodJob.perform_inline
     end
 
+    def run_pipeline_to_completion(pipeline_record, timeout: 15)
+      deadline = Time.current + timeout
+      loop do
+        perform_enqueued_jobs_inline
+        pipeline_record.reload
+        return pipeline_record if pipeline_record.terminal?
+
+        if Time.current > deadline
+          raise "Pipeline did not reach terminal state within #{timeout}s (status: #{pipeline_record.status})"
+        end
+
+        sleep 0.05
+      end
+    end
+
     def wait_until(timeout: 10, interval: 0.1)
       deadline = Time.current + timeout
       loop do

data/docs/architecture.md

@@ -180,3 +180,11 @@ This ensures a step is never prematurely marked `failed` on attempt 1 of 5.
 7. Separate atomic units per transaction boundary to minimize lock contention
 8. DAG validation runs at instantiation, before any database writes
 9. `failure_strategy` and `on_failure` are distinct concepts -- strategy vs. callback, no naming collision
+
+## Why these tradeoffs
+
+GoodPipeline is intentionally GoodJob-specific and Postgres-only. This is what enables atomic enqueue transactions — step status transitions and GoodJob record inserts happen in a single database transaction, eliminating an entire class of partial-state bugs that adapter-agnostic gems must work around.
+
+The DAG execution model (vs. strictly sequential steps) adds coordination complexity — row locks, atomic counters, fan-in race prevention — but unlocks parallel execution of independent steps. For workflows where steps have no dependency on each other, this means wall-clock time is bounded by the longest path through the graph, not the sum of all steps.
+
+The four-table data model (pipelines, steps, dependencies, chains) is more tables than a two-table approach, but dedicated dependency and chain tables enable efficient graph queries and keep the step table free of self-referential joins.

data/docs/branching.md

@@ -2,6 +2,8 @@
 
 Conditional branching lets a pipeline take different paths at runtime based on application state. The dashboard renders branches as diamond decision nodes.
 
+Runtime branching with `branch` blocks is uncommon among Ruby workflow gems — most offer only `skip_if` conditions on individual steps. GoodPipeline's branching evaluates a decision method when the branch is reached, runs the matching arm, marks non-matching arms as `skipped_by_branch`, and lets downstream steps wait on whichever arm was chosen.
+
 ## Defining a branch
 
 Use `branch` inside `configure` to define a decision point. The `by:` option names a method on your pipeline class that returns the arm to execute:

data/docs/callbacks.md

@@ -46,6 +46,8 @@ Note: `on_failure` does **not** fire for `skipped` pipelines. Being skipped by a
 
 Callbacks are dispatched via `PipelineCallbackJob`, a GoodJob job enqueued after the terminal state transaction commits. A slow external call (Slack, webhooks) cannot stall the coordinator, callback execution cannot corrupt pipeline state, and callbacks get GoodJob's retry mechanism if they fail.
 
+`PipelineCallbackJob` runs on the queue configured by `callback_queue_name` (default: `"good_pipeline_callbacks"`). This is separate from `coordination_queue_name` which controls the coordination jobs (`StepFinishedJob`, `PipelineReconciliationJob`), so slow callbacks don't block pipeline progression. See [Defining Pipelines](/defining-pipelines) for configuration options.
+
 ## Exactly-once guarantee
 
 The callback bundle (`on_complete` + one of `on_success`/`on_failure`) is dispatched as a **single unit**. A `callbacks_dispatched_at` timestamp is set atomically inside a `FOR UPDATE` locked transaction, ensuring the bundle fires exactly once even if `recompute_pipeline_status` is called from multiple code paths (coordinator or batch reconciliation).

data/docs/defining-pipelines.md

@@ -8,6 +8,8 @@ Every pipeline is a subclass of `GoodPipeline::Pipeline` that implements `config
 class VideoProcessingPipeline < GoodPipeline::Pipeline
   description "Downloads, transcodes and publishes a video"
   failure_strategy :halt
+  coordination_queue_name "video_coordination"
+  callback_queue_name "video_callbacks"
 
   on_complete :notify
   on_success  :celebrate
@@ -39,6 +41,8 @@ end
 | `on_complete` | Callback for any terminal state | `nil` |
 | `on_success` | Callback for succeeded | `nil` |
 | `on_failure` | Callback for failed or halted | `nil` |
+| `coordination_queue_name` | Queue for `StepFinishedJob` and `PipelineReconciliationJob` | `"good_pipeline_coordination"` |
+| `callback_queue_name` | Queue for `PipelineCallbackJob` | `"good_pipeline_callbacks"` |
 
 ## DSL verbs
 

data/docs/failure-strategies.md

@@ -127,6 +127,41 @@ A downstream step is eligible for enqueue when **all** of its incoming edges are
 
 A downstream step is marked `skipped` when it's still `pending` and at least one incoming edge is **permanently unsatisfied** — the upstream is terminal, cannot satisfy the edge, and no future event can change that.
 
+## Early termination with success
+
+Sometimes a job determines there is nothing to do — the account is deactivated, the resource was already processed, etc. Call `halt_pipeline!` to stop the pipeline early and mark it as `succeeded`:
+
+```ruby
+class FetchDataJob < ApplicationJob
+  def perform(account_id:)
+    account = Account.find(account_id)
+    return halt_pipeline! if account.deactivated?
+
+    # ... normal work
+  end
+end
+```
+
+The behavior:
+
+| Aspect | Value |
+|---|---|
+| Halting step status | `halted` |
+| Remaining pending steps | `skipped` |
+| Pipeline status | `succeeded` |
+| Callback triggered | `on_success` |
+| GoodJob record | Succeeded (no error, no discard) |
+
+No configuration or module includes are required. The Engine includes `GoodPipeline::Haltable` into `ActiveJob::Base` at boot, so `halt_pipeline!` is available in any job. For non-pipeline jobs, it's a no-op.
+
+::: tip Return early
+Remember to use `return halt_pipeline!` — without `return`, the job continues executing after the call.
+:::
+
+::: warning Parallel steps
+If another step is already running when `halt_pipeline!` is called, that step continues to completion. Only `pending` steps are skipped. If the running step fails, the pipeline will derive to `failed`, not `succeeded`.
+:::
+
 ## Failure resolution table
 
 | Pipeline strategy | Step override | Effect when step fails |

data/docs/getting-started.md

@@ -36,6 +36,18 @@ GoodJob.preserve_job_records = true
 
 GoodPipeline will raise `GoodPipeline::ConfigurationError` at boot if this is not set.
 
+## Configure queue names (optional)
+
+GoodPipeline routes its internal jobs to dedicated queues by default. You can override them globally:
+
+```ruby
+# config/initializers/good_pipeline.rb
+GoodPipeline.coordination_queue_name = "pipeline_coordination"  # StepFinishedJob, PipelineReconciliationJob
+GoodPipeline.callback_queue_name = "pipeline_callbacks"         # PipelineCallbackJob
+```
+
+Defaults are `"good_pipeline_coordination"` and `"good_pipeline_callbacks"`. Per-pipeline overrides are also available via the class DSL — see [Defining Pipelines](/defining-pipelines).
+
 ## Mount the dashboard (optional)
 
 ```ruby

data/docs/index.md

@@ -17,7 +17,7 @@ features:
   - title: Postgres only
     details: All state lives in Postgres. No Redis, no external dependencies. Step transitions and job enqueues happen in a single database transaction.
   - title: DAG orchestration
-    details: Define pipelines as directed acyclic graphs with the run and branch DSL. Steps run in parallel when possible, wait for dependencies automatically, and take different paths based on runtime decisions. Fan-out, fan-in, branching, and chaining are all there.
+    details: Define pipelines as directed acyclic graphs — not just linear chains. Steps run in parallel when possible, wait for dependencies automatically, and take different paths based on runtime decisions. Fan-out, fan-in, branching, and chaining are all first-class.
   - title: Web dashboard
     details: A mountable Rails engine with pipeline executions, step details, DAG visualization, and a pipeline definitions catalog. No build step.
 ---

data/docs/introduction.md

@@ -23,6 +23,24 @@ GoodJob's Batch feature fires a single `on_finish` callback when all jobs in a b
 
 GoodPipeline adds a coordination state machine, DAG validation, and atomic step transitions on top of Batch.
 
+### vs. Active Job Continuation (Rails 8.1)
+
+Rails 8.1 ships with `ActiveJob::Continuable`, which lets a single job define sequential steps with cursor-based progress tracking. If a deploy kills the process, the job resumes from its last checkpoint instead of restarting from scratch.
+
+This solves a different problem than GoodPipeline. Continuation makes one long-running job resilient to interruption. GoodPipeline orchestrates multiple independent jobs as a DAG with parallel execution, fan-out/fan-in, branching, and pipeline-level failure strategies.
+
+The two are complementary: a GoodPipeline step that processes millions of records could use `Continuable` internally for checkpoint/resume, while GoodPipeline handles the higher-level orchestration around it.
+
+### vs. Geneva Drive
+
+[Geneva Drive](https://github.com/julik/geneva_drive) is a durable workflow framework that executes steps strictly sequentially — one step at a time, like the mechanical gear it's named after. It works with any ActiveJob adapter (Sidekiq, Solid Queue, GoodJob) and supports PostgreSQL, MySQL, and SQLite.
+
+Geneva Drive is a strong choice for linear, long-lived workflows with pause/resume with human-in-the-loop recovery and per-hero workflow uniqueness constraints. Its layered exception policy system is particularly sophisticated.
+
+GoodPipeline takes a different approach: workflows are DAGs, not linear chains. Independent steps run in parallel across workers. Fan-out, fan-in, conditional branching, and pipeline chaining are first-class primitives. The tradeoff is that GoodPipeline requires GoodJob and PostgreSQL specifically, while Geneva Drive is adapter- and database-agnostic.
+
+Choose Geneva Drive when your workflow is inherently sequential and you need pause/resume or adapter flexibility. Choose GoodPipeline when steps can run concurrently, your workflow has branching or fan-in topology, or you want a built-in dashboard with DAG visualization.
+
 ## Features
 
 - `run` and `branch` DSL for defining step dependencies and conditional paths
@@ -39,4 +57,4 @@ GoodPipeline adds a coordination state machine, DAG validation, and atomic step
 - Ruby >= 3.2
 - Rails >= 7.1
 - PostgreSQL
-- GoodJob >= 3.10 with `preserve_job_records = true`
+- GoodJob >= 4.14 with `preserve_job_records = true`

data/docs/pipeline-chaining.md

@@ -79,6 +79,8 @@ GoodPipeline.run(
 
 Both pipelines start immediately. `MergeMediaPipeline` waits for both to succeed.
 
+Pipeline chaining is a first-class primitive — upstream/downstream relationships are tracked in a dedicated database table with atomic state propagation, rather than manually creating the next workflow in the last step of the current one.
+
 ## How `.then` works internally
 
 `.then` returns a `GoodPipeline::Chain` object which:

data/lib/generators/good_pipeline/install/templates/create_good_pipeline_tables.rb.erb

@@ -29,6 +29,7 @@ class CreateGoodPipelineTables < ActiveRecord::Migration[<%= ActiveRecord::Migra
       t.uuid :good_job_batch_id
       t.uuid :good_job_id
       t.integer :pending_upstream_count, null: false, default: 0
+      t.boolean :halt_requested, null: false, default: false
       t.integer :attempts
       t.string :error_class
       t.text :error_message
@@ -38,6 +39,7 @@ class CreateGoodPipelineTables < ActiveRecord::Migration[<%= ActiveRecord::Migra
 
     add_index :good_pipeline_steps, %i[pipeline_id key], unique: true
     add_index :good_pipeline_steps, :coordination_status
+    add_index :good_pipeline_steps, :good_job_id, unique: true, where: "good_job_id IS NOT NULL"
 
     create_table :good_pipeline_dependencies do |t|
       t.references :pipeline, null: false, foreign_key: { to_table: :good_pipeline_pipelines }, type: :uuid

data/lib/good_pipeline/chain_coordinator.rb

@@ -42,12 +42,8 @@ module GoodPipeline
 
       def start_pipeline(pipeline_record)
         pipeline_record.transition_to!(:running)
-
         root_step_ids = pipeline_record.steps.where.missing(:upstream_dependencies).pluck(:id)
-
-        root_step_ids.each do |step_id|
-          Coordinator.try_enqueue_step(step_id)
-        end
+        Coordinator.bulk_enqueue_steps(root_step_ids)
       end
     end
   end

data/lib/good_pipeline/coordinator.rb

@@ -3,9 +3,14 @@
 module GoodPipeline
   class Coordinator # rubocop:disable Metrics/ClassLength
     class << self
-      def complete_step(step, succeeded:)
+      def complete_step(step, succeeded:) # rubocop:disable Metrics/MethodLength
         return if step.terminal_coordination_status?
 
+        if succeeded && step.halt_requested?
+          handle_halt_execution(step)
+          return
+        end
+
         record_step_outcome(step, succeeded)
         propagate_halt(step) if !succeeded && step.pipeline.halt?
         return if unblock_downstream_steps(step)
@@ -44,6 +49,23 @@ module GoodPipeline
         step_was_enqueued || downstream_enqueued
       end
 
+      # Enqueues multiple steps in bulk using Batch.enqueue_all.
+      # Intended for root steps during pipeline startup where no
+      # concurrent enqueue risk exists and no upstream checks are needed.
+      def bulk_enqueue_steps(step_ids)
+        return if step_ids.empty?
+
+        steps = StepRecord.where(id: step_ids, coordination_status: "pending")
+                          .where(good_job_id: nil)
+                          .to_a
+
+        branch_steps, enqueueable_steps = steps.partition(&:branch_step?)
+
+        bulk_enqueue_user_jobs(enqueueable_steps) if enqueueable_steps.any?
+
+        branch_steps.each { |step| try_enqueue_step(step.id) }
+      end
+
       def recompute_pipeline_status(pipeline, has_active_steps: nil, has_downstream_chains: nil) # rubocop:disable Metrics/MethodLength
         return if pipeline.terminal?
 
@@ -68,12 +90,26 @@ module GoodPipeline
 
           return if rows_updated.zero?
 
-          PipelineCallbackJob.perform_later(pipeline.id, new_status.to_s)
+          queue = pipeline.type.constantize.callback_queue_name
+          PipelineCallbackJob.set(queue: queue).perform_later(pipeline.id, new_status.to_s)
         end
       end
 
       private
 
+      def handle_halt_execution(step)
+        step.transition_coordination_status_to!(:halted)
+        step.pipeline.steps.pending.update_all(coordination_status: "skipped")
+
+        pipeline = load_pipeline_with_active_check(step.pipeline_id)
+
+        recompute_pipeline_status(
+          pipeline,
+          has_active_steps: pipeline["has_active_steps"],
+          has_downstream_chains: pipeline["has_downstream_chains"]
+        )
+      end
+
       def record_step_outcome(step, succeeded)
         if succeeded
           step.transition_coordination_status_to!(:succeeded)
@@ -111,6 +147,18 @@ module GoodPipeline
         scope.update_all(coordination_status: "skipped")
       end
 
+      def transitive_downstream_ids(step)
+        visited = Set.new
+        queue = step.downstream_steps.pluck(:id)
+        while (current_id = queue.shift)
+          next if visited.include?(current_id)
+
+          visited << current_id
+          queue.concat(DependencyRecord.where(depends_on_step_id: current_id).pluck(:step_id))
+        end
+        visited
+      end
+
       def unblock_downstream_steps(step)
         sql = <<~SQL
           UPDATE good_pipeline_steps
@@ -173,6 +221,7 @@ module GoodPipeline
       def permanently_unsatisfied?(upstream)
         upstream.terminal_coordination_status? &&
           !upstream.succeeded? &&
+          !upstream.halted? &&
           !upstream.skipped_by_branch? &&
           effective_failure_strategy(upstream) != :ignore
       end
@@ -186,6 +235,7 @@ module GoodPipeline
       def all_upstreams_satisfied?(step)
         step.upstream_steps.all? do |upstream|
           upstream.succeeded? ||
+            upstream.halted? ||
             upstream.skipped_by_branch? ||
             (upstream.failed? && effective_failure_strategy(upstream) == :ignore)
         end
@@ -206,6 +256,7 @@ module GoodPipeline
       def build_step_batch(step)
         batch = GoodJob::Batch.new
         batch.on_finish = "GoodPipeline::StepFinishedJob"
+        batch.callback_queue_name = step.pipeline.type.constantize.coordination_queue_name
         batch.properties = { step_id: step.id }
         batch
       end
@@ -216,6 +267,14 @@ module GoodPipeline
         enqueued_job.provider_job_id || enqueued_job.job_id
       end
 
+      def fail_step_with_error(step, error)
+        step.transition_coordination_status_to!(:failed)
+        step.update_columns(
+          error_class: error.class.name,
+          error_message: error.message
+        )
+      end
+
       def derive_terminal_status(pipeline)
         has_failures = pipeline.steps.where(coordination_status: "failed").exists?
 
@@ -225,24 +284,66 @@ module GoodPipeline
         :failed
       end
 
-      def transitive_downstream_ids(step)
-        visited = Set.new
-        queue = step.downstream_steps.pluck(:id)
-        while (current_id = queue.shift)
-          next if visited.include?(current_id)
+      def bulk_enqueue_user_jobs(steps) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength,Metrics/CyclomaticComplexity
+        batch_job_pairs = []
+        step_metadata = {}
+        failed_steps = []
+        coordination_queue = steps.first.pipeline.type.constantize.coordination_queue_name
+
+        steps.each do |step|
+          job_class = begin
+            step.job_class.constantize
+          rescue NameError => error
+            failed_steps << [step, ConfigurationError.new(error.message)]
+            next
+          end
+
+          batch = GoodJob::Batch.new
+          batch.on_finish = "GoodPipeline::StepFinishedJob"
+          batch.callback_queue_name = coordination_queue
+          batch.properties = { step_id: step.id }
+
+          active_job = job_class.new(**step.params.symbolize_keys)
+          apply_enqueue_options(active_job, step.enqueue_options.symbolize_keys)
+
+          batch_job_pairs << [batch, [active_job]]
+          step_metadata[step.id] = { batch: batch, active_job: active_job }
+        end
 
-          visited << current_id
-          queue.concat(DependencyRecord.where(depends_on_step_id: current_id).pluck(:step_id))
+        StepRecord.transaction do
+          GoodJob::Batch.enqueue_all(batch_job_pairs) if batch_job_pairs.any?
+
+          now = Time.current
+          step_metadata.each do |step_id, metadata|
+            StepRecord.where(id: step_id).update_all(
+              coordination_status: "enqueued",
+              good_job_batch_id: metadata[:batch].id,
+              good_job_id: metadata[:active_job].provider_job_id || metadata[:active_job].job_id,
+              updated_at: now
+            )
+          end
+        end
+      ensure
+        failed_steps.each do |step, error|
+          fail_step_with_error(step, error)
+          propagate_halt(step) if step.pipeline.halt?
         end
-        visited
       end
 
-      def fail_step_with_error(step, error)
-        step.transition_coordination_status_to!(:failed)
-        step.update_columns(
-          error_class: error.class.name,
-          error_message: error.message
-        )
+      def apply_enqueue_options(active_job, options) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
+        return if options.blank?
+
+        if options[:good_job_labels] && active_job.respond_to?(:good_job_labels=)
+          active_job.good_job_labels = Array(options[:good_job_labels])
+        end
+
+        if options.key?(:good_job_notify) && active_job.respond_to?(:good_job_notify=)
+          active_job.good_job_notify = options[:good_job_notify]
+        end
+
+        active_job.queue_name = options[:queue].to_s if options[:queue]
+        active_job.priority = options[:priority] if options[:priority]
+        active_job.scheduled_at = Time.current + options[:wait] if options[:wait]
       end
 
       def effective_failure_strategy(step)

data/lib/good_pipeline/engine.rb

@@ -13,6 +13,12 @@ module GoodPipeline
       end
     end
 
+    initializer "good_pipeline.haltable" do
+      ActiveSupport.on_load(:active_job) do
+        include GoodPipeline::Haltable
+      end
+    end
+
     initializer "good_pipeline.cleanup_hook" do
       ActiveSupport::Notifications.subscribe("cleanup_preserved_jobs.good_job") do |event|
         timestamp = event.payload[:timestamp]

data/lib/good_pipeline/haltable.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module GoodPipeline
+  module Haltable
+    def halt_pipeline!
+      step = GoodPipeline::StepRecord.find_by(good_job_id: provider_job_id)
+      step&.update_columns(halt_requested: true, updated_at: Time.current)
+    end
+  end
+end