good_pipeline 0.3.1 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e9df8b5fbd57895f53adf1d3c5804ca9bd64ca792e880c2d4941957e4b8ca368
-  data.tar.gz: c4e2a7c4edbe27a0e40e7ff62a061f5ade53c44a74c0b824ce21bdf019cb8781
+  metadata.gz: 38ec7eb9fb3cec2b9109b9695b0084aff2fa111440cd8fb76dbe53be59bd8e06
+  data.tar.gz: 71094436abc0d2c393b3d188510608bf714867201f2fc13eac4cc218d6b8a420
 SHA512:
-  metadata.gz: e3c6e6940034efbc8679ede33b352022deb64a36b7a16a9dd3f02fd6dc32a837caf545407541bd0ca629f690d38cfa60342e6aa756e53ea321fb754eebbd772c
-  data.tar.gz: 878566f6d9ddc7b9ce6b4201f5bd241b21f2758f5475e3fa60e6c3fd629651bff06ff973af9f3f29ac4624bee58e78dc0f9628e8a1cdb503836e923a09e3944e
+  metadata.gz: c3520cff350de86f2d2820d7a4406f2b81407079c1d00ee8ecfc4dff02e23f54dcedb1ae61d3a32594a4680b111b4b127fe2808ecf08c4705c4e0732e04643b9
+  data.tar.gz: 2a91e6a2b050392b6e5e8c1af31bd93a8e05fe5d23f1f940d0bbe97df59c381b7286bf10a8f3590789b342d079d66b9e1c1433e00a03eaa42d31c277e8e6218f

data/CHANGELOG.md

@@ -1,5 +1,21 @@
 ## [Unreleased]
 
+## [0.4.0] - 2026-04-02
+
+### Performance
+
+- **Bulk root step enqueuing** — pipelines with multiple root steps now enqueue all of them via `GoodJob::Batch.enqueue_all` in a fixed number of queries instead of ~9 queries per step. Both `Runner#enqueue_root_steps` and `ChainCoordinator#start_pipeline` use the new `Coordinator.bulk_enqueue_steps` method.
+
+### Added
+
+- **Configurable queue names for internal jobs** — new `coordination_queue_name` and `callback_queue_name` settings control which queues `StepFinishedJob`, `PipelineReconciliationJob`, and `PipelineCallbackJob` run on. Configurable globally (`GoodPipeline.coordination_queue_name = "x"`) and per-pipeline via the class DSL. Defaults to `"good_pipeline_coordination"` and `"good_pipeline_callbacks"`.
+- **`Coordinator.bulk_enqueue_steps`** — public method that loads pending steps, partitions branch steps for individual handling, and bulk-enqueues the rest via `Batch.enqueue_all`. Invalid job classes are failed individually without blocking valid steps.
+
+### Changed
+
+- **Minimum GoodJob version** — bumped from `>= 3.10` to `>= 4.14` (required for `Batch.enqueue_all`).
+- **`run_pipeline_to_completion` test helper** — extracted from 3 integration test files into `test_helper.rb`.
+
 ## [0.3.1] - 2026-03-26
 
 ### Added

data/README.md

@@ -2,7 +2,7 @@
 
 DAG-based job pipeline orchestration for Rails, built on [GoodJob](https://github.com/bensheldon/good_job).
 
-Define multi-step workflows as directed acyclic graphs, where each step is a GoodJob job. GoodPipeline handles dependency resolution, parallel execution, failure strategies, pipeline chaining, and lifecycle callbacks. It also ships with a web dashboard.
+Define multi-step workflows as directed acyclic graphs — not linear chains. Steps run in parallel when they can and wait for dependencies when they must. GoodPipeline handles dependency resolution, parallel execution, failure strategies, conditional branching, pipeline chaining, and lifecycle callbacks. It also ships with a web dashboard.
 
 ## Requirements
 

data/demo/test/good_pipeline/test_bulk_enqueue.rb

@@ -0,0 +1,193 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class TestBulkEnqueue < ActiveSupport::TestCase
+  # --- basic enqueuing ---
+
+  def test_enqueues_multiple_steps
+    pipeline = create_pipeline(on_failure_strategy: "halt")
+    pipeline.update_columns(status: "running")
+    step_a = build_step(pipeline, key: "step_a", job_class: "DownloadJob")
+    step_b = build_step(pipeline, key: "step_b", job_class: "TranscodeJob")
+
+    GoodPipeline::Coordinator.bulk_enqueue_steps([step_a.id, step_b.id])
+
+    step_a.reload
+    step_b.reload
+
+    assert_equal "enqueued", step_a.coordination_status
+    assert_equal "enqueued", step_b.coordination_status
+    refute_nil step_a.good_job_batch_id
+    refute_nil step_b.good_job_batch_id
+    refute_nil step_a.good_job_id
+    refute_nil step_b.good_job_id
+  end
+
+  def test_each_step_gets_its_own_batch
+    pipeline = create_pipeline(on_failure_strategy: "halt")
+    pipeline.update_columns(status: "running")
+    step_a = build_step(pipeline, key: "step_a", job_class: "DownloadJob")
+    step_b = build_step(pipeline, key: "step_b", job_class: "TranscodeJob")
+
+    GoodPipeline::Coordinator.bulk_enqueue_steps([step_a.id, step_b.id])
+
+    refute_equal step_a.reload.good_job_batch_id, step_b.reload.good_job_batch_id
+  end
+
+  def test_good_job_id_points_to_real_job_record
+    pipeline = create_pipeline(on_failure_strategy: "halt")
+    pipeline.update_columns(status: "running")
+    step = build_step(pipeline, key: "step_a", job_class: "DownloadJob")
+
+    GoodPipeline::Coordinator.bulk_enqueue_steps([step.id])
+
+    step.reload
+    good_job = GoodJob::Job.find_by(id: step.good_job_id)
+
+    refute_nil good_job, "good_job_id should point to a real GoodJob::Job record"
+    assert_equal step.good_job_batch_id, good_job.batch_id
+  end
+
+  # --- batch callback setup ---
+
+  def test_batch_has_step_finished_callback
+    pipeline = create_pipeline(on_failure_strategy: "halt")
+    pipeline.update_columns(status: "running")
+    step = build_step(pipeline, key: "step_a", job_class: "DownloadJob")
+
+    GoodPipeline::Coordinator.bulk_enqueue_steps([step.id])
+
+    batch_record = GoodJob::BatchRecord.find(step.reload.good_job_batch_id)
+
+    assert_equal "GoodPipeline::StepFinishedJob", batch_record.on_finish
+    assert_equal({ step_id: step.id }, batch_record.properties)
+  end
+
+  # --- enqueue_options ---
+
+  def test_respects_queue_and_priority_options
+    pipeline = create_pipeline(on_failure_strategy: "halt")
+    pipeline.update_columns(status: "running")
+    step = build_step(pipeline, key: "step_a", job_class: "DownloadJob",
+                      enqueue_options: { "queue" => "critical", "priority" => 3 })
+
+    GoodPipeline::Coordinator.bulk_enqueue_steps([step.id])
+
+    good_job = GoodJob::Job.find_by(id: step.reload.good_job_id)
+
+    assert_equal "critical", good_job.queue_name
+    assert_equal 3, good_job.priority
+  end
+
+  def test_respects_wait_option
+    pipeline = create_pipeline(on_failure_strategy: "halt")
+    pipeline.update_columns(status: "running")
+    step = build_step(pipeline, key: "step_a", job_class: "DownloadJob",
+                      enqueue_options: { "wait" => 300 })
+
+    GoodPipeline::Coordinator.bulk_enqueue_steps([step.id])
+
+    good_job = GoodJob::Job.find_by(id: step.reload.good_job_id)
+
+    refute_nil good_job.scheduled_at
+    assert_in_delta 300, good_job.scheduled_at - good_job.created_at, 5
+  end
+
+  def test_passes_step_params_to_job
+    pipeline = create_pipeline(on_failure_strategy: "halt")
+    pipeline.update_columns(status: "running")
+    step = build_step(pipeline, key: "step_a", job_class: "DownloadJob",
+                      params: { "video_id" => 42 })
+
+    GoodPipeline::Coordinator.bulk_enqueue_steps([step.id])
+
+    good_job = GoodJob::Job.find_by(id: step.reload.good_job_id)
+    arguments = good_job.serialized_params["arguments"]
+
+    assert_equal 42, arguments.first["video_id"]
+  end
+
+  def test_handles_empty_params
+    pipeline = create_pipeline(on_failure_strategy: "halt")
+    pipeline.update_columns(status: "running")
+    step = build_step(pipeline, key: "step_a", job_class: "DownloadJob", params: {})
+
+    GoodPipeline::Coordinator.bulk_enqueue_steps([step.id])
+
+    assert_equal "enqueued", step.reload.coordination_status
+  end
+
+  # --- guard clauses ---
+
+  def test_skips_non_pending_steps
+    pipeline = create_pipeline(on_failure_strategy: "halt")
+    pipeline.update_columns(status: "running")
+    step_a = build_step(pipeline, key: "step_a", job_class: "DownloadJob")
+    step_a.update_columns(coordination_status: "enqueued")
+    step_b = build_step(pipeline, key: "step_b", job_class: "TranscodeJob")
+
+    GoodPipeline::Coordinator.bulk_enqueue_steps([step_a.id, step_b.id])
+
+    assert_equal "enqueued", step_b.reload.coordination_status
+    refute_nil step_b.good_job_id
+    assert_nil step_a.reload.good_job_id, "Non-pending step should not have been re-enqueued"
+  end
+
+  def test_skips_steps_with_good_job_id
+    pipeline = create_pipeline(on_failure_strategy: "halt")
+    pipeline.update_columns(status: "running")
+    step_a = build_step(pipeline, key: "step_a", job_class: "DownloadJob")
+    existing_job_id = SecureRandom.uuid
+    step_a.update_columns(good_job_id: existing_job_id)
+    step_b = build_step(pipeline, key: "step_b", job_class: "TranscodeJob")
+
+    GoodPipeline::Coordinator.bulk_enqueue_steps([step_a.id, step_b.id])
+
+    assert_equal "enqueued", step_b.reload.coordination_status
+    assert_equal existing_job_id, step_a.reload.good_job_id, "Step with good_job_id should be left alone"
+  end
+
+  def test_handles_empty_array
+    # Should not raise
+    result = GoodPipeline::Coordinator.bulk_enqueue_steps([])
+    assert_nil result
+  end
+
+  # --- branch step fallback ---
+
+  def test_falls_back_to_try_enqueue_step_for_branch_steps
+    pipeline = create_pipeline(on_failure_strategy: "halt")
+    pipeline.update_columns(status: "running")
+    branch_step = build_step(pipeline, key: "format_check",
+                             job_class: GoodPipeline::BRANCH_JOB_CLASS)
+    branch_step.update_columns(branch: { "decides" => "pick_format", "empty_arms" => %w[hd sd] })
+    normal_step = build_step(pipeline, key: "step_a", job_class: "DownloadJob")
+
+    GoodPipeline::Coordinator.bulk_enqueue_steps([branch_step.id, normal_step.id])
+
+    assert_equal "enqueued", normal_step.reload.coordination_status
+    refute_nil normal_step.good_job_id
+
+    branch_step.reload
+    refute_equal "pending", branch_step.coordination_status,
+                 "Branch step should have been processed by try_enqueue_step fallback"
+  end
+
+  # --- error handling ---
+
+  def test_invalid_job_class_fails_step_without_blocking_others
+    pipeline = create_pipeline(on_failure_strategy: "halt")
+    pipeline.update_columns(status: "running")
+    bad_step = build_step(pipeline, key: "bad_step", job_class: "NonExistentJob")
+    good_step = build_step(pipeline, key: "good_step", job_class: "DownloadJob")
+
+    GoodPipeline::Coordinator.bulk_enqueue_steps([bad_step.id, good_step.id])
+
+    assert_equal "enqueued", good_step.reload.coordination_status
+    refute_nil good_step.good_job_id
+
+    assert_equal "failed", bad_step.reload.coordination_status
+    assert_equal "GoodPipeline::ConfigurationError", bad_step.error_class
+  end
+end

data/demo/test/good_pipeline/test_queue_configuration.rb

@@ -0,0 +1,157 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class TestQueueConfiguration < ActiveSupport::TestCase
+  teardown do
+    GoodPipeline.coordination_queue_name = nil
+    GoodPipeline.callback_queue_name = nil
+  end
+
+  # --- global defaults ---
+
+  def test_default_coordination_queue_name
+    assert_equal "good_pipeline_coordination", GoodPipeline.coordination_queue_name
+  end
+
+  def test_default_callback_queue_name
+    assert_equal "good_pipeline_callbacks", GoodPipeline.callback_queue_name
+  end
+
+  # --- global override ---
+
+  def test_global_coordination_queue_override
+    GoodPipeline.coordination_queue_name = "custom_coordination"
+
+    assert_equal "custom_coordination", GoodPipeline.coordination_queue_name
+  end
+
+  def test_global_callback_queue_override
+    GoodPipeline.callback_queue_name = "custom_callbacks"
+
+    assert_equal "custom_callbacks", GoodPipeline.callback_queue_name
+  end
+
+  # --- pipeline DSL ---
+
+  def test_pipeline_dsl_coordination_queue
+    klass = Class.new(GoodPipeline::Pipeline) do
+      coordination_queue_name "pipeline_coordination"
+      def configure(**) = run(:a, DownloadJob)
+    end
+
+    assert_equal "pipeline_coordination", klass.coordination_queue_name
+  end
+
+  def test_pipeline_dsl_callback_queue
+    klass = Class.new(GoodPipeline::Pipeline) do
+      callback_queue_name "pipeline_callbacks"
+      def configure(**) = run(:a, DownloadJob)
+    end
+
+    assert_equal "pipeline_callbacks", klass.callback_queue_name
+  end
+
+  # --- pipeline DSL fallback to global ---
+
+  def test_pipeline_without_dsl_uses_global_config
+    GoodPipeline.coordination_queue_name = "global_coordination"
+
+    klass = Class.new(GoodPipeline::Pipeline) do
+      def configure(**) = run(:a, DownloadJob)
+    end
+
+    assert_equal "global_coordination", klass.coordination_queue_name
+  end
+
+  def test_pipeline_without_dsl_or_global_uses_default
+    klass = Class.new(GoodPipeline::Pipeline) do
+      def configure(**) = run(:a, DownloadJob)
+    end
+
+    assert_equal "good_pipeline_coordination", klass.coordination_queue_name
+  end
+
+  # --- pipeline DSL overrides global ---
+
+  def test_pipeline_dsl_overrides_global
+    GoodPipeline.coordination_queue_name = "global_coordination"
+
+    klass = Class.new(GoodPipeline::Pipeline) do
+      coordination_queue_name "pipeline_coordination"
+      def configure(**) = run(:a, DownloadJob)
+    end
+
+    assert_equal "pipeline_coordination", klass.coordination_queue_name
+  end
+
+  # --- inheritance ---
+
+  def test_pipeline_inherits_queue_from_parent
+    parent = Class.new(GoodPipeline::Pipeline) do
+      coordination_queue_name "parent_coordination"
+      callback_queue_name "parent_callbacks"
+    end
+
+    child = Class.new(parent) do
+      def configure(**) = run(:a, DownloadJob)
+    end
+
+    assert_equal "parent_coordination", child.coordination_queue_name
+    assert_equal "parent_callbacks", child.callback_queue_name
+  end
+
+  # --- step batch gets coordination queue ---
+
+  def test_step_batch_gets_coordination_queue
+    klass = Class.new(GoodPipeline::Pipeline) do
+      coordination_queue_name "step_coordination"
+      def configure(**) = run(:a, DownloadJob)
+    end
+    klass.define_singleton_method(:name) { "StepBatchQueueTestPipeline" }
+    Object.const_set(:StepBatchQueueTestPipeline, klass) unless defined?(::StepBatchQueueTestPipeline)
+
+    pipeline_record = StepBatchQueueTestPipeline.run
+
+    step = pipeline_record.steps.first
+    batch_record = GoodJob::BatchRecord.find(step.good_job_batch_id)
+
+    assert_equal "step_coordination", batch_record.callback_queue_name
+  end
+
+  # --- pipeline batch gets coordination queue ---
+
+  def test_pipeline_batch_gets_coordination_queue
+    klass = Class.new(GoodPipeline::Pipeline) do
+      coordination_queue_name "pipeline_coordination"
+      def configure(**) = run(:a, DownloadJob)
+    end
+    klass.define_singleton_method(:name) { "PipelineBatchQueueTestPipeline" }
+    Object.const_set(:PipelineBatchQueueTestPipeline, klass) unless defined?(::PipelineBatchQueueTestPipeline)
+
+    pipeline_record = PipelineBatchQueueTestPipeline.run
+
+    actual_record = GoodPipeline::PipelineRecord.find(pipeline_record.id)
+    batch_record = GoodJob::BatchRecord.find(actual_record.good_job_batch_id)
+
+    assert_equal "pipeline_coordination", batch_record.callback_queue_name
+  end
+
+  # --- PipelineCallbackJob gets callback queue ---
+
+  def test_callback_job_gets_callback_queue
+    klass = Class.new(GoodPipeline::Pipeline) do
+      callback_queue_name "my_callbacks"
+      def configure(**) = run(:a, DownloadJob)
+    end
+    klass.define_singleton_method(:name) { "CallbackQueueTestPipeline" }
+    Object.const_set(:CallbackQueueTestPipeline, klass) unless defined?(::CallbackQueueTestPipeline)
+
+    pipeline_record = CallbackQueueTestPipeline.run
+    run_pipeline_to_completion(pipeline_record)
+
+    callback_job = GoodJob::Job.where(job_class: "GoodPipeline::PipelineCallbackJob").last
+
+    assert_equal "my_callbacks", callback_job.queue_name
+  end
+end

data/demo/test/integration/test_bulk_enqueue_end_to_end.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class TestBulkEnqueueEndToEnd < ActiveSupport::TestCase
+  def test_fan_in_pipeline_with_multiple_root_steps_succeeds
+    pipeline_class = Class.new(GoodPipeline::Pipeline) do
+      failure_strategy :halt
+
+      define_method(:configure) do |**_kwargs|
+        run :root_a, DownloadJob
+        run :root_b, TranscodeJob
+        run :root_c, ThumbnailJob
+        run :collector, PublishJob, after: %i[root_a root_b root_c]
+      end
+    end
+    Object.const_set(:FanInBulkTestPipeline, pipeline_class) unless defined?(::FanInBulkTestPipeline)
+
+    pipeline_record = FanInBulkTestPipeline.run
+
+    # All 3 root steps should have been enqueued with distinct batches
+    root_steps = pipeline_record.steps.where(key: %w[root_a root_b root_c])
+    root_steps.each do |step|
+      refute_equal "pending", step.coordination_status,
+                   "Root step #{step.key} should have been enqueued"
+      refute_nil step.good_job_batch_id
+      refute_nil step.good_job_id
+    end
+
+    batch_ids = root_steps.pluck(:good_job_batch_id).uniq
+    assert_equal 3, batch_ids.size, "Each root step should have a unique batch"
+
+    result = run_pipeline_to_completion(pipeline_record)
+
+    assert_equal "succeeded", result.status
+    assert(result.steps.all? { |step| step.coordination_status == "succeeded" })
+  end
+
+  def test_all_root_steps_pipeline_succeeds
+    pipeline_class = Class.new(GoodPipeline::Pipeline) do
+      failure_strategy :continue
+
+      define_method(:configure) do |**_kwargs|
+        run :step_a, DownloadJob
+        run :step_b, TranscodeJob
+        run :step_c, ThumbnailJob
+        run :step_d, PublishJob
+        run :step_e, CleanupJob
+      end
+    end
+    Object.const_set(:AllRootsBulkTestPipeline, pipeline_class) unless defined?(::AllRootsBulkTestPipeline)
+
+    pipeline_record = AllRootsBulkTestPipeline.run
+    result = run_pipeline_to_completion(pipeline_record)
+
+    assert_equal "succeeded", result.status
+
+    result.steps.each do |step|
+      assert_equal "succeeded", step.coordination_status
+      refute_nil step.good_job_batch_id
+      refute_nil step.good_job_id
+    end
+  end
+
+  def test_fan_in_with_failing_root_step_halts
+    pipeline_class = Class.new(GoodPipeline::Pipeline) do
+      failure_strategy :halt
+
+      define_method(:configure) do |**_kwargs|
+        run :root_a, DownloadJob
+        run :root_b, FailingJob
+        run :root_c, ThumbnailJob
+        run :collector, PublishJob, after: %i[root_a root_b root_c]
+      end
+    end
+    Object.const_set(:FanInFailBulkTestPipeline, pipeline_class) unless defined?(::FanInFailBulkTestPipeline)
+
+    pipeline_record = FanInFailBulkTestPipeline.run
+    result = run_pipeline_to_completion(pipeline_record)
+
+    assert_equal "halted", result.status
+    assert_equal "failed", result.steps.find_by(key: "root_b").coordination_status
+  end
+
+  def test_enqueue_options_forwarded_to_good_job
+    pipeline_class = Class.new(GoodPipeline::Pipeline) do
+      failure_strategy :halt
+
+      define_method(:configure) do |**_kwargs|
+        run :step_a, DownloadJob, enqueue: { queue: "critical", priority: 1 }
+        run :step_b, TranscodeJob, enqueue: { queue: "low", priority: 10 }
+      end
+    end
+    Object.const_set(:EnqueueOptionsBulkTestPipeline, pipeline_class) unless defined?(::EnqueueOptionsBulkTestPipeline)
+
+    pipeline_record = EnqueueOptionsBulkTestPipeline.run
+
+    step_a = pipeline_record.steps.find_by(key: "step_a")
+    step_b = pipeline_record.steps.find_by(key: "step_b")
+
+    good_job_a = GoodJob::Job.find_by(id: step_a.good_job_id)
+    good_job_b = GoodJob::Job.find_by(id: step_b.good_job_id)
+
+    assert_equal "critical", good_job_a.queue_name
+    assert_equal 1, good_job_a.priority
+    assert_equal "low", good_job_b.queue_name
+    assert_equal 10, good_job_b.priority
+  end
+end

data/demo/test/integration/test_end_to_end.rb

@@ -3,21 +3,6 @@
 require "test_helper"
 
 class TestEndToEnd < ActiveSupport::TestCase
-  def run_pipeline_to_completion(pipeline_record, timeout: 15)
-    deadline = Time.current + timeout
-    loop do
-      perform_enqueued_jobs_inline
-      pipeline_record.reload
-      return pipeline_record if pipeline_record.terminal?
-
-      if Time.current > deadline
-        raise "Pipeline did not reach terminal state within #{timeout}s (status: #{pipeline_record.status})"
-      end
-
-      sleep 0.05
-    end
-  end
-
   def test_full_pipeline_succeeds
     pipeline_record = VideoProcessingPipeline.run(video_id: 123)
 

data/demo/test/integration/test_halt_execution.rb

@@ -3,19 +3,6 @@
 require "test_helper"
 
 class TestHaltExecution < ActiveSupport::TestCase
-  def run_pipeline_to_completion(pipeline_record, timeout: 15)
-    deadline = Time.current + timeout
-    loop do
-      perform_enqueued_jobs_inline
-      pipeline_record.reload
-      return pipeline_record if pipeline_record.terminal?
-
-      raise "Pipeline did not reach terminal state within #{timeout}s (status: #{pipeline_record.status})" if Time.current > deadline
-
-      sleep 0.05
-    end
-  end
-
   def test_halt_pipeline_marks_step_halted
     pipeline_class = Class.new(GoodPipeline::Pipeline) do
       failure_strategy :halt

data/demo/test/integration/test_queue_configuration.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class TestQueueConfigurationEndToEnd < ActiveSupport::TestCase
+  teardown do
+    GoodPipeline.coordination_queue_name = nil
+    GoodPipeline.callback_queue_name = nil
+  end
+
+  def test_full_pipeline_with_custom_queues
+    klass = Class.new(GoodPipeline::Pipeline) do
+      failure_strategy :halt
+      coordination_queue_name "e2e_coordination"
+      callback_queue_name "e2e_callbacks"
+
+      define_method(:configure) do |**_kwargs|
+        run :step_a, DownloadJob
+        run :step_b, TranscodeJob
+        run :step_c, PublishJob, after: %i[step_a step_b]
+      end
+    end
+    klass.define_singleton_method(:name) { "QueueE2ETestPipeline" }
+    Object.const_set(:QueueE2ETestPipeline, klass) unless defined?(::QueueE2ETestPipeline)
+
+    pipeline_record = QueueE2ETestPipeline.run
+
+    # Verify step batch queue names
+    pipeline_record.steps.each do |step|
+      next unless step.good_job_batch_id
+
+      batch_record = GoodJob::BatchRecord.find(step.good_job_batch_id)
+
+      assert_equal "e2e_coordination", batch_record.callback_queue_name,
+                   "Step #{step.key} batch should use coordination queue"
+    end
+
+    # Verify pipeline batch queue name
+    actual_record = GoodPipeline::PipelineRecord.find(pipeline_record.id)
+    pipeline_batch = GoodJob::BatchRecord.find(actual_record.good_job_batch_id)
+
+    assert_equal "e2e_coordination", pipeline_batch.callback_queue_name
+
+    # Run to completion and verify callback job queue
+    result = run_pipeline_to_completion(pipeline_record)
+
+    assert_equal "succeeded", result.status
+
+    callback_job = GoodJob::Job.where(job_class: "GoodPipeline::PipelineCallbackJob").last
+
+    assert_equal "e2e_callbacks", callback_job.queue_name
+  end
+
+  def test_global_config_applies_when_no_dsl
+    GoodPipeline.coordination_queue_name = "global_coord"
+    GoodPipeline.callback_queue_name = "global_cb"
+
+    klass = Class.new(GoodPipeline::Pipeline) do
+      failure_strategy :halt
+      define_method(:configure) do |**_kwargs|
+        run :step_a, DownloadJob
+      end
+    end
+    klass.define_singleton_method(:name) { "GlobalQueueTestPipeline" }
+    Object.const_set(:GlobalQueueTestPipeline, klass) unless defined?(::GlobalQueueTestPipeline)
+
+    pipeline_record = GlobalQueueTestPipeline.run
+
+    step = pipeline_record.steps.first
+    step_batch = GoodJob::BatchRecord.find(step.good_job_batch_id)
+
+    assert_equal "global_coord", step_batch.callback_queue_name
+
+    result = run_pipeline_to_completion(pipeline_record)
+
+    assert_equal "succeeded", result.status
+
+    callback_job = GoodJob::Job.where(job_class: "GoodPipeline::PipelineCallbackJob").last
+
+    assert_equal "global_cb", callback_job.queue_name
+  end
+end

data/demo/test/test_helper.rb

@@ -33,6 +33,21 @@ module ActiveSupport
       GoodJob.perform_inline
     end
 
+    def run_pipeline_to_completion(pipeline_record, timeout: 15)
+      deadline = Time.current + timeout
+      loop do
+        perform_enqueued_jobs_inline
+        pipeline_record.reload
+        return pipeline_record if pipeline_record.terminal?
+
+        if Time.current > deadline
+          raise "Pipeline did not reach terminal state within #{timeout}s (status: #{pipeline_record.status})"
+        end
+
+        sleep 0.05
+      end
+    end
+
     def wait_until(timeout: 10, interval: 0.1)
       deadline = Time.current + timeout
       loop do

data/docs/architecture.md

@@ -180,3 +180,11 @@ This ensures a step is never prematurely marked `failed` on attempt 1 of 5.
 7. Separate atomic units per transaction boundary to minimize lock contention
 8. DAG validation runs at instantiation, before any database writes
 9. `failure_strategy` and `on_failure` are distinct concepts -- strategy vs. callback, no naming collision
+
+## Why these tradeoffs
+
+GoodPipeline is intentionally GoodJob-specific and Postgres-only. This is what enables atomic enqueue transactions — step status transitions and GoodJob record inserts happen in a single database transaction, eliminating an entire class of partial-state bugs that adapter-agnostic gems must work around.
+
+The DAG execution model (vs. strictly sequential steps) adds coordination complexity — row locks, atomic counters, fan-in race prevention — but unlocks parallel execution of independent steps. For workflows where steps have no dependency on each other, this means wall-clock time is bounded by the longest path through the graph, not the sum of all steps.
+
+The four-table data model (pipelines, steps, dependencies, chains) is more tables than a two-table approach, but dedicated dependency and chain tables enable efficient graph queries and keep the step table free of self-referential joins.

data/docs/branching.md

@@ -2,6 +2,8 @@
 
 Conditional branching lets a pipeline take different paths at runtime based on application state. The dashboard renders branches as diamond decision nodes.
 
+Runtime branching with `branch` blocks is uncommon among Ruby workflow gems — most offer only `skip_if` conditions on individual steps. GoodPipeline's branching evaluates a decision method when the branch is reached, runs the matching arm, marks non-matching arms as `skipped_by_branch`, and lets downstream steps wait on whichever arm was chosen.
+
 ## Defining a branch
 
 Use `branch` inside `configure` to define a decision point. The `by:` option names a method on your pipeline class that returns the arm to execute:

data/docs/callbacks.md

@@ -46,6 +46,8 @@ Note: `on_failure` does **not** fire for `skipped` pipelines. Being skipped by a
 
 Callbacks are dispatched via `PipelineCallbackJob`, a GoodJob job enqueued after the terminal state transaction commits. A slow external call (Slack, webhooks) cannot stall the coordinator, callback execution cannot corrupt pipeline state, and callbacks get GoodJob's retry mechanism if they fail.
 
+`PipelineCallbackJob` runs on the queue configured by `callback_queue_name` (default: `"good_pipeline_callbacks"`). This is separate from `coordination_queue_name` which controls the coordination jobs (`StepFinishedJob`, `PipelineReconciliationJob`), so slow callbacks don't block pipeline progression. See [Defining Pipelines](/defining-pipelines) for configuration options.
+
 ## Exactly-once guarantee
 
 The callback bundle (`on_complete` + one of `on_success`/`on_failure`) is dispatched as a **single unit**. A `callbacks_dispatched_at` timestamp is set atomically inside a `FOR UPDATE` locked transaction, ensuring the bundle fires exactly once even if `recompute_pipeline_status` is called from multiple code paths (coordinator or batch reconciliation).

data/docs/defining-pipelines.md

@@ -8,6 +8,8 @@ Every pipeline is a subclass of `GoodPipeline::Pipeline` that implements `config
 class VideoProcessingPipeline < GoodPipeline::Pipeline
   description "Downloads, transcodes and publishes a video"
   failure_strategy :halt
+  coordination_queue_name "video_coordination"
+  callback_queue_name "video_callbacks"
 
   on_complete :notify
   on_success  :celebrate
@@ -39,6 +41,8 @@ end
 | `on_complete` | Callback for any terminal state | `nil` |
 | `on_success` | Callback for succeeded | `nil` |
 | `on_failure` | Callback for failed or halted | `nil` |
+| `coordination_queue_name` | Queue for `StepFinishedJob` and `PipelineReconciliationJob` | `"good_pipeline_coordination"` |
+| `callback_queue_name` | Queue for `PipelineCallbackJob` | `"good_pipeline_callbacks"` |
 
 ## DSL verbs
 

data/docs/getting-started.md

@@ -36,6 +36,18 @@ GoodJob.preserve_job_records = true
 
 GoodPipeline will raise `GoodPipeline::ConfigurationError` at boot if this is not set.
 
+## Configure queue names (optional)
+
+GoodPipeline routes its internal jobs to dedicated queues by default. You can override them globally:
+
+```ruby
+# config/initializers/good_pipeline.rb
+GoodPipeline.coordination_queue_name = "pipeline_coordination"  # StepFinishedJob, PipelineReconciliationJob
+GoodPipeline.callback_queue_name = "pipeline_callbacks"         # PipelineCallbackJob
+```
+
+Defaults are `"good_pipeline_coordination"` and `"good_pipeline_callbacks"`. Per-pipeline overrides are also available via the class DSL — see [Defining Pipelines](/defining-pipelines).
+
 ## Mount the dashboard (optional)
 
 ```ruby

data/docs/index.md

@@ -17,7 +17,7 @@ features:
   - title: Postgres only
     details: All state lives in Postgres. No Redis, no external dependencies. Step transitions and job enqueues happen in a single database transaction.
   - title: DAG orchestration
-    details: Define pipelines as directed acyclic graphs with the run and branch DSL. Steps run in parallel when possible, wait for dependencies automatically, and take different paths based on runtime decisions. Fan-out, fan-in, branching, and chaining are all there.
+    details: Define pipelines as directed acyclic graphs — not just linear chains. Steps run in parallel when possible, wait for dependencies automatically, and take different paths based on runtime decisions. Fan-out, fan-in, branching, and chaining are all first-class.
   - title: Web dashboard
     details: A mountable Rails engine with pipeline executions, step details, DAG visualization, and a pipeline definitions catalog. No build step.
 ---

data/docs/introduction.md

@@ -23,6 +23,24 @@ GoodJob's Batch feature fires a single `on_finish` callback when all jobs in a b
 
 GoodPipeline adds a coordination state machine, DAG validation, and atomic step transitions on top of Batch.
 
+### vs. Active Job Continuation (Rails 8.1)
+
+Rails 8.1 ships with `ActiveJob::Continuable`, which lets a single job define sequential steps with cursor-based progress tracking. If a deploy kills the process, the job resumes from its last checkpoint instead of restarting from scratch.
+
+This solves a different problem than GoodPipeline. Continuation makes one long-running job resilient to interruption. GoodPipeline orchestrates multiple independent jobs as a DAG with parallel execution, fan-out/fan-in, branching, and pipeline-level failure strategies.
+
+The two are complementary: a GoodPipeline step that processes millions of records could use `Continuable` internally for checkpoint/resume, while GoodPipeline handles the higher-level orchestration around it.
+
+### vs. Geneva Drive
+
+[Geneva Drive](https://github.com/julik/geneva_drive) is a durable workflow framework that executes steps strictly sequentially — one step at a time, like the mechanical gear it's named after. It works with any ActiveJob adapter (Sidekiq, Solid Queue, GoodJob) and supports PostgreSQL, MySQL, and SQLite.
+
+Geneva Drive is a strong choice for linear, long-lived workflows with pause/resume with human-in-the-loop recovery and per-hero workflow uniqueness constraints. Its layered exception policy system is particularly sophisticated.
+
+GoodPipeline takes a different approach: workflows are DAGs, not linear chains. Independent steps run in parallel across workers. Fan-out, fan-in, conditional branching, and pipeline chaining are first-class primitives. The tradeoff is that GoodPipeline requires GoodJob and PostgreSQL specifically, while Geneva Drive is adapter- and database-agnostic.
+
+Choose Geneva Drive when your workflow is inherently sequential and you need pause/resume or adapter flexibility. Choose GoodPipeline when steps can run concurrently, your workflow has branching or fan-in topology, or you want a built-in dashboard with DAG visualization.
+
 ## Features
 
 - `run` and `branch` DSL for defining step dependencies and conditional paths
@@ -39,4 +57,4 @@ GoodPipeline adds a coordination state machine, DAG validation, and atomic step
 - Ruby >= 3.2
 - Rails >= 7.1
 - PostgreSQL
-- GoodJob >= 3.10 with `preserve_job_records = true`
+- GoodJob >= 4.14 with `preserve_job_records = true`

data/docs/pipeline-chaining.md

@@ -79,6 +79,8 @@ GoodPipeline.run(
 
 Both pipelines start immediately. `MergeMediaPipeline` waits for both to succeed.
 
+Pipeline chaining is a first-class primitive — upstream/downstream relationships are tracked in a dedicated database table with atomic state propagation, rather than manually creating the next workflow in the last step of the current one.
+
 ## How `.then` works internally
 
 `.then` returns a `GoodPipeline::Chain` object which:

data/lib/good_pipeline/chain_coordinator.rb

@@ -42,12 +42,8 @@ module GoodPipeline
 
       def start_pipeline(pipeline_record)
         pipeline_record.transition_to!(:running)
-
         root_step_ids = pipeline_record.steps.where.missing(:upstream_dependencies).pluck(:id)
-
-        root_step_ids.each do |step_id|
-          Coordinator.try_enqueue_step(step_id)
-        end
+        Coordinator.bulk_enqueue_steps(root_step_ids)
       end
     end
   end

data/lib/good_pipeline/coordinator.rb

@@ -49,6 +49,23 @@ module GoodPipeline
         step_was_enqueued || downstream_enqueued
       end
 
+      # Enqueues multiple steps in bulk using Batch.enqueue_all.
+      # Intended for root steps during pipeline startup where no
+      # concurrent enqueue risk exists and no upstream checks are needed.
+      def bulk_enqueue_steps(step_ids)
+        return if step_ids.empty?
+
+        steps = StepRecord.where(id: step_ids, coordination_status: "pending")
+                          .where(good_job_id: nil)
+                          .to_a
+
+        branch_steps, enqueueable_steps = steps.partition(&:branch_step?)
+
+        bulk_enqueue_user_jobs(enqueueable_steps) if enqueueable_steps.any?
+
+        branch_steps.each { |step| try_enqueue_step(step.id) }
+      end
+
       def recompute_pipeline_status(pipeline, has_active_steps: nil, has_downstream_chains: nil) # rubocop:disable Metrics/MethodLength
         return if pipeline.terminal?
 
@@ -73,7 +90,8 @@ module GoodPipeline
 
           return if rows_updated.zero?
 
-          PipelineCallbackJob.perform_later(pipeline.id, new_status.to_s)
+          queue = pipeline.type.constantize.callback_queue_name
+          PipelineCallbackJob.set(queue: queue).perform_later(pipeline.id, new_status.to_s)
         end
       end
 
@@ -129,6 +147,18 @@ module GoodPipeline
         scope.update_all(coordination_status: "skipped")
       end
 
+      def transitive_downstream_ids(step)
+        visited = Set.new
+        queue = step.downstream_steps.pluck(:id)
+        while (current_id = queue.shift)
+          next if visited.include?(current_id)
+
+          visited << current_id
+          queue.concat(DependencyRecord.where(depends_on_step_id: current_id).pluck(:step_id))
+        end
+        visited
+      end
+
       def unblock_downstream_steps(step)
         sql = <<~SQL
           UPDATE good_pipeline_steps
@@ -226,6 +256,7 @@ module GoodPipeline
       def build_step_batch(step)
         batch = GoodJob::Batch.new
         batch.on_finish = "GoodPipeline::StepFinishedJob"
+        batch.callback_queue_name = step.pipeline.type.constantize.coordination_queue_name
         batch.properties = { step_id: step.id }
         batch
       end
@@ -236,6 +267,14 @@ module GoodPipeline
         enqueued_job.provider_job_id || enqueued_job.job_id
       end
 
+      def fail_step_with_error(step, error)
+        step.transition_coordination_status_to!(:failed)
+        step.update_columns(
+          error_class: error.class.name,
+          error_message: error.message
+        )
+      end
+
       def derive_terminal_status(pipeline)
         has_failures = pipeline.steps.where(coordination_status: "failed").exists?
 
@@ -245,24 +284,66 @@ module GoodPipeline
         :failed
       end
 
-      def transitive_downstream_ids(step)
-        visited = Set.new
-        queue = step.downstream_steps.pluck(:id)
-        while (current_id = queue.shift)
-          next if visited.include?(current_id)
+      def bulk_enqueue_user_jobs(steps) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength,Metrics/CyclomaticComplexity
+        batch_job_pairs = []
+        step_metadata = {}
+        failed_steps = []
+        coordination_queue = steps.first.pipeline.type.constantize.coordination_queue_name
+
+        steps.each do |step|
+          job_class = begin
+            step.job_class.constantize
+          rescue NameError => error
+            failed_steps << [step, ConfigurationError.new(error.message)]
+            next
+          end
+
+          batch = GoodJob::Batch.new
+          batch.on_finish = "GoodPipeline::StepFinishedJob"
+          batch.callback_queue_name = coordination_queue
+          batch.properties = { step_id: step.id }
+
+          active_job = job_class.new(**step.params.symbolize_keys)
+          apply_enqueue_options(active_job, step.enqueue_options.symbolize_keys)
+
+          batch_job_pairs << [batch, [active_job]]
+          step_metadata[step.id] = { batch: batch, active_job: active_job }
+        end
 
-          visited << current_id
-          queue.concat(DependencyRecord.where(depends_on_step_id: current_id).pluck(:step_id))
+        StepRecord.transaction do
+          GoodJob::Batch.enqueue_all(batch_job_pairs) if batch_job_pairs.any?
+
+          now = Time.current
+          step_metadata.each do |step_id, metadata|
+            StepRecord.where(id: step_id).update_all(
+              coordination_status: "enqueued",
+              good_job_batch_id: metadata[:batch].id,
+              good_job_id: metadata[:active_job].provider_job_id || metadata[:active_job].job_id,
+              updated_at: now
+            )
+          end
+        end
+      ensure
+        failed_steps.each do |step, error|
+          fail_step_with_error(step, error)
+          propagate_halt(step) if step.pipeline.halt?
         end
-        visited
       end
 
-      def fail_step_with_error(step, error)
-        step.transition_coordination_status_to!(:failed)
-        step.update_columns(
-          error_class: error.class.name,
-          error_message: error.message
-        )
+      def apply_enqueue_options(active_job, options) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
+        return if options.blank?
+
+        if options[:good_job_labels] && active_job.respond_to?(:good_job_labels=)
+          active_job.good_job_labels = Array(options[:good_job_labels])
+        end
+
+        if options.key?(:good_job_notify) && active_job.respond_to?(:good_job_notify=)
+          active_job.good_job_notify = options[:good_job_notify]
+        end
+
+        active_job.queue_name = options[:queue].to_s if options[:queue]
+        active_job.priority = options[:priority] if options[:priority]
+        active_job.scheduled_at = Time.current + options[:wait] if options[:wait]
       end
 
       def effective_failure_strategy(step)

data/lib/good_pipeline/pipeline.rb

@@ -5,7 +5,16 @@ module GoodPipeline
 
   class Pipeline # rubocop:disable Metrics/ClassLength
     VALID_FAILURE_STRATEGIES = %i[halt continue ignore].freeze
-    DSL_ATTRIBUTES = %i[display_name description failure_strategy on_complete on_success on_failure].freeze
+    DSL_ATTRIBUTES = %i[
+      display_name
+      description
+      failure_strategy
+      on_complete
+      on_success
+      on_failure
+      coordination_queue_name
+      callback_queue_name
+    ].freeze
 
     # --- Class-level DSL ---
 
@@ -58,6 +67,18 @@ module GoodPipeline
         @on_failure = method_name
       end
 
+      def coordination_queue_name(name = :__unset__)
+        return @coordination_queue_name || GoodPipeline.coordination_queue_name if name == :__unset__
+
+        @coordination_queue_name = name
+      end
+
+      def callback_queue_name(name = :__unset__)
+        return @callback_queue_name || GoodPipeline.callback_queue_name if name == :__unset__
+
+        @callback_queue_name = name
+      end
+
       alias build new
 
       def run(**)
@@ -95,6 +116,8 @@ module GoodPipeline
     def on_complete_callback = self.class.on_complete
     def on_success_callback = self.class.on_success
     def on_failure_callback = self.class.on_failure
+    def coordination_queue_name = self.class.coordination_queue_name
+    def callback_queue_name = self.class.callback_queue_name
 
     def initialize(**kwargs) # rubocop:disable Metrics/MethodLength
       @params = kwargs.freeze

data/lib/good_pipeline/runner.rb

@@ -32,6 +32,7 @@ module GoodPipeline
     def create_pipeline_batch(pipeline_id)
       batch = GoodJob::Batch.new
       batch.on_finish = "GoodPipeline::PipelineReconciliationJob"
+      batch.callback_queue_name = @pipeline.coordination_queue_name
       batch.properties = { pipeline_id: pipeline_id }
       batch.save
       batch
@@ -81,9 +82,8 @@ module GoodPipeline
     end
 
     def enqueue_root_steps(step_id_by_key)
-      @pipeline.root_steps.each do |step_definition|
-        Coordinator.try_enqueue_step(step_id_by_key[step_definition.key])
-      end
+      root_step_ids = @pipeline.root_steps.map { |step_definition| step_id_by_key[step_definition.key] }
+      Coordinator.bulk_enqueue_steps(root_step_ids)
     end
 
     def resolve_job_class(step_definition)

data/lib/good_pipeline/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module GoodPipeline
-  VERSION = "0.3.1"
+  VERSION = "0.4.0"
 end

data/lib/good_pipeline.rb

@@ -18,6 +18,21 @@ require_relative "good_pipeline/chain"
 require_relative "good_pipeline/engine" if defined?(Rails::Engine)
 
 module GoodPipeline
+  DEFAULT_COORDINATION_QUEUE_NAME = "good_pipeline_coordination"
+  DEFAULT_CALLBACK_QUEUE_NAME = "good_pipeline_callbacks"
+
+  class << self
+    attr_writer :coordination_queue_name, :callback_queue_name
+
+    def coordination_queue_name
+      @coordination_queue_name || DEFAULT_COORDINATION_QUEUE_NAME
+    end
+
+    def callback_queue_name
+      @callback_queue_name || DEFAULT_CALLBACK_QUEUE_NAME
+    end
+  end
+
   def self.run(*pipeline_configs)
     pipeline_records = pipeline_configs.map do |config|
       pipeline_class, pipeline_params = extract_pipeline_config(config)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: good_pipeline
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.0
 platform: ruby
 authors:
 - Ali Hamdi Ali Fadel
@@ -29,14 +29,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '3.10'
+        version: '4.14'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '3.10'
+        version: '4.14'
 - !ruby/object:Gem::Dependency
   name: railties
   requirement: !ruby/object:Gem::Requirement
@@ -120,6 +120,7 @@ files:
 - demo/db/seeds.rb
 - demo/docs/screenshots/definitions.png
 - demo/docs/screenshots/show.png
+- demo/test/good_pipeline/test_bulk_enqueue.rb
 - demo/test/good_pipeline/test_chain_record.rb
 - demo/test/good_pipeline/test_cleanup.rb
 - demo/test/good_pipeline/test_coordinator.rb
@@ -129,10 +130,12 @@ files:
 - demo/test/good_pipeline/test_pipeline_callback_job.rb
 - demo/test/good_pipeline/test_pipeline_reconciliation_job.rb
 - demo/test/good_pipeline/test_pipeline_record.rb
+- demo/test/good_pipeline/test_queue_configuration.rb
 - demo/test/good_pipeline/test_runner.rb
 - demo/test/good_pipeline/test_step_finished_job.rb
 - demo/test/good_pipeline/test_step_record.rb
 - demo/test/integration/test_branch_execution.rb
+- demo/test/integration/test_bulk_enqueue_end_to_end.rb
 - demo/test/integration/test_concurrent_fan_in.rb
 - demo/test/integration/test_end_to_end.rb
 - demo/test/integration/test_enqueue_atomicity.rb
@@ -142,6 +145,7 @@ files:
 - demo/test/integration/test_late_chain_registration.rb
 - demo/test/integration/test_missing_decision_method.rb
 - demo/test/integration/test_pipeline_chaining.rb
+- demo/test/integration/test_queue_configuration.rb
 - demo/test/integration/test_retry_scenarios.rb
 - demo/test/integration/test_sequential_branches.rb
 - demo/test/integration/test_step_finished_idempotency.rb