good_pipeline 0.4.0 → 0.6.0

data/demo/test/good_pipeline/test_bulk_enqueue.rb

@@ -61,7 +61,7 @@ class TestBulkEnqueue < ActiveSupport::TestCase
     batch_record = GoodJob::BatchRecord.find(step.reload.good_job_batch_id)
 
     assert_equal "GoodPipeline::StepFinishedJob", batch_record.on_finish
-    assert_equal({ step_id: step.id }, batch_record.properties)
+    assert_equal({ step_id: step.id, pipeline_id: pipeline.id }, batch_record.properties)
   end
 
   # --- enqueue_options ---
@@ -154,6 +154,22 @@ class TestBulkEnqueue < ActiveSupport::TestCase
     assert_nil result
   end
 
+  def test_rejects_steps_from_multiple_pipelines_before_enqueuing
+    pipeline_a = create_pipeline(on_failure_strategy: "halt", status: "running")
+    pipeline_b = create_pipeline(on_failure_strategy: "halt", status: "running")
+    step_a = build_step(pipeline_a, key: "step_a", job_class: "DownloadJob")
+    step_b = build_step(pipeline_b, key: "step_b", job_class: "TranscodeJob")
+
+    error = assert_raises(ArgumentError) do
+      GoodPipeline::Coordinator.bulk_enqueue_steps([step_a.id, step_b.id])
+    end
+
+    assert_match(/same pipeline/, error.message)
+    assert_nil step_a.reload.good_job_id
+    assert_nil step_b.reload.good_job_id
+    assert_equal %w[pending pending], [step_a.coordination_status, step_b.coordination_status]
+  end
+
   # --- branch step fallback ---
 
   def test_falls_back_to_try_enqueue_step_for_branch_steps
@@ -174,6 +190,17 @@ class TestBulkEnqueue < ActiveSupport::TestCase
                  "Branch step should have been processed by try_enqueue_step fallback"
   end
 
+  def test_falls_back_to_structural_resolution_for_barrier_steps
+    pipeline = create_pipeline(status: "running", on_failure_strategy: "halt")
+    barrier = build_step(pipeline, key: "barrier", job_class: GoodPipeline::BARRIER_JOB_CLASS)
+
+    GoodPipeline::Coordinator.bulk_enqueue_steps([barrier.id])
+
+    assert_equal "succeeded", barrier.reload.coordination_status
+    assert_nil barrier.good_job_id
+    assert_nil barrier.good_job_batch_id
+  end
+
   # --- error handling ---
 
   def test_invalid_job_class_fails_step_without_blocking_others
@@ -190,4 +217,30 @@ class TestBulkEnqueue < ActiveSupport::TestCase
     assert_equal "failed", bad_step.reload.coordination_status
     assert_equal "GoodPipeline::ConfigurationError", bad_step.error_class
   end
+
+  def test_each_bulk_configuration_failure_applies_its_halt_scope
+    pipeline = create_pipeline(on_failure_strategy: "halt")
+    pipeline.update_columns(status: "running")
+    ignored_failure = build_step(
+      pipeline,
+      id: "00000000-0000-4000-8000-000000000001",
+      key: "ignored_failure",
+      job_class: "MissingIgnoredJob",
+      on_failure_strategy: "ignore"
+    )
+    strict_failure = build_step(
+      pipeline,
+      id: "ffffffff-ffff-4fff-8fff-ffffffffffff",
+      key: "strict_failure",
+      job_class: "MissingStrictJob"
+    )
+    dependent = build_step(pipeline, key: "dependent", dependencies: [ignored_failure])
+
+    GoodPipeline::Coordinator.bulk_enqueue_steps([ignored_failure.id, strict_failure.id])
+
+    assert_equal "failed", ignored_failure.reload.coordination_status
+    assert_equal "failed", strict_failure.reload.coordination_status
+    assert_equal "skipped", dependent.reload.coordination_status
+    assert_equal "halted", pipeline.reload.status
+  end
 end

data/demo/test/good_pipeline/test_cancellation.rb

@@ -0,0 +1,312 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+# rubocop:disable Metrics/AbcSize, Metrics/ClassLength, Metrics/MethodLength
+class TestCancellation < ActiveSupport::TestCase
+  def test_cancel_pending_pipeline_cancels_pending_steps_and_finishes
+    pipeline = create_pipeline(on_failure_strategy: "halt")
+    step_a = build_step(pipeline, key: "a")
+    step_b = build_step(pipeline, key: "b", dependencies: [step_a])
+
+    result = GoodPipeline::Coordinator.cancel_pipeline(pipeline.id)
+
+    assert_equal pipeline.id, result.id
+    assert_equal "canceled", result.status
+    assert_equal "canceled", pipeline.reload.status
+    assert_equal %w[canceled canceled], [step_a.reload.coordination_status, step_b.reload.coordination_status]
+    refute_nil pipeline.callbacks_dispatched_at
+  end
+
+  def test_cancel_running_pipeline_without_enqueued_work_finishes_immediately
+    pipeline = create_pipeline(on_failure_strategy: "halt")
+    pipeline.update_columns(status: "running")
+    step = build_step(pipeline, key: "a")
+
+    GoodPipeline::Coordinator.cancel_pipeline(pipeline)
+
+    assert_equal "canceled", pipeline.reload.status
+    assert_equal "canceled", step.reload.coordination_status
+  end
+
+  def test_cancel_running_pipeline_drains_enqueued_work
+    pipeline = create_pipeline(on_failure_strategy: "halt")
+    pipeline.update_columns(status: "running")
+    enqueued_step = build_step(pipeline, key: "enqueued")
+    pending_step = build_step(pipeline, key: "pending", dependencies: [enqueued_step])
+    enqueued_step.update_columns(coordination_status: "enqueued")
+
+    result = GoodPipeline::Coordinator.cancel_pipeline(pipeline.id)
+
+    assert_equal "canceling", result.status
+    assert_equal "canceling", pipeline.reload.status
+    assert_equal "enqueued", enqueued_step.reload.coordination_status
+    assert_equal "canceled", pending_step.reload.coordination_status
+    assert_nil pipeline.callbacks_dispatched_at
+  end
+
+  def test_cancel_is_idempotent_while_canceling_and_after_canceled
+    pipeline = create_pipeline(on_failure_strategy: "halt")
+    pipeline.update_columns(status: "running")
+    step = build_step(pipeline, key: "a")
+    step.update_columns(coordination_status: "enqueued")
+
+    first = GoodPipeline::Coordinator.cancel_pipeline(pipeline.id)
+    second = GoodPipeline::Coordinator.cancel_pipeline(pipeline.id)
+
+    assert_equal "canceling", first.status
+    assert_equal "canceling", second.status
+    assert_nil pipeline.reload.callbacks_dispatched_at
+
+    GoodPipeline::Coordinator.complete_step(step.id, succeeded: true)
+    dispatched_at = pipeline.reload.callbacks_dispatched_at
+    third = GoodPipeline::Coordinator.cancel_pipeline(pipeline.id)
+
+    assert_equal "canceled", third.status
+    assert_equal dispatched_at, pipeline.reload.callbacks_dispatched_at
+    assert_equal 1, callback_jobs_for(pipeline).count
+  end
+
+  def test_cancel_rejects_unrelated_terminal_status
+    %w[succeeded failed halted skipped].each do |status|
+      pipeline = create_pipeline(on_failure_strategy: "halt")
+      pipeline.update_columns(status: status)
+
+      error = assert_raises(GoodPipeline::CancellationConflict) do
+        GoodPipeline::Coordinator.cancel_pipeline(pipeline.id)
+      end
+
+      assert_equal pipeline.id, error.pipeline_id
+      assert_equal status, error.status
+    end
+  end
+
+  def test_terminal_transition_and_callback_enqueue_roll_back_together
+    pipeline = create_pipeline(on_failure_strategy: "halt")
+    build_step(pipeline, key: "a")
+
+    GoodPipeline::PipelineRecord.transaction do
+      GoodPipeline::Coordinator.cancel_pipeline(pipeline.id)
+      raise ActiveRecord::Rollback
+    end
+
+    assert_equal "pending", pipeline.reload.status
+    assert_nil pipeline.callbacks_dispatched_at
+    assert_empty callback_jobs_for(pipeline)
+  end
+
+  def test_single_enqueue_is_guarded_once_canceling
+    pipeline = create_pipeline(on_failure_strategy: "halt")
+    pipeline.update_columns(status: "canceling")
+    step = build_step(pipeline, key: "a")
+
+    result = GoodPipeline::Coordinator.try_enqueue_step(step.id)
+
+    refute result
+    assert_equal "pending", step.reload.coordination_status
+    assert_nil step.good_job_id
+  end
+
+  def test_bulk_enqueue_is_guarded_once_canceling
+    pipeline = create_pipeline(on_failure_strategy: "halt")
+    pipeline.update_columns(status: "canceling")
+    step = build_step(pipeline, key: "a")
+
+    GoodPipeline::Coordinator.bulk_enqueue_steps([step.id])
+
+    assert_equal "pending", step.reload.coordination_status
+    assert_nil step.good_job_id
+  end
+
+  def test_successful_completion_during_drain_preserves_outcome_and_finalizes
+    pipeline, step = canceling_pipeline_with_enqueued_step
+
+    GoodPipeline::Coordinator.complete_step(step, succeeded: true)
+
+    assert_equal "succeeded", step.reload.coordination_status
+    assert_equal "canceled", pipeline.reload.status
+  end
+
+  def test_failed_completion_during_drain_preserves_outcome_and_metadata
+    pipeline, step = canceling_pipeline_with_enqueued_step
+    create_good_job_for_step(step, error: "RuntimeError: drain failed", executions_count: 2)
+
+    GoodPipeline::Coordinator.complete_step(step.id, succeeded: false)
+
+    step.reload
+
+    assert_equal "failed", step.coordination_status
+    assert_equal "RuntimeError", step.error_class
+    assert_equal "drain failed", step.error_message
+    assert_equal 2, step.attempts
+    assert_equal "canceled", pipeline.reload.status
+    refute_predicate pipeline, :halt_triggered?
+  end
+
+  def test_halt_requested_completion_during_drain_preserves_halted_step
+    pipeline, step = canceling_pipeline_with_enqueued_step
+    step.update_columns(halt_requested: true)
+
+    GoodPipeline::Coordinator.complete_step(step.id, succeeded: true)
+
+    assert_equal "halted", step.reload.coordination_status
+    assert_equal "canceled", pipeline.reload.status
+  end
+
+  def test_only_last_enqueued_completion_finalizes_cancellation
+    pipeline = create_pipeline(on_failure_strategy: "halt")
+    pipeline.update_columns(status: "running")
+    step_a = build_step(pipeline, key: "a")
+    step_b = build_step(pipeline, key: "b")
+    [step_a, step_b].each { |step| step.update_columns(coordination_status: "enqueued") }
+    GoodPipeline::Coordinator.cancel_pipeline(pipeline.id)
+
+    GoodPipeline::Coordinator.complete_step(step_a.id, succeeded: true)
+
+    assert_equal "canceling", pipeline.reload.status
+    assert_nil pipeline.callbacks_dispatched_at
+
+    GoodPipeline::Coordinator.complete_step(step_b.id, succeeded: false)
+
+    assert_equal "canceled", pipeline.reload.status
+    refute_nil pipeline.callbacks_dispatched_at
+    assert_equal 1, callback_jobs_for(pipeline).count
+  end
+
+  def test_recompute_uses_fresh_activity_and_prioritizes_cancellation
+    pipeline = create_pipeline(on_failure_strategy: "halt")
+    pipeline.update_columns(status: "canceling")
+    step = build_step(pipeline, key: "a")
+    pipeline.steps.load
+    step.update_columns(coordination_status: "succeeded")
+
+    GoodPipeline::Coordinator.recompute_pipeline_status(pipeline)
+
+    assert_equal "canceled", pipeline.reload.status
+  end
+
+  def test_cancel_preserves_real_scheduled_good_job
+    pipeline = create_pipeline(on_failure_strategy: "halt")
+    pipeline.update_columns(status: "running")
+    step = build_step(
+      pipeline,
+      key: "scheduled",
+      enqueue_options: { "wait" => 3600, "queue" => "slow" }
+    )
+    dependent = build_step(pipeline, key: "dependent", dependencies: [step])
+    GoodPipeline::Coordinator.bulk_enqueue_steps([step.id])
+    step.reload
+    good_job = GoodJob::Job.find(step.good_job_id)
+    job_count = GoodJob::Job.count
+    step_job_ids = [step.good_job_id, step.good_job_batch_id]
+    job_snapshot = good_job.attributes.slice("id", "batch_id", "scheduled_at", "finished_at", "queue_name", "priority")
+
+    GoodPipeline::Coordinator.cancel_pipeline(pipeline.id)
+
+    assert_equal "canceling", pipeline.reload.status
+    assert_equal "enqueued", step.reload.coordination_status
+    assert_equal step_job_ids, [step.good_job_id, step.good_job_batch_id]
+    assert_equal "canceled", dependent.reload.coordination_status
+    assert_equal job_count, GoodJob::Job.count
+    assert_equal job_snapshot, good_job.reload.attributes.slice(*job_snapshot.keys)
+    assert_operator good_job.scheduled_at, :>, Time.current
+
+    GoodPipeline::Coordinator.complete_step(step.id, succeeded: true)
+
+    assert_equal "succeeded", step.reload.coordination_status
+    assert_equal "canceled", pipeline.reload.status
+  end
+
+  def test_cancel_and_enqueue_race_has_only_serializable_outcomes
+    5.times do |iteration|
+      pipeline = create_pipeline(on_failure_strategy: "halt")
+      pipeline.update_columns(status: "running")
+      step = build_step(pipeline, key: "race_#{iteration}")
+      latch = Concurrent::CountDownLatch.new(2)
+
+      cancel = rails_promise do
+        latch.count_down
+        latch.wait(5)
+        GoodPipeline::Coordinator.cancel_pipeline(pipeline.id)
+      end
+      enqueue = rails_promise do
+        latch.count_down
+        latch.wait(5)
+        GoodPipeline::Coordinator.try_enqueue_step(step.id)
+      end
+
+      cancel.value!
+      enqueue.value!
+
+      assert_serialized_cancel_enqueue_outcome(pipeline, step, iteration)
+    end
+  end
+
+  def test_cancel_and_bulk_enqueue_race_has_only_serializable_outcomes
+    5.times do |iteration|
+      pipeline = create_pipeline(on_failure_strategy: "halt")
+      pipeline.update_columns(status: "running")
+      step = build_step(pipeline, key: "bulk_race_#{iteration}")
+      latch = Concurrent::CountDownLatch.new(2)
+
+      cancel = rails_promise do
+        latch.count_down
+        latch.wait(5)
+        GoodPipeline::Coordinator.cancel_pipeline(pipeline.id)
+      end
+      enqueue = rails_promise do
+        latch.count_down
+        latch.wait(5)
+        GoodPipeline::Coordinator.bulk_enqueue_steps([step.id])
+      end
+
+      cancel.value!
+      enqueue.value!
+
+      assert_serialized_cancel_enqueue_outcome(pipeline, step, iteration)
+    end
+  end
+
+  private
+
+  def canceling_pipeline_with_enqueued_step
+    pipeline = create_pipeline(on_failure_strategy: "halt")
+    pipeline.update_columns(status: "running")
+    step = build_step(pipeline, key: "a")
+    step.update_columns(coordination_status: "enqueued")
+    GoodPipeline::Coordinator.cancel_pipeline(pipeline.id)
+    [pipeline, step]
+  end
+
+  def create_good_job_for_step(step, error:, executions_count:)
+    good_job = GoodJob::Job.create!(
+      id: SecureRandom.uuid,
+      active_job_id: SecureRandom.uuid,
+      job_class: step.job_class,
+      error: error,
+      executions_count: executions_count,
+      finished_at: Time.current
+    )
+    step.update_column(:good_job_id, good_job.id)
+  end
+
+  def callback_jobs_for(pipeline)
+    GoodJob::Job.where(job_class: "GoodPipeline::PipelineCallbackJob")
+                .where("serialized_params -> 'arguments' ->> 0 = ?", pipeline.id)
+  end
+
+  def assert_serialized_cancel_enqueue_outcome(pipeline, step, iteration)
+    pipeline.reload
+    step.reload
+
+    if pipeline.canceled?
+      assert_equal "canceled", step.coordination_status, "iteration #{iteration}"
+      assert_nil step.good_job_id, "iteration #{iteration}"
+    else
+      assert_equal "canceling", pipeline.status, "iteration #{iteration}"
+      assert_equal "enqueued", step.coordination_status, "iteration #{iteration}"
+      refute_nil step.good_job_id, "iteration #{iteration}"
+    end
+  end
+end
+# rubocop:enable Metrics/AbcSize, Metrics/ClassLength, Metrics/MethodLength

data/demo/test/good_pipeline/test_cleanup.rb

@@ -28,6 +28,9 @@ class TestCleanup < ActiveSupport::TestCase
       pipeline: @running_pipeline, key: "step_a", job_class: "DownloadJob", coordination_status: "enqueued"
     )
 
+    @canceling_pipeline = create_old_pipeline(status: "canceling", step_status: "enqueued")
+    @canceled_pipeline = create_old_pipeline(status: "canceled", step_status: "canceled")
+
     # Recent terminal pipeline (should NOT be cleaned up)
     @recent_pipeline = GoodPipeline::PipelineRecord.create!(
       type: "TestPipeline", status: "failed", on_failure_strategy: "halt"
@@ -56,6 +59,18 @@ class TestCleanup < ActiveSupport::TestCase
     assert_not_nil GoodPipeline::PipelineRecord.find_by(id: @running_pipeline.id)
   end
 
+  test "preserves canceling pipelines" do
+    GoodPipeline.cleanup_preserved_pipelines(older_than: @now - 14.days)
+
+    assert_not_nil GoodPipeline::PipelineRecord.find_by(id: @canceling_pipeline.id)
+  end
+
+  test "cleans canceled pipelines" do
+    GoodPipeline.cleanup_preserved_pipelines(older_than: @now - 14.days)
+
+    assert_nil GoodPipeline::PipelineRecord.find_by(id: @canceled_pipeline.id)
+  end
+
   test "preserves recent terminal pipelines" do
     GoodPipeline.cleanup_preserved_pipelines(older_than: @now - 14.days)
 
@@ -80,14 +95,29 @@ class TestCleanup < ActiveSupport::TestCase
   test "noop when nothing to clean" do
     GoodPipeline.cleanup_preserved_pipelines(older_than: @now - 365.days)
 
-    assert_equal 3, GoodPipeline::PipelineRecord.count
+    assert_equal 5, GoodPipeline::PipelineRecord.count
   end
 
   test "triggers cleanup when GoodJob cleans preserved jobs" do
     GoodJob.cleanup_preserved_jobs(older_than: 14.days)
 
     assert_nil GoodPipeline::PipelineRecord.find_by(id: @old_pipeline.id)
+    assert_nil GoodPipeline::PipelineRecord.find_by(id: @canceled_pipeline.id)
     assert_not_nil GoodPipeline::PipelineRecord.find_by(id: @running_pipeline.id)
+    assert_not_nil GoodPipeline::PipelineRecord.find_by(id: @canceling_pipeline.id)
     assert_not_nil GoodPipeline::PipelineRecord.find_by(id: @recent_pipeline.id)
   end
+
+  private
+
+  def create_old_pipeline(status:, step_status:)
+    pipeline = GoodPipeline::PipelineRecord.create!(
+      type: "TestPipeline", status: status, on_failure_strategy: "halt"
+    )
+    pipeline.update_columns(updated_at: @now - 30.days)
+    GoodPipeline::StepRecord.create!(
+      pipeline: pipeline, key: "step_a", job_class: "DownloadJob", coordination_status: step_status
+    )
+    pipeline
+  end
 end

data/demo/test/good_pipeline/test_coordinator.rb

@@ -31,6 +31,18 @@ class TestCoordinator < ActiveSupport::TestCase
     assert_equal "succeeded", step.reload.coordination_status
   end
 
+  def test_terminal_step_redelivery_repairs_running_pipeline_status
+    pipeline = create_pipeline(on_failure_strategy: "halt")
+    pipeline.update_columns(status: "running")
+    step = build_step(pipeline, key: "a")
+    step.update_columns(coordination_status: "succeeded")
+
+    GoodPipeline::Coordinator.complete_step(step.id, succeeded: true)
+
+    assert_equal "succeeded", pipeline.reload.status
+    refute_nil pipeline.callbacks_dispatched_at
+  end
+
   # --- complete_step: succeeded ---
 
   def test_complete_step_succeeded_transitions
@@ -127,6 +139,14 @@ class TestCoordinator < ActiveSupport::TestCase
     assert_equal "succeeded", pipeline.reload.status
   end
 
+  def test_recompute_rejects_removed_activity_hints
+    pipeline = create_pipeline(on_failure_strategy: "halt")
+
+    assert_raises(ArgumentError) do
+      GoodPipeline::Coordinator.recompute_pipeline_status(pipeline, has_active_steps: true)
+    end
+  end
+
   # --- dispatch_callbacks_once ---
 
   def test_dispatch_callbacks_sets_callbacks_dispatched_at
@@ -261,6 +281,24 @@ class TestCoordinator < ActiveSupport::TestCase
 
   # --- try_enqueue_step ---
 
+  def test_try_enqueue_batch_includes_pipeline_ownership
+    pipeline = create_pipeline(on_failure_strategy: "halt")
+    pipeline.update_columns(status: "running")
+    step = build_step(pipeline, key: "a")
+
+    GoodPipeline::Coordinator.try_enqueue_step(step.id)
+
+    batch = GoodJob::BatchRecord.find(step.reload.good_job_batch_id)
+
+    assert_equal({ step_id: step.id, pipeline_id: pipeline.id }, batch.properties)
+  end
+
+  def test_try_enqueue_returns_false_for_missing_step
+    result = GoodPipeline::Coordinator.try_enqueue_step(SecureRandom.uuid)
+
+    refute result
+  end
+
   def test_try_enqueue_bails_on_non_pending_step
     pipeline = create_pipeline(on_failure_strategy: "halt")
     step = build_step(pipeline, key: "a")
@@ -293,6 +331,7 @@ class TestCoordinator < ActiveSupport::TestCase
 
   def test_try_enqueue_skips_permanently_unsatisfied_step
     pipeline = create_pipeline(on_failure_strategy: "continue")
+    pipeline.update_columns(status: "running")
     step_a = build_step(pipeline, key: "a")
     step_b = build_step(pipeline, key: "b", dependencies: [step_a])
     step_a.update_columns(coordination_status: "failed")