good_pipeline 0.1.0

data/demo/test/good_pipeline/test_pipeline_record.rb

@@ -0,0 +1,183 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class TestPipelineRecord < ActiveSupport::TestCase
+  # --- Defaults ---
+
+  def test_default_status_is_pending
+    pipeline = create_pipeline
+
+    assert_equal "pending", pipeline.status
+  end
+
+  def test_default_halt_triggered_is_false
+    pipeline = create_pipeline
+
+    refute pipeline.halt_triggered
+  end
+
+  def test_default_params_is_empty_hash
+    pipeline = create_pipeline
+
+    assert_equal({}, pipeline.params)
+  end
+
+  # --- UUID primary key ---
+
+  def test_id_is_uuid
+    pipeline = create_pipeline
+
+    assert_match(/\A[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\z/, pipeline.id)
+  end
+
+  # --- STI disabled ---
+
+  def test_type_column_does_not_trigger_sti
+    pipeline = create_pipeline(type: "VideoProcessingPipeline")
+    reloaded = GoodPipeline::PipelineRecord.find(pipeline.id)
+
+    assert_instance_of GoodPipeline::PipelineRecord, reloaded
+    assert_equal "VideoProcessingPipeline", reloaded.type
+  end
+
+  # --- terminal? ---
+
+  def test_terminal_returns_false_for_pending
+    pipeline = create_pipeline
+
+    refute_predicate pipeline, :terminal?
+  end
+
+  def test_terminal_returns_false_for_running
+    pipeline = create_pipeline
+    pipeline.update_columns(status: "running")
+
+    refute_predicate pipeline, :terminal?
+  end
+
+  def test_terminal_returns_true_for_succeeded
+    pipeline = create_pipeline
+    pipeline.update_columns(status: "succeeded")
+
+    assert_predicate pipeline, :terminal?
+  end
+
+  def test_terminal_returns_true_for_failed
+    pipeline = create_pipeline
+    pipeline.update_columns(status: "failed")
+
+    assert_predicate pipeline, :terminal?
+  end
+
+  def test_terminal_returns_true_for_halted
+    pipeline = create_pipeline
+    pipeline.update_columns(status: "halted")
+
+    assert_predicate pipeline, :terminal?
+  end
+
+  def test_terminal_returns_true_for_skipped
+    pipeline = create_pipeline
+    pipeline.update_columns(status: "skipped")
+
+    assert_predicate pipeline, :terminal?
+  end
+
+  # --- transition_to! valid transitions ---
+
+  def test_transition_pending_to_running
+    pipeline = create_pipeline
+    pipeline.transition_to!(:running)
+
+    assert_equal "running", pipeline.status
+  end
+
+  def test_transition_pending_to_skipped
+    pipeline = create_pipeline
+    pipeline.transition_to!(:skipped)
+
+    assert_equal "skipped", pipeline.status
+  end
+
+  def test_transition_running_to_succeeded
+    pipeline = create_pipeline
+    pipeline.transition_to!(:running)
+    pipeline.transition_to!(:succeeded)
+
+    assert_equal "succeeded", pipeline.status
+  end
+
+  def test_transition_running_to_failed
+    pipeline = create_pipeline
+    pipeline.transition_to!(:running)
+    pipeline.transition_to!(:failed)
+
+    assert_equal "failed", pipeline.status
+  end
+
+  def test_transition_running_to_halted
+    pipeline = create_pipeline
+    pipeline.transition_to!(:running)
+    pipeline.transition_to!(:halted)
+
+    assert_equal "halted", pipeline.status
+  end
+
+  # --- transition_to! invalid transitions ---
+
+  def test_transition_pending_to_succeeded_raises
+    pipeline = create_pipeline
+    error = assert_raises(GoodPipeline::InvalidTransition) { pipeline.transition_to!(:succeeded) }
+    assert_includes error.message, "cannot transition pipeline from 'pending' to 'succeeded'"
+  end
+
+  def test_transition_pending_to_failed_raises
+    pipeline = create_pipeline
+    error = assert_raises(GoodPipeline::InvalidTransition) { pipeline.transition_to!(:failed) }
+    assert_includes error.message, "from 'pending' to 'failed'"
+  end
+
+  def test_transition_running_to_pending_raises
+    pipeline = create_pipeline
+    pipeline.transition_to!(:running)
+    assert_raises(GoodPipeline::InvalidTransition) { pipeline.transition_to!(:pending) }
+  end
+
+  def test_transition_from_terminal_succeeded_raises
+    pipeline = create_pipeline
+    pipeline.update_columns(status: "succeeded")
+    pipeline.reload
+    assert_raises(GoodPipeline::InvalidTransition) { pipeline.transition_to!(:running) }
+  end
+
+  def test_transition_from_terminal_failed_raises
+    pipeline = create_pipeline
+    pipeline.update_columns(status: "failed")
+    pipeline.reload
+    assert_raises(GoodPipeline::InvalidTransition) { pipeline.transition_to!(:running) }
+  end
+
+  def test_transition_from_terminal_halted_raises
+    pipeline = create_pipeline
+    pipeline.update_columns(status: "halted")
+    pipeline.reload
+    assert_raises(GoodPipeline::InvalidTransition) { pipeline.transition_to!(:running) }
+  end
+
+  def test_transition_from_terminal_skipped_raises
+    pipeline = create_pipeline
+    pipeline.update_columns(status: "skipped")
+    pipeline.reload
+    assert_raises(GoodPipeline::InvalidTransition) { pipeline.transition_to!(:running) }
+  end
+
+  # --- transition_to! accepts symbols ---
+
+  def test_transition_to_accepts_symbols
+    pipeline = create_pipeline
+    pipeline.transition_to!(:running)
+
+    assert_equal "running", pipeline.status
+  end
+end

data/demo/test/good_pipeline/test_runner.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class TestRunner < ActiveSupport::TestCase
+  TestPipeline = Class.new(GoodPipeline::Pipeline) do
+    description "Test pipeline"
+    failure_strategy :halt
+
+    define_method(:configure) do |video_id:, **|
+      run :download, DownloadJob, with: { video_id: video_id }
+      run :transcode, TranscodeJob, after: :download
+      run :thumbnail, ThumbnailJob, after: :download
+    end
+  end
+
+  def test_creates_pipeline_record
+    klass = TestPipeline
+    instance = klass.build(video_id: 42)
+
+    record = GoodPipeline::Runner.call(instance)
+
+    assert_instance_of GoodPipeline::PipelineRecord, record
+    assert_equal klass.name, record.type
+    assert_equal({ "video_id" => 42 }, record.params)
+    assert_equal "running", record.status
+    assert_equal "halt", record.on_failure_strategy
+  end
+
+  def test_creates_step_records
+    klass = TestPipeline
+    instance = klass.build(video_id: 42)
+
+    record = GoodPipeline::Runner.call(instance)
+
+    steps = record.steps.order(:key)
+
+    assert_equal 3, steps.count
+    assert_equal %w[download thumbnail transcode], steps.map(&:key)
+    assert_equal "DownloadJob", steps.find_by(key: "download").job_class
+    assert_equal({ "video_id" => 42 }, steps.find_by(key: "download").params)
+  end
+
+  def test_creates_dependency_records
+    klass = TestPipeline
+    instance = klass.build(video_id: 42)
+
+    record = GoodPipeline::Runner.call(instance)
+
+    deps = record.dependencies
+
+    assert_equal 2, deps.count
+
+    download = record.steps.find_by(key: "download")
+    transcode = record.steps.find_by(key: "transcode")
+    thumbnail = record.steps.find_by(key: "thumbnail")
+
+    transcode_dep = deps.find_by(step: transcode)
+
+    assert_equal download.id, transcode_dep.depends_on_step_id
+
+    thumbnail_dep = deps.find_by(step: thumbnail)
+
+    assert_equal download.id, thumbnail_dep.depends_on_step_id
+  end
+
+  def test_enqueues_root_steps
+    klass = TestPipeline
+    instance = klass.build(video_id: 42)
+
+    record = GoodPipeline::Runner.call(instance)
+
+    download = record.steps.find_by(key: "download")
+    # Root step should have been enqueued (may have already completed)
+    refute_equal "pending", download.reload.coordination_status
+  end
+
+  def test_stores_pipeline_batch_id
+    klass = TestPipeline
+    instance = klass.build(video_id: 42)
+
+    record = GoodPipeline::Runner.call(instance)
+
+    assert_not_nil record.good_job_batch_id
+  end
+end

data/demo/test/good_pipeline/test_step_finished_job.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class TestStepFinishedJob < ActiveSupport::TestCase
+  def test_completes_step_as_succeeded_via_batch
+    pipeline = create_pipeline
+    pipeline.update_columns(status: "running")
+    step = create_step(pipeline, key: "a")
+    step.update_columns(coordination_status: "enqueued")
+
+    batch = GoodJob::Batch.new
+    batch.properties = { step_id: step.id }
+    batch.enqueue {}
+
+    GoodPipeline::StepFinishedJob.new.perform(batch, {})
+
+    assert_equal "succeeded", step.reload.coordination_status
+  end
+
+  def test_completes_step_as_failed_when_batch_discarded
+    pipeline = create_pipeline
+    pipeline.update_columns(status: "running")
+    step = create_step(pipeline, key: "a")
+    step.update_columns(coordination_status: "enqueued")
+
+    batch = GoodJob::Batch.new
+    batch.properties = { step_id: step.id }
+    batch.enqueue {}
+    GoodJob::BatchRecord.where(id: batch.id).update_all(discarded_at: Time.current)
+    batch = GoodJob::Batch.find(batch.id)
+
+    GoodPipeline::StepFinishedJob.new.perform(batch, {})
+
+    assert_equal "failed", step.reload.coordination_status
+  end
+end

data/demo/test/good_pipeline/test_step_record.rb

@@ -0,0 +1,208 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class TestStepRecord < ActiveSupport::TestCase
+  # --- Defaults ---
+
+  def test_default_coordination_status_is_pending
+    pipeline = create_pipeline
+    step = create_step(pipeline)
+
+    assert_equal "pending", step.coordination_status
+  end
+
+  # --- UUID primary key ---
+
+  def test_id_is_uuid
+    pipeline = create_pipeline
+    step = create_step(pipeline)
+
+    assert_match(/\A[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\z/, step.id)
+  end
+
+  # --- terminal_coordination_status? ---
+
+  def test_terminal_returns_false_for_pending
+    pipeline = create_pipeline
+    step = create_step(pipeline)
+
+    refute_predicate step, :terminal_coordination_status?
+  end
+
+  def test_terminal_returns_false_for_enqueued
+    pipeline = create_pipeline
+    step = create_step(pipeline)
+    step.update_columns(coordination_status: "enqueued")
+
+    refute_predicate step, :terminal_coordination_status?
+  end
+
+  def test_terminal_returns_true_for_succeeded
+    pipeline = create_pipeline
+    step = create_step(pipeline)
+    step.update_columns(coordination_status: "succeeded")
+
+    assert_predicate step, :terminal_coordination_status?
+  end
+
+  def test_terminal_returns_true_for_failed
+    pipeline = create_pipeline
+    step = create_step(pipeline)
+    step.update_columns(coordination_status: "failed")
+
+    assert_predicate step, :terminal_coordination_status?
+  end
+
+  def test_terminal_returns_true_for_skipped
+    pipeline = create_pipeline
+    step = create_step(pipeline)
+    step.update_columns(coordination_status: "skipped")
+
+    assert_predicate step, :terminal_coordination_status?
+  end
+
+  # --- transition_coordination_status_to! valid transitions ---
+
+  def test_transition_pending_to_enqueued
+    pipeline = create_pipeline
+    step = create_step(pipeline)
+    step.transition_coordination_status_to!(:enqueued)
+
+    assert_equal "enqueued", step.coordination_status
+  end
+
+  def test_transition_pending_to_skipped
+    pipeline = create_pipeline
+    step = create_step(pipeline)
+    step.transition_coordination_status_to!(:skipped)
+
+    assert_equal "skipped", step.coordination_status
+  end
+
+  def test_transition_enqueued_to_succeeded
+    pipeline = create_pipeline
+    step = create_step(pipeline)
+    step.transition_coordination_status_to!(:enqueued)
+    step.transition_coordination_status_to!(:succeeded)
+
+    assert_equal "succeeded", step.coordination_status
+  end
+
+  def test_transition_enqueued_to_failed
+    pipeline = create_pipeline
+    step = create_step(pipeline)
+    step.transition_coordination_status_to!(:enqueued)
+    step.transition_coordination_status_to!(:failed)
+
+    assert_equal "failed", step.coordination_status
+  end
+
+  # --- transition_coordination_status_to! invalid transitions ---
+
+  def test_transition_pending_to_succeeded_raises
+    pipeline = create_pipeline
+    step = create_step(pipeline)
+    assert_raises(GoodPipeline::InvalidTransition) do
+      step.transition_coordination_status_to!(:succeeded)
+    end
+  end
+
+  def test_transition_pending_to_failed_raises
+    pipeline = create_pipeline
+    step = create_step(pipeline)
+    assert_raises(GoodPipeline::InvalidTransition) do
+      step.transition_coordination_status_to!(:failed)
+    end
+  end
+
+  def test_transition_enqueued_to_pending_raises
+    pipeline = create_pipeline
+    step = create_step(pipeline)
+    step.transition_coordination_status_to!(:enqueued)
+    assert_raises(GoodPipeline::InvalidTransition) do
+      step.transition_coordination_status_to!(:pending)
+    end
+  end
+
+  def test_transition_enqueued_to_skipped_raises
+    pipeline = create_pipeline
+    step = create_step(pipeline)
+    step.transition_coordination_status_to!(:enqueued)
+    assert_raises(GoodPipeline::InvalidTransition) do
+      step.transition_coordination_status_to!(:skipped)
+    end
+  end
+
+  def test_transition_from_terminal_succeeded_raises
+    pipeline = create_pipeline
+    step = create_step(pipeline)
+    step.update_columns(coordination_status: "succeeded")
+    step.reload
+    assert_raises(GoodPipeline::InvalidTransition) do
+      step.transition_coordination_status_to!(:enqueued)
+    end
+  end
+
+  def test_transition_from_terminal_failed_raises
+    pipeline = create_pipeline
+    step = create_step(pipeline)
+    step.update_columns(coordination_status: "failed")
+    step.reload
+    assert_raises(GoodPipeline::InvalidTransition) do
+      step.transition_coordination_status_to!(:enqueued)
+    end
+  end
+
+  def test_transition_from_terminal_skipped_raises
+    pipeline = create_pipeline
+    step = create_step(pipeline)
+    step.update_columns(coordination_status: "skipped")
+    step.reload
+    assert_raises(GoodPipeline::InvalidTransition) do
+      step.transition_coordination_status_to!(:enqueued)
+    end
+  end
+
+  # --- Error message includes step key ---
+
+  def test_error_message_includes_step_key
+    pipeline = create_pipeline
+    step = create_step(pipeline, key: "transcode")
+    error = assert_raises(GoodPipeline::InvalidTransition) do
+      step.transition_coordination_status_to!(:succeeded)
+    end
+    assert_includes error.message, "transcode"
+    assert_includes error.message, "from 'pending' to 'succeeded'"
+  end
+
+  # --- Accepts symbols ---
+
+  def test_transition_accepts_symbols
+    pipeline = create_pipeline
+    step = create_step(pipeline)
+    step.transition_coordination_status_to!(:enqueued)
+
+    assert_equal "enqueued", step.coordination_status
+  end
+
+  # --- Unique constraint on (pipeline_id, key) ---
+
+  def test_duplicate_key_within_pipeline_raises
+    pipeline = create_pipeline
+    create_step(pipeline, key: "download")
+    assert_raises(ActiveRecord::RecordNotUnique) do
+      create_step(pipeline, key: "download", job_class: "OtherJob")
+    end
+  end
+
+  def test_same_key_in_different_pipelines_allowed
+    pipeline_a = create_pipeline(type: "PipelineA")
+    pipeline_b = create_pipeline(type: "PipelineB")
+    create_step(pipeline_a, key: "download")
+    create_step(pipeline_b, key: "download")
+
+    assert_equal 1, pipeline_a.steps.count
+    assert_equal 1, pipeline_b.steps.count
+  end
+end

data/demo/test/integration/test_concurrent_fan_in.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class TestConcurrentFanIn < ActiveSupport::TestCase
+  def test_simultaneous_upstream_completion_enqueues_downstream_exactly_once
+    50.times do |iteration|
+      pipeline = create_pipeline(on_failure_strategy: "halt")
+      pipeline.update_columns(status: "running")
+      step_a = build_step(pipeline, key: "a")
+      step_b = build_step(pipeline, key: "b")
+      step_c = build_step(pipeline, key: "c", dependencies: [step_a, step_b])
+      step_a.update_columns(coordination_status: "enqueued")
+      step_b.update_columns(coordination_status: "enqueued")
+
+      latch = Concurrent::CountDownLatch.new(2)
+
+      promise_a = rails_promise do
+        latch.count_down
+        latch.wait(5)
+        GoodPipeline::Coordinator.complete_step(step_a.reload, succeeded: true)
+      end
+
+      promise_b = rails_promise do
+        latch.count_down
+        latch.wait(5)
+        GoodPipeline::Coordinator.complete_step(step_b.reload, succeeded: true)
+      end
+
+      promise_a.value!
+      promise_b.value!
+
+      step_c.reload
+
+      refute_equal "pending", step_c.coordination_status,
+                   "Iteration #{iteration}: step_c should not still be pending"
+
+      ActiveRecord::Base.connection.truncate_tables(*ActiveRecord::Base.connection.tables)
+    end
+  end
+
+  def test_triple_fan_in_enqueues_downstream_exactly_once
+    pipeline = create_pipeline(on_failure_strategy: "halt")
+    pipeline.update_columns(status: "running")
+    step_a = build_step(pipeline, key: "a")
+    step_b = build_step(pipeline, key: "b")
+    step_c = build_step(pipeline, key: "c")
+    step_d = build_step(pipeline, key: "d", dependencies: [step_a, step_b, step_c])
+    [step_a, step_b, step_c].each { |step| step.update_columns(coordination_status: "enqueued") }
+
+    latch = Concurrent::CountDownLatch.new(3)
+
+    promises = [step_a, step_b, step_c].map do |step|
+      rails_promise do
+        latch.count_down
+        latch.wait(5)
+        GoodPipeline::Coordinator.complete_step(step.reload, succeeded: true)
+      end
+    end
+
+    promises.each(&:value!)
+
+    step_d.reload
+
+    refute_equal "pending", step_d.coordination_status
+  end
+
+  def test_good_job_id_guard_prevents_double_enqueue
+    pipeline = create_pipeline(on_failure_strategy: "halt")
+    pipeline.update_columns(status: "running")
+    step = build_step(pipeline, key: "a")
+    fake_job_id = SecureRandom.uuid
+    step.update_column(:good_job_id, fake_job_id)
+
+    GoodPipeline::Coordinator.try_enqueue_step(step.id)
+
+    step.reload
+
+    assert_equal "pending", step.coordination_status
+    assert_equal fake_job_id, step.good_job_id
+  end
+
+  def test_coordinator_interleave_enqueues_only_once
+    pipeline = create_pipeline(on_failure_strategy: "halt")
+    pipeline.update_columns(status: "running")
+    step = build_step(pipeline, key: "a")
+
+    latch = Concurrent::CountDownLatch.new(2)
+
+    promise_1 = rails_promise do
+      latch.count_down
+      latch.wait(5)
+      GoodPipeline::Coordinator.try_enqueue_step(step.id)
+    end
+
+    promise_2 = rails_promise do
+      latch.count_down
+      latch.wait(5)
+      GoodPipeline::Coordinator.try_enqueue_step(step.id)
+    end
+
+    promise_1.value!
+    promise_2.value!
+
+    step.reload
+
+    refute_equal "pending", step.coordination_status
+  end
+end

data/demo/test/integration/test_end_to_end.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class TestEndToEnd < ActiveSupport::TestCase
+  def run_pipeline_to_completion(pipeline_record, timeout: 15)
+    deadline = Time.current + timeout
+    loop do
+      perform_enqueued_jobs_inline
+      pipeline_record.reload
+      return pipeline_record if pipeline_record.terminal?
+
+      if Time.current > deadline
+        raise "Pipeline did not reach terminal state within #{timeout}s (status: #{pipeline_record.status})"
+      end
+
+      sleep 0.05
+    end
+  end
+
+  def test_full_pipeline_succeeds
+    pipeline_record = VideoProcessingPipeline.run(video_id: 123)
+
+    assert_instance_of GoodPipeline::Chain, pipeline_record
+    assert_equal "VideoProcessingPipeline", pipeline_record.type
+    assert_equal({ "video_id" => 123 }, pipeline_record.params)
+    assert_equal 5, pipeline_record.steps.count
+    assert_equal 5, pipeline_record.dependencies.count
+
+    result = run_pipeline_to_completion(pipeline_record)
+
+    assert_equal "succeeded", result.status
+    assert(result.steps.all? { |step| step.coordination_status == "succeeded" })
+    refute_nil result.callbacks_dispatched_at
+  end
+
+  def test_pipeline_with_failing_step_halts
+    failing_pipeline_class = Class.new(GoodPipeline::Pipeline) do
+      failure_strategy :halt
+
+      define_method(:configure) do |**_kwargs|
+        run :step_a, FailingJob
+        run :step_b, DownloadJob, after: :step_a
+      end
+    end
+    Object.const_set(:HaltTestPipeline, failing_pipeline_class) unless defined?(::HaltTestPipeline)
+
+    pipeline_record = HaltTestPipeline.run
+
+    result = run_pipeline_to_completion(pipeline_record)
+
+    assert_equal "halted", result.status
+    assert_predicate result, :halt_triggered?
+
+    step_a = result.steps.find_by(key: "step_a")
+    step_b = result.steps.find_by(key: "step_b")
+
+    assert_equal "failed", step_a.coordination_status
+    assert_equal "skipped", step_b.coordination_status
+  end
+
+  def test_pipeline_with_continue_strategy
+    continue_pipeline_class = Class.new(GoodPipeline::Pipeline) do
+      failure_strategy :continue
+
+      define_method(:configure) do |**_kwargs|
+        run :step_a, FailingJob
+        run :step_b, DownloadJob
+        run :step_c, DownloadJob, after: :step_a
+      end
+    end
+    Object.const_set(:ContinueTestPipeline, continue_pipeline_class) unless defined?(::ContinueTestPipeline)
+
+    pipeline_record = ContinueTestPipeline.run
+
+    result = run_pipeline_to_completion(pipeline_record)
+
+    assert_equal "failed", result.status
+    refute_predicate result, :halt_triggered?
+
+    step_a = result.steps.find_by(key: "step_a")
+    step_b = result.steps.find_by(key: "step_b")
+    step_c = result.steps.find_by(key: "step_c")
+
+    assert_equal "failed", step_a.coordination_status
+    assert_equal "succeeded", step_b.coordination_status
+    assert_equal "skipped", step_c.coordination_status
+  end
+end