good_pipeline 0.1.0

data/demo/test/integration/test_enqueue_atomicity.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class TestEnqueueAtomicity < ActiveSupport::TestCase
+  def test_rollback_cancels_step_transition_and_job_record
+    pipeline = create_pipeline(on_failure_strategy: "halt")
+    pipeline.update_columns(status: "running")
+    step = build_step(pipeline, key: "a")
+
+    GoodPipeline::StepRecord.transaction do
+      GoodPipeline::Coordinator.try_enqueue_step(step.id)
+      raise ActiveRecord::Rollback
+    end
+
+    step.reload
+
+    assert_equal "pending", step.coordination_status
+    assert_nil step.good_job_id
+    assert_nil step.good_job_batch_id
+    assert_equal 0, GoodJob::Job.count
+  end
+
+  def test_step_can_be_re_enqueued_after_rollback
+    pipeline = create_pipeline(on_failure_strategy: "halt")
+    pipeline.update_columns(status: "running")
+    step = build_step(pipeline, key: "a")
+
+    GoodPipeline::StepRecord.transaction do
+      GoodPipeline::Coordinator.try_enqueue_step(step.id)
+      raise ActiveRecord::Rollback
+    end
+
+    assert_equal "pending", step.reload.coordination_status
+
+    GoodPipeline::Coordinator.try_enqueue_step(step.id)
+
+    step.reload
+
+    refute_equal "pending", step.coordination_status
+    assert_not_nil step.good_job_id
+    assert_not_nil step.good_job_batch_id
+    assert GoodJob::Job.exists?(id: step.good_job_id)
+  end
+
+  def test_good_job_id_and_record_exist_together
+    pipeline = create_pipeline(on_failure_strategy: "halt")
+    pipeline.update_columns(status: "running")
+    step = build_step(pipeline, key: "a")
+
+    GoodPipeline::Coordinator.try_enqueue_step(step.id)
+
+    step.reload
+
+    assert_not_nil step.good_job_id, "good_job_id should be set after enqueue"
+    assert GoodJob::Job.exists?(id: step.good_job_id),
+           "GoodJob::Job record should exist for the good_job_id"
+  end
+end

data/demo/test/integration/test_pipeline_chaining.rb

@@ -0,0 +1,183 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class TestPipelineChaining < ActiveSupport::TestCase
+  def run_all_to_completion(pipeline_records, timeout: 15)
+    deadline = Time.current + timeout
+    loop do
+      perform_enqueued_jobs_inline
+      pipeline_records.each(&:reload)
+      return pipeline_records if pipeline_records.all?(&:terminal?)
+
+      if Time.current > deadline
+        statuses = pipeline_records.map { |pipeline| "#{pipeline.type}=#{pipeline.status}" }.join(", ")
+        raise "Pipelines did not reach terminal state within #{timeout}s (#{statuses})"
+      end
+
+      sleep 0.05
+    end
+  end
+
+  # --- Serial chain ---
+
+  def test_serial_chain_all_succeed
+    TestPipeline.run(video_id: 1)
+                .then(NotificationPipeline, with: { video_id: 1 })
+                .then(ArchivePipeline, with: { video_id: 1 })
+
+    all_records = GoodPipeline::PipelineRecord.all.to_a
+    run_all_to_completion(all_records)
+
+    assert all_records.all?(&:succeeded?), "All pipelines should have succeeded"
+  end
+
+  def test_serial_chain_first_fails_skips_rest
+    HaltTestPipeline.run
+                    .then(NotificationPipeline, with: {})
+
+    all_records = GoodPipeline::PipelineRecord.all.to_a
+    run_all_to_completion(all_records)
+
+    upstream = all_records.find { |pipeline| pipeline.type == "HaltTestPipeline" }
+    downstream = all_records.find { |pipeline| pipeline.type == "NotificationPipeline" }
+
+    assert_equal "halted", upstream.status
+    assert_equal "skipped", downstream.status
+  end
+
+  # --- Fan-out ---
+
+  def test_fan_out_both_start_after_upstream_succeeds
+    TestPipeline.run(video_id: 1)
+                .then(
+                  [NotificationPipeline, { with: {} }],
+                  [AnalyticsPipeline, { with: {} }]
+                )
+
+    all_records = GoodPipeline::PipelineRecord.all.to_a
+    run_all_to_completion(all_records)
+
+    notification = all_records.find { |pipeline| pipeline.type == "NotificationPipeline" }
+    analytics = all_records.find { |pipeline| pipeline.type == "AnalyticsPipeline" }
+
+    assert_equal "succeeded", notification.status
+    assert_equal "succeeded", analytics.status
+  end
+
+  def test_fan_out_upstream_fails_skips_all_downstream
+    HaltTestPipeline.run
+                    .then(
+                      [NotificationPipeline, { with: {} }],
+                      [AnalyticsPipeline, { with: {} }]
+                    )
+
+    all_records = GoodPipeline::PipelineRecord.all.to_a
+    run_all_to_completion(all_records)
+
+    notification = all_records.find { |pipeline| pipeline.type == "NotificationPipeline" }
+    analytics = all_records.find { |pipeline| pipeline.type == "AnalyticsPipeline" }
+
+    assert_equal "skipped", notification.status
+    assert_equal "skipped", analytics.status
+  end
+
+  # --- Fan-in ---
+
+  def test_fan_in_waits_for_all_upstreams
+    GoodPipeline.run(
+      [TestPipeline, { with: { video_id: 1 } }],
+      [NotificationPipeline, { with: {} }]
+    ).then(ArchivePipeline, with: {})
+
+    all_records = GoodPipeline::PipelineRecord.all.to_a
+    run_all_to_completion(all_records)
+
+    archive = all_records.find { |pipeline| pipeline.type == "ArchivePipeline" }
+
+    assert_equal "succeeded", archive.status
+  end
+
+  def test_fan_in_skips_when_any_upstream_fails
+    failing_pipeline_class = Class.new(GoodPipeline::Pipeline) do
+      failure_strategy :halt
+      define_method(:configure) { |**_kwargs| run :fail_step, FailingJob }
+    end
+    Object.const_set(:FanInFailPipeline, failing_pipeline_class) unless defined?(::FanInFailPipeline)
+
+    GoodPipeline.run(
+      [TestPipeline, { with: { video_id: 1 } }],
+      [FanInFailPipeline, { with: {} }]
+    ).then(ArchivePipeline, with: {})
+
+    all_records = GoodPipeline::PipelineRecord.all.to_a
+    run_all_to_completion(all_records)
+
+    archive = all_records.find { |pipeline| pipeline.type == "ArchivePipeline" }
+
+    assert_equal "skipped", archive.status
+  end
+
+  # --- Fan-out then fan-in ---
+
+  def test_fan_out_then_fan_in
+    TestPipeline.run(video_id: 1)
+                .then(
+                  [NotificationPipeline, { with: {} }],
+                  [AnalyticsPipeline, { with: {} }]
+                )
+                .then(ArchivePipeline, with: {})
+
+    all_records = GoodPipeline::PipelineRecord.all.to_a
+    run_all_to_completion(all_records)
+
+    assert all_records.all?(&:succeeded?), "All pipelines should have succeeded"
+
+    archive = all_records.find { |pipeline| pipeline.type == "ArchivePipeline" }
+
+    assert_equal 2, archive.upstream_pipelines.count
+  end
+
+  # --- Deep chain failure propagation ---
+
+  def test_deep_chain_failure_propagates_skips
+    HaltTestPipeline.run
+                    .then(NotificationPipeline, with: {})
+                    .then(AnalyticsPipeline, with: {})
+                    .then(ArchivePipeline, with: {})
+
+    all_records = GoodPipeline::PipelineRecord.all.to_a
+    run_all_to_completion(all_records)
+
+    notification = all_records.find { |pipeline| pipeline.type == "NotificationPipeline" }
+    analytics = all_records.find { |pipeline| pipeline.type == "AnalyticsPipeline" }
+    archive = all_records.find { |pipeline| pipeline.type == "ArchivePipeline" }
+
+    assert_equal "skipped", notification.status
+    assert_equal "skipped", analytics.status
+    assert_equal "skipped", archive.status
+  end
+
+  # --- Callbacks on skipped ---
+
+  def test_skipped_pipeline_fires_on_complete_but_not_on_failure
+    callback_pipeline_class = Class.new(GoodPipeline::Pipeline) do
+      on_complete :handle_complete
+
+      define_method(:configure) { |**_kwargs| run :step, DownloadJob }
+      define_method(:handle_complete) {} # no-op
+    end
+    Object.const_set(:SkipCallbackPipeline, callback_pipeline_class) unless defined?(::SkipCallbackPipeline)
+
+    HaltTestPipeline.run
+                    .then(SkipCallbackPipeline, with: {})
+
+    all_records = GoodPipeline::PipelineRecord.all.to_a
+    run_all_to_completion(all_records)
+
+    skipped_pipeline = all_records.find { |pipeline| pipeline.type == "SkipCallbackPipeline" }
+
+    assert_equal "skipped", skipped_pipeline.status
+    assert_not_nil skipped_pipeline.callbacks_dispatched_at
+  end
+end

data/demo/test/integration/test_retry_scenarios.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class TestRetryScenarios < ActiveSupport::TestCase
+  def test_retry_then_succeed_keeps_enqueued_during_retries
+    pipeline = create_pipeline(on_failure_strategy: "continue")
+    pipeline.update_columns(status: "running")
+    step_a = build_step(pipeline, key: "a", job_class: "RetryableJob")
+    step_b = build_step(pipeline, key: "b", dependencies: [step_a])
+
+    tracker_key = SecureRandom.hex(8)
+    ActiveRecord::Base.connection.execute(
+      ActiveRecord::Base.sanitize_sql(
+        ["INSERT INTO attempt_trackers (key, count) VALUES (?, 0)", tracker_key]
+      )
+    )
+    step_a.update_column(:params, { "tracker_key" => tracker_key })
+
+    GoodPipeline::Coordinator.try_enqueue_step(step_a.id)
+
+    saw_failed = false
+    wait_until(timeout: 15) do
+      perform_enqueued_jobs_inline
+      step_a.reload
+      saw_failed = true if step_a.coordination_status == "failed"
+      step_a.coordination_status == "succeeded"
+    end
+
+    refute saw_failed, "coordination_status should never have been 'failed' during retries"
+    assert_equal "succeeded", step_a.reload.coordination_status
+
+    perform_enqueued_jobs_inline
+    wait_until(timeout: 5) do
+      perform_enqueued_jobs_inline
+      step_b.reload
+      step_b.coordination_status != "pending"
+    end
+
+    refute_equal "pending", step_b.reload.coordination_status
+  end
+
+  def test_retry_then_exhaust_transitions_to_failed
+    pipeline = create_pipeline(on_failure_strategy: "continue")
+    pipeline.update_columns(status: "running")
+    step = build_step(pipeline, key: "a", job_class: "AlwaysFailingJob")
+
+    GoodPipeline::Coordinator.try_enqueue_step(step.id)
+
+    wait_until(timeout: 15) do
+      begin
+        perform_enqueued_jobs_inline
+      rescue AlwaysFailingJob::AlwaysFailingError
+        # Expected — GoodJob re-raises after exhausting retries in inline/external mode
+      end
+      step.reload
+      step.coordination_status == "failed"
+    end
+
+    step.reload
+
+    assert_equal "failed", step.coordination_status
+    assert_not_nil step.error_class
+    assert_not_nil step.error_message
+  end
+
+  def test_discard_on_immediate_failure_halts_pipeline
+    pipeline = create_pipeline(on_failure_strategy: "halt")
+    pipeline.update_columns(status: "running")
+    step_a = build_step(pipeline, key: "a", job_class: "FailingJob")
+    step_b = build_step(pipeline, key: "b", dependencies: [step_a])
+
+    GoodPipeline::Coordinator.try_enqueue_step(step_a.id)
+
+    wait_until(timeout: 10) do
+      perform_enqueued_jobs_inline
+      pipeline.reload
+      pipeline.terminal?
+    end
+
+    step_a.reload
+
+    assert_equal "failed", step_a.coordination_status
+    assert_equal "FailingJob::FailingError", step_a.error_class
+
+    assert_equal "skipped", step_b.reload.coordination_status
+    assert_predicate pipeline.reload, :halt_triggered?
+    assert_equal "halted", pipeline.status
+  end
+end

data/demo/test/integration/test_step_finished_idempotency.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class TestStepFinishedIdempotency < ActiveSupport::TestCase
+  def test_complete_step_on_already_succeeded_step_is_noop
+    pipeline = create_pipeline
+    pipeline.update_columns(status: "running")
+    step = build_step(pipeline, key: "a")
+    step.update_columns(coordination_status: "succeeded")
+
+    GoodPipeline::Coordinator.complete_step(step.reload, succeeded: true)
+
+    assert_equal "succeeded", step.reload.coordination_status
+  end
+
+  def test_complete_step_on_already_failed_step_is_noop
+    pipeline = create_pipeline
+    pipeline.update_columns(status: "running")
+    step = build_step(pipeline, key: "a")
+    step.update_columns(coordination_status: "failed")
+
+    GoodPipeline::Coordinator.complete_step(step.reload, succeeded: false)
+
+    assert_equal "failed", step.reload.coordination_status
+  end
+
+  def test_complete_step_on_already_skipped_step_is_noop
+    pipeline = create_pipeline
+    pipeline.update_columns(status: "running")
+    step = build_step(pipeline, key: "a")
+    step.update_columns(coordination_status: "skipped")
+
+    GoodPipeline::Coordinator.complete_step(step.reload, succeeded: true)
+
+    assert_equal "skipped", step.reload.coordination_status
+  end
+end

data/demo/test/test_helper.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+ENV["RAILS_ENV"] = "test"
+require_relative "../config/environment"
+require "rails/test_help"
+require "minitest/autorun"
+
+ActiveJob::Base.logger = Logger.new(nil)
+
+# Create attempt_trackers table for retry tests
+ActiveRecord::Base.connection.create_table :attempt_trackers, if_not_exists: true do |t|
+  t.string :key, null: false
+  t.integer :count, default: 0, null: false
+end
+
+module ActiveSupport
+  class TestCase
+    self.use_transactional_tests = false
+
+    teardown do
+      ActiveRecord::Base.connection.truncate_tables(*ActiveRecord::Base.connection.tables)
+    end
+
+    private
+
+    def rails_promise(&block)
+      Concurrent::Promises.future do
+        Rails.application.executor.wrap(&block)
+      end
+    end
+
+    def perform_enqueued_jobs_inline
+      GoodJob.perform_inline
+    end
+
+    def wait_until(timeout: 10, interval: 0.1)
+      deadline = Time.current + timeout
+      loop do
+        return if yield
+
+        raise "Timeout waiting for condition" if Time.current > deadline
+
+        sleep interval
+      end
+    end
+
+    def create_pipeline(**attributes)
+      GoodPipeline::PipelineRecord.create!(
+        { type: "TestPipeline" }.merge(attributes)
+      )
+    end
+
+    def create_step(pipeline, key: "step_a", job_class: "DownloadJob", **attributes)
+      GoodPipeline::StepRecord.create!(
+        {
+          pipeline: pipeline,
+          key: key,
+          job_class: job_class
+        }.merge(attributes)
+      )
+    end
+
+    def build_step(pipeline, key:, dependencies: [], on_failure_strategy: nil, **attributes)
+      step = create_step(pipeline, key: key, on_failure_strategy: on_failure_strategy, **attributes)
+      dependencies.each do |dependency_step|
+        GoodPipeline::DependencyRecord.create!(pipeline: pipeline, step: step, depends_on_step: dependency_step)
+      end
+      step
+    end
+  end
+end

data/dev-docker-compose.yml

@@ -0,0 +1,16 @@
+services:
+  postgres:
+    image: postgres:17.5
+    restart: unless-stopped
+    command: postgres -c "shared_preload_libraries=pg_stat_statements" -c "pg_stat_statements.track=all" -c "pg_stat_statements.max=10000" -c "track_activity_query_size=2048"
+    ports:
+      - "5432:5432"
+    volumes:
+      - postgres-data:/var/lib/postgresql/data
+    environment:
+      - POSTGRES_DB=postgres
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=postgres
+
+volumes:
+  postgres-data:

data/docs/.vitepress/config.mts

@@ -0,0 +1,66 @@
+import { defineConfig } from 'vitepress'
+
+export default defineConfig({
+  title: 'GoodPipeline',
+  description: 'DAG-based job pipeline orchestration for Rails, built on GoodJob.',
+  base: '/good_pipeline/',
+
+  themeConfig: {
+    nav: [
+      { text: 'Home', link: '/' },
+      { text: 'Docs', link: '/introduction' },
+    ],
+
+    sidebar: [
+      {
+        text: 'Getting Started',
+        items: [
+          { text: 'Introduction', link: '/introduction' },
+          { text: 'Installation & Setup', link: '/getting-started' },
+        ],
+      },
+      {
+        text: 'Core Guides',
+        items: [
+          { text: 'Defining Pipelines', link: '/defining-pipelines' },
+          { text: 'DAG Validation', link: '/dag-validation' },
+          { text: 'Failure Strategies', link: '/failure-strategies' },
+          { text: 'Pipeline Chaining', link: '/pipeline-chaining' },
+          { text: 'Lifecycle Callbacks', link: '/callbacks' },
+        ],
+      },
+      {
+        text: 'Operations',
+        items: [
+          { text: 'Monitoring & Introspection', link: '/monitoring' },
+          { text: 'Web Dashboard', link: '/dashboard' },
+          { text: 'Cleanup', link: '/cleanup' },
+        ],
+      },
+      {
+        text: 'Reference',
+        items: [
+          { text: 'Architecture', link: '/architecture' },
+        ],
+      },
+    ],
+
+    socialLinks: [
+      { icon: 'github', link: 'https://github.com/milkstrawai/good_pipeline' },
+    ],
+
+    editLink: {
+      pattern: 'https://github.com/milkstrawai/good_pipeline/edit/main/docs/:path',
+      text: 'Edit this page on GitHub',
+    },
+
+    search: {
+      provider: 'local',
+    },
+
+    footer: {
+      message: 'Released under the MIT License.',
+      copyright: 'Copyright 2026 MilkStraw AI',
+    },
+  },
+})

data/docs/.vitepress/theme/custom.css

@@ -0,0 +1,21 @@
+:root {
+  --vp-c-brand-1: #d97706;
+  --vp-c-brand-2: #b45309;
+  --vp-c-brand-3: #d97706;
+  --vp-c-brand-soft: rgba(217, 119, 6, 0.14);
+}
+
+.dark {
+  --vp-c-brand-1: #f59e0b;
+  --vp-c-brand-2: #fbbf24;
+  --vp-c-brand-3: #f59e0b;
+  --vp-c-brand-soft: rgba(245, 158, 11, 0.14);
+}
+
+.VPHero .name {
+  font-size: 2.5rem !important;
+}
+
+.VPHero .text {
+  font-size: 1.5rem !important;
+}

data/docs/.vitepress/theme/index.ts

@@ -0,0 +1,4 @@
+import DefaultTheme from 'vitepress/theme'
+import './custom.css'
+
+export default DefaultTheme