good_pipeline 0.1.0

data/demo/config/boot.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile", __dir__)
+require "bundler/setup" if File.exist?(ENV["BUNDLE_GEMFILE"])
+$LOAD_PATH.unshift File.expand_path("../../lib", __dir__)

data/demo/config/database.yml

@@ -0,0 +1,15 @@
+default: &default
+  adapter: postgresql
+  encoding: unicode
+  host: localhost
+  username: postgres
+  password: postgres
+  pool: 20
+
+development:
+  <<: *default
+  database: good_pipeline_development
+
+test:
+  <<: *default
+  database: good_pipeline_test

data/demo/config/environment.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+require_relative "application"
+
+Rails.application.initialize!

data/demo/config/environments/development.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+Rails.application.configure do
+  config.cache_classes = false
+  config.eager_load = false
+  config.consider_all_requests_local = true
+  config.active_support.deprecation = :log
+  config.good_job.execution_mode = :async
+end

data/demo/config/environments/test.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+Rails.application.configure do
+  config.cache_classes = true
+  config.eager_load = false
+  config.consider_all_requests_local = true
+  config.action_controller.perform_caching = false if config.respond_to?(:action_controller)
+  config.active_support.deprecation = :stderr
+  config.good_job.execution_mode = :external
+end

data/demo/config/routes.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+Rails.application.routes.draw do
+  mount GoodJob::Engine => "/good_job"
+  mount GoodPipeline::Engine => "/good_pipeline"
+end

data/demo/config.ru

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+require_relative "config/environment"
+
+run Rails.application

data/demo/db/migrate/20260319205325_create_good_jobs.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+class CreateGoodJobs < ActiveRecord::Migration[8.1]
+  def change
+    # Uncomment for Postgres v12 or earlier to enable gen_random_uuid() support
+    # enable_extension 'pgcrypto'
+
+    create_table :good_jobs, id: :uuid do |t|
+      t.text :queue_name
+      t.integer :priority
+      t.jsonb :serialized_params
+      t.datetime :scheduled_at
+      t.datetime :performed_at
+      t.datetime :finished_at
+      t.text :error
+
+      t.timestamps
+
+      t.uuid :active_job_id
+      t.text :concurrency_key
+      t.text :cron_key
+      t.uuid :retried_good_job_id
+      t.datetime :cron_at
+
+      t.uuid :batch_id
+      t.uuid :batch_callback_id
+
+      t.boolean :is_discrete
+      t.integer :executions_count
+      t.text :job_class
+      t.integer :error_event, limit: 2
+      t.text :labels, array: true
+      t.uuid :locked_by_id
+      t.datetime :locked_at
+    end
+
+    create_table :good_job_batches, id: :uuid do |t|
+      t.timestamps
+      t.text :description
+      t.jsonb :serialized_properties
+      t.text :on_finish
+      t.text :on_success
+      t.text :on_discard
+      t.text :callback_queue_name
+      t.integer :callback_priority
+      t.datetime :enqueued_at
+      t.datetime :discarded_at
+      t.datetime :finished_at
+      t.datetime :jobs_finished_at
+    end
+
+    create_table :good_job_executions, id: :uuid do |t|
+      t.timestamps
+
+      t.uuid :active_job_id, null: false
+      t.text :job_class
+      t.text :queue_name
+      t.jsonb :serialized_params
+      t.datetime :scheduled_at
+      t.datetime :finished_at
+      t.text :error
+      t.integer :error_event, limit: 2
+      t.text :error_backtrace, array: true
+      t.uuid :process_id
+      t.interval :duration
+    end
+
+    create_table :good_job_processes, id: :uuid do |t|
+      t.timestamps
+      t.jsonb :state
+      t.integer :lock_type, limit: 2
+    end
+
+    create_table :good_job_settings, id: :uuid do |t|
+      t.timestamps
+      t.text :key
+      t.jsonb :value
+      t.index :key, unique: true
+    end
+
+    add_index :good_jobs, :scheduled_at, where: "(finished_at IS NULL)", name: :index_good_jobs_on_scheduled_at
+    add_index :good_jobs, %i[queue_name scheduled_at], where: "(finished_at IS NULL)",
+                                                       name: :index_good_jobs_on_queue_name_and_scheduled_at
+    add_index :good_jobs, %i[active_job_id created_at], name: :index_good_jobs_on_active_job_id_and_created_at
+    add_index :good_jobs, :concurrency_key, where: "(finished_at IS NULL)",
+                                            name: :index_good_jobs_on_concurrency_key_when_unfinished
+    add_index :good_jobs, %i[concurrency_key created_at], name: :index_good_jobs_on_concurrency_key_and_created_at
+    add_index :good_jobs, %i[cron_key created_at], where: "(cron_key IS NOT NULL)",
+                                                   name: :index_good_jobs_on_cron_key_and_created_at_cond
+    add_index :good_jobs, %i[cron_key cron_at], where: "(cron_key IS NOT NULL)", unique: true,
+                                                name: :index_good_jobs_on_cron_key_and_cron_at_cond
+    add_index :good_jobs, [:finished_at], where: "finished_at IS NOT NULL",
+                                          name: :index_good_jobs_jobs_on_finished_at_only
+    add_index :good_jobs, %i[priority created_at], order: { priority: "DESC NULLS LAST", created_at: :asc },
+                                                   where: "finished_at IS NULL", name: :index_good_jobs_jobs_on_priority_created_at_when_unfinished
+    add_index :good_jobs, %i[priority created_at], order: { priority: "ASC NULLS LAST", created_at: :asc },
+                                                   where: "finished_at IS NULL", name: :index_good_job_jobs_for_candidate_lookup
+    add_index :good_jobs, [:batch_id], where: "batch_id IS NOT NULL"
+    add_index :good_jobs, [:batch_callback_id], where: "batch_callback_id IS NOT NULL"
+    add_index :good_jobs, :job_class, name: :index_good_jobs_on_job_class
+    add_index :good_jobs, :labels, using: :gin, where: "(labels IS NOT NULL)", name: :index_good_jobs_on_labels
+
+    add_index :good_job_executions, %i[active_job_id created_at],
+              name: :index_good_job_executions_on_active_job_id_and_created_at
+    add_index :good_jobs, %i[priority scheduled_at], order: { priority: "ASC NULLS LAST", scheduled_at: :asc },
+                                                     where: "finished_at IS NULL AND locked_by_id IS NULL", name: :index_good_jobs_on_priority_scheduled_at_unfinished_unlocked
+    add_index :good_jobs, :locked_by_id,
+              where: "locked_by_id IS NOT NULL", name: "index_good_jobs_on_locked_by_id"
+    add_index :good_job_executions, %i[process_id created_at],
+              name: :index_good_job_executions_on_process_id_and_created_at
+  end
+end

data/demo/db/migrate/20260319205326_create_good_pipeline_tables.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+class CreateGoodPipelineTables < ActiveRecord::Migration[8.1]
+  def change
+    # Uncomment for Postgres v12 or earlier to enable gen_random_uuid() support
+    # enable_extension 'pgcrypto'
+
+    create_table :good_pipeline_pipelines, id: :uuid do |t|
+      t.string :type, null: false
+      t.jsonb :params, null: false, default: {}
+      t.string :status, null: false, default: "pending"
+      t.boolean :halt_triggered, null: false, default: false
+      t.uuid :good_job_batch_id
+      t.string :on_failure_strategy, null: false, default: "halt"
+      t.datetime :callbacks_dispatched_at
+
+      t.timestamps
+    end
+
+    add_index :good_pipeline_pipelines, :status
+
+    create_table :good_pipeline_steps, id: :uuid do |t|
+      t.references :pipeline, null: false, foreign_key: { to_table: :good_pipeline_pipelines }, type: :uuid
+      t.string :key, null: false
+      t.string :job_class, null: false
+      t.jsonb :params, null: false, default: {}
+      t.string :coordination_status, null: false, default: "pending"
+      t.string :on_failure_strategy
+      t.string :queue
+      t.integer :priority
+      t.uuid :good_job_batch_id
+      t.uuid :good_job_id
+      t.integer :attempts
+      t.string :error_class
+      t.text :error_message
+
+      t.timestamps
+    end
+
+    add_index :good_pipeline_steps, %i[pipeline_id key], unique: true
+
+    create_table :good_pipeline_dependencies do |t|
+      t.references :pipeline, null: false, foreign_key: { to_table: :good_pipeline_pipelines }, type: :uuid
+      t.references :step, null: false, foreign_key: { to_table: :good_pipeline_steps }, type: :uuid
+      t.references :depends_on_step, null: false, foreign_key: { to_table: :good_pipeline_steps }, type: :uuid
+    end
+
+    create_table :good_pipeline_chains, id: :uuid do |t|
+      t.references :upstream_pipeline, null: false, foreign_key: { to_table: :good_pipeline_pipelines }, type: :uuid
+      t.references :downstream_pipeline, null: false, foreign_key: { to_table: :good_pipeline_pipelines }, type: :uuid
+    end
+  end
+end

data/demo/db/seeds.rb

@@ -0,0 +1,153 @@
+# frozen_string_literal: true
+
+puts "Seeding GoodPipeline demo data..."
+
+GoodPipeline::ChainRecord.delete_all
+GoodPipeline::DependencyRecord.delete_all
+GoodPipeline::StepRecord.delete_all
+GoodPipeline::PipelineRecord.delete_all
+
+NOW = Time.current # rubocop:disable Lint/ConstantDefinitionInBlock
+
+def create_pipeline(type:, status:, params:, strategy: "halt", age:, duration: nil, halt_triggered: false) # rubocop:disable Metrics/ParameterLists
+  pipeline = GoodPipeline::PipelineRecord.create!(
+    type: type, status: status, params: params,
+    on_failure_strategy: strategy, halt_triggered: halt_triggered
+  )
+  pipeline.update_columns(
+    created_at: NOW - age,
+    updated_at: duration ? NOW - age + duration : NOW - age
+  )
+  pipeline
+end
+
+def add_steps(pipeline, *step_defs)
+  records = {}
+  step_defs.each do |definition|
+    records[definition[:key]] = GoodPipeline::StepRecord.create!(
+      pipeline: pipeline, key: definition[:key], job_class: definition[:job],
+      coordination_status: definition.fetch(:status, "succeeded"),
+      error_class: definition[:error_class], error_message: definition[:error_message]
+    )
+  end
+  records
+end
+
+def add_edges(pipeline, steps, *edges)
+  edges.each do |from, to|
+    GoodPipeline::DependencyRecord.create!(pipeline: pipeline, step: steps[to], depends_on_step: steps[from])
+  end
+end
+
+# 1. VideoProcessingPipeline (succeeded) — fan-out + fan-in DAG
+video = create_pipeline(type: "VideoProcessingPipeline", status: "succeeded",
+                         params: { video_id: 8842, format: "mp4", resolution: "1080p" }, age: 2.hours, duration: 1.hour)
+video_steps = add_steps(video,
+                        { key: "download", job: "DownloadJob" },
+                        { key: "transcode", job: "TranscodeJob" },
+                        { key: "thumbnail", job: "ThumbnailJob" },
+                        { key: "publish", job: "PublishJob" },
+                        { key: "cleanup", job: "CleanupJob" })
+add_edges(video, video_steps,
+          %w[download transcode], %w[download thumbnail],
+          %w[transcode publish], %w[thumbnail publish],
+          %w[publish cleanup])
+
+# 2. DataIngestionPipeline (succeeded) — linear chain
+ingest = create_pipeline(type: "DataIngestionPipeline", status: "succeeded",
+                          params: { source: "s3://data-lake/events/2026-03-20", dataset: "user_events" },
+                          strategy: "continue", age: 5.hours, duration: 1.hour)
+ingest_steps = add_steps(ingest,
+                         { key: "extract", job: "ExtractJob" }, { key: "validate", job: "ValidateJob" },
+                         { key: "transform", job: "TransformJob" }, { key: "load", job: "LoadJob" })
+add_edges(ingest, ingest_steps, %w[extract validate], %w[validate transform], %w[transform load])
+
+# 3. ReportGenerationPipeline (running) — partially complete
+report = create_pipeline(type: "ReportGenerationPipeline", status: "running",
+                          params: { report_type: "monthly_revenue", month: "2026-02" }, age: 15.minutes)
+report_steps = add_steps(report,
+                         { key: "query_data", job: "QueryDataJob" },
+                         { key: "aggregate", job: "AggregateJob", status: "enqueued" },
+                         { key: "render_pdf", job: "RenderPdfJob", status: "pending" },
+                         { key: "email_report", job: "EmailReportJob", status: "pending" })
+add_edges(report, report_steps, %w[query_data aggregate], %w[aggregate render_pdf], %w[render_pdf email_report])
+
+# 4. UserOnboardingPipeline (succeeded) — fan-out
+onboarding = create_pipeline(type: "UserOnboardingPipeline", status: "succeeded",
+                              params: { user_id: 29_451, plan: "pro" }, age: 30.minutes, duration: 5.minutes)
+onboarding_steps = add_steps(onboarding,
+                             { key: "provision_account", job: "ProvisionAccountJob" },
+                             { key: "send_welcome_email", job: "SendWelcomeEmailJob" },
+                             { key: "sync_crm", job: "SyncCrmJob" })
+add_edges(onboarding, onboarding_steps, %w[provision_account send_welcome_email], %w[provision_account sync_crm])
+
+# 5. PaymentReconciliationPipeline (failed) — with error
+payment = create_pipeline(type: "PaymentReconciliationPipeline", status: "failed",
+                           params: { batch_date: "2026-03-19", gateway: "stripe" },
+                           strategy: "continue", age: 1.hour, duration: 15.minutes)
+payment_steps = add_steps(payment,
+                          { key: "fetch_transactions", job: "FetchTransactionsJob" },
+                          { key: "match_records", job: "MatchRecordsJob", status: "failed",
+                            error_class: "ReconciliationError",
+                            error_message: "Found 23 unmatched transactions totaling $4,892.50" },
+                          { key: "generate_report", job: "GenerateReportJob", status: "skipped" })
+add_edges(payment, payment_steps, %w[fetch_transactions match_records], %w[match_records generate_report])
+
+# 6. OrderFulfillmentPipeline → CustomerNotificationPipeline (chain)
+order = create_pipeline(type: "OrderFulfillmentPipeline", status: "succeeded",
+                         params: { order_id: 78_332, warehouse: "us-east-1" }, age: 3.hours, duration: 30.minutes)
+order_steps = add_steps(order,
+                        { key: "reserve_inventory", job: "ReserveInventoryJob" },
+                        { key: "pick_and_pack", job: "PickAndPackJob" },
+                        { key: "ship", job: "ShipJob" },
+                        { key: "update_tracking", job: "UpdateTrackingJob" })
+add_edges(order, order_steps,
+          %w[reserve_inventory pick_and_pack], %w[pick_and_pack ship], %w[ship update_tracking])
+
+notify = create_pipeline(type: "CustomerNotificationPipeline", status: "succeeded",
+                          params: { order_id: 78_332, channel: "email" }, age: 2.5.hours, duration: 30.minutes)
+add_steps(notify, { key: "send_shipping_email", job: "SendShippingEmailJob" }, { key: "send_sms", job: "SendSmsJob" })
+GoodPipeline::ChainRecord.create!(upstream_pipeline: order, downstream_pipeline: notify)
+
+# 7. ImageResizePipeline (halted) — first step failed
+image = create_pipeline(type: "ImageResizePipeline", status: "halted", halt_triggered: true,
+                         params: { image_id: 55_210, sizes: %w[sm md lg xl] }, age: 45.minutes, duration: 5.minutes)
+image_steps = add_steps(image,
+                        { key: "download_original", job: "DownloadOriginalJob", status: "failed",
+                          error_class: "Aws::S3::Errors::NoSuchKey", error_message: "The specified key does not exist." },
+                        { key: "resize", job: "ResizeJob", status: "skipped" },
+                        { key: "upload_resized", job: "UploadResizedJob", status: "skipped" })
+add_edges(image, image_steps, %w[download_original resize], %w[resize upload_resized])
+
+# 8. InvoiceProcessingPipeline (succeeded) — linear chain
+invoice = create_pipeline(type: "InvoiceProcessingPipeline", status: "succeeded",
+                           params: { invoice_id: 11_298, vendor: "Acme Corp" }, age: 6.hours, duration: 30.minutes)
+invoice_steps = add_steps(invoice,
+                          { key: "parse_pdf", job: "ParsePdfJob" }, { key: "validate_line_items", job: "ValidateLineItemsJob" },
+                          { key: "auto_approve", job: "AutoApproveJob" }, { key: "post_to_ledger", job: "PostToLedgerJob" })
+add_edges(invoice, invoice_steps, %w[parse_pdf validate_line_items], %w[validate_line_items auto_approve], %w[auto_approve post_to_ledger])
+
+# 9. VideoProcessingPipeline (succeeded, older) — second execution
+video2 = create_pipeline(type: "VideoProcessingPipeline", status: "succeeded",
+                          params: { video_id: 7_651, format: "webm", resolution: "720p" }, age: 8.hours, duration: 1.hour)
+video2_steps = add_steps(video2,
+                         { key: "download", job: "DownloadJob" }, { key: "transcode", job: "TranscodeJob" },
+                         { key: "thumbnail", job: "ThumbnailJob" }, { key: "publish", job: "PublishJob" },
+                         { key: "cleanup", job: "CleanupJob" })
+add_edges(video2, video2_steps,
+          %w[download transcode], %w[download thumbnail],
+          %w[transcode publish], %w[thumbnail publish],
+          %w[publish cleanup])
+
+# 10. DataIngestionPipeline (running) — just started
+ingest2 = create_pipeline(type: "DataIngestionPipeline", status: "running",
+                           params: { source: "s3://data-lake/events/2026-03-21", dataset: "page_views" },
+                           strategy: "continue", age: 3.minutes)
+ingest2_steps = add_steps(ingest2,
+                          { key: "extract", job: "ExtractJob" },
+                          { key: "validate", job: "ValidateJob", status: "enqueued" },
+                          { key: "transform", job: "TransformJob", status: "pending" },
+                          { key: "load", job: "LoadJob", status: "pending" })
+add_edges(ingest2, ingest2_steps, %w[extract validate], %w[validate transform], %w[transform load])
+
+puts "Done! #{GoodPipeline::PipelineRecord.count} pipelines seeded."

data/demo/test/good_pipeline/test_chain_record.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class TestChainRecord < ActiveSupport::TestCase
+  # --- Pipeline chain navigation ---
+  #
+  # A -> B -> C
+
+  def test_serial_chain_navigation
+    pipeline_a = create_pipeline(type: "PipelineA")
+    pipeline_b = create_pipeline(type: "PipelineB")
+    pipeline_c = create_pipeline(type: "PipelineC")
+
+    GoodPipeline::ChainRecord.create!(upstream_pipeline: pipeline_a, downstream_pipeline: pipeline_b)
+    GoodPipeline::ChainRecord.create!(upstream_pipeline: pipeline_b, downstream_pipeline: pipeline_c)
+
+    # A's downstream: B
+    assert_equal [pipeline_b.id], pipeline_a.downstream_chains.map(&:downstream_pipeline_id)
+
+    # B's upstream: A, B's downstream: C
+    assert_equal [pipeline_a.id], pipeline_b.upstream_chains.map(&:upstream_pipeline_id)
+    assert_equal [pipeline_c.id], pipeline_b.downstream_chains.map(&:downstream_pipeline_id)
+
+    # C's upstream: B
+    assert_equal [pipeline_b.id], pipeline_c.upstream_chains.map(&:upstream_pipeline_id)
+    assert_empty pipeline_c.downstream_chains
+  end
+end

data/demo/test/good_pipeline/test_cleanup.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require_relative "../test_helper"
+
+class TestCleanup < ActiveSupport::TestCase
+  setup do
+    @now = Time.current
+
+    # Old terminal pipeline (should be cleaned up)
+    @old_pipeline = GoodPipeline::PipelineRecord.create!(
+      type: "TestPipeline", status: "succeeded", on_failure_strategy: "halt"
+    )
+    @old_pipeline.update_columns(updated_at: @now - 30.days)
+
+    @old_step = GoodPipeline::StepRecord.create!(
+      pipeline: @old_pipeline, key: "step_a", job_class: "DownloadJob", coordination_status: "succeeded"
+    )
+    @old_dependency = GoodPipeline::DependencyRecord.create!(
+      pipeline: @old_pipeline, step: @old_step, depends_on_step: @old_step
+    )
+
+    # Old running pipeline (should NOT be cleaned up)
+    @running_pipeline = GoodPipeline::PipelineRecord.create!(
+      type: "TestPipeline", status: "running", on_failure_strategy: "halt"
+    )
+    @running_pipeline.update_columns(updated_at: @now - 30.days)
+    GoodPipeline::StepRecord.create!(
+      pipeline: @running_pipeline, key: "step_a", job_class: "DownloadJob", coordination_status: "enqueued"
+    )
+
+    # Recent terminal pipeline (should NOT be cleaned up)
+    @recent_pipeline = GoodPipeline::PipelineRecord.create!(
+      type: "TestPipeline", status: "failed", on_failure_strategy: "halt"
+    )
+    GoodPipeline::StepRecord.create!(
+      pipeline: @recent_pipeline, key: "step_a", job_class: "DownloadJob", coordination_status: "failed"
+    )
+  end
+
+  test "cleans old terminal pipelines" do
+    GoodPipeline.cleanup_preserved_pipelines(older_than: @now - 14.days)
+
+    assert_nil GoodPipeline::PipelineRecord.find_by(id: @old_pipeline.id)
+  end
+
+  test "cleans associated steps and dependencies" do
+    GoodPipeline.cleanup_preserved_pipelines(older_than: @now - 14.days)
+
+    assert_nil GoodPipeline::StepRecord.find_by(id: @old_step.id)
+    assert_nil GoodPipeline::DependencyRecord.find_by(id: @old_dependency.id)
+  end
+
+  test "preserves running pipelines" do
+    GoodPipeline.cleanup_preserved_pipelines(older_than: @now - 14.days)
+
+    assert_not_nil GoodPipeline::PipelineRecord.find_by(id: @running_pipeline.id)
+  end
+
+  test "preserves recent terminal pipelines" do
+    GoodPipeline.cleanup_preserved_pipelines(older_than: @now - 14.days)
+
+    assert_not_nil GoodPipeline::PipelineRecord.find_by(id: @recent_pipeline.id)
+  end
+
+  test "cleans chain records" do
+    downstream = GoodPipeline::PipelineRecord.create!(
+      type: "TestPipeline", status: "succeeded", on_failure_strategy: "halt"
+    )
+    downstream.update_columns(updated_at: @now - 30.days)
+
+    chain = GoodPipeline::ChainRecord.create!(
+      upstream_pipeline: @old_pipeline, downstream_pipeline: downstream
+    )
+
+    GoodPipeline.cleanup_preserved_pipelines(older_than: @now - 14.days)
+
+    assert_nil GoodPipeline::ChainRecord.find_by(id: chain.id)
+  end
+
+  test "noop when nothing to clean" do
+    GoodPipeline.cleanup_preserved_pipelines(older_than: @now - 365.days)
+
+    assert_equal 3, GoodPipeline::PipelineRecord.count
+  end
+
+  test "triggers cleanup when GoodJob cleans preserved jobs" do
+    GoodJob.cleanup_preserved_jobs(older_than: 14.days)
+
+    assert_nil GoodPipeline::PipelineRecord.find_by(id: @old_pipeline.id)
+    assert_not_nil GoodPipeline::PipelineRecord.find_by(id: @running_pipeline.id)
+    assert_not_nil GoodPipeline::PipelineRecord.find_by(id: @recent_pipeline.id)
+  end
+end