ductwork 0.26.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ad0ce6f40af82ff2fcc08abb65083e32fdee666be43646a9f53f59033da85fa6
-  data.tar.gz: 7ba3f5a80167323fff58ae89b82fbfa44d10e4d20d6f6046436906250f1d9715
+  metadata.gz: 7e6b0081c6aa307274ff6dd94cc9e1dc0c8840dd5e6c3fc1768d4b54adc511f6
+  data.tar.gz: e1e3f47b3c8ea24b70fc3a518b0cb61e484ea46bc1630b1c8fd911100f4b12e8
 SHA512:
-  metadata.gz: 34b29a0aaacc4994fadd9542c2d38696c15b88caaa6660f36d0b2cdac82aca27b818e7173ef633e03d2deea0fd162a24db41a7090feab2ff8c68d1fc6b8bcec3
-  data.tar.gz: 68bf238bdc9bdf4528cbbd5ba2554525d849d6008e5a7a6c5c0e35c96cf2896c04c2a8900ba29b69188b34f6ea1d30692dd25762b174c7b8bd7ee8afa146d9c1
+  metadata.gz: f8b8d8bd63a376180c162afb97f0feba1cd5e81dde9160501c441104fc6c00417185fcfa259a4a6fd2718de95d25d2069ea9dab2e037683fe69c2f9ff429e88a
+  data.tar.gz: a908516aeb3e7436f61e276e6b751e52e1b38c8f067eb0a4b55951957a7ef36e923599e2ba5484d8c46c058480c4d573b305d52b91d035822aa997a82123ce37

data/.claude/skills/audit-clock-drift/SKILL.md

@@ -0,0 +1,11 @@
+---
+name: audit-clock-drift
+description: Audit ductwork for time comparisons that are prone to clock drift
+allowed-tools: Read, Grep, Glob
+---
+
+# Audit Clock Drift
+
+Audit the entire OSS ductwork codebase for places where we are open to clock drift issues. Specifically, if ductwork is running across multiple hosts, where are we prone to comparing a database timestamp with an in-memory OS clock read generated with Ruby. Only look for comparisons that gate safety or visibility. For example: heartbeat, enqueueing, claiming, possibly ordering.
+
+For each finding: file:line, severity, why it matters, suggested fix. Do not modify files.

data/.claude/skills/audit-database-indexes/SKILL.md

@@ -0,0 +1,11 @@
+---
+name: audit-database-indexes
+description: Audit ductwork for missing database indexes
+allowed-tools: Read, Grep, Glob
+---
+
+# Audit Missing Database Indexes
+
+Audit the entire OSS ductwork codebase for queries that are missing a database index. All migrations live as templates under `lib/generators/ductwork/install/templates/db/**.rb`. Be sure to check all queries and determine if it is a hot path that needs an index. For example, reading next-to-be-claimed IDs, associations, etc. Ensure that suggestions work across at least PostgreSQL, MySQL, and SQLite.
+
+For each finding: file:line, severity, why it matters, suggested fix. Do not modify files.

data/.claude/skills/audit-database-support/SKILL.md

@@ -0,0 +1,18 @@
+---
+name: audit-database-support
+description: Audit ductwork for what database adapters and technologies are supported
+allowed-tools: Read, Grep, Glob
+---
+
+# Audit Clock Drift
+
+Audit the entire OSS ductwork codebase for code and queries that do not support a certain database adapters or technology. Ensure support for:
+
+* PostgreSQL
+* CockroachDB
+* MySQL 8+ (adapter and technology)
+* Trilogy (adapter)
+* SQLite
+* Oracle
+
+For each finding: file:line, severity, why it matters, suggested fix. Do not modify files.

data/.claude/skills/audit-durability/SKILL.md

@@ -0,0 +1,17 @@
+---
+name: audit-durability
+description: Audit ductwork for durability gaps
+allowed-tools: Read, Grep, Glob
+---
+
+# Durability Audit Gap
+
+Audit the entire OSS ductwork codebase for durability gaps.
+
+Check for:
+1. **Stuck pipelines**: claims without transition records, advancements without completion, missing reaper coverage
+2. **Lost data**: writes after observable side effects, missing "write before you act" ordering, places where partial failure is not handled
+3. **Double execution**: missing fencing on claim token or process ID, missing idempotency on transitions, gaps in two-phase commit
+4. **Reaper clobbering**: heartbeat updates racing reaper swwps, stale claim token assumptions, missing recoery count increments
+
+For each finding: file:line, severity, why it matters, suggested fix. Do not modify files.

data/.saturnci/database.yml

@@ -3,3 +3,4 @@ test:
   database: db/test.sqlite3
   pool: 5
   timeout: 5000
+  default_transaction_mode: immediate

data/CHANGELOG-PRO.md

@@ -1,5 +1,65 @@
 # Ductwork Pro Changelog
 
+## [1.0.0] (Unreleased)
+
+- chore: wire payloads to jobs without instantiating whole model objects
+- fix: avoid race condition of `nil`-ing out `execution` on job worker
+- fix: no longer strand `combine`/`collapse` branches in `advancing` when a run-row deadlock rolls back a transition that had already completed the branch (via the shared OSS claim-fence fix); previously the run stalled until the advancer process was reaped
+- fix: lock the run `FOR NO KEY UPDATE` on Postgres in `resolve_terminal_state!` to avoid the run-row deadlock between concurrent `combine`/`collapse` transitions
+- fix: resolve the `collapse` fan-in barrier via `barrier_node` so intermediate `divide`/`combine`/`chain` transitions and nested expands no longer create duplicate collapse targets
+- feat: record the matching `expand` node as `barrier_node` on `collapse` edges in the pipeline definition
+- fix: lower payload enveloped value limit to ~1GB
+- fix: print banner on boot
+- fix: avoid per-batch sort when wiring large collapse fan-ins
+- fix: remove unnecessary ordering so existing index is hit
+- fix: add index to support a keyset `ORDER BY` query for payloads
+- feat: stream large collapse fan-ins via lazy input payloads
+- feat: stream large expand fan-outs via lazy output payloads
+- fix: make changes to reach parity with OSS
+- fix: use existing count attribute on branch instead of `COUNT` query
+- fix: create composite index for the collapse fan-in read
+- chore: do not instantiate full payload activerecord models
+- fix: insert payload records in batches of 1_000
+- fix: harden the kill-and-restart path against thread hangs
+- chore: move configurable pipeline advancer thread pool to the OSS gem
+- fix: replace unnecessary lock with atomic, conditional increment
+- chore: add advancement integration durability tests
+- feat: make `collapse` interruptible, resumeable, and recoverable
+- feat: track `collapse` fan-in with counters on `ductwork_branches`
+- perf: collapse fan-in via atomic counter instead of scanning siblings
+- feat: read `ductwork_payloads` records when executing jobs
+- feat: set `ductwork_payloads.to_job_id` when advancing a branch via `collapse`
+- feat: set `ductwork_payloads.to_job_id` when advancing a branch via `expand`
+- fix: associate `ductwork_payloads` with `ductwork_executions` for origination
+- feat: set `ductwork_payloads.to_job_id` when resuming a `dampen`-ed pipeline run
+- feat: set `ductwork_payloads.to_job_id` when advancing a branch via `converge`
+- feat: set `ductwork_payloads.to_job_id` when advancing a branch via `divide`
+- feat: set `ductwork_payloads.to_job_id` when advancing a branch via `divert`
+- feat: set `ductwork_payloads.to_job_id` when advancing a branch via `combine`
+- feat: set `ductwork_payloads.to_job_id` when advancing a branch via `chain`
+- fix: add `position` column to `ductwork_payloads` table
+- feat: store step output payloads in `ductwork_payloads` records
+- feat: support all OSS v1.0 changes
+- fix: release branch in `Ductwork::Pro::Run#resume!`
+- fix: set status for pipeline, run, and branch when resuming
+- feat: add back dampening during advancement
+- feat: respect delay in definition when advancing branches
+- fix: properly release branch when execution times out
+- fix: call methods on `branch` in pipeline advancer
+- fix: use correct associations in `JobWorker#timed_out?`
+- fix: do not advance pipeline on job timeouts
+- feat: move dampening and resuming to `Run` with a top-level `Pipeline#resume!`
+- fix: properly report metrics when pipeline run completes or halts
+- fix: create `runs` and `branches` records when triggering pipeline
+- chore: change `dampers` association to `runs` instead of `pipelines`
+- chore: regenerate spec migrations to pick up `ductwork` v1.0.0 changes
+
+## [0.8.0]
+
+- feat: support delay and timeout arguments for `divert` and `converge` transitions - this is the last of adding support for the new transitions
+- fix: wrap code with rails app executor
+- feat: support `divert` and `converge` transitions in pipeline advancement
+
 ## [0.7.0]
 
 - feat: allow for passing an argument when resuming a pipeline - this will replace passing the previous step's output payload as the input arguments to the next step

data/CHANGELOG.md

@@ -1,5 +1,148 @@
 # Ductwork Changelog
 
+## [1.0.0]
+
+- chore: add `trilogy` database adapter to the CI test matrix
+- chore: read workflow/pipelines definitions from `app/workflows` directory
+- fix: rescue transient errors in supervisor/runner work loops so one bad iteration doesn't skip graceful shutdown or kill the process
+- chore: rescue invalid foreign key error as to not let thread die
+- fix: reap claims that lost their process ID on a process destroy racing a fresh claim
+- fix: always recover incomplete claims when reusing a PID + machine identifier
+- fix: recover an advancement whose transition completed the branch and then rolled back (e.g. a run-row deadlock victim) instead of stranding the branch in `advancing` — the claim fence and `release!` now use the token captured when advancement starts rather than the in-memory attribute that `complete!`/`halt!` null mid-transition
+- fix: lock the run `FOR NO KEY UPDATE` on Postgres when resolving terminal state so it does not deadlock upgrading past the `FOR KEY SHARE` locks concurrent transitions hold from inserting run-referencing rows
+- fix: fan-in `collapse` correctly when siblings are not direct children of the expanding branch (intermediate `divide`/`combine`/`chain` or nested expands no longer create duplicate collapse targets)
+- feat: record the matching `expand` node as `barrier_node` on `collapse` edges in the pipeline definition
+- fix: use database clock instead of app-server time for execution, job, and process timestamps
+- feat: add `Ductwork::DatabaseClock.now` to read current time from the database
+- fix: add missing indexes for runs and transitions
+- fix: use correct index for claiming for mysql databases
+- chore: add optional force argument on process reap
+- chore: do not raise `StaleClaimError` in `Branch#with_claim_fence`
+- fix: add missing claim fence around branch/advancement mutation queries
+- fix: add a branch claiming guard to skip when no live process record exists
+- fix: cleanup crashed thread and release branch atomically
+- feat: cap pipeline advancer crashes and halt with `advancer_crashes_exhausted`
+- feat: create configuration for pipeline advancer max crash count
+- chore: conditionally update step, run, and pipeline status when claiming
+- fix: fire `on_halt` once and outside of all database transactions
+- fix: add missing database indexes (mostly) for dashboard pages
+- fix: only `sleep` pipeline advancer when no branch is claimed
+- feat: add configurable pipeline advancer thread pool via `pipeline_advancer.count`
+- fix: do not let one wedged worker freeze the whole supervisor work loop
+- fix: ignore already-exited child processes during graceful shutdown
+- chore: set `job_crashes_exhausted` as pipeline halt reason when exhausted crash budget
+- feat: cap and tail back-off execution crashes
+- feat: create configuration for job worker max crash count
+- feat: add `Ductwork::Workflow` alias of `Ductwork::Pipeline`
+- fix: skip "internal" errors when calculating advancement retry budget
+- chore: bump advancement retry count to 10
+- feat: automatically restart threads that are stuck in framework code
+- fix: add secondary order by UUID v7 to get latest step on branch
+- feat: make supervisor reaper timeout configurable
+- chore: add composite index for branch claim step subquery
+- chore: add composite index for `latest_step` lookup
+- chore: add index on processes for last_heartbeat_at timestamp
+- fix: use partial index for resolving run's terminal state for performance
+- fix: use partial index in branch claim query for performance
+- fix: use original claim token when cleaning up dead pipeline advancer threads
+- fix: do not call `on_halt` lifecycle event if transaction rolls back
+- fix: cleanup claimed resources on worker/advancer restart
+- fix: crash execution if thread crashes without process crashing
+- fix: abandon advancements if thread crashes without process crashing
+- fix: prevent OS-level PID reuse from adopting stale `Ductwork::Process` record
+- fix: fence `Execution#crashed!` with process ID so a reclaimed execution isn't clobbered by the reaper
+- fix: check claim tokens before advancing branch
+- feat: create in-scope `idempotency_key` attribute for the `Step` model
+- fix: raise `Execution::CommitFailed` from `#succeeded!`/`#errored!` instead of silently dropping the result when the reaper has clobbered an in-flight execution
+- fix: do not re-query current process ID in execution claiming
+- fix: use database clock in process supervisor to avoid cross-host clock drift
+- feat: add CLI health check command that reports supervisor processes health
+- fix: protect against cross-host clock skew with database time comparisons
+- chore: rescue heartbeat errors to prevent killing process
+- fix: duplicate `Ductwork::BranchLink` records when reviving a pipeline
+- fix: prevent possible double execution in `Pipeline#revive!` by only retrying advancing if it was the halt reason, otherwise retry the job
+- fix: pass failed job input arguments to new job when reviving a pipeline
+- fix: pass in owner process id when executing job - this ensures the process finishing the job is the one that claimed it
+- fix: make `Execution#succeeded!` and `#errored!` idempotent and fence with process id
+- feat: set `ductwork_execution.process_id` when claiming availability
+- chore: associate `ductwork_executions` to `ductwork_processes`
+- chore: hoist usages of app executor out of models and services
+- chore: refactor `Job` methods and claiming onto `Execution`
+- fix: prevent possible double job execution by using "current" `execution` record
+- chore: swap locking for atomic update in `Ductwork::Availability#abandon!`
+- chore: swap locking for atomic update in `Ductwork::Advancement#abandon!`
+- fix: make `Ductwork::Job#execution_crashed!` idempotent to prevent race condition with the reaper
+- fix: prevent possible double job execution by only wrapping user code in rescue
+- fix: atomically release branches conditioned on claim token
+- fix: fail abandoned availabilities and executions when reaping jobs
+- fix: fail abandoned advancements when reaping branches
+- fix: prevent double execution by explicitly killing process before replacing it if it failed health check
+- fix: close durability gap by rescuing branch halting
+- fix: close durability gap by releasing branch in advancement rescue
+- fix: permit `id` parameter when paginating
+- chore: set new execution time to be "now" when job worker crashes (ie. don't set a retry back-off as with job erroring)
+- feat: add a crash count to executions and increment when job worker crashes
+- chore: setup and add first durability integration test
+- chore: nullify claim token when releasing, completing, or halting branch
+- feat: prevent branch claim/release race condition by comparing tokens
+- chore: generate and set claim token on branch record during claiming
+- chore: add `ductwork_branches.claim_token` string column
+- chore: set `@branch` ivar after branch is claimed
+- fix: only use `unique_by` options for non-MySQL db adapters
+- feat: add mysql and postgresql databases to CI test matrix
+- fix: don't use partial indexes for MySQL
+- fix: adopt or create process record when reporting heartbeat
+- fix: fan-in advancement checks branch status instead of step status
+- fix: add back in calling `on_halt` DSL method
+- feat: set `halt_reason` when halting branch
+- chore: add `halt_reason` column to `ductwork_branches` table
+- fix: update proper records and state for advancer retry
+- feat: respect pipeline advancer max retry configuration
+- feat: add pipeline advancer max retry configuration
+- feat: properly set pipeline/run terminal states
+- fix: check for terminal status before setting status on pipeline
+- chore: do not halt pipeline from job worker
+- feat: update branch claim query
+- feat: implement `Ductwork::Pipeline#revive!`
+- chore: add `source_step_id` column on `ductwork_steps`
+- fix: protect against null error backtraces for job results
+- fix: update dashboard pages to handle `ductwork_runs`
+- fix: create `ductwork_runs` records opaquely in rspec test helpers
+- feat: associate `branches`, `steps`, and `tuples` to `runs` instead of `pipelines`
+- feat: add `ductwork_runs` table and model to represent pipeline runs
+- chore: rename `ductwork_runs` to `ductwork_attempts`
+- chore: log when pipeline advancement errors
+- fix: move branch release into main advance transaction
+- fix: set `last_advanced_at` timestamp when releasing branch
+- chore: derive process dead threshold from process reap threshold
+- fix: move transition and advancement creation into claim transaction
+- fix: guard against PID reuse with guard statement
+- fix: use safe navigation operator on process records
+- fix: rescue all job worker errors as to not let thread die
+- fix: reap process (and claims) on process restart or immediate shutdown
+- chore: bump ruby versions in CI
+- chore: bump rails versions in appraisals file
+- chore: DRY up process record destruction
+- fix: release job availabilities during process reaping
+- fix: use proper association to `ductwork_processes` in `ductwork_availabilities`
+- fix: release branches during process reaping
+- feat: add process reaper check to supervisors' run loop
+- chore: DRY up process record adoption and creation
+- fix: create or adopt process records in pipeline advancer and job worker
+- fix: create top-level process record for thread supervisor
+- fix: use safe navigation in rescue when advancing branch
+- fix: use correct column name in migration
+- feat!: advance branches instead of pipelines - BREAKING CHANGE: this completely changes how pipelines are advanced. to migrate let all your current pipelines complete then deploy
+- fix: delete child process records when restarting within supervisor
+- fix: wrap optimistic job claiming in a single transaction
+- chore: bump project and CI ruby versions to v4.0.2
+- fix: add missing index when migrating to UUIDs
+- fix: job availability claim and update state in a single transaction
+- feat: complete and halt active branches for pipelines
+- feat: create initial branch when a pipeline is triggered
+- feat: introduce `Branch` and `BranchJunction` models and tables
+- fix: correctly log transition names via edge
+
 ## [0.26.0]
 
 - feat: add `divert` and complementary `converge` transitions - this is essentially a conditional/case statement transition for pipelines including it's "fan-in" method
@@ -256,4 +399,4 @@
 
 ## [0.1.0]
 
-- Initial release - see [documentation](https://docs.getductwork.io/) for details
+- Initial release - see [documentation](https://www.getductwork.io/docs/) for details

data/CLAUDE.md

@@ -0,0 +1,34 @@
+# Ductwork OSS Architecture Context
+
+This is the OSS `ductwork` gem (LGPL v3). The paid `ductwork-pro` gem
+extends it via `prepend` and adds features that MUST NOT be reimplemented
+or referenced here.
+
+## Lives in OSS (this repo)
+- Core workflow transitions: `chain`, `expand`, `divide`, `divert`, `combine`, `converge`, and `collapse`
+- Core pipeline DSL
+- Two-phase commit (transition + advancement records) for pipeline advancement
+- Supervisor / advancer / worker process hierarchy
+- Forking + threaded concurrency modes
+- Configurable pipeline advancer thread pool
+- Heartbeat-based orphan detection
+- SKIP LOCKED claiming with atomic UPDATE...WHERE fallback
+- Reaper with global-timeout sweeps
+- Automatic restart of worker threads stuck in framework code (no execution claimed)
+- `Ductwork::Pipeline#revive!` API
+- UUID v7 primary keys across PG/MySQL/SQLite
+- Rails engine-mountable web dashboard
+
+## Lives in Pro
+- Human-in-the-loop functionality with `dampen` transition
+- Step timeout feature defined in pipeline definition DSL
+- Step delay feature defined in pipeline definition DSL
+- Automatic restart of worker threads stuck inside job execution (claimed execution that won't return; via step timeout)
+- Large payload support
+- Resumable batched fan-out/fan-in
+- Interruptible pipeline advancement
+- Metric reporting to StatsD
+
+## Hard rules
+- Never reference `Ductwork::Pro::*` constants from OSS code.
+- Pro extends OSS via `prepend`; OSS must remain functional standalone.

data/README.md

@@ -3,13 +3,13 @@
 [![CI](https://github.com/ductwork/ductwork/actions/workflows/main.yml/badge.svg)](https://github.com/ductwork/ductwork/actions/workflows/main.yml)
 [![Gem Version](https://badge.fury.io/rb/ductwork.svg?icon=si%3Arubygems)](https://rubygems.org/gems/ductwork)
 
-A Ruby pipeline and workflow framework.
+A durable workflow orchestration framework for Ruby.
 
-Ductwork lets you build complex pipelines quickly and easily using intuitive Ruby tooling and a natural DSL. No need to learn complicated unified object models or stand up separate runner instances—just write Ruby code and let Ductwork handle the orchestration.
+Ductwork lets you build durable pipelines and workflows quickly and easily using intuitive Ruby tooling and a natural DSL. No need to learn complicated unified object models or stand up separate runner instances, just write Ruby code and let Ductwork handle the orchestration.
 
 There is also a paid [Ductwork Pro](https://www.getductwork.io/) version with more features and support. See the [Pricing](https://www.getductwork.io/#pricing) page to buy a license.
 
-**[Full Documentation](https://docs.getductwork.io/)**
+**[Full Documentation](https://www.getductwork.io/docs/)**
 
 ## Installation
 
@@ -31,34 +31,34 @@ bin/rails generate ductwork:install
 bin/rails generate ductwork:update
 ```
 
-Run migrations and you're ready to start building pipelines!
+Run migrations and you're ready to start building workflows!
 
-## Configuration
 
+## Configuration
 
-The only required configuration is specifying which pipelines to run. Edit the default configuration file `config/ductwork.yml`:
+The only required configuration is specifying which workflows and pipelines to run. Edit the default configuration file `config/ductwork.yml`:
 
 ```yaml
 default: &default
   pipelines:
     - EnrichUserDataPipeline
-    - SendMonthlyStatusReportsPipeline
+    - SendMonthlyStatusReportsWorkflow
 ```
 
-Or use the wildcard to run all pipelines (use cautiously—this can consume significant resources):
+Or use the wildcard to run all pipelines (use cautiously as this can consume significant resources):
 
 ```yaml
 default: &default
   pipelines: "*"
 ```
 
-See the [Configuration Guide](https://docs.getductwork.io/getting-started/configuration.html) for all available options including thread counts, timeouts, and database settings.
+See the [Configuration Guide](https://www.getductwork.io/docs/getting-started/configuration/) for all available options including thread counts, timeouts, and database settings.
 
 ## Usage
 
-### 1. Create a Pipeline Class
+### 1. Create a Workflow Class
 
-Pipeline classes live in `app/pipelines` and inherit from `Ductwork::Pipeline`. While the "Pipeline" suffix is optional, it can help avoid naming collisions:
+Your workflow and pipeline classes live in `app/pipelines` or `app/workflows` and inherit from `Ductwork::Pipeline` or `Ductwork::Workflow` — `Ductwork::Workflow` is an alias for `Ductwork::Pipeline`, so pick whichever name reads best for the process you're modeling. While the "Pipeline" or "Workflow" suffix is optional, it can help avoid naming collisions:
 
 ```ruby
 # app/pipelines/enrich_user_data_pipeline.rb
@@ -111,7 +111,7 @@ end
 
 **Important:** Return values must be JSON-serializable.
 
-See [Defining Pipelines](https://docs.getductwork.io/getting-started/defining-pipelines.html) for detailed documentation.
+See [Defining Pipelines](https://www.getductwork.io/docs/getting-started/defining-pipelines/) for detailed documentation.
 
 ### 4. Run Ductwork
 
@@ -129,7 +129,7 @@ bin/ductwork -c config/ductwork.0.yml
 
 ### 5. Trigger Your Pipeline
 
-Trigger pipelines from anywhere in your Rails application. The `trigger` method returns a `Ductwork::Pipeline` instance for monitoring:
+Trigger workflows from anywhere in your Rails application. The `trigger` method returns a `Ductwork::Pipeline` instance for monitoring:
 
 ```ruby
 # In a Rake task
@@ -142,10 +142,20 @@ end
 def create
   pipeline = EnrichUserDataPipeline.trigger(params[:days_outdated])
 
-  render json: { pipeline_id: pipeline.id, status: pipeline.status }
+  render json: { id: pipeline.id, status: pipeline.status }
 end
 ```
 
+## Delivery Guarantees
+
+Ductwork guarantees **at-least-once**, never exactly-once, execution of each step.
+
+If a worker process is killed (`kill -9`, OOM, host failure, deploy) mid-job, Ductwork can't know whether the step's side effects already ran. Rather than risk silently dropping work, it favors re-running it: a reaper detects the orphaned claim via missed heartbeats and, after a timeout, makes the job eligible to be claimed and executed again, potentially re-running side effects that already completed.
+
+**Write step side effects to be idempotent.** Prefer upserts over inserts, guard non-idempotent external calls (charges, emails, webhooks) with your own dedupe key, etc. Every `Ductwork::Step` exposes `idempotency_key` (a stable ID for that step's execution) for exactly this purpose. Keep steps as small as possible and limit each one to as few side effects as you can; the smaller the blast radius of a re-run, the easier it is to make idempotent.
+
+Pipeline advancement (moving a branch from one step to the next) is tracked separately via its own claim/commit records, so a crash between "step finished" and "pipeline advanced" is handled the same way: the stalled advancement is reaped and retried rather than left stuck.
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/app/controllers/ductwork/application_controller.rb

@@ -4,8 +4,8 @@ module Ductwork
   class ApplicationController < ActionController::Base
     DEFAULT_PER_PAGE = 50
 
-    def query_pipelines
-      Ductwork::Pipeline
+    def query_pipeline_runs
+      Ductwork::Run
         .includes(steps: { job: { executions: :result } })
         .then(&method(:filter_by_klass))
         .then(&method(:filter_by_status))
@@ -15,7 +15,7 @@ module Ductwork
 
     def filter_by_klass(relation)
       if params[:klass].present?
-        relation.where(klass: params[:klass])
+        relation.where(pipeline_klass: params[:klass])
       else
         relation
       end

data/app/controllers/ductwork/dashboards_controller.rb

@@ -13,7 +13,7 @@ module Ductwork
       @klasses = Ductwork::Pipeline.group(:klass).pluck(:klass).sort
       @statuses = Ductwork::Pipeline.statuses.keys
       @per_page = 25
-      @pipelines = query_pipelines
+      @runs = query_pipeline_runs
     end
   end
 end

data/app/controllers/ductwork/pipelines_controller.rb

@@ -3,23 +3,24 @@
 module Ductwork
   class PipelinesController < Ductwork::ApplicationController
     def index
-      @pipelines = query_pipelines
+      @runs = query_pipeline_runs
       @klasses = Ductwork::Pipeline.group(:klass).pluck(:klass).sort
       @statuses = Ductwork::Pipeline.statuses.keys
     end
 
     def show
       @pipeline = Ductwork::Pipeline.find(params[:id])
+      @last_run = @pipeline.runs.order(started_at: :desc).first
       @per_page = 10
       @steps = query_steps
-      @klasses = @pipeline.steps.group(:klass).pluck(:klass).sort
+      @klasses = @last_run.steps.group(:klass).pluck(:klass).sort
       @statuses = Ductwork::Step.statuses.keys
     end
 
     private
 
     def query_steps
-      @pipeline
+      @last_run
         .steps
         .then(&method(:filter_by_klass))
         .then(&method(:filter_by_status))

data/app/helpers/ductwork/application_helper.rb

@@ -21,7 +21,7 @@ module Ductwork
     def next_page_path
       next_page = params[:page].to_i + 1
       next_params = params
-                    .permit(:controller, :action, :klass, :status, :page)
+                    .permit(:controller, :action, :id, :klass, :status, :page)
                     .merge(page: next_page)
 
       url_for(**next_params)
@@ -30,7 +30,7 @@ module Ductwork
     def previous_page_path
       previous_page = params[:page].to_i - 1
       previous_params = params
-                        .permit(:controller, :action, :klass, :status, :page)
+                        .permit(:controller, :action, :id, :klass, :status, :page)
                         .merge(page: previous_page)
 
       url_for(**previous_params)

data/app/views/ductwork/dashboards/show.html.erb

@@ -80,29 +80,29 @@
         </tr>
       </thead>
       <tbody>
-        <% @pipelines.each do |pipeline| %>
-          <tr class="row-link" data-href="<%= pipeline_path(pipeline.id) %>">
+        <% @runs.each do |run| %>
+          <tr class="row-link" data-href="<%= pipeline_path(run.pipeline_id) %>">
             <td>
               <code>
-                <%= pipeline.id %>
+                <%= run.pipeline_id %>
               </code>
             </td>
             <td>
               <code>
-                <%= pipeline.klass %>
+                <%= run.pipeline_klass %>
               </code>
             </td>
             <td>
-              <div class="status-pill <%= pipeline.status %>">
-                <%= pipeline.status.gsub("_", "-") %>
+              <div class="status-pill <%= run.status %>">
+                <%= run.status.gsub("_", "-") %>
               </div>
             </td>
             <td>
               <%
                 count = Ductwork::Result
-                        .joins(execution: { job: { step: :pipeline }})
+                        .joins(execution: { job: { step: { run: :pipeline }}})
                         .failure
-                        .where(ductwork_pipelines: { id: pipeline.id })
+                        .where(ductwork_pipelines: { id: run.pipeline_id })
                         .count
               %>
               <% if count.zero? %>
@@ -116,16 +116,16 @@
             </td>
             <td>
               <div class="tight-timestamp">
-                <%= pipeline.started_at.iso8601(3) %>
+                <%= run.started_at.iso8601(3) %>
               </div>
             </td>
             <td>
-              <% if pipeline.completed_at.nil? %>
-                <div data-started-at=<%= pipeline.started_at.iso8601 %>>
+              <% if run.completed_at.nil? %>
+                <div data-started-at=<%= run.started_at.iso8601 %>>
                   <span id="elapsed-timer">0h 0m 0s</span>
                 </div>
               <% else %>
-                <%= formatted_time_distance(pipeline.started_at, pipeline.completed_at) %>
+                <%= formatted_time_distance(run.started_at, run.completed_at) %>
               <% end %>
             </td>
           </tr>

data/app/views/ductwork/pipelines/index.html.erb

@@ -54,25 +54,25 @@
       </tr>
     </thead>
     <tbody>
-      <% @pipelines.each do |pipeline| %>
-        <tr class="row-link" data-href="<%= pipeline_path(pipeline.id) %>">
+      <% @runs.each do |run| %>
+        <tr class="row-link" data-href="<%= pipeline_path(run.pipeline_id) %>">
           <td>
-            <code><%= pipeline.id %></code>
+            <code><%= run.pipeline_id %></code>
           </td>
           <td>
-            <code><%= pipeline.klass %></code>
+            <code><%= run.pipeline_klass %></code>
           </td>
           <td>
-            <div class="status-pill <%= pipeline.status %>">
-              <%= pipeline.status.gsub("_", "-") %>
+            <div class="status-pill <%= run.status %>">
+              <%= run.status.gsub("_", "-") %>
             </div>
           </td>
           <td>
             <%
               count = Ductwork::Result
-                      .joins(execution: { job: { step: :pipeline }})
+                      .joins(execution: { job: { step: { run: :pipeline }}})
                       .failure
-                      .where(ductwork_pipelines: { id: pipeline.id })
+                      .where(ductwork_pipelines: { id: run.pipeline_id })
                       .count
             %>
             <% if count.zero? %>
@@ -86,16 +86,16 @@
           </td>
           <td>
             <div class="tight-timestamp">
-              <%= pipeline.started_at.iso8601(3) %>
+              <%= run.started_at.iso8601(3) %>
             </div>
           </td>
           <td>
-            <% if pipeline.completed_at.nil? %>
-              <div data-started-at=<%= pipeline.started_at.iso8601 %>>
+            <% if run.completed_at.nil? %>
+              <div data-started-at=<%= run.started_at.iso8601 %>>
                 <span id="elapsed-timer">0h 0m 0s</span>
               </div>
             <% else %>
-              <%= formatted_time_distance(pipeline.started_at, pipeline.completed_at) %>
+              <%= formatted_time_distance(run.started_at, run.completed_at) %>
             <% end %>
           </td>
         </tr>