good_pipeline 0.5.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e8d45ee63fb102b327e2e847cd277c911c625711562117c172119b3384d08447
-  data.tar.gz: 888ac688b1e600a74fbc70b36c4ab0aa1d7747d4f986bd8cc9d688a6ab8c8dbc
+  metadata.gz: 4294d8f713a4aa48f6d9680f1b3d1c264f9ff51ec5fb0f38c1347bf3f7f44e19
+  data.tar.gz: 428d2ba77ef93e773b6016c3220eeb76174fcdea9463f71e3b150766e66f15e8
 SHA512:
-  metadata.gz: 642f603b2991bb82256510cf3ea1a13f511f7009a7ba82f10d9fb8c264acdc27d213629842414293a6728c13d055ecf808ba71c555fc9bf4705f3b3b8a23ab5f
-  data.tar.gz: 7941dcef31ca4d86f5fb796803a61d3c3d9af3ed8442aaace983c5a6c5d62360c16a4458ab7130f34764d14e37d53cc79167e4296a52b31bc0381300a3bc74b6
+  metadata.gz: df4a09f4fe2b67119f633e10ef83a05f24315b0017b8b634acfca6a9efc18182b9c6419cab8e0b1cdd0587be694e1c84c779b85c4875b5aef4ee13e4eedc3388
+  data.tar.gz: 975e92dfb856c37fa1806951798fbb449b4c1e6bec3df3bb241c1621c1bf3f88b81e61c3c039c3f0a64ea6620c084048b5a8ba5afb959921edadf9c533210745

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 ## [Unreleased]
 
+## [0.6.0] - 2026-10-29
+
+### Breaking changes
+
+- **Chain failure strategies** — chained pipelines now use each upstream pipeline's stored `failure_strategy`. An upstream that succeeds or fails with `:ignore` satisfies its chain dependency; any other terminal outcome skips dependents. Downstream pipelines wait for every dependency, including when chains are registered after an upstream finishes. Ignored failures keep their `failed` status and failure callbacks. No new setting or migration is required.
+- **Ruby 4 minimum** — GoodPipeline now requires Ruby 4.0 or newer. Development, CI, and gem releases use Ruby 4.0.9.
+
 ## [0.5.0] - 2026-09-01
 
 ### Added

data/README.md

@@ -6,7 +6,7 @@ Define multi-step workflows as directed acyclic graphs — not linear chains. St
 
 ## Requirements
 
-- Ruby >= 3.2
+- Ruby >= 4.0
 - Rails >= 7.2
 - PostgreSQL
 - GoodJob >= 4.14 with `preserve_job_records = true`
@@ -126,7 +126,7 @@ Set at the pipeline level with `failure_strategy`:
 |---|---|
 | `:halt` (default) | Stop all pending steps when any step fails |
 | `:continue` | Let independent branches continue; skip only blocked downstream steps |
-| `:ignore` | Treat failures as successes for dependency resolution |
+| `:ignore` | Treat failed steps and pipelines as satisfied for dependency resolution |
 
 Per-step overrides via `on_failure:` in `run` apply to that step's outgoing edges only.
 
@@ -201,7 +201,7 @@ GoodPipeline.run(
 ).then(MergeMediaPipeline, with: { video_id: 123, audio_id: 456 })
 ```
 
-If an upstream pipeline fails, halts, is canceled, or is skipped, downstream pipelines are automatically skipped.
+Chained pipelines wait for every upstream to succeed or fail with `failure_strategy :ignore`. Any other terminal outcome skips dependent pipelines. Ignored failures keep their `failed` status and still trigger failure callbacks.
 
 ### Monitoring
 

data/demo/test/good_pipeline/test_dashboard.rb

@@ -32,15 +32,16 @@ module GoodPipeline
       assert_select ".gp-status-filter-scroll > .gp-segment[role=group][aria-label='Status filter']", count: 1
       assert_includes response.body, %(id="gp-table")
       assert_no_match(/data-href/, response.body)
-      asset_href = response.body[%r{href="([^"]*/frontend/static/0-5-0/dashboard\.css)"}, 1]
+      asset_version = GoodPipeline::VERSION.tr(".", "-")
+      asset_href = response.body[%r{href="([^"]*/frontend/static/#{asset_version}/dashboard\.css)"}, 1]
 
-      assert asset_href&.end_with?("/frontend/static/0-5-0/dashboard.css")
+      assert asset_href&.end_with?("/frontend/static/#{asset_version}/dashboard.css")
       refute_includes response.body, "style.css"
       refute_includes response.body, "<script>"
 
       assert_dashboard_css(asset_href)
 
-      get "/good_pipeline/frontend/static/0-5-0/dashboard.js"
+      get "/good_pipeline/frontend/static/#{asset_version}/dashboard.js"
 
       assert_response :success
       assert_includes response.media_type, "javascript"

data/demo/test/integration/test_pipeline_chaining.rb

@@ -2,6 +2,10 @@
 
 require "test_helper"
 
+class IgnoreChainPipeline < ContinueTestPipeline
+  failure_strategy :ignore
+end
+
 class TestPipelineChaining < ActiveSupport::TestCase
   def run_all_to_completion(pipeline_records, timeout: 15)
     deadline = Time.current + timeout
@@ -46,6 +50,32 @@ class TestPipelineChaining < ActiveSupport::TestCase
     assert_equal "skipped", downstream.status
   end
 
+  def test_ignored_failure_stays_failed_and_allows_the_chain_to_finish
+    IgnoreChainPipeline.run
+                       .then(NotificationPipeline, with: {})
+                       .then(ArchivePipeline, with: {})
+
+    run_all_to_completion(GoodPipeline::PipelineRecord.all.to_a)
+
+    assert_equal(
+      { "IgnoreChainPipeline" => "failed", "NotificationPipeline" => "succeeded", "ArchivePipeline" => "succeeded" },
+      GoodPipeline::PipelineRecord.pluck(:type, :status).to_h
+    )
+  end
+
+  def test_continue_failure_skips_dependents_even_when_they_use_ignore
+    ContinueTestPipeline.run
+                        .then(IgnoreChainPipeline, with: {})
+                        .then(NotificationPipeline, with: {})
+
+    run_all_to_completion(GoodPipeline::PipelineRecord.all.to_a)
+
+    assert_equal(
+      { "ContinueTestPipeline" => "failed", "IgnoreChainPipeline" => "skipped", "NotificationPipeline" => "skipped" },
+      GoodPipeline::PipelineRecord.pluck(:type, :status).to_h
+    )
+  end
+
   # --- Fan-out ---
 
   def test_fan_out_both_start_after_upstream_succeeds
@@ -118,6 +148,41 @@ class TestPipelineChaining < ActiveSupport::TestCase
     assert_equal "skipped", archive.status
   end
 
+  def test_ignored_failure_waits_for_the_other_upstream_to_finish
+    ignored = create_pipeline(status: "running", on_failure_strategy: "ignore")
+    ignored_step = build_step(ignored, key: "optional", coordination_status: "enqueued")
+    waiting = create_pipeline(status: "running", on_failure_strategy: "halt")
+    waiting_step = build_step(waiting, key: "required", coordination_status: "enqueued")
+    downstream = GoodPipeline::Chain.new([ignored, waiting]).then(NotificationPipeline)
+
+    GoodPipeline::Coordinator.complete_step(ignored_step.id, succeeded: false)
+
+    assert_equal "pending", downstream.reload.status
+
+    GoodPipeline::Coordinator.complete_step(waiting_step.id, succeeded: true)
+
+    assert_equal "running", downstream.reload.status
+  end
+
+  def test_fan_in_requires_every_failure_to_be_ignored
+    ignored = create_pipeline(status: "failed", on_failure_strategy: "ignore")
+    blocking = create_pipeline(status: "failed", on_failure_strategy: "continue")
+
+    downstream = GoodPipeline::Chain.new([ignored, blocking]).then(NotificationPipeline)
+
+    assert_equal "skipped", downstream.reload.status
+  end
+
+  def test_late_chain_registration_accepts_an_ignored_failure
+    upstream = IgnoreChainPipeline.run
+    run_pipeline_to_completion(upstream)
+
+    downstream = upstream.then(NotificationPipeline)
+    run_pipeline_to_completion(downstream)
+
+    assert_equal "succeeded", downstream.reload.status
+  end
+
   # --- Fan-out then fan-in ---
 
   def test_fan_out_then_fan_in
@@ -159,7 +224,7 @@ class TestPipelineChaining < ActiveSupport::TestCase
   end
 
   def test_canceling_pending_upstream_automatically_skips_downstream
-    upstream = create_pipeline(type: "TestPipeline", status: "pending", on_failure_strategy: "halt")
+    upstream = create_pipeline(type: "TestPipeline", status: "pending", on_failure_strategy: "ignore")
     upstream_step = build_step(upstream, key: "work")
     downstream = create_pipeline(type: "NotificationPipeline", status: "pending", on_failure_strategy: "halt")
     GoodPipeline::ChainRecord.create!(upstream_pipeline: upstream, downstream_pipeline: downstream)
@@ -172,7 +237,7 @@ class TestPipelineChaining < ActiveSupport::TestCase
   end
 
   def test_canceling_running_upstream_waits_for_drain_before_skipping_downstream
-    upstream = create_pipeline(type: "TestPipeline", status: "running", on_failure_strategy: "halt")
+    upstream = create_pipeline(type: "TestPipeline", status: "running", on_failure_strategy: "ignore")
     upstream_step = build_step(upstream, key: "work")
     upstream_step.update_columns(coordination_status: "enqueued")
     downstream = create_pipeline(type: "NotificationPipeline", status: "pending", on_failure_strategy: "halt")

data/docs/failure-strategies.md

@@ -1,6 +1,6 @@
 # Failure Strategies
 
-Three failure strategies control what happens when a step fails. Set one at the pipeline level, then override per step if needed.
+Three failure strategies control what happens when a step fails and whether a failed pipeline allows its chain to continue. Set one at the pipeline level, then override per step if needed.
 
 ## Pipeline-level strategy
 
@@ -140,6 +140,10 @@ A barrier depends on every step in its preceding phase, so the same rules apply
 
 Whenever eligible work remains, every new terminal step releases its outgoing dependency counter exactly once. This includes enqueue-time configuration failures and steps skipped by partial halt propagation, preventing a barrier or ordinary fan-in from remaining pending after all upstreams are terminal.
 
+## Pipeline chains
+
+Pipeline chains use each upstream pipeline's stored `failure_strategy`. Step-level overrides do not change the chain policy. See [Pipeline Chaining](./pipeline-chaining.md#unsuccessful-upstream-propagation) for how failures affect dependent pipelines.
+
 ## Early termination with success
 
 Sometimes a job determines there is nothing to do — the account is deactivated, the resource was already processed, etc. Call `halt_pipeline!` to stop the pipeline early and mark it as `succeeded`:

data/docs/introduction.md

@@ -54,7 +54,7 @@ Choose Geneva Drive when your workflow is inherently sequential and you need pau
 
 ## Requirements
 
-- Ruby >= 3.2
+- Ruby >= 4.0
 - Rails >= 7.2
 - PostgreSQL
 - GoodJob >= 4.14 with `preserve_job_records = true`

data/docs/monitoring.md

@@ -45,7 +45,7 @@ step.duration             # => 12.34 (Float seconds, from GoodJob record)
 | `succeeded` | All steps terminal, none failed |
 | `failed` | One or more steps failed; `:continue` or `:ignore` strategy was used |
 | `halted` | `:halt` strategy was applied — `halt_triggered` is `true` |
-| `skipped` | Skipped because an upstream pipeline in a chain failed |
+| `skipped` | An upstream pipeline failed without `:ignore`, halted, was canceled, or was skipped |
 | `canceled` | Cancellation completed, or a pending pipeline was canceled before it started — terminal |
 
 ## Step statuses

data/docs/pipeline-chaining.md

@@ -1,6 +1,8 @@
 # Pipeline Chaining
 
-Pipelines can be wired together into pipeline-level DAGs. A downstream pipeline starts only after all its upstream pipelines succeed.
+Pipelines can be wired together into pipeline-level DAGs. A downstream pipeline starts after every upstream has succeeded or failed with `failure_strategy :ignore`.
+
+The examples below use the default `:halt` strategy unless another strategy is shown.
 
 ## Serial chain
 
@@ -88,9 +90,9 @@ Pipeline chaining is a first-class primitive — upstream/downstream relationshi
 1. Creates downstream pipeline records with `status: pending` — params are stored immediately at chain registration time
 2. Creates `good_pipeline_chains` rows linking upstream to downstream pipeline IDs
 3. If any upstream has already reached a terminal state, immediately triggers chain propagation so the downstream is started or skipped
-4. After any upstream pipeline reaches a terminal state, the chain coordinator checks if all upstreams for each downstream have succeeded
-5. If all upstreams succeeded, the downstream pipeline starts (root steps are enqueued)
-6. If any upstream fails, halts, is canceled, or is skipped, the downstream pipeline is set to `skipped`
+4. After any upstream pipeline reaches a terminal state, the chain coordinator checks each upstream's status and stored failure strategy
+5. If every upstream succeeded or failed with `:ignore`, the downstream pipeline starts (root steps are enqueued)
+6. If any terminal upstream does not satisfy its dependency, the downstream pipeline is set to `skipped`
 
 The chain coordinator uses a blocking `FOR UPDATE` row lock. A contending propagation waits, then rechecks the downstream pipeline while locked, so a fan-in wake-up cannot be silently dropped. This also makes `.then` safe to call after an upstream has already completed.
 
@@ -98,14 +100,16 @@ Individual chain transitions are atomic and propagation is idempotent, but deliv
 
 ## Unsuccessful upstream propagation
 
-If any upstream pipeline in a chain fails, halts, is canceled, or is skipped:
+A failed upstream using `failure_strategy :ignore` allows the chain to continue after it finishes. Its status remains `failed`, and failure callbacks still run. The pipeline-level strategy controls chain dependencies; a step-level `on_failure:` override only controls that step's outgoing edges inside its pipeline.
+
+If any upstream pipeline fails with `:continue`, halts, is canceled, or is skipped:
 
 - The downstream pipeline transitions to `skipped`
 - Any further downstream pipelines are also recursively `skipped`
 - `on_complete` callbacks fire on skipped pipelines, but `on_failure` does **not** — being skipped is not considered a failure
 
 ```
-A (failed) → B (skipped) → C (skipped) → D (skipped)
+A (failed, :continue) → B (skipped) → C (skipped) → D (skipped)
 ```
 
 A `canceling` upstream is still nonterminal, so its downstream pipelines remain pending while already-enqueued work drains. Once the upstream reaches `canceled`, normal chain propagation skips the pending downstream graph.

data/gemfiles/rails_7_2.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    good_pipeline (0.5.0)
+    good_pipeline (0.6.0)
       activerecord (>= 7.2)
       good_job (>= 4.14)
       railties (>= 7.2)
@@ -336,7 +336,7 @@ CHECKSUMS
   fugit (1.13.0) sha256=a4f093fce740da52f216740a5041e2a594ea763cdb89e8b2754ca4399634ab18
   globalid (1.4.0) sha256=037f12fbf1d9d7a014d501c2d5c77356fd4ddd96d7a7991d6700bba96706f427
   good_job (4.19.2) sha256=73f4ea0595a29d28b9d9c0bf23361cc7434542b4d286f59b803819110c26e4b0
-  good_pipeline (0.5.0)
+  good_pipeline (0.6.0)
   i18n (1.15.2) sha256=00f9eb62412fe593b2a65a97daa75300d37abb8f7202ec748e94b6d46a9dd1b5
   io-console (0.9.2) sha256=efa74f891dd03c0939a931dfc6e74c2813d904763d456ea9762b0525e748db08
   irb (1.18.0) sha256=de9454a0703a54704b9811a5ef31a60c86949fbf4013fcf244fabc7c775248e3

data/gemfiles/rails_8_0.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    good_pipeline (0.5.0)
+    good_pipeline (0.6.0)
       activerecord (>= 7.2)
       good_job (>= 4.14)
       railties (>= 7.2)
@@ -334,7 +334,7 @@ CHECKSUMS
   fugit (1.13.0) sha256=a4f093fce740da52f216740a5041e2a594ea763cdb89e8b2754ca4399634ab18
   globalid (1.4.0) sha256=037f12fbf1d9d7a014d501c2d5c77356fd4ddd96d7a7991d6700bba96706f427
   good_job (4.19.2) sha256=73f4ea0595a29d28b9d9c0bf23361cc7434542b4d286f59b803819110c26e4b0
-  good_pipeline (0.5.0)
+  good_pipeline (0.6.0)
   i18n (1.15.2) sha256=00f9eb62412fe593b2a65a97daa75300d37abb8f7202ec748e94b6d46a9dd1b5
   io-console (0.9.2) sha256=efa74f891dd03c0939a931dfc6e74c2813d904763d456ea9762b0525e748db08
   irb (1.18.0) sha256=de9454a0703a54704b9811a5ef31a60c86949fbf4013fcf244fabc7c775248e3

data/gemfiles/rails_8_1.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    good_pipeline (0.5.0)
+    good_pipeline (0.6.0)
       activerecord (>= 7.2)
       good_job (>= 4.14)
       railties (>= 7.2)
@@ -338,7 +338,7 @@ CHECKSUMS
   fugit (1.13.0) sha256=a4f093fce740da52f216740a5041e2a594ea763cdb89e8b2754ca4399634ab18
   globalid (1.4.0) sha256=037f12fbf1d9d7a014d501c2d5c77356fd4ddd96d7a7991d6700bba96706f427
   good_job (4.19.2) sha256=73f4ea0595a29d28b9d9c0bf23361cc7434542b4d286f59b803819110c26e4b0
-  good_pipeline (0.5.0)
+  good_pipeline (0.6.0)
   i18n (1.15.2) sha256=00f9eb62412fe593b2a65a97daa75300d37abb8f7202ec748e94b6d46a9dd1b5
   io-console (0.9.2) sha256=efa74f891dd03c0939a931dfc6e74c2813d904763d456ea9762b0525e748db08
   irb (1.18.0) sha256=de9454a0703a54704b9811a5ef31a60c86949fbf4013fcf244fabc7c775248e3

data/lib/good_pipeline/chain_coordinator.rb

@@ -22,7 +22,7 @@ module GoodPipeline
             locked.transition_to!(:skipped)
             Coordinator.dispatch_callbacks_once(locked, :skipped)
             skipped_downstream_ids = locked.downstream_pipelines.pluck(:id)
-          elsif all_upstreams_succeeded?(locked)
+          elsif all_upstreams_satisfied?(locked)
             start_pipeline(locked)
           end
         end
@@ -32,12 +32,16 @@ module GoodPipeline
 
       def should_skip_downstream?(pipeline)
         pipeline.upstream_pipelines.any? do |upstream|
-          upstream.failed? || upstream.halted? || upstream.skipped? || upstream.canceled?
+          upstream.terminal? && !upstream_satisfied?(upstream)
         end
       end
 
-      def all_upstreams_succeeded?(pipeline)
-        pipeline.upstream_pipelines.all?(&:succeeded?)
+      def upstream_satisfied?(upstream)
+        upstream.succeeded? || (upstream.failed? && upstream.ignore?)
+      end
+
+      def all_upstreams_satisfied?(pipeline)
+        pipeline.upstream_pipelines.all? { |upstream| upstream_satisfied?(upstream) }
       end
 
       def start_pipeline(pipeline_record)

data/lib/good_pipeline/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module GoodPipeline
-  VERSION = "0.5.0"
+  VERSION = "0.6.0"
 end

data/mise.toml

@@ -1,5 +1,5 @@
 [tools]
-ruby = "3.4.7"
+ruby = "4.0.9"
 
 [tasks."docker:start"]
 description = "Start Docker services using docker-compose"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: good_pipeline
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Ali Hamdi Ali Fadel
@@ -248,14 +248,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 3.2.0
+      version: 4.0.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.9
+rubygems_version: 4.0.20
 specification_version: 4
 summary: DAG-based job pipeline orchestration for Rails, built on GoodJob
 test_files: []