good_pipeline 0.1.0

data/docs/package.json

@@ -0,0 +1,11 @@
+{
+  "private": true,
+  "scripts": {
+    "docs:dev": "vitepress dev",
+    "docs:build": "vitepress build",
+    "docs:preview": "vitepress preview"
+  },
+  "devDependencies": {
+    "vitepress": "^1.6.3"
+  }
+}

data/docs/pipeline-chaining.md

@@ -0,0 +1,104 @@
+# Pipeline Chaining
+
+GoodPipeline supports wiring pipelines together into pipeline-level DAGs. A downstream pipeline starts only after all its upstream pipelines succeed.
+
+## Serial chain
+
+```ruby
+VideoProcessingPipeline
+  .run(video_id: 123)
+  .then(NotificationPipeline, with: { video_id: 123 })
+```
+
+`NotificationPipeline` starts only after `VideoProcessingPipeline` succeeds.
+
+## Multi-step serial chain
+
+```ruby
+VideoProcessingPipeline
+  .run(video_id: 123)
+  .then(QualityCheckPipeline, with: { video_id: 123 })
+  .then(NotificationPipeline, with: { video_id: 123 })
+```
+
+Each pipeline waits for the previous one to succeed.
+
+## Fan-out
+
+Multiple downstream pipelines start in parallel when the upstream succeeds:
+
+```ruby
+VideoProcessingPipeline
+  .run(video_id: 123)
+  .then(
+    [NotificationPipeline, with: { video_id: 123 }],
+    [AnalyticsPipeline,    with: { video_id: 123 }]
+  )
+```
+
+Both `NotificationPipeline` and `AnalyticsPipeline` start simultaneously.
+
+## Fan-out then fan-in
+
+```ruby
+VideoProcessingPipeline
+  .run(video_id: 123)
+  .then(QualityCheckPipeline, with: { video_id: 123 })
+  .then(
+    [NotificationPipeline, with: { video_id: 123 }],
+    [AnalyticsPipeline,    with: { video_id: 123 }]
+  )
+  .then(ArchivePipeline, with: { video_id: 123 })
+```
+
+```
+VideoProcessingPipeline
+          ↓
+  QualityCheckPipeline
+          ↓
+     ┌────┴────┐
+     ↓         ↓
+Notification Analytics
+     └────┬────┘
+          ↓
+   ArchivePipeline
+```
+
+`ArchivePipeline` waits for **both** `NotificationPipeline` and `AnalyticsPipeline` to succeed.
+
+## Parallel start
+
+Run multiple pipelines in parallel from the start using `GoodPipeline.run`:
+
+```ruby
+GoodPipeline.run(
+  [VideoProcessingPipeline, with: { video_id: 123 }],
+  [AudioProcessingPipeline, with: { audio_id: 456 }]
+).then(MergeMediaPipeline, with: { video_id: 123, audio_id: 456 })
+```
+
+Both pipelines start immediately. `MergeMediaPipeline` waits for both to succeed.
+
+## How `.then` works internally
+
+`.then` returns a `GoodPipeline::Chain` object which:
+
+1. Creates downstream pipeline records with `status: pending` — params are stored immediately at chain registration time
+2. Creates `good_pipeline_chains` rows linking upstream to downstream pipeline IDs
+3. After any upstream pipeline reaches a terminal state, the chain coordinator checks if all upstreams for each downstream have succeeded
+4. If all upstreams succeeded, the downstream pipeline starts (root steps are enqueued)
+5. If any upstream fails, halts, or is skipped, the downstream pipeline is set to `skipped`
+
+The chain coordinator uses the **same atomic row-locking pattern** (`FOR UPDATE SKIP LOCKED`) as the step-level coordinator to prevent double-start races.
+
+## Failure propagation
+
+If any upstream pipeline in a chain fails, halts, or is skipped:
+
+- The downstream pipeline transitions to `skipped`
+- Any further downstream pipelines are also recursively `skipped`
+- `on_complete` callbacks fire on skipped pipelines, but `on_failure` does **not** — being skipped is not considered a failure
+
+```
+A (failed) → B (skipped) → C (skipped) → D (skipped)
+```

data/lib/generators/good_pipeline/install/install_generator.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+require "rails/generators/active_record"
+
+module GoodPipeline
+  class InstallGenerator < Rails::Generators::Base
+    include ActiveRecord::Generators::Migration
+
+    source_root File.expand_path("templates", __dir__)
+
+    desc "Creates the GoodPipeline migration file."
+    def create_migration_file
+      migration_template(
+        "create_good_pipeline_tables.rb.erb",
+        "db/migrate/create_good_pipeline_tables.rb"
+      )
+    end
+  end
+end

data/lib/generators/good_pipeline/install/templates/create_good_pipeline_tables.rb.erb

@@ -0,0 +1,51 @@
+class CreateGoodPipelineTables < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
+  def change
+    # Uncomment for Postgres v12 or earlier to enable gen_random_uuid() support
+    # enable_extension 'pgcrypto'
+
+    create_table :good_pipeline_pipelines, id: :uuid do |t|
+      t.string :type, null: false
+      t.jsonb :params, null: false, default: {}
+      t.string :status, null: false, default: "pending"
+      t.boolean :halt_triggered, null: false, default: false
+      t.uuid :good_job_batch_id
+      t.string :on_failure_strategy, null: false, default: "halt"
+      t.datetime :callbacks_dispatched_at
+
+      t.timestamps
+    end
+
+    add_index :good_pipeline_pipelines, :status
+
+    create_table :good_pipeline_steps, id: :uuid do |t|
+      t.references :pipeline, null: false, foreign_key: { to_table: :good_pipeline_pipelines }, type: :uuid
+      t.string :key, null: false
+      t.string :job_class, null: false
+      t.jsonb :params, null: false, default: {}
+      t.string :coordination_status, null: false, default: "pending"
+      t.string :on_failure_strategy
+      t.string :queue
+      t.integer :priority
+      t.uuid :good_job_batch_id
+      t.uuid :good_job_id
+      t.integer :attempts
+      t.string :error_class
+      t.text :error_message
+
+      t.timestamps
+    end
+
+    add_index :good_pipeline_steps, %i[pipeline_id key], unique: true
+
+    create_table :good_pipeline_dependencies do |t|
+      t.references :pipeline, null: false, foreign_key: { to_table: :good_pipeline_pipelines }, type: :uuid
+      t.references :step, null: false, foreign_key: { to_table: :good_pipeline_steps }, type: :uuid
+      t.references :depends_on_step, null: false, foreign_key: { to_table: :good_pipeline_steps }, type: :uuid
+    end
+
+    create_table :good_pipeline_chains, id: :uuid do |t|
+      t.references :upstream_pipeline, null: false, foreign_key: { to_table: :good_pipeline_pipelines }, type: :uuid
+      t.references :downstream_pipeline, null: false, foreign_key: { to_table: :good_pipeline_pipelines }, type: :uuid
+    end
+  end
+end

data/lib/good_pipeline/chain.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require "forwardable"
+
+module GoodPipeline
+  class Chain
+    extend Forwardable
+
+    attr_reader :pipeline_records
+
+    def_delegators :first_pipeline_record,
+                   :id, :status, :params, :type, :terminal?, :reload,
+                   :steps, :dependencies, :halt_triggered?,
+                   :callbacks_dispatched_at, :on_failure_strategy
+
+    def initialize(pipeline_records)
+      @pipeline_records = Array(pipeline_records)
+    end
+
+    def then(*arguments) # rubocop:disable Metrics/MethodLength
+      configs = normalize_arguments(arguments)
+      downstream_records = []
+
+      configs.each do |pipeline_class, pipeline_params|
+        instance = pipeline_class.build(**pipeline_params)
+        downstream_record = Runner.call(instance, start: false)
+        downstream_records << downstream_record
+
+        @pipeline_records.each do |upstream_record|
+          ChainRecord.create!(
+            upstream_pipeline: upstream_record,
+            downstream_pipeline: downstream_record
+          )
+        end
+      end
+
+      Chain.new(downstream_records)
+    end
+
+    private
+
+    def first_pipeline_record
+      @pipeline_records.first
+    end
+
+    def normalize_arguments(arguments)
+      if arguments.first.is_a?(Array)
+        arguments.map { |config| GoodPipeline.extract_pipeline_config(config) }
+      else
+        [GoodPipeline.extract_pipeline_config(arguments)]
+      end
+    end
+  end
+end

data/lib/good_pipeline/chain_coordinator.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module GoodPipeline
+  class ChainCoordinator
+    def self.propagate_terminal_state(pipeline)
+      pipeline.downstream_pipelines.each do |downstream_pipeline|
+        try_start_downstream(downstream_pipeline.id)
+      end
+    end
+
+    def self.try_start_downstream(pipeline_id) # rubocop:disable Metrics/MethodLength
+      skipped_downstream_ids = nil
+
+      PipelineRecord.transaction do
+        locked = PipelineRecord.lock("FOR UPDATE SKIP LOCKED").find_by(id: pipeline_id)
+        return unless locked&.pending?
+
+        if should_skip_downstream?(locked)
+          locked.transition_to!(:skipped)
+          Coordinator.dispatch_callbacks_once(locked, :skipped)
+          skipped_downstream_ids = locked.downstream_pipelines.pluck(:id)
+        elsif all_upstreams_succeeded?(locked)
+          start_pipeline(locked)
+        end
+      end
+
+      skipped_downstream_ids&.each { |downstream_id| try_start_downstream(downstream_id) }
+    end
+
+    def self.should_skip_downstream?(pipeline)
+      pipeline.upstream_pipelines.any? do |upstream|
+        upstream.failed? || upstream.halted? || upstream.skipped?
+      end
+    end
+
+    def self.all_upstreams_succeeded?(pipeline)
+      pipeline.upstream_pipelines.all?(&:succeeded?)
+    end
+
+    def self.start_pipeline(pipeline_record)
+      pipeline_record.transition_to!(:running)
+
+      root_step_ids = pipeline_record.steps.where.missing(:upstream_dependencies).pluck(:id)
+
+      root_step_ids.each do |step_id|
+        Coordinator.try_enqueue_step(step_id)
+      end
+    end
+
+    private_class_method :try_start_downstream, :all_upstreams_succeeded?,
+                         :should_skip_downstream?, :start_pipeline
+  end
+end

data/lib/good_pipeline/coordinator.rb

@@ -0,0 +1,176 @@
+# frozen_string_literal: true
+
+module GoodPipeline
+  class Coordinator # rubocop:disable Metrics/ClassLength
+    def self.complete_step(step, succeeded:)
+      return if step.terminal_coordination_status?
+
+      record_step_outcome(step, succeeded)
+      propagate_halt(step) if !succeeded && step.pipeline.halt?
+      unblock_downstream_steps(step)
+      recompute_pipeline_status(step.pipeline.reload)
+    end
+
+    def self.record_step_outcome(step, succeeded)
+      StepRecord.transaction do
+        if succeeded
+          step.transition_coordination_status_to!(:succeeded)
+        else
+          record_step_failure(step)
+        end
+      end
+    end
+
+    def self.record_step_failure(step)
+      metadata = FailureMetadata.extract(step)
+      step.transition_coordination_status_to!(:failed)
+      step.update_columns(
+        error_class: metadata.error_class,
+        error_message: metadata.error_message,
+        attempts: metadata.attempts
+      )
+    end
+
+    def self.propagate_halt(step)
+      pipeline = step.pipeline
+      StepRecord.transaction do
+        pipeline.update_column(:halt_triggered, true)
+        skip_all_pending_steps(pipeline, except_dependents_of: step)
+      end
+    end
+
+    def self.unblock_downstream_steps(step)
+      step.downstream_steps.each do |downstream_step|
+        try_enqueue_step(downstream_step.id)
+      end
+    end
+
+    def self.try_enqueue_step(step_id)
+      skipped_downstream_ids = nil
+
+      StepRecord.transaction do
+        locked_step = StepRecord.lock("FOR UPDATE SKIP LOCKED").find_by(id: step_id)
+        return unless locked_step&.pending?
+        return if locked_step.good_job_id.present?
+
+        skipped_downstream_ids = resolve_step(locked_step)
+      end
+
+      skipped_downstream_ids&.each { |downstream_step_id| try_enqueue_step(downstream_step_id) }
+    end
+
+    def self.resolve_step(locked_step)
+      if should_skip?(locked_step)
+        locked_step.transition_coordination_status_to!(:skipped)
+        locked_step.downstream_steps.pluck(:id)
+      else
+        enqueue_user_job(locked_step) if all_upstreams_satisfied?(locked_step)
+        nil
+      end
+    end
+
+    def self.enqueue_user_job(step)
+      step.transition_coordination_status_to!(:enqueued)
+
+      batch = build_step_batch(step)
+      batch.enqueue { enqueue_step_job(step) }
+      step.update_column(:good_job_batch_id, batch.id)
+    end
+
+    def self.build_step_batch(step)
+      batch = GoodJob::Batch.new
+      batch.on_finish = "GoodPipeline::StepFinishedJob"
+      batch.properties = { step_id: step.id }
+      batch
+    end
+
+    def self.enqueue_step_job(step)
+      job = build_job_instance(step)
+      enqueued_job = job.enqueue
+      step.update_column(:good_job_id, enqueued_job.provider_job_id || enqueued_job.job_id)
+    end
+
+    def self.build_job_instance(step)
+      job = step.job_class.constantize.new(**step.params.symbolize_keys)
+      job.queue_name = step.queue if step.queue.present?
+      job.priority = step.priority if step.priority.present?
+      job
+    end
+
+    def self.recompute_pipeline_status(pipeline)
+      steps = pipeline.steps.reload
+
+      return if steps.any? { |step| step.pending? || step.enqueued? }
+      return if pipeline.terminal?
+
+      new_status = derive_terminal_status(steps, pipeline)
+      pipeline.transition_to!(new_status)
+      dispatch_callbacks_once(pipeline, new_status)
+      ChainCoordinator.propagate_terminal_state(pipeline)
+    end
+
+    def self.derive_terminal_status(steps, pipeline)
+      has_failures = steps.any?(&:failed?)
+
+      return :succeeded unless has_failures
+      return :halted if pipeline.halt_triggered?
+
+      :failed
+    end
+
+    def self.dispatch_callbacks_once(pipeline, new_status)
+      PipelineRecord.transaction do
+        locked = PipelineRecord.lock("FOR UPDATE").find(pipeline.id)
+        return if locked.callbacks_dispatched_at.present?
+
+        locked.update!(callbacks_dispatched_at: Time.current)
+        PipelineCallbackJob.perform_later(locked.id, new_status.to_s)
+      end
+    end
+
+    # --- Private helpers ---
+
+    def self.all_upstreams_satisfied?(step)
+      step.upstream_steps.all? do |upstream|
+        upstream.succeeded? ||
+          (upstream.failed? && effective_strategy(upstream) == :ignore)
+      end
+    end
+
+    def self.should_skip?(step)
+      step.pending? &&
+        step.upstream_steps.any? { |upstream| permanently_unsatisfied?(upstream) }
+    end
+
+    def self.permanently_unsatisfied?(upstream)
+      upstream.terminal_coordination_status? &&
+        !upstream.succeeded? &&
+        effective_strategy(upstream) != :ignore
+    end
+
+    def self.skip_all_pending_steps(pipeline, except_dependents_of:)
+      exempt_step_ids = if effective_strategy(except_dependents_of) == :ignore
+                          except_dependents_of.downstream_steps.pluck(:id)
+                        else
+                          []
+                        end
+
+      pipeline.steps.pending.find_each do |pending_step|
+        next if exempt_step_ids.include?(pending_step.id)
+
+        pending_step.transition_coordination_status_to!(:skipped)
+      end
+    end
+
+    def self.effective_strategy(step)
+      step.on_failure_strategy&.to_sym || step.pipeline.on_failure_strategy.to_sym
+    end
+
+    private_class_method :record_step_outcome, :record_step_failure,
+                         :propagate_halt, :unblock_downstream_steps,
+                         :all_upstreams_satisfied?, :should_skip?, :permanently_unsatisfied?,
+                         :skip_all_pending_steps, :effective_strategy,
+                         :enqueue_user_job, :build_step_batch, :enqueue_step_job,
+                         :build_job_instance, :derive_terminal_status, :resolve_step
+  end
+end

data/lib/good_pipeline/cycle_detector.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module GoodPipeline
+  class CycleDetector
+    def self.check!(steps, edges)
+      color = Hash.new(:white)
+      path = []
+
+      steps.each_key do |key|
+        next if color[key] == :black
+
+        dfs(key, edges, color, path)
+      end
+    end
+
+    def self.dfs(node, edges, color, path)
+      color[node] = :grey
+      path.push(node)
+
+      (edges[node] || []).each do |neighbor|
+        raise_cycle!(path, neighbor) if color[neighbor] == :grey
+        dfs(neighbor, edges, color, path) if color[neighbor] == :white
+      end
+
+      path.pop
+      color[node] = :black
+    end
+
+    def self.raise_cycle!(path, neighbor)
+      cycle = path.drop_while { |node| node != neighbor } + [neighbor]
+      raise InvalidPipelineError, "cycle detected: #{cycle.map { |key| ":#{key}" }.join(" -> ")}"
+    end
+
+    private_class_method :dfs, :raise_cycle!
+  end
+end

data/lib/good_pipeline/engine.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module GoodPipeline
+  class Engine < ::Rails::Engine
+    engine_name "good_pipeline"
+    isolate_namespace GoodPipeline
+
+    initializer "good_pipeline.check_good_job_config" do
+      ActiveSupport.on_load(:active_job) do
+        next if GoodJob.preserve_job_records == true
+
+        raise GoodPipeline::ConfigurationError, "GoodPipeline requires GoodJob.preserve_job_records = true"
+      end
+    end
+
+    initializer "good_pipeline.cleanup_hook" do
+      ActiveSupport::Notifications.subscribe("cleanup_preserved_jobs.good_job") do |event|
+        timestamp = event.payload[:timestamp]
+        GoodPipeline.cleanup_preserved_pipelines(older_than: timestamp) if timestamp
+      end
+    end
+  end
+end

data/lib/good_pipeline/errors.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module GoodPipeline
+  class Error < StandardError; end
+
+  class InvalidPipelineError < Error; end
+
+  class InvalidTransition < Error; end
+
+  class ConfigurationError < Error; end
+end

data/lib/good_pipeline/failure_metadata.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module GoodPipeline
+  class FailureMetadata
+    Result = Struct.new(:error_class, :error_message, :attempts)
+
+    def self.extract(step)
+      good_job = GoodJob::Job.find_by(id: step.good_job_id)
+      return Result.new(error_class: nil, error_message: nil, attempts: 0) unless good_job
+
+      error_class, error_message = parse_error(good_job.error)
+
+      Result.new(
+        error_class: error_class,
+        error_message: error_message,
+        attempts: good_job.executions_count
+      )
+    end
+
+    def self.parse_error(error_string)
+      return [nil, nil] if error_string.blank?
+
+      parts = error_string.split(GoodJob::Job::ERROR_MESSAGE_SEPARATOR, 2)
+      [parts[0]&.strip, parts[1]&.strip]
+    end
+
+    private_class_method :parse_error
+  end
+end

data/lib/good_pipeline/graph_validator.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module GoodPipeline
+  class GraphValidator
+    def self.validate!(step_definitions)
+      new(step_definitions).validate!
+    end
+
+    def initialize(step_definitions)
+      @step_definitions = step_definitions
+    end
+
+    def validate!
+      check_empty_pipeline!
+      check_duplicate_keys!
+      build_steps_by_key!
+      check_self_dependencies!
+      check_unknown_references!
+      check_cycles!
+    end
+
+    private
+
+    def check_empty_pipeline!
+      raise InvalidPipelineError, "pipeline has no steps" if @step_definitions.empty?
+    end
+
+    def check_duplicate_keys!
+      seen = {}
+      @step_definitions.each do |step|
+        raise InvalidPipelineError, "duplicate step key :#{step.key}" if seen.key?(step.key)
+
+        seen[step.key] = true
+      end
+    end
+
+    def build_steps_by_key!
+      @steps_by_key = @step_definitions.to_h { |step| [step.key, step] }
+    end
+
+    def check_self_dependencies!
+      @steps_by_key.each_value do |step|
+        raise InvalidPipelineError, "step :#{step.key} depends on itself" if step.dependencies.include?(step.key)
+      end
+    end
+
+    def check_unknown_references!
+      @steps_by_key.each_value do |step|
+        step.dependencies.each do |dependency_key|
+          unless @steps_by_key.key?(dependency_key)
+            raise InvalidPipelineError, "step :#{step.key} references unknown dependency :#{dependency_key}"
+          end
+        end
+      end
+    end
+
+    def check_cycles!
+      CycleDetector.check!(@steps_by_key, build_forward_edges)
+    end
+
+    def build_forward_edges
+      edges = Hash.new { |h, k| h[k] = [] }
+      @steps_by_key.each_value do |step|
+        step.dependencies.each do |dependency_key|
+          edges[dependency_key] << step.key
+        end
+      end
+      edges
+    end
+  end
+end