good_pipeline 0.2.2 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e52af813bd86f0de3d905a3f591209e0a155e0b5c28e3a76f963f1c2cdbd8232
-  data.tar.gz: 3a48213db593949e7f9afff0f341cf29dfdb90d4972b20b4c5e48091cb86bab1
+  metadata.gz: 350a7e051f704db8ee906a90bb8f641f1373be815aceb9adccc8cd17b2d38640
+  data.tar.gz: f0cebfb8f77d35e043df87e71e5698b4e6ec08a468d95702a5a3a489eb1debf9
 SHA512:
-  metadata.gz: 7bc3d7e1181b852bbc795f1e65ef9b2764645ffc13d7d860f444074158c741202ddae188253c50cb12a44da9958fe6403b2435fef6ec71d9172a35b46816da55
-  data.tar.gz: 22b59ba7dba841a6d55559c27a7427032bad94ed1fd3e336f2d53997ee4a24bf2f8c193d77e743095e4c5c73b948975b0f85228d63839eba367af1dc441f8456
+  metadata.gz: 682725126bc0643cd8ec88d249ef7e93b760d646a4aac30cbb693c1bde6d1c0b3d30494623eb5ffed8cc2845f62dc3642501648bf65cbf3adc43577f13734270
+  data.tar.gz: 5e2aae2df9e5b8997b85a4aff15d0989fa24f27b6e607332d82421c5e5b33d0a4171e455ddc04eab71f790657fe9f71d59b412b7ffa054b7f15c60148b1a1536

data/CHANGELOG.md

@@ -1,5 +1,34 @@
 ## [Unreleased]
 
+## [0.3.0] - 2026-03-25
+
+### Performance
+
+- **Bulk insert steps and dependencies** — `Runner` uses `insert_all!` with `RETURNING` for steps and dependencies instead of individual `create!` calls, reducing pipeline creation from N+M queries to 2.
+- **Pre-generated pipeline UUID** — `Runner` generates the pipeline UUID upfront, folding batch ID and initial status into a single INSERT instead of separate UPDATEs.
+- **Atomic upstream counter** — new `pending_upstream_count` column on steps tracks how many upstreams remain. `unblock_downstream_steps` atomically decrements via `UPDATE ... RETURNING` and only calls `try_enqueue_step` when the count reaches zero, eliminating O(N) wasted lock acquisitions for fan-in and diamond topologies.
+- **Merged UPDATE round-trips** — `enqueue_user_job` folds status transition, batch ID, and job ID into one `update_columns`. `record_step_failure` merges status and error metadata into one `update_columns`.
+- **Removed redundant transaction** — `record_step_outcome` no longer wraps a single `update_columns` in an explicit transaction.
+- **`update_columns` in transition methods** — `transition_coordination_status_to!` and `transition_to!` use `update_columns` instead of `update!`, skipping AR dirty tracking overhead.
+- **SQL EXISTS for status checks** — `recompute_pipeline_status` and `derive_terminal_status` use `EXISTS` queries instead of loading all step records.
+- **Pipeline load with EXISTS** — `load_pipeline_with_active_check` combines pipeline load with active-step and downstream-chain EXISTS checks in a single query.
+- **Conditional callback dispatch** — `dispatch_callbacks_once` uses `UPDATE WHERE callbacks_dispatched_at IS NULL` instead of `SELECT FOR UPDATE` + `UPDATE`.
+- **Early return on active pipeline** — `complete_step` skips pipeline status recomputation when `unblock_downstream_steps` enqueued any downstream step.
+- **Bulk skip on halt** — `skip_all_pending_steps` uses `update_all` instead of iterating with individual updates.
+- **Single-pass graph validation** — `GraphValidator` merges duplicate-key check, self-dependency check, steps-by-key index, and forward-edges construction into one O(n) pass and returns `steps_by_key` for reuse by `Pipeline`.
+- **Frozen constant defaults** — `EMPTY_HASH` and `EMPTY_ARRAY` shared constants avoid allocating fresh empty containers on every `StepDefinition` and `Pipeline#run` call.
+- **Fast-path shortcuts** — `validate_enqueue_options!` returns immediately for empty options. `expand_branch_aliases` skips `flat_map` when no branches are defined.
+
+### Added
+
+- **Benchmarking scripts** — `bench/memory_bench.rb` (in-memory, no DB) and `bench/database_bench.rb` (PostgreSQL) with `--json` flag for structured output. Covers pipeline construction, graph validation, cycle detection, step enqueue, step completion, status recomputation, halt propagation, and full pipeline run across linear, fan-out, fan-in, and diamond topologies.
+- **`pending_upstream_count` column** — integer column on steps table, set by `Runner` at creation time, decremented atomically by `Coordinator` on step completion.
+
+### Changed
+
+- **`Runner` refactored** — `call` method extracted into `create_pipeline_batch`, `create_pipeline_record`, `insert_steps`, `insert_dependencies`, and `enqueue_root_steps` for readability. Pipeline record is a local variable passed to methods instead of an instance variable.
+- **`Coordinator` method reordering** — private methods grouped by concern (outcome recording, downstream unblocking, step resolution, pipeline status) rather than call order.
+
 ## [0.2.2] - 2026-03-24
 
 ### Fixed

data/app/models/good_pipeline/pipeline_record.rb

@@ -1,6 +1,9 @@
 # frozen_string_literal: true
 
 module GoodPipeline
+  # This model intentionally has no AR callbacks or validations. Status transitions
+  # use update_columns throughout the coordinator layer. If you need lifecycle hooks,
+  # ensure all update_columns call sites are updated accordingly.
   class PipelineRecord < ActiveRecord::Base
     self.table_name = "good_pipeline_pipelines"
     self.inheritance_column = nil
@@ -67,7 +70,7 @@ module GoodPipeline
         raise InvalidTransition, "cannot transition pipeline from '#{status}' to '#{new_status}'"
       end
 
-      update!(status: new_status)
+      update_columns(status: new_status, updated_at: Time.current)
     end
   end
 end

data/app/models/good_pipeline/step_record.rb

@@ -1,6 +1,9 @@
 # frozen_string_literal: true
 
 module GoodPipeline
+  # This model intentionally has no AR callbacks or validations. Status transitions
+  # use update_columns throughout the coordinator layer. If you need lifecycle hooks,
+  # ensure all update_columns call sites are updated accordingly.
   class StepRecord < ActiveRecord::Base
     self.table_name = "good_pipeline_steps"
 
@@ -74,7 +77,7 @@ module GoodPipeline
               "cannot transition step '#{key}' coordination_status from '#{coordination_status}' to '#{new_status}'"
       end
 
-      update!(coordination_status: new_status)
+      update_columns(coordination_status: new_status, updated_at: Time.current)
     end
   end
 end

data/demo/db/migrate/20260319205326_create_good_pipeline_tables.rb

@@ -30,6 +30,7 @@ class CreateGoodPipelineTables < ActiveRecord::Migration[8.1]
       t.jsonb :branch, null: false, default: {}
       t.uuid :good_job_batch_id
       t.uuid :good_job_id
+      t.integer :pending_upstream_count, null: false, default: 0
       t.integer :attempts
       t.string :error_class
       t.text :error_message

data/demo/test/test_helper.rb

@@ -65,6 +65,7 @@ module ActiveSupport
       dependencies.each do |dependency_step|
         GoodPipeline::DependencyRecord.create!(pipeline: pipeline, step: step, depends_on_step: dependency_step)
       end
+      step.update_column(:pending_upstream_count, dependencies.size)
       step
     end
   end

data/lib/generators/good_pipeline/install/templates/create_good_pipeline_tables.rb.erb

@@ -28,6 +28,7 @@ class CreateGoodPipelineTables < ActiveRecord::Migration[<%= ActiveRecord::Migra
       t.jsonb :branch, null: false, default: {}
       t.uuid :good_job_batch_id
       t.uuid :good_job_id
+      t.integer :pending_upstream_count, null: false, default: 0
       t.integer :attempts
       t.string :error_class
       t.text :error_message

data/lib/good_pipeline/constants.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module GoodPipeline
+  EMPTY_HASH  = {}.freeze
+  EMPTY_ARRAY = [].freeze
+end

data/lib/good_pipeline/coordinator.rb

@@ -8,20 +8,29 @@ module GoodPipeline
 
         record_step_outcome(step, succeeded)
         propagate_halt(step) if !succeeded && step.pipeline.halt?
-        unblock_downstream_steps(step)
-        recompute_pipeline_status(step.pipeline.reload)
+        return if unblock_downstream_steps(step)
+
+        pipeline = load_pipeline_with_active_check(step.pipeline_id)
+
+        recompute_pipeline_status(
+          pipeline,
+          has_active_steps: pipeline["has_active_steps"],
+          has_downstream_chains: pipeline["has_downstream_chains"]
+        )
       end
 
       def try_enqueue_step(step_id) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+        step_was_enqueued = false
         skipped_downstream_ids = nil
         recompute_pipeline = nil
 
         StepRecord.transaction do
           locked_step = StepRecord.lock("FOR UPDATE").find_by(id: step_id)
-          return unless locked_step&.pending?
-          return if locked_step.good_job_id.present?
+          return false unless locked_step&.pending?
+          return false if locked_step.good_job_id.present?
 
           skipped_downstream_ids = resolve_step(locked_step)
+          step_was_enqueued = skipped_downstream_ids.nil?
         rescue ConfigurationError => error
           fail_step_with_error(locked_step, error)
           propagate_halt(locked_step) if locked_step.pipeline.halt?
@@ -29,48 +38,55 @@ module GoodPipeline
           recompute_pipeline = locked_step.pipeline
         end
 
-        skipped_downstream_ids&.each { |downstream_step_id| try_enqueue_step(downstream_step_id) }
+        downstream_enqueued = false
+        skipped_downstream_ids&.each { |id| downstream_enqueued = true if try_enqueue_step(id) }
         recompute_pipeline_status(recompute_pipeline.reload) if recompute_pipeline
+        step_was_enqueued || downstream_enqueued
       end
 
-      def recompute_pipeline_status(pipeline)
-        steps = pipeline.steps.reload
-
-        return if steps.any? { |step| step.pending? || step.enqueued? }
+      def recompute_pipeline_status(pipeline, has_active_steps: nil, has_downstream_chains: nil) # rubocop:disable Metrics/MethodLength
         return if pipeline.terminal?
 
-        new_status = derive_terminal_status(steps, pipeline)
+        active = if has_active_steps.nil?
+                   pipeline.steps.where(coordination_status: %w[pending enqueued]).exists?
+                 else
+                   has_active_steps
+                 end
+
+        return if active
+
+        new_status = derive_terminal_status(pipeline)
         pipeline.transition_to!(new_status)
         dispatch_callbacks_once(pipeline, new_status)
-        ChainCoordinator.propagate_terminal_state(pipeline)
+        ChainCoordinator.propagate_terminal_state(pipeline) unless has_downstream_chains == false
       end
 
       def dispatch_callbacks_once(pipeline, new_status)
         PipelineRecord.transaction do
-          locked = PipelineRecord.lock("FOR UPDATE").find(pipeline.id)
-          return if locked.callbacks_dispatched_at.present?
+          rows_updated = PipelineRecord.where(id: pipeline.id, callbacks_dispatched_at: nil)
+                                       .update_all(callbacks_dispatched_at: Time.current)
+
+          return if rows_updated.zero?
 
-          locked.update!(callbacks_dispatched_at: Time.current)
-          PipelineCallbackJob.perform_later(locked.id, new_status.to_s)
+          PipelineCallbackJob.perform_later(pipeline.id, new_status.to_s)
         end
       end
 
       private
 
       def record_step_outcome(step, succeeded)
-        StepRecord.transaction do
-          if succeeded
-            step.transition_coordination_status_to!(:succeeded)
-          else
-            record_step_failure(step)
-          end
+        if succeeded
+          step.transition_coordination_status_to!(:succeeded)
+        else
+          record_step_failure(step)
         end
       end
 
       def record_step_failure(step)
         metadata = FailureMetadata.extract(step)
-        step.transition_coordination_status_to!(:failed)
         step.update_columns(
+          coordination_status: "failed",
+          updated_at: Time.current,
           error_class: metadata.error_class,
           error_message: metadata.error_message,
           attempts: metadata.attempts
@@ -78,28 +94,71 @@ module GoodPipeline
       end
 
       def propagate_halt(step)
-        pipeline = step.pipeline
         StepRecord.transaction do
-          pipeline.update_column(:halt_triggered, true)
-          skip_all_pending_steps(pipeline, except_dependents_of: step)
+          step.pipeline.update_column(:halt_triggered, true)
+          skip_all_pending_steps(step.pipeline, except_dependents_of: step)
+        end
+      end
+
+      def skip_all_pending_steps(pipeline, except_dependents_of:)
+        scope = pipeline.steps.pending
+
+        if effective_failure_strategy(except_dependents_of) == :ignore
+          exempt_ids = transitive_downstream_ids(except_dependents_of)
+          scope = scope.where.not(id: exempt_ids.to_a) if exempt_ids.any?
         end
+
+        scope.update_all(coordination_status: "skipped")
       end
 
       def unblock_downstream_steps(step)
-        step.downstream_steps.each do |downstream_step|
-          try_enqueue_step(downstream_step.id)
+        sql = <<~SQL
+          UPDATE good_pipeline_steps
+             SET pending_upstream_count = pending_upstream_count - 1
+          WHERE id IN (
+            SELECT step_id FROM good_pipeline_dependencies
+             WHERE depends_on_step_id = $1
+          )
+            AND coordination_status = 'pending'
+          RETURNING id, pending_upstream_count
+        SQL
+
+        any_enqueued = false
+        StepRecord.connection.exec_query(sql, "SQL", [step.id]).each do |row|
+          any_enqueued = true if row["pending_upstream_count"].zero? && try_enqueue_step(row["id"])
         end
+        any_enqueued
       end
 
-      def resolve_step(locked_step) # rubocop:disable Metrics/MethodLength
+      def load_pipeline_with_active_check(pipeline_id)
+        sql = <<~SQL.squish
+          good_pipeline_pipelines.*,
+          EXISTS(
+            SELECT 1 FROM good_pipeline_steps
+             WHERE pipeline_id = good_pipeline_pipelines.id
+               AND coordination_status IN ('pending', 'enqueued')
+          ) AS has_active_steps,
+          EXISTS(
+            SELECT 1 FROM good_pipeline_chains
+             WHERE upstream_pipeline_id = good_pipeline_pipelines.id
+          ) AS has_downstream_chains
+        SQL
+
+        PipelineRecord.select(sql).where(id: pipeline_id).first!
+      end
+
+      def resolve_step(locked_step) # rubocop:disable Metrics/MethodLength,Metrics/AbcSize
         if should_skip?(locked_step)
           locked_step.transition_coordination_status_to!(:skipped)
+          decrement_upstream_counts_for_terminal_step(locked_step.id)
           locked_step.downstream_steps.pluck(:id)
         elsif locked_step.branch_step? && all_upstreams_satisfied?(locked_step)
           BranchResolver.resolve(locked_step)
+          decrement_upstream_counts_for_terminal_step(locked_step.id)
           locked_step.downstream_steps.pluck(:id)
         elsif BranchResolver.skipped_by_branch?(locked_step)
           locked_step.transition_coordination_status_to!(:skipped_by_branch)
+          decrement_upstream_counts_for_terminal_step(locked_step.id)
           locked_step.downstream_steps.pluck(:id)
         else
           enqueue_user_job(locked_step) if all_upstreams_satisfied?(locked_step)
@@ -107,12 +166,41 @@ module GoodPipeline
         end
       end
 
-      def enqueue_user_job(step)
-        step.transition_coordination_status_to!(:enqueued)
+      def should_skip?(step)
+        step.pending? && step.upstream_steps.any? { |upstream| permanently_unsatisfied?(upstream) }
+      end
+
+      def permanently_unsatisfied?(upstream)
+        upstream.terminal_coordination_status? &&
+          !upstream.succeeded? &&
+          !upstream.skipped_by_branch? &&
+          effective_failure_strategy(upstream) != :ignore
+      end
+
+      def decrement_upstream_counts_for_terminal_step(step_id)
+        downstream_ids = DependencyRecord.where(depends_on_step_id: step_id).select(:step_id)
+        StepRecord.where(id: downstream_ids, coordination_status: "pending")
+                  .update_all("pending_upstream_count = pending_upstream_count - 1")
+      end
+
+      def all_upstreams_satisfied?(step)
+        step.upstream_steps.all? do |upstream|
+          upstream.succeeded? ||
+            upstream.skipped_by_branch? ||
+            (upstream.failed? && effective_failure_strategy(upstream) == :ignore)
+        end
+      end
 
+      def enqueue_user_job(step)
         batch = build_step_batch(step)
-        batch.enqueue { enqueue_step_job(step) }
-        step.update_column(:good_job_batch_id, batch.id)
+        good_job_id = nil
+        batch.enqueue { good_job_id = enqueue_step_job(step) }
+        step.update_columns(
+          coordination_status: "enqueued",
+          good_job_batch_id: batch.id,
+          good_job_id: good_job_id,
+          updated_at: Time.current
+        )
       end
 
       def build_step_batch(step)
@@ -125,11 +213,11 @@ module GoodPipeline
       def enqueue_step_job(step)
         job = step.job_class.constantize.new(**step.params.symbolize_keys)
         enqueued_job = job.enqueue(**step.enqueue_options.symbolize_keys)
-        step.update_column(:good_job_id, enqueued_job.provider_job_id || enqueued_job.job_id)
+        enqueued_job.provider_job_id || enqueued_job.job_id
       end
 
-      def derive_terminal_status(steps, pipeline)
-        has_failures = steps.any?(&:failed?)
+      def derive_terminal_status(pipeline)
+        has_failures = pipeline.steps.where(coordination_status: "failed").exists?
 
         return :succeeded unless has_failures
         return :halted if pipeline.halt_triggered?
@@ -137,40 +225,6 @@ module GoodPipeline
         :failed
       end
 
-      def all_upstreams_satisfied?(step)
-        step.upstream_steps.all? do |upstream|
-          upstream.succeeded? ||
-            upstream.skipped_by_branch? ||
-            (upstream.failed? && effective_failure_strategy(upstream) == :ignore)
-        end
-      end
-
-      def should_skip?(step)
-        step.pending? &&
-          step.upstream_steps.any? { |upstream| permanently_unsatisfied?(upstream) }
-      end
-
-      def permanently_unsatisfied?(upstream)
-        upstream.terminal_coordination_status? &&
-          !upstream.succeeded? &&
-          !upstream.skipped_by_branch? &&
-          effective_failure_strategy(upstream) != :ignore
-      end
-
-      def skip_all_pending_steps(pipeline, except_dependents_of:)
-        exempt_step_ids = if effective_failure_strategy(except_dependents_of) == :ignore
-                            transitive_downstream_ids(except_dependents_of)
-                          else
-                            Set.new
-                          end
-
-        pipeline.steps.pending.find_each do |pending_step|
-          next if exempt_step_ids.include?(pending_step.id)
-
-          pending_step.transition_coordination_status_to!(:skipped)
-        end
-      end
-
       def transitive_downstream_ids(step)
         visited = Set.new
         queue = step.downstream_steps.pluck(:id)

data/lib/good_pipeline/graph_validator.rb

@@ -12,11 +12,10 @@ module GoodPipeline
 
     def validate!
       check_empty_pipeline!
-      check_duplicate_keys!
-      build_steps_by_key!
-      check_self_dependencies!
+      build_index!
       check_unknown_references!
       check_cycles!
+      @steps_by_key
     end
 
     private
@@ -25,22 +24,20 @@ module GoodPipeline
       raise InvalidPipelineError, "pipeline has no steps" if @step_definitions.empty?
     end
 
-    def check_duplicate_keys!
-      seen = {}
+    def build_index! # rubocop:disable Metrics/AbcSize
+      @steps_by_key = {}
+      @forward_edges = Hash.new { |h, k| h[k] = [] }
+
       @step_definitions.each do |step|
-        raise InvalidPipelineError, "duplicate step key :#{step.key}" if seen.key?(step.key)
+        raise InvalidPipelineError, "duplicate step key :#{step.key}" if @steps_by_key.key?(step.key)
 
-        seen[step.key] = true
-      end
-    end
+        step.dependencies.each do |dependency_key|
+          raise InvalidPipelineError, "step :#{step.key} depends on itself" if dependency_key == step.key
 
-    def build_steps_by_key!
-      @steps_by_key = @step_definitions.to_h { |step| [step.key, step] }
-    end
+          @forward_edges[dependency_key] << step.key
+        end
 
-    def check_self_dependencies!
-      @steps_by_key.each_value do |step|
-        raise InvalidPipelineError, "step :#{step.key} depends on itself" if step.dependencies.include?(step.key)
+        @steps_by_key[step.key] = step
       end
     end
 
@@ -55,17 +52,7 @@ module GoodPipeline
     end
 
     def check_cycles!
-      CycleDetector.check!(@steps_by_key, build_forward_edges)
-    end
-
-    def build_forward_edges
-      edges = Hash.new { |h, k| h[k] = [] }
-      @steps_by_key.each_value do |step|
-        step.dependencies.each do |dependency_key|
-          edges[dependency_key] << step.key
-        end
-      end
-      edges
+      CycleDetector.check!(@steps_by_key, @forward_edges)
     end
   end
 end

data/lib/good_pipeline/pipeline.rb

@@ -103,11 +103,10 @@ module GoodPipeline
       @branch_context_stack = []
       @building = true
       configure(**kwargs)
-      GraphValidator.validate!(@step_definitions)
+      @steps_by_key = GraphValidator.validate!(@step_definitions).freeze
       @step_definitions.freeze
       @branch_aliases.freeze
       @building = false
-      @steps_by_key = @step_definitions.to_h { |step| [step.key, step] }.freeze
       @root_steps = @step_definitions.select { |step| step.dependencies.empty? }.freeze
       freeze
     end
@@ -118,7 +117,7 @@ module GoodPipeline
       raise NotImplementedError, "#{self.class} must implement #configure"
     end
 
-    def run(key, job_class, with: {}, after: [], on_failure: nil, enqueue: {}) # rubocop:disable Metrics/MethodLength
+    def run(key, job_class, with: EMPTY_HASH, after: EMPTY_ARRAY, on_failure: nil, enqueue: EMPTY_HASH) # rubocop:disable Metrics/MethodLength
       raise ConfigurationError, "run can only be called inside configure" unless @building
 
       expanded_after = expand_branch_aliases(after)
@@ -186,6 +185,8 @@ module GoodPipeline
     # NOTE: Single-level expansion only. If nested branches are added in the future,
     # this must become recursive to expand inner branch aliases.
     def expand_branch_aliases(dependencies)
+      return Array(dependencies) if @branch_aliases.empty?
+
       Array(dependencies).flat_map { |dependency| @branch_aliases.fetch(dependency, [dependency]) }
     end
   end

data/lib/good_pipeline/runner.rb

@@ -10,61 +10,81 @@ module GoodPipeline
       @pipeline = pipeline_instance
     end
 
-    def call(start: true) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength
+    def call(start: true) # rubocop:disable Metrics/MethodLength
+      pipeline_id = SecureRandom.uuid
       pipeline_record = nil
-      step_records = {}
-
-      PipelineRecord.transaction do # rubocop:disable Metrics/BlockLength
-        pipeline_record = PipelineRecord.create!(
-          type: @pipeline.class.name,
-          params: @pipeline.params,
-          status: :pending,
-          on_failure_strategy: @pipeline.failure_strategy.to_s
-        )
-
-        # Two passes: create all step records first, then dependencies.
-        # Branch steps may appear after their dependents in step_definitions.
-        @pipeline.step_definitions.each do |step_definition|
-          step_records[step_definition.key] = StepRecord.create!(
-            pipeline: pipeline_record,
-            key: step_definition.key.to_s,
-            job_class: resolve_job_class(step_definition),
-            params: step_definition.params,
-            on_failure_strategy: step_definition.failure_strategy&.to_s,
-            enqueue_options: step_definition.enqueue_options,
-            branch: build_branch_hash(step_definition)
-          )
-        end
+      step_id_by_key = {}
 
-        @pipeline.step_definitions.each do |step_definition| # rubocop:disable Style/CombinableLoops
-          step_definition.dependencies.each do |dependency_key|
-            DependencyRecord.create!(
-              pipeline: pipeline_record,
-              step: step_records[step_definition.key],
-              depends_on_step: step_records[dependency_key]
-            )
-          end
-        end
+      PipelineRecord.transaction do
+        batch = create_pipeline_batch(pipeline_id)
+        pipeline_record = create_pipeline_record(pipeline_id, batch.id, start: start)
+        step_id_by_key = insert_steps(pipeline_record)
+        insert_dependencies(pipeline_record, step_id_by_key)
+      end
+
+      enqueue_root_steps(step_id_by_key) if start
+
+      pipeline_record
+    end
+
+    private
 
-        pipeline_batch = GoodJob::Batch.new
-        pipeline_batch.on_finish = "GoodPipeline::PipelineReconciliationJob"
-        pipeline_batch.properties = { pipeline_id: pipeline_record.id }
-        pipeline_batch.save
-        pipeline_record.update_column(:good_job_batch_id, pipeline_batch.id)
+    def create_pipeline_batch(pipeline_id)
+      batch = GoodJob::Batch.new
+      batch.on_finish = "GoodPipeline::PipelineReconciliationJob"
+      batch.properties = { pipeline_id: pipeline_id }
+      batch.save
+      batch
+    end
+
+    def create_pipeline_record(pipeline_id, batch_id, start:)
+      PipelineRecord.create!(
+        id: pipeline_id,
+        type: @pipeline.class.name,
+        params: @pipeline.params,
+        status: start ? :running : :pending,
+        on_failure_strategy: @pipeline.failure_strategy.to_s,
+        good_job_batch_id: batch_id
+      )
+    end
 
-        pipeline_record.transition_to!(:running) if start
+    def insert_steps(pipeline_record) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+      step_rows = @pipeline.step_definitions.map do |step_definition|
+        {
+          pipeline_id: pipeline_record.id,
+          key: step_definition.key.to_s,
+          job_class: resolve_job_class(step_definition),
+          params: step_definition.params,
+          on_failure_strategy: step_definition.failure_strategy&.to_s,
+          enqueue_options: step_definition.enqueue_options,
+          branch: build_branch_hash(step_definition),
+          pending_upstream_count: step_definition.dependencies.size
+        }
       end
 
-      if start
-        @pipeline.root_steps.each do |step_definition|
-          Coordinator.try_enqueue_step(step_records[step_definition.key].id)
+      result = StepRecord.insert_all!(step_rows, returning: %w[id key])
+      result.rows.each_with_object({}) { |(id, key), hash| hash[key.to_sym] = id }
+    end
+
+    def insert_dependencies(pipeline_record, step_id_by_key)
+      dependency_rows = @pipeline.step_definitions.flat_map do |step_definition|
+        step_definition.dependencies.map do |dependency_key|
+          {
+            pipeline_id: pipeline_record.id,
+            step_id: step_id_by_key[step_definition.key],
+            depends_on_step_id: step_id_by_key[dependency_key]
+          }
         end
       end
 
-      pipeline_record
+      DependencyRecord.insert_all!(dependency_rows) if dependency_rows.any?
     end
 
-    private
+    def enqueue_root_steps(step_id_by_key)
+      @pipeline.root_steps.each do |step_definition|
+        Coordinator.try_enqueue_step(step_id_by_key[step_definition.key])
+      end
+    end
 
     def resolve_job_class(step_definition)
       step_definition.job_class.is_a?(String) ? step_definition.job_class : step_definition.job_class.name

data/lib/good_pipeline/step_definition.rb

@@ -4,11 +4,29 @@ module GoodPipeline
   class StepDefinition
     SUPPORTED_ENQUEUE_OPTIONS = %i[queue priority wait good_job_labels good_job_notify].freeze
 
-    attr_reader :key, :job_class, :params, :dependencies, :failure_strategy, :enqueue_options,
-                :branch_key, :branch_arm, :decides, :empty_arms
+    attr_reader :key,
+                :job_class,
+                :params,
+                :dependencies,
+                :failure_strategy,
+                :enqueue_options,
+                :branch_key,
+                :branch_arm,
+                :decides,
+                :empty_arms
 
-    def initialize(key:, job_class:, params: {}, dependencies: [], failure_strategy: nil, enqueue_options: {}, # rubocop:disable Metrics/MethodLength
-                   branch_key: nil, branch_arm: nil, decides: nil, empty_arms: [])
+    def initialize( # rubocop:disable Metrics/MethodLength
+      key:,
+      job_class:,
+      params: EMPTY_HASH,
+      dependencies: EMPTY_ARRAY,
+      failure_strategy: nil,
+      enqueue_options: EMPTY_HASH,
+      branch_key: nil,
+      branch_arm: nil,
+      decides: nil,
+      empty_arms: EMPTY_ARRAY
+    )
       @key = key
       @job_class = job_class
       @params = params.freeze
@@ -37,6 +55,8 @@ module GoodPipeline
     end
 
     def validate_enqueue_options!(options)
+      return if options.empty?
+
       unsupported = options.keys.map(&:to_sym) - SUPPORTED_ENQUEUE_OPTIONS
       return if unsupported.empty?
 

data/lib/good_pipeline/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module GoodPipeline
-  VERSION = "0.2.2"
+  VERSION = "0.3.0"
 end

data/lib/good_pipeline.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative "good_pipeline/version"
+require_relative "good_pipeline/constants"
 require_relative "good_pipeline/errors"
 require_relative "good_pipeline/step_definition"
 require_relative "good_pipeline/branch_builder"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: good_pipeline
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.3.0
 platform: ruby
 authors:
 - Ali Hamdi Ali Fadel
@@ -177,6 +177,7 @@ files:
 - lib/good_pipeline/branch_resolver.rb
 - lib/good_pipeline/chain.rb
 - lib/good_pipeline/chain_coordinator.rb
+- lib/good_pipeline/constants.rb
 - lib/good_pipeline/coordinator.rb
 - lib/good_pipeline/cycle_detector.rb
 - lib/good_pipeline/engine.rb