ductwork 0.26.0 → 1.0.0

data/lib/generators/ductwork/install/templates/db/create_ductwork_branch_links.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+class CreateDuctworkBranchLinks < Ductwork::Migration
+  def change
+    create_ductwork_table :ductwork_branch_links do |table|
+      belongs_to(
+        table,
+        :parent_branch,
+        index: true,
+        null: false,
+        foreign_key: { to_table: :ductwork_branches }
+      )
+      belongs_to(
+        table,
+        :child_branch,
+        index: true,
+        null: false,
+        foreign_key: { to_table: :ductwork_branches }
+      )
+      table.timestamps null: false
+    end
+
+    add_index :ductwork_branch_links,
+              %w[parent_branch_id child_branch_id],
+              unique: true
+  end
+end

data/lib/generators/ductwork/install/templates/db/create_ductwork_branches.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+class CreateDuctworkBranches < Ductwork::Migration
+  def change
+    create_ductwork_table :ductwork_branches do |table|
+      belongs_to(
+        table,
+        :run,
+        index: true,
+        null: false,
+        foreign_key: { to_table: :ductwork_runs }
+      )
+      table.string :pipeline_klass, null: false
+      table.string :status, null: false
+      table.string :halt_reason
+      table.string :claim_token
+      table.datetime :claimed_for_advancing_at
+      table.datetime :last_advanced_at, null: false
+      table.datetime :started_at, null: false
+      table.datetime :completed_at
+      table.timestamps null: false
+    end
+
+    if mysql?
+      add_index :ductwork_branches,
+                %w[pipeline_klass status claimed_for_advancing_at last_advanced_at],
+                name: "index_ductwork_branches_on_claim_latest"
+    else
+      add_index :ductwork_branches,
+                %w[pipeline_klass status last_advanced_at],
+                where: "claimed_for_advancing_at IS NULL",
+                name: "index_ductwork_branches_on_claim_latest"
+    end
+    add_index :ductwork_branches, %w[run_id started_at]
+    add_index :ductwork_branches, %w[run_id status]
+  end
+end

data/lib/generators/ductwork/install/templates/db/create_ductwork_executions.rb

@@ -10,10 +10,17 @@ class CreateDuctworkExecutions < Ductwork::Migration
         null: false,
         foreign_key: { to_table: :ductwork_jobs }
       )
+      belongs_to(
+        table,
+        :process,
+        index: true,
+        null: true,
+        foreign_key: { to_table: :ductwork_processes }
+      )
       table.timestamp :started_at, null: false
       table.timestamp :completed_at
       table.integer :retry_count, null: false
-      table.integer :process_id
+      table.integer :crash_count, null: false
       table.timestamps null: false
     end
 

data/lib/generators/ductwork/install/templates/db/create_ductwork_pipelines.rb

@@ -4,14 +4,6 @@ class CreateDuctworkPipelines < Ductwork::Migration
   def change
     create_ductwork_table :ductwork_pipelines do |table|
       table.string :klass, null: false
-      table.text :definition, null: false
-      table.string :definition_sha1, null: false
-      table.timestamp :triggered_at, null: false
-      table.timestamp :started_at, null: false
-      table.timestamp :completed_at
-      table.timestamp :halted_at
-      table.timestamp :claimed_for_advancing_at
-      table.timestamp :last_advanced_at, null: false
       table.string :status, null: false
       table.timestamps null: false
     end

data/lib/generators/ductwork/install/templates/db/create_ductwork_processes.rb

@@ -5,10 +5,12 @@ class CreateDuctworkProcesses < Ductwork::Migration
     create_ductwork_table :ductwork_processes do |table|
       table.integer :pid, null: false
       table.string :machine_identifier, null: false
+      table.string :role, null: false
       table.timestamp :last_heartbeat_at, null: false
       table.timestamps null: false
     end
 
+    add_index :ductwork_processes, :last_heartbeat_at
     add_index :ductwork_processes, %i[pid machine_identifier], unique: true
   end
 end

data/lib/generators/ductwork/install/templates/db/create_ductwork_results.rb

@@ -18,5 +18,6 @@ class CreateDuctworkResults < Ductwork::Migration
     end
 
     add_index :ductwork_results, :execution_id, unique: true
+    add_index :ductwork_results, %i[result_type created_at]
   end
 end

data/lib/generators/ductwork/install/templates/db/create_ductwork_runs.rb

@@ -5,16 +5,55 @@ class CreateDuctworkRuns < Ductwork::Migration
     create_ductwork_table :ductwork_runs do |table|
       belongs_to(
         table,
-        :execution,
-        index: false,
+        :pipeline,
+        index: true,
         null: false,
-        foreign_key: { to_table: :ductwork_executions }
+        foreign_key: { to_table: :ductwork_pipelines }
       )
-      table.timestamp :started_at, null: false
-      table.timestamp :completed_at
+      table.string :pipeline_klass, null: false
+      table.text :definition, null: false
+      table.string :definition_sha1, null: false
+      table.string :status, null: false
+      table.datetime :triggered_at, null: false
+      table.datetime :started_at, null: false
+      table.datetime :completed_at
+      table.datetime :halted_at
+      table.datetime :on_halt_dispatched_at
       table.timestamps null: false
     end
 
-    add_index :ductwork_runs, :execution_id, unique: true
+    add_index :ductwork_runs, %i[pipeline_id status]
+    add_index :ductwork_runs, %i[pipeline_id started_at]
+    add_index :ductwork_runs, :started_at
+    add_index :ductwork_runs, %i[pipeline_klass started_at]
+    add_index :ductwork_runs, %i[status started_at]
+
+    if mysql?
+      reversible do |direction|
+        direction.up do
+          execute <<~SQL
+            ALTER TABLE ductwork_runs
+              ADD COLUMN active_pipeline_id VARCHAR(36)
+                GENERATED ALWAYS AS (
+                  IF(status IN ('in_progress', 'paused'), pipeline_id, NULL)
+                )
+          SQL
+          add_index :ductwork_runs, :active_pipeline_id,
+                    unique: true,
+                    name: :idx_unique_active_run
+        end
+
+        direction.down do
+          remove_index :ductwork_runs, name: :idx_unique_active_run
+          remove_column :ductwork_runs, :active_pipeline_id
+        end
+      end
+    else
+      add_index :ductwork_runs,
+                :pipeline_id,
+                unique: true,
+                where: "status IN ('in_progress', 'paused')",
+                name: :idx_unique_active_run
+    end
   end
 end

data/lib/generators/ductwork/install/templates/db/create_ductwork_steps.rb

@@ -5,10 +5,24 @@ class CreateDuctworkSteps < Ductwork::Migration
     create_ductwork_table :ductwork_steps do |table|
       belongs_to(
         table,
-        :pipeline,
+        :run,
         index: true,
         null: false,
-        foreign_key: { to_table: :ductwork_pipelines }
+        foreign_key: { to_table: :ductwork_runs }
+      )
+      belongs_to(
+        table,
+        :branch,
+        index: true,
+        null: false,
+        foreign_key: { to_table: :ductwork_branches }
+      )
+      belongs_to(
+        table,
+        :source_step,
+        index: true,
+        null: true,
+        foreign_key: { to_table: :ductwork_steps }
       )
       table.string :node, null: false
       table.string :klass, null: false
@@ -21,9 +35,10 @@ class CreateDuctworkSteps < Ductwork::Migration
       table.timestamps null: false
     end
 
-    add_index :ductwork_steps, %i[pipeline_id status node]
-    add_index :ductwork_steps, %i[pipeline_id node status]
+    add_index :ductwork_steps, %i[branch_id started_at]
+    add_index :ductwork_steps, %i[run_id node status]
+    add_index :ductwork_steps, %i[run_id status node]
+    add_index :ductwork_steps, %i[status branch_id]
     add_index :ductwork_steps, %i[status node]
-    add_index :ductwork_steps, %i[pipeline_id status]
   end
 end

data/lib/generators/ductwork/install/templates/db/create_ductwork_transitions.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+class CreateDuctworkTransitions < Ductwork::Migration
+  def change
+    create_ductwork_table :ductwork_transitions do |table|
+      belongs_to(
+        table,
+        :branch,
+        index: true,
+        null: false,
+        foreign_key: { to_table: :ductwork_branches }
+      )
+      belongs_to(
+        table,
+        :in_step,
+        index: true,
+        null: false,
+        foreign_key: { to_table: :ductwork_steps }
+      )
+      belongs_to(
+        table,
+        :out_step,
+        index: true,
+        null: true,
+        foreign_key: { to_table: :ductwork_steps }
+      )
+      table.datetime :started_at, null: false
+      table.datetime :completed_at
+      table.timestamps null: false
+    end
+
+    if mysql?
+      add_index :ductwork_transitions,
+                %i[branch_id completed_at started_at],
+                name: "index_ductwork_transitions_on_latest_open"
+    else
+      add_index :ductwork_transitions,
+                %i[branch_id started_at],
+                where: "completed_at IS NULL",
+                name: "index_ductwork_transitions_on_latest_open"
+    end
+  end
+end

data/lib/generators/ductwork/install/templates/db/create_ductwork_tuples.rb

@@ -5,10 +5,10 @@ class CreateDuctworkTuples < Ductwork::Migration
     create_ductwork_table :ductwork_tuples do |table|
       belongs_to(
         table,
-        :pipeline,
+        :run,
         index: true,
         null: false,
-        foreign_key: { to_table: :ductwork_pipelines }
+        foreign_key: { to_table: :ductwork_runs }
       )
       table.string :key, null: false
       table.string :serialized_value
@@ -17,6 +17,6 @@ class CreateDuctworkTuples < Ductwork::Migration
       table.timestamps null: false
     end
 
-    add_index :ductwork_tuples, %i[key pipeline_id], unique: true
+    add_index :ductwork_tuples, %i[run_id key], unique: true
   end
 end

data/lib/generators/ductwork/update/templates/db/add_crash_count_to_ductwork_advancements.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class AddCrashCountToDuctworkAdvancements < Ductwork::Migration
+  def change
+    add_column :ductwork_advancements, :crash_count, :integer
+  end
+end

data/lib/generators/ductwork/update/templates/db/add_crash_count_to_ductwork_executions.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+class AddCrashCountToDuctworkExecutions < Ductwork::Migration
+  def change
+    add_column :ductwork_executions, :crash_count, :integer
+
+    # NOTE: change this how you see fit for your scale
+    Ductwork::Execution
+      .where(crash_count: nil)
+      .update_all(crash_count: 0)
+
+    change_column_null :ductwork_executions, :crash_count, false
+  end
+end

data/lib/generators/ductwork/update/templates/db/add_indexes_to_ductwork_results.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class AddIndexesToDuctworkResults < Ductwork::Migration
+  def change
+    add_index :ductwork_results, %i[result_type created_at]
+  end
+end

data/lib/generators/ductwork/update/templates/db/add_indexes_to_ductwork_runs.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class AddIndexesToDuctworkRuns < Ductwork::Migration
+  def change
+    add_index :ductwork_runs, :started_at
+    add_index :ductwork_runs, %i[pipeline_klass started_at]
+    add_index :ductwork_runs, %i[status started_at]
+  end
+end

data/lib/generators/ductwork/update/templates/db/add_indexes_to_ductwork_transitions.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+class AddIndexesToDuctworkTransitions < Ductwork::Migration
+  def change
+    if mysql?
+      add_index :ductwork_transitions,
+                %i[branch_id completed_at started_at],
+                name: "index_ductwork_transitions_on_latest_open"
+    else
+      add_index :ductwork_transitions,
+                %i[branch_id started_at],
+                where: "completed_at IS NULL",
+                name: "index_ductwork_transitions_on_latest_open"
+    end
+  end
+end

data/lib/generators/ductwork/update/templates/db/add_pipeline_started_index_to_ductwork_runs.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class AddPipelineStartedIndexToDuctworkRuns < Ductwork::Migration
+  def change
+    add_index :ductwork_runs, %i[pipeline_id started_at]
+  end
+end

data/lib/generators/ductwork/update/templates/db/add_process_id_to_ductwork_executions.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+class AddProcessIdToDuctworkExecution < Ductwork::Migration
+  def up
+    options = if postgresql?
+                {
+                  type: uuid_column_type,
+                  index: true,
+                  null: true,
+                  foreign_key: { to_table: :ductwork_processes },
+                }
+              else
+                {
+                  type: uuid_column_type,
+                  limit: 36,
+                  index: true,
+                  null: true,
+                  foreign_key: { to_table: :ductwork_processes },
+                }
+              end
+
+    add_reference :ductwork_executions, :process, **options
+
+    Ductwork::Execution.find_each do |execution|
+      execution.update!(process: execution.availability.process)
+    end
+  end
+
+  def down
+    remove_reference :ductwork_executions,
+                     :process,
+                     foreign_key: { to_table: :ductwork_processes },
+                     index: true
+  end
+end

data/lib/generators/ductwork/update/templates/db/add_role_to_ductwork_processes.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+class AddRoleToDuctworkProcesses < Ductwork::Migration
+  def up
+    add_column :ductwork_processes, :role, :string
+
+    # NOTE: ideally there are no process records that exist because the
+    # ductwork process should not be running. either way, we explicitly
+    # don't set it to supervisor so it won't report in the health checks
+    Ductwork::Process
+      .where(role: nil)
+      .update_all(role: "pipeline_advancer")
+
+    change_column_null :ductwork_executions, :crash_count, false
+  end
+
+  def down
+    remove_column :ductwork_processes, :role, :string
+  end
+end

data/lib/generators/ductwork/update/templates/db/associate_branches_to_runs.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+class AssociateBranchesToRuns < Ductwork::Migration
+  def up
+    options = if postgresql?
+                {
+                  type: uuid_column_type,
+                  index: true,
+                  null: true,
+                  foreign_key: { to_table: :ductwork_runs },
+                }
+              else
+                {
+                  type: uuid_column_type,
+                  limit: 36,
+                  index: true,
+                  null: true,
+                  foreign_key: { to_table: :ductwork_runs },
+                }
+              end
+
+    add_reference :ductwork_branches, :run, **options
+
+    Ductwork::Pipeline.find_each do |pipeline|
+      run = Ductwork::Run.create!(
+        pipeline: pipeline,
+        pipeline_klass: pipeline.klass,
+        definition: pipeline.definition,
+        definition_sha1: pipeline.definition_sha1,
+        status: pipeline.status,
+        triggered_at: pipeline.triggered_at,
+        started_at: pipeline.started_at,
+        completed_at: pipeline.completed_at,
+        halted_at: pipeline.halted_at
+      )
+      pipeline.branches.update_all(run_id: run.id)
+    end
+
+    remove_index :ductwork_branches, %w[pipeline_id started_at]
+    remove_reference :ductwork_branches, :pipeline, index: true, foreign_key: { to_table: :ductwork_pipelines }
+
+    add_index :ductwork_branches, %w[run_id started_at]
+  end
+
+  def down
+    options = if postgresql?
+                {
+                  type: uuid_column_type,
+                  index: true,
+                  null: true,
+                  foreign_key: { to_table: :ductwork_pipelines },
+                }
+              else
+                {
+                  type: uuid_column_type,
+                  limit: 36,
+                  index: true,
+                  null: true,
+                  foreign_key: { to_table: :ductwork_pipelines },
+                }
+              end
+
+    add_reference :ductwork_branches, :pipeline, **options
+
+    Ductwork::Run.find_each do |run|
+      Ductwork::Branch.where(run_id: run.id).update_all(pipeline_id: run.pipeline_id)
+    end
+
+    remove_index :ductwork_branches, %w[run_id started_at]
+    remove_reference :ductwork_branches, :run, index: true, foreign_key: { to_table: :ductwork_runs }
+
+    add_index :ductwork_branches, %w[pipeline_id started_at]
+  end
+end

data/lib/generators/ductwork/update/templates/db/associate_steps_to_branches.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+class AssociateStepsToBranches < Ductwork::Migration
+  def change
+    options = if postgresql?
+                {
+                  type: uuid_column_type,
+                  index: true,
+                  null: true,
+                  foreign_key: { to_table: :ductwork_branches },
+                }
+              else
+                {
+                  type: uuid_column_type,
+                  limit: 36,
+                  index: true,
+                  null: true,
+                  foreign_key: { to_table: :ductwork_branches },
+                }
+              end
+
+    add_reference :ductwork_steps, :branch, **options
+  end
+end

data/lib/generators/ductwork/update/templates/db/associate_steps_to_runs.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+class AssociateStepsToRuns < Ductwork::Migration
+  def up
+    options = if postgresql?
+                {
+                  type: uuid_column_type,
+                  index: true,
+                  null: true,
+                  foreign_key: { to_table: :ductwork_runs },
+                }
+              else
+                {
+                  type: uuid_column_type,
+                  limit: 36,
+                  index: true,
+                  null: true,
+                  foreign_key: { to_table: :ductwork_runs },
+                }
+              end
+
+    add_reference :ductwork_steps, :run, **options
+
+    Ductwork::Pipeline.find_each do |pipeline|
+      run = Ductwork::Run.find_by(pipeline_id: pipeline.id)
+      pipeline.steps.update_all(run_id: run.id) if run
+    end
+
+    remove_index :ductwork_steps, %i[pipeline_id status node]
+    remove_index :ductwork_steps, %i[pipeline_id node status]
+    remove_index :ductwork_steps, %i[pipeline_id status]
+    remove_reference :ductwork_steps, :pipeline, index: true, foreign_key: { to_table: :ductwork_pipelines }
+
+    add_index :ductwork_steps, %i[run_id status node]
+    add_index :ductwork_steps, %i[run_id node status]
+    add_index :ductwork_steps, %i[run_id status]
+  end
+
+  def down
+    options = if postgresql?
+                {
+                  type: uuid_column_type,
+                  index: true,
+                  null: true,
+                  foreign_key: { to_table: :ductwork_pipelines },
+                }
+              else
+                {
+                  type: uuid_column_type,
+                  limit: 36,
+                  index: true,
+                  null: true,
+                  foreign_key: { to_table: :ductwork_pipelines },
+                }
+              end
+
+    add_reference :ductwork_steps, :pipeline, **options
+
+    Ductwork::Run.find_each do |run|
+      Ductwork::Step.where(run_id: run.id).update_all(pipeline_id: run.pipeline_id)
+    end
+
+    remove_index :ductwork_steps, %i[run_id status node]
+    remove_index :ductwork_steps, %i[run_id node status]
+    remove_index :ductwork_steps, %i[run_id status]
+    remove_reference :ductwork_steps, :run, index: true, foreign_key: { to_table: :ductwork_runs }
+
+    add_index :ductwork_steps, %i[pipeline_id status node]
+    add_index :ductwork_steps, %i[pipeline_id node status]
+    add_index :ductwork_steps, %i[pipeline_id status]
+  end
+end

data/lib/generators/ductwork/update/templates/db/associate_tuples_to_runs.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+class AssociateTuplesToRuns < Ductwork::Migration
+  def up
+    options = if postgresql?
+                {
+                  type: uuid_column_type,
+                  index: true,
+                  null: true,
+                  foreign_key: { to_table: :ductwork_runs },
+                }
+              else
+                {
+                  type: uuid_column_type,
+                  limit: 36,
+                  index: true,
+                  null: true,
+                  foreign_key: { to_table: :ductwork_runs },
+                }
+              end
+
+    add_reference :ductwork_tuples, :run, **options
+
+    Ductwork::Pipeline.find_each do |pipeline|
+      run = Ductwork::Run.find_by(pipeline_id: pipeline.id)
+
+      pipeline.tuples.update_all(run_id: run.id) if run
+    end
+
+    remove_index :ductwork_tuples, %i[pipeline_id status node]
+    remove_index :ductwork_tuples, %i[pipeline_id node status]
+    remove_index :ductwork_tuples, %i[pipeline_id status]
+    remove_reference :ductwork_tuples, :pipeline, index: true, foreign_key: { to_table: :ductwork_pipelines }
+
+    add_index :ductwork_tuples, %i[run_id status node]
+    add_index :ductwork_tuples, %i[run_id node status]
+    add_index :ductwork_tuples, %i[run_id status]
+  end
+
+  def down
+    options = if postgresql?
+                {
+                  type: uuid_column_type,
+                  index: true,
+                  null: true,
+                  foreign_key: { to_table: :ductwork_pipelines },
+                }
+              else
+                {
+                  type: uuid_column_type,
+                  limit: 36,
+                  index: true,
+                  null: true,
+                  foreign_key: { to_table: :ductwork_pipelines },
+                }
+              end
+
+    add_reference :ductwork_tuples, :pipeline, **options
+
+    Ductwork::Run.find_each do |run|
+      Ductwork::Tuple.where(run_id: run.id).update_all(pipeline_id: run.pipeline_id)
+    end
+
+    remove_index :ductwork_tuples, %i[run_id status node]
+    remove_index :ductwork_tuples, %i[run_id node status]
+    remove_index :ductwork_tuples, %i[run_id status]
+    remove_reference :ductwork_tuples, :run, index: true, foreign_key: { to_table: :ductwork_runs }
+
+    add_index :ductwork_tuples, %i[pipeline_id status node]
+    add_index :ductwork_tuples, %i[pipeline_id node status]
+    add_index :ductwork_tuples, %i[pipeline_id status]
+  end
+end