ductwork 0.25.0 → 1.0.0

data/lib/generators/ductwork/update/templates/db/add_pipeline_started_index_to_ductwork_runs.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class AddPipelineStartedIndexToDuctworkRuns < Ductwork::Migration
+  def change
+    add_index :ductwork_runs, %i[pipeline_id started_at]
+  end
+end

data/lib/generators/ductwork/update/templates/db/add_process_id_to_ductwork_executions.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+class AddProcessIdToDuctworkExecution < Ductwork::Migration
+  def up
+    options = if postgresql?
+                {
+                  type: uuid_column_type,
+                  index: true,
+                  null: true,
+                  foreign_key: { to_table: :ductwork_processes },
+                }
+              else
+                {
+                  type: uuid_column_type,
+                  limit: 36,
+                  index: true,
+                  null: true,
+                  foreign_key: { to_table: :ductwork_processes },
+                }
+              end
+
+    add_reference :ductwork_executions, :process, **options
+
+    Ductwork::Execution.find_each do |execution|
+      execution.update!(process: execution.availability.process)
+    end
+  end
+
+  def down
+    remove_reference :ductwork_executions,
+                     :process,
+                     foreign_key: { to_table: :ductwork_processes },
+                     index: true
+  end
+end

data/lib/generators/ductwork/update/templates/db/add_role_to_ductwork_processes.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+class AddRoleToDuctworkProcesses < Ductwork::Migration
+  def up
+    add_column :ductwork_processes, :role, :string
+
+    # NOTE: ideally there are no process records that exist because the
+    # ductwork process should not be running. either way, we explicitly
+    # don't set it to supervisor so it won't report in the health checks
+    Ductwork::Process
+      .where(role: nil)
+      .update_all(role: "pipeline_advancer")
+
+    change_column_null :ductwork_executions, :crash_count, false
+  end
+
+  def down
+    remove_column :ductwork_processes, :role, :string
+  end
+end

data/lib/generators/ductwork/update/templates/db/associate_branches_to_runs.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+class AssociateBranchesToRuns < Ductwork::Migration
+  def up
+    options = if postgresql?
+                {
+                  type: uuid_column_type,
+                  index: true,
+                  null: true,
+                  foreign_key: { to_table: :ductwork_runs },
+                }
+              else
+                {
+                  type: uuid_column_type,
+                  limit: 36,
+                  index: true,
+                  null: true,
+                  foreign_key: { to_table: :ductwork_runs },
+                }
+              end
+
+    add_reference :ductwork_branches, :run, **options
+
+    Ductwork::Pipeline.find_each do |pipeline|
+      run = Ductwork::Run.create!(
+        pipeline: pipeline,
+        pipeline_klass: pipeline.klass,
+        definition: pipeline.definition,
+        definition_sha1: pipeline.definition_sha1,
+        status: pipeline.status,
+        triggered_at: pipeline.triggered_at,
+        started_at: pipeline.started_at,
+        completed_at: pipeline.completed_at,
+        halted_at: pipeline.halted_at
+      )
+      pipeline.branches.update_all(run_id: run.id)
+    end
+
+    remove_index :ductwork_branches, %w[pipeline_id started_at]
+    remove_reference :ductwork_branches, :pipeline, index: true, foreign_key: { to_table: :ductwork_pipelines }
+
+    add_index :ductwork_branches, %w[run_id started_at]
+  end
+
+  def down
+    options = if postgresql?
+                {
+                  type: uuid_column_type,
+                  index: true,
+                  null: true,
+                  foreign_key: { to_table: :ductwork_pipelines },
+                }
+              else
+                {
+                  type: uuid_column_type,
+                  limit: 36,
+                  index: true,
+                  null: true,
+                  foreign_key: { to_table: :ductwork_pipelines },
+                }
+              end
+
+    add_reference :ductwork_branches, :pipeline, **options
+
+    Ductwork::Run.find_each do |run|
+      Ductwork::Branch.where(run_id: run.id).update_all(pipeline_id: run.pipeline_id)
+    end
+
+    remove_index :ductwork_branches, %w[run_id started_at]
+    remove_reference :ductwork_branches, :run, index: true, foreign_key: { to_table: :ductwork_runs }
+
+    add_index :ductwork_branches, %w[pipeline_id started_at]
+  end
+end

data/lib/generators/ductwork/update/templates/db/associate_steps_to_branches.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+class AssociateStepsToBranches < Ductwork::Migration
+  def change
+    options = if postgresql?
+                {
+                  type: uuid_column_type,
+                  index: true,
+                  null: true,
+                  foreign_key: { to_table: :ductwork_branches },
+                }
+              else
+                {
+                  type: uuid_column_type,
+                  limit: 36,
+                  index: true,
+                  null: true,
+                  foreign_key: { to_table: :ductwork_branches },
+                }
+              end
+
+    add_reference :ductwork_steps, :branch, **options
+  end
+end

data/lib/generators/ductwork/update/templates/db/associate_steps_to_runs.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+class AssociateStepsToRuns < Ductwork::Migration
+  def up
+    options = if postgresql?
+                {
+                  type: uuid_column_type,
+                  index: true,
+                  null: true,
+                  foreign_key: { to_table: :ductwork_runs },
+                }
+              else
+                {
+                  type: uuid_column_type,
+                  limit: 36,
+                  index: true,
+                  null: true,
+                  foreign_key: { to_table: :ductwork_runs },
+                }
+              end
+
+    add_reference :ductwork_steps, :run, **options
+
+    Ductwork::Pipeline.find_each do |pipeline|
+      run = Ductwork::Run.find_by(pipeline_id: pipeline.id)
+      pipeline.steps.update_all(run_id: run.id) if run
+    end
+
+    remove_index :ductwork_steps, %i[pipeline_id status node]
+    remove_index :ductwork_steps, %i[pipeline_id node status]
+    remove_index :ductwork_steps, %i[pipeline_id status]
+    remove_reference :ductwork_steps, :pipeline, index: true, foreign_key: { to_table: :ductwork_pipelines }
+
+    add_index :ductwork_steps, %i[run_id status node]
+    add_index :ductwork_steps, %i[run_id node status]
+    add_index :ductwork_steps, %i[run_id status]
+  end
+
+  def down
+    options = if postgresql?
+                {
+                  type: uuid_column_type,
+                  index: true,
+                  null: true,
+                  foreign_key: { to_table: :ductwork_pipelines },
+                }
+              else
+                {
+                  type: uuid_column_type,
+                  limit: 36,
+                  index: true,
+                  null: true,
+                  foreign_key: { to_table: :ductwork_pipelines },
+                }
+              end
+
+    add_reference :ductwork_steps, :pipeline, **options
+
+    Ductwork::Run.find_each do |run|
+      Ductwork::Step.where(run_id: run.id).update_all(pipeline_id: run.pipeline_id)
+    end
+
+    remove_index :ductwork_steps, %i[run_id status node]
+    remove_index :ductwork_steps, %i[run_id node status]
+    remove_index :ductwork_steps, %i[run_id status]
+    remove_reference :ductwork_steps, :run, index: true, foreign_key: { to_table: :ductwork_runs }
+
+    add_index :ductwork_steps, %i[pipeline_id status node]
+    add_index :ductwork_steps, %i[pipeline_id node status]
+    add_index :ductwork_steps, %i[pipeline_id status]
+  end
+end

data/lib/generators/ductwork/update/templates/db/associate_tuples_to_runs.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+class AssociateTuplesToRuns < Ductwork::Migration
+  def up
+    options = if postgresql?
+                {
+                  type: uuid_column_type,
+                  index: true,
+                  null: true,
+                  foreign_key: { to_table: :ductwork_runs },
+                }
+              else
+                {
+                  type: uuid_column_type,
+                  limit: 36,
+                  index: true,
+                  null: true,
+                  foreign_key: { to_table: :ductwork_runs },
+                }
+              end
+
+    add_reference :ductwork_tuples, :run, **options
+
+    Ductwork::Pipeline.find_each do |pipeline|
+      run = Ductwork::Run.find_by(pipeline_id: pipeline.id)
+
+      pipeline.tuples.update_all(run_id: run.id) if run
+    end
+
+    remove_index :ductwork_tuples, %i[pipeline_id status node]
+    remove_index :ductwork_tuples, %i[pipeline_id node status]
+    remove_index :ductwork_tuples, %i[pipeline_id status]
+    remove_reference :ductwork_tuples, :pipeline, index: true, foreign_key: { to_table: :ductwork_pipelines }
+
+    add_index :ductwork_tuples, %i[run_id status node]
+    add_index :ductwork_tuples, %i[run_id node status]
+    add_index :ductwork_tuples, %i[run_id status]
+  end
+
+  def down
+    options = if postgresql?
+                {
+                  type: uuid_column_type,
+                  index: true,
+                  null: true,
+                  foreign_key: { to_table: :ductwork_pipelines },
+                }
+              else
+                {
+                  type: uuid_column_type,
+                  limit: 36,
+                  index: true,
+                  null: true,
+                  foreign_key: { to_table: :ductwork_pipelines },
+                }
+              end
+
+    add_reference :ductwork_tuples, :pipeline, **options
+
+    Ductwork::Run.find_each do |run|
+      Ductwork::Tuple.where(run_id: run.id).update_all(pipeline_id: run.pipeline_id)
+    end
+
+    remove_index :ductwork_tuples, %i[run_id status node]
+    remove_index :ductwork_tuples, %i[run_id node status]
+    remove_index :ductwork_tuples, %i[run_id status]
+    remove_reference :ductwork_tuples, :run, index: true, foreign_key: { to_table: :ductwork_runs }
+
+    add_index :ductwork_tuples, %i[pipeline_id status node]
+    add_index :ductwork_tuples, %i[pipeline_id node status]
+    add_index :ductwork_tuples, %i[pipeline_id status]
+  end
+end

data/lib/generators/ductwork/update/templates/db/backfill_branch_ids_on_steps.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+class BackfillBranchIdsOnSteps < Ductwork::Migration
+  def up
+    Ductwork::Pipeline.find_each do |pipeline|
+      branch = Ductwork::Branch.create!(
+        pipeline: pipeline,
+        pipeline_klass: pipeline.klass,
+        status: pipeline.status,
+        started_at: pipeline.started_at,
+        last_advanced_at: pipeline.last_advanced_at,
+        completed_at: pipeline.completed_at
+      )
+      pipeline.steps.update_all(branch_id: branch.id)
+    end
+  end
+
+  def down
+    Ductwork::Step.update_all(branch_id: nil)
+  end
+end

data/lib/generators/ductwork/update/templates/db/create_ductwork_advancements.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+class CreateDuctworkAdvancements < Ductwork::Migration
+  def change
+    create_ductwork_table :ductwork_advancements do |table|
+      belongs_to(
+        table,
+        :process,
+        index: true,
+        null: true,
+        foreign_key: { to_table: :ductwork_processes }
+      )
+      belongs_to(
+        table,
+        :transition,
+        index: true,
+        null: false,
+        foreign_key: { to_table: :ductwork_transitions }
+      )
+      table.datetime :started_at, null: false
+      table.datetime :completed_at
+      table.string :error_klass
+      table.string :error_message
+      table.text :error_backtrace
+      table.timestamps null: false
+    end
+  end
+end

data/lib/generators/ductwork/update/templates/db/create_ductwork_branch_links.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+class CreateDuctworkBranchLinks < Ductwork::Migration
+  def change
+    create_ductwork_table :ductwork_branch_links do |table|
+      belongs_to(
+        table,
+        :parent_branch,
+        index: true,
+        null: false,
+        foreign_key: { to_table: :ductwork_branches }
+      )
+      belongs_to(
+        table,
+        :child_branch,
+        index: true,
+        null: false,
+        foreign_key: { to_table: :ductwork_branches }
+      )
+      table.timestamps null: false
+    end
+
+    add_index :ductwork_branch_links,
+              %w[parent_branch_id child_branch_id],
+              unique: true
+  end
+end

data/lib/generators/ductwork/update/templates/db/create_ductwork_branches.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+class CreateDuctworkBranches < Ductwork::Migration
+  def change
+    create_ductwork_table :ductwork_branches do |table|
+      belongs_to(
+        table,
+        :pipeline,
+        index: true,
+        null: false,
+        foreign_key: { to_table: :ductwork_pipelines }
+      )
+      table.string :pipeline_klass, null: false
+      table.string :status, null: false
+      table.string :halt_reason
+      table.string :claim_token
+      table.datetime :claimed_for_advancing_at
+      table.datetime :last_advanced_at, null: false
+      table.datetime :started_at, null: false
+      table.datetime :completed_at
+      table.timestamps null: false
+    end
+
+    if mysql?
+      add_index :ductwork_branches,
+                %w[pipeline_klass status claimed_for_advancing_at last_advanced_at],
+                name: "index_ductwork_branches_on_claim_latest"
+    else
+      add_index :ductwork_branches,
+                %w[pipeline_klass status last_advanced_at],
+                where: "claimed_for_advancing_at IS NULL",
+                name: "index_ductwork_branches_on_claim_latest"
+    end
+    add_index :ductwork_branches, %w[pipeline_id started_at]
+    add_index :ductwork_branches, %w[run_id status]
+  end
+end

data/lib/generators/ductwork/update/templates/db/create_ductwork_runs.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+class CreateDuctworkRuns < Ductwork::Migration
+  def change
+    create_ductwork_table :ductwork_runs do |table|
+      belongs_to(
+        table,
+        :pipeline,
+        index: true,
+        null: false,
+        foreign_key: { to_table: :ductwork_pipelines }
+      )
+      table.string :pipeline_klass, null: false
+      table.text :definition, null: false
+      table.string :definition_sha1, null: false
+      table.string :status, null: false
+      table.datetime :triggered_at, null: false
+      table.datetime :started_at, null: false
+      table.datetime :completed_at
+      table.datetime :halted_at
+      table.datetime :on_halt_dispatched_at
+      table.timestamps null: false
+    end
+
+    add_index :ductwork_runs, %i[pipeline_id status]
+
+    if mysql?
+      reversible do |direction|
+        direction.up do
+          execute <<~SQL
+            ALTER TABLE ductwork_runs
+              ADD COLUMN active_pipeline_id VARCHAR(36)
+                GENERATED ALWAYS AS (
+                  IF(status IN ('in_progress', 'paused'), pipeline_id, NULL)
+                )
+          SQL
+          add_index :ductwork_runs, :active_pipeline_id,
+                    unique: true,
+                    name: :idx_unique_active_run
+        end
+
+        direction.down do
+          remove_index :ductwork_runs, name: :idx_unique_active_run
+          remove_column :ductwork_runs, :active_pipeline_id
+        end
+      end
+    else
+      add_index :ductwork_runs,
+                :pipeline_id,
+                unique: true,
+                where: "status IN ('in_progress', 'paused')",
+                name: :idx_unique_active_run
+    end
+  end
+end

data/lib/generators/ductwork/update/templates/db/create_ductwork_transitions.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+class CreateDuctworkTransitions < Ductwork::Migration
+  def change
+    create_ductwork_table :ductwork_transitions do |table|
+      belongs_to(
+        table,
+        :branches,
+        index: true,
+        null: false,
+        foreign_key: { to_table: :ductwork_branches }
+      )
+      belongs_to(
+        table,
+        :in_step,
+        index: true,
+        null: false,
+        foreign_key: { to_table: :ductwork_steps }
+      )
+      belongs_to(
+        table,
+        :out_step,
+        index: true,
+        null: true,
+        foreign_key: { to_table: :ductwork_steps }
+      )
+      table.datetime :started_at, null: false
+      table.datetime :completed_at
+      table.timestamps null: false
+    end
+  end
+end

data/lib/generators/ductwork/update/templates/db/denormalize_pipeline_klass_on_availabilities.rb

@@ -12,9 +12,16 @@ class DenormalizePipelineKlassOnAvailabilities < Ductwork::Migration
 
     change_column_null :ductwork_availabilities, :pipeline_klass, false
     remove_index :ductwork_availabilities, name: "index_ductwork_availabilities_on_claim_latest"
-    add_index :ductwork_availabilities,
-              %i[pipeline_klass started_at],
-              name: "index_ductwork_availabilities_on_claim_latest",
-              where: "completed_at IS NULL"
+
+    if mysql?
+      add_index :ductwork_availabilities,
+                %i[pipeline_klass completed_at started_at],
+                name: "index_ductwork_availabilities_on_claim_latest"
+    else
+      add_index :ductwork_availabilities,
+                %i[pipeline_klass started_at],
+                name: "index_ductwork_availabilities_on_claim_latest",
+                where: "completed_at IS NULL"
+    end
   end
 end

data/lib/generators/ductwork/update/templates/db/migrate_tables_to_uuid_primary_key.rb

@@ -31,6 +31,7 @@ class MigrateTablesToUuidPrimaryKey < Ductwork::Migration
     drop_old_foreign_keys
     swap_primary_keys
     swap_foreign_keys
+    create_missing_indexes
   end
 
   def down
@@ -115,4 +116,8 @@ class MigrateTablesToUuidPrimaryKey < Ductwork::Migration
       end
     end
   end
+
+  def create_missing_indexes
+    add_index :ductwork_tuples, %i[key pipeline_id], unique: true
+  end
 end

data/lib/generators/ductwork/update/templates/db/rename_runs_to_attempts.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class RenameRunsToAttempts < Ductwork::Migration
+  def change
+    rename_table :ductwork_runs, :ductwork_attempts
+  end
+end

data/lib/generators/ductwork/update/templates/db/update_process_associations.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+class UpdateProcessAssociations < Ductwork::Migration
+  def change
+    remove_column :ductwork_availabilities, :process_id, :integer
+
+    options = if postgresql?
+                {
+                  type: uuid_column_type,
+                  index: true,
+                  null: true,
+                  foreign_key: { to_table: :ductwork_processes },
+                }
+              else
+                {
+                  type: uuid_column_type,
+                  limit: 36,
+                  index: true,
+                  null: true,
+                  foreign_key: { to_table: :ductwork_processes },
+                }
+              end
+
+    add_reference :ductwork_availabilities, :process, **options
+    add_index :ductwork_availabilities, %i[id process_id]
+
+    remove_column :ductwork_executions, :process_id, :integer
+  end
+end