ductwork 0.26.0 → 1.0.0

data/lib/generators/ductwork/update/templates/db/backfill_branch_ids_on_steps.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+class BackfillBranchIdsOnSteps < Ductwork::Migration
+  def up
+    Ductwork::Pipeline.find_each do |pipeline|
+      branch = Ductwork::Branch.create!(
+        pipeline: pipeline,
+        pipeline_klass: pipeline.klass,
+        status: pipeline.status,
+        started_at: pipeline.started_at,
+        last_advanced_at: pipeline.last_advanced_at,
+        completed_at: pipeline.completed_at
+      )
+      pipeline.steps.update_all(branch_id: branch.id)
+    end
+  end
+
+  def down
+    Ductwork::Step.update_all(branch_id: nil)
+  end
+end

data/lib/generators/ductwork/update/templates/db/create_ductwork_advancements.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+class CreateDuctworkAdvancements < Ductwork::Migration
+  def change
+    create_ductwork_table :ductwork_advancements do |table|
+      belongs_to(
+        table,
+        :process,
+        index: true,
+        null: true,
+        foreign_key: { to_table: :ductwork_processes }
+      )
+      belongs_to(
+        table,
+        :transition,
+        index: true,
+        null: false,
+        foreign_key: { to_table: :ductwork_transitions }
+      )
+      table.datetime :started_at, null: false
+      table.datetime :completed_at
+      table.string :error_klass
+      table.string :error_message
+      table.text :error_backtrace
+      table.timestamps null: false
+    end
+  end
+end

data/lib/generators/ductwork/update/templates/db/create_ductwork_branch_links.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+class CreateDuctworkBranchLinks < Ductwork::Migration
+  def change
+    create_ductwork_table :ductwork_branch_links do |table|
+      belongs_to(
+        table,
+        :parent_branch,
+        index: true,
+        null: false,
+        foreign_key: { to_table: :ductwork_branches }
+      )
+      belongs_to(
+        table,
+        :child_branch,
+        index: true,
+        null: false,
+        foreign_key: { to_table: :ductwork_branches }
+      )
+      table.timestamps null: false
+    end
+
+    add_index :ductwork_branch_links,
+              %w[parent_branch_id child_branch_id],
+              unique: true
+  end
+end

data/lib/generators/ductwork/update/templates/db/create_ductwork_branches.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+class CreateDuctworkBranches < Ductwork::Migration
+  def change
+    create_ductwork_table :ductwork_branches do |table|
+      belongs_to(
+        table,
+        :pipeline,
+        index: true,
+        null: false,
+        foreign_key: { to_table: :ductwork_pipelines }
+      )
+      table.string :pipeline_klass, null: false
+      table.string :status, null: false
+      table.string :halt_reason
+      table.string :claim_token
+      table.datetime :claimed_for_advancing_at
+      table.datetime :last_advanced_at, null: false
+      table.datetime :started_at, null: false
+      table.datetime :completed_at
+      table.timestamps null: false
+    end
+
+    if mysql?
+      add_index :ductwork_branches,
+                %w[pipeline_klass status claimed_for_advancing_at last_advanced_at],
+                name: "index_ductwork_branches_on_claim_latest"
+    else
+      add_index :ductwork_branches,
+                %w[pipeline_klass status last_advanced_at],
+                where: "claimed_for_advancing_at IS NULL",
+                name: "index_ductwork_branches_on_claim_latest"
+    end
+    add_index :ductwork_branches, %w[pipeline_id started_at]
+    add_index :ductwork_branches, %w[run_id status]
+  end
+end

data/lib/generators/ductwork/update/templates/db/create_ductwork_runs.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+class CreateDuctworkRuns < Ductwork::Migration
+  def change
+    create_ductwork_table :ductwork_runs do |table|
+      belongs_to(
+        table,
+        :pipeline,
+        index: true,
+        null: false,
+        foreign_key: { to_table: :ductwork_pipelines }
+      )
+      table.string :pipeline_klass, null: false
+      table.text :definition, null: false
+      table.string :definition_sha1, null: false
+      table.string :status, null: false
+      table.datetime :triggered_at, null: false
+      table.datetime :started_at, null: false
+      table.datetime :completed_at
+      table.datetime :halted_at
+      table.datetime :on_halt_dispatched_at
+      table.timestamps null: false
+    end
+
+    add_index :ductwork_runs, %i[pipeline_id status]
+
+    if mysql?
+      reversible do |direction|
+        direction.up do
+          execute <<~SQL
+            ALTER TABLE ductwork_runs
+              ADD COLUMN active_pipeline_id VARCHAR(36)
+                GENERATED ALWAYS AS (
+                  IF(status IN ('in_progress', 'paused'), pipeline_id, NULL)
+                )
+          SQL
+          add_index :ductwork_runs, :active_pipeline_id,
+                    unique: true,
+                    name: :idx_unique_active_run
+        end
+
+        direction.down do
+          remove_index :ductwork_runs, name: :idx_unique_active_run
+          remove_column :ductwork_runs, :active_pipeline_id
+        end
+      end
+    else
+      add_index :ductwork_runs,
+                :pipeline_id,
+                unique: true,
+                where: "status IN ('in_progress', 'paused')",
+                name: :idx_unique_active_run
+    end
+  end
+end

data/lib/generators/ductwork/update/templates/db/create_ductwork_transitions.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+class CreateDuctworkTransitions < Ductwork::Migration
+  def change
+    create_ductwork_table :ductwork_transitions do |table|
+      belongs_to(
+        table,
+        :branches,
+        index: true,
+        null: false,
+        foreign_key: { to_table: :ductwork_branches }
+      )
+      belongs_to(
+        table,
+        :in_step,
+        index: true,
+        null: false,
+        foreign_key: { to_table: :ductwork_steps }
+      )
+      belongs_to(
+        table,
+        :out_step,
+        index: true,
+        null: true,
+        foreign_key: { to_table: :ductwork_steps }
+      )
+      table.datetime :started_at, null: false
+      table.datetime :completed_at
+      table.timestamps null: false
+    end
+  end
+end

data/lib/generators/ductwork/update/templates/db/denormalize_pipeline_klass_on_availabilities.rb

@@ -12,9 +12,16 @@ class DenormalizePipelineKlassOnAvailabilities < Ductwork::Migration
 
     change_column_null :ductwork_availabilities, :pipeline_klass, false
     remove_index :ductwork_availabilities, name: "index_ductwork_availabilities_on_claim_latest"
-    add_index :ductwork_availabilities,
-              %i[pipeline_klass started_at],
-              name: "index_ductwork_availabilities_on_claim_latest",
-              where: "completed_at IS NULL"
+
+    if mysql?
+      add_index :ductwork_availabilities,
+                %i[pipeline_klass completed_at started_at],
+                name: "index_ductwork_availabilities_on_claim_latest"
+    else
+      add_index :ductwork_availabilities,
+                %i[pipeline_klass started_at],
+                name: "index_ductwork_availabilities_on_claim_latest",
+                where: "completed_at IS NULL"
+    end
   end
 end

data/lib/generators/ductwork/update/templates/db/migrate_tables_to_uuid_primary_key.rb

@@ -31,6 +31,7 @@ class MigrateTablesToUuidPrimaryKey < Ductwork::Migration
     drop_old_foreign_keys
     swap_primary_keys
     swap_foreign_keys
+    create_missing_indexes
   end
 
   def down
@@ -115,4 +116,8 @@ class MigrateTablesToUuidPrimaryKey < Ductwork::Migration
       end
     end
   end
+
+  def create_missing_indexes
+    add_index :ductwork_tuples, %i[key pipeline_id], unique: true
+  end
 end

data/lib/generators/ductwork/update/templates/db/rename_runs_to_attempts.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class RenameRunsToAttempts < Ductwork::Migration
+  def change
+    rename_table :ductwork_runs, :ductwork_attempts
+  end
+end

data/lib/generators/ductwork/update/templates/db/update_process_associations.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+class UpdateProcessAssociations < Ductwork::Migration
+  def change
+    remove_column :ductwork_availabilities, :process_id, :integer
+
+    options = if postgresql?
+                {
+                  type: uuid_column_type,
+                  index: true,
+                  null: true,
+                  foreign_key: { to_table: :ductwork_processes },
+                }
+              else
+                {
+                  type: uuid_column_type,
+                  limit: 36,
+                  index: true,
+                  null: true,
+                  foreign_key: { to_table: :ductwork_processes },
+                }
+              end
+
+    add_reference :ductwork_availabilities, :process, **options
+    add_index :ductwork_availabilities, %i[id process_id]
+
+    remove_column :ductwork_executions, :process_id, :integer
+  end
+end

data/lib/generators/ductwork/update/update_generator.rb

@@ -10,6 +10,8 @@ module Ductwork
     source_root File.expand_path("templates", __dir__)
 
     def create_files
+      connection = Ductwork::Record.connection
+
       if Ductwork::Availability.column_names.exclude?("pipeline_klass")
         migration_template "db/denormalize_pipeline_klass_on_availabilities.rb",
                            "db/migrate/denormalize_pipeline_klass_on_availabilities.rb"
@@ -19,6 +21,103 @@ module Ductwork
         migration_template "db/migrate_tables_to_uuid_primary_key.rb",
                            "db/migrate/migrate_tables_to_uuid_primary_key.rb"
       end
+
+      if !connection.table_exists?(:ductwork_branches)
+        migration_template "db/create_ductwork_branches.rb",
+                           "db/migrate/create_ductwork_branches.rb"
+      end
+
+      if !connection.table_exists?(:ductwork_branch_links)
+        migration_template "db/create_ductwork_branch_links.rb",
+                           "db/migrate/create_ductwork_branch_links.rb"
+      end
+
+      if Ductwork::Step.column_names.exclude?("branch_id")
+        migration_template "db/associate_steps_to_branches.rb",
+                           "db/migrate/associate_steps_to_branches.rb"
+        migration_template "db/backfill_branch_ids_on_steps.rb",
+                           "db/migrate/backfill_branch_ids_on_steps.rb"
+      end
+
+      if !connection.table_exists?(:ductwork_transitions)
+        migration_template "db/create_ductwork_transitions.rb",
+                           "db/migrate/create_ductwork_transitions.rb"
+      end
+
+      if !connection.table_exists?(:ductwork_advancements)
+        migration_template "db/create_ductwork_advancements.rb",
+                           "db/migrate/create_ductwork_advancements.rb"
+      end
+
+      if Ductwork::Execution.column_names.include?("process_id")
+        migration_template "db/update_process_associations.rb",
+                           "db/migrate/update_process_associations.rb"
+      end
+
+      if connection.table_exists?(:ductwork_runs)
+        migration_template "db/rename_runs_to_attempts.rb",
+                           "db/migrate/rename_runs_to_attempts.rb"
+      end
+
+      if !connection.table_exists?(:ductwork_runs)
+        migration_template "db/create_ductwork_runs.rb",
+                           "db/migrate/create_ductwork_runs.rb"
+      end
+
+      if Ductwork::Branch.column_names.include?("pipeline_id")
+        migration_template "db/associate_branches_to_runs.rb",
+                           "db/migrate/associate_branches_to_runs.rb"
+      end
+
+      if Ductwork::Step.column_names.include?("pipeline_id")
+        migration_template "db/associate_steps_to_runs.rb",
+                           "db/migrate/associate_steps_to_runs.rb"
+      end
+
+      if Ductwork::Tuple.column_names.include?("pipeline_id")
+        migration_template "db/associate_tuples_to_runs.rb",
+                           "db/migrate/associate_tuples_to_runs.rb"
+      end
+
+      if Ductwork::Execution.column_names.exclude?("crash_count")
+        migration_template "db/add_crash_count_to_ductwork_executions.rb",
+                           "db/migrate/add_crash_count_to_ductwork_executions.rb"
+      end
+
+      if Ductwork::Advancement.column_names.exclude?("crash_count")
+        migration_template "db/add_crash_count_to_ductwork_advancements.rb",
+                           "db/migrate/add_crash_count_to_ductwork_advancements.rb"
+      end
+
+      if Ductwork::Execution.column_names.exclude?("process_id")
+        migration_template "db/add_process_id_to_ductwork_executions.rb",
+                           "db/migrate/add_process_id_to_ductwork_executions.rb"
+      end
+
+      if Ductwork::Process.column_names.exclude?("role")
+        migration_template "db/add_role_to_ductwork_processes.rb",
+                           "db/migrate/add_role_to_ductwork_processes.rb"
+      end
+
+      if !connection.index_exists?(:ductwork_results, %i[result_type created_at])
+        migration_template "db/add_indexes_to_ductwork_results.rb",
+                           "db/migrate/add_indexes_to_ductwork_results.rb"
+      end
+
+      if !connection.index_exists?(:ductwork_runs, :started_at)
+        migration_template "db/add_indexes_to_ductwork_runs.rb",
+                           "db/migrate/add_indexes_to_ductwork_runs.rb"
+      end
+
+      if !connection.index_exists?(:ductwork_runs, %i[pipeline_id started_at])
+        migration_template "db/add_pipeline_started_index_to_ductwork_runs.rb",
+                           "db/migrate/add_pipeline_started_index_to_ductwork_runs.rb"
+      end
+
+      if !connection.index_exists?(:ductwork_transitions, name: "index_ductwork_transitions_on_latest_open")
+        migration_template "db/add_indexes_to_ductwork_transitions.rb",
+                           "db/migrate/add_indexes_to_ductwork_transitions.rb"
+      end
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ductwork
 version: !ruby/object:Gem::Version
-  version: 0.26.0
+  version: 1.0.0
 platform: ruby
 authors:
 - Tyler Ewing
@@ -111,12 +111,17 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".claude/skills/audit-clock-drift/SKILL.md"
+- ".claude/skills/audit-database-indexes/SKILL.md"
+- ".claude/skills/audit-database-support/SKILL.md"
+- ".claude/skills/audit-durability/SKILL.md"
 - ".saturnci/Dockerfile"
 - ".saturnci/database.yml"
 - ".saturnci/docker-compose.yml"
 - ".saturnci/pre.sh"
 - CHANGELOG-PRO.md
 - CHANGELOG.md
+- CLAUDE.md
 - LICENSE.txt
 - PRO-LICENSE.txt
 - README.md
@@ -139,17 +144,24 @@ files:
 - app/views/layouts/ductwork/application.html.erb
 - config/routes.rb
 - lib/ductwork.rb
+- lib/ductwork/branch_claim.rb
 - lib/ductwork/cli.rb
 - lib/ductwork/configuration.rb
 - lib/ductwork/context.rb
+- lib/ductwork/database_clock.rb
 - lib/ductwork/dsl/branch_builder.rb
 - lib/ductwork/dsl/definition_builder.rb
 - lib/ductwork/engine.rb
-- lib/ductwork/job_claim.rb
+- lib/ductwork/execution_claim.rb
+- lib/ductwork/fault_injection.rb
 - lib/ductwork/machine_identifier.rb
 - lib/ductwork/migration.rb
 - lib/ductwork/migration_helper.rb
+- lib/ductwork/models/advancement.rb
+- lib/ductwork/models/attempt.rb
 - lib/ductwork/models/availability.rb
+- lib/ductwork/models/branch.rb
+- lib/ductwork/models/branch_link.rb
 - lib/ductwork/models/execution.rb
 - lib/ductwork/models/job.rb
 - lib/ductwork/models/pipeline.rb
@@ -158,8 +170,11 @@ files:
 - lib/ductwork/models/result.rb
 - lib/ductwork/models/run.rb
 - lib/ductwork/models/step.rb
+- lib/ductwork/models/transition.rb
 - lib/ductwork/models/tuple.rb
-- lib/ductwork/optimistic_locking_job_claim.rb
+- lib/ductwork/models/workflow.rb
+- lib/ductwork/optimistic_locking_execution_claim.rb
+- lib/ductwork/processes/health_check.rb
 - lib/ductwork/processes/job_worker.rb
 - lib/ductwork/processes/job_worker_runner.rb
 - lib/ductwork/processes/launcher.rb
@@ -169,7 +184,8 @@ files:
 - lib/ductwork/processes/process_supervisor_runner.rb
 - lib/ductwork/processes/thread_supervisor.rb
 - lib/ductwork/processes/thread_supervisor_runner.rb
-- lib/ductwork/row_locking_job_claim.rb
+- lib/ductwork/processes/worker_health_check.rb
+- lib/ductwork/row_locking_execution_claim.rb
 - lib/ductwork/running_context.rb
 - lib/ductwork/testing.rb
 - lib/ductwork/testing/helpers.rb
@@ -180,7 +196,11 @@ files:
 - lib/generators/ductwork/install/install_generator.rb
 - lib/generators/ductwork/install/templates/bin/ductwork
 - lib/generators/ductwork/install/templates/config/ductwork.yml
+- lib/generators/ductwork/install/templates/db/create_ductwork_advancements.rb
+- lib/generators/ductwork/install/templates/db/create_ductwork_attempts.rb
 - lib/generators/ductwork/install/templates/db/create_ductwork_availabilities.rb
+- lib/generators/ductwork/install/templates/db/create_ductwork_branch_links.rb
+- lib/generators/ductwork/install/templates/db/create_ductwork_branches.rb
 - lib/generators/ductwork/install/templates/db/create_ductwork_executions.rb
 - lib/generators/ductwork/install/templates/db/create_ductwork_jobs.rb
 - lib/generators/ductwork/install/templates/db/create_ductwork_pipelines.rb
@@ -188,10 +208,31 @@ files:
 - lib/generators/ductwork/install/templates/db/create_ductwork_results.rb
 - lib/generators/ductwork/install/templates/db/create_ductwork_runs.rb
 - lib/generators/ductwork/install/templates/db/create_ductwork_steps.rb
+- lib/generators/ductwork/install/templates/db/create_ductwork_transitions.rb
 - lib/generators/ductwork/install/templates/db/create_ductwork_tuples.rb
 - lib/generators/ductwork/update/USAGE
+- lib/generators/ductwork/update/templates/db/add_crash_count_to_ductwork_advancements.rb
+- lib/generators/ductwork/update/templates/db/add_crash_count_to_ductwork_executions.rb
+- lib/generators/ductwork/update/templates/db/add_indexes_to_ductwork_results.rb
+- lib/generators/ductwork/update/templates/db/add_indexes_to_ductwork_runs.rb
+- lib/generators/ductwork/update/templates/db/add_indexes_to_ductwork_transitions.rb
+- lib/generators/ductwork/update/templates/db/add_pipeline_started_index_to_ductwork_runs.rb
+- lib/generators/ductwork/update/templates/db/add_process_id_to_ductwork_executions.rb
+- lib/generators/ductwork/update/templates/db/add_role_to_ductwork_processes.rb
+- lib/generators/ductwork/update/templates/db/associate_branches_to_runs.rb
+- lib/generators/ductwork/update/templates/db/associate_steps_to_branches.rb
+- lib/generators/ductwork/update/templates/db/associate_steps_to_runs.rb
+- lib/generators/ductwork/update/templates/db/associate_tuples_to_runs.rb
+- lib/generators/ductwork/update/templates/db/backfill_branch_ids_on_steps.rb
+- lib/generators/ductwork/update/templates/db/create_ductwork_advancements.rb
+- lib/generators/ductwork/update/templates/db/create_ductwork_branch_links.rb
+- lib/generators/ductwork/update/templates/db/create_ductwork_branches.rb
+- lib/generators/ductwork/update/templates/db/create_ductwork_runs.rb
+- lib/generators/ductwork/update/templates/db/create_ductwork_transitions.rb
 - lib/generators/ductwork/update/templates/db/denormalize_pipeline_klass_on_availabilities.rb
 - lib/generators/ductwork/update/templates/db/migrate_tables_to_uuid_primary_key.rb
+- lib/generators/ductwork/update/templates/db/rename_runs_to_attempts.rb
+- lib/generators/ductwork/update/templates/db/update_process_associations.rb
 - lib/generators/ductwork/update/update_generator.rb
 homepage: https://github.com/ductwork/ductwork
 licenses:
@@ -200,7 +241,7 @@ metadata:
   homepage_uri: https://github.com/ductwork/ductwork
   changelog_uri: https://github.com/ductwork/ductwork/blob/main/CHANGELOG.md
   rubygems_mfa_required: 'true'
-  documentation_uri: https://docs.getductwork.io/
+  documentation_uri: https://www.getductwork.io/docs/
 rdoc_options: []
 require_paths:
 - app
@@ -217,7 +258,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.6
+rubygems_version: 4.0.16
 specification_version: 4
 summary: A Ruby pipeline framework
 test_files: []

data/lib/ductwork/optimistic_locking_job_claim.rb

@@ -1,88 +0,0 @@
-# frozen_string_literal: true
-
-module Ductwork
-  class OptimisticLockingJobClaim
-    def initialize(klass)
-      @id = nil
-      @job = nil
-      @klass = klass
-      @process_id = ::Process.pid
-    end
-
-    def latest
-      @id = latest_availability_id
-
-      if id.present?
-        rows_updated = claim_availability
-
-        if rows_updated == 1
-          Ductwork.logger.debug(
-            msg: "Job claimed",
-            role: :job_worker,
-            process_id: process_id,
-            availability_id: id
-          )
-
-          @job = find_job
-
-          update_state
-        else
-          Ductwork.logger.debug(
-            msg: "Did not claim job, avoided race condition",
-            role: :job_worker,
-            process_id: process_id,
-            availability_id: id
-          )
-        end
-      else
-        Ductwork.logger.debug(
-          msg: "No available job to claim",
-          role: :job_worker,
-          process_id: process_id,
-          pipeline: klass
-        )
-      end
-
-      job
-    end
-
-    private
-
-    attr_reader :id, :job, :klass, :process_id
-
-    def latest_availability_id
-      Ductwork::Availability
-        .where("ductwork_availabilities.started_at <= ?", Time.current)
-        .where(completed_at: nil, pipeline_klass: klass)
-        .order(:started_at)
-        .limit(1)
-        .pluck(:id)
-        .first
-    end
-
-    def claim_availability
-      Ductwork::Availability
-        .where(id: id, completed_at: nil)
-        .update_all(completed_at: Time.current, process_id: process_id)
-    end
-
-    def find_job
-      Ductwork::Job
-        .joins(executions: :availability)
-        .find_by!(ductwork_availabilities: { id:, process_id: })
-    end
-
-    def update_state
-      Ductwork::Record.transaction do
-        Ductwork::Execution
-          .joins(:availability)
-          .where(completed_at: nil)
-          .where(ductwork_availabilities: { id: })
-          .sole
-          .update!(process_id:)
-        job.step.in_progress!
-        job.step.pipeline.in_progress!
-      end
-    end
-  end
-end