ductwork 0.25.0 → 1.0.0

data/lib/generators/ductwork/update/update_generator.rb

@@ -10,6 +10,8 @@ module Ductwork
     source_root File.expand_path("templates", __dir__)
 
     def create_files
+      connection = Ductwork::Record.connection
+
       if Ductwork::Availability.column_names.exclude?("pipeline_klass")
         migration_template "db/denormalize_pipeline_klass_on_availabilities.rb",
                            "db/migrate/denormalize_pipeline_klass_on_availabilities.rb"
@@ -19,6 +21,103 @@ module Ductwork
         migration_template "db/migrate_tables_to_uuid_primary_key.rb",
                            "db/migrate/migrate_tables_to_uuid_primary_key.rb"
       end
+
+      if !connection.table_exists?(:ductwork_branches)
+        migration_template "db/create_ductwork_branches.rb",
+                           "db/migrate/create_ductwork_branches.rb"
+      end
+
+      if !connection.table_exists?(:ductwork_branch_links)
+        migration_template "db/create_ductwork_branch_links.rb",
+                           "db/migrate/create_ductwork_branch_links.rb"
+      end
+
+      if Ductwork::Step.column_names.exclude?("branch_id")
+        migration_template "db/associate_steps_to_branches.rb",
+                           "db/migrate/associate_steps_to_branches.rb"
+        migration_template "db/backfill_branch_ids_on_steps.rb",
+                           "db/migrate/backfill_branch_ids_on_steps.rb"
+      end
+
+      if !connection.table_exists?(:ductwork_transitions)
+        migration_template "db/create_ductwork_transitions.rb",
+                           "db/migrate/create_ductwork_transitions.rb"
+      end
+
+      if !connection.table_exists?(:ductwork_advancements)
+        migration_template "db/create_ductwork_advancements.rb",
+                           "db/migrate/create_ductwork_advancements.rb"
+      end
+
+      if Ductwork::Execution.column_names.include?("process_id")
+        migration_template "db/update_process_associations.rb",
+                           "db/migrate/update_process_associations.rb"
+      end
+
+      if connection.table_exists?(:ductwork_runs)
+        migration_template "db/rename_runs_to_attempts.rb",
+                           "db/migrate/rename_runs_to_attempts.rb"
+      end
+
+      if !connection.table_exists?(:ductwork_runs)
+        migration_template "db/create_ductwork_runs.rb",
+                           "db/migrate/create_ductwork_runs.rb"
+      end
+
+      if Ductwork::Branch.column_names.include?("pipeline_id")
+        migration_template "db/associate_branches_to_runs.rb",
+                           "db/migrate/associate_branches_to_runs.rb"
+      end
+
+      if Ductwork::Step.column_names.include?("pipeline_id")
+        migration_template "db/associate_steps_to_runs.rb",
+                           "db/migrate/associate_steps_to_runs.rb"
+      end
+
+      if Ductwork::Tuple.column_names.include?("pipeline_id")
+        migration_template "db/associate_tuples_to_runs.rb",
+                           "db/migrate/associate_tuples_to_runs.rb"
+      end
+
+      if Ductwork::Execution.column_names.exclude?("crash_count")
+        migration_template "db/add_crash_count_to_ductwork_executions.rb",
+                           "db/migrate/add_crash_count_to_ductwork_executions.rb"
+      end
+
+      if Ductwork::Advancement.column_names.exclude?("crash_count")
+        migration_template "db/add_crash_count_to_ductwork_advancements.rb",
+                           "db/migrate/add_crash_count_to_ductwork_advancements.rb"
+      end
+
+      if Ductwork::Execution.column_names.exclude?("process_id")
+        migration_template "db/add_process_id_to_ductwork_executions.rb",
+                           "db/migrate/add_process_id_to_ductwork_executions.rb"
+      end
+
+      if Ductwork::Process.column_names.exclude?("role")
+        migration_template "db/add_role_to_ductwork_processes.rb",
+                           "db/migrate/add_role_to_ductwork_processes.rb"
+      end
+
+      if !connection.index_exists?(:ductwork_results, %i[result_type created_at])
+        migration_template "db/add_indexes_to_ductwork_results.rb",
+                           "db/migrate/add_indexes_to_ductwork_results.rb"
+      end
+
+      if !connection.index_exists?(:ductwork_runs, :started_at)
+        migration_template "db/add_indexes_to_ductwork_runs.rb",
+                           "db/migrate/add_indexes_to_ductwork_runs.rb"
+      end
+
+      if !connection.index_exists?(:ductwork_runs, %i[pipeline_id started_at])
+        migration_template "db/add_pipeline_started_index_to_ductwork_runs.rb",
+                           "db/migrate/add_pipeline_started_index_to_ductwork_runs.rb"
+      end
+
+      if !connection.index_exists?(:ductwork_transitions, name: "index_ductwork_transitions_on_latest_open")
+        migration_template "db/add_indexes_to_ductwork_transitions.rb",
+                           "db/migrate/add_indexes_to_ductwork_transitions.rb"
+      end
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ductwork
 version: !ruby/object:Gem::Version
-  version: 0.25.0
+  version: 1.0.0
 platform: ruby
 authors:
 - Tyler Ewing
@@ -111,12 +111,17 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".claude/skills/audit-clock-drift/SKILL.md"
+- ".claude/skills/audit-database-indexes/SKILL.md"
+- ".claude/skills/audit-database-support/SKILL.md"
+- ".claude/skills/audit-durability/SKILL.md"
 - ".saturnci/Dockerfile"
 - ".saturnci/database.yml"
 - ".saturnci/docker-compose.yml"
 - ".saturnci/pre.sh"
 - CHANGELOG-PRO.md
 - CHANGELOG.md
+- CLAUDE.md
 - LICENSE.txt
 - PRO-LICENSE.txt
 - README.md
@@ -139,17 +144,24 @@ files:
 - app/views/layouts/ductwork/application.html.erb
 - config/routes.rb
 - lib/ductwork.rb
+- lib/ductwork/branch_claim.rb
 - lib/ductwork/cli.rb
 - lib/ductwork/configuration.rb
 - lib/ductwork/context.rb
+- lib/ductwork/database_clock.rb
 - lib/ductwork/dsl/branch_builder.rb
 - lib/ductwork/dsl/definition_builder.rb
 - lib/ductwork/engine.rb
-- lib/ductwork/job_claim.rb
+- lib/ductwork/execution_claim.rb
+- lib/ductwork/fault_injection.rb
 - lib/ductwork/machine_identifier.rb
 - lib/ductwork/migration.rb
 - lib/ductwork/migration_helper.rb
+- lib/ductwork/models/advancement.rb
+- lib/ductwork/models/attempt.rb
 - lib/ductwork/models/availability.rb
+- lib/ductwork/models/branch.rb
+- lib/ductwork/models/branch_link.rb
 - lib/ductwork/models/execution.rb
 - lib/ductwork/models/job.rb
 - lib/ductwork/models/pipeline.rb
@@ -158,8 +170,11 @@ files:
 - lib/ductwork/models/result.rb
 - lib/ductwork/models/run.rb
 - lib/ductwork/models/step.rb
+- lib/ductwork/models/transition.rb
 - lib/ductwork/models/tuple.rb
-- lib/ductwork/optimistic_locking_job_claim.rb
+- lib/ductwork/models/workflow.rb
+- lib/ductwork/optimistic_locking_execution_claim.rb
+- lib/ductwork/processes/health_check.rb
 - lib/ductwork/processes/job_worker.rb
 - lib/ductwork/processes/job_worker_runner.rb
 - lib/ductwork/processes/launcher.rb
@@ -169,7 +184,8 @@ files:
 - lib/ductwork/processes/process_supervisor_runner.rb
 - lib/ductwork/processes/thread_supervisor.rb
 - lib/ductwork/processes/thread_supervisor_runner.rb
-- lib/ductwork/row_locking_job_claim.rb
+- lib/ductwork/processes/worker_health_check.rb
+- lib/ductwork/row_locking_execution_claim.rb
 - lib/ductwork/running_context.rb
 - lib/ductwork/testing.rb
 - lib/ductwork/testing/helpers.rb
@@ -180,7 +196,11 @@ files:
 - lib/generators/ductwork/install/install_generator.rb
 - lib/generators/ductwork/install/templates/bin/ductwork
 - lib/generators/ductwork/install/templates/config/ductwork.yml
+- lib/generators/ductwork/install/templates/db/create_ductwork_advancements.rb
+- lib/generators/ductwork/install/templates/db/create_ductwork_attempts.rb
 - lib/generators/ductwork/install/templates/db/create_ductwork_availabilities.rb
+- lib/generators/ductwork/install/templates/db/create_ductwork_branch_links.rb
+- lib/generators/ductwork/install/templates/db/create_ductwork_branches.rb
 - lib/generators/ductwork/install/templates/db/create_ductwork_executions.rb
 - lib/generators/ductwork/install/templates/db/create_ductwork_jobs.rb
 - lib/generators/ductwork/install/templates/db/create_ductwork_pipelines.rb
@@ -188,10 +208,31 @@ files:
 - lib/generators/ductwork/install/templates/db/create_ductwork_results.rb
 - lib/generators/ductwork/install/templates/db/create_ductwork_runs.rb
 - lib/generators/ductwork/install/templates/db/create_ductwork_steps.rb
+- lib/generators/ductwork/install/templates/db/create_ductwork_transitions.rb
 - lib/generators/ductwork/install/templates/db/create_ductwork_tuples.rb
 - lib/generators/ductwork/update/USAGE
+- lib/generators/ductwork/update/templates/db/add_crash_count_to_ductwork_advancements.rb
+- lib/generators/ductwork/update/templates/db/add_crash_count_to_ductwork_executions.rb
+- lib/generators/ductwork/update/templates/db/add_indexes_to_ductwork_results.rb
+- lib/generators/ductwork/update/templates/db/add_indexes_to_ductwork_runs.rb
+- lib/generators/ductwork/update/templates/db/add_indexes_to_ductwork_transitions.rb
+- lib/generators/ductwork/update/templates/db/add_pipeline_started_index_to_ductwork_runs.rb
+- lib/generators/ductwork/update/templates/db/add_process_id_to_ductwork_executions.rb
+- lib/generators/ductwork/update/templates/db/add_role_to_ductwork_processes.rb
+- lib/generators/ductwork/update/templates/db/associate_branches_to_runs.rb
+- lib/generators/ductwork/update/templates/db/associate_steps_to_branches.rb
+- lib/generators/ductwork/update/templates/db/associate_steps_to_runs.rb
+- lib/generators/ductwork/update/templates/db/associate_tuples_to_runs.rb
+- lib/generators/ductwork/update/templates/db/backfill_branch_ids_on_steps.rb
+- lib/generators/ductwork/update/templates/db/create_ductwork_advancements.rb
+- lib/generators/ductwork/update/templates/db/create_ductwork_branch_links.rb
+- lib/generators/ductwork/update/templates/db/create_ductwork_branches.rb
+- lib/generators/ductwork/update/templates/db/create_ductwork_runs.rb
+- lib/generators/ductwork/update/templates/db/create_ductwork_transitions.rb
 - lib/generators/ductwork/update/templates/db/denormalize_pipeline_klass_on_availabilities.rb
 - lib/generators/ductwork/update/templates/db/migrate_tables_to_uuid_primary_key.rb
+- lib/generators/ductwork/update/templates/db/rename_runs_to_attempts.rb
+- lib/generators/ductwork/update/templates/db/update_process_associations.rb
 - lib/generators/ductwork/update/update_generator.rb
 homepage: https://github.com/ductwork/ductwork
 licenses:
@@ -200,7 +241,7 @@ metadata:
   homepage_uri: https://github.com/ductwork/ductwork
   changelog_uri: https://github.com/ductwork/ductwork/blob/main/CHANGELOG.md
   rubygems_mfa_required: 'true'
-  documentation_uri: https://docs.getductwork.io/
+  documentation_uri: https://www.getductwork.io/docs/
 rdoc_options: []
 require_paths:
 - app
@@ -217,7 +258,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.6
+rubygems_version: 4.0.16
 specification_version: 4
 summary: A Ruby pipeline framework
 test_files: []

data/lib/ductwork/optimistic_locking_job_claim.rb

@@ -1,88 +0,0 @@
-# frozen_string_literal: true
-
-module Ductwork
-  class OptimisticLockingJobClaim
-    def initialize(klass)
-      @id = nil
-      @job = nil
-      @klass = klass
-      @process_id = ::Process.pid
-    end
-
-    def latest
-      @id = latest_availability_id
-
-      if id.present?
-        rows_updated = claim_availability
-
-        if rows_updated == 1
-          Ductwork.logger.debug(
-            msg: "Job claimed",
-            role: :job_worker,
-            process_id: process_id,
-            availability_id: id
-          )
-
-          @job = find_job
-
-          update_state
-        else
-          Ductwork.logger.debug(
-            msg: "Did not claim job, avoided race condition",
-            role: :job_worker,
-            process_id: process_id,
-            availability_id: id
-          )
-        end
-      else
-        Ductwork.logger.debug(
-          msg: "No available job to claim",
-          role: :job_worker,
-          process_id: process_id,
-          pipeline: klass
-        )
-      end
-
-      job
-    end
-
-    private
-
-    attr_reader :id, :job, :klass, :process_id
-
-    def latest_availability_id
-      Ductwork::Availability
-        .where("ductwork_availabilities.started_at <= ?", Time.current)
-        .where(completed_at: nil, pipeline_klass: klass)
-        .order(:started_at)
-        .limit(1)
-        .pluck(:id)
-        .first
-    end
-
-    def claim_availability
-      Ductwork::Availability
-        .where(id: id, completed_at: nil)
-        .update_all(completed_at: Time.current, process_id: process_id)
-    end
-
-    def find_job
-      Ductwork::Job
-        .joins(executions: :availability)
-        .find_by!(ductwork_availabilities: { id:, process_id: })
-    end
-
-    def update_state
-      Ductwork::Record.transaction do
-        Ductwork::Execution
-          .joins(:availability)
-          .where(completed_at: nil)
-          .where(ductwork_availabilities: { id: })
-          .sole
-          .update!(process_id:)
-        job.step.in_progress!
-        job.step.pipeline.in_progress!
-      end
-    end
-  end
-end

data/lib/ductwork/row_locking_job_claim.rb

@@ -1,75 +0,0 @@
-# frozen_string_literal: true
-
-module Ductwork
-  class RowLockingJobClaim
-    def initialize(klass)
-      @id = nil
-      @job = nil
-      @klass = klass
-      @process_id = ::Process.pid
-    end
-
-    def latest
-      rows_updated = attempt_availability_claim
-
-      if rows_updated == 1
-        Ductwork.logger.debug(
-          msg: "Job claimed",
-          role: :job_worker,
-          process_id: process_id,
-          availability_id: id
-        )
-        update_state
-      else
-        Ductwork.logger.debug(
-          msg: "No available job to claim",
-          role: :job_worker,
-          process_id: process_id,
-          pipeline: klass
-        )
-      end
-
-      job
-    end
-
-    private
-
-    attr_reader :id, :job, :klass, :process_id
-
-    def attempt_availability_claim
-      Ductwork::Record.transaction do
-        @id = Ductwork::Availability
-              .where("ductwork_availabilities.started_at <= ?", Time.current)
-              .where(completed_at: nil, pipeline_klass: klass)
-              .order(:started_at)
-              .lock("FOR UPDATE SKIP LOCKED")
-              .limit(1)
-              .ids
-              .first
-
-        if id.present?
-          Ductwork::Availability
-            .where(id: id, completed_at: nil)
-            .update_all(completed_at: Time.current, process_id: process_id)
-        else
-          0
-        end
-      end
-    end
-
-    def update_state
-      Ductwork::Record.transaction do
-        execution = Ductwork::Execution
-                    .joins(:availability)
-                    .where(completed_at: nil)
-                    .where(ductwork_availabilities: { id: })
-                    .sole
-        @job = execution.job
-
-        execution.update!(process_id:)
-        job.step.in_progress!
-        job.step.pipeline.in_progress!
-      end
-    end
-  end
-end