ductwork 1.2.0 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 402e7de6f5703214be13a443e54a5b74f113500a22ad42727db969a7bcb39b69
-  data.tar.gz: 9889dd598476068548da0f6cf9e63cf94f72e6e124b837d3abe2994de6a2770f
+  metadata.gz: 94c5c2250e4a96fee3aaeac379ec1ee58ac85e6aa5c2f0ccffc67c9e66d4499d
+  data.tar.gz: ac6aac4135803b6d0712db4b4491202b64280a506a3101baba6563f549e5ee9d
 SHA512:
-  metadata.gz: 2eb520b852ff56484b7afd3f89a525f4e444ba2a04a03ff3d97152bdad625cf5d393a2500a72bcdd2ca8a4122f726caaadcd8aa4c626e70e8ba6397ec84bd5df
-  data.tar.gz: 8667ad5225297be4b5c98383fbe0ad3766e18a33391912d110e9aae94bcaa5c9949089fb2c8f4eb0d6f0f17c8588626e82e6c7abe90aa0293a2081dffe8aef82
+  metadata.gz: d7305c2c9a26693cfa92dce562016420aafe4d745cb3efb4450b5192a8c036d7db5d96023b71979bd15c817059c2f8f2933f974a441066ec891cb2132d802c99
+  data.tar.gz: 54dfdadfcafe3e1e4a214f5d11599e87bc3b114387702b426aad5c146ec7ca2042c42e0fa001cd0d598e11b515a342f8f01d5ac3b96a4f6b982d6579f93b50a7

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Ductwork Changelog
 
+## [1.3.0]
+
+- feat: add more `MigrationHelper` methods for swapping and updating the availability claim index
+- fix: resolve the database adapter from the configuration (`Ductwork::Record.adapter`) instead of `Ductwork::Record.connection.adapter_name`, and take the database clock's reading through `connection_pool.with_connection` - `ActiveRecord::Base#connection` is soft deprecated and takes a *sticky* lease, so `ConnectionPool#with_connection` skips its own checkin (it releases only `unless lease.sticky`) and every advancer or worker thread that touched the claim hot path pinned a pool slot for its lifetime; the Rails executor checked those back in per unit of work so long-running processes did not leak, but a host app setting `config.active_record.permanent_connection_checkout = :disallowed` would have raised, and `:deprecated` emitted a warning per claim
+- fix: `ExecutionClaim` now selects row locking on Oracle as it always intended - the predicate tests `/oracle_enhanced/i` but was matched against `connection.adapter_name.downcase`, which renders as `oracleenhanced` with no underscore, so Oracle silently fell through to optimistic locking; it is matched against the configured adapter (`oracle_enhanced`) now. Oracle is not covered by CI
+- fix: make `Ductwork::Processes::ProcessSupervisor#shutdown` idempotent and serialize it on a mutex - `run` calls `shutdown` itself once its loop sees the running context go down, so a caller that stops the supervisor from another thread raced the supervisor's own call and both ran `reap_own_process_record!` at the same time
+
 ## [1.2.0]
 
 - feat: add `add_availability_claim_index` and `remove_availability_claim_index` to `Ductwork::MigrationHelper`, which own the adapter-specific shape and the name of the availability claim index

data/lib/ductwork/context.rb

@@ -26,7 +26,7 @@ module Ductwork
         first_set_at: Time.current,
         last_set_at: Time.current,
       }
-      opts = if Ductwork::Tuple.connection.adapter_name.match?(/mysql|trilogy/i)
+      opts = if Ductwork::Tuple.adapter.match?(/mysql|trilogy/i)
                {}
              else
                { unique_by: %i[run_id key] }

data/lib/ductwork/database_clock.rb

@@ -1,10 +1,6 @@
 # frozen_string_literal: true
 
 module Ductwork
-  # NOTE: these are SQL fragments that resolve against the database server's
-  # clock instead of the calling Ruby process's clock. use them in `WHERE`
-  # clauses that compare between a stored timestamp and "now" so that NTP
-  # drift between hosts cannot make healthy work look stale (or vice versa)
   class DatabaseClock
     def self.ago_sql(column, interval)
       new(column).ago_sql(interval)
@@ -15,7 +11,7 @@ module Ductwork
     end
 
     def self.now
-      adapter = Ductwork::Record.connection.adapter_name.downcase
+      adapter = Ductwork::Record.adapter
       sql =
         case adapter
         when /postgresql|cockroach/i
@@ -29,7 +25,9 @@ module Ductwork
         else
           raise NotImplementedError, "Database clock does not support adapter #{adapter}"
         end
-      raw = Ductwork::Record.connection.select_value(sql)
+      raw = Ductwork::Record.connection_pool.with_connection do |conn|
+        conn.select_value(sql)
+      end
 
       case raw
       when ::Time, ::DateTime
@@ -41,7 +39,7 @@ module Ductwork
     end
 
     def initialize(column)
-      @adapter = Ductwork::Record.connection.adapter_name.downcase
+      @adapter = Ductwork::Record.adapter
       @column = column
     end
 

data/lib/ductwork/execution_claim.rb

@@ -5,7 +5,7 @@ module Ductwork
     def initialize(klass, owner_process_id)
       @klass = klass
       @owner_process_id = owner_process_id
-      @adapter = Ductwork::Record.connection.adapter_name.downcase
+      @adapter = Ductwork::Record.adapter
     end
 
     def latest

data/lib/ductwork/migration_helper.rb

@@ -2,29 +2,113 @@
 
 module Ductwork
   module MigrationHelper
+    class OpenTransactionError < StandardError; end
+
     AVAILABILITY_CLAIM_INDEX_NAME = "index_ductwork_availabilities_on_claim_latest"
+    AVAILABILITY_CLAIM_INDEX_NAME_V2 = "index_ductwork_availabilities_on_claim_latest_v2"
+    AVAILABILITY_CLAIM_INDEX_NAMES = [
+      AVAILABILITY_CLAIM_INDEX_NAME,
+      AVAILABILITY_CLAIM_INDEX_NAME_V2,
+    ].freeze
 
-    def add_availability_claim_index(extra_columns: [], where_extra: nil)
+    def add_availability_claim_index(
+      extra_columns: [],
+      where_extra: nil,
+      name: AVAILABILITY_CLAIM_INDEX_NAME,
+      online: false
+    )
       if mysql?
-        add_index :ductwork_availabilities,
-                  [:pipeline_klass, :completed_at, *extra_columns, :started_at],
-                  name: AVAILABILITY_CLAIM_INDEX_NAME
+        columns = [:pipeline_klass, :completed_at, *extra_columns, :started_at]
+        options = { name: }
       else
-        predicate = ["completed_at IS NULL", where_extra].compact.join(" AND ")
+        columns = %i[pipeline_klass started_at]
+        options = {
+          name: name,
+          where: ["completed_at IS NULL", where_extra].compact.join(" AND "),
+        }
+      end
 
-        add_index :ductwork_availabilities,
-                  %i[pipeline_klass started_at],
-                  name: AVAILABILITY_CLAIM_INDEX_NAME,
-                  where: predicate
+      if online
+        add_index_online :ductwork_availabilities, columns, **options
+      else
+        add_index :ductwork_availabilities, columns, **options
       end
     end
 
-    def remove_availability_claim_index
-      remove_index :ductwork_availabilities,
-                   name: AVAILABILITY_CLAIM_INDEX_NAME,
+    def remove_availability_claim_index(name: AVAILABILITY_CLAIM_INDEX_NAME)
+      remove_index :ductwork_availabilities, name: name, if_exists: true
+    end
+
+    def swap_availability_claim_index(to:, extra_columns: [], where_extra: nil)
+      unless AVAILABILITY_CLAIM_INDEX_NAMES.include?(to)
+        raise ArgumentError, "unrecognized availability claim index name: #{to}"
+      end
+
+      add_availability_claim_index(
+        extra_columns: extra_columns,
+        where_extra: where_extra,
+        name: to,
+        online: true
+      )
+
+      (AVAILABILITY_CLAIM_INDEX_NAMES - [to]).each do |stale|
+        remove_index_online :ductwork_availabilities, name: stale
+      end
+    end
+
+    def add_index_online(table_name, column_names, **options)
+      ensure_ddl_transaction_disabled!
+
+      name = options[:name] || connection.index_name(table_name, column_names)
+
+      drop_invalid_index(table_name, name)
+
+      return if connection.index_name_exists?(table_name, name)
+
+      full_options = options.merge(name:)
+      full_options[:algorithm] = :concurrently if postgresql?
+
+      add_index table_name, column_names, **full_options
+    end
+
+    def remove_index_online(table_name, name:)
+      ensure_ddl_transaction_disabled!
+
+      return unless connection.index_name_exists?(table_name, name)
+
+      options = { name: name, if_exists: true }
+      options[:algorithm] = :concurrently if postgresql?
+
+      remove_index table_name, **options
+    end
+
+    def drop_invalid_index(table_name, index_name)
+      return unless postgresql?
+
+      invalid = connection.select_value(<<~SQL.squish)
+        SELECT 1 FROM pg_index
+        WHERE indexrelid = to_regclass(#{connection.quote(index_name)})
+          AND NOT indisvalid
+      SQL
+
+      return if invalid.blank?
+
+      remove_index table_name,
+                   name: index_name,
+                   algorithm: :concurrently,
                    if_exists: true
     end
 
+    def ensure_ddl_transaction_disabled!
+      return unless postgresql?
+      return unless connection.transaction_open?
+
+      raise OpenTransactionError, <<~MESSAGE.squish
+        online index changes cannot run inside a transaction on PostgreSQL;
+        declare `disable_ddl_transaction!` in the migration
+      MESSAGE
+    end
+
     def create_ductwork_table(table_name, &block)
       if postgresql?
         create_table table_name, id: :uuid, &block

data/lib/ductwork/models/record.rb

@@ -14,6 +14,10 @@ module Ductwork
       "ductwork_"
     end
 
+    def self.adapter
+      connection_db_config.adapter.to_s.downcase
+    end
+
     private
 
     def generate_uuid_v7

data/lib/ductwork/models/run.rb

@@ -118,7 +118,7 @@ module Ductwork
     # relies on the advancer retrying a deadlock victim; SQLite has no row-level
     # locks, so `lock!` is a no-op there and the deadlock cannot occur.
     def lock_for_terminal_resolution!
-      if Ductwork::Record.connection.adapter_name.downcase.match?(/postgresql|cockroach/)
+      if Ductwork::Record.adapter.match?(/postgresql|cockroach/)
         lock!("FOR NO KEY UPDATE")
       else
         lock!

data/lib/ductwork/processes/process_supervisor.rb

@@ -3,8 +3,6 @@
 module Ductwork
   module Processes
     class ProcessSupervisor # rubocop:todo Metrics/ClassLength
-      # Upper bound on how long sig_kill_process will block the supervisor
-      # loop waiting for a KILL'd child to be reaped (see sig_kill_process).
       KILL_REAP_TIMEOUT = 5 # seconds
       KILL_REAP_INTERVAL = 0.1
 
@@ -13,6 +11,8 @@ module Ductwork
       def initialize
         @running_context = Ductwork::RunningContext.new
         @workers = []
+        @shutdown_mutex = Mutex.new
+        @shutdown_complete = false
 
         run_hooks_for(:start)
 
@@ -53,22 +53,28 @@ module Ductwork
       end
 
       def shutdown
-        running_context.shutdown!
-        Ductwork.logger.debug(
-          msg: "Beginning shutdown",
-          role: :process_supervisor
-        )
+        shutdown_mutex.synchronize do
+          next if shutdown_complete
+
+          running_context.shutdown!
+          Ductwork.logger.debug(
+            msg: "Beginning shutdown",
+            role: :process_supervisor
+          )
 
-        terminate_gracefully
-        wait_for_workers_to_exit
-        terminate_immediately
-        reap_own_process_record!
-        run_hooks_for(:stop)
+          terminate_gracefully
+          wait_for_workers_to_exit
+          terminate_immediately
+          reap_own_process_record!
+          run_hooks_for(:stop)
+
+          @shutdown_complete = true
+        end
       end
 
       private
 
-      attr_reader :running_context
+      attr_reader :running_context, :shutdown_mutex, :shutdown_complete
 
       def adopt_or_create_process!
         Ductwork.wrap_with_app_executor do

data/lib/ductwork/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Ductwork
-  VERSION = "1.2.0"
+  VERSION = "1.3.0"
 end

data/lib/generators/ductwork/update/update_generator.rb

@@ -10,113 +10,113 @@ module Ductwork
     source_root File.expand_path("templates", __dir__)
 
     def create_files
-      connection = Ductwork::Record.connection
-
-      if Ductwork::Availability.column_names.exclude?("pipeline_klass")
-        migration_template "db/denormalize_pipeline_klass_on_availabilities.rb",
-                           "db/migrate/denormalize_pipeline_klass_on_availabilities.rb"
-      end
-
-      if Ductwork::Pipeline.column_for_attribute("id").type != :uuid
-        migration_template "db/migrate_tables_to_uuid_primary_key.rb",
-                           "db/migrate/migrate_tables_to_uuid_primary_key.rb"
-      end
-
-      if !connection.table_exists?(:ductwork_branches)
-        migration_template "db/create_ductwork_branches.rb",
-                           "db/migrate/create_ductwork_branches.rb"
-      end
-
-      if !connection.table_exists?(:ductwork_branch_links)
-        migration_template "db/create_ductwork_branch_links.rb",
-                           "db/migrate/create_ductwork_branch_links.rb"
-      end
-
-      if Ductwork::Step.column_names.exclude?("branch_id")
-        migration_template "db/associate_steps_to_branches.rb",
-                           "db/migrate/associate_steps_to_branches.rb"
-        migration_template "db/backfill_branch_ids_on_steps.rb",
-                           "db/migrate/backfill_branch_ids_on_steps.rb"
-      end
-
-      if !connection.table_exists?(:ductwork_transitions)
-        migration_template "db/create_ductwork_transitions.rb",
-                           "db/migrate/create_ductwork_transitions.rb"
-      end
-
-      if !connection.table_exists?(:ductwork_advancements)
-        migration_template "db/create_ductwork_advancements.rb",
-                           "db/migrate/create_ductwork_advancements.rb"
-      end
-
-      if Ductwork::Execution.column_names.include?("process_id")
-        migration_template "db/update_process_associations.rb",
-                           "db/migrate/update_process_associations.rb"
-      end
-
-      if connection.table_exists?(:ductwork_runs)
-        migration_template "db/rename_runs_to_attempts.rb",
-                           "db/migrate/rename_runs_to_attempts.rb"
-      end
-
-      if !connection.table_exists?(:ductwork_runs)
-        migration_template "db/create_ductwork_runs.rb",
-                           "db/migrate/create_ductwork_runs.rb"
-      end
-
-      if Ductwork::Branch.column_names.include?("pipeline_id")
-        migration_template "db/associate_branches_to_runs.rb",
-                           "db/migrate/associate_branches_to_runs.rb"
-      end
-
-      if Ductwork::Step.column_names.include?("pipeline_id")
-        migration_template "db/associate_steps_to_runs.rb",
-                           "db/migrate/associate_steps_to_runs.rb"
-      end
-
-      if Ductwork::Tuple.column_names.include?("pipeline_id")
-        migration_template "db/associate_tuples_to_runs.rb",
-                           "db/migrate/associate_tuples_to_runs.rb"
-      end
-
-      if Ductwork::Execution.column_names.exclude?("crash_count")
-        migration_template "db/add_crash_count_to_ductwork_executions.rb",
-                           "db/migrate/add_crash_count_to_ductwork_executions.rb"
-      end
-
-      if Ductwork::Advancement.column_names.exclude?("crash_count")
-        migration_template "db/add_crash_count_to_ductwork_advancements.rb",
-                           "db/migrate/add_crash_count_to_ductwork_advancements.rb"
-      end
-
-      if Ductwork::Execution.column_names.exclude?("process_id")
-        migration_template "db/add_process_id_to_ductwork_executions.rb",
-                           "db/migrate/add_process_id_to_ductwork_executions.rb"
-      end
-
-      if Ductwork::Process.column_names.exclude?("role")
-        migration_template "db/add_role_to_ductwork_processes.rb",
-                           "db/migrate/add_role_to_ductwork_processes.rb"
-      end
-
-      if !connection.index_exists?(:ductwork_results, %i[result_type created_at])
-        migration_template "db/add_indexes_to_ductwork_results.rb",
-                           "db/migrate/add_indexes_to_ductwork_results.rb"
-      end
-
-      if !connection.index_exists?(:ductwork_runs, :started_at)
-        migration_template "db/add_indexes_to_ductwork_runs.rb",
-                           "db/migrate/add_indexes_to_ductwork_runs.rb"
-      end
-
-      if !connection.index_exists?(:ductwork_runs, %i[pipeline_id started_at])
-        migration_template "db/add_pipeline_started_index_to_ductwork_runs.rb",
-                           "db/migrate/add_pipeline_started_index_to_ductwork_runs.rb"
-      end
-
-      if !connection.index_exists?(:ductwork_transitions, name: "index_ductwork_transitions_on_latest_open")
-        migration_template "db/add_indexes_to_ductwork_transitions.rb",
-                           "db/migrate/add_indexes_to_ductwork_transitions.rb"
+      Ductwork::Record.connection_pool.with_connection do |connection|
+        if Ductwork::Availability.column_names.exclude?("pipeline_klass")
+          migration_template "db/denormalize_pipeline_klass_on_availabilities.rb",
+                             "db/migrate/denormalize_pipeline_klass_on_availabilities.rb"
+        end
+
+        if Ductwork::Pipeline.column_for_attribute("id").type != :uuid
+          migration_template "db/migrate_tables_to_uuid_primary_key.rb",
+                             "db/migrate/migrate_tables_to_uuid_primary_key.rb"
+        end
+
+        if !connection.table_exists?(:ductwork_branches)
+          migration_template "db/create_ductwork_branches.rb",
+                             "db/migrate/create_ductwork_branches.rb"
+        end
+
+        if !connection.table_exists?(:ductwork_branch_links)
+          migration_template "db/create_ductwork_branch_links.rb",
+                             "db/migrate/create_ductwork_branch_links.rb"
+        end
+
+        if Ductwork::Step.column_names.exclude?("branch_id")
+          migration_template "db/associate_steps_to_branches.rb",
+                             "db/migrate/associate_steps_to_branches.rb"
+          migration_template "db/backfill_branch_ids_on_steps.rb",
+                             "db/migrate/backfill_branch_ids_on_steps.rb"
+        end
+
+        if !connection.table_exists?(:ductwork_transitions)
+          migration_template "db/create_ductwork_transitions.rb",
+                             "db/migrate/create_ductwork_transitions.rb"
+        end
+
+        if !connection.table_exists?(:ductwork_advancements)
+          migration_template "db/create_ductwork_advancements.rb",
+                             "db/migrate/create_ductwork_advancements.rb"
+        end
+
+        if Ductwork::Execution.column_names.include?("process_id")
+          migration_template "db/update_process_associations.rb",
+                             "db/migrate/update_process_associations.rb"
+        end
+
+        if connection.table_exists?(:ductwork_runs)
+          migration_template "db/rename_runs_to_attempts.rb",
+                             "db/migrate/rename_runs_to_attempts.rb"
+        end
+
+        if !connection.table_exists?(:ductwork_runs)
+          migration_template "db/create_ductwork_runs.rb",
+                             "db/migrate/create_ductwork_runs.rb"
+        end
+
+        if Ductwork::Branch.column_names.include?("pipeline_id")
+          migration_template "db/associate_branches_to_runs.rb",
+                             "db/migrate/associate_branches_to_runs.rb"
+        end
+
+        if Ductwork::Step.column_names.include?("pipeline_id")
+          migration_template "db/associate_steps_to_runs.rb",
+                             "db/migrate/associate_steps_to_runs.rb"
+        end
+
+        if Ductwork::Tuple.column_names.include?("pipeline_id")
+          migration_template "db/associate_tuples_to_runs.rb",
+                             "db/migrate/associate_tuples_to_runs.rb"
+        end
+
+        if Ductwork::Execution.column_names.exclude?("crash_count")
+          migration_template "db/add_crash_count_to_ductwork_executions.rb",
+                             "db/migrate/add_crash_count_to_ductwork_executions.rb"
+        end
+
+        if Ductwork::Advancement.column_names.exclude?("crash_count")
+          migration_template "db/add_crash_count_to_ductwork_advancements.rb",
+                             "db/migrate/add_crash_count_to_ductwork_advancements.rb"
+        end
+
+        if Ductwork::Execution.column_names.exclude?("process_id")
+          migration_template "db/add_process_id_to_ductwork_executions.rb",
+                             "db/migrate/add_process_id_to_ductwork_executions.rb"
+        end
+
+        if Ductwork::Process.column_names.exclude?("role")
+          migration_template "db/add_role_to_ductwork_processes.rb",
+                             "db/migrate/add_role_to_ductwork_processes.rb"
+        end
+
+        if !connection.index_exists?(:ductwork_results, %i[result_type created_at])
+          migration_template "db/add_indexes_to_ductwork_results.rb",
+                             "db/migrate/add_indexes_to_ductwork_results.rb"
+        end
+
+        if !connection.index_exists?(:ductwork_runs, :started_at)
+          migration_template "db/add_indexes_to_ductwork_runs.rb",
+                             "db/migrate/add_indexes_to_ductwork_runs.rb"
+        end
+
+        if !connection.index_exists?(:ductwork_runs, %i[pipeline_id started_at])
+          migration_template "db/add_pipeline_started_index_to_ductwork_runs.rb",
+                             "db/migrate/add_pipeline_started_index_to_ductwork_runs.rb"
+        end
+
+        if !connection.index_exists?(:ductwork_transitions, name: "index_ductwork_transitions_on_latest_open")
+          migration_template "db/add_indexes_to_ductwork_transitions.rb",
+                             "db/migrate/add_indexes_to_ductwork_transitions.rb"
+        end
       end
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ductwork
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.3.0
 platform: ruby
 authors:
 - Tyler Ewing