ductwork 0.20.2 → 0.21.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 66eeccc14571dbd9d0970843c38da7601f090a9fdcc3b1bbd3b0cad083d37197
-  data.tar.gz: f277974af71bcbbbd33f1ad1398e79acac2f88c2c0afbbf0143b019e16e65d48
+  metadata.gz: 2029d11f63583fd4b5cb774bf3c249b30f933700852ea286c56ae9f6262aa584
+  data.tar.gz: 5c32000bd3409f54af3fc43aac6296e7fad458d8ae74336b4edb84310b22bf2a
 SHA512:
-  metadata.gz: 9ebc366c05370b908eb35789aa096733cd2171023843b2472602b307182d063a22980559ffed3278ef2ce5a6a98ad41a4d2d9221d216ee4fc3259fedf0319e99
-  data.tar.gz: 48211addcab05625ce589767370df9c28c7e1a4c44d2d69a71c4ec20906d371844af92ff4c5c937a5300fc490a6cca758fce6896bbc7e80b359d3a5105ec84bc
+  metadata.gz: 543e4ab981017a51580b7b3a2a750bbd52a1de313ec27556d1cbb72045b2f6a8fbe743e15585cc56e2682914f0040c91b31a1328be86097f1c08b0dadfcdc73a
+  data.tar.gz: 88da2d0dfa602e6850aacbe4fa20a256606eff5084b4d13826a0b715bf46a6a611dc39b4afa13cda35160a66bbf8226b45fc2e618a06178b023632ff98c6c35a

data/.saturnci/Dockerfile

@@ -0,0 +1,22 @@
+FROM ruby:4.0.1-slim
+
+RUN apt-get update -qq && \
+    apt-get install --no-install-recommends -y \
+    build-essential \
+    git \
+    libsqlite3-dev \
+    libyaml-dev \
+    sqlite3 \
+    && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/* \
+    && echo "saturnci-test" > /etc/machine-id
+
+WORKDIR /code
+
+COPY Gemfile Gemfile.lock ductwork.gemspec ./
+COPY lib/ductwork/version.rb ./lib/ductwork/
+
+RUN bundle install
+
+COPY . .
+
+CMD ["bundle", "exec", "rspec"]

data/.saturnci/database.yml

@@ -0,0 +1,5 @@
+test:
+  adapter: sqlite3
+  database: db/test.sqlite3
+  pool: 5
+  timeout: 5000

data/.saturnci/docker-compose.yml

@@ -0,0 +1,10 @@
+services:
+  saturn_test_app:
+    build:
+      context: ..
+      dockerfile: .saturnci/Dockerfile
+    environment:
+      - RAILS_ENV=test
+    volumes:
+      - ..:/code
+    working_dir: /code

data/.saturnci/pre.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -e
+
+bundle install

data/CHANGELOG.md

@@ -1,5 +1,15 @@
 # Ductwork Changelog
 
+## [0.21.0]
+
+- feat: enqueue jobs in batches when expanding - this marks a major performance improvement
+- fix: use node's defined klass when looking up max expansion depth
+- feat!: change all tables' primary key type to UUID v7 - BREAKING CHANGE: be sure to run `bin/rails g ductwork:update` to get the database migrations required to migrate existing data
+- chore!: drop support for ruby 3.2 - BREAKING CHANGE: we need UUID v7 from `SecureRandom` to use as primary keys for our tables; it is only available on Ruby 3.3+ (support ends in a ~month so this was going to happen anyway)
+- feat: use row locking job claiming when the database supports it
+- feat: denormalize pipeline klass column on `ductwork_availabilities` - be sure to run `bin/rails g ductwork:update` to get the latest migration
+- fix: use pipeline class name in log message
+
 ## [0.20.2]
 
 - fix: complete pipeline when there are no steps to expand to

data/README.md

@@ -25,6 +25,12 @@ Run the Rails generator to create the binstub, configuration file, and migration
 bin/rails generate ductwork:install
 ```
 
+**NOTE**: run the update generator if you've already installed ductwork to get updates:
+
+```bash
+bin/rails generate ductwork:update
+```
+
 Run migrations and you're ready to start building pipelines!
 
 ## Configuration

data/app/views/ductwork/pipelines/show.html.erb

@@ -122,7 +122,7 @@
             <div class="grid">
               <div>
                 <label>ID</label>
-                <code>
+                <code style="display: block;">
                   <%= step.id %>
                 </code>
               </div>

data/lib/ductwork/context.rb

@@ -21,6 +21,7 @@ module Ductwork
 
     def set(key, value, overwrite: false)
       attributes = {
+        id: SecureRandom.uuid_v7,
         pipeline_id: pipeline_id,
         key: key,
         serialized_value: Ductwork::Tuple.serialize(value),

data/lib/ductwork/job_claim.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Ductwork
+  class JobClaim
+    def initialize(klass)
+      @klass = klass
+      @adapter = Ductwork::Record.connection.adapter_name.downcase
+    end
+
+    def latest
+      claim = if supports_row_locking?
+                RowLockingJobClaim
+              else
+                OptimisticLockingJobClaim
+              end
+
+      claim.new(klass).latest
+    end
+
+    private
+
+    attr_reader :klass, :adapter
+
+    def supports_row_locking?
+      adapter.match?(/postgresql/i) ||
+        adapter.match?(/mysql2/i) ||
+        adapter.match?(/trilogy/i) ||
+        adapter.match?(/oracle_enhanced/i)
+    end
+  end
+end

data/lib/ductwork/migration.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Ductwork
+  class Migration < ActiveRecord::Migration["#{Rails::VERSION::MAJOR}.#{Rails::VERSION::MINOR}".to_f]
+    include Ductwork::MigrationHelper
+  end
+end

data/lib/ductwork/migration_helper.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Ductwork
+  module MigrationHelper
+    def create_ductwork_table(table_name, &block)
+      if postgresql?
+        create_table table_name, id: :uuid, &block
+      else
+        create_table table_name, id: false do |table|
+          table.string :id, limit: 36, null: false, primary_key: true
+          block.call(table)
+        end
+      end
+    end
+
+    def belongs_to(table_object, association_name, **options)
+      full_options = if postgresql?
+                       { type: uuid_column_type }.merge(options)
+                     else
+                       { type: uuid_column_type, limit: 36 }.merge(options)
+                     end
+
+      table_object.belongs_to association_name, **full_options
+    end
+
+    def uuid_column_type
+      if postgresql?
+        :uuid
+      else
+        :string
+      end
+    end
+
+    def postgresql?
+      connection.adapter_name.match?(/postgresql/i)
+    end
+
+    def mysql?
+      connection.adapter_name.match?(/mysql/i)
+    end
+  end
+end

data/lib/ductwork/models/job.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Ductwork
-  class Job < Ductwork::Record # rubocop:todo Metrics/ClassLength
+  class Job < Ductwork::Record
     belongs_to :step, class_name: "Ductwork::Step"
     has_many :executions, class_name: "Ductwork::Execution", foreign_key: "job_id", dependent: :destroy
 
@@ -11,60 +11,8 @@ module Ductwork
 
     FAILED_EXECUTION_TIMEOUT = 10.seconds
 
-    def self.claim_latest(klass) # rubocop:todo Metrics
-      process_id = ::Process.pid
-      id = Ductwork::Availability
-           .joins(execution: { job: { step: :pipeline } })
-           .where("ductwork_availabilities.started_at <= ?", Time.current)
-           .where(completed_at: nil)
-           .where(ductwork_pipelines: { klass: })
-           .order(:created_at)
-           .limit(1)
-           .pluck(:id)
-           .first
-
-      if id.present?
-        # TODO: probably makes sense to use SQL here instead of relying
-        # on ActiveRecord to construct the correct `UPDATE` query
-        rows_updated = nil
-        Ductwork::Record.transaction do
-          rows_updated = Ductwork::Availability
-                         .where(id: id, completed_at: nil)
-                         .update_all(completed_at: Time.current, process_id: process_id)
-          Ductwork::Execution
-            .joins(:availability)
-            .where(completed_at: nil)
-            .where(ductwork_availabilities: { id: })
-            .update_all(process_id:)
-        end
-
-        if rows_updated == 1
-          Ductwork.logger.debug(
-            msg: "Job claimed",
-            role: :job_worker,
-            process_id: process_id,
-            availability_id: id
-          )
-          job = Ductwork::Job
-                .joins(executions: :availability)
-                .find_by(ductwork_availabilities: { id:, process_id: })
-
-          Ductwork::Record.transaction do
-            job.step.in_progress!
-            job.step.pipeline.in_progress!
-          end
-
-          job
-        else
-          Ductwork.logger.debug(
-            msg: "Did not claim job, avoided race condition",
-            role: :job_worker,
-            process_id: process_id,
-            availability_id: id
-          )
-          nil
-        end
-      end
+    def self.claim_latest(klass)
+      Ductwork::JobClaim.new(klass).latest
     end
 
     def self.enqueue(step, *args)
@@ -78,7 +26,8 @@ module Ductwork
         retry_count: 0
       )
       execution.create_availability!(
-        started_at: Time.current
+        started_at: Time.current,
+        pipeline_klass: step.pipeline.klass
       )
 
       Ductwork.logger.info(
@@ -168,7 +117,8 @@ module Ductwork
             started_at: FAILED_EXECUTION_TIMEOUT.from_now
           )
           new_execution.create_availability!(
-            started_at: FAILED_EXECUTION_TIMEOUT.from_now
+            started_at: FAILED_EXECUTION_TIMEOUT.from_now,
+            pipeline_klass: pipeline.klass
           )
         elsif execution.retry_count >= max_retry
           halted = true

data/lib/ductwork/models/pipeline.rb

@@ -259,7 +259,7 @@ module Ductwork
     end
 
     def expand_to_next_steps(step_id, edge)
-      next_klass = edge[:to].sole
+      next_klass = parsed_definition.dig(:edges, edge[:to].sole, :klass)
       return_value = Ductwork::Job
                      .find_by(step_id:)
                      .return_value
@@ -270,12 +270,78 @@ module Ductwork
       elsif return_value.none?
         complete!
       else
-        # TODO: Brainstorm on using `insert_all` instead of iterating.
-        # Performance is bad when the return value has a lot of elements
-        # and we create a step and job individually
-        Array(return_value).each do |input_arg|
-          create_step_and_enqueue_job(edge:, input_arg:)
+        bulk_create_steps_and_jobs(edge:, return_value:)
+      end
+    end
+
+    def bulk_create_steps_and_jobs(edge:, return_value:) # rubocop:todo Metrics
+      # NOTE: "chain" is used by ActiveRecord so we have to call
+      # this enum value "default" :sad:
+      to_transition = edge[:type] == "chain" ? "default" : edge[:type]
+      node ||= edge[:to].sole
+      step_klass = parsed_definition.dig(:edges, node, :klass)
+      now = Time.current
+
+      Array(return_value).each_slice(1_000).each do |batch| # rubocop:todo Metrics/BlockLength
+        step_rows = []
+        job_rows = []
+        execution_rows = []
+        availability_rows = []
+
+        batch.each do |value| # rubocop:todo Metrics/BlockLength
+          step_id = SecureRandom.uuid_v7
+          job_id = SecureRandom.uuid_v7
+          execution_id = SecureRandom.uuid_v7
+          availability_id = SecureRandom.uuid_v7
+
+          step_rows << {
+            id: step_id,
+            pipeline_id: id,
+            node: node,
+            klass: step_klass,
+            status: "in_progress",
+            to_transition: to_transition,
+            started_at: now,
+            created_at: now,
+            updated_at: now,
+          }
+          job_rows << {
+            id: job_id,
+            step_id: step_id,
+            input_args: JSON.dump({ args: [value] }),
+            klass: step_klass,
+            started_at: now,
+            created_at: now,
+            updated_at: now,
+          }
+          execution_rows << {
+            id: execution_id,
+            job_id: job_id,
+            retry_count: 0,
+            started_at: now,
+            created_at: now,
+            updated_at: now,
+          }
+          availability_rows << {
+            id: availability_id,
+            execution_id: execution_id,
+            pipeline_klass: klass,
+            started_at: now,
+            created_at: now,
+            updated_at: now,
+          }
         end
+
+        Ductwork::Step.insert_all!(step_rows)
+        Ductwork::Job.insert_all!(job_rows)
+        Ductwork::Execution.insert_all!(execution_rows)
+        Ductwork::Availability.insert_all!(availability_rows)
+
+        Ductwork.logger.info(
+          msg: "Job batch enqueued",
+          count: batch.count,
+          job_klass: step_klass
+        )
       end
     end
 

data/lib/ductwork/models/record.rb

@@ -8,8 +8,16 @@ module Ductwork
       connects_to(database: { writing: Ductwork.configuration.database.to_sym })
     end
 
+    before_create :generate_uuid_v7
+
     def self.table_name_prefix
       "ductwork_"
     end
+
+    private
+
+    def generate_uuid_v7
+      self.id ||= SecureRandom.uuid_v7
+    end
   end
 end

data/lib/ductwork/optimistic_locking_job_claim.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module Ductwork
+  class OptimisticLockingJobClaim
+    def initialize(klass)
+      @id = nil
+      @job = nil
+      @klass = klass
+      @process_id = ::Process.pid
+    end
+
+    def latest
+      @id = latest_availability_id
+
+      if id.present?
+        rows_updated = claim_availability
+
+        if rows_updated == 1
+          Ductwork.logger.debug(
+            msg: "Job claimed",
+            role: :job_worker,
+            process_id: process_id,
+            availability_id: id
+          )
+
+          @job = find_job
+
+          update_state
+        else
+          Ductwork.logger.debug(
+            msg: "Did not claim job, avoided race condition",
+            role: :job_worker,
+            process_id: process_id,
+            availability_id: id
+          )
+        end
+      else
+        Ductwork.logger.debug(
+          msg: "No available job to claim",
+          role: :job_worker,
+          process_id: process_id,
+          pipeline: klass
+        )
+      end
+
+      job
+    end
+
+    private
+
+    attr_reader :id, :job, :klass, :process_id
+
+    def latest_availability_id
+      Ductwork::Availability
+        .where("ductwork_availabilities.started_at <= ?", Time.current)
+        .where(completed_at: nil, pipeline_klass: klass)
+        .order(:started_at)
+        .limit(1)
+        .pluck(:id)
+        .first
+    end
+
+    def claim_availability
+      Ductwork::Availability
+        .where(id: id, completed_at: nil)
+        .update_all(completed_at: Time.current, process_id: process_id)
+    end
+
+    def find_job
+      Ductwork::Job
+        .joins(executions: :availability)
+        .find_by!(ductwork_availabilities: { id:, process_id: })
+    end
+
+    def update_state
+      Ductwork::Record.transaction do
+        Ductwork::Execution
+          .joins(:availability)
+          .where(completed_at: nil)
+          .where(ductwork_availabilities: { id: })
+          .sole
+          .update!(process_id:)
+        job.step.in_progress!
+        job.step.pipeline.in_progress!
+      end
+    end
+  end
+end

data/lib/ductwork/processes/pipeline_advancer_runner.rb

@@ -75,7 +75,7 @@ module Ductwork
             Ductwork.logger.warn(
               msg: "Restarted pipeline advancer",
               role: :pipeline_advancer_runner,
-              pipeline: advancer.pipeline,
+              pipeline: advancer.pipeline.class.to_s,
               thread: advancer.name
             )
           end

data/lib/ductwork/row_locking_job_claim.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module Ductwork
+  class RowLockingJobClaim
+    def initialize(klass)
+      @id = nil
+      @job = nil
+      @klass = klass
+      @process_id = ::Process.pid
+    end
+
+    def latest
+      rows_updated = attempt_availability_claim
+
+      if rows_updated == 1
+        Ductwork.logger.debug(
+          msg: "Job claimed",
+          role: :job_worker,
+          process_id: process_id,
+          availability_id: id
+        )
+        update_state
+      else
+        Ductwork.logger.debug(
+          msg: "No available job to claim",
+          role: :job_worker,
+          process_id: process_id,
+          pipeline: klass
+        )
+      end
+
+      job
+    end
+
+    private
+
+    attr_reader :id, :job, :klass, :process_id
+
+    def attempt_availability_claim
+      Ductwork::Record.transaction do
+        @id = Ductwork::Availability
+              .where("ductwork_availabilities.started_at <= ?", Time.current)
+              .where(completed_at: nil, pipeline_klass: klass)
+              .order(:started_at)
+              .lock("FOR UPDATE SKIP LOCKED")
+              .limit(1)
+              .ids
+              .first
+
+        if id.present?
+          Ductwork::Availability
+            .where(id: id, completed_at: nil)
+            .update_all(completed_at: Time.current, process_id: process_id)
+        else
+          0
+        end
+      end
+    end
+
+    def update_state
+      Ductwork::Record.transaction do
+        execution = Ductwork::Execution
+                    .joins(:availability)
+                    .where(completed_at: nil)
+                    .where(ductwork_availabilities: { id: })
+                    .sole
+        @job = execution.job
+
+        execution.update!(process_id:)
+        job.step.in_progress!
+        job.step.pipeline.in_progress!
+      end
+    end
+  end
+end

data/lib/ductwork/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Ductwork
-  VERSION = "0.20.2"
+  VERSION = "0.21.0"
 end

data/lib/generators/ductwork/install/templates/db/create_ductwork_availabilities.rb

@@ -1,19 +1,27 @@
 # frozen_string_literal: true
 
-class CreateDuctworkAvailabilities < ActiveRecord::Migration[<%= Rails::VERSION::MAJOR %>.<%= Rails::VERSION::MINOR %>]
+class CreateDuctworkAvailabilities < Ductwork::Migration
   def change
-    create_table :ductwork_availabilities do |table|
-      table.belongs_to :execution, index: false, null: false, foreign_key: { to_table: :ductwork_executions }
+    create_ductwork_table :ductwork_availabilities do |table|
+      belongs_to(
+        table,
+        :execution,
+        index: false,
+        null: false,
+        foreign_key: { to_table: :ductwork_executions }
+      )
       table.timestamp :started_at, null: false
       table.timestamp :completed_at
       table.integer :process_id
+      table.string :pipeline_klass, null: false
       table.timestamps null: false
     end
 
     add_index :ductwork_availabilities, :execution_id, unique: true
     add_index :ductwork_availabilities, %i[id process_id]
     add_index :ductwork_availabilities,
-              %i[completed_at started_at created_at],
-              name: "index_ductwork_availabilities_on_claim_latest"
+              %i[pipeline_klass started_at],
+              name: "index_ductwork_availabilities_on_claim_latest",
+              where: "completed_at IS NULL"
   end
 end

data/lib/generators/ductwork/install/templates/db/create_ductwork_executions.rb

@@ -1,9 +1,15 @@
 # frozen_string_literal: true
 
-class CreateDuctworkExecutions < ActiveRecord::Migration[<%= Rails::VERSION::MAJOR %>.<%= Rails::VERSION::MINOR %>]
+class CreateDuctworkExecutions < Ductwork::Migration
   def change
-    create_table :ductwork_executions do |table|
-      table.belongs_to :job, index: true, null: false, foreign_key: { to_table: :ductwork_jobs }
+    create_ductwork_table :ductwork_executions do |table|
+      belongs_to(
+        table,
+        :job,
+        index: true,
+        null: false,
+        foreign_key: { to_table: :ductwork_jobs }
+      )
       table.timestamp :started_at, null: false
       table.timestamp :completed_at
       table.integer :retry_count, null: false

data/lib/generators/ductwork/install/templates/db/create_ductwork_jobs.rb

@@ -1,9 +1,15 @@
 # frozen_string_literal: true
 
-class CreateDuctworkJobs < ActiveRecord::Migration[<%= Rails::VERSION::MAJOR %>.<%= Rails::VERSION::MINOR %>]
+class CreateDuctworkJobs < Ductwork::Migration
   def change
-    create_table :ductwork_jobs do |table|
-      table.belongs_to :step, index: false, null: false, foreign_key: { to_table: :ductwork_steps }
+    create_ductwork_table :ductwork_jobs do |table|
+      belongs_to(
+        table,
+        :step,
+        index: false,
+        null: false,
+        foreign_key: { to_table: :ductwork_steps }
+      )
       table.string :klass, null: false
       table.timestamp :started_at, null: false
       table.timestamp :completed_at

data/lib/generators/ductwork/install/templates/db/create_ductwork_pipelines.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
-class CreateDuctworkPipelines < ActiveRecord::Migration[<%= Rails::VERSION::MAJOR %>.<%= Rails::VERSION::MINOR %>]
+class CreateDuctworkPipelines < Ductwork::Migration
   def change
-    create_table :ductwork_pipelines do |table|
+    create_ductwork_table :ductwork_pipelines do |table|
       table.string :klass, null: false
       table.text :definition, null: false
       table.string :definition_sha1, null: false

data/lib/generators/ductwork/install/templates/db/create_ductwork_processes.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
-class CreateDuctworkProcesses < ActiveRecord::Migration[<%= Rails::VERSION::MAJOR %>.<%= Rails::VERSION::MINOR %>]
+class CreateDuctworkProcesses < Ductwork::Migration
   def change
-    create_table :ductwork_processes do |table|
+    create_ductwork_table :ductwork_processes do |table|
       table.integer :pid, null: false
       table.string :machine_identifier, null: false
       table.timestamp :last_heartbeat_at, null: false

data/lib/generators/ductwork/install/templates/db/create_ductwork_results.rb

@@ -1,9 +1,15 @@
 # frozen_string_literal: true
 
-class CreateDuctworkResults < ActiveRecord::Migration[<%= Rails::VERSION::MAJOR %>.<%= Rails::VERSION::MINOR %>]
+class CreateDuctworkResults < Ductwork::Migration
   def change
-    create_table :ductwork_results do |table|
-      table.belongs_to :execution, index: false, null: false, foreign_key: { to_table: :ductwork_executions }
+    create_ductwork_table :ductwork_results do |table|
+      belongs_to(
+        table,
+        :execution,
+        index: false,
+        null: false,
+        foreign_key: { to_table: :ductwork_executions }
+      )
       table.string :result_type, null: false
       table.string :error_klass
       table.string :error_message

data/lib/generators/ductwork/install/templates/db/create_ductwork_runs.rb

@@ -1,9 +1,15 @@
 # frozen_string_literal: true
 
-class CreateDuctworkRuns < ActiveRecord::Migration[<%= Rails::VERSION::MAJOR %>.<%= Rails::VERSION::MINOR %>]
+class CreateDuctworkRuns < Ductwork::Migration
   def change
-    create_table :ductwork_runs do |table|
-      table.belongs_to :execution, index: false, null: false, foreign_key: { to_table: :ductwork_executions }
+    create_ductwork_table :ductwork_runs do |table|
+      belongs_to(
+        table,
+        :execution,
+        index: false,
+        null: false,
+        foreign_key: { to_table: :ductwork_executions }
+      )
       table.timestamp :started_at, null: false
       table.timestamp :completed_at
       table.timestamps null: false

data/lib/generators/ductwork/install/templates/db/create_ductwork_steps.rb

@@ -1,9 +1,15 @@
 # frozen_string_literal: true
 
-class CreateDuctworkSteps < ActiveRecord::Migration[<%= Rails::VERSION::MAJOR %>.<%= Rails::VERSION::MINOR %>]
+class CreateDuctworkSteps < Ductwork::Migration
   def change
-    create_table :ductwork_steps do |table|
-      table.belongs_to :pipeline, index: true, null: false, foreign_key: { to_table: :ductwork_pipelines }
+    create_ductwork_table :ductwork_steps do |table|
+      belongs_to(
+        table,
+        :pipeline,
+        index: true,
+        null: false,
+        foreign_key: { to_table: :ductwork_pipelines }
+      )
       table.string :node, null: false
       table.string :klass, null: false
       table.string :to_transition, null: false

data/lib/generators/ductwork/install/templates/db/create_ductwork_tuples.rb

@@ -1,9 +1,15 @@
 # frozen_string_literal: true
 
-class CreateDuctworkTuples < ActiveRecord::Migration[<%= Rails::VERSION::MAJOR %>.<%= Rails::VERSION::MINOR %>]
+class CreateDuctworkTuples < Ductwork::Migration
   def change
-    create_table :ductwork_tuples do |table|
-      table.belongs_to :pipeline, index: true, null: false, foreign_key: { to_table: :ductwork_pipelines }
+    create_ductwork_table :ductwork_tuples do |table|
+      belongs_to(
+        table,
+        :pipeline,
+        index: true,
+        null: false,
+        foreign_key: { to_table: :ductwork_pipelines }
+      )
       table.string :key, null: false
       table.string :serialized_value
       table.datetime :first_set_at, null: false

data/lib/generators/ductwork/update/USAGE

@@ -0,0 +1,9 @@
+Description:
+    Updates ductwork with necessary files and changes
+
+Example:
+    bin/rails generate ductwork:update
+
+    This will perform the following:
+        Adds migrations
+

data/lib/generators/ductwork/update/templates/db/denormalize_pipeline_klass_on_availabilities.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+class DenormalizePipelineKlassOnAvailabilities < Ductwork::Migration
+  def change
+    add_column :ductwork_availabilities, :pipeline_klass, :string
+
+    # NOTE: change this how you see fit. everything is updated to a static,
+    # bogus value in case there are a lot of records.
+    Ductwork::Availability
+      .where(pipeline_klass: nil)
+      .update_all(pipeline_klass: "Pipeline")
+
+    change_column_null :ductwork_availabilities, :pipeline_klass, false
+    remove_index :ductwork_availabilities, name: "index_ductwork_availabilities_on_claim_latest"
+    add_index :ductwork_availabilities,
+              %i[pipeline_klass started_at],
+              name: "index_ductwork_availabilities_on_claim_latest",
+              where: "completed_at IS NULL"
+  end
+end

data/lib/generators/ductwork/update/templates/db/migrate_tables_to_uuid_primary_key.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+class MigrateTablesToUuidPrimaryKey < Ductwork::Migration
+  TABLE_TO_FOREIGN_KEYS = {
+    ductwork_pipelines: [],
+    ductwork_steps: [:pipeline],
+    ductwork_jobs: [:step],
+    ductwork_executions: [:job],
+    ductwork_availabilities: [:execution],
+    ductwork_runs: [:execution],
+    ductwork_results: [:execution],
+    ductwork_tuples: [:pipeline],
+    ductwork_processes: [],
+  }.freeze
+  FOREIGN_KEY_TO_TABLES = {
+    pipeline: %i[ductwork_steps ductwork_tuples],
+    step: %i[ductwork_jobs],
+    job: %i[ductwork_executions],
+    execution: %i[ductwork_availabilities ductwork_runs ductwork_results],
+    availabilitie: [],
+    run: [],
+    result: [],
+    tuple: [],
+    processe: [],
+  }.freeze
+
+  def up
+    create_primary_key_uuid_columns
+    create_foreign_key_uuid_columns
+    backfill_uuid_columns
+    drop_old_foreign_keys
+    swap_primary_keys
+    swap_foreign_keys
+  end
+
+  def down
+    raise ActiveRecord::IrreversibleMigration,
+          "Cannot revert UUID migration — old integer IDs have been dropped. Restore from backup."
+  end
+
+  private
+
+  def create_primary_key_uuid_columns
+    TABLE_TO_FOREIGN_KEYS.each_key do |table|
+      add_column table, :uuid, uuid_column_type
+    end
+  end
+
+  def create_foreign_key_uuid_columns
+    TABLE_TO_FOREIGN_KEYS.each do |table, column_prefixes|
+      column_prefixes.each do |column_prefix|
+        add_column table, "#{column_prefix}_uuid", uuid_column_type
+      end
+    end
+  end
+
+  def backfill_uuid_columns
+    FOREIGN_KEY_TO_TABLES.each do |prefix, tables|
+      select_all("SELECT id from ductwork_#{prefix}s WHERE uuid IS NULL").each do |row|
+        uuid = connection.quote(SecureRandom.uuid_v7)
+        id = row["id"]
+
+        execute("UPDATE ductwork_#{prefix}s SET uuid = #{uuid} WHERE id = #{id}")
+        tables.each do |table|
+          execute("UPDATE #{table} SET #{prefix}_uuid = #{uuid} WHERE #{prefix}_id = #{id}")
+        end
+      end
+    end
+  end
+
+  def drop_old_foreign_keys
+    FOREIGN_KEY_TO_TABLES.each do |prefix, tables|
+      tables.each do |table|
+        remove_foreign_key table, "ductwork_#{prefix}s", column: "#{prefix}_id", if_exists: true
+      end
+    end
+  end
+
+  def swap_primary_keys # rubocop:disable Metrics/PerceivedComplexity
+    TABLE_TO_FOREIGN_KEYS.each_key do |table|
+      if postgresql?
+        execute "ALTER TABLE #{table} DROP CONSTRAINT IF EXISTS #{table}_pkey"
+      elsif mysql?
+        execute "ALTER TABLE #{table} MODIFY id BIGINT NOT NULL"
+        execute "ALTER TABLE #{table} DROP PRIMARY KEY"
+      end
+
+      remove_column table, :id
+      rename_column table, :uuid, :id
+
+      if postgresql?
+        execute "ALTER TABLE #{table} ADD PRIMARY KEY (id)"
+      elsif mysql?
+        execute "ALTER TABLE #{table} MODIFY id VARCHAR(36) NOT NULL"
+        execute "ALTER TABLE #{table} ADD PRIMARY KEY (id)"
+      elsif !index_exists?(table, :id)
+        add_index table, :id, unique: true
+      end
+
+      change_column_null table, :id, false
+    end
+  end
+
+  def swap_foreign_keys
+    FOREIGN_KEY_TO_TABLES.each do |prefix, tables|
+      tables.each do |table|
+        remove_index table, "#{prefix}_id", if_exists: true
+        remove_column table, "#{prefix}_id"
+        rename_column table, "#{prefix}_uuid", "#{prefix}_id"
+        if !index_exists?(table, "#{prefix}_id")
+          add_index table, "#{prefix}_id"
+        end
+        change_column_null table, "#{prefix}_id", false
+        add_foreign_key table, "ductwork_#{prefix}s", column: "#{prefix}_id"
+      end
+    end
+  end
+end

data/lib/generators/ductwork/update/update_generator.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require "rails/generators/migration"
+require "rails/generators/active_record/migration"
+
+module Ductwork
+  class UpdateGenerator < Rails::Generators::Base
+    include ActiveRecord::Generators::Migration
+
+    source_root File.expand_path("templates", __dir__)
+
+    def create_files
+      if Ductwork::Availability.column_names.exclude?("pipeline_klass")
+        migration_template "db/denormalize_pipeline_klass_on_availabilities.rb",
+                           "db/migrate/denormalize_pipeline_klass_on_availabilities.rb"
+      end
+
+      if Ductwork::Pipeline.column_for_attribute("id").type != :uuid
+        migration_template "db/migrate_tables_to_uuid_primary_key.rb",
+                           "db/migrate/migrate_tables_to_uuid_primary_key.rb"
+      end
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ductwork
 version: !ruby/object:Gem::Version
-  version: 0.20.2
+  version: 0.21.0
 platform: ruby
 authors:
 - Tyler Ewing
@@ -111,10 +111,14 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".saturnci/Dockerfile"
+- ".saturnci/database.yml"
+- ".saturnci/docker-compose.yml"
+- ".saturnci/pre.sh"
 - CHANGELOG-PRO.md
 - CHANGELOG.md
-- COMM-LICENSE.txt
 - LICENSE.txt
+- PRO-LICENSE.txt
 - README.md
 - Rakefile
 - app/assets/images/ductwork/logo.png
@@ -141,7 +145,10 @@ files:
 - lib/ductwork/dsl/branch_builder.rb
 - lib/ductwork/dsl/definition_builder.rb
 - lib/ductwork/engine.rb
+- lib/ductwork/job_claim.rb
 - lib/ductwork/machine_identifier.rb
+- lib/ductwork/migration.rb
+- lib/ductwork/migration_helper.rb
 - lib/ductwork/models/availability.rb
 - lib/ductwork/models/execution.rb
 - lib/ductwork/models/job.rb
@@ -152,6 +159,7 @@ files:
 - lib/ductwork/models/run.rb
 - lib/ductwork/models/step.rb
 - lib/ductwork/models/tuple.rb
+- lib/ductwork/optimistic_locking_job_claim.rb
 - lib/ductwork/processes/job_worker.rb
 - lib/ductwork/processes/job_worker_runner.rb
 - lib/ductwork/processes/launcher.rb
@@ -161,6 +169,7 @@ files:
 - lib/ductwork/processes/process_supervisor_runner.rb
 - lib/ductwork/processes/thread_supervisor.rb
 - lib/ductwork/processes/thread_supervisor_runner.rb
+- lib/ductwork/row_locking_job_claim.rb
 - lib/ductwork/running_context.rb
 - lib/ductwork/testing.rb
 - lib/ductwork/testing/helpers.rb
@@ -180,6 +189,10 @@ files:
 - lib/generators/ductwork/install/templates/db/create_ductwork_runs.rb
 - lib/generators/ductwork/install/templates/db/create_ductwork_steps.rb
 - lib/generators/ductwork/install/templates/db/create_ductwork_tuples.rb
+- lib/generators/ductwork/update/USAGE
+- lib/generators/ductwork/update/templates/db/denormalize_pipeline_klass_on_availabilities.rb
+- lib/generators/ductwork/update/templates/db/migrate_tables_to_uuid_primary_key.rb
+- lib/generators/ductwork/update/update_generator.rb
 homepage: https://github.com/ductwork/ductwork
 licenses:
 - LGPL-3.0-or-later
@@ -197,14 +210,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 3.2.0
+      version: 3.3.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.3
+rubygems_version: 4.0.6
 specification_version: 4
 summary: A Ruby pipeline framework
 test_files: []