ductwork 0.25.0 → 1.0.0

data/lib/ductwork/row_locking_execution_claim.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module Ductwork
+  class RowLockingExecutionClaim
+    def initialize(klass, owner_process_id)
+      @availability = nil
+      @execution = nil
+      @klass = klass
+      @process_id = owner_process_id
+    end
+
+    def latest
+      Ductwork::Record.transaction do
+        claim_availability
+
+        if availability.present?
+          Ductwork.logger.debug(
+            msg: "Job claimed",
+            role: :job_worker,
+            process_id: process_id,
+            availability_id: availability.id
+          )
+          update_state
+        else
+          Ductwork.logger.debug(
+            msg: "No available job to claim",
+            role: :job_worker,
+            process_id: process_id,
+            pipeline: klass
+          )
+        end
+      end
+
+      execution
+    end
+
+    private
+
+    attr_reader :availability, :execution, :klass, :process_id
+
+    def claim_availability
+      sql = Ductwork::DatabaseClock.now_sql("ductwork_availabilities.started_at")
+      @availability = Ductwork::Availability
+                      .where(sql)
+                      .where(completed_at: nil, pipeline_klass: klass)
+                      .order(:started_at)
+                      .lock("FOR UPDATE SKIP LOCKED")
+                      .limit(1)
+                      .first
+
+      return unless availability
+
+      completed_at = Time.current
+      @execution = availability.execution
+
+      availability.update_columns(completed_at:, process_id:)
+      execution.update_columns(process_id:)
+    end
+
+    def update_state
+      step = execution.job.step
+
+      Ductwork::Step
+        .where(id: step.id)
+        .where.not(status: "in_progress")
+        .update_all(status: "in_progress", updated_at: Time.current)
+      Ductwork::Run
+        .where(id: step.run_id)
+        .where.not(status: "in_progress")
+        .update_all(status: "in_progress", updated_at: Time.current)
+      Ductwork::Pipeline
+        .where(id: Ductwork::Run.where(id: step.run_id).select(:pipeline_id))
+        .where.not(status: "in_progress")
+        .update_all(status: "in_progress", updated_at: Time.current)
+    end
+  end
+end

data/lib/ductwork/testing/rspec.rb

@@ -66,7 +66,7 @@ RSpec::Matchers.define(:have_set_context) do |expected|
   supports_block_expectations
 
   match do |block|
-    ctx = Ductwork::Context.new(pipeline_id)
+    ctx = Ductwork::Context.new(run_id)
     before_values = expected.map { |k, _| ctx.get(k.to_s) }
 
     block.call
@@ -76,7 +76,21 @@ RSpec::Matchers.define(:have_set_context) do |expected|
     before_values.all?(&:nil?) && after_context == expected
   end
 
-  chain :for_pipeline, :pipeline_id
+  chain :for_pipeline do |pipeline|
+    @run_id = pipeline.current_run.id
+  end
+
+  chain :for_run do |given_run_id|
+    @run_id = given_run_id
+  end
+
+  def run_id
+    if @run_id.blank?
+      raise ArgumentError, "Must chain with .for_pipeline or .for_run"
+    end
+
+    @run_id
+  end
 
   failure_message do
     "Context does not match expected result"
@@ -88,25 +102,28 @@ module Ductwork
     module RSpec
       def pipeline_for(klass, **attrs)
         definition = klass.pipeline_definition.to_json
-        definition_sha1 = OpenSSL::Digest::SHA256.hexdigest(definition)
+        definition_sha1 = Digest::SHA1.hexdigest(definition)
+        pipeline_klass = klass.name.to_s
+        now = Time.current
         status = attrs[:status] || "in_progress"
-        triggered_at = attrs[:triggered_at] || Time.current
-        started_at = attrs[:started_at] || Time.current
-        last_advanced_at = attrs[:last_advanced_at] || Time.current
+        triggered_at = attrs[:triggered_at] || now
+        started_at = attrs[:started_at] || now
 
-        Ductwork::Pipeline.create!(
-          klass:,
+        pipeline = Ductwork::Pipeline.create!(klass:, status:)
+        pipeline.runs.create!(
+          pipeline_klass:,
           definition:,
           definition_sha1:,
           status:,
           triggered_at:,
-          started_at:,
-          last_advanced_at:
+          started_at:
         )
+
+        pipeline
       end
 
       def set_pipeline_context(pipeline, **key_values)
-        ctx = Ductwork::Context.new(pipeline.id)
+        ctx = Ductwork::Context.new(pipeline.current_run.id)
         key_values.each do |key, value|
           ctx.set(key.to_s, value)
         end

data/lib/ductwork/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Ductwork
-  VERSION = "0.25.0"
+  VERSION = "1.0.0"
 end

data/lib/ductwork.rb

@@ -65,6 +65,7 @@ module Ductwork
     def validate!
       step_directory = Rails.root.join("app/steps")
       pipeline_directory = Rails.root.join("app/pipelines")
+      workflow_directory = Rails.root.join("app/workflows")
       loader = if defined?(Rails.autoloaders)
                  Rails.autoloaders.main
                else
@@ -79,6 +80,10 @@ module Ductwork
         loader.eager_load_dir(pipeline_directory)
       end
 
+      if workflow_directory.exist?
+        loader.eager_load_dir(workflow_directory)
+      end
+
       true
     end
 

data/lib/generators/ductwork/install/install_generator.rb

@@ -17,20 +17,30 @@ module Ductwork
 
       migration_template "db/create_ductwork_pipelines.rb",
                          "db/migrate/create_ductwork_pipelines.rb"
+      migration_template "db/create_ductwork_runs.rb",
+                         "db/migrate/create_ductwork_runs.rb"
+      migration_template "db/create_ductwork_branches.rb",
+                         "db/migrate/create_ductwork_branches.rb"
+      migration_template "db/create_ductwork_branch_links.rb",
+                         "db/migrate/create_ductwork_branch_links.rb"
       migration_template "db/create_ductwork_steps.rb",
                          "db/migrate/create_ductwork_steps.rb"
       migration_template "db/create_ductwork_jobs.rb",
                          "db/migrate/create_ductwork_jobs.rb"
+      migration_template "db/create_ductwork_processes.rb",
+                         "db/migrate/create_ductwork_processes.rb"
       migration_template "db/create_ductwork_executions.rb",
                          "db/migrate/create_ductwork_executions.rb"
       migration_template "db/create_ductwork_availabilities.rb",
                          "db/migrate/create_ductwork_availabilities.rb"
-      migration_template "db/create_ductwork_runs.rb",
-                         "db/migrate/create_ductwork_runs.rb"
+      migration_template "db/create_ductwork_attempts.rb",
+                         "db/migrate/create_ductwork_attempts.rb"
       migration_template "db/create_ductwork_results.rb",
                          "db/migrate/create_ductwork_results.rb"
-      migration_template "db/create_ductwork_processes.rb",
-                         "db/migrate/create_ductwork_processes.rb"
+      migration_template "db/create_ductwork_transitions.rb",
+                         "db/migrate/create_ductwork_transitions.rb"
+      migration_template "db/create_ductwork_advancements.rb",
+                         "db/migrate/create_ductwork_advancements.rb"
       migration_template "db/create_ductwork_tuples.rb",
                          "db/migrate/create_ductwork_tuples.rb"
 

data/lib/generators/ductwork/install/templates/config/ductwork.yml

@@ -9,12 +9,16 @@ default: &default
     polling_timeout: 1
     shutdown_timeout: 20
   pipeline_advancer:
+    count: 3
+    max_crash: 10
+    max_retry: 3
     polling_timeout: 1
     shutdown_timeout: 20
     steps:
       max_depth: -1
   supervisor:
     polling_timeout: 1
+    reaper_timeout: 300
     shutdown_timeout: 30
 
 development:

data/lib/generators/ductwork/install/templates/db/create_ductwork_advancements.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+class CreateDuctworkAdvancements < Ductwork::Migration
+  def change
+    create_ductwork_table :ductwork_advancements do |table|
+      belongs_to(
+        table,
+        :process,
+        index: true,
+        null: true,
+        foreign_key: { to_table: :ductwork_processes }
+      )
+      belongs_to(
+        table,
+        :transition,
+        index: true,
+        null: false,
+        foreign_key: { to_table: :ductwork_transitions }
+      )
+      table.datetime :started_at, null: false
+      table.datetime :completed_at
+      table.integer :crash_count
+      table.string :error_klass
+      table.string :error_message
+      table.text :error_backtrace
+      table.timestamps null: false
+    end
+  end
+end

data/lib/generators/ductwork/install/templates/db/create_ductwork_attempts.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+class CreateDuctworkAttempts < Ductwork::Migration
+  def change
+    create_ductwork_table :ductwork_attempts do |table|
+      belongs_to(
+        table,
+        :execution,
+        index: false,
+        null: false,
+        foreign_key: { to_table: :ductwork_executions }
+      )
+      table.timestamp :started_at, null: false
+      table.timestamp :completed_at
+      table.timestamps null: false
+    end
+
+    add_index :ductwork_attempts, :execution_id, unique: true
+  end
+end

data/lib/generators/ductwork/install/templates/db/create_ductwork_availabilities.rb

@@ -10,18 +10,31 @@ class CreateDuctworkAvailabilities < Ductwork::Migration
         null: false,
         foreign_key: { to_table: :ductwork_executions }
       )
+      belongs_to(
+        table,
+        :process,
+        index: true,
+        null: true,
+        foreign_key: { to_table: :ductwork_processes }
+      )
       table.timestamp :started_at, null: false
       table.timestamp :completed_at
-      table.integer :process_id
       table.string :pipeline_klass, null: false
       table.timestamps null: false
     end
 
     add_index :ductwork_availabilities, :execution_id, unique: true
     add_index :ductwork_availabilities, %i[id process_id]
-    add_index :ductwork_availabilities,
-              %i[pipeline_klass started_at],
-              name: "index_ductwork_availabilities_on_claim_latest",
-              where: "completed_at IS NULL"
+
+    if mysql?
+      add_index :ductwork_availabilities,
+                %i[pipeline_klass completed_at started_at],
+                name: "index_ductwork_availabilities_on_claim_latest"
+    else
+      add_index :ductwork_availabilities,
+                %i[pipeline_klass started_at],
+                name: "index_ductwork_availabilities_on_claim_latest",
+                where: "completed_at IS NULL"
+    end
   end
 end

data/lib/generators/ductwork/install/templates/db/create_ductwork_branch_links.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+class CreateDuctworkBranchLinks < Ductwork::Migration
+  def change
+    create_ductwork_table :ductwork_branch_links do |table|
+      belongs_to(
+        table,
+        :parent_branch,
+        index: true,
+        null: false,
+        foreign_key: { to_table: :ductwork_branches }
+      )
+      belongs_to(
+        table,
+        :child_branch,
+        index: true,
+        null: false,
+        foreign_key: { to_table: :ductwork_branches }
+      )
+      table.timestamps null: false
+    end
+
+    add_index :ductwork_branch_links,
+              %w[parent_branch_id child_branch_id],
+              unique: true
+  end
+end

data/lib/generators/ductwork/install/templates/db/create_ductwork_branches.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+class CreateDuctworkBranches < Ductwork::Migration
+  def change
+    create_ductwork_table :ductwork_branches do |table|
+      belongs_to(
+        table,
+        :run,
+        index: true,
+        null: false,
+        foreign_key: { to_table: :ductwork_runs }
+      )
+      table.string :pipeline_klass, null: false
+      table.string :status, null: false
+      table.string :halt_reason
+      table.string :claim_token
+      table.datetime :claimed_for_advancing_at
+      table.datetime :last_advanced_at, null: false
+      table.datetime :started_at, null: false
+      table.datetime :completed_at
+      table.timestamps null: false
+    end
+
+    if mysql?
+      add_index :ductwork_branches,
+                %w[pipeline_klass status claimed_for_advancing_at last_advanced_at],
+                name: "index_ductwork_branches_on_claim_latest"
+    else
+      add_index :ductwork_branches,
+                %w[pipeline_klass status last_advanced_at],
+                where: "claimed_for_advancing_at IS NULL",
+                name: "index_ductwork_branches_on_claim_latest"
+    end
+    add_index :ductwork_branches, %w[run_id started_at]
+    add_index :ductwork_branches, %w[run_id status]
+  end
+end

data/lib/generators/ductwork/install/templates/db/create_ductwork_executions.rb

@@ -10,10 +10,17 @@ class CreateDuctworkExecutions < Ductwork::Migration
         null: false,
         foreign_key: { to_table: :ductwork_jobs }
       )
+      belongs_to(
+        table,
+        :process,
+        index: true,
+        null: true,
+        foreign_key: { to_table: :ductwork_processes }
+      )
       table.timestamp :started_at, null: false
       table.timestamp :completed_at
       table.integer :retry_count, null: false
-      table.integer :process_id
+      table.integer :crash_count, null: false
       table.timestamps null: false
     end
 

data/lib/generators/ductwork/install/templates/db/create_ductwork_pipelines.rb

@@ -4,14 +4,6 @@ class CreateDuctworkPipelines < Ductwork::Migration
   def change
     create_ductwork_table :ductwork_pipelines do |table|
       table.string :klass, null: false
-      table.text :definition, null: false
-      table.string :definition_sha1, null: false
-      table.timestamp :triggered_at, null: false
-      table.timestamp :started_at, null: false
-      table.timestamp :completed_at
-      table.timestamp :halted_at
-      table.timestamp :claimed_for_advancing_at
-      table.timestamp :last_advanced_at, null: false
       table.string :status, null: false
       table.timestamps null: false
     end

data/lib/generators/ductwork/install/templates/db/create_ductwork_processes.rb

@@ -5,10 +5,12 @@ class CreateDuctworkProcesses < Ductwork::Migration
     create_ductwork_table :ductwork_processes do |table|
       table.integer :pid, null: false
       table.string :machine_identifier, null: false
+      table.string :role, null: false
       table.timestamp :last_heartbeat_at, null: false
       table.timestamps null: false
     end
 
+    add_index :ductwork_processes, :last_heartbeat_at
     add_index :ductwork_processes, %i[pid machine_identifier], unique: true
   end
 end

data/lib/generators/ductwork/install/templates/db/create_ductwork_results.rb

@@ -18,5 +18,6 @@ class CreateDuctworkResults < Ductwork::Migration
     end
 
     add_index :ductwork_results, :execution_id, unique: true
+    add_index :ductwork_results, %i[result_type created_at]
   end
 end

data/lib/generators/ductwork/install/templates/db/create_ductwork_runs.rb

@@ -5,16 +5,55 @@ class CreateDuctworkRuns < Ductwork::Migration
     create_ductwork_table :ductwork_runs do |table|
       belongs_to(
         table,
-        :execution,
-        index: false,
+        :pipeline,
+        index: true,
         null: false,
-        foreign_key: { to_table: :ductwork_executions }
+        foreign_key: { to_table: :ductwork_pipelines }
       )
-      table.timestamp :started_at, null: false
-      table.timestamp :completed_at
+      table.string :pipeline_klass, null: false
+      table.text :definition, null: false
+      table.string :definition_sha1, null: false
+      table.string :status, null: false
+      table.datetime :triggered_at, null: false
+      table.datetime :started_at, null: false
+      table.datetime :completed_at
+      table.datetime :halted_at
+      table.datetime :on_halt_dispatched_at
       table.timestamps null: false
     end
 
-    add_index :ductwork_runs, :execution_id, unique: true
+    add_index :ductwork_runs, %i[pipeline_id status]
+    add_index :ductwork_runs, %i[pipeline_id started_at]
+    add_index :ductwork_runs, :started_at
+    add_index :ductwork_runs, %i[pipeline_klass started_at]
+    add_index :ductwork_runs, %i[status started_at]
+
+    if mysql?
+      reversible do |direction|
+        direction.up do
+          execute <<~SQL
+            ALTER TABLE ductwork_runs
+              ADD COLUMN active_pipeline_id VARCHAR(36)
+                GENERATED ALWAYS AS (
+                  IF(status IN ('in_progress', 'paused'), pipeline_id, NULL)
+                )
+          SQL
+          add_index :ductwork_runs, :active_pipeline_id,
+                    unique: true,
+                    name: :idx_unique_active_run
+        end
+
+        direction.down do
+          remove_index :ductwork_runs, name: :idx_unique_active_run
+          remove_column :ductwork_runs, :active_pipeline_id
+        end
+      end
+    else
+      add_index :ductwork_runs,
+                :pipeline_id,
+                unique: true,
+                where: "status IN ('in_progress', 'paused')",
+                name: :idx_unique_active_run
+    end
   end
 end

data/lib/generators/ductwork/install/templates/db/create_ductwork_steps.rb

@@ -5,10 +5,24 @@ class CreateDuctworkSteps < Ductwork::Migration
     create_ductwork_table :ductwork_steps do |table|
       belongs_to(
         table,
-        :pipeline,
+        :run,
         index: true,
         null: false,
-        foreign_key: { to_table: :ductwork_pipelines }
+        foreign_key: { to_table: :ductwork_runs }
+      )
+      belongs_to(
+        table,
+        :branch,
+        index: true,
+        null: false,
+        foreign_key: { to_table: :ductwork_branches }
+      )
+      belongs_to(
+        table,
+        :source_step,
+        index: true,
+        null: true,
+        foreign_key: { to_table: :ductwork_steps }
       )
       table.string :node, null: false
       table.string :klass, null: false
@@ -21,9 +35,10 @@ class CreateDuctworkSteps < Ductwork::Migration
       table.timestamps null: false
     end
 
-    add_index :ductwork_steps, %i[pipeline_id status node]
-    add_index :ductwork_steps, %i[pipeline_id node status]
+    add_index :ductwork_steps, %i[branch_id started_at]
+    add_index :ductwork_steps, %i[run_id node status]
+    add_index :ductwork_steps, %i[run_id status node]
+    add_index :ductwork_steps, %i[status branch_id]
     add_index :ductwork_steps, %i[status node]
-    add_index :ductwork_steps, %i[pipeline_id status]
   end
 end

data/lib/generators/ductwork/install/templates/db/create_ductwork_transitions.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+class CreateDuctworkTransitions < Ductwork::Migration
+  def change
+    create_ductwork_table :ductwork_transitions do |table|
+      belongs_to(
+        table,
+        :branch,
+        index: true,
+        null: false,
+        foreign_key: { to_table: :ductwork_branches }
+      )
+      belongs_to(
+        table,
+        :in_step,
+        index: true,
+        null: false,
+        foreign_key: { to_table: :ductwork_steps }
+      )
+      belongs_to(
+        table,
+        :out_step,
+        index: true,
+        null: true,
+        foreign_key: { to_table: :ductwork_steps }
+      )
+      table.datetime :started_at, null: false
+      table.datetime :completed_at
+      table.timestamps null: false
+    end
+
+    if mysql?
+      add_index :ductwork_transitions,
+                %i[branch_id completed_at started_at],
+                name: "index_ductwork_transitions_on_latest_open"
+    else
+      add_index :ductwork_transitions,
+                %i[branch_id started_at],
+                where: "completed_at IS NULL",
+                name: "index_ductwork_transitions_on_latest_open"
+    end
+  end
+end

data/lib/generators/ductwork/install/templates/db/create_ductwork_tuples.rb

@@ -5,10 +5,10 @@ class CreateDuctworkTuples < Ductwork::Migration
     create_ductwork_table :ductwork_tuples do |table|
       belongs_to(
         table,
-        :pipeline,
+        :run,
         index: true,
         null: false,
-        foreign_key: { to_table: :ductwork_pipelines }
+        foreign_key: { to_table: :ductwork_runs }
       )
       table.string :key, null: false
       table.string :serialized_value
@@ -17,6 +17,6 @@ class CreateDuctworkTuples < Ductwork::Migration
       table.timestamps null: false
     end
 
-    add_index :ductwork_tuples, %i[key pipeline_id], unique: true
+    add_index :ductwork_tuples, %i[run_id key], unique: true
   end
 end

data/lib/generators/ductwork/update/templates/db/add_crash_count_to_ductwork_advancements.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class AddCrashCountToDuctworkAdvancements < Ductwork::Migration
+  def change
+    add_column :ductwork_advancements, :crash_count, :integer
+  end
+end

data/lib/generators/ductwork/update/templates/db/add_crash_count_to_ductwork_executions.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+class AddCrashCountToDuctworkExecutions < Ductwork::Migration
+  def change
+    add_column :ductwork_executions, :crash_count, :integer
+
+    # NOTE: change this how you see fit for your scale
+    Ductwork::Execution
+      .where(crash_count: nil)
+      .update_all(crash_count: 0)
+
+    change_column_null :ductwork_executions, :crash_count, false
+  end
+end

data/lib/generators/ductwork/update/templates/db/add_indexes_to_ductwork_results.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class AddIndexesToDuctworkResults < Ductwork::Migration
+  def change
+    add_index :ductwork_results, %i[result_type created_at]
+  end
+end

data/lib/generators/ductwork/update/templates/db/add_indexes_to_ductwork_runs.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class AddIndexesToDuctworkRuns < Ductwork::Migration
+  def change
+    add_index :ductwork_runs, :started_at
+    add_index :ductwork_runs, %i[pipeline_klass started_at]
+    add_index :ductwork_runs, %i[status started_at]
+  end
+end

data/lib/generators/ductwork/update/templates/db/add_indexes_to_ductwork_transitions.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+class AddIndexesToDuctworkTransitions < Ductwork::Migration
+  def change
+    if mysql?
+      add_index :ductwork_transitions,
+                %i[branch_id completed_at started_at],
+                name: "index_ductwork_transitions_on_latest_open"
+    else
+      add_index :ductwork_transitions,
+                %i[branch_id started_at],
+                where: "completed_at IS NULL",
+                name: "index_ductwork_transitions_on_latest_open"
+    end
+  end
+end