ductwork 0.26.0 → 1.0.0

data/lib/ductwork/models/branch_link.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Ductwork
+  class BranchLink < Ductwork::Record
+    belongs_to :parent_branch, class_name: "Ductwork::Branch"
+    belongs_to :child_branch, class_name: "Ductwork::Branch"
+
+    validates :parent_branch_id, uniqueness: { scope: :child_branch_id }
+  end
+end

data/lib/ductwork/models/execution.rb

@@ -1,13 +1,210 @@
 # frozen_string_literal: true
 
 module Ductwork
-  class Execution < Ductwork::Record
+  class Execution < Ductwork::Record # rubocop:todo Metrics/ClassLength
     belongs_to :job, class_name: "Ductwork::Job"
+    belongs_to :process, class_name: "Ductwork::Process", optional: true
     has_one :availability, class_name: "Ductwork::Availability", foreign_key: "execution_id", dependent: :destroy
-    has_one :run, class_name: "Ductwork::Run", foreign_key: "execution_id", dependent: :destroy
+    has_one :attempt, class_name: "Ductwork::Attempt", foreign_key: "execution_id", dependent: :destroy
     has_one :result, class_name: "Ductwork::Result", foreign_key: "execution_id", dependent: :destroy
 
     validates :retry_count, presence: true
     validates :started_at, presence: true
+
+    FAILED_EXECUTION_TIMEOUT = 10.seconds
+
+    class CommitFailed < StandardError; end
+
+    def call(pipeline, owner_process_id)
+      Ductwork.logger.debug(
+        msg: "Executing job",
+        role: :job_worker,
+        pipeline: pipeline,
+        job_klass: job.klass
+      )
+      args = JSON.parse(job.input_args)["args"]
+      instance = Object.const_get(job.klass).build_for_execution(job.step.run_id, *args)
+      create_attempt!(started_at: Time.current)
+      output_payload = nil
+
+      begin
+        output_payload = instance.execute
+        Ductwork::FaultInjection.checkpoint(:during_job_execution)
+      rescue StandardError => e
+        errored!(e, owner_process_id)
+        log_job_executed(pipeline, "error")
+
+        return
+      end
+
+      # AT-LEAST-ONCE CONTRACT: `instance.execute` has already run and any
+      # side effects it performed are now durable. The commit below can still
+      # fail (CommitFailed) if the reaper clobbered this claim, in which case
+      # `crashed!` creates a fresh availability and the job runs AGAIN.
+      # Ductwork guarantees at-least-once, never exactly-once, execution for
+      # the forking worker model: a job body may be re-run after a successful
+      # side effect. Jobs with non-idempotent effects MUST guard them at the
+      # application layer (use `Step#idempotency_key`). The reuse window is
+      # widened by transient stale claims; narrowing that is a separate fix,
+      # but it cannot eliminate this contract.
+      succeeded!(output_payload, owner_process_id)
+      log_job_executed(pipeline, "succeeded")
+    end
+
+    def succeeded!(output_payload, owner_process_id)
+      completed_at = Time.current
+      payload = JSON.dump({ payload: output_payload })
+
+      Ductwork::Record.transaction do
+        rows_updated = Ductwork::Execution
+                       .where(id: id, completed_at: nil, process_id: owner_process_id)
+                       .update_all(completed_at:)
+
+        if rows_updated.zero?
+          raise Ductwork::Execution::CommitFailed, "Reaper clobbered claimed job execution"
+        end
+
+        job.update!(output_payload: payload, completed_at: Time.current)
+        attempt.update!(completed_at: Time.current)
+        create_result!(result_type: "success")
+        job.step.update!(status: :advancing)
+      end
+    end
+
+    def crashed! # rubocop:todo Metrics
+      run = job.step.run
+      max_crash = Ductwork.configuration.job_worker_max_crash(
+        pipeline: run.pipeline_klass,
+        step: job.klass
+      )
+
+      Ductwork::Record.transaction do # rubocop:todo Metrics/BlockLength
+        rows_updated = Ductwork::Execution
+                       .where(id: id, process_id: process_id, completed_at: nil)
+                       .update_all(completed_at: Time.current)
+
+        return if rows_updated.zero?
+
+        reload
+        attempt&.update!(completed_at: Time.current)
+        create_result!(result_type: "process_crashed")
+
+        if crash_count < max_crash
+          new_crash_count = crash_count + 1
+          started_at = crash_backoff_at(new_crash_count, max_crash)
+
+          new_execution = job.executions.create!(
+            retry_count: retry_count,
+            crash_count: new_crash_count,
+            started_at: started_at
+          )
+          new_execution.create_availability!(
+            started_at: started_at,
+            pipeline_klass: run.pipeline_klass
+          )
+        else
+          job.step.update!(status: :failed)
+
+          Ductwork.logger.error(
+            msg: "Job exceeded crash limit and failed",
+            job_id: job.id,
+            job_klass: job.klass,
+            run_id: run.id,
+            role: :job_worker
+          )
+        end
+      end
+    end
+
+    def errored!(error, owner_process_id) # rubocop:todo Metrics
+      run = job.step.run
+      completed_at = Time.current
+      max_retry = Ductwork.configuration.job_worker_max_retry(
+        pipeline: run.pipeline_klass,
+        step: job.klass
+      )
+
+      Ductwork::Record.transaction do # rubocop:todo Metrics/BlockLength
+        rows_updated = Ductwork::Execution
+                       .where(id: id, completed_at: nil, process_id: owner_process_id)
+                       .update_all(completed_at:)
+
+        if rows_updated.zero?
+          raise Ductwork::Execution::CommitFailed, "Reaper clobbered claimed job execution"
+        end
+
+        attempt.update!(completed_at: Time.current)
+        create_result!(
+          result_type: "failure",
+          error_klass: error.class.to_s,
+          error_message: error.message,
+          error_backtrace: error.backtrace.join("\n")
+        )
+
+        if retry_count < max_retry
+          retry_at = Ductwork::DatabaseClock.now + FAILED_EXECUTION_TIMEOUT
+          new_execution = job.executions.create!(
+            retry_count: retry_count + 1,
+            crash_count: crash_count,
+            started_at: retry_at
+          )
+
+          new_execution.create_availability!(
+            started_at: retry_at,
+            pipeline_klass: run.pipeline_klass
+          )
+
+          Ductwork.logger.warn(
+            msg: "Job errored",
+            error_klass: error.class.name,
+            error_message: error.message,
+            job_id: job.id,
+            job_klass: job.klass,
+            run_id: run.id,
+            role: :job_worker
+          )
+        elsif retry_count >= max_retry
+          job.step.update!(status: :failed)
+
+          Ductwork.logger.error(
+            msg: "Job exhausted retries and failed",
+            error_klass: error.class.name,
+            error_message: error.message,
+            job_id: job.id,
+            job_klass: job.klass,
+            run_id: run.id,
+            role: :job_worker
+          )
+        end
+      end
+    end
+
+    private
+
+    # NOTE: the first third (floored) of the crash budget retries immediately
+    # so a one-off reaper clobber / deploy restart recovers fast; past that,
+    # delay linearly so sub-cap crash loops stop hammering the database and
+    # worker pool
+    def crash_backoff_at(crash_count, max_crash)
+      immediate_threshold = max_crash / 3
+      now = Ductwork::DatabaseClock.now
+
+      if crash_count <= immediate_threshold
+        now
+      else
+        now + (crash_count * FAILED_EXECUTION_TIMEOUT)
+      end
+    end
+
+    def log_job_executed(pipeline, result_status)
+      Ductwork.logger.info(
+        msg: "Job executed",
+        pipeline: pipeline,
+        job_id: job.id,
+        job_klass: job.klass,
+        result: result_status,
+        role: :job_worker
+      )
+    end
   end
 end

data/lib/ductwork/models/job.rb

@@ -9,25 +9,22 @@ module Ductwork
     validates :started_at, presence: true
     validates :input_args, presence: true
 
-    FAILED_EXECUTION_TIMEOUT = 10.seconds
-
-    def self.claim_latest(klass)
-      Ductwork::JobClaim.new(klass).latest
-    end
-
     def self.enqueue(step, *args)
+      now = Ductwork::DatabaseClock.now
       job = step.create_job!(
         klass: step.klass,
-        started_at: Time.current,
+        started_at: now,
         input_args: JSON.dump({ args: })
       )
       execution = job.executions.create!(
-        started_at: Time.current,
-        retry_count: 0
+        started_at: now,
+        retry_count: 0,
+        crash_count: 0
       )
+
       execution.create_availability!(
-        started_at: Time.current,
-        pipeline_klass: step.pipeline.klass
+        started_at: now,
+        pipeline_klass: step.run.pipeline_klass
       )
 
       Ductwork.logger.info(
@@ -39,120 +36,10 @@ module Ductwork
       job
     end
 
-    def execute(pipeline)
-      # i don't _really_ like this, but it should be fine for now...
-      execution = executions.order(:created_at).last
-      Ductwork.logger.debug(
-        msg: "Executing job",
-        role: :job_worker,
-        pipeline: pipeline,
-        job_klass: klass
-      )
-      args = JSON.parse(input_args)["args"]
-      instance = Object.const_get(klass).build_for_execution(step.pipeline_id, *args)
-      run = execution.create_run!(
-        started_at: Time.current
-      )
-      result = nil
-
-      begin
-        output_payload = instance.execute
-        execution_succeeded!(execution, run, output_payload)
-        result = "success"
-      rescue StandardError => e
-        execution_failed!(execution, run, e)
-        result = "failure"
-      ensure
-        Ductwork.logger.info(
-          msg: "Job executed",
-          pipeline: pipeline,
-          job_id: id,
-          job_klass: klass,
-          result: result || "killed",
-          role: :job_worker
-        )
-      end
-    end
-
     def return_value
       if output_payload.present?
         JSON.parse(output_payload).fetch("payload", nil)
       end
     end
-
-    private
-
-    def execution_succeeded!(execution, run, output_payload)
-      payload = JSON.dump({ payload: output_payload })
-
-      Ductwork::Record.transaction do
-        update!(output_payload: payload, completed_at: Time.current)
-        execution.update!(completed_at: Time.current)
-        run.update!(completed_at: Time.current)
-        execution.create_result!(result_type: "success")
-        step.update!(status: :advancing)
-      end
-    end
-
-    def execution_failed!(execution, run, error) # rubocop:todo Metrics
-      halted = false
-      pipeline = step.pipeline
-      max_retry = Ductwork
-                  .configuration
-                  .job_worker_max_retry(pipeline: pipeline.klass, step: klass)
-
-      Ductwork::Record.transaction do
-        execution.update!(completed_at: Time.current)
-        run.update!(completed_at: Time.current)
-        execution.create_result!(
-          result_type: "failure",
-          error_klass: error.class.to_s,
-          error_message: error.message,
-          error_backtrace: error.backtrace.join("\n")
-        )
-
-        if execution.retry_count < max_retry
-          new_execution = executions.create!(
-            retry_count: execution.retry_count + 1,
-            started_at: FAILED_EXECUTION_TIMEOUT.from_now
-          )
-          new_execution.create_availability!(
-            started_at: FAILED_EXECUTION_TIMEOUT.from_now,
-            pipeline_klass: pipeline.klass
-          )
-        elsif execution.retry_count >= max_retry
-          halted = true
-
-          step.update!(status: :failed)
-          pipeline.halt!
-        end
-      end
-
-      Ductwork.logger.warn(
-        msg: "Job errored",
-        error_klass: error.class.name,
-        error_message: error.message,
-        job_id: id,
-        job_klass: klass,
-        pipeline_id: pipeline.id,
-        role: :job_worker
-      )
-
-      # NOTE: perform lifecycle hook execution outside of the transaction as
-      # to not unnecessarily hold it open
-      if halted
-        execute_on_halt(pipeline, error)
-      end
-    end
-
-    def execute_on_halt(pipeline, error)
-      klass = JSON
-              .parse(pipeline.definition)
-              .dig("metadata", "on_halt", "klass")
-
-      if klass.present?
-        Object.const_get(klass).new(error).execute
-      end
-    end
   end
 end