cpflow 5.2.0 → 6.0.0.rc.0

data/lib/command/generate_github_actions.rb

@@ -16,7 +16,7 @@ module Command
     def copy_files
       relative_paths = generated_files
       replacements = template_variables
-      copy_template_files(relative_paths)
+      copy_generated_files(relative_paths)
       substitute_template_variables(relative_paths, replacements)
       make_shell_scripts_executable(relative_paths)
     end
@@ -27,11 +27,11 @@ module Command
 
     private
 
-    def copy_template_files(relative_paths)
+    def copy_generated_files(relative_paths)
       relative_paths.each do |relative_path|
         empty_directory(File.dirname(relative_path), verbose: false)
         copy_file(
-          File.join("github_flow_templates", relative_path),
+          GenerateGithubActions.source_file(relative_path),
           relative_path,
           force: true,
           verbose: ENV.fetch("HIDE_COMMAND_OUTPUT", nil) != "true"
@@ -99,6 +99,9 @@ module Command
       - manual promotion from staging to production
       - nightly cleanup and PR help workflows
 
+      It also copies cpflow's composite actions into `.github/actions/cpflow-*`
+      so every local `uses:` target is checked in and can be audited directly.
+
       Pass `--staging-branch BRANCH` when staging should auto-deploy from a branch
       other than `main` or `master`; the generator will bake that branch into the
       GitHub Actions push trigger and use it as the default STAGING_APP_BRANCH.
@@ -108,13 +111,13 @@ module Command
     DESC
     EXAMPLES = <<~EX
       ```sh
-      # Creates thin .github/workflows wrappers for the Control Plane flow
+      # Creates workflow wrappers, local composite actions, and validation helpers
       cpflow generate-github-actions
 
       # Creates the flow with staging deploys triggered from develop
       cpflow generate-github-actions --staging-branch develop
 
-      # Overwrites existing generated wrappers from the installed cpflow gem
+      # Overwrites existing generated GitHub Actions files from the installed cpflow gem
       cpflow generate-github-actions --force
       ```
     EX
@@ -122,22 +125,43 @@ module Command
     VALIDATIONS = [].freeze
     REQUIRES_STARTUP_CHECKS = false
 
-    # Resolve template root from __dir__ rather than Cpflow.root_path because this file is
+    # Resolve source roots from __dir__ rather than Cpflow.root_path because this file is
     # loaded before `module Cpflow` finishes defining its class methods.
+    REPOSITORY_ROOT = Pathname.new(File.expand_path("../..", __dir__))
     TEMPLATE_ROOT = Pathname.new(File.expand_path("../github_flow_templates", __dir__))
+    ACTIONS_ROOT = REPOSITORY_ROOT.join(".github/actions")
 
-    def self.generated_files
+    def self.generated_file_sources
       ensure_template_root!
 
-      Dir.glob(TEMPLATE_ROOT.join("**", "*").to_s, File::FNM_DOTMATCH)
-         .select { |path| File.file?(path) }
-         .map { |path| Pathname.new(path).relative_path_from(TEMPLATE_ROOT).to_s }
-         .sort
-         .freeze
+      template_sources = files_beneath(TEMPLATE_ROOT).to_h do |source|
+        [source.relative_path_from(TEMPLATE_ROOT).to_s, source]
+      end
+      action_sources = files_beneath(ACTIONS_ROOT, "cpflow-*/**/*").to_h do |source|
+        [source.relative_path_from(REPOSITORY_ROOT).to_s, source]
+      end
+
+      template_sources.merge(action_sources).sort.to_h.freeze
+    end
+
+    def self.generated_files
+      generated_file_sources.keys.freeze
+    end
+
+    def self.source_file(relative_path)
+      generated_file_sources.fetch(relative_path).to_s
     end
 
     def self.ensure_template_root!
       raise "cpflow template directory not found: #{TEMPLATE_ROOT}" unless TEMPLATE_ROOT.directory?
+      raise "cpflow action directory not found: #{ACTIONS_ROOT}" unless ACTIONS_ROOT.directory?
+    end
+
+    def self.files_beneath(root, pattern = "**/*")
+      Dir.glob(root.join(pattern).to_s, File::FNM_DOTMATCH)
+         .select { |path| File.file?(path) }
+         .map { |path| Pathname.new(path) }
+         .sort
     end
 
     def call

data/lib/command/run.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "timeout"
+
 module Command
   class Run < Base # rubocop:disable Metrics/ClassLength
     INTERACTIVE_COMMANDS = [
@@ -47,8 +49,27 @@ module Command
         and also overridden per job through `--cpu` and `--memory`)
       - By default, the job is stopped if it takes longer than 6 hours to finish
         (can be configured though `runner_job_timeout` in `controlplane.yml`)
+      - Waiting for a runner replica is limited to the smaller of `runner_job_timeout` and 1000 seconds.
+        A terminal cron status fails immediately, and reaching the observation deadline reports the last safe status
+      - With non-interactive log methods 2 and 3, after a command prints its completion marker, Control Plane
+        has up to 20 minutes to reconcile the cron job to a terminal status. This can be configured through
+        `runner_job_status_reconciliation_timeout` in `controlplane.yml`; timing out exits nonzero and reports
+        the job, replica, and last observed status
+      - Log method 1 does not emit a completion marker, so its job-status polling is not covered by the
+        post-command reconciliation timeout
       - Non-interactive jobs return the Control Plane cron job status even when the job finishes before
         Control Plane exposes a runner replica to attach logs to
+      - Injects `CPFLOW_GVC_ID` and `CPFLOW_GVC_CREATED` into the job, exposing the app's immutable GVC
+        identity, so that a command such as a release script can tell which GVC incarnation it is running in.
+        These change when a GVC is deleted and recreated under the same name, and only then, unlike
+        `CPLN_GVC_ALIAS`, which is also embedded in mutable derived values such as the app domain and so
+        cannot be attributed to recreation alone
+      - `CPFLOW_GVC_CREATED` is the GVC's creation timestamp as returned by the Control Plane API and passed
+        through unmodified, currently an ISO 8601 UTC timestamp with millisecond precision and a `Z` suffix
+        (e.g. `2026-08-28T00:54:48.648Z`)
+      - Both variables are always set, and are empty when the GVC cannot be read, so that a consumer can
+        fail closed. They are never omitted, because the runner inherits the original workload's
+        environment and an omitted variable could otherwise expose a stale inherited value
     DESC
     EXAMPLES = <<~EX.freeze
       ```sh
@@ -69,7 +90,8 @@ module Command
       # - stop the job
       cpflow run -a $APP_NAME --detached -- rails db:migrate
 
-      # The command needs to be quoted if setting an env variable or passing args.
+      # Quote the whole command to intentionally opt into shell syntax such as an env assignment.
+      # Separately supplied command arguments are passed literally.
       cpflow run -a $APP_NAME -- 'SOME_ENV_VAR=some_value rails db:migrate'
 
       # Uses a different image (which may not be promoted yet).
@@ -94,12 +116,24 @@ module Command
     DEFAULT_JOB_CPU = "1"
     DEFAULT_JOB_MEMORY = "2Gi"
     DEFAULT_JOB_TIMEOUT = 21_600 # 6 hours
+    DEFAULT_JOB_STATUS_RECONCILIATION_TIMEOUT = 1_200 # 20 minutes
     DEFAULT_JOB_HISTORY_LIMIT = 10
+    MAX_REPLICA_OBSERVATION_SECONDS = 1_000
+    REPLICA_OBSERVATION_POLL_INTERVAL_SECONDS = 1
+    LOG_QUERY_LOOKBACK_SECONDS = 60
+    POST_TERMINAL_LOG_DRAIN_SECONDS = 120
+    POST_TERMINAL_LOG_POLL_INTERVAL_SECONDS = 1
+    LOG_REQUEST_TIMEOUT_SECONDS = 30
+    JOB_STATUS_UNAVAILABLE_RETRY_LIMIT = 5
+    JOB_STATUS_POLL_INTERVAL_SECONDS = 1
+    JOB_STATUS_REQUEST_TIMEOUT_SECONDS = 30
+    NORMALIZED_JOB_STATUS_PATTERN = /\A[a-z][a-z0-9_-]{0,31}\z/
     MAGIC_END = "---cpflow run command finished---"
 
     attr_reader :interactive, :detached, :location, :original_workload, :runner_workload,
                 :default_image, :default_cpu, :default_memory, :job_timeout, :job_history_limit,
-                :container, :job, :replica, :command, :job_completed_before_replica_exit_status
+                :job_status_reconciliation_timeout, :container, :job, :replica, :command,
+                :job_completed_before_replica_exit_status
 
     def call # rubocop:disable Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
       @interactive = config.options[:interactive] || interactive_command?
@@ -113,6 +147,8 @@ module Command
       @default_cpu = config.current[:runner_job_default_cpu] || DEFAULT_JOB_CPU
       @default_memory = config.current[:runner_job_default_memory] || DEFAULT_JOB_MEMORY
       @job_timeout = config.current[:runner_job_timeout] || DEFAULT_JOB_TIMEOUT
+      @job_status_reconciliation_timeout =
+        config.current[:runner_job_status_reconciliation_timeout] || DEFAULT_JOB_STATUS_RECONCILIATION_TIMEOUT
       @job_history_limit = DEFAULT_JOB_HISTORY_LIMIT
 
       unless interactive
@@ -268,27 +304,80 @@ module Command
     end
 
     def wait_for_replica_for_job
-      step("Waiting for replica to start, which runs job '#{job}'", retry_on_failure: true) do
-        result = cp.fetch_workload_replicas(runner_workload, location: location)
-        @replica = result&.dig("items")&.find { |item| item.include?(job) }
+      observation_limit = [job_timeout, MAX_REPLICA_OBSERVATION_SECONDS].min
+      observation_deadline = monotonic_time + observation_limit
+
+      step("Waiting for runner replica to start") do
+        observe_replica_until(observation_deadline, observation_limit)
+      end
+    end
+
+    def observe_replica_until(observation_deadline, observation_limit)
+      last_status = nil
+
+      loop do
+        ensure_before_replica_observation_deadline!(observation_deadline, observation_limit, last_status)
+
+        @replica = replica_for_job
+        return replica if replica
+
+        ensure_before_replica_observation_deadline!(observation_deadline, observation_limit, last_status)
 
-        replica || completed_job_before_replica? || false
+        last_status = current_job_status
+        return true if completed_job_before_replica?(last_status)
+
+        sleep_before_replica_observation_retry(observation_deadline, observation_limit, last_status)
       end
     end
 
-    def completed_job_before_replica?
-      case current_job_status
+    def ensure_before_replica_observation_deadline!(observation_deadline, observation_limit, status)
+      return if monotonic_time < observation_deadline
+
+      raise replica_observation_timeout_message(observation_limit, status)
+    end
+
+    def replica_for_job
+      result = cp.fetch_workload_replicas(runner_workload, location: location)
+      result&.dig("items")&.find { |item| item.include?(job) }
+    end
+
+    def sleep_before_replica_observation_retry(observation_deadline, observation_limit, status)
+      progress.print(".")
+      remaining = observation_deadline - monotonic_time
+      raise replica_observation_timeout_message(observation_limit, status) unless remaining.positive?
+
+      Kernel.sleep([REPLICA_OBSERVATION_POLL_INTERVAL_SECONDS, remaining].min)
+    end
+
+    # Returns true for success, false while pending, and raises for a terminal non-success status.
+    def completed_job_before_replica?(status)
+      case status
       when "successful"
         @job_completed_before_replica_exit_status = ExitCode::SUCCESS
         true
       when nil, "active", "pending"
         false
       else
-        @job_completed_before_replica_exit_status = ExitCode::ERROR_DEFAULT
-        true
+        raise "Runner job ended before a replica was observed (status: #{normalized_job_status(status)})."
       end
     end
 
+    def normalized_job_status(status)
+      return "unavailable" if status.nil?
+
+      token = status.to_s.downcase
+      token.match?(NORMALIZED_JOB_STATUS_PATTERN) ? token : "unknown"
+    end
+
+    def replica_observation_timeout_message(observation_limit, status)
+      "Runner replica was not observed before the observation limit: #{format('%g', observation_limit)} seconds " \
+        "(status: #{normalized_job_status(status)})."
+    end
+
+    def monotonic_time
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+
     def run_interactive
       progress.puts("Connecting to replica '#{replica}'...\n\n")
       # workload_exec returns false on non-zero exit, nil when signal-killed (e.g. Ctrl-C).
@@ -389,11 +478,12 @@ module Command
       job_start_hash["args"].push("-c")
       job_start_hash["env"] ||= []
       job_start_hash["env"].push({ "name" => "CPFLOW_RUNNER_SCRIPT", "value" => runner_script })
+      job_start_hash["env"].concat(gvc_identity_env_vars)
       if interactive
         job_start_hash["env"].push({ "name" => "CPFLOW_MONITORING_SCRIPT", "value" => interactive_monitoring_script })
 
         job_start_hash["args"].push('eval "$CPFLOW_MONITORING_SCRIPT"')
-        @command = %(bash -c 'eval "$CPFLOW_RUNNER_SCRIPT"')
+        @command = ["bash", "-c", 'eval "$CPFLOW_RUNNER_SCRIPT"']
       else
         job_start_hash["args"].push('eval "$CPFLOW_RUNNER_SCRIPT"')
       end
@@ -413,6 +503,58 @@ module Command
       job_start_hash.to_yaml
     end
 
+    # Exposes the GVC's immutable identity to the job. cpflow authenticates client-side with the
+    # operator/CI credentials, so reading the GVC here adds no GVC-view binding to the app's workload
+    # identity and leaves nothing behind for the delete lifecycle to clean up.
+    #
+    # Both variables are always emitted, empty when unknown, rather than omitted. `update_runner_workload`
+    # copies the original workload's env wholesale into the runner, so omitting them would let a value
+    # inherited from the workload survive a failed read and be mistaken for a live identity. Emitting an
+    # explicit empty value is the only way a consumer can actually fail closed.
+    def gvc_identity_env_vars
+      gvc_data = fetch_gvc_for_identity
+      gvc_id = (gvc_data && gvc_data["id"]).to_s
+      # Gated on the id so a fresh id can never be paired with an inherited timestamp.
+      gvc_created = gvc_id.empty? ? "" : gvc_data["created"].to_s
+
+      [
+        { "name" => "CPFLOW_GVC_ID", "value" => gvc_id },
+        { "name" => "CPFLOW_GVC_CREATED", "value" => gvc_created }
+      ]
+    end
+
+    # The identity is an optional enrichment, so failing to read it must never stop the job, and the
+    # variables are still emitted (empty) rather than dropped.
+    # Two reasons the rescue is deliberately broad rather than a narrow class list:
+    #
+    # 1. Availability. `cpflow run` is also the release-phase mechanism for `deploy-image`, `setup-app`,
+    #    and `delete`, so a transient error on the GVC endpoint would otherwise fail a deploy over a
+    #    variable the command does not need.
+    # 2. Taxonomy. `handle_response` raises a bare `RuntimeError` for 401 and 5xx responses, so any
+    #    "narrow" list that actually covered the real failures would have to include `RuntimeError`.
+    #
+    # `MaintenanceMode#domain_workload_update_confirmed?` rescues `StandardError` against this same API
+    # client for the same reason. The rescued body is a single external call, so a bug in this file's own
+    # logic still raises.
+    def fetch_gvc_for_identity
+      cp.fetch_gvc
+    rescue StandardError => e
+      # Deliberately `Shell.warn` rather than a `step`. `step_finish` prints a red "failed!" banner
+      # whenever its block is falsy, regardless of `abort_on_error`, so routing an optional lookup
+      # through it would show a failure on every deploy for anyone whose token omits `gvc` view -- for
+      # a read that stops nothing and leaves the exit code at 0.
+      Shell.warn(gvc_identity_error_message(e))
+      nil
+    end
+
+    # A 403 here is normally a missing `view` grant on kind `gvc`, but `ForbiddenError` renders any
+    # `/org/...` URL as "Double check your org", which sends the operator after the wrong thing.
+    def gvc_identity_error_message(error)
+      "Continuing without the GVC identity, so CPFLOW_GVC_ID and CPFLOW_GVC_CREATED are set to empty " \
+        "strings. A permission failure here usually means the token lacks `view` on kind `gvc` for this " \
+        "app, even when the message below mentions the org. #{error.message}"
+    end
+
     def interactive_monitoring_script
       <<~SCRIPT
         primary_pid=""
@@ -473,10 +615,12 @@ module Command
         if @log_method == 1 || @interactive
           args_join(config.args)
         else
+          # MAGIC_END must be a same-stream barrier after all payload output. Merge stderr into stdout before
+          # capturing the payload exit status and emitting the marker.
           <<~SCRIPT
-            ( #{args_join(config.args)} )
+            ( #{args_join(config.args)} ) 2>&1
             CPFLOW_EXIT_CODE=$?
-            echo '#{MAGIC_END}'
+            printf '\\n%s\\n' '#{MAGIC_END}'
             exit $CPFLOW_EXIT_CODE
           SCRIPT
         end
@@ -491,6 +635,7 @@ module Command
 
     def wait_for_job_status_and_log(logs_pipe) # rubocop:disable Metrics/MethodLength
       no_logs_counter = 0
+      command_finished = false
 
       loop do
         no_logs_counter += 1
@@ -500,12 +645,16 @@ module Command
 
         no_logs_counter = 0
         line = logs_pipe.gets
-        break if line.chomp == MAGIC_END
+        if line.chomp == MAGIC_END
+          command_finished = true
+          break
+        end
 
         puts(line)
       end
 
-      resolve_job_status
+      status_deadline = job_status_reconciliation_deadline if command_finished
+      resolve_job_status(status_deadline: status_deadline)
     end
 
     def print_detached_commands
@@ -519,51 +668,198 @@ module Command
       )
     end
 
-    def resolve_job_status # rubocop:disable Metrics/MethodLength
+    # rubocop:disable Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+    def resolve_job_status(status_deadline: nil)
+      last_status = @last_job_status
       loop do
-        status = current_job_status
-
-        Shell.debug("JOB STATUS", status)
-
-        case status
-        when "active", "pending"
-          sleep 1
-        when "successful"
-          break ExitCode::SUCCESS
-        else
+        remaining_seconds = status_deadline && (status_deadline - monotonic_time)
+        if remaining_seconds && remaining_seconds <= 0
+          progress.puts(Shell.color(job_status_reconciliation_timeout_message(last_status), :red))
           break ExitCode::ERROR_DEFAULT
         end
+
+        request_timeout = if remaining_seconds
+                            [JOB_STATUS_REQUEST_TIMEOUT_SECONDS, remaining_seconds].min
+                          else
+                            JOB_STATUS_REQUEST_TIMEOUT_SECONDS
+                          end
+        begin
+          status = current_job_status(timeout_seconds: request_timeout)
+        rescue Shell::CommandTimeout
+          next
+        end
+        last_status = status
+        @last_job_status = status
+
+        exit_status = job_exit_status(status, unavailable_is_pending: false)
+        break exit_status if exit_status
+
+        sleep_seconds = JOB_STATUS_POLL_INTERVAL_SECONDS
+        sleep_seconds = [sleep_seconds, remaining_seconds].min if remaining_seconds
+        Kernel.sleep(sleep_seconds) if sleep_seconds.positive?
+      end
+    end
+
+    def reconciled_job_status(unavailable_status_streak, reconciliation_deadline, reconciliation_deadline_origin,
+                              request_timeout:)
+      status = begin
+        current_job_status(timeout_seconds: request_timeout)
+      rescue Shell::CommandTimeout
+        nil
       end
+      next_unavailable_status_streak = status.nil? ? unavailable_status_streak + 1 : 0
+      exit_status = job_exit_status(status, unavailable_is_pending: status.nil?)
+
+      if next_unavailable_status_streak > JOB_STATUS_UNAVAILABLE_RETRY_LIMIT && reconciliation_deadline.nil?
+        reconciliation_deadline = start_reconciliation_deadline(
+          reconciliation_deadline,
+          duration: POST_TERMINAL_LOG_DRAIN_SECONDS
+        )
+        reconciliation_deadline_origin = :status_outage
+      elsif reconciliation_deadline_origin == :status_outage && %w[active pending].include?(status)
+        reconciliation_deadline = reconciliation_deadline_origin = @post_terminal_log_from = nil
+      end
+
+      [exit_status, next_unavailable_status_streak, status, reconciliation_deadline, reconciliation_deadline_origin]
     end
 
-    def current_job_status
-      result = cp.fetch_cron_workload(runner_workload, location: location)
+    def job_exit_status(status, unavailable_is_pending:)
+      Shell.debug("JOB STATUS", normalized_job_status(status))
+
+      case status
+      when nil then unavailable_is_pending ? nil : ExitCode::ERROR_DEFAULT
+      when "active", "pending" then nil
+      when "successful" then ExitCode::SUCCESS
+      else ExitCode::ERROR_DEFAULT
+      end
+    end
+    # rubocop:enable Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+
+    def current_job_status(timeout_seconds: nil)
+      result = if timeout_seconds
+                 cp.fetch_cron_workload(runner_workload, location: location, timeout_seconds: timeout_seconds)
+               else
+                 cp.fetch_cron_workload(runner_workload, location: location)
+               end
       job_details = result&.dig("items")&.find { |item| item["id"] == job }
       job_details&.dig("status")
     end
 
+    def job_status_reconciliation_deadline
+      monotonic_time + job_status_reconciliation_timeout
+    end
+
+    def job_status_reconciliation_timeout_message(status)
+      replica_args = app_workload_replica_args.join(" ")
+
+      <<~MESSAGE.chomp
+        ERROR: Control Plane job status did not reconcile within #{job_status_reconciliation_timeout} seconds after the command finished.
+        job: #{job}
+        replica: #{replica}
+        status: #{status || 'unknown'}
+        Inspect logs: cpflow logs #{replica_args}
+        Stop the stale runner: cpflow ps:stop #{replica_args}
+      MESSAGE
+    end
+
     ###########################################
     ### temporary extaction from run:detached
     ###########################################
-    def show_logs_waiting # rubocop:disable Metrics/MethodLength
+    # Tracks log completion, authoritative job status, provisional status outages, and bounded deadlines.
+    def show_logs_waiting # rubocop:disable Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
       retries = 0
+      exit_status = nil
+      reconciliation_deadline = nil
+      reconciliation_deadline_origin = nil
+      unavailable_status_streak = 0
+      finish_marker_seen = false
+      last_job_status = nil
+
       begin
-        job_finished_count = 0
+        # rubocop:disable Metrics/BlockLength
         loop do
-          case print_uniq_logs
-          when :finished
-            break
-          when :changed
-            next
-          else
-            job_finished_count += 1 if resolve_job_status
-            break if job_finished_count > 5
-
-            sleep(1)
+          remaining = reconciliation_deadline && (reconciliation_deadline - monotonic_time)
+          break if remaining && !remaining.positive?
+
+          unless finish_marker_seen
+            log_request_timeout = remaining ? [LOG_REQUEST_TIMEOUT_SECONDS, remaining].min : LOG_REQUEST_TIMEOUT_SECONDS
+            log_status = begin
+              print_uniq_logs(timeout_seconds: log_request_timeout)
+            rescue Shell::CommandTimeout
+              :unchanged
+            end
+            finish_marker_seen = log_status == :finished
+            if finish_marker_seen && reconciliation_deadline.nil?
+              reconciliation_deadline = start_reconciliation_deadline(
+                reconciliation_deadline,
+                duration: job_status_reconciliation_timeout || POST_TERMINAL_LOG_DRAIN_SECONDS
+              )
+              reconciliation_deadline_origin = :finish_marker
+            elsif finish_marker_seen && reconciliation_deadline_origin == :status_outage
+              reconciliation_deadline = monotonic_time +
+                                        (job_status_reconciliation_timeout || POST_TERMINAL_LOG_DRAIN_SECONDS)
+              reconciliation_deadline_origin = :finish_marker
+            end
           end
-        end
+          remaining = reconciliation_deadline && (reconciliation_deadline - monotonic_time)
+          break if remaining && !remaining.positive?
+
+          request_timeout = if remaining
+                              [JOB_STATUS_REQUEST_TIMEOUT_SECONDS, remaining].min
+                            else
+                              JOB_STATUS_REQUEST_TIMEOUT_SECONDS
+                            end
+
+          exit_status ||= begin
+            job_status_state = reconciled_job_status(
+              unavailable_status_streak, reconciliation_deadline, reconciliation_deadline_origin,
+              request_timeout: request_timeout
+            )
+            current_exit_status, unavailable_status_streak, observed_status,
+              reconciliation_deadline, reconciliation_deadline_origin = job_status_state
+            last_job_status = observed_status unless observed_status.nil?
+            current_exit_status
+          end
+          if exit_status && reconciliation_deadline_origin == :status_outage
+            reconciliation_deadline = monotonic_time + POST_TERMINAL_LOG_DRAIN_SECONDS
+            reconciliation_deadline_origin = :terminal
+          elsif exit_status && reconciliation_deadline.nil?
+            reconciliation_deadline = start_reconciliation_deadline(
+              reconciliation_deadline,
+              duration: POST_TERMINAL_LOG_DRAIN_SECONDS
+            )
+            reconciliation_deadline_origin = :terminal
+          end
+          break if finish_marker_seen && exit_status
+
+          remaining = reconciliation_deadline && (reconciliation_deadline - monotonic_time)
+          break if remaining && !remaining.positive?
 
-        resolve_job_status
+          poll_interval = POST_TERMINAL_LOG_POLL_INTERVAL_SECONDS
+          poll_interval = [poll_interval, remaining].min if remaining
+          Kernel.sleep(poll_interval)
+        end
+        # rubocop:enable Metrics/BlockLength
+
+        if exit_status && finish_marker_seen
+          exit_status
+        elsif exit_status
+          Shell.warn(
+            "Runner job reached terminal status #{normalized_job_status(last_job_status)} but the command " \
+            "completion marker was unavailable after #{POST_TERMINAL_LOG_DRAIN_SECONDS} seconds; " \
+            "returning authoritative job exit status #{exit_status} with incomplete output."
+          )
+          exit_status
+        elsif reconciliation_deadline_origin == :finish_marker && job_status_reconciliation_timeout
+          progress.puts(Shell.color(job_status_reconciliation_timeout_message(last_job_status), :red))
+          ExitCode::ERROR_DEFAULT
+        else
+          Shell.warn(
+            "Runner job status reconciliation reached the #{POST_TERMINAL_LOG_DRAIN_SECONDS}-second deadline " \
+            "(last_status: #{normalized_job_status(last_job_status)}); returning exit status #{ExitCode::ERROR_DEFAULT}."
+          )
+          ExitCode::ERROR_DEFAULT
+        end
       rescue RuntimeError => e
         raise "#{e} Exiting..." unless retries < 10 # MAX_RETRIES
 
@@ -573,12 +869,20 @@ module Command
       end
     end
 
-    def print_uniq_logs
+    def start_reconciliation_deadline(current_deadline, duration:)
+      return current_deadline if current_deadline
+
+      # Freeze the pre-terminal overlap so late-ingested entries do not age out of the query window.
+      @post_terminal_log_from = Time.now.to_i - LOG_QUERY_LOOKBACK_SECONDS
+      monotonic_time + duration
+    end
+
+    def print_uniq_logs(timeout_seconds: nil)
       status = nil
 
       @printed_log_entries ||= []
       ts = Time.now.to_i
-      entries = normalized_log_entries(from: ts - 60, to: ts)
+      entries = fetch_log_entries(ts, timeout_seconds)
 
       (entries - @printed_log_entries).sort.each do |(_ts, val)|
         status ||= :changed
@@ -590,6 +894,15 @@ module Command
       status || :unchanged
     end
 
+    def fetch_log_entries(to, timeout_seconds)
+      from = @post_terminal_log_from || (to - LOG_QUERY_LOOKBACK_SECONDS)
+      return normalized_log_entries(from: from, to: to) unless timeout_seconds
+
+      Timeout.timeout(timeout_seconds, Shell::CommandTimeout) do
+        normalized_log_entries(from: from, to: to)
+      end
+    end
+
     def normalized_log_entries(from:, to:)
       log = cp.log_get(workload: runner_workload, from: from, to: to, replica: replica)