cpflow 5.3.0 → 6.0.0.rc.0

data/lib/command/deploy_image.rb

@@ -4,6 +4,12 @@ require "resolv"
 
 module Command
   class DeployImage < Base # rubocop:disable Metrics/ClassLength
+    WORKLOAD_IMAGE_UPDATE_MAX_ATTEMPTS = 30
+    # Preserve the general failure cap; only optimistic-concurrency conflicts get the longer convergence window.
+    WORKLOAD_IMAGE_UPDATE_CONFLICT_MAX_ATTEMPTS = 120
+    WORKLOAD_IMAGE_UPDATE_CONFLICT_PATTERN =
+      /(?:\b409\b|(?:\A|\n)[ \t]*(?:error:[ \t]*)?conflict[ \t]*(?:\r?\n|\z))/i
+
     NAME = "deploy-image"
     OPTIONS = [
       app_option(required: true),
@@ -122,7 +128,8 @@ module Command
 
           container_name = container["name"]
           step("Deploying image '#{image}' for workload '#{workload}'") do
-            cp.workload_set_image_ref(workload, container: container_name, image: image)
+            update_workload_image_ref(workload, container_name, image)
+
             deployed_endpoints[workload] = endpoint_for_workload(workload_data)
             # A missing public endpoint is valid; the image update still completed successfully.
             true
@@ -136,6 +143,36 @@ module Command
       deployed_endpoints
     end
 
+    def update_workload_image_ref(workload, container, image)
+      attempts = 0
+
+      loop do
+        attempts += 1
+        result = cp.workload_set_image_ref(workload, container: container, image: image)
+        return true if result.fetch(:success)
+
+        output = result.fetch(:output).to_s
+        raise workload_image_update_error(output) if attempts >= workload_image_update_limit(output)
+
+        wait_before_workload_image_update_retry
+      end
+    end
+
+    def workload_image_update_limit(output)
+      return WORKLOAD_IMAGE_UPDATE_CONFLICT_MAX_ATTEMPTS if output.match?(WORKLOAD_IMAGE_UPDATE_CONFLICT_PATTERN)
+
+      WORKLOAD_IMAGE_UPDATE_MAX_ATTEMPTS
+    end
+
+    def workload_image_update_error(output)
+      output.strip.empty? ? "Command exited with non-zero status." : output
+    end
+
+    def wait_before_workload_image_update_retry
+      progress.print(".")
+      Kernel.sleep(1)
+    end
+
     def print_deployed_endpoints(deployed_endpoints)
       progress.puts("\nDeployed endpoints:")
       deployed_endpoints.each do |workload, endpoint|

data/lib/command/generate_github_actions.rb

@@ -16,7 +16,7 @@ module Command
     def copy_files
       relative_paths = generated_files
       replacements = template_variables
-      copy_template_files(relative_paths)
+      copy_generated_files(relative_paths)
       substitute_template_variables(relative_paths, replacements)
       make_shell_scripts_executable(relative_paths)
     end
@@ -27,11 +27,11 @@ module Command
 
     private
 
-    def copy_template_files(relative_paths)
+    def copy_generated_files(relative_paths)
       relative_paths.each do |relative_path|
         empty_directory(File.dirname(relative_path), verbose: false)
         copy_file(
-          File.join("github_flow_templates", relative_path),
+          GenerateGithubActions.source_file(relative_path),
           relative_path,
           force: true,
           verbose: ENV.fetch("HIDE_COMMAND_OUTPUT", nil) != "true"
@@ -99,6 +99,9 @@ module Command
       - manual promotion from staging to production
       - nightly cleanup and PR help workflows
 
+      It also copies cpflow's composite actions into `.github/actions/cpflow-*`
+      so every local `uses:` target is checked in and can be audited directly.
+
       Pass `--staging-branch BRANCH` when staging should auto-deploy from a branch
       other than `main` or `master`; the generator will bake that branch into the
       GitHub Actions push trigger and use it as the default STAGING_APP_BRANCH.
@@ -108,13 +111,13 @@ module Command
     DESC
     EXAMPLES = <<~EX
       ```sh
-      # Creates thin .github/workflows wrappers for the Control Plane flow
+      # Creates workflow wrappers, local composite actions, and validation helpers
       cpflow generate-github-actions
 
       # Creates the flow with staging deploys triggered from develop
       cpflow generate-github-actions --staging-branch develop
 
-      # Overwrites existing generated wrappers from the installed cpflow gem
+      # Overwrites existing generated GitHub Actions files from the installed cpflow gem
       cpflow generate-github-actions --force
       ```
     EX
@@ -122,22 +125,43 @@ module Command
     VALIDATIONS = [].freeze
     REQUIRES_STARTUP_CHECKS = false
 
-    # Resolve template root from __dir__ rather than Cpflow.root_path because this file is
+    # Resolve source roots from __dir__ rather than Cpflow.root_path because this file is
     # loaded before `module Cpflow` finishes defining its class methods.
+    REPOSITORY_ROOT = Pathname.new(File.expand_path("../..", __dir__))
     TEMPLATE_ROOT = Pathname.new(File.expand_path("../github_flow_templates", __dir__))
+    ACTIONS_ROOT = REPOSITORY_ROOT.join(".github/actions")
 
-    def self.generated_files
+    def self.generated_file_sources
       ensure_template_root!
 
-      Dir.glob(TEMPLATE_ROOT.join("**", "*").to_s, File::FNM_DOTMATCH)
-         .select { |path| File.file?(path) }
-         .map { |path| Pathname.new(path).relative_path_from(TEMPLATE_ROOT).to_s }
-         .sort
-         .freeze
+      template_sources = files_beneath(TEMPLATE_ROOT).to_h do |source|
+        [source.relative_path_from(TEMPLATE_ROOT).to_s, source]
+      end
+      action_sources = files_beneath(ACTIONS_ROOT, "cpflow-*/**/*").to_h do |source|
+        [source.relative_path_from(REPOSITORY_ROOT).to_s, source]
+      end
+
+      template_sources.merge(action_sources).sort.to_h.freeze
+    end
+
+    def self.generated_files
+      generated_file_sources.keys.freeze
+    end
+
+    def self.source_file(relative_path)
+      generated_file_sources.fetch(relative_path).to_s
     end
 
     def self.ensure_template_root!
       raise "cpflow template directory not found: #{TEMPLATE_ROOT}" unless TEMPLATE_ROOT.directory?
+      raise "cpflow action directory not found: #{ACTIONS_ROOT}" unless ACTIONS_ROOT.directory?
+    end
+
+    def self.files_beneath(root, pattern = "**/*")
+      Dir.glob(root.join(pattern).to_s, File::FNM_DOTMATCH)
+         .select { |path| File.file?(path) }
+         .map { |path| Pathname.new(path) }
+         .sort
     end
 
     def call

data/lib/command/run.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "timeout"
+
 module Command
   class Run < Base # rubocop:disable Metrics/ClassLength
     INTERACTIVE_COMMANDS = [
@@ -49,6 +51,12 @@ module Command
         (can be configured though `runner_job_timeout` in `controlplane.yml`)
       - Waiting for a runner replica is limited to the smaller of `runner_job_timeout` and 1000 seconds.
         A terminal cron status fails immediately, and reaching the observation deadline reports the last safe status
+      - With non-interactive log methods 2 and 3, after a command prints its completion marker, Control Plane
+        has up to 20 minutes to reconcile the cron job to a terminal status. This can be configured through
+        `runner_job_status_reconciliation_timeout` in `controlplane.yml`; timing out exits nonzero and reports
+        the job, replica, and last observed status
+      - Log method 1 does not emit a completion marker, so its job-status polling is not covered by the
+        post-command reconciliation timeout
       - Non-interactive jobs return the Control Plane cron job status even when the job finishes before
         Control Plane exposes a runner replica to attach logs to
       - Injects `CPFLOW_GVC_ID` and `CPFLOW_GVC_CREATED` into the job, exposing the app's immutable GVC
@@ -108,15 +116,24 @@ module Command
     DEFAULT_JOB_CPU = "1"
     DEFAULT_JOB_MEMORY = "2Gi"
     DEFAULT_JOB_TIMEOUT = 21_600 # 6 hours
+    DEFAULT_JOB_STATUS_RECONCILIATION_TIMEOUT = 1_200 # 20 minutes
     DEFAULT_JOB_HISTORY_LIMIT = 10
     MAX_REPLICA_OBSERVATION_SECONDS = 1_000
     REPLICA_OBSERVATION_POLL_INTERVAL_SECONDS = 1
+    LOG_QUERY_LOOKBACK_SECONDS = 60
+    POST_TERMINAL_LOG_DRAIN_SECONDS = 120
+    POST_TERMINAL_LOG_POLL_INTERVAL_SECONDS = 1
+    LOG_REQUEST_TIMEOUT_SECONDS = 30
+    JOB_STATUS_UNAVAILABLE_RETRY_LIMIT = 5
+    JOB_STATUS_POLL_INTERVAL_SECONDS = 1
+    JOB_STATUS_REQUEST_TIMEOUT_SECONDS = 30
     NORMALIZED_JOB_STATUS_PATTERN = /\A[a-z][a-z0-9_-]{0,31}\z/
     MAGIC_END = "---cpflow run command finished---"
 
     attr_reader :interactive, :detached, :location, :original_workload, :runner_workload,
                 :default_image, :default_cpu, :default_memory, :job_timeout, :job_history_limit,
-                :container, :job, :replica, :command, :job_completed_before_replica_exit_status
+                :job_status_reconciliation_timeout, :container, :job, :replica, :command,
+                :job_completed_before_replica_exit_status
 
     def call # rubocop:disable Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
       @interactive = config.options[:interactive] || interactive_command?
@@ -130,6 +147,8 @@ module Command
       @default_cpu = config.current[:runner_job_default_cpu] || DEFAULT_JOB_CPU
       @default_memory = config.current[:runner_job_default_memory] || DEFAULT_JOB_MEMORY
       @job_timeout = config.current[:runner_job_timeout] || DEFAULT_JOB_TIMEOUT
+      @job_status_reconciliation_timeout =
+        config.current[:runner_job_status_reconciliation_timeout] || DEFAULT_JOB_STATUS_RECONCILIATION_TIMEOUT
       @job_history_limit = DEFAULT_JOB_HISTORY_LIMIT
 
       unless interactive
@@ -596,10 +615,12 @@ module Command
         if @log_method == 1 || @interactive
           args_join(config.args)
         else
+          # MAGIC_END must be a same-stream barrier after all payload output. Merge stderr into stdout before
+          # capturing the payload exit status and emitting the marker.
           <<~SCRIPT
-            ( #{args_join(config.args)} )
+            ( #{args_join(config.args)} ) 2>&1
             CPFLOW_EXIT_CODE=$?
-            echo '#{MAGIC_END}'
+            printf '\\n%s\\n' '#{MAGIC_END}'
             exit $CPFLOW_EXIT_CODE
           SCRIPT
         end
@@ -614,6 +635,7 @@ module Command
 
     def wait_for_job_status_and_log(logs_pipe) # rubocop:disable Metrics/MethodLength
       no_logs_counter = 0
+      command_finished = false
 
       loop do
         no_logs_counter += 1
@@ -623,12 +645,16 @@ module Command
 
         no_logs_counter = 0
         line = logs_pipe.gets
-        break if line.chomp == MAGIC_END
+        if line.chomp == MAGIC_END
+          command_finished = true
+          break
+        end
 
         puts(line)
       end
 
-      resolve_job_status
+      status_deadline = job_status_reconciliation_deadline if command_finished
+      resolve_job_status(status_deadline: status_deadline)
     end
 
     def print_detached_commands
@@ -642,51 +668,198 @@ module Command
       )
     end
 
-    def resolve_job_status # rubocop:disable Metrics/MethodLength
+    # rubocop:disable Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+    def resolve_job_status(status_deadline: nil)
+      last_status = @last_job_status
       loop do
-        status = current_job_status
-
-        Shell.debug("JOB STATUS", status)
-
-        case status
-        when "active", "pending"
-          sleep 1
-        when "successful"
-          break ExitCode::SUCCESS
-        else
+        remaining_seconds = status_deadline && (status_deadline - monotonic_time)
+        if remaining_seconds && remaining_seconds <= 0
+          progress.puts(Shell.color(job_status_reconciliation_timeout_message(last_status), :red))
           break ExitCode::ERROR_DEFAULT
         end
+
+        request_timeout = if remaining_seconds
+                            [JOB_STATUS_REQUEST_TIMEOUT_SECONDS, remaining_seconds].min
+                          else
+                            JOB_STATUS_REQUEST_TIMEOUT_SECONDS
+                          end
+        begin
+          status = current_job_status(timeout_seconds: request_timeout)
+        rescue Shell::CommandTimeout
+          next
+        end
+        last_status = status
+        @last_job_status = status
+
+        exit_status = job_exit_status(status, unavailable_is_pending: false)
+        break exit_status if exit_status
+
+        sleep_seconds = JOB_STATUS_POLL_INTERVAL_SECONDS
+        sleep_seconds = [sleep_seconds, remaining_seconds].min if remaining_seconds
+        Kernel.sleep(sleep_seconds) if sleep_seconds.positive?
       end
     end
 
-    def current_job_status
-      result = cp.fetch_cron_workload(runner_workload, location: location)
+    def reconciled_job_status(unavailable_status_streak, reconciliation_deadline, reconciliation_deadline_origin,
+                              request_timeout:)
+      status = begin
+        current_job_status(timeout_seconds: request_timeout)
+      rescue Shell::CommandTimeout
+        nil
+      end
+      next_unavailable_status_streak = status.nil? ? unavailable_status_streak + 1 : 0
+      exit_status = job_exit_status(status, unavailable_is_pending: status.nil?)
+
+      if next_unavailable_status_streak > JOB_STATUS_UNAVAILABLE_RETRY_LIMIT && reconciliation_deadline.nil?
+        reconciliation_deadline = start_reconciliation_deadline(
+          reconciliation_deadline,
+          duration: POST_TERMINAL_LOG_DRAIN_SECONDS
+        )
+        reconciliation_deadline_origin = :status_outage
+      elsif reconciliation_deadline_origin == :status_outage && %w[active pending].include?(status)
+        reconciliation_deadline = reconciliation_deadline_origin = @post_terminal_log_from = nil
+      end
+
+      [exit_status, next_unavailable_status_streak, status, reconciliation_deadline, reconciliation_deadline_origin]
+    end
+
+    def job_exit_status(status, unavailable_is_pending:)
+      Shell.debug("JOB STATUS", normalized_job_status(status))
+
+      case status
+      when nil then unavailable_is_pending ? nil : ExitCode::ERROR_DEFAULT
+      when "active", "pending" then nil
+      when "successful" then ExitCode::SUCCESS
+      else ExitCode::ERROR_DEFAULT
+      end
+    end
+    # rubocop:enable Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+
+    def current_job_status(timeout_seconds: nil)
+      result = if timeout_seconds
+                 cp.fetch_cron_workload(runner_workload, location: location, timeout_seconds: timeout_seconds)
+               else
+                 cp.fetch_cron_workload(runner_workload, location: location)
+               end
       job_details = result&.dig("items")&.find { |item| item["id"] == job }
       job_details&.dig("status")
     end
 
+    def job_status_reconciliation_deadline
+      monotonic_time + job_status_reconciliation_timeout
+    end
+
+    def job_status_reconciliation_timeout_message(status)
+      replica_args = app_workload_replica_args.join(" ")
+
+      <<~MESSAGE.chomp
+        ERROR: Control Plane job status did not reconcile within #{job_status_reconciliation_timeout} seconds after the command finished.
+        job: #{job}
+        replica: #{replica}
+        status: #{status || 'unknown'}
+        Inspect logs: cpflow logs #{replica_args}
+        Stop the stale runner: cpflow ps:stop #{replica_args}
+      MESSAGE
+    end
+
     ###########################################
     ### temporary extaction from run:detached
     ###########################################
-    def show_logs_waiting # rubocop:disable Metrics/MethodLength
+    # Tracks log completion, authoritative job status, provisional status outages, and bounded deadlines.
+    def show_logs_waiting # rubocop:disable Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
       retries = 0
+      exit_status = nil
+      reconciliation_deadline = nil
+      reconciliation_deadline_origin = nil
+      unavailable_status_streak = 0
+      finish_marker_seen = false
+      last_job_status = nil
+
       begin
-        job_finished_count = 0
+        # rubocop:disable Metrics/BlockLength
         loop do
-          case print_uniq_logs
-          when :finished
-            break
-          when :changed
-            next
-          else
-            job_finished_count += 1 if resolve_job_status
-            break if job_finished_count > 5
-
-            sleep(1)
+          remaining = reconciliation_deadline && (reconciliation_deadline - monotonic_time)
+          break if remaining && !remaining.positive?
+
+          unless finish_marker_seen
+            log_request_timeout = remaining ? [LOG_REQUEST_TIMEOUT_SECONDS, remaining].min : LOG_REQUEST_TIMEOUT_SECONDS
+            log_status = begin
+              print_uniq_logs(timeout_seconds: log_request_timeout)
+            rescue Shell::CommandTimeout
+              :unchanged
+            end
+            finish_marker_seen = log_status == :finished
+            if finish_marker_seen && reconciliation_deadline.nil?
+              reconciliation_deadline = start_reconciliation_deadline(
+                reconciliation_deadline,
+                duration: job_status_reconciliation_timeout || POST_TERMINAL_LOG_DRAIN_SECONDS
+              )
+              reconciliation_deadline_origin = :finish_marker
+            elsif finish_marker_seen && reconciliation_deadline_origin == :status_outage
+              reconciliation_deadline = monotonic_time +
+                                        (job_status_reconciliation_timeout || POST_TERMINAL_LOG_DRAIN_SECONDS)
+              reconciliation_deadline_origin = :finish_marker
+            end
           end
-        end
+          remaining = reconciliation_deadline && (reconciliation_deadline - monotonic_time)
+          break if remaining && !remaining.positive?
+
+          request_timeout = if remaining
+                              [JOB_STATUS_REQUEST_TIMEOUT_SECONDS, remaining].min
+                            else
+                              JOB_STATUS_REQUEST_TIMEOUT_SECONDS
+                            end
+
+          exit_status ||= begin
+            job_status_state = reconciled_job_status(
+              unavailable_status_streak, reconciliation_deadline, reconciliation_deadline_origin,
+              request_timeout: request_timeout
+            )
+            current_exit_status, unavailable_status_streak, observed_status,
+              reconciliation_deadline, reconciliation_deadline_origin = job_status_state
+            last_job_status = observed_status unless observed_status.nil?
+            current_exit_status
+          end
+          if exit_status && reconciliation_deadline_origin == :status_outage
+            reconciliation_deadline = monotonic_time + POST_TERMINAL_LOG_DRAIN_SECONDS
+            reconciliation_deadline_origin = :terminal
+          elsif exit_status && reconciliation_deadline.nil?
+            reconciliation_deadline = start_reconciliation_deadline(
+              reconciliation_deadline,
+              duration: POST_TERMINAL_LOG_DRAIN_SECONDS
+            )
+            reconciliation_deadline_origin = :terminal
+          end
+          break if finish_marker_seen && exit_status
+
+          remaining = reconciliation_deadline && (reconciliation_deadline - monotonic_time)
+          break if remaining && !remaining.positive?
 
-        resolve_job_status
+          poll_interval = POST_TERMINAL_LOG_POLL_INTERVAL_SECONDS
+          poll_interval = [poll_interval, remaining].min if remaining
+          Kernel.sleep(poll_interval)
+        end
+        # rubocop:enable Metrics/BlockLength
+
+        if exit_status && finish_marker_seen
+          exit_status
+        elsif exit_status
+          Shell.warn(
+            "Runner job reached terminal status #{normalized_job_status(last_job_status)} but the command " \
+            "completion marker was unavailable after #{POST_TERMINAL_LOG_DRAIN_SECONDS} seconds; " \
+            "returning authoritative job exit status #{exit_status} with incomplete output."
+          )
+          exit_status
+        elsif reconciliation_deadline_origin == :finish_marker && job_status_reconciliation_timeout
+          progress.puts(Shell.color(job_status_reconciliation_timeout_message(last_job_status), :red))
+          ExitCode::ERROR_DEFAULT
+        else
+          Shell.warn(
+            "Runner job status reconciliation reached the #{POST_TERMINAL_LOG_DRAIN_SECONDS}-second deadline " \
+            "(last_status: #{normalized_job_status(last_job_status)}); returning exit status #{ExitCode::ERROR_DEFAULT}."
+          )
+          ExitCode::ERROR_DEFAULT
+        end
       rescue RuntimeError => e
         raise "#{e} Exiting..." unless retries < 10 # MAX_RETRIES
 
@@ -696,12 +869,20 @@ module Command
       end
     end
 
-    def print_uniq_logs
+    def start_reconciliation_deadline(current_deadline, duration:)
+      return current_deadline if current_deadline
+
+      # Freeze the pre-terminal overlap so late-ingested entries do not age out of the query window.
+      @post_terminal_log_from = Time.now.to_i - LOG_QUERY_LOOKBACK_SECONDS
+      monotonic_time + duration
+    end
+
+    def print_uniq_logs(timeout_seconds: nil)
       status = nil
 
       @printed_log_entries ||= []
       ts = Time.now.to_i
-      entries = normalized_log_entries(from: ts - 60, to: ts)
+      entries = fetch_log_entries(ts, timeout_seconds)
 
       (entries - @printed_log_entries).sort.each do |(_ts, val)|
         status ||= :changed
@@ -713,6 +894,15 @@ module Command
       status || :unchanged
     end
 
+    def fetch_log_entries(to, timeout_seconds)
+      from = @post_terminal_log_from || (to - LOG_QUERY_LOOKBACK_SECONDS)
+      return normalized_log_entries(from: from, to: to) unless timeout_seconds
+
+      Timeout.timeout(timeout_seconds, Shell::CommandTimeout) do
+        normalized_log_entries(from: from, to: to)
+      end
+    end
+
     def normalized_log_entries(from:, to:)
       log = cp.log_get(workload: runner_workload, from: from, to: to, replica: replica)
 

data/lib/command/update_github_actions.rb

@@ -10,12 +10,13 @@ module Command
 
     NAME = "update-github-actions"
     OPTIONS = [staging_branch_option].freeze
-    DESCRIPTION = "Regenerates generated GitHub Actions wrappers for the installed cpflow version"
+    DESCRIPTION = "Regenerates GitHub Actions files for the installed cpflow version"
     LONG_DESCRIPTION = <<~DESC.freeze
-      Regenerates the generated cpflow GitHub Actions wrappers and helper files
-      from the currently installed cpflow gem. Use this after updating the
-      cpflow gem so checked-in workflow wrappers move to the matching upstream
-      release tag, for example `v#{Cpflow::VERSION}`.
+      Regenerates the cpflow workflow wrappers, local composite actions, and
+      helper files from the currently installed cpflow gem. Use this after
+      updating the cpflow gem so checked-in workflow wrappers move to the
+      matching upstream release tag, for example `v#{Cpflow::VERSION}`, and the
+      generated action implementations move with them.
 
       If the existing generated staging workflow uses a custom single staging
       branch, the command preserves it. Pass `--staging-branch BRANCH` to set or
@@ -23,7 +24,7 @@ module Command
     DESC
     EXAMPLES = <<~EX
       ```sh
-      # After updating the cpflow gem, refresh generated GitHub Actions wrappers
+      # After updating the cpflow gem, refresh every generated GitHub Actions file
       cpflow update-github-actions
 
       # When running cpflow through Bundler