cpflow 5.0.4 → 5.1.0

data/lib/github_flow_templates/.github/workflows/cpflow-promote-staging-to-production.yml

@@ -9,22 +9,590 @@ on:
         type: string
 
 permissions:
-  # The upstream reusable workflow's create-github-release job needs
-  # contents: write, and callers must grant the union of callee permissions.
-  contents: write
+  contents: read
+
+env:
+  # Override these by editing this file or by setting the matching repository variable.
+  # Worst-case wall time per attempt is HEALTH_CHECK_INTERVAL plus the curl --max-time below
+  # (10s), so the defaults give a ~10 minute window (24 × (15 + 10) = 600s) — enough for
+  # most Rails cold boots (asset precompile + db:migrate + workload readiness).
+  HEALTH_CHECK_RETRIES: ${{ vars.HEALTH_CHECK_RETRIES || '24' }}
+  HEALTH_CHECK_INTERVAL: ${{ vars.HEALTH_CHECK_INTERVAL || '15' }}
+  # Space-separated list of HTTP statuses considered healthy. The default accepts 301/302
+  # because `curl` is invoked without `-L`, so a root `/` that redirects to a login page
+  # (common for Rails apps that auth-gate `/`) would otherwise be reported as unhealthy
+  # despite the workload itself being up.
+  #
+  # Strongly recommended: expose a dedicated `/health` endpoint that returns `200` and set
+  # HEALTH_CHECK_ACCEPTED_STATUSES to `"200"` in repository variables. The 301/302 default
+  # trades correctness for ergonomics — a maintenance-mode redirect or an auth-gate redirect
+  # to a login page can pass this check even when the underlying app is broken. Override
+  # via the HEALTH_CHECK_ACCEPTED_STATUSES repo variable to tighten this for apps that
+  # expose a dedicated health endpoint (e.g. "200" for a plain /health, or "200 401 403"
+  # for apps that auth-gate / without redirecting).
+  HEALTH_CHECK_ACCEPTED_STATUSES: ${{ vars.HEALTH_CHECK_ACCEPTED_STATUSES || '200 301 302' }}
+  COPY_IMAGE_RETRIES: ${{ vars.COPY_IMAGE_RETRIES || '3' }}
+  COPY_IMAGE_RETRY_INTERVAL: ${{ vars.COPY_IMAGE_RETRY_INTERVAL || '20' }}
+  ROLLBACK_READINESS_RETRIES: ${{ vars.ROLLBACK_READINESS_RETRIES || '24' }}
+  ROLLBACK_READINESS_INTERVAL: ${{ vars.ROLLBACK_READINESS_INTERVAL || '15' }}
+
+concurrency:
+  # Single global group: only one production promotion may run at a time across the
+  # whole repo. Independent of staging deploys and review-app workflows (different
+  # GVCs / different concurrency keys), so those can still run in parallel.
+  group: cpflow-promote-staging-to-production
+  # Don't cancel an in-flight promotion: a half-finished `cpflow deploy-image` plus a
+  # rollback can leave production in a worse state than letting the first run finish.
+  cancel-in-progress: false
 
 jobs:
   promote-to-production:
     if: github.event.inputs.confirm_promotion == 'promote'
-    uses: shakacode/control-plane-flow/.github/workflows/cpflow-promote-staging-to-production.yml@__CPFLOW_GITHUB_ACTIONS_REF__
-    with:
-      # Keep CPLN_TOKEN_PRODUCTION as a secret on this protected GitHub
-      # Environment. The caller passes the environment name, the upstream
-      # reusable workflow runs its production job in that environment, and
-      # GitHub exposes environment secrets only after required reviewers approve.
-      production_environment: production
-    # Only pass the staging token explicitly. CPLN_TOKEN_PRODUCTION must live on
-    # the protected production Environment, where GitHub exposes it only after
-    # the required reviewers approve this job.
-    secrets:
-      CPLN_TOKEN_STAGING: ${{ secrets.CPLN_TOKEN_STAGING }}
+    runs-on: ubuntu-latest
+    # This normal caller-repo job declares the protected production Environment
+    # directly, so GitHub exposes environment secrets in this job after the
+    # environment gate. Do not move production promotion back behind a
+    # cross-repo reusable workflow; environment secrets are not available there.
+    environment: production
+    timeout-minutes: 45
+    outputs:
+      staging_app_name: ${{ steps.release-context.outputs.staging_app_name }}
+      production_app_name: ${{ steps.release-context.outputs.production_app_name }}
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+        with:
+          persist-credentials: false
+
+      - name: Checkout control-plane-flow actions
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+        with:
+          repository: shakacode/control-plane-flow
+          ref: __CPFLOW_GITHUB_ACTIONS_REF__
+          path: .cpflow
+          persist-credentials: false
+
+      - name: Validate production token
+        shell: bash
+        env:
+          # GitHub does not expose which secret scope supplied this value.
+          # Keep CPLN_TOKEN_PRODUCTION absent from repo/org secrets so the
+          # protected production Environment is the only configured source.
+          CPLN_TOKEN_PRODUCTION: ${{ secrets.CPLN_TOKEN_PRODUCTION }}
+          PRODUCTION_ENVIRONMENT: production
+        run: |
+          set -euo pipefail
+
+          if [[ -z "${CPLN_TOKEN_PRODUCTION}" ]]; then
+            echo "::error::CPLN_TOKEN_PRODUCTION is not set. Add it as a secret on the '${PRODUCTION_ENVIRONMENT}' GitHub Environment."
+            exit 1
+          fi
+
+      - name: Validate required secrets and variables
+        uses: ./.cpflow/.github/actions/cpflow-validate-config
+        # Pass secrets via env so the composite action checks indirect shell
+        # variables instead of interpolating secret values into a run script.
+        env:
+          CPLN_TOKEN_STAGING: ${{ secrets.CPLN_TOKEN_STAGING }}
+          CPLN_TOKEN_PRODUCTION: ${{ secrets.CPLN_TOKEN_PRODUCTION }}
+          CPLN_ORG_STAGING: ${{ vars.CPLN_ORG_STAGING }}
+          CPLN_ORG_PRODUCTION: ${{ vars.CPLN_ORG_PRODUCTION }}
+          STAGING_APP_NAME: ${{ vars.STAGING_APP_NAME }}
+          PRODUCTION_APP_NAME: ${{ vars.PRODUCTION_APP_NAME }}
+        with:
+          required: |
+            secret:CPLN_TOKEN_STAGING
+            secret:CPLN_TOKEN_PRODUCTION
+            variable:CPLN_ORG_STAGING
+            variable:CPLN_ORG_PRODUCTION
+            variable:STAGING_APP_NAME
+            variable:PRODUCTION_APP_NAME
+
+      - name: Capture release context
+        id: release-context
+        env:
+          STAGING_APP_NAME: ${{ vars.STAGING_APP_NAME }}
+          PRODUCTION_APP_NAME: ${{ vars.PRODUCTION_APP_NAME }}
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          {
+            echo "staging_app_name=${STAGING_APP_NAME}"
+            echo "production_app_name=${PRODUCTION_APP_NAME}"
+          } >> "$GITHUB_OUTPUT"
+
+      - name: Setup production environment
+        uses: ./.cpflow/.github/actions/cpflow-setup-environment
+        with:
+          token: ${{ secrets.CPLN_TOKEN_PRODUCTION }}
+          org: ${{ vars.CPLN_ORG_PRODUCTION }}
+          working_directory: .cpflow
+          cpln_cli_version: ${{ vars.CPLN_CLI_VERSION }}
+          cpflow_version: ${{ vars.CPFLOW_VERSION }}
+          # The setup action validates CPFLOW_VERSION against this full workflow ref.
+          control_plane_flow_ref: shakacode/control-plane-flow/.github/workflows/cpflow-promote-staging-to-production.yml@__CPFLOW_GITHUB_ACTIONS_REF__
+
+      # Runs after Setup production environment so the pinned Ruby (>= 3.1) is on PATH.
+      # YAML.load_file(..., aliases: true) is not supported on Ruby 3.0 (system Ruby on ubuntu-22.04).
+      - name: Resolve production app workloads
+        id: workloads
+        env:
+          PRODUCTION_APP_NAME: ${{ vars.PRODUCTION_APP_NAME }}
+          PRIMARY_WORKLOAD: ${{ vars.PRIMARY_WORKLOAD }}
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          ruby - "${PRODUCTION_APP_NAME}" "${PRIMARY_WORKLOAD}" "${GITHUB_OUTPUT}" <<'RUBY'
+          require "yaml"
+
+          app = ARGV.fetch(0)
+          requested_primary = ARGV.fetch(1, "").to_s.strip
+          output_path = ARGV.fetch(2)
+          data = YAML.safe_load(File.read(".controlplane/controlplane.yml"), aliases: true)
+          apps = data["apps"] || {}
+          app_config = apps[app]
+
+          unless app_config
+            warn "Error: app '#{app}' is not defined under `apps:` in `.controlplane/controlplane.yml`."
+            warn "       Fix the PRODUCTION_APP_NAME repository variable or add the app to controlplane.yml."
+            exit 1
+          end
+
+          workloads = Array(app_config["app_workloads"]).map(&:to_s).reject(&:empty?)
+          workloads = ["rails"] if workloads.empty?
+
+          primary =
+            if requested_primary.empty?
+              if workloads.length == 1
+                workloads.first
+              elsif workloads.include?("rails")
+                "rails"
+              else
+                puts "::error::PRIMARY_WORKLOAD is not configured and app '#{app}' has multiple workloads: #{workloads.join(', ')}."
+                warn "       Set the PRIMARY_WORKLOAD repository variable to one of these workloads."
+                exit 1
+              end
+            elsif workloads.include?(requested_primary)
+              requested_primary
+            else
+              puts "::error::PRIMARY_WORKLOAD '#{requested_primary}' is not one of: #{workloads.join(', ')}."
+              exit 1
+            end
+
+          File.open(output_path, "a") do |output|
+            output.puts "names=#{workloads.join(',')}"
+            output.puts "primary=#{primary}"
+          end
+          RUBY
+
+      - name: Detect release phase support
+        id: release-phase
+        uses: ./.cpflow/.github/actions/cpflow-detect-release-phase
+        with:
+          app_name: ${{ vars.PRODUCTION_APP_NAME }}
+
+      - name: Verify production environment variables
+        env:
+          CPLN_TOKEN_STAGING: ${{ secrets.CPLN_TOKEN_STAGING }}
+          CPLN_TOKEN_PRODUCTION: ${{ secrets.CPLN_TOKEN_PRODUCTION }}
+          STAGING_APP_NAME: ${{ vars.STAGING_APP_NAME }}
+          PRODUCTION_APP_NAME: ${{ vars.PRODUCTION_APP_NAME }}
+          CPLN_ORG_STAGING: ${{ vars.CPLN_ORG_STAGING }}
+          CPLN_ORG_PRODUCTION: ${{ vars.CPLN_ORG_PRODUCTION }}
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          staging_vars="$(CPLN_TOKEN="${CPLN_TOKEN_STAGING}" cpln gvc get "${STAGING_APP_NAME}" --org "${CPLN_ORG_STAGING}" -o json | jq -r '.spec.env // [] | .[].name' | sort)"
+          production_vars="$(CPLN_TOKEN="${CPLN_TOKEN_PRODUCTION}" cpln gvc get "${PRODUCTION_APP_NAME}" --org "${CPLN_ORG_PRODUCTION}" -o json | jq -r '.spec.env // [] | .[].name' | sort)"
+
+          if [[ -z "${staging_vars}" ]]; then
+            echo "Staging GVC exposes no environment variables; skipping parity check."
+            exit 0
+          fi
+
+          # Treat staging as the promotion source of truth: fail when a variable
+          # present in staging is missing in production. Production-only variables
+          # are allowed, but surface them so teams can spot drift.
+          missing_vars="$(comm -23 <(printf '%s\n' "${staging_vars}") <(printf '%s\n' "${production_vars}"))"
+          production_only_vars="$(comm -13 <(printf '%s\n' "${staging_vars}") <(printf '%s\n' "${production_vars}"))"
+
+          if [[ -n "${production_only_vars}" ]]; then
+            echo "::warning::Production has environment variables that are not present in staging:"
+            echo "${production_only_vars}"
+          fi
+
+          if [[ -n "${missing_vars}" ]]; then
+            echo "::error::Production is missing environment variables that exist in staging"
+            echo "${missing_vars}"
+            exit 1
+          fi
+
+      - name: Capture current production image
+        id: capture-current
+        env:
+          PRODUCTION_APP_NAME: ${{ vars.PRODUCTION_APP_NAME }}
+          CPLN_ORG_PRODUCTION: ${{ vars.CPLN_ORG_PRODUCTION }}
+          WORKLOAD_NAMES: ${{ steps.workloads.outputs.names }}
+          PRIMARY_WORKLOAD: ${{ steps.workloads.outputs.primary }}
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          selected_workload="${PRIMARY_WORKLOAD}"
+          selected_image=""
+          selected_version=""
+          rollback_state='{}'
+
+          # Validate all workloads have images, then promote the primary workload's
+          # image as the canonical image for this GVC.
+          while IFS= read -r workload_name; do
+            [[ -n "${workload_name}" ]] || continue
+
+            workload_json="$(cpln workload get "${workload_name}" --gvc "${PRODUCTION_APP_NAME}" --org "${CPLN_ORG_PRODUCTION}" -o json)"
+            workload_image="$(echo "${workload_json}" | jq -r '.spec.containers[0].image // empty')"
+            workload_containers="$(echo "${workload_json}" | jq -c '.spec.containers | map({name, image})')"
+            workload_version="$(echo "${workload_json}" | jq -r '.version')"
+
+            if [[ "${workload_name}" == "${selected_workload}" ]]; then
+              selected_image="${workload_image}"
+              selected_version="${workload_version}"
+            fi
+
+            rollback_state="$(
+              jq -c \
+                --arg workload "${workload_name}" \
+                --arg image "${workload_image}" \
+                --arg version "${workload_version}" \
+                --argjson containers "${workload_containers}" \
+                '. + {($workload): {image: $image, version: $version, containers: $containers}}' \
+                <<< "${rollback_state}"
+            )"
+          done < <(tr ',' '\n' <<< "${WORKLOAD_NAMES}")
+
+          if [[ -z "${selected_image}" || -z "${selected_version}" ]]; then
+            echo "::error::Could not capture current image/version for primary workload '${selected_workload}'." >&2
+            exit 1
+          fi
+
+          echo "current_image=${selected_image}" >> "$GITHUB_OUTPUT"
+          echo "current_version=${selected_version}" >> "$GITHUB_OUTPUT"
+          # Randomize the heredoc delimiter so a stray "EOF" line inside rollback_state can't terminate it early.
+          delim="EOF_$(openssl rand -hex 8)"
+          {
+            echo "rollback_state<<${delim}"
+            echo "${rollback_state}"
+            echo "${delim}"
+          } >> "$GITHUB_OUTPUT"
+
+      - name: Capture deployed staging image
+        id: staging-image
+        env:
+          CPLN_TOKEN_STAGING: ${{ secrets.CPLN_TOKEN_STAGING }}
+          STAGING_APP_NAME: ${{ vars.STAGING_APP_NAME }}
+          CPLN_ORG_STAGING: ${{ vars.CPLN_ORG_STAGING }}
+          WORKLOAD_NAMES: ${{ steps.workloads.outputs.names }}
+          PRIMARY_WORKLOAD: ${{ steps.workloads.outputs.primary }}
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          selected_workload="${PRIMARY_WORKLOAD}"
+          selected_image=""
+
+          while IFS= read -r workload_name; do
+            [[ -n "${workload_name}" ]] || continue
+
+            workload_json="$(CPLN_TOKEN="${CPLN_TOKEN_STAGING}" cpln workload get "${workload_name}" --gvc "${STAGING_APP_NAME}" --org "${CPLN_ORG_STAGING}" -o json)"
+            workload_image="$(echo "${workload_json}" | jq -r '.spec.containers[0].image // empty')"
+
+            if [[ -z "${workload_image}" ]]; then
+              echo "::error::Could not find an image on staging workload '${workload_name}'." >&2
+              exit 1
+            fi
+
+            if [[ "${workload_name}" == "${selected_workload}" ]]; then
+              selected_image="${workload_image}"
+            fi
+          done < <(tr ',' '\n' <<< "${WORKLOAD_NAMES}")
+
+          staging_image_ref="${selected_image}"
+          if [[ -z "${staging_image_ref}" ]]; then
+            echo "::error::Could not determine the deployed staging image for primary workload '${selected_workload}'." >&2
+            exit 1
+          fi
+
+          if [[ "${staging_image_ref}" == /org/*/image/* ]]; then
+            staging_image="${staging_image_ref##*/image/}"
+          elif [[ "${staging_image_ref}" == *.registry.cpln.io/* ]]; then
+            staging_image="${staging_image_ref#*.registry.cpln.io/}"
+          else
+            staging_image="${staging_image_ref}"
+          fi
+
+          echo "image=${staging_image}" >> "$GITHUB_OUTPUT"
+
+      - name: Copy image from staging
+        env:
+          # Pass the upstream token via env rather than `-t` so it doesn't appear in /proc/<pid>/cmdline.
+          CPLN_TOKEN_STAGING: ${{ secrets.CPLN_TOKEN_STAGING }}
+          CPLN_UPSTREAM_TOKEN: ${{ secrets.CPLN_TOKEN_STAGING }}
+          PRODUCTION_APP_NAME: ${{ vars.PRODUCTION_APP_NAME }}
+          CPLN_ORG_STAGING: ${{ vars.CPLN_ORG_STAGING }}
+          CPLN_ORG_PRODUCTION: ${{ vars.CPLN_ORG_PRODUCTION }}
+          STAGING_IMAGE: ${{ steps.staging-image.outputs.image }}
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          if ! [[ "${COPY_IMAGE_RETRIES}" =~ ^[0-9]+$ ]]; then
+            echo "::error::COPY_IMAGE_RETRIES must be a non-negative integer."
+            exit 1
+          fi
+
+          if ! [[ "${COPY_IMAGE_RETRY_INTERVAL}" =~ ^[0-9]+$ ]]; then
+            echo "::error::COPY_IMAGE_RETRY_INTERVAL must be a non-negative integer."
+            exit 1
+          fi
+
+          copy_image_retries=$((10#${COPY_IMAGE_RETRIES}))
+          copy_image_attempts=$((copy_image_retries + 1))
+          copy_image_retry_interval=$((10#${COPY_IMAGE_RETRY_INTERVAL}))
+
+          if ! CPLN_TOKEN="${CPLN_TOKEN_STAGING}" cpln image get "${STAGING_IMAGE}" --org "${CPLN_ORG_STAGING}" -o json >/dev/null; then
+            echo "::error::Staging image '${STAGING_IMAGE}' was not found in org '${CPLN_ORG_STAGING}'; aborting promotion."
+            exit 1
+          fi
+
+          copy_status=1
+          for attempt in $(seq 1 "${copy_image_attempts}"); do
+            if cpflow copy-image-from-upstream -a "${PRODUCTION_APP_NAME}" --org "${CPLN_ORG_PRODUCTION}" --image "${STAGING_IMAGE}"; then
+              copy_status=0
+              break
+            else
+              copy_status=$?
+            fi
+
+            if [[ "${attempt}" -lt "${copy_image_attempts}" ]]; then
+              echo "::warning::Image copy attempt ${attempt}/${copy_image_attempts} failed with exit ${copy_status}; retrying in ${copy_image_retry_interval}s."
+              sleep "${copy_image_retry_interval}"
+            else
+              echo "::warning::Image copy attempt ${attempt}/${copy_image_attempts} failed with exit ${copy_status}; no attempts remain."
+            fi
+          done
+
+          if [[ "${copy_status}" -ne 0 ]]; then
+            echo "::error::Could not copy staging image '${STAGING_IMAGE}' from '${CPLN_ORG_STAGING}' to '${CPLN_ORG_PRODUCTION}' after ${copy_image_attempts} attempt(s)."
+            exit "${copy_status}"
+          fi
+
+      - name: Deploy image to production
+        env:
+          PRODUCTION_APP_NAME: ${{ vars.PRODUCTION_APP_NAME }}
+          CPLN_ORG_PRODUCTION: ${{ vars.CPLN_ORG_PRODUCTION }}
+          RELEASE_PHASE_FLAG: ${{ steps.release-phase.outputs.flag }}
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          deploy_args=(-a "${PRODUCTION_APP_NAME}")
+          if [[ -n "${RELEASE_PHASE_FLAG}" ]]; then
+            deploy_args+=("${RELEASE_PHASE_FLAG}")
+          fi
+          deploy_args+=(--org "${CPLN_ORG_PRODUCTION}" --verbose)
+
+          cpflow deploy-image "${deploy_args[@]}"
+
+      - name: Wait for deployment health
+        id: health-check
+        uses: ./.cpflow/.github/actions/cpflow-wait-for-health
+        with:
+          workload_name: ${{ steps.workloads.outputs.primary }}
+          app_name: ${{ vars.PRODUCTION_APP_NAME }}
+          org: ${{ vars.CPLN_ORG_PRODUCTION }}
+          max_retries: ${{ env.HEALTH_CHECK_RETRIES }}
+          interval_seconds: ${{ env.HEALTH_CHECK_INTERVAL }}
+          accepted_statuses: ${{ env.HEALTH_CHECK_ACCEPTED_STATUSES }}
+
+      - name: Roll back on failure
+        if: failure() && steps.capture-current.outcome == 'success'
+        env:
+          ROLLBACK_STATE: ${{ steps.capture-current.outputs.rollback_state }}
+          PRODUCTION_APP_NAME: ${{ vars.PRODUCTION_APP_NAME }}
+          CPLN_ORG_PRODUCTION: ${{ vars.CPLN_ORG_PRODUCTION }}
+        shell: bash
+        run: |
+          # Best-effort rollback: try every workload, aggregate failures, exit non-zero at the end
+          # if any failed. A single cpln hiccup shouldn't leave other workloads mid-promotion.
+          # Keep -e disabled here so rollback can aggregate failures across workloads.
+          set -uo pipefail
+
+          rollback_failures=0
+          if ! rollback_entries="$(echo "${ROLLBACK_STATE}" | jq -r 'to_entries[] | "\(.key)\t\(.value.containers | @json)"')"; then
+            echo "::error::Could not parse rollback state; manual recovery may be required." >&2
+            exit 1
+          fi
+
+          while IFS=$'\t' read -r workload_name previous_containers; do
+            rollback_args=()
+            if ! current_names="$(cpln workload get "${workload_name}" --gvc "${PRODUCTION_APP_NAME}" --org "${CPLN_ORG_PRODUCTION}" -o json | jq -c '.spec.containers | map(.name)')"; then
+              echo "::warning::Could not retrieve current containers for workload '${workload_name}'; skipping rollback for this workload." >&2
+              rollback_failures=$((rollback_failures + 1))
+              continue
+            fi
+            if ! previous_names="$(echo "${previous_containers}" | jq -c 'map(.name)')"; then
+              echo "::warning::Could not parse captured containers for workload '${workload_name}'; skipping rollback for this workload." >&2
+              rollback_failures=$((rollback_failures + 1))
+              continue
+            fi
+
+            if [[ "$(echo "${current_names}" | jq -c 'sort')" != "$(echo "${previous_names}" | jq -c 'sort')" ]]; then
+              echo "::error::Container set changed for workload '${workload_name}'; refusing rollback." >&2
+              rollback_failures=$((rollback_failures + 1))
+              continue
+            fi
+
+            if ! rollback_container_entries="$(jq -r '.[] | "\(.name)\t\(.image)"' <<< "${previous_containers}")"; then
+              echo "::warning::Could not build rollback image list for workload '${workload_name}'; skipping rollback for this workload." >&2
+              rollback_failures=$((rollback_failures + 1))
+              continue
+            fi
+
+            while IFS=$'\t' read -r container_name image; do
+              rollback_args+=(--set "spec.containers.${container_name}.image=${image}")
+            done <<< "${rollback_container_entries}"
+
+            if ! cpln workload update "${workload_name}" \
+              --gvc "${PRODUCTION_APP_NAME}" \
+              --org "${CPLN_ORG_PRODUCTION}" \
+              "${rollback_args[@]}"; then
+              echo "::warning::Rollback failed for workload '${workload_name}'; continuing with remaining workloads." >&2
+              rollback_failures=$((rollback_failures + 1))
+            fi
+          done <<< "${rollback_entries}"
+
+          if [[ "${rollback_failures}" -gt 0 ]]; then
+            echo "::error::${rollback_failures} workload(s) failed to roll back; inspect the logs above." >&2
+            exit 1
+          fi
+
+      - name: Wait for rollback readiness
+        if: failure() && steps.capture-current.outcome == 'success'
+        env:
+          ROLLBACK_STATE: ${{ steps.capture-current.outputs.rollback_state }}
+          PRODUCTION_APP_NAME: ${{ vars.PRODUCTION_APP_NAME }}
+          CPLN_ORG_PRODUCTION: ${{ vars.CPLN_ORG_PRODUCTION }}
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          mapfile -t workloads < <(echo "${ROLLBACK_STATE}" | jq -r 'keys[]')
+
+          # Poll workloads in parallel so the worst-case wall time during a
+          # production incident is `retries × interval` rather than scaling
+          # linearly with the number of workloads. Each per-workload retry
+          # loop runs in a backgrounded subshell that writes its final state
+          # to a status file; the parent waits for all of them before
+          # aggregating warnings, keeping output ordered and deterministic.
+          status_dir="$(mktemp -d)"
+          trap 'rm -rf "${status_dir}"' EXIT
+
+          pids=()
+          for workload_name in "${workloads[@]}"; do
+            [[ -n "${workload_name}" ]] || continue
+            status_name="${workload_name//\//_}"
+
+            echo "Polling rollback readiness for workload '${workload_name}'..."
+            (
+              set -euo pipefail
+              ready=false
+              for attempt in $(seq 1 "${ROLLBACK_READINESS_RETRIES}"); do
+                deployment_ready="$(cpln workload get "${workload_name}" --gvc "${PRODUCTION_APP_NAME}" --org "${CPLN_ORG_PRODUCTION}" -o json | jq -r '.status.ready // false')"
+                if [[ "${deployment_ready}" == "true" ]]; then
+                  ready=true
+                  break
+                fi
+
+                if [[ "${attempt}" -lt "${ROLLBACK_READINESS_RETRIES}" ]]; then
+                  sleep "${ROLLBACK_READINESS_INTERVAL}"
+                fi
+              done
+
+              if [[ "${ready}" == "true" ]]; then
+                printf 'ready\n' > "${status_dir}/${status_name}"
+              else
+                printf 'not_ready\n' > "${status_dir}/${status_name}"
+              fi
+            ) &
+            pids+=("$!")
+          done
+
+          # `|| true` so a single workload that fails to poll (e.g. transient
+          # cpln API error) doesn't abort the parent before the others finish.
+          # Missing or non-`ready` status files are surfaced in the aggregation
+          # loop below, so the failure is still visible to operators.
+          for pid in "${pids[@]}"; do
+            wait "${pid}" || true
+          done
+
+          for workload_name in "${workloads[@]}"; do
+            [[ -n "${workload_name}" ]] || continue
+            status_name="${workload_name//\//_}"
+            status_file="${status_dir}/${status_name}"
+            if [[ ! -f "${status_file}" ]] || [[ "$(<"${status_file}")" != "ready" ]]; then
+              echo "::warning::Workload '${workload_name}' did not report ready after rollback."
+            fi
+          done
+
+      - name: Promotion summary
+        if: always()
+        env:
+          HEALTHY: ${{ steps.health-check.outputs.healthy }}
+          PREVIOUS_IMAGE: ${{ steps.capture-current.outputs.current_image }}
+          PREVIOUS_VERSION: ${{ steps.capture-current.outputs.current_version }}
+          DEPLOYED_IMAGE: ${{ steps.staging-image.outputs.image }}
+        shell: bash
+        run: |
+          {
+            echo "## Promotion Summary"
+            echo
+            if [[ "${HEALTHY}" == "true" ]]; then
+              echo "✅ Status: deployment successful"
+            else
+              echo "❌ Status: deployment failed"
+            fi
+            echo
+            echo "Previous image: \`${PREVIOUS_IMAGE}\`"
+            echo "Previous version: ${PREVIOUS_VERSION}"
+            echo "Deployed image: \`${DEPLOYED_IMAGE}\`"
+          } >> "$GITHUB_STEP_SUMMARY"
+
+  create-github-release:
+    needs: promote-to-production
+    if: needs.promote-to-production.result == 'success'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+
+    steps:
+      - name: Create GitHub release
+        env:
+          GH_REPO: ${{ github.repository }}
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_RUN_ID: ${{ github.run_id }}
+          STAGING_APP_NAME: ${{ needs.promote-to-production.outputs.staging_app_name }}
+          PRODUCTION_APP_NAME: ${{ needs.promote-to-production.outputs.production_app_name }}
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          release_date="$(date '+%Y-%m-%d')"
+          timestamp="$(date '+%H%M%S')"
+          release_tag="production-${release_date}-${timestamp}-${GITHUB_RUN_ID}"
+
+          gh release create "${release_tag}" \
+            --title "Production Release ${release_date} ${timestamp}" \
+            --notes "Promoted ${STAGING_APP_NAME} to ${PRODUCTION_APP_NAME} on ${release_date} at ${timestamp}."

data/lib/github_flow_templates/bin/pin-cpflow-github-ref

@@ -8,15 +8,25 @@ USAGE = <<~USAGE
 
   Use a release tag for normal operation, e.g. v5.0.0.
   Use a full 40-character commit SHA for temporary unreleased upstream testing.
-  This only updates generated reusable-workflow `uses:` refs. The called
-  workflows load their own matching shared actions from that same workflow
-  commit automatically. Regenerate from the cpflow gem when templates changed.
+  This updates generated reusable-workflow `uses:` refs plus the production
+  workflow's pinned control-plane-flow checkout and setup validation ref.
+  Regenerate from the cpflow gem when templates changed.
   Use --allow-moving-ref only for short-lived local branch/ref experiments.
 USAGE
 
 ALLOWED_OPTIONS = ["--allow-moving-ref"].freeze
 FULL_COMMIT_SHA = /\A[0-9a-f]{40}\z/i
 RELEASE_TAG = /\Av\d+\.\d+\.\d+(?:[-.][0-9A-Za-z][0-9A-Za-z.-]*)?\z/
+PRODUCTION_WORKFLOW_REF = "shakacode/control-plane-flow/.github/workflows/" \
+                          "cpflow-promote-staging-to-production.yml"
+CPFLOW_CHECKOUT_REF_PATTERN = %r{
+  (^\s*-\s+name:\s+Checkout\ control-plane-flow\ actions\s*\n
+    (?:(?!^\s*-\s+name:).)*?
+    ^\s+repository:\s+shakacode/control-plane-flow\s*\n
+    (?:(?!^\s*-\s+name:).)*?
+    ^\s+ref:\s+)
+  [^\s]+
+}mx
 
 options, positional = ARGV.partition { |arg| arg.start_with?("--") }
 unknown_options = options - ALLOWED_OPTIONS
@@ -57,8 +67,12 @@ end
 changed = []
 workflow_paths.each do |path|
   text = File.read(path)
+  production_setup_ref_pattern =
+    /(control_plane_flow_ref:\s+#{Regexp.escape(PRODUCTION_WORKFLOW_REF)}@)[^\s]+/
   updated = text
             .gsub(%r{(uses:\s+shakacode/control-plane-flow/\.github/workflows/[^@\s]+@)[^\s]+}, "\\1#{ref}")
+            .gsub(production_setup_ref_pattern, "\\1#{ref}")
+            .gsub(CPFLOW_CHECKOUT_REF_PATTERN, "\\1#{ref}")
 
   next if updated == text