amsd-pipeline-shared 0.2.0 → 0.4.0

package/orchestrations/scripts/preflight-check.sh

@@ -0,0 +1,696 @@
+#!/usr/bin/env bash
+# Service endpoints come from orchestrations/config/services.json — one place to change
+# a port, instead of the 20+ copies these URLs used to have across the pipeline.
+. "$(dirname "${BASH_SOURCE[0]}")/lib/service-urls.sh"
+_DASH="$(service_url dashboard)"
+
+# preflight-check.sh — validates everything before a paid pipeline run.
+# Catches missing exports, bad PRD config, missing API keys, and wrong paths.
+# Called automatically by pre-run-reset.sh, or run standalone.
+#
+# Usage:
+#   bash orchestrations/scripts/preflight-check.sh --runner tier3-travel-app-run.sh \
+#        --prd orchestrations/travel-app-prd.json
+#
+# Exit 0 = all checks pass. Exit 1 = one or more checks failed (do NOT run).
+
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=lib/resume-semantics.sh
+. "$SCRIPT_DIR/lib/resume-semantics.sh"
+REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+# Config files are DATA: load them without executing them. See lib/env-file.sh.
+. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/lib/env-file.sh"
+
+# ── Args ─────────────────────────────────────────────────────────────────────
+# Inherit from the launching environment when present: a launcher that already resolved
+# PRD_FILE should not have to pass it again, and blanking it here made the pre-flight
+# report "PRD_FILE is unset" about an environment where it was plainly set.
+RUNNER_SCRIPT="${RUNNER_SCRIPT:-}"
+PRD_FILE="${PRD_FILE:-}"
+PROJECT_CONFIG_DIR="${EPAM_PROJECT_CONFIG_DIR:-}"
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --runner) RUNNER_SCRIPT="$SCRIPT_DIR/$2"; shift 2 ;;
+    --prd)    PRD_FILE="$(cd "$(dirname "$2")" && pwd)/$(basename "$2")"; shift 2 ;;
+    --project-config) PROJECT_CONFIG_DIR="$2"; shift 2 ;;
+    *) echo "Unknown arg: $1" >&2; exit 1 ;;
+  esac
+done
+
+PASS=0; FAIL=0
+ok()   { echo "  ✓ $*"; PASS=$((PASS+1)); }
+fail() { echo "  ✗ $*" >&2; FAIL=$((FAIL+1)); }
+
+echo ""
+echo "━━━ Pre-flight checks ━━━"
+
+# ── 1. Runner script exists and is shellcheck-clean ──────────────────────────
+echo "[ Runner script ]"
+if [[ -z "$RUNNER_SCRIPT" ]]; then
+  fail "No --runner specified"
+elif [[ ! -f "$RUNNER_SCRIPT" ]]; then
+  fail "Runner not found: $RUNNER_SCRIPT"
+else
+  ok "Runner exists: $(basename "$RUNNER_SCRIPT")"
+  if command -v shellcheck &>/dev/null; then
+    # THE VERDICT IS A FUNCTION OF THE BYTES, SO IT IS KEPT PER DIGEST. shellcheck holds ~3.3 GB
+    # for the 11,000-line orchestrator and takes ten seconds; running it on every launch cost that
+    # every launch, and under a memory cap sized to the host it was the one step that could not
+    # fit (2026-09-13: the £0 greenfield rehearsal was killed here twice). The same file yields the
+    # same verdict, so a clean verdict is recorded beside the script under its sha256 and a launch
+    # of unchanged bytes reads the record. A failing verdict is never recorded: it is re-run and
+    # re-reported until the script changes.
+    _sc_cache_dir="${EPAM_PREFLIGHT_CACHE_DIR:-$SCRIPT_DIR/.preflight-cache}"
+    _sc_digest="$(sha256sum "$RUNNER_SCRIPT" 2>/dev/null | cut -d' ' -f1)"
+    _sc_mark="$_sc_cache_dir/shellcheck-${_sc_digest:-nodigest}.ok"
+    if [[ -n "$_sc_digest" && -f "$_sc_mark" ]]; then
+      ok "shellcheck clean (verdict recorded for these exact bytes, ${_sc_digest:0:12}; not re-run)"
+    elif shellcheck --severity=error "$RUNNER_SCRIPT" 2>/dev/null; then
+      ok "shellcheck clean"
+      if [[ -n "$_sc_digest" ]]; then
+        mkdir -p "$_sc_cache_dir" 2>/dev/null && : > "$_sc_mark" 2>/dev/null || true
+      fi
+    else
+      fail "shellcheck errors in $(basename "$RUNNER_SCRIPT")"
+    fi
+  else
+    ok "shellcheck not installed — skipping"
+  fi
+fi
+
+# ── 2. Critical variables RESOLVE in the launching environment ───────────────
+# This used to grep the launcher's source for "export VAR". A string in a file is not an
+# environment: it missed tier3-metrolinx-run.sh's shared `export A B C` line and demanded
+# OUTPUT_DIR/PROJECT_ROOT from a launcher that correctly sets them per lane downstream.
+# Which variables matter is declared in config, not here.
+echo "[ Critical variables ]"
+_req_cfg="$REPO_ROOT/orchestrations/config/preflight-required-env.json"
+if [ -f "$_req_cfg" ]; then
+  while IFS=$'\t' read -r _var _why; do
+    [ -n "$_var" ] || continue
+    if [ -n "${!_var:-}" ]; then
+      ok "$_var is resolved"
+    else
+      fail "$_var is unset — $_why"
+    fi
+  done < <(jq -r '.required | to_entries[] | "\(.key)\t\(.value)"' "$_req_cfg" 2>/dev/null)
+  while IFS=$'\t' read -r _var _why; do
+    [ -n "$_var" ] || continue
+    [ -n "${!_var:-}" ] && ok "$_var is resolved" || echo "  – $_var unset — $_why"
+  done < <(jq -r '.advisory | to_entries[] | "\(.key)\t\(.value)"' "$_req_cfg" 2>/dev/null)
+else
+  fail "preflight-required-env.json missing — cannot tell which variables this run needs"
+fi
+
+# Is PRD_FILE about to be OVERWRITTEN by this run's own Jira ingest, before anything
+# downstream ever reads today's content? ingest-jira-tickets.sh writes to
+# JIRA_SYNTH_PRD_PATH if set, else falls through to PRD_FILE (run-agent-orchestration.sh
+# `"${JIRA_SYNTH_PRD_PATH:-${PRD_FILE:-...}}"`). Whenever that target IS PRD_FILE, whatever
+# is on disk right now is LAST run's leftover, not this run's input — checking its content
+# is checking the wrong PRD.
+#
+# Caught live 2026-08-05: a metrolinx pre-flight failed on "pre-baked specification blocks"
+# on AMSD-2041 in orchestrations/projects/metrolinx/prd.json — real contamination, but from
+# the PREVIOUS run, about to be discarded by ingest before the writer ever runs. The earlier
+# version of this guard only recognised a genuinely EMPTY placeholder (mock1's shape) and
+# missed this — a non-empty stale file about to be overwritten is the same situation.
+# The project's declared codeline root, read the same way the launcher reads it. Its presence is
+# what tells this gate that scope — and therefore outputDir — is resolved during the run.
+# A KEY FROM THE ENVIRONMENT, ELSE FROM THE PROJECT'S OWN ENV FILES — read the way the launcher
+# reads it. One lookup for every key this gate needs from a project (JIRA_CODELINE_ROOT,
+# EPAM_ESTIMATION_MANIFEST): a second copy of the rule would be free to drift.
+_project_env_value() {
+  local _key="$1" _val="${!1:-}"
+  if [[ -z "$_val" ]] && [[ -n "${PROJECT_CONFIG_DIR:-}" ]]; then
+    # THE FILENAME IS NOT SPELLED HERE. The registry names a project's env files
+    # (config/provider-sets.json); a literal would be a second home for that name, free to drift
+    # with nothing failing. Both halves are scanned because the key may live in either, and the
+    # two are disjoint so at most one can answer.
+    local _pe_lib _pe_f
+    _pe_lib="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/lib/llm-settings-resolve.js"
+    if [[ -f "$_pe_lib" ]] && { [[ -n "${NODE_BIN:-}" ]] || command -v node >/dev/null 2>&1; }; then
+      while IFS= read -r _pe_f; do
+        [[ -n "$_pe_f" ]] && [[ -f "$_pe_f" ]] || continue
+        _val=$(sed -n "s/^[[:space:]]*${_key}=//p" "$_pe_f" \
+                 | tail -1 | tr -d '"'"'"'"' )
+        [[ -n "$_val" ]] && break
+      done < <("${NODE_BIN:-node}" -e '
+        const { projectEnvFiles } = require(process.argv[1]);
+        const f = projectEnvFiles(process.argv[2]);
+        if (f) process.stdout.write(f.base + "\n" + f.overlay + "\n");
+      ' "$_pe_lib" "$PROJECT_CONFIG_DIR" 2>/dev/null)
+    fi
+  fi
+  printf '%s' "$_val"
+}
+_codeline_root="$(_project_env_value JIRA_CODELINE_ROOT)"
+
+_prd_pending_ingest=0
+if [[ -n "${JIRA_URL:-}" ]] && [[ -n "$PRD_FILE" ]]; then
+  _synth_target="${JIRA_SYNTH_PRD_PATH:-$PRD_FILE}"
+  if [[ -n "$_synth_target" ]] && [[ "$(cd "$(dirname "$_synth_target")" 2>/dev/null && pwd)/$(basename "$_synth_target")" == "$PRD_FILE" ]]; then
+    _prd_pending_ingest=1
+  fi
+fi
+
+# ── 3. PRD integrity gate ────────────────────────────────────────────────────
+echo "[ PRD integrity ]"
+if [[ -z "$PRD_FILE" ]]; then
+  fail "No --prd specified (required for integrity gate)"
+elif [[ ! -f "$PRD_FILE" ]] && { [[ "$_prd_pending_ingest" == "1" ]] || [[ -n "$_codeline_root" ]]; }; then
+  # A PRD THAT DOES NOT EXIST YET IS NOT A FAILURE WHEN THE RUN CREATES IT.
+  #
+  # prd.json is REGENERATED every run (Jira ingest, or resolve-codeline-scope.sh for an authored
+  # PRD) and is no longer committed, so a fresh install legitimately has none at this point. This
+  # gate demanded the file anyway and refused every first launch on a clean install.
+  ok "no PRD on disk yet — this run creates it (ingest / scope resolution); integrity check deferred"
+elif [[ ! -f "$PRD_FILE" ]]; then
+  fail "PRD not found: $PRD_FILE"
+else
+  # Detect canonical (pre-spec-pass) PRD: no story has specification.createdFrom set.
+  # Strict phase/field checks only apply to elaborated PRDs — skip them for canonical.
+  _prd_is_canonical=$(python3 "$SCRIPT_DIR/lib/handlers/prd-is-canonical.py" "$PRD_FILE" 2>/dev/null || echo "false")
+
+  if [[ "$_prd_is_canonical" == "true" ]]; then
+    # Even on a canonical (pre-spec-pass) PRD, no base story should carry a
+    # pre-baked 'specification' block (status, coordinatorReview, etc.) from
+    # a prior run — the spec-mode coordinator reads this field to decide
+    # whether to (re)assign agents, and stale "completed" data silently makes
+    # it skip re-elaboration (and the split-mandate check inside that loop)
+    # for that story. This has happened once already (2026-07-06); the
+    # `has_splits` bypass above wouldn't catch it because these stories have
+    # no `createdFrom`, only stale status data. A story already completed=true
+    # in THIS run (e.g. SKY-001 after the scaffold phase) legitimately carries
+    # its own real specification data from this run's own spec pass — only
+    # PENDING stories carrying specification data indicate prior-run
+    # contamination baked into canonical.
+    _stale_spec=$(python3 "$SCRIPT_DIR/lib/handlers/prd-stale-specification-stories.py" "$PRD_FILE" 2>/dev/null || echo "")
+    # A RESUME LEGITIMATELY CARRIES ITS OWN SPEC PASS.
+    #
+    # This gate reads a pending story's specification block as a PRIOR run baked into the
+    # canonical, which is right for a fresh run and has fired for real (2026-07-06). A resume is
+    # the second case it cannot distinguish, alongside the pending-ingest one below: the blocks
+    # belong to the run being resumed, and that run IS this run. On 2026-08-18 it refused the
+    # resume of 20260818T101809Z — whose canonical was correctly lean while the runtime PRD held
+    # 6 and 3 verification criteria, 2 fix sites each and both roles — for carrying exactly the
+    # output the resume exists to reuse.
+    if [[ -n "$_stale_spec" ]] && resume_preserves spec-blocks; then
+      ok "PRD carries specification data for the run being resumed (EPAM_RESUME_RUN=${EPAM_RESUME_RUN:-}) — this is the resumed run's own output, not a prior run's"
+      PASS=$((PASS+1))
+    elif [[ -n "$_stale_spec" ]] && [[ "$_prd_pending_ingest" != "1" ]]; then
+      fail "Canonical PRD has pre-baked 'specification' blocks on base stories (must be lean/unelaborated): $_stale_spec"
+    elif [[ -n "$_stale_spec" ]]; then
+      ok "PRD carries stale specification data from a prior run, but Jira ingest overwrites this exact file before anything reads it — deferred"
+      PASS=$((PASS+1))
+    else
+      _base_count=$(python3 "$SCRIPT_DIR/lib/handlers/prd-story-count.py" "$PRD_FILE" 2>/dev/null || echo "?")
+      ok "PRD integrity OK — $_base_count base user stories (canonical/pre-spec-pass — strict phase checks deferred until after spec pass elaboration)"
+      PASS=$((PASS+1))
+    fi
+  else
+    integrity_out=$(bash "$SCRIPT_DIR/preflight-prd-integrity.sh" --prd "$PRD_FILE" 2>&1) || integrity_exit=$?
+    echo "$integrity_out"
+    if [[ "${integrity_exit:-0}" -ne 0 ]]; then
+      FAIL=$((FAIL+1))
+    else
+      PASS=$((PASS+1))
+    fi
+  fi
+fi
+
+# ── 4. PRD file valid JSON with required fields ───────────────────────────────
+echo "[ PRD file ]"
+if [[ -z "$PRD_FILE" ]]; then
+  fail "No --prd specified"
+elif [[ ! -f "$PRD_FILE" ]] && { [[ "$_prd_pending_ingest" == "1" ]] || [[ -n "$_codeline_root" ]]; }; then
+  # Same reason as the integrity gate above: the run writes this file, so its absence here is the
+  # expected state of a fresh install, not a fault to refuse the launch over.
+  ok "PRD is written by this run (ingest / scope resolution) — file checks deferred"
+elif [[ ! -f "$PRD_FILE" ]]; then
+  fail "PRD not found: $PRD_FILE"
+else
+  ok "PRD exists: $(basename "$PRD_FILE")"
+  if python3 -c "import json; json.load(open('$PRD_FILE'))" 2>/dev/null; then
+    ok "PRD valid JSON"
+  else
+    fail "PRD is not valid JSON"
+  fi
+
+  # outputDir must be set
+  #
+  # THE CONDITION IS TESTED BEFORE THE VALUE, and that ordering is the whole point.
+  #
+  # This read the field FIRST and reported whatever it found, so both deferral branches below were
+  # unreachable whenever the file on disk happened to carry a value. That is every fresh install:
+  # pre-flight runs BEFORE ingest, so prd.json at this moment is not this run's PRD at all — it is
+  # the file the install shipped, or the PREVIOUS run's output, and the run overwrites it minutes
+  # later (resolve-codeline-scope.sh -> apply-codeline-scope.js).
+  #
+  # Live 2026-09-04, pipeline-tests-10: a run correctly targeting the test codelines reported
+  # "✓ PRD project.outputDir = /home/…/metrolinx/next.gotransit.com" — a REAL CLIENT PATH, out of a
+  # dead file — and the comparison below it turns the same stale value into a HARD FAILURE
+  # ("does NOT match PRD outputDir") for a run whose real target is simply different.
+  #
+  # When the run resolves scope, the field is not read, not reported, and not compared.
+  OUTPUT_DIR_VAL=""
+  if [[ "$_prd_pending_ingest" == "1" ]]; then
+    ok "PRD content is pending Jira ingest for this run — outputDir check deferred"
+  elif [[ -n "$_codeline_root" ]]; then
+    # DEFERRED FOR THE SAME REASON, ARRIVED AT DIFFERENTLY.
+    #
+    # project.outputDir is WRITTEN BY THE RUN: resolve-codeline-scope.sh calls
+    # apply-codeline-scope.js, which sets outputDirs and outputDir from the codelines it
+    # discovers. So this gate demanded a field the pipeline had not created yet, and no
+    # PRD-authored project could ever pass it.
+    #
+    # The deferral above already exists for the Jira path — the same reasoning, keyed to
+    # ingest. That is the Step 0.3 shape again: a mechanism written only for the branch it was
+    # first needed on. A project that declares a codeline root resolves its scope the same way,
+    # whether its PRD arrived from Jira or was authored by hand.
+    ok "codeline scope is resolved during the run (root: $_codeline_root) — outputDir check deferred"
+  else
+    # NOTHING WILL RESOLVE THIS LATER, so the PRD's own declared value is the answer and IS checked.
+    # Read here rather than above, so a deferred run never touches the field at all.
+    OUTPUT_DIR_VAL=$(python3 -c "import json; d=json.load(open('$PRD_FILE')); print(d.get('project',{}).get('outputDir',''))" 2>/dev/null || true)
+    if [[ -n "$OUTPUT_DIR_VAL" ]]; then
+      ok "PRD project.outputDir = $OUTPUT_DIR_VAL"
+    else
+      fail "PRD project.outputDir is NOT set, and no codeline root is declared to resolve it from"
+    fi
+  fi
+
+  # The RESOLVED OUTPUT_DIR must agree with the PRD's. This used to scrape `OUTPUT_DIR=`
+  # out of the launcher's SOURCE with grep+sed, which is the same anti-pattern as the export
+  # check above and failed the same way: tier3-mock-run.sh takes its output directory from
+  # --project-root and never spells OUTPUT_DIR= at all, so the scrape produced '' and the
+  # pre-flight declared a mismatch against a perfectly correct launcher. A launcher that
+  # resolves the directory per lane has nothing to compare here, and saying so is honest.
+  if [[ -n "$OUTPUT_DIR_VAL" ]]; then
+    if [[ -z "${OUTPUT_DIR:-}" ]]; then
+      echo "  – OUTPUT_DIR not resolved in this environment (set per lane downstream) — nothing to compare"
+    elif [[ "${OUTPUT_DIR}" == "$OUTPUT_DIR_VAL" ]]; then
+      ok "OUTPUT_DIR agrees with PRD outputDir ($OUTPUT_DIR_VAL)"
+    else
+      fail "OUTPUT_DIR='${OUTPUT_DIR}' does NOT match PRD outputDir='$OUTPUT_DIR_VAL' — deliverables would be checked in the wrong place"
+    fi
+  fi
+
+  # Story field checks — a canonical PRD is judged on what it must NOT carry; an elaborated one on
+  # what it must.
+  #
+  # A RESUME IS JUDGED BY NEITHER. Its PRD is the run's own working copy, restored from the
+  # checkpoint, and carries the run's own assignments; a story that was never split carries no
+  # specification.createdFrom, so prd-is-canonical calls that copy canonical and the pin refusal
+  # aborted every resume of the brownfield rehearsal on the assigner's own agentRole, the ladder's
+  # own model and the set's own aiProvider (£0 brownfield harness run 5, 2026-09-14).
+  if resume_preserves prd-state; then
+    ok "resuming run ${EPAM_RESUME_RUN}: the PRD is the run's own working copy — the authored-PRD checks do not apply"
+  elif [[ "$_prd_is_canonical" == "true" ]]; then
+    # A CANONICAL PRD PINS NOTHING THE RUN DECIDES. A story's agent is the assigner's decision, its
+    # model the ladder's, its provider the set's. Authored with any of them, the PRD bypasses that
+    # stage and the mint refuses MID-RUN — "assigned python-engineer, which is not in the roster" —
+    # after the roster was minted, the prompts generated and the spend made (2026-09-13, a
+    # generated greenfield PRD). This used to say "deferred" and pass. Refused here by story and
+    # field, before any model is called, so the correction costs nothing.
+    _pinned=$(jq -r '
+      [ .stories[]? | select(.id) as $s
+        | ["agentRole","model","aiProvider"][] as $k
+        | select(($s[$k] // "") != "") | "\($s.id): \($k)=\($s[$k])" ] | .[]' "$PRD_FILE" 2>/dev/null || true)
+    if [[ -n "$_pinned" ]] && [[ "$_prd_pending_ingest" == "1" ]]; then
+      # THE FILE ON DISK IS THE PREVIOUS RUN'S, AND THIS RUN OVERWRITES IT. A Jira project's PRD is
+      # synthesised by ingest every run; what sits here now carries the previous run's own
+      # assignments, exactly as the stale-specification check above already understands. Refusing on
+      # it refused the second launch of every Jira project (£0 replay of a Sept 9 brownfield
+      # cassette, 2026-09-17). Reported, not refused — ingest replaces it before anything reads it.
+      ok "PRD on disk carries a previous run's assignments (agentRole/model/aiProvider), but this run's Jira ingest overwrites this exact file before anything reads it — deferred"
+    elif [[ -n "$_pinned" ]]; then
+      fail "the canonical PRD pins what the run decides — the assigner owns agentRole, the ladder owns model, the set owns aiProvider. Remove these from the authored PRD:"
+      while IFS= read -r _line; do [[ -n "$_line" ]] && echo "      $_line" >&2; done <<< "$_pinned"
+    else
+      ok "canonical PRD pins no agent, model or provider — the run will decide each"
+    fi
+  else
+    python3 "$SCRIPT_DIR/lib/handlers/prd-story-assignment-check.py" "$PRD_FILE" "$SCRIPT_DIR/../config/providers.json"
+    story_exit=$?
+    [[ $story_exit -eq 0 ]] && ((PASS++)) || ((FAIL++))
+  fi
+fi
+
+# ── 4. Required API keys ──────────────────────────────────────────────────────
+echo "[ API keys ]"
+# THE ACTIVE SET SAYS WHICH KEYS A RUN NEEDS; THE PROJECT SAYS WHICH IT NEEDS BESIDES.
+# This named OPENROUTER_API_KEY and OPENAI_API_KEY as required for every run — two vendors, in the
+# engine, whatever set was active — and RAPIDAPI_KEY, one project's contract-discovery key, as a
+# warning for all of them. On the claude, codemie and mockserver sets the first two are absent by
+# design (the mockserver set REMOVES vendor credentials so a rehearsal cannot spend), so pre-flight
+# refused every £0 rehearsal and every subscription-billed run over keys nothing would use.
+# Found 2026-09-13 opening the greenfield rehearsal. Derived: the set's own credential declarations
+# (config/provider-sets.json, via lib/set-credentials.sh) and the project's REQUIRED_KEYS.
+load_env_file_safe "$REPO_ROOT/.env" keep-existing
+. "$SCRIPT_DIR/lib/set-credentials.sh"
+_pf_set_keys="$(set_required_keys 2>/dev/null || printf '')"
+if [[ -z "$_pf_set_keys" ]]; then
+  ok "the active provider set (${EPAM_PROVIDER_SET:-default}) declares no required credentials"
+fi
+IFS=',' read -ra _pf_keys <<< "${_pf_set_keys}"
+for key in "${_pf_keys[@]}"; do
+  key="${key// /}"; [[ -z "$key" ]] && continue
+  if [[ -n "${!key:-}" ]]; then
+    ok "$key is set (required by the active provider set)"
+  else
+    fail "$key is NOT set — the active provider set requires it; the run will fail"
+  fi
+done
+
+IFS=',' read -ra _pf_project_keys <<< "${REQUIRED_KEYS:-}"
+for key in "${_pf_project_keys[@]}"; do
+  key="${key// /}"; [[ -z "$key" ]] && continue
+  if [[ -n "${!key:-}" ]]; then
+    ok "$key is set (required by this project)"
+  else
+    echo "  ⚠ $key not set — this project declares it in REQUIRED_KEYS; the story that needs it may fail"
+  fi
+done
+
+# ── Balance: what the account can still spend, against what this launch expects to ─────────
+# regintel 20260919T224649Z resume 6 burned attempts into 402s on an empty account; a launch on
+# 2026-09-20 was about to start on $13.50 for a ~$20 run. The set declares how to read the
+# balance (balanceProbe); the project may declare the launch's budget (costControls.runBudgetUsd).
+. "$SCRIPT_DIR/lib/spend-probe.sh" 2>/dev/null || true
+_pf_balance="$(balance_probe_read 2>/dev/null || true)"
+_pf_budget="${EPAM_RUN_BUDGET_USD:-}"
+if [[ -z "$_pf_balance" ]]; then
+  ok "balance: the active provider set (${EPAM_PROVIDER_SET:-default}) declares no balance probe, or it answered no figure — not judged"
+elif [[ -n "$_pf_budget" ]] && "${NODE_BIN:-node}" -e 'process.exit(Number(process.argv[1]) < Number(process.argv[2]) ? 0 : 1)' "$_pf_balance" "$_pf_budget" 2>/dev/null; then
+  fail "balance \$${_pf_balance} is below this project's declared run budget \$${_pf_budget} (costControls.runBudgetUsd) — top up before launching"
+else
+  ok "balance \$${_pf_balance}${_pf_budget:+ covers the declared run budget \$${_pf_budget}}"
+fi
+
+# ── Sign-in: the set's own check that this machine can call its stack ─────────────────────
+# A set whose calls ride a session (CodeMie web: the CLI's SSO) declares signInProbe — a command,
+# relative to orchestrations/, that exits 0 when a call can be made and names what is missing when
+# not. Without it a signed-out session is found by the first story, on every rung of its ladder.
+# A set declaring none is not judged.
+preflight_sign_in() {
+  local _pf_probe_argv=() _a _l _pf_probe_out _pf_probe_err _n=0
+  # One argument per line, each newline-terminated — `read` drops an unterminated last line, and a
+  # probe command without its script is `node` alone, which reads an empty stdin and "passes".
+  while IFS= read -r _a; do [[ -n "$_a" ]] && _pf_probe_argv+=("$_a"); done < <("${NODE_BIN:-node}" -e '
+    const fs=require("fs"),path=require("path");
+    const reg=JSON.parse(fs.readFileSync(path.join(process.argv[1],"..","config","provider-sets.json"),"utf8"));
+    const p=((reg.sets||{})[process.env.EPAM_PROVIDER_SET||reg.defaultSet||""]||{}).signInProbe;
+    if(p&&Array.isArray(p.command)) for(const a of p.command) process.stdout.write(a+"\n");
+  ' "$SCRIPT_DIR" 2>/dev/null || true)
+  if [[ "${#_pf_probe_argv[@]}" -eq 0 ]]; then
+    ok "sign-in: the active provider set (${EPAM_PROVIDER_SET:-default}) declares no sign-in probe — not judged"
+    return 0
+  fi
+  [[ "${_pf_probe_argv[0]}" = "node" ]] && _pf_probe_argv[0]="${NODE_BIN:-node}"
+  _pf_probe_err="$(mktemp)"
+  # stdout: one line per check that held; stderr: the sign-in chatter and, on refusal, the reason last.
+  if _pf_probe_out="$(cd "$SCRIPT_DIR/.." && "${_pf_probe_argv[@]}" 2>"$_pf_probe_err" </dev/null)"; then
+    while IFS= read -r _l; do [[ -n "$_l" ]] && { ok "sign-in: $_l"; _n=$((_n + 1)); }; done <<< "$_pf_probe_out"
+    [[ "$_n" -gt 0 ]] || ok "sign-in: the ${EPAM_PROVIDER_SET:-default} set's probe passed"
+  else
+    fail "sign-in: the ${EPAM_PROVIDER_SET:-default} set's probe refused — $(grep -v '^[[:space:]]*$' "$_pf_probe_err" | tail -1)"
+  fi
+  rm -f "$_pf_probe_err"
+}
+preflight_sign_in
+
+# ── 5. Every assigned model is a rung of a declared ladder ───────────────────
+#
+# A model absent from the chain cannot escalate: the successor lookup returns EMPTY,
+# which every call site reads as "already at the top rung". The story then burns all of
+# its attempts on one model and the log is indistinguishable from a legitimate ceiling.
+#
+# Live, run 20260814T224748Z: the prd-model-coordinator assigned `gpt-5-codex`, a model
+# on no ladder this project declares. Its own deterministic reviewer checks STRUCTURE —
+# which fields changed, whether stories were added — and never membership. The run only
+# escaped because ladder position had persisted from an earlier run and it resumed
+# mid-chain, ignoring the PRD's assignment entirely.
+#
+# Here rather than in one launcher: this file is what all eight launchers pre-flight
+# through, and the same coordinator writes the PRD for every one of them.
+echo "[ Model ladders ]"
+_mlm_lib="$SCRIPT_DIR/lib/model-ladder-membership.sh"
+_mlm_settings="${EPAM_LLM_SETTINGS_FILE:-${PROJECT_CONFIG_DIR:-}/llm-settings.json}"
+if [[ ! -f "$_mlm_lib" ]]; then
+  fail "lib/model-ladder-membership.sh missing — cannot verify assigned models can escalate"
+elif [[ -z "${PRD_FILE:-}" || ! -f "${PRD_FILE:-}" ]]; then
+  echo "  ⚠ no PRD to check model assignments against"
+elif [[ ! -f "$_mlm_settings" ]]; then
+  # UNKNOWN, not fine: a project whose ladders cannot be read must not look verified.
+  fail "no llm-settings.json at '$_mlm_settings' — cannot tell whether assigned models can escalate"
+elif [[ "$_prd_pending_ingest" == "1" ]]; then
+  # THE SAME DEFERRAL THE PRD CHECKS ABOVE ALREADY MAKE. Jira ingest rewrites this file before
+  # anything reads it, so whatever model the LAST run assigned is not this run's assignment.
+  # Live 2026-09-11: the previous run was openrouter and left `AMSD-1919 -> MiniMax-M3`; the next
+  # launch was the claude set, on whose ladders that model does not exist, and pre-flight refused
+  # the run over a value it was about to discard. Provider sets are hot-swappable on one install;
+  # a pre-flight that fails the swap on the last run's leftovers breaks that.
+  ok "model assignments belong to the last run; Jira ingest rewrites the PRD before anything reads it — deferred"
+  PASS=$((PASS+1))
+else
+  # shellcheck source=lib/model-ladder-membership.sh
+  . "$_mlm_lib"
+  if stories_with_unladdered_models "$PRD_FILE" "$_mlm_settings"; then
+    ok "every assigned model is a rung of a declared ladder"
+  elif resume_preserves prd-state; then
+    # GREENFIELD RESUME WITH A PROVIDER-SET SWAP. The PRD carries model assignments from the
+    # prior set; _mc_enforce_ladder (story-watchdog.sh) corrects every unladdered model to the
+    # current set's start rung at the top of run-agent-orchestration.sh, before any coordinator
+    # call or story run. Refusing here blocks a legitimate swap — the same class of problem the
+    # Jira ingest deferral above already solves for brownfield projects.
+    # Live 2026-09-17, regintel 20260916T200108Z: resumed with EPAM_PROVIDER_SET=claude after
+    # an openrouter run; stories carried MiniMax-M2.7-highspeed, which is on no claude ladder.
+    ok "resume — stale model assignments from prior provider set; _mc_enforce_ladder corrects them at orchestration start"
+    PASS=$((PASS+1))
+  else
+    fail "a story is assigned a model that is on no declared ladder (see above) — it could never escalate"
+  fi
+fi
+
+# ── Machine-environment checks: on by default ────────────────────────────────
+# Sections 5 and 6 and the service probes in 7 assess THIS MACHINE — is the dashboard up,
+# is the snapshot watcher alive, is Langfuse answering. A real launch must pass them.
+#
+# EPAM_PREFLIGHT_ENVIRONMENT=0 assesses the PROJECT only. It exists for callers that invoke
+# a launcher without launching on the machine's behalf — mock-launcher-parity.test.ts drives
+# the real launcher into a forced-failure phase to prove a failed run still archives its
+# evidence, and would otherwise pass or fail with whether a watcher happened to be running.
+# It cannot hide a project defect: every project-readiness check still runs and still blocks.
+_assess_environment="${EPAM_PREFLIGHT_ENVIRONMENT:-1}"
+if [ "$_assess_environment" = "0" ]; then
+  echo "[ Machine environment ]"
+  echo "  – SKIPPED by EPAM_PREFLIGHT_ENVIRONMENT=0 — project checks below still apply"
+  echo ""
+fi
+
+# ── 5. Dashboard is up ───────────────────────────────────────────────────────
+if [ "$_assess_environment" != "0" ]; then
+echo "[ Dashboard ]"
+# The dashboards are how a run is watched, so pre-flight BRINGS THEM UP rather than reporting that
+# they are down. See lib/dashboard-ensure.sh — it restarts the container and re-checks before it
+# fails. The old probe curled prd.json, a file the dashboards no longer read.
+# shellcheck source=lib/dashboard-ensure.sh
+. "$SCRIPT_DIR/lib/dashboard-ensure.sh"
+if ensure_dashboards_up "$_DASH"; then
+  ok "Dashboard is up at ${_DASH}"
+else
+  fail "Dashboard not serving at ${_DASH} — a run started now cannot be watched"
+fi
+
+# ── 6. Self-heal observability routing ───────────────────────────────────────
+# Verifies the full chain: claude.sh writes healing-events.jsonl to LOG_DIR
+# (orchestrations/logs), docker mounts orchestrations/logs at /logs-dir, and
+# nginx serves /logs/healing-events.jsonl so agent-activity.html and health.html
+# both read real data — not stale events from a previous run.
+echo "[ Self-heal observability ]"
+
+# 6a. LOG_DIR write path is writable
+# Honour an inherited LOG_DIR. This resolved ../logs unconditionally, so a run told to keep
+# its artefacts elsewhere — a test run inside its own perimeter — still had the SHARED client
+# log directory probed and reported here, and the operator was told artefacts would land in a
+# directory the run never writes to.
+LOG_DIR_DEFAULT="${LOG_DIR:-$(cd "$(dirname "$0")/../logs" 2>/dev/null && pwd || echo '')}"
+if [[ -d "$LOG_DIR_DEFAULT" ]] && touch "$LOG_DIR_DEFAULT/.preflight-write-test" 2>/dev/null; then
+  rm -f "$LOG_DIR_DEFAULT/.preflight-write-test"
+  ok "LOG_DIR ($LOG_DIR_DEFAULT) is writable — healing-events.jsonl will land here"
+else
+  fail "LOG_DIR ($LOG_DIR_DEFAULT) is not writable — self-heal events will be lost"
+fi
+
+# 6b. healing-events.jsonl at LOG_DIR is NOT stale from a prior run
+# (pre-run-reset.sh should have cleared it; non-empty means it was not reset)
+HEAL_LOG="$LOG_DIR_DEFAULT/healing-events.jsonl"
+# A RESUME CARRIES ITS OWN LEDGERS. pre-run-reset keeps them on a resume — they are this run's
+# record, and clearing them re-ran a finished phase (regintel 20260918T132928Z). Non-empty here
+# is the resumed run's own history, not a prior run's leftovers.
+if resume_preserves ledgers; then
+  ok "resuming run ${EPAM_RESUME_RUN}: healing-events.jsonl is the run's own record — kept, not stale"
+elif [[ -s "$HEAL_LOG" ]]; then
+  fail "healing-events.jsonl is non-empty from a prior run — run pre-run-reset.sh to clear it (stale data pollutes health.html)"
+else
+  ok "healing-events.jsonl is empty/absent — clean slate for this run"
+fi
+
+# 6c. Dashboard /logs/healing-events.jsonl is served by nginx (routing sanity)
+#
+# WAITED FOR, NOT SAMPLED ONCE. pre-run-reset.sh restarts agent-monitor with --force-recreate one
+# step before this runs, so the container being probed was deliberately destroyed and rebuilt
+# seconds earlier and is not yet accepting connections. A single curl therefore read a healthy
+# stack as a broken mount and refused the launch -- live 2026-09-08, pipeline-tests-40, where the
+# same URL answered 200 moments later with the file present and empty exactly as intended.
+#
+# That message has now been misread twice, once as a container-runtime fault and once as a missing
+# file. It was neither: the probe raced a restart the pipeline itself had just forced. Bounded, so
+# a mount that genuinely never comes up still refuses the launch rather than hanging.
+_logs_ok=0
+for _try in $(seq 1 15); do
+  if curl -sf ${_DASH}/logs/healing-events.jsonl >/dev/null 2>&1; then _logs_ok=1; break; fi
+  sleep 1
+done
+if [ "$_logs_ok" = "1" ]; then
+  ok "nginx serves /logs/healing-events.jsonl — agent-activity.html and health.html will read it"
+else
+  fail "nginx /logs/healing-events.jsonl not reachable after 15s — docker /logs-dir mount may be wrong; run pre-run-reset.sh"
+fi
+
+# 6d. build-info.json is fresh and contains selfHealing (health.html depends on it)
+if curl -sf "${_DASH}/build-info.json" 2>/dev/null | python3 -c "import json,sys; d=json.load(sys.stdin); assert 'selfHealing' in d.get('metrics',{}), 'selfHealing missing'" 2>/dev/null; then
+  ok "build-info.json has metrics.selfHealing — health.html will render correctly"
+else
+  fail "build-info.json missing or lacks metrics.selfHealing — health.html will show nothing; check snapshot-watch.js process"
+fi
+
+# 6e. Snapshot watcher (snapshot-watch.js) is running — health.html OOMs when
+# Eleventy's full watcher runs; snapshot-watch.js is the lightweight replacement.
+# If the PID file is missing or the process is dead, build-info.json will never
+# update during the run, leaving health.html permanently stale.
+# The watcher is a MACHINE-level daemon feeding the dashboards, not a per-run artefact, so its
+# PID file lives with those dashboards rather than in whichever log directory this run happens
+# to own. Reading it from LOG_DIR meant a run keeping its artefacts elsewhere looked for the
+# file in its own empty directory and concluded the watcher was dead.
+SNAP_PID_FILE="$(cd "$(dirname "$0")/../logs" 2>/dev/null && pwd || echo '')/dashboards-watch.pid"
+_snap_ok=false
+if [[ -f "$SNAP_PID_FILE" ]]; then
+  _snap_pid="$(cat "$SNAP_PID_FILE" 2>/dev/null || echo '')"
+  if [[ -n "$_snap_pid" ]] && ps -p "$_snap_pid" > /dev/null 2>&1; then
+    _snap_ok=true
+  fi
+fi
+# A RUNNING WATCHER WITH NO PID FILE IS STILL A RUNNING WATCHER.
+#
+# This fallback used to sit INSIDE the `-f "$SNAP_PID_FILE"` branch above, so it was reachable
+# only when a PID file already existed. Started by hand — which is exactly what the check's own
+# error message tells you to do — the watcher leaves no PID file, and the check then reported it
+# dead while it was demonstrably polling every ten seconds. The next check (6f) proves it is
+# actually refreshing, so this one only has to establish that it exists.
+if [[ "$_snap_ok" != "true" ]] && pgrep -f 'snapshot-watch.js' > /dev/null 2>&1; then
+  _snap_ok=true
+fi
+if [[ "$_snap_ok" == "true" ]]; then
+  ok "snapshot-watch.js is running — build-info.json will refresh every 10s"
+else
+  fail "snapshot-watch.js is NOT running — health.html will stay permanently stale during the run; start it with: node orchestrations/scripts/snapshot-watch.js 10 &"
+fi
+
+# 6f. build-info.json was generated recently (within 120s) — proves the watcher
+# is actively refreshing, not just present-but-stalled.
+_generated_at=$(curl -sf "${_DASH}/build-info.json" 2>/dev/null \
+  | python3 "$SCRIPT_DIR/lib/handlers/json-field.py" generatedAt 2>/dev/null || echo '')
+if [[ -n "$_generated_at" ]]; then
+  _age_s=$(python3 "$SCRIPT_DIR/lib/handlers/iso-timestamp-age-seconds.py" "$_generated_at" 2>/dev/null || echo 9999)
+  if [[ "$_age_s" -lt 120 ]]; then
+    ok "build-info.json is ${_age_s}s old — snapshot watcher is actively refreshing"
+  else
+    fail "build-info.json is ${_age_s}s old (> 120s) — snapshot watcher may be stalled; kill and restart snapshot-watch.js"
+  fi
+else
+  fail "build-info.json missing generatedAt field — snapshot watcher may be writing corrupted output"
+fi
+
+fi
+
+# ── 7. PROJECT READINESS ─────────────────────────────────────────────────────
+# Added 2026-08-05. Four launches that day died on conditions nothing checked before
+# spending: a stale dist (twice, caught by a phase gate AFTER the run started), a dead
+# observability stack, and a project with no synthesis template that silently ran under
+# ANOTHER project's identity. A pre-flight that does not assess the PROJECT is not
+# assessing what actually breaks.
+echo "[ Project readiness ]"
+
+# dist/ must be newer than src/ — the pipeline executes dist/epam.js, so a stale bundle
+# means the code under test is not the code that runs.
+_newest_src=$(find "$REPO_ROOT/src" -name '*.ts' -newer "$REPO_ROOT/dist/epam.js" -print -quit 2>/dev/null)
+if [ ! -f "$REPO_ROOT/dist/epam.js" ]; then
+  fail "dist/epam.js missing — the pipeline runs dist, not src. Build with: node ./node_modules/.bin/tsup"
+elif [ -n "$_newest_src" ]; then
+  fail "dist/ is STALE (e.g. $(basename "$_newest_src")) — source changes would NOT execute. Rebuild with tsup."
+else
+  ok "dist/ is newer than src/"
+fi
+
+# AN ESTIMATION MANIFEST THE RUN WAS TOLD TO USE MUST BE THERE (2026-09-28). Unset is nothing to
+# check — the run proceeds exactly as it did before the variable existed. Set, it is read right
+# after ingest and every ticket's assessment reaches six agents; a path that is missing or
+# unreadable would stop the run there, after the spend on ingest, so it is refused here instead.
+_estimation_manifest="$(_project_env_value EPAM_ESTIMATION_MANIFEST)"
+if [[ -n "$_estimation_manifest" ]]; then
+  if [[ -r "$_estimation_manifest" ]]; then
+    ok "estimation manifest readable: $_estimation_manifest"
+  else
+    fail "EPAM_ESTIMATION_MANIFEST=$_estimation_manifest is missing or unreadable — fix the path or unset it"
+  fi
+fi
+
+# The project must own its synthesis template. Without one, synthesize-prd-from-jira.js
+# falls back to another project's canonical and inherits ITS project block — observed
+# 2026-08-05: hello-dolly runs were labelled project.name: skyscanner-app.
+# THE CHECK IS GONE BECAUSE THE TEMPLATE IS. It required every project to ship its own stored PRD
+# template, because synthesis filled one and would otherwise borrow another project's identity.
+# Synthesis now builds the PRD from the tracker and the project's own config, so there is no
+# template to own and no identity to borrow.
+if [ -n "${PROJECT_CONFIG_DIR:-}" ]; then
+  if [ -n "${PROJECT_NAME:-}" ]; then
+    ok "project declares its own name (${PROJECT_NAME}) — synthesis has an identity of its own"
+  else
+    fail "PROJECT_NAME is not set — the synthesised PRD would carry no project identity at all"
+  fi
+else
+  echo "  – project config dir not given (--project-config); skipping identity check"
+fi
+
+# Observability: a run aborts at the tier launcher's own preflight when these are down, so
+# finding out here costs nothing and saves a launch cycle.
+if [ "${EPAM_PREFLIGHT_SKIP_NETWORK:-0}" != "1" ] && [ "$_assess_environment" != "0" ]; then
+  for _svc in langfuse grafana; do
+    _url=""
+    if [ -f "$SCRIPT_DIR/lib/service-urls.sh" ]; then
+      # shellcheck source=lib/service-urls.sh
+      . "$SCRIPT_DIR/lib/service-urls.sh" 2>/dev/null || true
+      _url="$(service_url "$_svc" 2>/dev/null || true)"
+    fi
+    [ -n "$_url" ] || continue
+    _code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 "$_url" 2>/dev/null || echo "000")
+    if [ "$_code" = "000" ]; then
+      fail "${_svc} NOT serving at ${_url} — the run will abort at the observability preflight"
+    else
+      ok "${_svc} serving (HTTP ${_code})"
+    fi
+  done
+fi
+echo ""
+
+# ── Summary ──────────────────────────────────────────────────────────────────
+echo ""
+if [[ $FAIL -eq 0 ]]; then
+  echo "━━━ ✓ All $PASS checks passed — safe to run ━━━"
+  exit 0
+else
+  echo "━━━ ✗ $FAIL check(s) FAILED — DO NOT run pipeline ━━━" >&2
+  exit 1
+fi