amsd-pipeline-shared 0.5.23 → 0.5.24

package/docs/gateway-baseline-2.0.80.md5

@@ -244,3 +244,4 @@ d4992178558476af4c4c228e6aa63b2d  scripts/lib/mint-and-spec.sh
 fac1b26443bb52ba45ba072f4d654920  scripts/lib/story-guards.sh
 ada237373615a26d924226184ed7614b  scripts/lib/prd-change-review.sh
 c2dfd2381db0242fdbc487a948fd7feb  scripts/lib/agent-si.js
+03bde875977ab741c54c3b38a236a496  scripts/pre-run-reset.sh

package/docs/gateway-files.txt

@@ -165,6 +165,7 @@ scripts/lib/content-retry.js
 scripts/lib/control-plane.sh
 scripts/lib/cost-emitter.js
 scripts/lib/cost-record.sh
+scripts/lib/dashboard-target.sh
 scripts/lib/doctor/agent.js
 scripts/lib/doctor/classify.js
 scripts/lib/doctor/cli.js
@@ -254,6 +255,7 @@ scripts/lib/transcript-tool-calls.js
 scripts/lib/worktree-rungs.sh
 scripts/llm-handler.sh
 scripts/mint-agents-step.js
+scripts/pre-run-reset.sh
 scripts/preflight-check.sh
 scripts/preflight-prd-integrity.sh
 scripts/preflight-static.sh

package/orchestrations/scripts/lib/dashboard-target.sh

@@ -0,0 +1,219 @@
+#!/usr/bin/env bash
+# THE DASHBOARDS SERVE A RUN — pre-run-reset.sh's Step 1, lifted into a function so every pipeline points them the same way (operator,
+# 2026-10-22: launch checklist item 12, "Dashboards all up and running, controlled by eleventy", in scope for the Claude workflows).
+# The lines below are pre-run-reset.sh's Step 1 as released (2.0.80), unchanged: the active-PRD and active-output-dir pointers, the compose
+# override mounting the run's PRD directory at /prd-dir and its LOG_DIR at /logs-dir, nginx's alias, and agent-monitor recreated.
+#   point_dashboards_at_run <prd_file> <log_dir>
+# What pre-run-reset.sh sets before its Step 1 is taken from the caller when set, else derived from this install as pre-run-reset.sh derives it.
+point_dashboards_at_run() {
+  local PRD_FILE="${1:?prd file}" LOG_DIR="${2:?log dir}"
+  local _here; _here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+  local REPO_ROOT="${REPO_ROOT:-$(cd "$_here/../../.." && pwd)}"
+  local PRD_PRESENT=1; [ -f "$PRD_FILE" ] || PRD_PRESENT=0
+  if [ -z "${OBS_PROJECT:-}" ]; then
+    local _identity_lib="$REPO_ROOT/orchestrations-installer/lib/isolated-compose-identity.sh"
+    [ -f "$_identity_lib" ] && . "$_identity_lib" || { echo "[dashboards] missing $_identity_lib — cannot resolve the observability project name" >&2; return 1; }
+    local OBS_PROJECT; OBS_PROJECT="$(isolated_project_name "$REPO_ROOT" obs)"
+  fi
+  local _STATE_FILE="${PIPELINE_SERVICES_STATE_FILE:-$REPO_ROOT/.pipeline-services-state.env}"
+  if [ -z "${OBS_DASHBOARD_PORT:-}" ] && [ -f "$_STATE_FILE" ]; then set -a; . "$_STATE_FILE"; set +a; fi
+  local COMPOSE_BASE="${COMPOSE_BASE:-$REPO_ROOT/docker-compose.observability.yml}"
+  local COMPOSE_OVERRIDE="${COMPOSE_OVERRIDE:-$REPO_ROOT/docker-compose.observability.override.yml}"
+  local DASHBOARD_STATE_DIR="${DASHBOARD_STATE_DIR:-$REPO_ROOT/orchestrations/dashboards}"
+  declare -F info >/dev/null 2>&1 || info() { echo "[dashboards] $*" >&2; }
+  declare -F success >/dev/null 2>&1 || success() { echo "[dashboards] ✓ $*" >&2; }
+# ── Step 1: Wire agent-monitor to serve the live PRD ─────────────────────────
+# Generate a compose override that mounts the PRD's PARENT DIRECTORY (not the
+# file itself) at /prd-dir inside the nginx container, and patch nginx.conf's
+# alias to the correct basename. nginx.conf routes GET /prd.json to that path.
+#
+# Directory mount, not file mount — deliberately. A single-FILE bind mount is
+# fixed to the inode present at mount time; since the orchestration pipeline
+# always writes the PRD via atomic tmp+rename (jq ... > tmp && mv tmp file /
+# fs.renameSync — correct, corruption-safe, used throughout this codebase),
+# every write creates a NEW inode at that path, silently orphaning a file
+# mount the moment the PRD is rewritten after this script runs (found live
+# 2026-07-13: the dashboard had been serving a frozen, stale PRD snapshot
+# indefinitely). A directory mount doesn't have this problem — renames INSIDE
+# the mounted directory are visible immediately, no container restart needed
+# for subsequent PRD writes during the run.
+# THE LOGS MOUNT DOES NOT DEPEND ON THE PRD. This whole block used to be skipped when the PRD did
+# not exist yet, taking /logs-dir down with it — but LOG_DIR is this run's own log directory and is
+# knowable with or without a PRD. The consequence was a chain: a fresh install has no prd.json (it
+# is REGENERATED by ingest, and no longer committed — see .gitignore), so the dashboard mount was
+# skipped, so nginx had no /logs-dir, so pre-flight's own reachability check failed
+# ("nginx /logs/healing-events.jsonl not reachable") and refused the launch — on an install where
+# nothing was actually wrong.
+#
+# /prd-dir is the only part that genuinely needs the file, so that is the only part now conditional.
+# THE PRD'S DIRECTORY IS KNOWABLE EVEN WHEN THE FILE IS NOT THERE YET, and mounting it is the whole
+# reason this is a DIRECTORY mount rather than a file mount: a file appearing inside a mounted
+# directory is visible immediately, with no container restart.
+#
+# Conditioning the /prd-dir mount on the file already existing was half a fix. On a fresh install
+# the PRD does not exist when this runs — ingest writes it minutes LATER — so the mount was skipped,
+# nothing re-mounted afterwards, and nginx served 404 for /prd.json for the whole run. The dashboard
+# showed "data offline" against a perfectly healthy run (live 2026-09-04, pipeline-tests-13).
+PRD_DIR="$(dirname "$PRD_FILE")"
+PRD_BASENAME="$(basename "$PRD_FILE")"
+# The directory itself must exist to be mounted; docker would otherwise create it as root.
+mkdir -p "$PRD_DIR" 2>/dev/null || true
+
+if [ "$PRD_PRESENT" = "0" ]; then
+  info "Configuring agent-monitor to serve $PRD_BASENAME (written by ingest during this run) and this run's logs..."
+else
+info "Configuring agent-monitor to serve: $(basename "$PRD_FILE")..."
+
+# Ensure the PRD file is world-readable so nginx (non-root) can serve it
+chmod o+r "$PRD_FILE"
+
+# build/snapshot.js (the Eleventy watcher — a separate Node process, no access
+# to the Docker mounts above) needs its own way to know which PRD is active.
+# It reads this pointer file directly; falls back to the old dead
+# orchestrations/prd.json symlink when absent so an unrun/older setup doesn't
+# break. Truncate-write (>), not atomic rename — this file itself must never
+# be subject to the inode-staleness bug it exists to work around.
+echo -n "$PRD_FILE" > "$DASHBOARD_STATE_DIR/.active-prd-path"
+
+# Same pointer-file pattern for the project output directory: snapshot.js
+# (the Eleventy watcher) reads EPAM_PROJECT_OUTPUT_DIR from its process env,
+# but since pre-run-reset.sh is invoked as a subprocess (not sourced), the
+# export above does not propagate to the parent shell or to the Eleventy
+# watcher started later by run-agent-orchestration.sh. Writing the path here
+# lets snapshot.js read it without requiring the env var in the process chain.
+echo -n "$LOG_DIR" > "$DASHBOARD_STATE_DIR/.active-output-dir"
+fi   # end: the PRD-only half (pointer files, chmod, nginx alias below)
+
+# /logs-dir gets the same directory-mount treatment as /prd-dir, and for the
+# identical reason: step-status.json/agent-status.json/agent-activity.jsonl/
+# phase-cost.jsonl are all rewritten via atomic tmp+rename or repeated append
+# during a run, and for external-project runs LOG_DIR is a directory the
+# dashboard has otherwise never been told about at all.
+#
+# WRITTEN WHETHER OR NOT A PRD EXISTS YET — the /prd-dir line is the only part that needs one.
+{
+  echo "services:"
+  echo "  agent-monitor:"
+  echo "    volumes:"
+  echo "      - ${PRD_DIR}:/prd-dir:ro"
+  echo "      - ${LOG_DIR}:/logs-dir:ro"
+} > "$COMPOSE_OVERRIDE"
+
+# nginx.conf's /prd.json alias basename must match whatever PRD was passed —
+# patch it in place (truncate-write via sed -i, not atomic rename, so this
+# single-file mount for nginx.conf itself is not subject to the same bug).
+# Pointed whether or not the file exists yet: the basename is known from --prd, and the alias must
+# already be correct when ingest writes the file into the mounted directory mid-run.
+sed -i "s|alias /prd-dir/[^;]*;|alias /prd-dir/${PRD_BASENAME};|" \
+  "$REPO_ROOT/orchestrations/dashboards/nginx.conf"
+
+# --force-recreate, not just `up -d`: nginx.conf itself is STILL a single-
+# FILE bind mount (the sed -i just above rewrites it via a new inode, same as
+# every other atomic-write case this session). Plain `up -d` only recreates a
+# container when the COMPOSE CONFIG changed — it does not notice an existing
+# bind-mounted file's inode changing underneath it, so without
+# --force-recreate the container keeps serving the nginx.conf it started
+# with, silently ignoring this run's PRD-basename patch (found live
+# 2026-07-13 verifying this exact fix — even the fix's own docs file wasn't
+# safe from the bug it documents).
+# THE COMPOSE THAT MATCHES THE INSTALL, resolved once.
+#
+# This shelled out to `docker compose` literally. A podman install therefore never remounted
+# agent-monitor, so the run's own log dir was never served and the pre-flight gate
+# "nginx /logs/healing-events.jsonl not reachable" failed on every podman run -- after the
+# installer's health check had already reported the box "ready", because that check probes host
+# ports and cannot see a mount that was never made.
+#
+# container_compose() already resolves the runtime and knows podman needs PODMAN_COMPOSE_PROVIDER
+# and a real runtime dir, and the installer lib ships inside every install. Sourced, never
+# reimplemented: a second copy of the resolution is how this drifted from install.sh in the first
+# place.
+_obs_runtime_lib() {
+  local _c
+  for _c in "${INSTALLER_LIB:-}" \
+            "$(dirname "${BASH_SOURCE[0]}")/../../orchestrations-installer/lib/container-runtime.sh" \
+            "${REPO_ROOT:-}/orchestrations-installer/lib/container-runtime.sh"; do
+    [ -n "$_c" ] && [ -f "$_c" ] && { printf '%s' "$_c"; return 0; }
+  done
+  return 1
+}
+_obs_runtime_name() {
+  local _l; _l="$(_obs_runtime_lib)" || { printf 'none'; return 1; }
+  # shellcheck disable=SC1090
+  . "$_l" 2>/dev/null || { printf 'none'; return 1; }
+  container_runtime 2>/dev/null || printf 'none'
+}
+_obs_have_runtime() {
+  local _n; _n="$(_obs_runtime_name)"
+  [ -n "$_n" ] && [ "$_n" != "none" ]
+}
+_obs_compose() {
+  local _l; _l="$(_obs_runtime_lib)" || return 1
+  # shellcheck disable=SC1090
+  . "$_l" 2>/dev/null || return 1
+  container_compose "$@"
+}
+
+# THE DASHBOARD RESTART IS THE ONLY SLOW STEP, AND A TEST DOES NOT NEED IT.
+#
+# `docker compose up --force-recreate` costs ~3 seconds. Everything else this script does — the
+# roster restore, the PRD restore, clearing generated artefacts and prompts — is file work that
+# takes milliseconds. A test that exercises the RESET therefore paid three seconds per case for a
+# container it never looks at: two test files spent 23 of the 24 seconds my tests take, purely
+# here, and that is the difference between a suite that gets run and one that does not.
+#
+# Skipped only when asked. A run never sets this, so the live path is unchanged.
+# REPLACING THE CONTAINER, NOT RESTARTING IT.
+#
+# --force-recreate is honoured by docker compose and IGNORED by podman-compose, which restarts the
+# existing container instead. A restarted container keeps the volumes it was created with, so the
+# override this script just wrote -- carrying THIS run's /prd-dir and /logs-dir -- was reported as
+# applied and had no effect. Live 2026-09-08, pipeline-tests-41: override written 08:28,
+# agent-monitor created 08:23 at install time and merely started at 08:28, mounts holding neither
+# /logs-dir nor /prd-dir, and the run then died at a pre-flight check for a /logs path nginx could
+# not serve.
+#
+# The same class this file already records for nginx.conf ("without --force-recreate the container
+# keeps serving the nginx.conf it started with") -- one runtime further down. Removing the service
+# first makes the replacement explicit and true on both runtimes; --force-recreate stays for
+# docker, where it is meaningful and cheap.
+_obs_replace_agent_monitor() {
+  EPAM_OBS_SUBNET="${OBS_SUBNET:-}" \
+  EPAM_OBS_CLICKHOUSE_PORT="${OBS_CLICKHOUSE_PORT:-}" \
+  EPAM_OBS_LANGFUSE_PORT="${OBS_LANGFUSE_PORT:-}" \
+  EPAM_OBS_DASHBOARD_PORT="${OBS_DASHBOARD_PORT:-}" \
+  EPAM_OBS_GRAFANA_PORT="${OBS_GRAFANA_PORT:-}" \
+  EPAM_PRD_DIR="$PRD_DIR" EPAM_LOG_DIR="$LOG_DIR" \
+  _obs_compose -f "$COMPOSE_BASE" -f "$COMPOSE_OVERRIDE" -p "$OBS_PROJECT" \
+    rm -sf agent-monitor >/dev/null 2>&1 || true
+  EPAM_OBS_SUBNET="${OBS_SUBNET:-}" \
+  EPAM_OBS_CLICKHOUSE_PORT="${OBS_CLICKHOUSE_PORT:-}" \
+  EPAM_OBS_LANGFUSE_PORT="${OBS_LANGFUSE_PORT:-}" \
+  EPAM_OBS_DASHBOARD_PORT="${OBS_DASHBOARD_PORT:-}" \
+  EPAM_OBS_GRAFANA_PORT="${OBS_GRAFANA_PORT:-}" \
+  EPAM_PRD_DIR="$PRD_DIR" EPAM_LOG_DIR="$LOG_DIR" \
+  _obs_compose -f "$COMPOSE_BASE" -f "$COMPOSE_OVERRIDE" -p "$OBS_PROJECT" \
+    up -d --force-recreate agent-monitor 2>/dev/null
+}
+
+if [ "${EPAM_SKIP_CONTAINER_RESTART:-0}" = "1" ]; then
+  info "  Container restart skipped (EPAM_SKIP_CONTAINER_RESTART=1)"
+
+elif _obs_replace_agent_monitor; then
+  if [ "$PRD_PRESENT" = "1" ]; then
+    success "agent-monitor restarted → /prd-dir = $PRD_DIR (serving $PRD_BASENAME), /logs-dir = $LOG_DIR"
+  else
+    success "agent-monitor restarted → /prd-dir = $PRD_DIR (serving $PRD_BASENAME once ingest writes it), /logs-dir = $LOG_DIR"
+  fi
+elif ! _obs_have_runtime; then
+  info "  no container runtime found — skipping container restart"
+else
+  # NOT "docker is missing". The runtime IS here and the remount FAILED, which is a different
+  # fact and a different fix. The old text blamed an absent docker for every outcome, and on
+  # 2026-09-08 that sent the reader hunting containers "stuck in Created" while the real cause
+  # was this call naming a runtime the box does not use.
+  info "  agent-monitor remount FAILED with $(_obs_runtime_name) — this run's /logs-dir and /prd-dir are NOT mounted;"
+  info "  the pre-flight check 'nginx /logs/healing-events.jsonl not reachable' will fail because of this"
+fi
+  return 0
+}