amsd-pipeline 1.37.0 → 2.0.0

package/install.sh

@@ -4,6 +4,10 @@
 #   ./orchestrations-installer/install.sh                      install for the default stack, with dashboards if docker is up
 #   ./install.sh --stack codemie      install for a specific stack
 #   ./install.sh --no-docker          skip dashboards entirely
+#   ./install.sh --replay on          record every run to Langfuse so it can be replayed for $0
+#                                     (default off; needs LANGFUSE_SECRET_KEY + LANGFUSE_PUBLIC_KEY
+#                                      in .env or the environment — recording without them is
+#                                      silent, and a run not recorded can never be replayed)
 #   ./install.sh --check              verify an existing install, change nothing
 #   ./install.sh --dest ~/somewhere --ref v1.7   package that ref into a NEW tree, then install it
 #
@@ -60,6 +64,20 @@ while [ $# -gt 0 ]; do
         --stack)     STACK="${2:-}"; shift 2 ;;
         --no-docker) USE_DOCKER=no; shift ;;
         --docker)    USE_DOCKER=yes; shift ;;
+        # DOCUMENTED SINCE BEFORE IT EXISTED. orchestrations/config/env-vars.json describes both
+        # Langfuse keys as belonging to "replay (--replay on / EPAM_REPLAY=on)", and this parser
+        # had no --replay arm — so an operator following the configuration got
+        # "unknown option '--replay'" and exit 1. The env var keeps working; this is the flag the
+        # docs already promised.
+        --replay)
+            REPLAY_MODE="${2:-}"
+            case "$REPLAY_MODE" in
+                on|off) ;;
+                # Never silently fall back to off: that produces an install recording nothing
+                # while the operator believes it records, and the loss is one-way.
+                *) _bad "--replay takes 'on' or 'off', not '${2:-}'"; exit 1 ;;
+            esac
+            shift 2 ;;
         --check)     CHECK_ONLY=1; shift ;;
         --uninstall) UNINSTALL=1; shift ;;
         --help|-h)   sed -n '2,15p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
@@ -340,6 +358,7 @@ RUNNER="$("$NODE_BIN" -e '
 if [ -n "$RUNNER" ]; then
     if command -v "$RUNNER" >/dev/null 2>&1; then _ok "$RUNNER (the '$STACK' runner)"
     else _bad "'$RUNNER' is not on PATH — the '$STACK' stack cannot run without it"; FAILED=1; fi
+fi
 
 # PYTHON IS A RUNTIME DEPENDENCY, not an optional extra: 88 handlers under
 # orchestrations/scripts/lib/handlers are executed with it.
@@ -348,12 +367,17 @@ if [ -n "$RUNNER" ]; then
 # stdlib, plus one LOCAL module (_testfile, imported by siblings in the same directory). So there
 # is no venv to provision, no pip install, no requirements.txt — the interpreter is the whole
 # requirement, and checking for it is the whole job.
+#
+# UNCONDITIONAL, and it was not. This check sat INSIDE the `[ -n "$RUNNER" ]` block above, so a
+# stack whose settings file declares no runner skipped a stated hard requirement entirely and the
+# install reported ready without it. pipeline-health.sh has always checked python3
+# unconditionally, so the two disagreed about whether the machine was fit to run — and the
+# installer, the one an operator trusts first, was the lenient one.
 if command -v python3 >/dev/null 2>&1; then
     _ok "python3 ($(python3 -V 2>&1 | awk '{print $2}')) — 88 handlers need it"
 else
     _bad "python3 is not on PATH — 88 pipeline handlers cannot run without it"; FAILED=1
 fi
-fi
 
 # ── Credentials: what this stack needs, from what it declares ───────────────
 _head "Credentials"
@@ -591,9 +615,29 @@ case "$REPLAY_MODE" in
         # LangfuseTracer.ts:30 gates on BOTH keys. A fresh install has empty volumes, so no project
         # and no keys exist — and recording is silently off while the containers run and capture
         # nothing. That is the one case where a warning is not enough.
+        # READ THE INSTALL'S .env, NOT THIS SHELL.
+        #
+        # This tested ${LANGFUSE_SECRET_KEY:-} from the installer's own environment. .env IS loaded
+        # earlier — but inside a command substitution (_missing_creds="$( set -a; . .env; ... )"),
+        # so it dies with that subshell and never reaches here. An operator whose .env was
+        # correctly filled in was told "replay: on but missing: LANGFUSE_SECRET_KEY
+        # LANGFUSE_PUBLIC_KEY" and the install FAILED — while the credentials step three sections
+        # above read the same file and reported it filled. One file, one run, two answers.
+        #
+        # A value already exported still wins: an operator who exports the keys in their shell
+        # must not start failing because .env does not repeat them. Read in a subshell so the keys
+        # are not leaked into everything the installer runs afterwards.
+        _lf_sk="${LANGFUSE_SECRET_KEY:-}"
+        _lf_pk="${LANGFUSE_PUBLIC_KEY:-}"
+        if [ -f "$ROOT/.env" ]; then
+            _lf_from_env="$( set -a; . "$ROOT/.env" 2>/dev/null; set +a
+                             printf '%s\t%s' "${LANGFUSE_SECRET_KEY:-}" "${LANGFUSE_PUBLIC_KEY:-}" )"
+            [ -z "$_lf_sk" ] && _lf_sk="$(printf '%s' "$_lf_from_env" | cut -f1)"
+            [ -z "$_lf_pk" ] && _lf_pk="$(printf '%s' "$_lf_from_env" | cut -f2)"
+        fi
         _lf_missing=""
-        [ -z "${LANGFUSE_SECRET_KEY:-}" ] && _lf_missing="$_lf_missing LANGFUSE_SECRET_KEY"
-        [ -z "${LANGFUSE_PUBLIC_KEY:-}" ] && _lf_missing="$_lf_missing LANGFUSE_PUBLIC_KEY"
+        [ -z "$_lf_sk" ] && _lf_missing="$_lf_missing LANGFUSE_SECRET_KEY"
+        [ -z "$_lf_pk" ] && _lf_missing="$_lf_missing LANGFUSE_PUBLIC_KEY"
         if [ -n "$_lf_missing" ]; then
             _bad "replay: on but missing:$_lf_missing — nothing would be recorded, and a run not recorded can never be replayed"
             FAILED=1
@@ -649,6 +693,22 @@ compose_up() {
     fi
     local _up=1 _log _subnet _i=0
     _log="$(mktemp)"
+
+    # A CONTAINER GETS ITS SERVICE ALIAS WHEN IT IS CREATED, AND ONLY THEN.
+    #
+    # `up -d` leaves an existing container alone. If that container was created while the network
+    # create was failing — the pool-overlap case this loop exists to walk past — compose made it
+    # unattached and CONNECTED it afterwards, and a plain connect carries no service alias. The
+    # stack then comes up with every container healthy and no DNS at all: `getent hosts postgres`
+    # unresolved, `nc: bad address`, and langfuse crash-looping on "Can't reach database server"
+    # while postgres sits healthy beside it. Measured 2026-09-07 on pipeline-tests-29 — postgres
+    # aliases=[] before, aliases=[...-postgres-1 postgres] after a down/up — and it survived three
+    # re-installs because every one of them was an `up` over containers that already existed.
+    #
+    # So a re-install starts them fresh. Volumes are untouched (no -v), which is what keeps the
+    # langfuse database and its migrations across installs; only the containers are rebuilt.
+    (cd "$ROOT" && container_compose -f "$COMPOSE_FILE" -p "$_OBS_PROJECT" down) >/dev/null 2>&1 || true
+
     for _subnet in $(isolated_subnet_candidates "$ROOT"); do
         # ATTEMPT 0 KEEPS THE WELL-KNOWN PORTS EXACTLY (offset 0) — a normal single-install machine
         # sees no change at all, still :3100, :8092, :8123, :3001. Only a genuine collision (this
@@ -866,7 +926,14 @@ else
         _LD_UP=1
         _LD_LOG="$(mktemp)"
         _LD_SUBNET=""; _LD_I=0
-        for _LD_SUBNET in $(isolated_subnet_candidates "$ROOT"); do
+        # SEEDED APART FROM THE OBSERVABILITY STACK. Both used the bare $ROOT, so one install's
+        # two stacks were handed the SAME ordered candidates and raced for the first one. Live
+        # 2026-09-06 on pipeline-tests-29: the launch dashboard took the subnet the observability
+        # stack wanted, that stack's network create then failed, and its retry came up with
+        # containers carrying NO network aliases — `getent hosts postgres` unresolved, so langfuse
+        # could never reach its database however healthy postgres reported itself. The mock stack
+        # has been seeded with "$ROOT-mock" since it was added; this one never was.
+        for _LD_SUBNET in $(isolated_subnet_candidates "$ROOT-launch"); do
             _LD_TRY_PORT=$((_LD_PORT + _LD_I * 10))
             if (cd "$LAUNCH_DIR" && LAUNCH_SUBNET="$_LD_SUBNET" LAUNCH_UI_PORT="$_LD_TRY_PORT" \
                     container_compose -f "$LAUNCH_COMPOSE" -p "$_LD_PROJECT" up -d --build) >"$_LD_LOG" 2>&1; then

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "amsd-pipeline",
-  "version": "1.37.0",
+  "version": "2.0.0",
   "description": "Installer for the amsd-pipeline orchestration stack. Clones, packages and provisions the full stack with one command — no separate git clone step.",
   "bin": {
     "amsd-pipeline": "bin/amsd-pipeline.js"