amsd-pipeline 1.9.0 → 1.10.0

package/install.sh

@@ -93,6 +93,18 @@ if [ "$UNINSTALL" = "1" ]; then
             exit 1
         fi
     fi
+    # STOP THE RUNNER HOST FIRST — it is NOT docker-related (a plain host process), so it must not
+    # be skipped by the "no container runtime, nothing to uninstall" exit below.
+    _UN_RH_PIDFILE="$_UN_ROOT/launch-dashboard/.runner-host.pid"
+    if [ -f "$_UN_RH_PIDFILE" ]; then
+        _UN_RH_PID="$(cat "$_UN_RH_PIDFILE" 2>/dev/null)"
+        if [ -n "$_UN_RH_PID" ] && kill -0 "$_UN_RH_PID" 2>/dev/null; then
+            kill "$_UN_RH_PID" 2>/dev/null
+            _ok "stopped runner-host (pid $_UN_RH_PID)"
+        fi
+        rm -f "$_UN_RH_PIDFILE"
+    fi
+
     . "$INSTALLER_DIR/lib/isolated-compose-identity.sh"
     if [ -f "$INSTALLER_DIR/lib/container-runtime.sh" ]; then
         . "$INSTALLER_DIR/lib/container-runtime.sh"
@@ -204,14 +216,33 @@ if [ -n "$DEST" ]; then
             [ -n "$_excl" ] && _RUN_STATE_EXCLUDES+=("$_excl")
         done < <(run_state_exclude_args "$INSTALLER_DIR/run-state-paths.json")
     fi
+
+    # AN UPDATE MUST NEVER OVERWRITE AN OPERATOR'S EXISTING PROJECT CONFIG EITHER — a DIFFERENT
+    # mechanism from the excludes above, deliberately: a blanket exclude would also block a
+    # brand-new project's config.env from ever being extracted the first time a later ref adds
+    # one. Snapshot whatever already exists now, extract, then restore it over what the ref just
+    # wrote — so a first install (or a genuinely new project) is unaffected either way.
+    _OPCFG_TMP=""
+    if [ -f "$INSTALLER_DIR/operator-config-paths.json" ]; then
+        . "$INSTALLER_DIR/lib/preserve-operator-config.sh"
+        _OPCFG_TMP="$(mktemp -d)"
+        snapshot_operator_config "$DEST" "$INSTALLER_DIR/operator-config-paths.json" "$_OPCFG_TMP"
+    fi
+
     # The ${arr[@]+"${arr[@]}"} form, not bare "${arr[@]}": bash <4.4 (macOS ships 3.2 by default,
     # GPLv3 licensing) throws "unbound variable" under `set -u` expanding an empty array the plain
     # way. This form is safe on every bash this installer might run under.
     if ! git -C "$_GIT_ROOT" archive "$_PKG_REF" \
             | tar -x -C "$DEST" "${_RUN_STATE_EXCLUDES[@]+"${_RUN_STATE_EXCLUDES[@]}"}"; then
         _bad "packaging '$_PKG_REF' into $DEST failed"
+        [ -n "$_OPCFG_TMP" ] && rm -rf "$_OPCFG_TMP"
         exit 1
     fi
+
+    if [ -n "$_OPCFG_TMP" ]; then
+        restore_operator_config "$DEST" "$_OPCFG_TMP"
+        rm -rf "$_OPCFG_TMP"
+    fi
     _ok "packaged $_PKG_REF into $DEST"
     ROOT="$DEST"
     CONFIG="$ROOT/orchestrations/config"
@@ -519,6 +550,24 @@ esac
 
 # ── Dashboards: OPTIONAL, and never a reason to fail ────────────────────────
 _head "Dashboards (optional)"
+
+# BUILD BEFORE UP, OR THE HEALTHCHECK FAILS BY CONSTRUCTION. orchestrations/dashboards/live/ is
+# gitignored — eleventy's own build output, never tracked — so on every fresh install it starts
+# EMPTY. agent-monitor's healthcheck probes `/`, nginx has no index and autoindex is off, so a
+# brand-new install was 403-unhealthy FOREVER regardless of docker, subnet or port: found live
+# 2026-09-03, grafana (depends_on agent-monitor: condition service_healthy) never even started —
+# stuck at "Created". Only a completed pipeline run (or this build) ever populated live/ before.
+#
+# Only when there is something to build FROM (src/ present, same test the Build section above
+# uses) — a packaged, src/-less install ships no eleventy at all.
+if [ -d "$ROOT/src" ] && [ -f "$ROOT/package.json" ]; then
+    if (cd "$ROOT" && npm run dashboards:build --silent >/dev/null 2>&1); then
+        _ok "dashboards built — agent-monitor has real content to serve"
+    else
+        _warn "dashboards:build failed — agent-monitor's healthcheck may fail until a run populates it"
+    fi
+fi
+
 # THE PROBE ASKS THE RESOLVED RUNTIME. It said `docker` literally, so on a podman-only machine the
 # installer announced "runtime: podman" and then started nothing — the report and the behaviour
 # disagreeing, which is this file's recurring defect.
@@ -615,25 +664,70 @@ else
     . "$INSTALLER_DIR/lib/wait-for-health.sh"
     . "$INSTALLER_DIR/lib/isolated-compose-identity.sh"
 
-    # A REAL PASSWORD IS A DECISION ONLY A HUMAN MAKES — never synthesized here. Mirrors the root
-    # .env handling: copy the template so there is something to fill in, never invent a secret.
+    # A VENDOR/API CREDENTIAL IS A DECISION ONLY A HUMAN MAKES — never synthesized (see the root
+    # .env handling: copy the template, never invent a secret). LAUNCH_PASSWORD is a DIFFERENT
+    # risk class: it gates a loopback-only local UI, not a billed vendor account or a shared
+    # system — a blank one previously meant install.sh already knew this stack could not start
+    # (it had just written this exact warning) and then attempted `up -d` anyway, hard-failing on
+    # compose's `${LAUNCH_PASSWORD:?...}` interpolation instead of the warning it already gave.
+    # Operator decision 2026-09-03: generate one so the dashboard starts unattended; changeable
+    # any time by editing launch-dashboard/.env directly.
     if [ ! -f "$LAUNCH_DIR/.env" ]; then
         if [ -f "$LAUNCH_DIR/.env.example" ]; then
             cp "$LAUNCH_DIR/.env.example" "$LAUNCH_DIR/.env"
-            _warn "launch-dashboard/.env created from .env.example — FILL IN LAUNCH_PASSWORD before it can start"
+            _GENERATED_PW="$("$NODE_BIN" -e 'process.stdout.write(require("crypto").randomBytes(18).toString("base64url"))' 2>/dev/null)"
+            if [ -n "$_GENERATED_PW" ]; then
+                # REPLACE the template's blank line in place — never append a second
+                # LAUNCH_PASSWORD= key. Both parse fine (bash sourcing takes the last one) but a
+                # duplicate key is a needless trap for whoever reads this file by hand next.
+                if grep -q '^LAUNCH_PASSWORD=' "$LAUNCH_DIR/.env"; then
+                    _LD_TMP="$(mktemp)"
+                    sed "s|^LAUNCH_PASSWORD=.*|LAUNCH_PASSWORD=$_GENERATED_PW|" "$LAUNCH_DIR/.env" > "$_LD_TMP" \
+                        && mv "$_LD_TMP" "$LAUNCH_DIR/.env"
+                else
+                    printf '\nLAUNCH_PASSWORD=%s\n' "$_GENERATED_PW" >> "$LAUNCH_DIR/.env"
+                fi
+                _ok "launch-dashboard/.env created with a generated LAUNCH_PASSWORD"
+                # SHOWN ONCE, HERE — otherwise the only way to learn it is to already know to go
+                # read the file by hand, which is exactly the gap an operator hit live: the
+                # dashboard was up and healthy with no way to log into it from the install output
+                # alone. Also saved in launch-dashboard/.env for every time after this one.
+                printf '      LAUNCH_PASSWORD: %s\n' "$_GENERATED_PW"
+                printf '      (also saved in launch-dashboard/.env — edit that file to change it)\n'
+            else
+                _warn "launch-dashboard/.env created from .env.example — FILL IN LAUNCH_PASSWORD before it can start"
+            fi
         else
             _bad "launch-dashboard/.env is missing and there is no .env.example to create one from"
             FAILED=1
         fi
     fi
 
+    # PRE-CREATE THE BIND-MOUNT SOURCES, AS THE HOST USER, BEFORE DOCKER EVER SEES THEM.
+    #
+    # The compose file's own comment above (services.launch-api) already names this exact trap: a
+    # bind mount onto a directory that doesn't exist yet gets auto-created BY DOCKER, as root — and
+    # launch-api's own `user: "${LAUNCH_UID:-1000}:..."` then cannot write to it. Found live
+    # 2026-09-03 against a genuinely fresh install: "unable to open database file", launch-api
+    # crash-looping, nginx's launch-ui reporting "host not found in upstream" as a downstream
+    # symptom of the crash — same bug CLASS as the dashboards live/ directory fixed above, here for
+    # ./data and ./spool specifically.
+    mkdir -p "$LAUNCH_DIR/data" "$LAUNCH_DIR/spool"
+
     _LD_PORT="$(grep -E '^LAUNCH_UI_PORT=' "$LAUNCH_DIR/.env" 2>/dev/null | tail -1 | cut -d= -f2)"
     _LD_PORT="${_LD_PORT:-8099}"
     _LD_PROJECT="$(isolated_project_name "$ROOT" launch)"
     _LD_HEALTH_URL="http://localhost:${_LD_PORT}/api/health"
 
+    _LD_PW="$(grep -E '^LAUNCH_PASSWORD=' "$LAUNCH_DIR/.env" 2>/dev/null | tail -1 | cut -d= -f2-)"
     if [ ! -f "$LAUNCH_DIR/.env" ]; then
         LAUNCH_STATUS=failed
+    elif [ -z "$_LD_PW" ] && [ "$CHECK_ONLY" != "1" ]; then
+        # KNOWN ALREADY, NEVER A SURPRISE CRASH. A .env from before LAUNCH_PASSWORD was
+        # auto-generated (or one an operator deliberately blanked) still fails compose's
+        # `${LAUNCH_PASSWORD:?...}` interpolation — skip the attempt instead of hitting it.
+        LAUNCH_STATUS=failed
+        _warn "launch-dashboard/.env has no LAUNCH_PASSWORD — skipping start; set one and re-run to bring it up"
     elif [ "$CHECK_ONLY" = "1" ]; then
         if wait_for_health "$_LD_HEALTH_URL" 3 1; then
             LAUNCH_STATUS=up
@@ -692,6 +786,68 @@ else
     fi
 fi
 
+# ── Runner host: what actually launches a pipeline run from the dashboard ────
+# "the install script must start all services" (operator, 2026-09-04) — a saved launch request
+# sat "pending" forever with nothing polling for it, because nothing ever started this.
+#
+# NOT DOCKERIZED, DELIBERATELY — same as runner-host.js's own header says: "a container cannot
+# exec a host process." It spawns the pipeline's real launcher on the HOST, which needs git access
+# to the codeline root, the claude/codemie-claude CLI's host auth (~/.claude — a container has none
+# of this unless it were bind-mounted in), and host git credentials for anything that pushes.
+# Containerizing the poll loop alone is easy (the spool it watches is already bind-mounted into
+# launch-api); containerizing what it SPAWNS on a hit would mean containerizing the whole pipeline.
+_head "Runner host (launches pipeline runs the dashboard queues)"
+if [ "$LAUNCH_STATUS" = "up" ]; then
+    _RH_PIDFILE="$LAUNCH_DIR/.runner-host.pid"
+    _RH_LOG="$LAUNCH_DIR/.runner-host.log"
+    _RH_OLD_PID=""
+    [ -f "$_RH_PIDFILE" ] && _RH_OLD_PID="$(cat "$_RH_PIDFILE" 2>/dev/null)"
+    if [ -n "$_RH_OLD_PID" ] && kill -0 "$_RH_OLD_PID" 2>/dev/null; then
+        _ok "already running (pid $_RH_OLD_PID)"
+    else
+        # setsid, NEVER nohup — found live: nohup here made install.sh hang forever whenever its
+        # own stdio is piped (any parent that captures its output, including this test suite).
+        # setsid fully detaches into a new session (immune to SIGHUP by construction, survives the
+        # launching shell/terminal closing — the WSL-restart case this exists for). Falls back to
+        # a plain backgrounded process on a host with no setsid (macOS ships none by default).
+        #
+        # `</dev/null >>log 2>&1` on the command ALONE was still not enough — the daemon kept the
+        # pipe to install.sh's own stdout open regardless (Node's spawn() never saw 'close', even
+        # though every byte of real output arrived and install.sh itself had long since exited).
+        # bash forking a background job inherits ALL open fds, not just 0/1/2; a plain per-command
+        # redirect only dup2's those three. `exec` with no command applies the redirect to the
+        # CURRENT shell — including whatever else it inherited — before the second `exec` replaces
+        # that shell's own process image with the daemon, so nothing is left holding the pipe open.
+        _RH_DAEMONIZE="setsid"
+        command -v setsid >/dev/null 2>&1 || _RH_DAEMONIZE=""
+        # launch-dashboard/.env MUST BE SOURCED HERE. Docker Compose auto-loads a .env file next
+        # to the compose file into the CONTAINER's environment; a bare host process gets none of
+        # that for free. Found live: runner-host.js's own config.js hard-requires LAUNCH_PASSWORD
+        # from process.env ("gates a button that spends real money") and crashed instantly with it
+        # unset, even though the value was sitting right there in the file the whole time.
+        # SPOOL_DIR's default ('/spool') is the CONTAINER's bind-mount path — correct for
+        # launch-api running inside docker, meaningless for a bare host process. Found live, right
+        # after the LAUNCH_PASSWORD fix above stopped masking it: EACCES on mkdir '/spool/requests'
+        # (no permission to create a directory at the filesystem root). The real, same, host
+        # directory this container has bind-mounted as /spool is $LAUNCH_DIR/spool.
+        ( exec </dev/null >>"$_RH_LOG" 2>&1
+          cd "$ROOT" && set -a && . "$LAUNCH_DIR/.env" 2>/dev/null; set +a
+          EPAM_HOME="$ROOT" SPOOL_DIR="$LAUNCH_DIR/spool" RUNS_DB="$LAUNCH_DIR/data/runs.db" \
+              exec $_RH_DAEMONIZE "$NODE_BIN" "$LAUNCH_DIR/backend/src/runner-host.js" ) &
+        echo $! > "$_RH_PIDFILE"
+        sleep 0.3
+        _RH_NEW_PID="$(cat "$_RH_PIDFILE" 2>/dev/null)"
+        if [ -n "$_RH_NEW_PID" ] && kill -0 "$_RH_NEW_PID" 2>/dev/null; then
+            _ok "started (pid $_RH_NEW_PID, log: $_RH_LOG)"
+        else
+            _bad "runner-host failed to start — see $_RH_LOG"
+            FAILED=1
+        fi
+    fi
+else
+    _ok "skipped — launch dashboard status is '$LAUNCH_STATUS', nothing to poll for"
+fi
+
 # ── The command people will actually type ───────────────────────────────────
 # ── What this install IS ──────────────────────────────────────────────────────
 # An install whose mode can only be inferred from which containers happen to be running is an

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "amsd-pipeline",
-  "version": "1.9.0",
+  "version": "1.10.0",
   "description": "Installer for the amsd-pipeline orchestration stack. Clones, packages and provisions the full stack with one command — no separate git clone step.",
   "bin": {
     "amsd-pipeline": "bin/amsd-pipeline.js"