@kybernesis/create 0.14.7 → 0.14.8

package/dist/deploy.js

@@ -1,4 +1,5 @@
 import { execFileSync, spawnSync } from "node:child_process";
+import { TEMPLATE_CONTAINER_PROBE, assessRestart, parseTemplateContainers } from "./sandbox-orphans.js";
 import { existsSync, readFileSync } from "node:fs";
 import { join } from "node:path";
 import { upsertEnv } from "./envfile.js";
@@ -239,6 +240,14 @@ export async function deploy(options) {
         '  SCRIPT="scripts/eve-server.sh"',
         "fi",
         'if [ -z "$SCRIPT" ]; then echo "FAILED: no restart script on the host"; exit 1; fi',
+        // An orphaned sandbox template container (a start killed mid-build; only
+        // `sleep` inside) makes every eve start hang forever. The packaged start
+        // script clears it too, but a host still running its own older copy
+        // would not, so the deploy does it first and says so (KYB-531).
+        "for c in $(command -v docker >/dev/null 2>&1 && docker ps --filter name=eve-sbx-tpl- --format '{{.Names}}' 2>/dev/null); do",
+        "  live=$(docker top \"$c\" 2>/dev/null | tail -n +2 | awk '{print $8}' | grep -vc '^sleep$')",
+        '  if [ "${live:-0}" = "0" ]; then echo "removing orphaned sandbox template container $c (only sleep inside; it would block every start)"; docker rm -f "$c" >/dev/null 2>&1 || true; fi',
+        "done",
         'echo "using $SCRIPT"',
         'setsid nohup bash "$SCRIPT" > /tmp/kyb-deploy.log 2>&1 < /dev/null &',
         "sleep 2",
@@ -266,23 +275,45 @@ export async function deploy(options) {
     let last = "";
     let seen = "";
     let lastChange = Date.now();
+    let lastBuildingNote = 0;
+    let stuck = null;
     while (Date.now() - startedAt < CEILING_MS) {
-        const out = execFileSync("ssh", [target, `tail -40 /tmp/kyb-deploy.log 2>/dev/null || true`], {
-            encoding: "utf8",
-        });
-        last = out;
-        if (out !== seen) {
-            for (const line of out.split("\n")) {
-                if (line && !seen.includes(line) && /npm|install|build|SOURCE IS NEWER|waiting|pid=|health:|FAILED/i.test(line)) {
+        // Two witnesses: the log, and the sandbox template containers. A cold
+        // template build is silent in the log for longer than the quiet limit,
+        // and judging it on the log alone is how a fine build got killed (KYB-531).
+        const out = execFileSync("ssh", [target, `tail -40 /tmp/kyb-deploy.log 2>/dev/null || true; echo __TEMPLATES__; ${TEMPLATE_CONTAINER_PROBE}`], { encoding: "utf8" });
+        const [logPart, probePart = ""] = out.split("__TEMPLATES__");
+        last = logPart;
+        if (logPart !== seen) {
+            for (const line of logPart.split("\n")) {
+                if (line && !seen.includes(line) && /npm|install|build|SOURCE IS NEWER|waiting|pid=|health:|FAILED|template|removing/i.test(line)) {
                     console.log(dim(`    ${line.trim().slice(0, 120)}`));
                 }
             }
-            seen = out;
+            seen = logPart;
+            lastChange = Date.now();
+        }
+        const verdict = assessRestart({
+            log: logPart,
+            templates: parseTemplateContainers(probePart),
+            quietMs: Date.now() - lastChange,
+            quietLimitMs: QUIET_LIMIT_MS,
+        });
+        if (verdict.state === "healthy" || verdict.state === "failed")
+            break;
+        if (verdict.state === "building") {
             lastChange = Date.now();
+            if (Date.now() - lastBuildingNote > 60_000) {
+                console.log(dim(`    still building the sandbox template (${Math.round((Date.now() - startedAt) / 60_000)} min): ${verdict.detail}`));
+                lastBuildingNote = Date.now();
+            }
         }
-        if (/health:|FAILED/.test(out))
+        else if (verdict.state === "orphaned") {
+            stuck = verdict.detail ?? null;
+            console.log(yellow(`\n  ! an orphaned sandbox template container is blocking the start: ${stuck}`));
             break;
-        if (Date.now() - lastChange > QUIET_LIMIT_MS) {
+        }
+        else if (verdict.state === "quiet") {
             console.log(yellow(`\n  ! nothing new on the host for ${Math.round(QUIET_LIMIT_MS / 60_000)} minutes — giving up on the wait`));
             break;
         }
@@ -296,8 +327,11 @@ export async function deploy(options) {
         .join("\n"));
     console.log(healthy
         ? green("\n  ✓ deployed and serving the current build")
-        : yellow("\n  ! the restart did not report health. It may still be building — this stops watching, it does not stop the host.\n" +
-            `    Watch it:  ssh ${target} 'tail -f /tmp/kyb-deploy.log'`));
+        : stuck
+            ? yellow(`\n  ! the restart cannot finish while that container exists. Remove it on the host, then run kyb deploy again.\n` +
+                `    ssh ${target} '${stuck.replace(/^.*remove it: /, "").replace(/\).*$/, "")}'`)
+            : yellow("\n  ! the restart did not report health. It may still be building — this stops watching, it does not stop the host.\n" +
+                `    Watch it:  ssh ${target} 'tail -f /tmp/kyb-deploy.log'`));
     if (!healthy)
         process.exitCode = 1;
 }

package/dist/doctor.js

@@ -1,4 +1,5 @@
 import { existsSync, readFileSync, readdirSync } from "node:fs";
+import { TEMPLATE_CONTAINER_PROBE, parseTemplateContainers, templateContainerDoctorChecks } from "./sandbox-orphans.js";
 import { join } from "node:path";
 import { pathToFileURL } from "node:url";
 import { bold, capture, dim, green, parseEnv, red, yellow } from "./util.js";
@@ -377,6 +378,11 @@ export async function doctor() {
          * eventually produces looks like anything except a full disk.
          */
         if (capture("sh", ["-c", "command -v docker >/dev/null && echo yes"])?.trim() === "yes") {
+            // A template container left behind by a killed start blocks every
+            // later start forever; a live one means "do not restart yet" (KYB-531).
+            for (const check of templateContainerDoctorChecks(parseTemplateContainers(capture("sh", ["-c", TEMPLATE_CONTAINER_PROBE])))) {
+                checks.push(check);
+            }
             const job = capture("sh", ["-c", "test -x /etc/cron.daily/kyb-docker-prune && echo yes"])?.trim();
             const percent = Number(capture("sh", ["-c", "df / | awk 'NR==2{print $5}' | tr -d '%'"])?.trim() ?? 0);
             if (job !== "yes") {

package/dist/sandbox-orphans.d.ts

@@ -0,0 +1,56 @@
+import type { Check } from "./doctor.js";
+/**
+ * Docker sandbox template containers: the ones still building, and the ones
+ * left behind.
+ *
+ * eve builds each Docker sandbox template inside an `eve-sbx-tpl-*` container
+ * on first start. A cold build (apt-get, pnpm, playwright with its browsers)
+ * runs for minutes and prints nothing to the deploy log while it does. Two
+ * things went wrong with that on one deployment (KYB-531):
+ *
+ * - the host start script judged the start on a 45-second clock and printed
+ *   "FAILED to start" while the template was building, so the operator killed
+ *   the start;
+ * - the killed start left the container behind with `sleep 2147483647` as its
+ *   only process, and while it existed every later `eve start` blocked forever
+ *   at "initializing N sandbox templates" with nothing in any log to say why.
+ *
+ * Everything here reads one probe: each template container's name and how
+ * many processes inside it are not `sleep`. Building means at least one; an
+ * orphan means none.
+ */
+export interface TemplateContainer {
+    name: string;
+    /** Processes inside the container other than `sleep`. */
+    liveProcesses: number;
+}
+/**
+ * Shell that prints one `name count` line per `eve-sbx-tpl-*` container, or
+ * nothing when Docker is absent. The same probe serves doctor (locally), the
+ * deploy (over ssh), and the host start script.
+ */
+export declare const TEMPLATE_CONTAINER_PROBE: string;
+export declare function parseTemplateContainers(output: string | null | undefined): TemplateContainer[];
+export declare function orphanedTemplates(containers: readonly TemplateContainer[]): TemplateContainer[];
+export declare function buildingTemplates(containers: readonly TemplateContainer[]): TemplateContainer[];
+/** The exact command that clears an orphan; printed as the remedy and run by the start script. */
+export declare function removeOrphanCommand(name: string): string;
+export declare function templateContainerDoctorChecks(containers: readonly TemplateContainer[]): Check[];
+export type RestartState = "healthy" | "failed" | "building" | "orphaned" | "quiet" | "waiting";
+/**
+ * What a deploy should conclude from the host right now.
+ *
+ * The log alone was the whole verdict before, and a template build is silent
+ * in it. The template probe is the second witness: while something is building,
+ * silence is progress and the wait goes on; an orphan with nothing building
+ * and no server is a stuck host with a known remedy, not a slow one.
+ */
+export declare function assessRestart(input: {
+    log: string;
+    templates: readonly TemplateContainer[];
+    quietMs: number;
+    quietLimitMs: number;
+}): {
+    state: RestartState;
+    detail?: string;
+};

package/dist/sandbox-orphans.js

@@ -0,0 +1,77 @@
+/**
+ * Shell that prints one `name count` line per `eve-sbx-tpl-*` container, or
+ * nothing when Docker is absent. The same probe serves doctor (locally), the
+ * deploy (over ssh), and the host start script.
+ */
+export const TEMPLATE_CONTAINER_PROBE = [
+    "command -v docker >/dev/null 2>&1 || exit 0",
+    "for c in $(docker ps --filter name=eve-sbx-tpl- --format '{{.Names}}' 2>/dev/null); do",
+    "  live=$(docker top \"$c\" 2>/dev/null | tail -n +2 | awk '{print $8}' | grep -vc '^sleep$')",
+    '  echo "$c ${live:-0}"',
+    "done",
+].join("\n");
+export function parseTemplateContainers(output) {
+    return (output ?? "")
+        .split("\n")
+        .map((line) => line.trim())
+        .filter((line) => line.startsWith("eve-sbx-tpl-"))
+        .map((line) => {
+        const [name, count] = line.split(/\s+/);
+        return { name, liveProcesses: Number(count) || 0 };
+    });
+}
+export function orphanedTemplates(containers) {
+    return containers.filter((c) => c.liveProcesses === 0);
+}
+export function buildingTemplates(containers) {
+    return containers.filter((c) => c.liveProcesses > 0);
+}
+/** The exact command that clears an orphan; printed as the remedy and run by the start script. */
+export function removeOrphanCommand(name) {
+    return `docker rm -f ${name}`;
+}
+export function templateContainerDoctorChecks(containers) {
+    const orphans = orphanedTemplates(containers);
+    const building = buildingTemplates(containers);
+    const checks = orphans.map((c) => ({
+        verdict: "fail",
+        label: `orphaned sandbox template container ${c.name}: every eve start will hang until it is removed`,
+        detail: `its only process is sleep; a start was killed mid-build. Remove it: ${removeOrphanCommand(c.name)}`,
+    }));
+    if (building.length) {
+        checks.push({
+            verdict: "warn",
+            label: `sandbox template building: ${building.map((c) => c.name).join(", ")}`,
+            detail: "a cold build takes minutes and the server binds its port only after it finishes; do not restart",
+        });
+    }
+    return checks;
+}
+/**
+ * What a deploy should conclude from the host right now.
+ *
+ * The log alone was the whole verdict before, and a template build is silent
+ * in it. The template probe is the second witness: while something is building,
+ * silence is progress and the wait goes on; an orphan with nothing building
+ * and no server is a stuck host with a known remedy, not a slow one.
+ */
+export function assessRestart(input) {
+    if (/health:\s*200/.test(input.log))
+        return { state: "healthy" };
+    if (/FAILED/.test(input.log))
+        return { state: "failed" };
+    const building = buildingTemplates(input.templates);
+    if (building.length) {
+        return { state: "building", detail: building.map((c) => c.name).join(", ") };
+    }
+    const orphans = orphanedTemplates(input.templates);
+    if (orphans.length) {
+        return {
+            state: "orphaned",
+            detail: orphans.map((c) => `${c.name} (remove it: ${removeOrphanCommand(c.name)})`).join("; "),
+        };
+    }
+    if (input.quietMs > input.quietLimitMs)
+        return { state: "quiet" };
+    return { state: "waiting" };
+}

package/dist/upgrade.js

@@ -1,4 +1,4 @@
-import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { chmodSync, copyFileSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
 import { spawnSync } from "node:child_process";
 import { join } from "node:path";
 import { upsertEnv } from "./envfile.js";
@@ -237,6 +237,33 @@ function repairHostArtifacts(cwd, deps) {
     const pruneSource = join(scripts, "docker-prune.sh");
     const pruneTarget = "/etc/cron.daily/kyb-docker-prune";
     const hasDocker = capture("sh", ["-c", "command -v docker >/dev/null && echo yes"])?.trim() === "yes";
+    // The host start script is package-owned too. It carried a 45-second
+    // verdict clock that called a cold sandbox template build a failure, and no
+    // sweep for the orphaned template container that a killed start leaves
+    // behind (KYB-531). An agent still running the copy `kyb init` made keeps
+    // both faults until something refreshes it; this does, on every upgrade.
+    const serverSource = join(scripts, "eve-server.sh");
+    const serverTarget = join(cwd, "scripts/eve-server.sh");
+    if (existsSync(serverSource) && existsSync(serverTarget)) {
+        reconcileHostArtifact({
+            targetPath: serverTarget,
+            desiredContent: readFileSync(serverSource),
+            expectedMode: 0o755,
+            installIfMissing: false,
+            owner: "@kybernesis/exe/scripts/eve-server.sh",
+            update: () => {
+                try {
+                    copyFileSync(serverSource, serverTarget);
+                    chmodSync(serverTarget, 0o755);
+                    return true;
+                }
+                catch {
+                    return false;
+                }
+            },
+            manualCommand: `cp ${shellQuote(serverSource)} scripts/eve-server.sh && chmod +x scripts/eve-server.sh`,
+        });
+    }
     if (existsSync(pruneSource)) {
         const pruneResult = reconcileHostArtifact({
             targetPath: pruneTarget,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kybernesis/create",
-  "version": "0.14.7",
+  "version": "0.14.8",
   "description": "The Kybernesis agent scaffolder and FDE toolkit: one command to a governed, remembering, multiplayer, self-testing eve agent — plus doctor and upgrade.",
   "license": "Apache-2.0",
   "type": "module",