jishushell 0.5.15 → 0.5.22

package/dist/services/nomad-manager.js

@@ -20,7 +20,7 @@ import { StringDecoder } from "string_decoder";
 import { promisify } from "util";
 import { parse } from "yaml";
 import * as config from "../config.js";
-import { getGatewayPort, getInstance, getInstanceRuntime, instanceMetaPath, getRuntimeEnv, isPortInUse, reallocateGatewayPort, } from "./instance-manager.js";
+import { extractGatewayPort, getGatewayPort, getInstance, getInstanceRuntime, instanceMetaPath, getRuntimeEnv, isPortInUse, reallocateGatewayPort, } from "./instance-manager.js";
 import { getAdapter, resolveAgentType } from "./runtime/index.js";
 function getConfigValue(name) {
     return name in config ? config[name] : undefined;
@@ -540,24 +540,35 @@ async function injectConnectionsRuntimeEnv(instanceId, task) {
     }
 }
 async function getRunningAlloc(instanceId) {
+    const allocs = await getAllocs(instanceId);
+    if (!allocs)
+        return null;
+    for (const status of ["running", "pending"]) {
+        for (const alloc of allocs) {
+            if (alloc.ClientStatus === status)
+                return alloc;
+        }
+    }
+    return null;
+}
+async function getAllocs(instanceId) {
     const jid = jobId(instanceId);
     try {
         const resp = await nomadGet(`/v1/job/${jid}/allocations`);
         if (resp.status === 404)
-            return null;
+            return [];
         const allocs = await resp.json();
-        for (const status of ["running", "pending"]) {
-            for (const alloc of allocs) {
-                if (alloc.ClientStatus === status)
-                    return alloc;
-            }
-        }
-        return null;
+        return Array.isArray(allocs) ? allocs : [];
     }
     catch {
         return null;
     }
 }
+function latestAlloc(allocs) {
+    if (!allocs.length)
+        return null;
+    return [...allocs].sort((a, b) => ((b.ModifyIndex ?? b.CreateIndex ?? 0) - (a.ModifyIndex ?? a.CreateIndex ?? 0)))[0] ?? null;
+}
 // Returns true if the Nomad job exists and was NOT explicitly stopped by the user (Stop=false).
 // Used on jishushell startup to auto-restart instances that were running before a reboot.
 export async function shouldAutoStart(instanceId) {
@@ -604,7 +615,12 @@ export async function getStatus(instanceId) {
     catch {
         return { ...stopped, status: "unknown", error: "Nomad unreachable" };
     }
-    const alloc = await getRunningAlloc(instanceId);
+    const allocs = await getAllocs(instanceId);
+    if (allocs == null || allocs.length === 0)
+        return { ...stopped, status: "pending" };
+    const alloc = allocs.find((entry) => entry.ClientStatus === "running")
+        ?? allocs.find((entry) => entry.ClientStatus === "pending")
+        ?? latestAlloc(allocs);
     if (!alloc)
         return { ...stopped, status: "pending" };
     const allocId = alloc.ID;
@@ -642,32 +658,19 @@ export async function getStatus(instanceId) {
         }
     }
     catch { /* ignore */ }
-    // Fallback: Nomad cgroup stats are often zero on cgroup v2 (e.g. Raspberry Pi).
-    // Use `docker stats` directly when Nomad reports 0.
-    if (!result.memory_mb && allocId) {
-        try {
-            // Validate allocId to prevent shell injection (Nomad UUIDs are hex + hyphens)
-            if (!/^[a-f0-9-]+$/i.test(allocId))
-                throw new Error("invalid allocId");
-            const containerName = `${resolveTaskName(instanceId)}-${allocId}`;
-            const { execFile } = await import("child_process");
-            const { promisify } = await import("util");
-            const execFileAsync = promisify(execFile);
-            const { stdout } = await execFileAsync("docker", ["stats", "--no-stream", "--format", "{{.MemUsage}}", containerName], { timeout: 5000 });
-            const raw = stdout.trim();
-            // Format: "499.6MiB / 3GiB" or "123.4MB / 2GB"
-            const match = raw.match(/^([\d.]+)\s*(MiB|GiB|MB|GB|KiB|KB)/i);
-            if (match) {
-                let mb = parseFloat(match[1]);
-                const unit = match[2].toLowerCase();
-                if (unit === "gib" || unit === "gb")
-                    mb *= 1024;
-                else if (unit === "kib" || unit === "kb")
-                    mb /= 1024;
-                result.memory_mb = Math.round(mb * 10) / 10;
-            }
+    // Fallback: Nomad cgroup stats are often zero on cgroup v2 (e.g. Raspberry
+    // Pi / CIX). Read from the shared, cached, single-flight `docker stats`
+    // snapshot instead of forking one `docker stats` per instance — see
+    // getDockerMemSnapshot for why per-instance forking was the cold-path cost.
+    if (!result.memory_mb && allocId && /^[a-f0-9-]+$/i.test(allocId)) {
+        const containerName = `${resolveTaskName(instanceId)}-${allocId}`;
+        const stat = (await getDockerMemSnapshot()).get(containerName);
+        if (stat) {
+            if (stat.memory_mb)
+                result.memory_mb = stat.memory_mb;
+            if (!result.cpu_percent && stat.cpu_percent)
+                result.cpu_percent = stat.cpu_percent;
         }
-        catch { /* ignore */ }
     }
     return result;
 }
@@ -679,6 +682,20 @@ async function phaseRunningCheck(instanceId) {
     }
     return { ok: true };
 }
+async function phaseResetTerminalJobBeforeStart(instanceId) {
+    const status = await getStatus(instanceId);
+    if (!["failed", "dead", "complete"].includes(String(status.status)))
+        return;
+    try {
+        const resp = await nomadDelete(`/v1/job/${jobId(instanceId)}?purge=false`);
+        if (!resp.ok && resp.status !== 404) {
+            console.warn(`[nomad] ${instanceId}: failed to stop terminal job before start (HTTP ${resp.status}): ${await resp.text()}`);
+        }
+    }
+    catch (e) {
+        console.warn(`[nomad] ${instanceId}: failed to stop terminal job before start: ${e?.message ?? e}`);
+    }
+}
 /**
  * Phase 2: home-conflict check — dispatched through the adapter so
  * framework code carries no agentType-specific knowledge. Adapters that
@@ -890,6 +907,7 @@ export async function startInstance(instanceId) {
             extra.code = running.code;
         return failed("running_check", extra);
     }
+    await phaseResetTerminalJobBeforeStart(instanceId);
     const legacyManager = await getLegacyAppManager(instanceId);
     if (legacyManager) {
         const prep = await legacyManager.prepareStart(instanceId);
@@ -1127,6 +1145,82 @@ export async function getLogs(instanceId, lines = 200, logType = "stderr") {
     return [];
 }
 const execFileAsync = promisify(execFileCb);
+const DOCKER_STATS_TTL_MS = 30_000;
+/** Field separator for the batched `docker stats --format` line. Exported so
+ *  tests can construct mock output without hardcoding the literal. */
+export const DOCKER_STATS_FIELD_SEP = "__JS__";
+let _dockerStatsEntry = null;
+let _dockerStatsInFlight = null;
+/** Test-only: reset the shared docker-stats snapshot so each test starts from
+ *  a cold cache (the 30s TTL + single-flight would otherwise leak one test's
+ *  mocked snapshot into the next). Not used by production code paths. */
+export function __resetDockerStatsCacheForTests() {
+    _dockerStatsEntry = null;
+    _dockerStatsInFlight = null;
+}
+function parseDockerMemUsageMb(memUsage) {
+    // Format: "499.6MiB / 3GiB" — the used side is everything before "/".
+    const used = (memUsage.split("/")[0] ?? "").trim();
+    const match = used.match(/^([\d.]+)\s*(MiB|GiB|MB|GB|KiB|KB|B)?/i);
+    if (!match)
+        return 0;
+    let mb = parseFloat(match[1]);
+    if (!Number.isFinite(mb))
+        return 0;
+    const unit = (match[2] ?? "MiB").toLowerCase();
+    if (unit === "gib" || unit === "gb")
+        mb *= 1024;
+    else if (unit === "kib" || unit === "kb")
+        mb /= 1024;
+    else if (unit === "b")
+        mb /= 1024 * 1024;
+    return Math.round(mb * 10) / 10;
+}
+async function loadDockerStatsSnapshot() {
+    const snapshot = new Map();
+    try {
+        const fmt = `{{.Name}}${DOCKER_STATS_FIELD_SEP}{{.MemUsage}}${DOCKER_STATS_FIELD_SEP}{{.CPUPerc}}`;
+        const { stdout } = await execFileAsync("docker", ["stats", "--no-stream", "--format", fmt], { timeout: 8_000 });
+        for (const line of stdout.split("\n")) {
+            const trimmed = line.trim();
+            if (!trimmed)
+                continue;
+            const [name, memUsage, cpuPerc] = trimmed.split(DOCKER_STATS_FIELD_SEP);
+            if (!name)
+                continue;
+            snapshot.set(name, {
+                memory_mb: parseDockerMemUsageMb(memUsage ?? ""),
+                cpu_percent: Math.round((parseFloat(cpuPerc ?? "") || 0) * 10) / 10,
+            });
+        }
+    }
+    catch {
+        /* docker missing / timeout / daemon down → empty map, caller degrades */
+    }
+    return snapshot;
+}
+/**
+ * Returns a per-container stats map, refreshing at most once per
+ * DOCKER_STATS_TTL_MS. Concurrent callers (the `Promise.all` over every
+ * instance in `GET /api/instances`) share a single in-flight docker call.
+ */
+async function getDockerMemSnapshot() {
+    const now = Date.now();
+    if (_dockerStatsEntry && now - _dockerStatsEntry.ts < DOCKER_STATS_TTL_MS) {
+        return _dockerStatsEntry.data;
+    }
+    if (_dockerStatsInFlight)
+        return _dockerStatsInFlight;
+    _dockerStatsInFlight = loadDockerStatsSnapshot()
+        .then((data) => {
+        _dockerStatsEntry = { data, ts: Date.now() };
+        return data;
+    })
+        .finally(() => {
+        _dockerStatsInFlight = null;
+    });
+    return _dockerStatsInFlight;
+}
 export async function exec(instanceId, command, timeoutMs = 120_000) {
     const alloc = await getRunningAlloc(instanceId);
     if (!alloc || alloc.ClientStatus !== "running") {
@@ -1509,6 +1603,96 @@ var UnifiedNomadJobs;
                 : {}),
         }));
     }
+    function isExternalAppTaskPort(port) {
+        return (port.visibility ?? "external") !== "internal";
+    }
+    function readDeclaredHostPort(port) {
+        const candidate = port.host_port ?? port.port;
+        return Number.isInteger(candidate) && candidate > 0 ? candidate : null;
+    }
+    function applyPersistedAppSpecPortOverrides(appId, spec) {
+        const meta = getInstance(appId);
+        if (!meta)
+            return spec;
+        const runtime = getInstanceRuntime(appId);
+        const runtimePorts = Array.isArray(runtime.ports) ? runtime.ports : [];
+        const persistedGatewayPort = extractGatewayPort(runtime, resolveAgentType(meta));
+        const totalExternalPorts = spec.tasks.reduce((count, task) => count + (task.ports ?? []).filter((port) => isExternalAppTaskPort(port)).length, 0);
+        let changed = false;
+        const tasks = spec.tasks.map((task) => {
+            if (!Array.isArray(task.ports) || task.ports.length === 0)
+                return task;
+            let taskChanged = false;
+            const ports = task.ports.map((port) => {
+                if (!isExternalAppTaskPort(port))
+                    return port;
+                const currentHostPort = readDeclaredHostPort(port);
+                if (!currentHostPort)
+                    return port;
+                let nextHostPort = null;
+                const namedRuntimePort = typeof port.name === "string" && port.name
+                    ? runtimePorts.find((candidate) => candidate?.name === port.name
+                        && Number.isInteger(candidate?.hostPort)
+                        && candidate.hostPort > 0)
+                    : null;
+                if (namedRuntimePort) {
+                    nextHostPort = namedRuntimePort.hostPort;
+                }
+                else if (runtimePorts.length === 1
+                    && totalExternalPorts === 1
+                    && Number.isInteger(runtimePorts[0]?.hostPort)
+                    && runtimePorts[0].hostPort > 0) {
+                    nextHostPort = runtimePorts[0].hostPort;
+                }
+                else if (totalExternalPorts === 1
+                    && persistedGatewayPort != null
+                    && persistedGatewayPort > 0) {
+                    nextHostPort = persistedGatewayPort;
+                }
+                if (!nextHostPort || nextHostPort === currentHostPort)
+                    return port;
+                changed = true;
+                taskChanged = true;
+                return { ...port, host_port: nextHostPort };
+            });
+            return taskChanged ? { ...task, ports } : task;
+        });
+        return changed ? { ...spec, tasks } : spec;
+    }
+    async function maybeReallocateAppSpecHostPort(appId, spec, reason) {
+        if (!getInstance(appId))
+            return { spec, changed: false };
+        const effectiveSpec = applyPersistedAppSpecPortOverrides(appId, spec);
+        const currentGatewayPort = getGatewayPort(appId);
+        if (!Number.isInteger(currentGatewayPort) || currentGatewayPort <= 0) {
+            return { spec: effectiveSpec, changed: false };
+        }
+        const declaredPorts = effectiveSpec.tasks.flatMap((task) => (task.ports ?? [])
+            .filter((port) => isExternalAppTaskPort(port))
+            .map((port) => readDeclaredHostPort(port))
+            .filter((port) => port != null));
+        if (!declaredPorts.includes(currentGatewayPort)) {
+            return { spec: effectiveSpec, changed: false };
+        }
+        if (!(await isPortInUse(currentGatewayPort))) {
+            return { spec: effectiveSpec, changed: false };
+        }
+        try {
+            const reallocation = await reallocateGatewayPort(appId);
+            console.log(`[nomad] ${appId}: reallocated AppSpec host port ${reallocation.from} -> ${reallocation.to} (${reason})`);
+            return {
+                spec: applyPersistedAppSpecPortOverrides(appId, spec),
+                changed: true,
+            };
+        }
+        catch (e) {
+            return {
+                spec: effectiveSpec,
+                changed: false,
+                error: `AppSpec host port ${currentGatewayPort} is held by another process and reallocation failed: ${e?.message ?? e}`,
+            };
+        }
+    }
     // ── Health check → Nomad service check builder ────────────────────────────
     function buildServiceCheck(task, appId) {
         const health = task.health;
@@ -2648,28 +2832,17 @@ var UnifiedNomadJobs;
             }
         }
         catch { /* ignore */ }
-        // Fallback: use `docker stats` when Nomad cgroup stats are zero (cgroup v2 / Pi).
-        // Only applicable for docker-driver tasks.
-        if (!result.memory_mb && allocId && ptName) {
-            try {
-                if (!/^[a-f0-9-]+$/i.test(allocId))
-                    throw new Error("invalid allocId");
-                const containerName = `${ptName}-${allocId}`;
-                const execFileAsync = promisify(execFileCb);
-                const { stdout } = await execFileAsync("docker", ["stats", "--no-stream", "--format", "{{.MemUsage}}", containerName], { timeout: 5_000 });
-                const raw = stdout.trim();
-                const match = raw.match(/^([\d.]+)\s*(MiB|GiB|MB|GB|KiB|KB)/i);
-                if (match) {
-                    let mb = parseFloat(match[1]);
-                    const unit = match[2].toLowerCase();
-                    if (unit === "gib" || unit === "gb")
-                        mb *= 1024;
-                    else if (unit === "kib" || unit === "kb")
-                        mb /= 1024;
-                    result.memory_mb = Math.round(mb * 10) / 10;
-                }
+        // Fallback: cgroup v2 (Pi / CIX) → Nomad alloc-stats are zero. Use the
+        // shared cached `docker stats` snapshot rather than forking per-instance.
+        if (!result.memory_mb && allocId && ptName && /^[a-f0-9-]+$/i.test(allocId)) {
+            const containerName = `${ptName}-${allocId}`;
+            const stat = (await getDockerMemSnapshot()).get(containerName);
+            if (stat) {
+                if (stat.memory_mb)
+                    result.memory_mb = stat.memory_mb;
+                if (!result.cpu_percent && stat.cpu_percent)
+                    result.cpu_percent = stat.cpu_percent;
             }
-            catch { /* ignore */ }
         }
         return result;
     }
@@ -2757,7 +2930,7 @@ var UnifiedNomadJobs;
         if (adoptedExternal.conflicts.length > 0) {
             return { ok: false, error: adoptedExternal.conflicts.join("; ") };
         }
-        const effectiveSpec = adoptedExternal.spec;
+        let effectiveSpec = applyPersistedAppSpecPortOverrides(appId, adoptedExternal.spec);
         // Validate all images before submitting
         for (const task of effectiveSpec.tasks) {
             if (task.runtime === "container") {
@@ -2784,31 +2957,54 @@ var UnifiedNomadJobs;
         if (hostNetworkError) {
             return { ok: false, error: hostNetworkError };
         }
-        let jobDef;
-        try {
-            jobDef = buildAppJob(effectiveSpec, appId, driver, extraEnv);
-        }
-        catch (e) {
-            return { ok: false, error: `Job build failed: ${e.message}` };
+        if (driver === "docker") {
+            const preflight = await maybeReallocateAppSpecHostPort(appId, effectiveSpec, "host_port_busy");
+            if (preflight.error)
+                return { ok: false, error: preflight.error };
+            effectiveSpec = preflight.spec;
         }
-        try {
-            const resp = await nomadPost("/v1/jobs", jobDef);
-            if (resp.ok) {
-                const data = await resp.json();
-                return { ok: true, eval_id: data.EvalID };
+        for (let attempt = 0; attempt < 2; attempt++) {
+            let jobDef;
+            try {
+                jobDef = buildAppJob(effectiveSpec, appId, driver, extraEnv);
             }
-            const text = await resp.text();
-            return { ok: false, error: text };
-        }
-        catch (e) {
-            const isNetErr = e?.message === "fetch failed" || e?.cause?.code === "ECONNREFUSED";
-            return {
-                ok: false,
-                error: isNetErr
-                    ? `Nomad 服务不可达 (${getNomadAddr()})，请先启动 Nomad`
-                    : e.message,
-            };
+            catch (e) {
+                return { ok: false, error: `Job build failed: ${e.message}` };
+            }
+            let submitError = null;
+            let netErr = false;
+            try {
+                const resp = await nomadPost("/v1/jobs", jobDef);
+                if (resp.ok) {
+                    const data = await resp.json();
+                    // When the app was previously failed, verify it actually transitions
+                    // away from the failed state rather than reporting false success.
+                    if (status.status === "failed") {
+                        const recovered = await waitForRecovery(appId, 15_000, 2_000);
+                        if (!recovered) {
+                            return { ok: false, error: "App start submitted but instance remains in failed state. Check app logs for details.", eval_id: data.EvalID };
+                        }
+                    }
+                    return { ok: true, eval_id: data.EvalID };
+                }
+                submitError = await resp.text();
+            }
+            catch (e) {
+                netErr = e?.message === "fetch failed" || e?.cause?.code === "ECONNREFUSED";
+                submitError = netErr ? `Nomad 服务不可达 (${getNomadAddr()})，请先启动 Nomad` : e.message;
+            }
+            if (attempt === 0 && driver === "docker" && !netErr) {
+                const retry = await maybeReallocateAppSpecHostPort(appId, effectiveSpec, "docker_race");
+                if (retry.error)
+                    return { ok: false, error: retry.error };
+                if (retry.changed) {
+                    effectiveSpec = retry.spec;
+                    continue;
+                }
+            }
+            return { ok: false, error: submitError ?? "unknown error" };
         }
+        return { ok: false, error: "start retry exhausted" };
     }
     UnifiedNomadJobs.startAppJob = startAppJob;
     /**
@@ -2828,6 +3024,21 @@ var UnifiedNomadJobs;
         return false;
     }
     UnifiedNomadJobs.waitForRunning = waitForRunning;
+    /**
+     * Poll until the app job leaves the "failed" state or times out.
+     * Used after start submission to verify actual recovery before reporting success.
+     * Returns true if the app transitions away from "failed" (to pending/running/etc).
+     */
+    async function waitForRecovery(appId, timeoutMs = 15_000, pollIntervalMs = 2_000) {
+        const deadline = Date.now() + timeoutMs;
+        while (Date.now() < deadline) {
+            await new Promise((r) => setTimeout(r, pollIntervalMs));
+            const status = await getAppStatus(appId);
+            if (status.status !== "failed")
+                return true;
+        }
+        return false;
+    }
     async function checkDependencies(spec) {
         if (!spec.depends_on || Object.keys(spec.depends_on).length === 0) {
             return { ok: true, errors: [] };