jishushell 0.4.10 → 0.4.17

package/dist/services/instance-manager.js

@@ -1,6 +1,7 @@
 import { execFile, execFileSync } from "child_process";
 import { randomBytes } from "crypto";
-import { chmodSync, chownSync, copyFileSync, cpSync, existsSync, readFileSync, readdirSync, realpathSync, renameSync, rmSync, statSync, } from "fs";
+import { chmodSync, chownSync, closeSync, copyFileSync, cpSync, existsSync, openSync, readFileSync, readdirSync, realpathSync, renameSync, rmSync, statSync, } from "fs";
+import { rm as rmAsync } from "fs/promises";
 import { createServer as netCreateServer } from "net";
 import { userInfo } from "os";
 import { dirname, join, resolve } from "path";
@@ -8,6 +9,7 @@ import { BACKUPS_DIR, INSTANCES_DIR, JISHUSHELL_HOME, getPanelConfig } from "../
 import { LEGACY_PROVIDER_API_ALIASES } from "../constants.js";
 import { safeReadJson, safeWriteJson } from "../utils/safe-json.js";
 import { ensureDirContainer, writeConfigFile, writeSecretFile } from "../utils/fs.js";
+import { compileTaskRuntime } from "./app-compiler.js";
 const _configChangeListeners = [];
 export function onConfigChange(listener) {
     _configChangeListeners.push(listener);
@@ -119,7 +121,16 @@ function safePort(port) {
         throw new Error(`Invalid port: ${port}`);
     return String(port);
 }
-function isPortInUse(port) {
+/**
+ * Probes whether a port is currently held by any process on the host.
+ *
+ * Binds `0.0.0.0:port` — under the Linux default socket semantics, this fails
+ * with EADDRINUSE whenever another listener holds the port on `0.0.0.0`, on
+ * any specific interface (e.g. `127.0.0.1`), or via Docker's published port
+ * map. An IPv6-only listener on `::1` is not detected, but that edge case
+ * never comes up in the Nomad+Docker path this project uses.
+ */
+export function isPortInUse(port) {
     if (!Number.isInteger(port) || port < 1 || port > 65535)
         return Promise.resolve(false);
     return new Promise((resolve) => {
@@ -131,13 +142,25 @@ function isPortInUse(port) {
         server.listen(port, "0.0.0.0");
     });
 }
-async function defaultGatewayPort(instanceId) {
+/**
+ * Picks a gateway port for an instance, starting at {@link DEFAULT_GATEWAY_PORT}
+ * and walking upward until a port is both unknown to jishushell's instance
+ * metadata and reports as free on the host. Concurrent callers coordinate
+ * through `_pendingPorts` so they never hand out the same port.
+ *
+ * Intentionally does not hold a reservation indefinitely — the caller is
+ * expected to either persist the port into instance metadata (which then
+ * shows up in `usedGatewayPorts`) or clear `_pendingPorts` on failure.
+ */
+async function allocateGatewayPort(instanceId) {
     const used = usedGatewayPorts(instanceId);
+    const skipped = [];
     let port = DEFAULT_GATEWAY_PORT;
     while (true) {
         if (port > 65535)
             throw new Error("No available gateway port found (all ports 18789-65535 in use)");
         if (used.has(port) || _pendingPorts.has(port)) {
+            skipped.push(port);
             port++;
             continue;
         }
@@ -147,16 +170,18 @@ async function defaultGatewayPort(instanceId) {
         try {
             if (await isPortInUse(port)) {
                 _pendingPorts.delete(port);
+                skipped.push(port);
                 port++;
                 continue;
             }
-            return port;
+            return { port, skipped };
         }
         catch {
             _pendingPorts.delete(port);
             // Skip this port on a transient OS error rather than failing the entire
             // allocation — a single bad port check should not prevent instance creation.
             console.warn(`[instance] Port ${port} availability check failed, trying next port`);
+            skipped.push(port);
             port++;
             continue;
         }
@@ -243,8 +268,7 @@ function chownToServiceUser(...paths) {
         }
     }
 }
-async function defaultRuntime(instanceId, openclawHome) {
-    const port = await defaultGatewayPort(instanceId);
+function buildDefaultRuntime(instanceId, port, openclawHome) {
     const home = openclawHome || defaultOpenclawHome(instanceId);
     return {
         command: resolveOpenclawBin(),
@@ -755,6 +779,26 @@ function getStockExtensionsDir() {
     return join(JISHUSHELL_HOME, "packages", "openclaw", "lib", "node_modules", "openclaw", "extensions");
 }
 // ── Public API ──
+/**
+ * Probe whether a file is readable by the current process. Used to
+ * distinguish "primary missing / corrupted" (recoverable via safeReadJson's
+ * .bak chain) from "primary exists but permission denied" (EACCES — the
+ * common sudo-script footgun that leaves root-owned files). safeReadJson
+ * swallows every read error internally and returns null, so without this
+ * probe an unreadable primary looks identical to a truly-gone instance.
+ */
+function probeReadable(path) {
+    if (!existsSync(path))
+        return null;
+    try {
+        const fd = openSync(path, "r");
+        closeSync(fd);
+        return null;
+    }
+    catch (e) {
+        return e;
+    }
+}
 export function listInstances() {
     if (!existsSync(INSTANCES_DIR))
         return [];
@@ -764,14 +808,33 @@ export function listInstances() {
         const metaPath = join(INSTANCES_DIR, name, "instance.json");
         const dirPath = join(INSTANCES_DIR, name);
         try {
-            if (statSync(dirPath).isDirectory() && existsSync(metaPath)) {
-                instances.push(JSON.parse(readFileSync(metaPath, "utf-8")));
+            if (!statSync(dirPath).isDirectory())
+                continue;
+            // Use safeReadJson so primary instance.json that was corrupted or
+            // deleted mid-rename falls back to the .bak chain maintained by
+            // safeWriteJson. Without this, an interrupted safeWriteJson call
+            // would silently drop the instance from every list/get hot path
+            // even though the backup chain still holds valid content on disk.
+            const meta = safeReadJson(metaPath, `instance:${name}`);
+            if (meta) {
+                instances.push(meta);
+                continue;
+            }
+            // safeReadJson → null can mean any of (a) primary missing + no
+            // backups, (b) all candidates unparseable, (c) permission denied.
+            // (a) and (b) are legitimate "drop from list" cases; (c) is the
+            // sudo-script footgun and must be logged loudly so the operator
+            // doesn't just see an empty instance list with no hint why.
+            const readErr = probeReadable(metaPath);
+            if (readErr && readErr.code === "EACCES") {
+                console.error(`[instance-manager] cannot read instance '${name}': ${readErr.message}. ` +
+                    `Check file ownership with: ls -la ${metaPath}`);
             }
         }
         catch (e) {
-            // Permission errors (EACCES) caused by root-owned files are a common deployment
-            // mistake (e.g. running a maintenance script as sudo). Log clearly instead of
-            // silently dropping the instance from the list.
+            // Fallback for failures before the safeReadJson call (e.g. statSync
+            // on a directory we can't enter). Still log instead of silently
+            // dropping the entry.
             console.error(`[instance-manager] cannot read instance '${name}': ${e.message}`);
         }
     }
@@ -779,17 +842,26 @@ export function listInstances() {
 }
 export function getInstance(instanceId) {
     const metaPath = instanceMetaPath(instanceId);
-    if (!existsSync(metaPath))
-        return null;
-    try {
-        return JSON.parse(readFileSync(metaPath, "utf-8"));
-    }
-    catch (e) {
-        // Surface permission errors clearly (EACCES: instance.json owned by root after a sudo script)
-        throw new Error(`Cannot read instance '${instanceId}' metadata: ${e.message}. Check file ownership with: ls -la ${metaPath}`);
+    // Go through safeReadJson so primary missing/corrupted instance.json
+    // is still served from the .bak chain. Returning null on "truly gone"
+    // (no primary, no backups) keeps the existing 404 behavior intact.
+    const meta = safeReadJson(metaPath, `instance:${instanceId}`);
+    if (meta)
+        return meta;
+    // safeReadJson swallows every read error internally, which is exactly
+    // wrong for the EACCES case — a root-owned primary would silently
+    // return null and callers would report "Instance not found" instead
+    // of the actionable "check file ownership" message. Re-probe to
+    // distinguish and throw on permission denial. Missing/corrupted with
+    // no backup still returns null (→ 404 upstream).
+    const readErr = probeReadable(metaPath);
+    if (readErr && readErr.code === "EACCES") {
+        throw new Error(`Cannot read instance '${instanceId}' metadata: ${readErr.message}. ` +
+            `Check file ownership with: ls -la ${metaPath}`);
     }
+    return null;
 }
-export async function createInstance(instanceId, name, description = "", cloneFrom, openclawHome, cloneOptions) {
+export async function createInstance(instanceId, name, description = "", cloneFrom, openclawHome, appSpec, cloneOptions) {
     const d = instanceDir(instanceId);
     if (existsSync(d))
         throw new Error(`Instance '${instanceId}' already exists`);
@@ -837,9 +909,18 @@ export async function createInstance(instanceId, name, description = "", cloneFr
     catch { /* non-root without CAP_CHOWN — already correct owner */ }
     ensureDirContainer(home);
     ensureDirContainer(join(home, OPENCLAW_STATE_DIRNAME));
-    const runtime = await defaultRuntime(instanceId, home);
+    const portAlloc = await allocateGatewayPort(instanceId);
+    const baseRuntime = buildDefaultRuntime(instanceId, portAlloc.port, home);
+    let runtime = baseRuntime;
+    if (appSpec) {
+        const serviceTask = appSpec.tasks.find((t) => t.role === "service");
+        if (serviceTask) {
+            const compiled = compileTaskRuntime(serviceTask, instanceId);
+            runtime = { ...baseRuntime, ...compiled };
+        }
+    }
     const allocatedPort = extractGatewayPort(runtime);
-    // Port already reserved inside defaultGatewayPort; just track for cleanup
+    // Port already reserved inside allocateGatewayPort; just track for cleanup
     try {
         const meta = {
             id: instanceId,
@@ -848,6 +929,7 @@ export async function createInstance(instanceId, name, description = "", cloneFr
             openclaw_home: home,
             runtime,
             created_at: new Date().toISOString(),
+            ...(appSpec ? { app_id: appSpec.id } : {}),
         };
         safeWriteJson(instanceMetaPath(instanceId), meta);
         const envFiles = (runtime.env_files || []).map((p) => normalizePath(p));
@@ -959,6 +1041,34 @@ export async function createInstance(instanceId, name, description = "", cloneFr
                 updateEnvFile(providerEnv, { UPSTREAM_API_KEY: dp.apiKey });
             }
         }
+        // Merge App-level config_defaults into openclaw.json (shallow merge, app values win)
+        if (appSpec?.openclaw?.config_defaults && existsSync(configPath)) {
+            try {
+                const existing = JSON.parse(readFileSync(configPath, "utf-8"));
+                const defaults = appSpec.openclaw.config_defaults;
+                // Deep merge top-level keys
+                for (const [key, value] of Object.entries(defaults)) {
+                    if (typeof value === "object" && value !== null && !Array.isArray(value) && typeof existing[key] === "object" && existing[key] !== null) {
+                        existing[key] = { ...existing[key], ...value };
+                    }
+                    else {
+                        existing[key] = value;
+                    }
+                }
+                writeConfigFile(configPath, JSON.stringify(existing, null, 2));
+            }
+            catch { /* ignore merge errors, keep existing config */ }
+        }
+        // Record App-level skills for later installation into the instance
+        if (appSpec?.openclaw?.skills && Array.isArray(appSpec.openclaw.skills)) {
+            try {
+                const skillsDir = join(dirname(configPath), "skills");
+                ensureDirContainer(skillsDir);
+                const skillMeta = join(skillsDir, ".app-skills.json");
+                safeWriteJson(skillMeta, { app_id: appSpec.id, skills: appSpec.openclaw.skills });
+            }
+            catch { /* ignore */ }
+        }
         // Copy cloned provider.env BEFORE proxy bootstrap so bootstrap can find the API key
         if (cloneFrom && envFiles.length) {
             const srcEnvFiles = getRuntimeEnvFiles(cloneFrom);
@@ -997,6 +1107,17 @@ export async function createInstance(instanceId, name, description = "", cloneFr
                 console.warn(`[instance] chown for ${instanceId} failed:`, e.message);
             }
         }
+        // Attach transient port allocation info for the API response only — never
+        // persisted in instance.json. If the caller (e.g. the create route) sees
+        // skipped ports it can tell the user the default was busy.
+        if (portAlloc.skipped.length > 0) {
+            meta.port_allocation = {
+                assigned: portAlloc.port,
+                requested: DEFAULT_GATEWAY_PORT,
+                reason: "default_busy",
+                skipped: portAlloc.skipped,
+            };
+        }
         return meta;
     }
     finally {
@@ -1023,7 +1144,7 @@ export function updateInstanceMeta(instanceId, patch) {
     Object.assign(meta, patch);
     safeWriteJson(metaPath, meta);
 }
-export function deleteInstance(instanceId, purgeBackups = false) {
+export async function deleteInstance(instanceId, purgeBackups = false) {
     const d = instanceDir(instanceId);
     if (!existsSync(d))
         return { ok: false, warnings: ["Instance directory not found"] };
@@ -1049,14 +1170,18 @@ export function deleteInstance(instanceId, purgeBackups = false) {
     }).catch((e) => {
         console.warn(`[instance] Could not load nomad-manager for cleanup:`, e.message);
     });
+    // Async rm so the Node event loop stays responsive during large deletes:
+    // a fresh instance with a just-installed openclaw package can be 1+ GB
+    // with hundreds of nested dirs, which takes 30-60s to unlink on SD storage.
+    // rmSync would block every other HTTP request for that whole window.
     let dirDeleted = false;
     try {
-        rmSync(d, { recursive: true, force: true });
+        await rmAsync(d, { recursive: true, force: true });
         dirDeleted = true;
     }
     catch {
         try {
-            execFileSync("sudo", ["rm", "-rf", d], { timeout: 10000 });
+            execFileSync("sudo", ["rm", "-rf", d], { timeout: 300000 });
             dirDeleted = true;
         }
         catch (e) {
@@ -1067,11 +1192,14 @@ export function deleteInstance(instanceId, purgeBackups = false) {
     if (home && !home.startsWith(d) && existsSync(home)) {
         warnings.push(`Custom openclaw_home '${home}' was preserved. Delete manually if no longer needed.`);
     }
-    // Handle backups (stored in separate directory, not affected by instance rmSync)
+    // Handle backups (stored in separate directory, not affected by the
+    // instance rm above). Backups can be hundreds of MB each and accumulate
+    // across retention windows, so use the same async rm path to keep the
+    // event loop responsive.
     const backupDir = join(BACKUPS_DIR, instanceId);
     if (purgeBackups && existsSync(backupDir)) {
         try {
-            rmSync(backupDir, { recursive: true, force: true });
+            await rmAsync(backupDir, { recursive: true, force: true });
         }
         catch (e) {
             warnings.push(`Failed to delete backups: ${e.message}`);
@@ -1416,11 +1544,30 @@ export async function getGatewayHost(instanceId) {
                     const detail = await fetch(`${getNomadAddr()}/v1/allocation/${encodeURIComponent(alloc.ID)}`, { headers, signal: AbortSignal.timeout(5000) });
                     if (detail.ok) {
                         const d = await detail.json();
-                        const ports = d?.AllocatedResources?.Shared?.Ports ?? [];
-                        const gwPort = ports.find((p) => p.Label === "gateway");
+                        // Preferred source: AllocatedResources.Shared.Ports (bridge mode).
+                        const sharedPorts = d?.AllocatedResources?.Shared?.Ports ?? [];
+                        const gwPort = sharedPorts.find((p) => p.Label === "gateway");
                         if (gwPort?.HostIP && gwPort.HostIP !== "0.0.0.0") {
                             result = gwPort.HostIP;
                         }
+                        else {
+                            // Host mode / task-level reservation: address lives under
+                            // AllocatedResources.Tasks.<task>.Networks[*].IP. On Nomad
+                            // 1.6.5 with `network_interface = "lo"`, the IP is whichever
+                            // address the OS enumerates first — which can be IPv6 `::1`
+                            // on systems where lo has both `127.0.0.1/8` and `::1/128`
+                            // (the default on most modern Linux distros). Reading it from
+                            // here is the authoritative source and matches what nomad
+                            // configures the docker-proxy bind to use.
+                            const taskNets = d?.AllocatedResources?.Tasks?.gateway?.Networks ?? [];
+                            const net = taskNets.find((n) => {
+                                const rps = n?.ReservedPorts ?? [];
+                                return rps.some((p) => p.Label === "gateway");
+                            });
+                            if (net?.IP && net.IP !== "0.0.0.0") {
+                                result = net.IP;
+                            }
+                        }
                     }
                 }
             }
@@ -1436,7 +1583,11 @@ export async function getGatewayHost(instanceId) {
     try {
         const out = execFileSync("ss", ["-tlnH", "sport", "=", ":" + safePort(port)], { encoding: "utf-8", timeout: 3000, stdio: ["pipe", "pipe", "pipe"] });
         for (const line of out.split("\n")) {
-            const match = line.match(/\s([\d.]+):(\d+)\s/);
+            // IPv4 dotted-quad: "... 127.0.0.1:18789 ..."
+            // IPv6 bracketed:   "... [::1]:18789 ..."
+            let match = line.match(/\s([\d.]+):(\d+)\s/);
+            if (!match)
+                match = line.match(/\s\[([0-9a-fA-F:]+)\]:(\d+)\s/);
             if (match && match[2] === String(port)) {
                 const addr = match[1];
                 result = addr === "0.0.0.0" ? "127.0.0.1" : addr;
@@ -1448,6 +1599,16 @@ export async function getGatewayHost(instanceId) {
     _gwHostCache.set(instanceId, { host: result, ts: Date.now() });
     return result;
 }
+/**
+ * Wrap an IPv6 literal in brackets for safe URL host-component / Host-header
+ * use. Bare names ("gateway.local") and IPv4 ("127.0.0.1") contain no colon
+ * and pass through unchanged; anything with a colon is an IPv6 literal and
+ * MUST be bracketed before being concatenated with a port, otherwise
+ * `http://::1:18789/` is unparseable.
+ */
+export function urlHost(host) {
+    return host.includes(":") ? `[${host}]` : host;
+}
 export function findInstancesSharingOpenclawHome(instanceId) {
     const targetHome = normalizePath(getOpenclawHome(instanceId));
     return listInstances()
@@ -1455,6 +1616,48 @@ export function findInstancesSharingOpenclawHome(instanceId) {
         .filter((inst) => normalizePath(inst.openclaw_home || defaultOpenclawHome(inst.id)) === targetHome)
         .map((inst) => inst.id);
 }
+/**
+ * Re-pick a gateway port for an existing instance and rewrite its persisted
+ * runtime metadata (`runtime.args` and `runtime.env.OPENCLAW_GATEWAY_PORT`).
+ *
+ * Used when {@link isPortInUse} reports that the previously-assigned port has
+ * been taken by something else between create-time and start-time (e.g. a
+ * host-side openclaw started by the user, an unrelated service that grabbed
+ * the port at boot, or a Docker race on the next allocation). The Nomad job
+ * spec is rebuilt from instance metadata on every submit, so updating
+ * `instance.json` here is sufficient — no other files need patching.
+ */
+export async function reallocateGatewayPort(instanceId) {
+    const meta = safeReadJson(instanceMetaPath(instanceId), "instance-meta");
+    if (!meta)
+        throw new Error(`Cannot reallocate port for unknown instance '${instanceId}'`);
+    const fromPort = extractGatewayPort(meta.runtime) ?? DEFAULT_GATEWAY_PORT;
+    const alloc = await allocateGatewayPort(instanceId);
+    try {
+        const runtime = (meta.runtime ?? {});
+        const args = Array.isArray(runtime.args) ? [...runtime.args] : [];
+        for (let i = 0; i < args.length; i++) {
+            if (args[i] === "--port" && i + 1 < args.length) {
+                args[i + 1] = String(alloc.port);
+            }
+            else if (typeof args[i] === "string" && args[i].startsWith("--port=")) {
+                args[i] = `--port=${alloc.port}`;
+            }
+        }
+        runtime.args = args;
+        const env = (runtime.env ?? {});
+        env.OPENCLAW_GATEWAY_PORT = String(alloc.port);
+        runtime.env = env;
+        meta.runtime = runtime;
+        safeWriteJson(instanceMetaPath(instanceId), meta);
+        chownToServiceUser(instanceMetaPath(instanceId));
+        console.log(`[instance] ${instanceId}: gateway port reallocated ${fromPort} -> ${alloc.port}`);
+        return { from: fromPort, to: alloc.port, skipped: alloc.skipped };
+    }
+    finally {
+        _pendingPorts.delete(alloc.port);
+    }
+}
 export function findInstancesSharingGatewayPort(instanceId) {
     const targetPort = getGatewayPort(instanceId);
     return listInstances()