jishushell 0.0.1 → 0.4.2

package/dist/services/nomad-manager.js

@@ -0,0 +1,828 @@
+/**
+ * Nomad-based service manager for OpenClaw instances.
+ * Communicates with Nomad via its HTTP API.
+ */
+import { execFile as execFileCb, execFileSync } from "child_process";
+import { chmodSync, existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "fs";
+import { homedir, platform, userInfo } from "os";
+import { dirname } from "path";
+import { join } from "path";
+import { promisify } from "util";
+import { getNomadAddr, getNomadDriver, getNomadToken, getOpenclawDockerImage, isOfficialImage, JISHUSHELL_HOME } from "../config.js";
+import { TtlCache } from "../utils/ttl-cache.js";
+import { findInstancesSharingGatewayPort, findInstancesSharingOpenclawHome, getGatewayPort, getInstanceRuntime, getOpenclawConfigPath, getOpenclawHome, getRuntimeEnv } from "./instance-manager.js";
+import { getLegacyStatus, stopInstance as stopLegacyInstance } from "./process-manager.js";
+// Docker image names must match this pattern to prevent command injection.
+export const DOCKER_IMAGE_RE = /^[a-zA-Z0-9][a-zA-Z0-9\-_.:/@]*$/;
+// Maximum allowed length for a Docker image reference.
+export const MAX_DOCKER_IMAGE_NAME_LEN = 256;
+const JOB_PREFIX = "openclaw-";
+// Tracks the panel's listening port so bridge-mode containers can reach it via host.docker.internal.
+let _panelPort = 8090;
+export function setPanelPort(port) { _panelPort = port; }
+/**
+ * When running in docker bridge mode, 127.0.0.1 inside the container resolves to
+ * the container's own loopback, not the host. Rewrite the jsproxy provider baseUrl
+ * in openclaw.json to use host.docker.internal instead so the container can reach
+ * the JishuShell LLM proxy.
+ */
+function patchJsproxyBaseUrl(configPath) {
+    try {
+        const raw = readFileSync(configPath, "utf-8");
+        const patched = raw.replace(/http:\/\/127\.0\.0\.1:(\d+)\/proxy/g, `http://host.docker.internal:$1/proxy`);
+        if (patched !== raw) {
+            writeFileSync(configPath, patched, "utf-8");
+            console.log(`[nomad] Patched jsproxy baseUrl in ${configPath} (127.0.0.1 → host.docker.internal)`);
+        }
+    }
+    catch (e) {
+        console.warn(`[nomad] Failed to patch jsproxy baseUrl in ${configPath}: ${e.message}`);
+    }
+}
+/**
+ * Docker bridge port publishing cannot reach a process that only binds the
+ * container loopback. Normalize default/loopback gateway binds to `lan` so
+ * Nomad's published host port can reach the gateway.
+ *
+ * OpenClaw will seed localhost Control UI origins automatically for non-loopback
+ * binds on startup when they are missing, so persisting the bind mode here keeps
+ * startup and runtime behavior aligned.
+ */
+function patchDockerBridgeGatewayBind(configPath) {
+    try {
+        const raw = readFileSync(configPath, "utf-8");
+        const parsed = JSON.parse(raw);
+        if (!parsed || typeof parsed !== "object" || Array.isArray(parsed))
+            return;
+        const gatewayRaw = parsed.gateway;
+        const gateway = gatewayRaw && typeof gatewayRaw === "object" && !Array.isArray(gatewayRaw)
+            ? gatewayRaw
+            : (parsed.gateway = {});
+        const bind = typeof gateway.bind === "string" ? gateway.bind.trim() : "";
+        if (bind && bind !== "loopback")
+            return;
+        gateway.bind = "lan";
+        const next = JSON.stringify(parsed, null, 2);
+        const output = raw.endsWith("\n") ? `${next}\n` : next;
+        if (output === raw)
+            return;
+        writeFileSync(configPath, output, "utf-8");
+        console.log(`[nomad] Normalized gateway.bind to "lan" in ${configPath} for Docker bridge networking`);
+    }
+    catch (e) {
+        console.warn(`[nomad] Failed to patch gateway.bind in ${configPath}: ${e.message}`);
+    }
+}
+const DEFAULT_COMMAND = "/usr/bin/openclaw";
+const DEFAULT_PIDS_LIMIT = 512;
+export const VALID_LOG_TYPES = new Set(["stdout", "stderr"]);
+const memoryOversubscriptionCache = new TtlCache(30_000);
+function nomadAuthHeaders() {
+    const token = getNomadToken();
+    return token ? { "X-Nomad-Token": token } : {};
+}
+const DEFAULT_ARGS = ["gateway", "run", "--port", "18789", "--allow-unconfigured"];
+const DEFAULT_USER = userInfo().username;
+const DEFAULT_CWD = homedir();
+const DEFAULT_ENV = {
+    HOME: homedir(),
+    TMPDIR: "/tmp",
+    PATH: `${homedir()}/.local/bin:${homedir()}/.npm-global/bin:${homedir()}/bin:${homedir()}/.volta/bin:`
+        + `${homedir()}/.asdf/shims:${homedir()}/.bun/bin:${homedir()}/.nvm/current/bin:${homedir()}/.fnm/current/bin:`
+        + `${homedir()}/.local/share/pnpm:/usr/local/bin:/usr/bin:/bin`,
+};
+const DEFAULT_RESOURCES = { CPU: 500, MemoryMB: 512 };
+// Hard upper bounds applied before submitting any Nomad job. Prevents a
+// misconfigured or malicious instance config from exhausting scheduler
+// resources on the host (no Nomad Enterprise Resource Quotas in OSS).
+const MAX_CPU_MHZ = 4000; // 4 GHz — sane ceiling for a single task
+const MAX_MEMORY_MB = 4096; // 4 GB reservation
+const MAX_MEMORY_MAX_MB = 4096; // 4 GB hard limit (memory_max)
+function jobId(instanceId) {
+    return `${JOB_PREFIX}${instanceId}`;
+}
+// Nomad Template metacharacters that must not appear in values interpolated
+// into EmbeddedTmpl. Defense-in-depth: instanceId is already validated by the
+// route layer, but this guard makes the template-building code self-contained.
+export const NOMAD_TEMPLATE_UNSAFE_RE = /[{}"\\]/;
+function assertSafeTemplateId(id) {
+    if (NOMAD_TEMPLATE_UNSAFE_RE.test(id)) {
+        throw new Error(`Job ID "${id}" contains characters unsafe for Nomad Template interpolation`);
+    }
+}
+async function nomadGet(path) {
+    const resp = await fetch(`${getNomadAddr()}${path}`, {
+        headers: nomadAuthHeaders(),
+        signal: AbortSignal.timeout(10000),
+    });
+    if (!resp.ok && resp.status !== 404)
+        throw new Error(`Nomad ${path}: ${resp.status}`);
+    return resp;
+}
+async function nomadPost(path, body) {
+    return fetch(`${getNomadAddr()}${path}`, {
+        method: "POST",
+        headers: { "Content-Type": "application/json", ...nomadAuthHeaders() },
+        body: JSON.stringify(body),
+        signal: AbortSignal.timeout(10000),
+    });
+}
+async function nomadDelete(path) {
+    return fetch(`${getNomadAddr()}${path}`, {
+        method: "DELETE",
+        headers: nomadAuthHeaders(),
+        signal: AbortSignal.timeout(10000),
+    });
+}
+async function nomadPut(path, body) {
+    return fetch(`${getNomadAddr()}${path}`, {
+        method: "PUT",
+        headers: { "Content-Type": "application/json", ...nomadAuthHeaders() },
+        body: JSON.stringify(body),
+        signal: AbortSignal.timeout(10000),
+    });
+}
+async function getMemoryOversubscriptionState() {
+    const cached = memoryOversubscriptionCache.peek();
+    if (cached)
+        return cached;
+    try {
+        const resp = await nomadGet("/v1/operator/scheduler/configuration");
+        if (!resp.ok)
+            return "unknown";
+        const payload = await resp.json();
+        const state = payload?.SchedulerConfig?.MemoryOversubscriptionEnabled === true ? "enabled" : "disabled";
+        memoryOversubscriptionCache.set(state);
+        return state;
+    }
+    catch {
+        return "unknown";
+    }
+}
+// ── Nomad Variables (secrets) ──
+async function writeInstanceVariables(instanceId) {
+    const jid = jobId(instanceId);
+    // (short-term mitigation): variable path follows Nomad's workload-identity
+    // convention. Each job's workload identity has implicit read/write access only
+    // to variables under its own nomad/jobs/<job-id>/ prefix, providing per-job
+    // secret isolation within the shared "default" namespace. Per-instance Nomad
+    // namespaces remain a planned future improvement.
+    const ns = "default";
+    const varPath = `nomad/jobs/${jid}/openclaw/gateway`;
+    const encodedPath = encodeURIComponent(varPath);
+    // Read proxy token from env file
+    const env = getRuntimeEnv(instanceId);
+    const proxyToken = env.JSPROXY_API_KEY || "";
+    // Nothing to store when proxy token is unconfigured.
+    if (!proxyToken)
+        return;
+    const items = { JSPROXY_API_KEY: proxyToken };
+    // retry with exponential back-off on CAS conflicts (409) so concurrent
+    // startInstance calls do not silently discard the latest token. Throw after
+    // MAX_ATTEMPTS so the caller can surface the error instead of continuing with
+    // a missing proxy token.
+    const MAX_ATTEMPTS = 3;
+    for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
+        // Re-read ModifyIndex on every attempt to always CAS against the latest version.
+        let cas = 0;
+        try {
+            const existing = await nomadGet(`/v1/var/${encodedPath}?namespace=${ns}`);
+            if (existing.ok) {
+                const data = await existing.json();
+                cas = data.ModifyIndex || 0;
+            }
+        }
+        catch { /* variable may not exist yet — cas=0 creates a new one */ }
+        const resp = await nomadPut(`/v1/var/${encodedPath}?cas=${cas}&namespace=${ns}`, {
+            Namespace: ns,
+            Path: varPath,
+            Items: items,
+        });
+        if (resp.ok)
+            return;
+        const text = await resp.text();
+        // 409 Conflict = CAS mismatch; another writer won the race — retry.
+        if (resp.status === 409 && attempt < MAX_ATTEMPTS - 1) {
+            await new Promise(r => setTimeout(r, 100 * Math.pow(2, attempt)));
+            continue;
+        }
+        throw new Error(`Failed to write Nomad Variables for ${instanceId}` +
+            ` (attempt ${attempt + 1}/${MAX_ATTEMPTS}): HTTP ${resp.status} ${text}`);
+    }
+}
+export async function purgeInstanceVariables(instanceId) {
+    const jid = jobId(instanceId);
+    const varPath = `nomad/jobs/${jid}/openclaw/gateway`;
+    const encodedPath = encodeURIComponent(varPath);
+    try {
+        const resp = await nomadDelete(`/v1/var/${encodedPath}`);
+        if (!resp.ok && resp.status !== 404) {
+            console.warn(`[nomad] Failed to purge variables for ${instanceId}: HTTP ${resp.status}`);
+        }
+    }
+    catch (e) {
+        console.warn(`[nomad] Failed to purge variables for ${instanceId}: ${e.message}`);
+    }
+}
+export const VALID_USER_RE = /^[a-z0-9._-]{1,32}$/;
+/**
+ * Resolve the numeric uid:gid for a given username by reading /etc/passwd.
+ * Falls back to process.getuid!():process.getgid!() when the lookup fails
+ * (e.g. the user doesn't exist on this host or /etc/passwd is unreadable).
+ */
+function resolveUidGid(username) {
+    try {
+        const passwd = readFileSync("/etc/passwd", "utf-8");
+        const line = passwd.split("\n").find(l => l.startsWith(username + ":"));
+        if (line) {
+            const parts = line.split(":");
+            const uid = parseInt(parts[2], 10);
+            const gid = parseInt(parts[3], 10);
+            if (!isNaN(uid) && !isNaN(gid))
+                return `${uid}:${gid}`;
+        }
+    }
+    catch { /* ignore */ }
+    return `${process.getuid()}:${process.getgid()}`;
+}
+function buildRuntime(instanceId) {
+    const runtime = getInstanceRuntime(instanceId);
+    const openclawHome = getOpenclawHome(instanceId);
+    // Validate user to prevent injection via Nomad job spec
+    if (runtime.user && !VALID_USER_RE.test(runtime.user)) {
+        throw new Error(`Invalid runtime user: ${runtime.user}`);
+    }
+    const command = runtime.command || DEFAULT_COMMAND;
+    let args = runtime.args;
+    if (!Array.isArray(args))
+        args = [...DEFAULT_ARGS];
+    else
+        args = args.map(String);
+    const env = { ...DEFAULT_ENV };
+    Object.assign(env, getRuntimeEnv(instanceId));
+    delete env.JSPROXY_API_KEY; // Injected by Nomad template from Variables
+    env.OPENCLAW_HOME = openclawHome;
+    env.OPENCLAW_INSTANCE_ID = instanceId;
+    const resources = { ...DEFAULT_RESOURCES };
+    for (const [key, value] of Object.entries(runtime.resources || {})) {
+        if (value != null)
+            resources[key] = Number(value);
+    }
+    // Clamp to sane upper bounds — guards against arbitrarily large values that
+    // would exhaust Nomad scheduler capacity or system memory.
+    resources.CPU = Math.max(1, Math.min(resources.CPU, MAX_CPU_MHZ));
+    resources.MemoryMB = Math.max(1, Math.min(resources.MemoryMB, MAX_MEMORY_MB));
+    return {
+        command: String(command),
+        args,
+        user: runtime.user || DEFAULT_USER,
+        cwd: runtime.cwd || DEFAULT_CWD,
+        env,
+        resources,
+    };
+}
+function normalizeDockerResources(instanceId, runtime, oversubState) {
+    const requestedMemoryMB = Number(runtime.resources.MemoryMB ?? DEFAULT_RESOURCES.MemoryMB);
+    let effectiveMemoryMB = requestedMemoryMB;
+    let effectiveMemoryMaxMB = Math.min(Number(runtime.resources.MemoryMaxMB ?? MAX_MEMORY_MAX_MB), MAX_MEMORY_MAX_MB);
+    if (effectiveMemoryMaxMB < effectiveMemoryMB) {
+        console.warn(`[nomad] ${instanceId}: MemoryMaxMB (${effectiveMemoryMaxMB}) is below MemoryMB (${effectiveMemoryMB}); clamping max to reservation.`);
+        effectiveMemoryMaxMB = effectiveMemoryMB;
+    }
+    if (oversubState === "disabled" && effectiveMemoryMaxMB > effectiveMemoryMB) {
+        console.warn(`[nomad] ${instanceId}: memory oversubscription is disabled; promoting MemoryMB ` +
+            `from ${effectiveMemoryMB}MB to ${effectiveMemoryMaxMB}MB so the Docker limit matches MemoryMaxMB.`);
+        effectiveMemoryMB = effectiveMemoryMaxMB;
+    }
+    return {
+        ...runtime.resources,
+        MemoryMB: effectiveMemoryMB,
+        MemoryMaxMB: effectiveMemoryMaxMB,
+    };
+}
+function buildTaskDocker(instanceId, runtime, oversubState) {
+    // Guard against Nomad Template injection: validate the job ID contains no
+    // template metacharacters before interpolating it into EmbeddedTmpl.
+    const safeJobId = jobId(instanceId);
+    assertSafeTemplateId(safeJobId);
+    const openclawHome = getOpenclawHome(instanceId);
+    const image = getOpenclawDockerImage();
+    // Image classification:
+    // - Official: ghcr.io/openclaw/openclaw:* — binary baked in, no bind-mount needed
+    // - Slim base: jishushell-base:* — binary bind-mounted from host npm package (legacy)
+    // - Local/other: openclaw:* or custom images
+    const _isOfficialImage = isOfficialImage(image);
+    const isSlimBaseImage = /^jishushell-base:/i.test(image);
+    // node_modules parent dir — contains openclaw package AND all its sibling dependencies
+    const openclawNmDir = join(JISHUSHELL_HOME, "packages", "openclaw", "node_modules");
+    const openclawAppDir = join(openclawHome, "app");
+    const volumes = [
+        `${openclawHome}:${openclawHome}:rw`,
+        // Only slim base needs the bind-mount; official and local images have the binary baked in.
+        ...(isSlimBaseImage ? [`${openclawNmDir}:/usr/lib/node_modules:ro`] : []),
+        // Official image: persist /app so OpenClaw can self-upgrade and install plugins.
+        ...(_isOfficialImage ? [`${openclawAppDir}:/app:rw`] : []),
+    ];
+    const containerEnv = { ...runtime.env };
+    containerEnv.PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin";
+    const isLocalImage = !image.includes("/"); // e.g. "openclaw:v1.0" vs "ghcr.io/..."
+    // Container runs as uid 1000 (node user in the image). Always set HOME to
+    // /home/node which exists in the image and is writable (npm cache, etc).
+    // The host HOME (/home/pi) does not exist inside the container.
+    containerEnv.HOME = "/home/node";
+    // Plugins (e.g. openclaw-weixin) use OPENCLAW_STATE_DIR to find credentials.
+    // Inside Docker, HOME differs from the host, so set this explicitly.
+    if (!containerEnv.OPENCLAW_STATE_DIR) {
+        containerEnv.OPENCLAW_STATE_DIR = `${openclawHome}/.openclaw`;
+    }
+    const runtimeArgs = [...(runtime.args || [])];
+    let commandConfig;
+    if (_isOfficialImage) {
+        // Official/custom image ENTRYPOINT is docker-entrypoint.sh (from node base).
+        // Prepend "openclaw" so the entrypoint execs: openclaw gateway run ...
+        commandConfig = { args: ["openclaw", ...runtimeArgs] };
+    }
+    else if (isSlimBaseImage) {
+        // Slim base image: ENTRYPOINT is ["node", "/usr/lib/node_modules/openclaw/openclaw.mjs"]
+        // which uses the bind-mounted binary. Just pass gateway args, no command override needed.
+        commandConfig = { args: runtimeArgs };
+    }
+    else if (isLocalImage) {
+        // Local image has /usr/local/bin/openclaw symlink baked in.
+        commandConfig = { command: "/usr/local/bin/openclaw", args: runtimeArgs };
+    }
+    else {
+        commandConfig = { command: "/usr/local/bin/openclaw", args: runtimeArgs };
+    }
+    // Only the gateway port is published to the host; all other container ports stay
+    // hidden. Bridge networking gives each container an isolated network namespace;
+    // extra_hosts injects the host gateway IP so the container can still reach the
+    // JishuShell LLM proxy on the host without needing host-mode networking.
+    const gatewayPort = getGatewayPort(instanceId);
+    const normalizedResources = normalizeDockerResources(instanceId, runtime, oversubState);
+    return {
+        Name: "gateway",
+        Driver: "docker",
+        // Task-level User field — Nomad passes this as --user to docker run.
+        User: resolveUidGid(runtime.user),
+        Config: {
+            image,
+            // Local build: never force pull. Use local image cache.
+            force_pull: false,
+            ...commandConfig,
+            volumes,
+            // Bridge mode (default): container gets an isolated network namespace.
+            // host.docker.internal resolves to the host's gateway IP on the bridge
+            // (172.17.0.1 or equivalent) so the container can call 127.0.0.1-bound
+            // host services via http://host.docker.internal:<port> instead.
+            extra_hosts: ["host.docker.internal:host-gateway"],
+            cap_drop: ["ALL"],
+            security_opt: ["no-new-privileges"], // block setuid/setgid escalation
+            pids_limit: DEFAULT_PIDS_LIMIT, // prevent fork bomb
+            // Official image: writable rootfs (users may install packages / upgrade inside the container).
+            // Other images: read-only rootfs for security.
+            readonly_rootfs: !_isOfficialImage,
+            // Provide a writable /tmp via mount config (Nomad docker driver
+            // doesn't support top-level "tmpfs" field in older versions).
+            mounts: [{ type: "tmpfs", target: "/tmp", tmpfs_options: { size: 67108864 } }],
+        },
+        Env: containerEnv,
+        Resources: {
+            ...normalizedResources,
+            // When memory oversubscription is enabled, MemoryMB is the scheduler
+            // reservation and MemoryMaxMB is the burst ceiling / container hard limit.
+            // When it is disabled, Nomad ignores MemoryMaxMB and enforces MemoryMB as
+            // the Docker limit, so normalizeDockerResources() promotes MemoryMB to the
+            // configured max to preserve user intent.
+            // Statically reserve the gateway port on the host so Nomad can track it and
+            // detect conflicts across instances before the container even starts.
+            // In bridge mode Nomad maps this host port to the same container port.
+            Networks: [{ ReservedPorts: [{ Label: "gateway", Value: gatewayPort }] }],
+        },
+        LogConfig: { MaxFiles: 3, MaxFileSizeMB: 10 },
+        Templates: [{
+                DestPath: "secrets/instance.env",
+                Envvars: true,
+                EmbeddedTmpl: [
+                    `{{ if nomadVarExists "nomad/jobs/${safeJobId}/openclaw/gateway" }}`,
+                    `JSPROXY_API_KEY={{ with nomadVar "nomad/jobs/${safeJobId}/openclaw/gateway" }}{{ .JSPROXY_API_KEY }}{{ end }}`,
+                    `{{ end }}`,
+                ].join("\n"),
+                ChangeMode: "restart",
+            }],
+    };
+}
+async function buildJob(instanceId) {
+    const jid = jobId(instanceId);
+    const runtime = buildRuntime(instanceId);
+    const driver = getNomadDriver();
+    const oversubState = await getMemoryOversubscriptionState();
+    if (driver !== "docker") {
+        throw new Error(`Unsupported Nomad driver: ${driver}. Only "docker" is supported.`);
+    }
+    const task = buildTaskDocker(instanceId, runtime, oversubState);
+    return {
+        Job: {
+            ID: jid,
+            Name: jid,
+            Namespace: "default",
+            Type: "service",
+            Datacenters: ["*"],
+            TaskGroups: [{
+                    Name: "openclaw",
+                    Count: 1,
+                    RestartPolicy: {
+                        Attempts: 3,
+                        Interval: 300000000000, // 5 min (nanoseconds)
+                        Delay: 15000000000, // 15 s  (nanoseconds)
+                        // "fail" mode: once attempts are exhausted the alloc is marked failed
+                        // and triggers reschedule evaluation, making failures visible.
+                        // "delay" (old default) silently retries forever without ever
+                        // setting the alloc to failed or triggering reschedule.
+                        Mode: "fail",
+                    },
+                    // Single-node (Raspberry Pi) environment: reschedule is meaningless
+                    // because there is only one node. Explicitly disable it so Nomad
+                    // doesn't spin trying to place the job on a non-existent second node.
+                    Reschedule: {
+                        Attempts: 0,
+                        Unlimited: false,
+                    },
+                    // Update policy: use task_states health check because no service
+                    // checks are registered. Without this, Nomad defaults to
+                    // health_check="checks" and waits forever for a signal that never comes,
+                    // hanging every job re-submission indefinitely.
+                    Update: {
+                        MaxParallel: 1,
+                        HealthCheck: "task_states",
+                        MinHealthyTime: 5000000000, // 5 s
+                        HealthyDeadline: 60000000000, // 60 s
+                        AutoRevert: false,
+                    },
+                    Tasks: [task],
+                }],
+        },
+    };
+}
+async function getRunningAlloc(instanceId) {
+    const jid = jobId(instanceId);
+    try {
+        const resp = await nomadGet(`/v1/job/${jid}/allocations`);
+        if (resp.status === 404)
+            return null;
+        const allocs = await resp.json();
+        for (const status of ["running", "pending"]) {
+            for (const alloc of allocs) {
+                if (alloc.ClientStatus === status)
+                    return alloc;
+            }
+        }
+        return null;
+    }
+    catch {
+        return null;
+    }
+}
+// Returns true if the Nomad job exists and was NOT explicitly stopped by the user (Stop=false).
+// Used on jishushell startup to auto-restart instances that were running before a reboot.
+export async function shouldAutoStart(instanceId) {
+    const jid = jobId(instanceId);
+    try {
+        const resp = await nomadGet(`/v1/job/${jid}`);
+        if (!resp.ok || resp.status === 404)
+            return false;
+        const job = await resp.json();
+        // Stop=true means user explicitly stopped it; Stop=false means it was running.
+        // Also skip dead jobs — all allocs failed, resubmitting would fail again.
+        return job.Stop === false && job.Status !== "dead";
+    }
+    catch {
+        return false;
+    }
+}
+export async function getStatus(instanceId) {
+    const jid = jobId(instanceId);
+    const stopped = { status: "stopped", pid: null, uptime: null, memory_mb: null, cpu_percent: null };
+    try {
+        const resp = await nomadGet(`/v1/job/${jid}`);
+        if (resp.status === 404)
+            return stopped;
+        const job = await resp.json();
+        if (job.Stop)
+            return stopped;
+    }
+    catch {
+        return { ...stopped, status: "unknown", error: "Nomad unreachable" };
+    }
+    const alloc = await getRunningAlloc(instanceId);
+    if (!alloc)
+        return { ...stopped, status: "pending" };
+    const allocId = alloc.ID;
+    const result = {
+        status: alloc.ClientStatus || "unknown",
+        alloc_id: allocId,
+        pid: null,
+        uptime: null,
+        memory_mb: null,
+        cpu_percent: null,
+        restarts: 0,
+    };
+    const gwState = alloc.TaskStates?.gateway || {};
+    result.restarts = gwState.Restarts || 0;
+    const startedAt = gwState.StartedAt;
+    if (startedAt) {
+        try {
+            const start = new Date(startedAt);
+            result.uptime = Math.floor((Date.now() - start.getTime()) / 1000);
+        }
+        catch { /* ignore */ }
+    }
+    try {
+        const statsResp = await nomadGet(`/v1/client/allocation/${allocId}/stats`);
+        if (statsResp.ok) {
+            const stats = await statsResp.json();
+            // raw_exec: stats nested under Tasks.gateway; docker: top-level ResourceUsage
+            const taskStats = stats.Tasks?.gateway?.ResourceUsage || stats.ResourceUsage || {};
+            const memStats = taskStats.MemoryStats || {};
+            const cpuStats = taskStats.CpuStats || {};
+            const memBytes = memStats.RSS || memStats.Usage || 0;
+            result.memory_mb = Math.round(memBytes / (1024 * 1024) * 10) / 10;
+            result.cpu_percent = Math.round((cpuStats.Percent || 0) * 10) / 10;
+        }
+    }
+    catch { /* ignore */ }
+    // Fallback: Nomad cgroup stats are often zero on cgroup v2 (e.g. Raspberry Pi).
+    // Use `docker stats` directly when Nomad reports 0.
+    if (!result.memory_mb && allocId) {
+        try {
+            // Validate allocId to prevent shell injection (Nomad UUIDs are hex + hyphens)
+            if (!/^[a-f0-9-]+$/i.test(allocId))
+                throw new Error("invalid allocId");
+            const containerName = `gateway-${allocId}`;
+            const { execFile } = await import("child_process");
+            const { promisify } = await import("util");
+            const execFileAsync = promisify(execFile);
+            const { stdout } = await execFileAsync("docker", ["stats", "--no-stream", "--format", "{{.MemUsage}}", containerName], { timeout: 5000 });
+            const raw = stdout.trim();
+            // Format: "499.6MiB / 3GiB" or "123.4MB / 2GB"
+            const match = raw.match(/^([\d.]+)\s*(MiB|GiB|MB|GB|KiB|KB)/i);
+            if (match) {
+                let mb = parseFloat(match[1]);
+                const unit = match[2].toLowerCase();
+                if (unit === "gib" || unit === "gb")
+                    mb *= 1024;
+                else if (unit === "kib" || unit === "kb")
+                    mb /= 1024;
+                result.memory_mb = Math.round(mb * 10) / 10;
+            }
+        }
+        catch { /* ignore */ }
+    }
+    return result;
+}
+export async function startInstance(instanceId) {
+    const status = await getStatus(instanceId);
+    if (status.status === "running") {
+        return { ok: false, error: "Instance is already running" };
+    }
+    const homeConflicts = [];
+    for (const otherId of findInstancesSharingOpenclawHome(instanceId)) {
+        const otherStatus = await getStatus(otherId);
+        if (otherStatus.status === "running")
+            homeConflicts.push(otherId);
+    }
+    if (homeConflicts.length) {
+        return {
+            ok: false,
+            error: `This instance shares OPENCLAW_HOME with running instance(s): ${homeConflicts.join(", ")}. Move it to its own instance directory before starting it.`,
+        };
+    }
+    const portConflicts = [];
+    for (const otherId of findInstancesSharingGatewayPort(instanceId)) {
+        const otherStatus = await getStatus(otherId);
+        if (otherStatus.status === "running")
+            portConflicts.push(otherId);
+    }
+    if (portConflicts.length) {
+        const port = getGatewayPort(instanceId);
+        return {
+            ok: false,
+            error: `Gateway port ${port} is already in use by running instance(s): ${portConflicts.join(", ")}. Assign a different port before starting this instance.`,
+        };
+    }
+    const legacyStatus = await getLegacyStatus(instanceId);
+    if (legacyStatus.status === "running") {
+        console.log(`[nomad] Stopping legacy process for ${instanceId} (pid=${legacyStatus.pid}) before Nomad start...`);
+        await stopLegacyInstance(instanceId);
+        // Give it a moment to exit
+        await new Promise((r) => setTimeout(r, 2000));
+    }
+    const configPath = getOpenclawConfigPath(instanceId);
+    if (!existsSync(configPath)) {
+        return { ok: false, error: "Config file not found" };
+    }
+    mkdirSync(dirname(configPath), { recursive: true, mode: 0o750 });
+    // Ensure Docker image exists when using docker driver
+    if (getNomadDriver() === "docker") {
+        // Docker Desktop on macOS uses virtio-fs: the container process isn't recognized
+        // as the file owner even when uid matches. The .openclaw dir needs 0o777 so the
+        // container can create .tmp files for atomic config writes. Files need 0o644 so
+        // the container can read the config. Applied here to fix existing instances too.
+        // On Linux/RPi 0o755 is sufficient — Docker maps the host uid correctly.
+        const stateDir = dirname(configPath);
+        chmodSync(stateDir, platform() === "darwin" ? 0o777 : 0o750);
+        chmodSync(configPath, platform() === "darwin" ? 0o644 : 0o600);
+        // Bridge mode needs a non-loopback gateway bind inside the container.
+        patchDockerBridgeGatewayBind(configPath);
+        // Bridge mode: rewrite 127.0.0.1 → host.docker.internal in jsproxy baseUrl
+        // so the container can reach the JishuShell LLM proxy on the host.
+        patchJsproxyBaseUrl(configPath);
+        const image = getOpenclawDockerImage();
+        // validate image name format and length.
+        if (!DOCKER_IMAGE_RE.test(image) || image.length > MAX_DOCKER_IMAGE_NAME_LEN) {
+            return { ok: false, error: `Invalid Docker image name: "${image}"` };
+        }
+        // Initialize persistent /app directory for official images on first start.
+        // Copy the image's /app contents to the host so OpenClaw can self-upgrade.
+        if (isOfficialImage(image)) {
+            const appDir = join(getOpenclawHome(instanceId), "app");
+            mkdirSync(appDir, { recursive: true, mode: 0o755 });
+            const isEmpty = readdirSync(appDir).length === 0;
+            if (isEmpty) {
+                console.log(`[nomad] Initializing /app for ${instanceId} from image ${image}...`);
+                const tmpName = `jishushell-init-${instanceId}-${Date.now()}`;
+                try {
+                    execFileSync("docker", ["create", "--name", tmpName, image], { timeout: 30000, stdio: "ignore" });
+                    execFileSync("docker", ["cp", `${tmpName}:/app/.`, appDir], { timeout: 120000, stdio: "ignore" });
+                    execFileSync("docker", ["rm", tmpName], { timeout: 10000, stdio: "ignore" });
+                }
+                catch (e) {
+                    try {
+                        execFileSync("docker", ["rm", "-f", tmpName], { timeout: 10000, stdio: "ignore" });
+                    }
+                    catch { }
+                    console.error(`[nomad] Failed to initialize /app for ${instanceId}: ${e.message}`);
+                    return { ok: false, error: `Failed to initialize OpenClaw app directory: ${e.message}` };
+                }
+            }
+        }
+        try {
+            execFileSync("docker", ["image", "inspect", image], { timeout: 10000, stdio: "ignore" });
+        }
+        catch {
+            // Image not found locally — kick off a background build and return
+            // immediately so the API doesn't block for 5-10 minutes on RPi.
+            console.log(`[nomad] Docker image ${image} not found, starting background build...`);
+            try {
+                const setupManager = await import("./setup-manager.js");
+                const result = isOfficialImage(image)
+                    ? setupManager.startBuildCustomOpenclawImage(image)
+                    : setupManager.startBuildOpenclawDockerImage(image);
+                return {
+                    ok: false,
+                    error: `Docker image ${image} not found. Build started in background.`,
+                    building: true,
+                    taskId: result.taskId,
+                };
+            }
+            catch (e) {
+                return { ok: false, error: `Docker image ${image} not available: ${e.message}` };
+            }
+        }
+    }
+    // Write instance secrets to Nomad Variables before starting the job.
+    // propagate failure — a missing proxy token causes 401 on every LLM
+    // request, so it is better to surface the error here than start a broken instance.
+    try {
+        await writeInstanceVariables(instanceId);
+    }
+    catch (e) {
+        return { ok: false, error: `Failed to store instance secrets in Nomad Variables: ${e.message}` };
+    }
+    const jobDef = await buildJob(instanceId);
+    try {
+        const resp = await nomadPost("/v1/jobs", jobDef);
+        if (resp.ok) {
+            const data = await resp.json();
+            return { ok: true, eval_id: data.EvalID };
+        }
+        return { ok: false, error: await resp.text() };
+    }
+    catch (e) {
+        const isNetErr = e?.message === "fetch failed" || e?.cause?.code === "ECONNREFUSED";
+        return { ok: false, error: isNetErr ? `Nomad 服务不可达 (${getNomadAddr()})，请先启动 Nomad` : e.message };
+    }
+}
+export async function stopInstance(instanceId, purge = false) {
+    const jid = jobId(instanceId);
+    try {
+        const resp = await nomadDelete(`/v1/job/${jid}?purge=${purge}`);
+        if (resp.ok) {
+            if (purge) {
+                try {
+                    await purgeInstanceVariables(instanceId);
+                }
+                catch { /* ignore */ }
+            }
+            return { ok: true };
+        }
+        if (resp.status === 404)
+            return { ok: false, error: "Instance is not running" };
+        return { ok: false, error: await resp.text() };
+    }
+    catch (e) {
+        const isNetErr = e?.message === "fetch failed" || e?.cause?.code === "ECONNREFUSED";
+        return { ok: false, error: isNetErr ? `Nomad 服务不可达 (${getNomadAddr()})，请先启动 Nomad` : e.message };
+    }
+}
+export async function restartInstance(instanceId) {
+    // Prefer the native Nomad allocation restart API over stop+start:
+    //   PUT /v1/client/allocation/{allocID}/restart
+    // This preserves alloc history, EvalID, and avoids rescheduling.
+    // Only falls back to stop+start when no running/pending alloc exists.
+    const alloc = await getRunningAlloc(instanceId);
+    if (alloc) {
+        try {
+            const resp = await nomadPut(`/v1/client/allocation/${alloc.ID}/restart`, {
+                TaskName: "gateway",
+                AllTasks: false,
+            });
+            if (resp.ok)
+                return { ok: true, alloc_id: alloc.ID };
+            // Non-2xx from the restart endpoint falls through to stop+start
+            const errText = await resp.text();
+            console.warn(`[nomad] Native restart failed for ${instanceId} (HTTP ${resp.status}): ${errText} — falling back to stop+start`);
+        }
+        catch (e) {
+            console.warn(`[nomad] Native restart error for ${instanceId}: ${e.message} — falling back to stop+start`);
+        }
+    }
+    // Fallback: stop the job and re-submit it
+    await stopInstance(instanceId);
+    await new Promise((r) => setTimeout(r, 2000));
+    return startInstance(instanceId);
+}
+export async function getLogs(instanceId, lines = 200, logType = "stderr") {
+    // Defense-in-depth: only allow known log types to prevent path/query injection
+    if (!VALID_LOG_TYPES.has(logType))
+        logType = "stderr";
+    let alloc = await getRunningAlloc(instanceId);
+    if (!alloc) {
+        const jid = jobId(instanceId);
+        try {
+            const resp = await nomadGet(`/v1/job/${jid}/allocations`);
+            if (resp.ok) {
+                const allocs = await resp.json();
+                if (allocs.length) {
+                    alloc = allocs.sort((a, b) => (b.CreateIndex || 0) - (a.CreateIndex || 0))[0];
+                }
+            }
+        }
+        catch { /* ignore */ }
+    }
+    if (!alloc)
+        return [];
+    try {
+        const params = new URLSearchParams({
+            task: "gateway",
+            type: logType,
+            plain: "true",
+            origin: "end",
+            offset: String(Math.max(lines * 512, 100000)),
+            follow: "false",
+        });
+        const resp = await nomadGet(`/v1/client/fs/logs/${alloc.ID}?${params}`);
+        if (resp.ok) {
+            const text = await resp.text();
+            return text.split("\n").slice(-lines);
+        }
+    }
+    catch { /* ignore */ }
+    return [];
+}
+const execFileAsync = promisify(execFileCb);
+export async function exec(instanceId, command, timeoutMs = 120_000) {
+    const alloc = await getRunningAlloc(instanceId);
+    if (!alloc || alloc.ClientStatus !== "running") {
+        throw new Error("Instance is not running");
+    }
+    const allocId = alloc.ID;
+    if (!/^[a-f0-9-]+$/i.test(allocId))
+        throw new Error("invalid allocId");
+    const containerName = `gateway-${allocId}`;
+    // Use the same user as the container's main process (runtime.user uid:gid)
+    const runtime = getInstanceRuntime(instanceId);
+    const userFlag = resolveUidGid(runtime.user);
+    try {
+        const { stdout, stderr } = await execFileAsync("docker", ["exec", "--user", userFlag, containerName, ...command], { timeout: timeoutMs });
+        return { stdout, stderr, exitCode: 0 };
+    }
+    catch (e) {
+        return {
+            stdout: e.stdout || "",
+            stderr: e.stderr || e.message,
+            exitCode: e.code ?? 1,
+        };
+    }
+}
+//# sourceMappingURL=nomad-manager.js.map