jishushell 0.4.24 → 0.5.15

package/dist/services/nomad-manager.js

@@ -30,7 +30,7 @@ function resolveConfigPath(value, fallback) {
 }
 const JISHUSHELL_HOME = resolveConfigPath(getConfigValue("JISHUSHELL_HOME"), join(process.env.HOME ?? homedir(), ".jishushell"));
 const APPS_DIR = resolveConfigPath(getConfigValue("APPS_DIR"), join(JISHUSHELL_HOME, "apps"));
-const INSTANCES_DIR = resolveConfigPath(getConfigValue("INSTANCES_DIR"), join(JISHUSHELL_HOME, "instances"));
+const _INSTANCES_DIR = resolveConfigPath(getConfigValue("INSTANCES_DIR"), join(JISHUSHELL_HOME, "instances"));
 const getNomadAddrValue = getConfigValue("getNomadAddr");
 const getNomadDriverValue = getConfigValue("getNomadDriver");
 const getNomadTokenValue = getConfigValue("getNomadToken");
@@ -415,6 +415,20 @@ function getInstanceAgentType(instanceId) {
     }
 }
 function wrapNomadJob(jid, groupName, task) {
+    // Adapters declare port reservations on `task.Resources.Networks` (legacy
+    // schema). The docker driver's `Config.ports = [<label>]` lookup, however,
+    // resolves labels against the TaskGroup-level `Networks` block. Move (not
+    // copy) the network block so the docker driver can find the port and so
+    // HostNetwork ("external") is honored — without this, ports publish to
+    // 127.0.0.1 by default. Keeping it on both levels would make Nomad reject
+    // the job with "port label already in use".
+    const taskNetworks = Array.isArray(task?.Resources?.Networks)
+        ? task.Resources.Networks
+        : [];
+    const groupNetworks = taskNetworks.length > 0 ? taskNetworks.map((n) => ({ ...n })) : undefined;
+    if (groupNetworks && task?.Resources && typeof task.Resources === "object") {
+        delete task.Resources.Networks;
+    }
     return {
         Job: {
             ID: jid,
@@ -425,14 +439,24 @@ function wrapNomadJob(jid, groupName, task) {
             TaskGroups: [{
                     Name: groupName,
                     Count: 1,
+                    ...(groupNetworks ? { Networks: groupNetworks } : {}),
                     RestartPolicy: {
-                        Attempts: 3,
-                        Interval: 300000000000,
+                        // 10 attempts × 15s delay = ~2.5min recovery window. Multi-task
+                        // groups (e.g. weknora-app racing paradedb cold-init) commonly
+                        // need 3-5 restarts before the dependency's external port-publish
+                        // settles. Previous 3-attempt cap caused alloc-fail cascades on
+                        // first-boot of stacks with DB sidecars.
+                        Attempts: 10,
+                        Interval: 600000000000,
                         Delay: 15000000000,
                         Mode: "fail",
                     },
                     Reschedule: {
-                        Attempts: 0,
+                        // Allow Nomad to reschedule the whole alloc up to 3 times if all
+                        // restarts fail (e.g. transient image pull or host reboot).
+                        // Unlimited stays false to keep alloc churn bounded.
+                        Attempts: 3,
+                        Interval: 3600000000000,
                         Unlimited: false,
                     },
                     Update: {
@@ -457,6 +481,7 @@ async function buildJob(instanceId) {
     if (legacyManager) {
         const runtime = legacyManager.buildRuntime(instanceId);
         const task = legacyManager.buildNomadTask(instanceId, runtime, jid);
+        await injectConnectionsRuntimeEnv(instanceId, task);
         return wrapNomadJob(jid, legacyManager.nomadTaskGroupName(), task);
     }
     // Pure adapter dispatch — no more `isHermesInstance()` / kind literals.
@@ -466,11 +491,54 @@ async function buildJob(instanceId) {
         throw new Error(`Runtime adapter "${agentType}" does not implement buildNomadTask(); cannot schedule Nomad job`);
     }
     const task = await adapter.buildNomadTask(instanceId);
+    await injectConnectionsRuntimeEnv(instanceId, task);
     // Task group name mirrors the agentType. Log/status helpers resolve the
     // Nomad task name via resolveTaskName(instanceId) → adapter.nomadTaskName.
     const groupName = agentType;
     return wrapNomadJob(jid, groupName, task);
 }
+/**
+ * Re-resolve `instance.connections` against the live capability registry
+ * and merge the resulting env into the freshly-built Nomad task. Idempotent
+ * — empty meta.connections short-circuits to a no-op.
+ *
+ * Resolving at start time (rather than reading the frozen
+ * `instance.json["connections-env"]` written by PUT /connections) means
+ * provider port / address changes in the registry propagate on next
+ * restart without requiring the user to re-bind. Failures here are
+ * logged but never block start: a missing required binding still surfaces
+ * via the Connections UI status badge.
+ */
+async function injectConnectionsRuntimeEnv(instanceId, task) {
+    try {
+        const meta = getInstance(instanceId);
+        const connections = meta?.connections;
+        if (!meta || !connections || Object.keys(connections).length === 0)
+            return;
+        const { loadCapabilitySpecForLegacyInstance } = await import("./runtime/migrations.js");
+        const spec = loadCapabilitySpecForLegacyInstance(meta);
+        if (!spec)
+            return;
+        const { resolveConnections } = await import("./connection-resolver.js");
+        const { resolved } = resolveConnections(spec, { connections }, "preCreate");
+        if (resolved.length === 0)
+            return;
+        const { RUNTIME_HOOKS } = await import("./connection-apply.js");
+        const merged = {};
+        for (const binding of resolved) {
+            const hook = RUNTIME_HOOKS[binding.category];
+            if (!hook)
+                continue;
+            Object.assign(merged, await hook(meta, binding));
+        }
+        if (Object.keys(merged).length === 0)
+            return;
+        task.Env = { ...(task.Env ?? {}), ...merged };
+    }
+    catch (e) {
+        console.warn(`[nomad] connections runtime env merge failed for ${instanceId}: ${e?.message ?? e}`);
+    }
+}
 async function getRunningAlloc(instanceId) {
     const jid = jobId(instanceId);
     try {
@@ -607,7 +675,7 @@ export async function getStatus(instanceId) {
 async function phaseRunningCheck(instanceId) {
     const status = await getStatus(instanceId);
     if (status.status === "running") {
-        return { ok: false, error: "Instance is already running" };
+        return { ok: false, error: "Instance is already running", code: "INSTANCE_ALREADY_RUNNING" };
     }
     return { ok: true };
 }
@@ -618,7 +686,7 @@ async function phaseRunningCheck(instanceId) {
  * each instance owns its own bind-mount) leave the hook unset and this
  * phase is a no-op.
  */
-async function phaseHomeConflict(instanceId, sharedHomeIds) {
+async function phaseHomeConflict(_instanceId, sharedHomeIds) {
     const homeConflicts = [];
     for (const otherId of sharedHomeIds) {
         const otherStatus = await getStatus(otherId);
@@ -674,6 +742,81 @@ async function phasePreStartHook(adapter, instanceId) {
         return { ok: false, error: e?.message || String(e) };
     }
 }
+/**
+ * §17 / PR 9 — re-render adapter-managed connection config from the
+ * current capability registry before each instance start.
+ *
+ * Without this hook, env values (like `SEARCH_API_BASE_URL` =
+ * `http://<host>:<port>/search`) are frozen into the adapter's config
+ * files at PUT /connections time. When the host IP changes (DHCP
+ * renewal, pi reboot picking up a new lease, network move) or a
+ * provider gets re-deployed at a different host:port, the consumer
+ * keeps trying the stale address and search/llm/etc. silently fail.
+ *
+ * What this does on every start:
+ *   1. Read connections from instance.json
+ *   2. Re-resolve them in `runtime` mode (tolerant: ambiguous/missing
+ *      becomes empty resolved instead of throwing — start should still
+ *      proceed even if one binding can't be re-rendered)
+ *   3. Collect env via the same persist hooks PUT /connections uses
+ *   4. Call adapter.applyConnectionEnv with the fresh env so the
+ *      adapter rewrites its config files (mcp_servers / openclaw.json /
+ *      etc.) with the current address
+ *
+ * Failures here are logged but never block start: a stale config is
+ * better than no start. If something is genuinely wrong with the
+ * registry, the user will see a connection error in the UI on next
+ * use — at which point they can re-bind manually.
+ */
+async function phaseRefreshConnections(adapter, instanceId) {
+    if (!adapter.applyConnectionEnv)
+        return;
+    try {
+        const meta = getInstance(instanceId);
+        const connections = meta?.connections;
+        if (!meta || !connections || Object.keys(connections).length === 0)
+            return;
+        const { loadCapabilitySpecForLegacyInstance } = await import("./runtime/migrations.js");
+        const spec = loadCapabilitySpecForLegacyInstance(meta);
+        if (!spec)
+            return;
+        const { resolveConnections } = await import("./connection-resolver.js");
+        const { resolved } = resolveConnections(spec, { connections }, "preCreate");
+        if (resolved.length === 0)
+            return;
+        const { PERSIST_HOOKS } = await import("./connection-apply.js");
+        const merged = {};
+        const seenEnvKeys = new Set();
+        // Accumulate env across all resolved bindings via a stub
+        // writeConnectionEnv that just collects into `merged`. We don't
+        // run the real persist hooks here because those would re-write
+        // generic-app `connections-env` (already handled by
+        // injectConnectionsRuntimeEnv); we only want the env so we can
+        // pass it to adapter.applyConnectionEnv below.
+        const stubCtx = {
+            registry: await import("./capability-registry.js"),
+            adapter: { applyConnectionEnv: undefined },
+            async writeConnectionEnv(_inst, env) {
+                for (const [k, v] of Object.entries(env)) {
+                    merged[k] = v;
+                    seenEnvKeys.add(k);
+                }
+            },
+        };
+        for (const binding of resolved) {
+            const hook = PERSIST_HOOKS[binding.category];
+            if (!hook)
+                continue;
+            await hook(meta, binding, stubCtx);
+        }
+        if (seenEnvKeys.size === 0)
+            return;
+        await adapter.applyConnectionEnv(instanceId, merged);
+    }
+    catch (e) {
+        console.warn(`[nomad] connections refresh failed for ${instanceId}: ${e?.message ?? e}`);
+    }
+}
 /**
  * Phase 5: submit to Nomad with a single retry on port race. Between our
  * earlier host probe and Docker's actual bind another process could have
@@ -741,8 +884,12 @@ export async function startInstance(instanceId) {
         return { ok: false, phase, ...rest };
     };
     const running = await phaseRunningCheck(instanceId);
-    if (!running.ok)
-        return failed("running_check", { error: running.error });
+    if (!running.ok) {
+        const extra = { error: running.error };
+        if (running.code)
+            extra.code = running.code;
+        return failed("running_check", extra);
+    }
     const legacyManager = await getLegacyAppManager(instanceId);
     if (legacyManager) {
         const prep = await legacyManager.prepareStart(instanceId);
@@ -761,6 +908,11 @@ export async function startInstance(instanceId) {
         const home = await phaseHomeConflict(instanceId, adapter.findInstancesSharingHome?.(instanceId) ?? []);
         if (!home.ok)
             return failed("home_conflict", { error: home.error });
+        // PR 9 — refresh adapter-managed connection config from current
+        // capability registry before adapter pre-start. Best-effort: never
+        // blocks start (any failure is logged and we proceed with the
+        // existing on-disk config). See phaseRefreshConnections doc.
+        await phaseRefreshConnections(adapter, instanceId);
         const hook = await phasePreStartHook(adapter, instanceId);
         if (!hook.ok) {
             const extra = { error: hook.error };
@@ -777,12 +929,65 @@ export async function startInstance(instanceId) {
     const submit = await phaseSubmit(instanceId, port.portAllocation);
     if (!submit.ok)
         return failed("submit", { error: submit.error });
+    // Auto-register capability providers for legacy instances so they appear
+    // in the Connections UI alongside app-installed apps. App-dir installed
+    // apps short-circuit at the top of this function and don't reach here.
+    await registerLegacyCapabilitiesTopLevel(instanceId);
     return {
         ok: true,
         eval_id: submit.evalId,
         ...(submit.portAllocation ? { port_allocation: submit.portAllocation } : {}),
     };
 }
+/**
+ * Best-effort capability registration for legacy (non-app-installed)
+ * hermes/openclaw instances. Loaded synthetic spec via the migrations
+ * helper; failures are logged but never block start/stop.
+ */
+async function registerLegacyCapabilitiesTopLevel(instanceId) {
+    try {
+        const meta = getInstance(instanceId);
+        if (!meta)
+            return;
+        const { loadCapabilitySpecForLegacyInstance } = await import("./runtime/migrations.js");
+        const synthSpec = loadCapabilitySpecForLegacyInstance(meta);
+        if (!synthSpec?.provides?.length)
+            return;
+        // The synthetic spec's `name` is the yaml template's display name
+        // ("Hermes Agent" / "OpenClaw Container"). For Connections-tab UX we
+        // want the candidate to surface the user's instance name (e.g. "h",
+        // "claw11"), so override before handing it to registerCapabilities.
+        const instanceName = typeof meta.name === "string" && meta.name
+            ? meta.name
+            : instanceId;
+        const namedSpec = { ...synthSpec, name: instanceName };
+        // The synthetic spec has `tasks: []`, so `resolveProvideEndpoint`
+        // can't compute the actual gateway port and falls back to the yaml's
+        // declared port (e.g. openclaw default 18789). Legacy instances may
+        // have been allocated a different port at creation time (port
+        // collision avoidance). Pass the live `getGatewayPort` so the
+        // capability registry advertises the port consumers can actually
+        // reach.
+        const portOverride = getGatewayPort(instanceId);
+        const { registerCapabilities } = await import("./app/app-manager.js");
+        registerCapabilities(instanceId, namedSpec, portOverride > 0 ? portOverride : undefined);
+    }
+    catch (e) {
+        console.warn(`[legacy-capabilities] register failed for ${instanceId}: ${e?.message ?? e}`);
+    }
+}
+async function unregisterLegacyCapabilitiesTopLevel(instanceId, purge) {
+    try {
+        const { markCapabilitiesStopped, unregisterCapabilities } = await import("./app/app-manager.js");
+        if (purge)
+            unregisterCapabilities(instanceId);
+        else
+            markCapabilitiesStopped(instanceId);
+    }
+    catch (e) {
+        console.warn(`[legacy-capabilities] unregister failed for ${instanceId}: ${e?.message ?? e}`);
+    }
+}
 export async function stopInstance(instanceId, purge = false) {
     const jid = jobId(instanceId);
     try {
@@ -794,10 +999,14 @@ export async function stopInstance(instanceId, purge = false) {
                 }
                 catch { /* ignore */ }
             }
+            await unregisterLegacyCapabilitiesTopLevel(instanceId, purge);
             return { ok: true };
         }
-        if (resp.status === 404)
+        if (resp.status === 404) {
+            // Already stopped — still mark capabilities stopped so the UI reflects state.
+            await unregisterLegacyCapabilitiesTopLevel(instanceId, purge);
             return { ok: false, error: "Instance is not running" };
+        }
         return { ok: false, error: await resp.text() };
     }
     catch (e) {
@@ -828,6 +1037,11 @@ export async function restartInstance(instanceId) {
                 const meta = getInstance(instanceId);
                 const agentType = resolveAgentType(meta);
                 const adapter = getAdapter(agentType);
+                // PR 9 — refresh connection-derived config (mcp_servers /
+                // openclaw.json plugins) from current capability registry so
+                // host IP changes propagate on restart without manual re-bind.
+                // Symmetric with the same call in startInstance.
+                await phaseRefreshConnections(adapter, instanceId);
                 if (adapter.hooks?.onBeforeStart) {
                     await adapter.hooks.onBeforeStart({ instanceId });
                 }
@@ -841,8 +1055,12 @@ export async function restartInstance(instanceId) {
                 TaskName: resolveTaskName(instanceId),
                 AllTasks: false,
             });
-            if (resp.ok)
+            if (resp.ok) {
+                // Re-register capabilities — yaml provides may have changed since
+                // the last start (e.g. a panel upgrade adding `llm-agent`).
+                await registerLegacyCapabilitiesTopLevel(instanceId);
                 return { ok: true, alloc_id: alloc.ID };
+            }
             // Non-2xx from the restart endpoint falls through to stop+start
             const errText = await resp.text();
             console.warn(`[nomad] Native restart failed for ${instanceId} (HTTP ${resp.status}): ${errText} — falling back to stop+start`);
@@ -1005,7 +1223,7 @@ var UnifiedNomadJobs;
     const MAX_MEMORY_MAX_MB = 4096; // 4 GB hard limit
     const DEFAULT_PIDS_LIMIT = 512;
     const NOMAD_CONFIG_PATH = join(JISHUSHELL_HOME, "nomad", "nomad.hcl");
-    const DEFAULT_CWD = homedir();
+    const _DEFAULT_CWD = homedir();
     function appDirForId(appId) {
         return join(APPS_DIR, appId);
     }
@@ -1199,12 +1417,12 @@ var UnifiedNomadJobs;
         if (!s)
             return defaultNs;
         if (s.endsWith("ms"))
-            return parseInt(s) * 1_000_000;
+            return parseInt(s, 10) * 1_000_000;
         if (s.endsWith("s"))
-            return parseInt(s) * 1_000_000_000;
+            return parseInt(s, 10) * 1_000_000_000;
         if (s.endsWith("m"))
-            return parseInt(s) * 60_000_000_000;
-        return parseInt(s) * 1_000_000_000;
+            return parseInt(s, 10) * 60_000_000_000;
+        return parseInt(s, 10) * 1_000_000_000;
     }
     function portLabel(taskName, portName) {
         const sanitize = (value) => value.replace(/[^a-zA-Z0-9_-]/g, "-");
@@ -1225,10 +1443,19 @@ var UnifiedNomadJobs;
     function hostNetworkForPort(port) {
         if ((port.visibility ?? "external") === "internal")
             return undefined;
+        // Honor explicit host_network from spec (e.g. weknora's ports declare
+        // `host_network: docker_bridge` so peer tasks reach the published port
+        // via host.docker.internal). Verify the named network is actually
+        // declared in nomad.hcl — otherwise Nomad rejects job submission with
+        // "unknown host network". Falls back to "external" when unset.
+        const requested = (port.host_network ?? "").trim();
+        if (requested && nomadConfigDeclaresHostNetwork(requested))
+            return requested;
         return nomadConfigDeclaresHostNetwork("external") ? "external" : undefined;
     }
     function specRequiresExternalHostNetwork(spec) {
-        return spec.tasks.some((task) => (task.ports ?? []).some((port) => (port.visibility ?? "external") !== "internal"));
+        return spec.tasks.some((task) => task.runtime !== "container"
+            && (task.ports ?? []).some((port) => (port.visibility ?? "external") !== "internal"));
     }
     async function validateRequiredHostNetworks(spec) {
         if (!specRequiresExternalHostNetwork(spec))
@@ -1268,7 +1495,18 @@ var UnifiedNomadJobs;
             Label: portLabel(task.name, port.name),
             Value: port.host_port ?? port.port,
             ...(task.runtime === "container" ? { To: port.container_port ?? port.port } : {}),
-            ...(hostNetworkForPort(port) ? { HostNetwork: hostNetworkForPort(port) } : {}),
+            // Attach the named host_network for any externally-visible port —
+            // including container tasks. Without it, Nomad falls back to
+            // HostNetwork="default" (loopback) and the docker driver publishes
+            // to 127.0.0.1, breaking cross-container consumers (e.g. OpenWebUI
+            // calling hermes / openclaw via the connections page). The earlier
+            // restriction to non-container tasks was overcautious — our task
+            // groups use host networking (Mode: "") rather than Nomad bridge
+            // mode, so attaching host_network is safe and is exactly what
+            // searxng-container has been doing in practice all along.
+            ...(hostNetworkForPort(port)
+                ? { HostNetwork: hostNetworkForPort(port) }
+                : {}),
         }));
     }
     // ── Health check → Nomad service check builder ────────────────────────────
@@ -1489,12 +1727,6 @@ var UnifiedNomadJobs;
             return null;
         return command.replace(/^~(?=\/|$)/, homedir());
     }
-    function taskCommandLine(task) {
-        const command = expandTaskCommand(task.command);
-        if (!command)
-            return null;
-        return [command, ...(task.args ?? []).map(String)].join(" ").trim();
-    }
     function commandLineMatchesTask(commandLine, task) {
         const normalized = commandLine.trim();
         const command = expandTaskCommand(task.command);
@@ -2013,17 +2245,73 @@ var UnifiedNomadJobs;
             const src = v.source.replace(/^~(?=\/|$)/, homedir());
             return `${src}:${v.target}${v.readonly ? ":ro" : ":rw"}`;
         });
+        // Resolve container task user. On Linux we default to the panel
+        // process's host uid:gid so bind-mounted data dirs (owned by the panel
+        // user, typically `pi`) are writable without forcing the container to
+        // run as root and without needing chown / DAC_OVERRIDE gymnastics.
+        // yaml override `user: "<uid>:<gid>"` wins; explicit `user: "root"` or
+        // `user: "0:0"` keeps the image's root default.
+        //
+        // On macOS we skip the host-uid default. Docker on Mac runs inside a
+        // Linux VM (Colima/Docker Desktop) with its own uid namespace — host
+        // uids like 501 (the standard macOS first-user) are virtualised away
+        // by virtiofs and almost never exist in the image's /etc/passwd. Some
+        // images crash hard when started as an unknown uid: e.g. browserless
+        // calls Node.js `os.userInfo()` very early, which throws
+        // `uv_os_get_passwd returned ENOENT` and the container exits before
+        // the port ever binds. Letting the image's default USER directive
+        // take effect is correct on Mac; users who do need bind-mount
+        // ownership control can still set yaml `user:` explicitly.
+        const containerUser = (() => {
+            if (task.runtime !== "container")
+                return undefined;
+            const declared = typeof task.user === "string" ? task.user.trim() : "";
+            if (declared === "host" || declared === "") {
+                if (process.platform === "darwin")
+                    return undefined;
+                const uid = process.getuid?.() ?? 1000;
+                const gid = process.getgid?.() ?? 1000;
+                return `${uid}:${gid}`;
+            }
+            return declared;
+        })();
+        // Validate cap_add against a tight allowlist. The full Linux cap surface
+        // is large; we only honor capabilities image entrypoints actually need
+        // for the canonical "start as root + gosu to user" pattern. Anything
+        // outside this list is silently dropped — failing closed protects
+        // against typo'd / hostile specs widening the container's capability
+        // bounds beyond what the panel author signed off on.
+        const ALLOWED_CAPS = new Set([
+            "CHOWN", "DAC_OVERRIDE", "FOWNER",
+            "SETUID", "SETGID", "SETPCAP", "NET_BIND_SERVICE",
+        ]);
+        const capAdd = Array.isArray(task.cap_add)
+            ? task.cap_add
+                .map((c) => typeof c === "string" ? c.trim().toUpperCase().replace(/^CAP_/, "") : "")
+                .filter((c) => ALLOWED_CAPS.has(c))
+            : [];
         const taskDef = {
             Name: task.name,
             Driver: "docker",
+            ...(containerUser ? { User: containerUser } : {}),
             Config: {
                 image,
                 force_pull: false,
+                // Nomad's docker driver default `image_pull_timeout` is 5 minutes;
+                // on Raspberry Pi or other constrained networks a 1+ GiB image
+                // (Open WebUI, OpenClaw, Hermes) can exceed that and the alloc
+                // restart-loops with "Failed to pull: context deadline exceeded"
+                // before it ever starts. Raise to 15 minutes — long enough for
+                // realistic Pi-class pulls, short enough that a genuinely
+                // unreachable registry still surfaces as a failure within a
+                // bounded window.
+                image_pull_timeout: "15m",
                 ...(task.command ? { command: String(task.command) } : {}),
                 args,
                 ...(publishedPorts.length > 0 ? { ports: publishedPorts } : {}),
                 extra_hosts: ["host.docker.internal:host-gateway"],
                 cap_drop: ["ALL"],
+                ...(capAdd.length > 0 ? { cap_add: capAdd } : {}),
                 security_opt: ["no-new-privileges"],
                 pids_limit: DEFAULT_PIDS_LIMIT,
                 readonly_rootfs: false,
@@ -2093,13 +2381,17 @@ var UnifiedNomadJobs;
                             ? { Networks: [{ ReservedPorts: groupReservedPorts }] }
                             : {}),
                         RestartPolicy: {
-                            Attempts: 3,
-                            Interval: 300_000_000_000,
+                            // 10 attempts × 15s delay = ~2.5min recovery window. Multi-task
+                            // app groups commonly need 3-5 restarts before sidecar
+                            // dependencies' external port-publish settles.
+                            Attempts: 10,
+                            Interval: 600_000_000_000,
                             Delay: 15_000_000_000,
                             Mode: "fail",
                         },
                         Reschedule: {
-                            Attempts: 0,
+                            Attempts: 3,
+                            Interval: 3_600_000_000_000,
                             Unlimited: false,
                         },
                         Update: {
@@ -2224,6 +2516,7 @@ var UnifiedNomadJobs;
         }
         return statuses[0];
     }
+    UnifiedNomadJobs.aggregateHealthStatus = aggregateHealthStatus;
     async function getRunningAlloc(appId) {
         return pickLiveAlloc(await getAllocs(appId));
     }
@@ -2602,6 +2895,10 @@ var UnifiedNomadJobs;
         if (nomadStopped) {
             const allocsStopped = await waitForAllocationsToStop(liveAllocIds);
             if (!allocsStopped) {
+                const lingeringAlloc = await getRunningAlloc(appId);
+                if (!lingeringAlloc) {
+                    return { ok: true };
+                }
                 return { ok: false, error: `App '${appId}' allocations did not stop in time` };
             }
             return { ok: true };
@@ -3155,16 +3452,87 @@ var UnifiedNomadJobs;
         return getGenericJobStatus(nomadJobId);
     }
     UnifiedNomadJobs.getInstanceStatus = getInstanceStatus;
+    /**
+     * Capability registration shim for **legacy** (non-app-installed)
+     * hermes/openclaw instances. Loads the synthetic spec via
+     * loadCapabilitySpecForLegacyInstance and routes provides through the
+     * app-manager registry helpers, so legacy instances surface in
+     * Connections candidate lists like app-installed ones.
+     *
+     * Errors are swallowed and logged — capability registration is best-
+     * effort; a failure here must not block start/stop.
+     */
+    async function registerLegacyCapabilities(instanceId) {
+        try {
+            const meta = getInstance(instanceId);
+            if (!meta)
+                return;
+            const { loadCapabilitySpecForLegacyInstance } = await import("./runtime/migrations.js");
+            const synthSpec = loadCapabilitySpecForLegacyInstance(meta);
+            if (!synthSpec?.provides?.length)
+                return;
+            // Same instance-name override + portOverride passthrough as
+            // registerLegacyCapabilitiesTopLevel — Connections candidates should
+            // show the user's instance name and advertise the actually-allocated
+            // gateway port (not the yaml default).
+            const instanceName = typeof meta.name === "string" && meta.name
+                ? meta.name
+                : instanceId;
+            const namedSpec = { ...synthSpec, name: instanceName };
+            const portOverride = getGatewayPort(instanceId);
+            const { registerCapabilities } = await import("./app/app-manager.js");
+            registerCapabilities(instanceId, namedSpec, portOverride > 0 ? portOverride : undefined);
+        }
+        catch (e) {
+            console.warn(`[legacy-capabilities] register failed for ${instanceId}: ${e?.message ?? e}`);
+        }
+    }
+    async function unregisterLegacyCapabilities(instanceId, purge) {
+        try {
+            const { markCapabilitiesStopped, unregisterCapabilities } = await import("./app/app-manager.js");
+            if (purge) {
+                unregisterCapabilities(instanceId);
+            }
+            else {
+                markCapabilitiesStopped(instanceId);
+            }
+        }
+        catch (e) {
+            console.warn(`[legacy-capabilities] unregister failed for ${instanceId}: ${e?.message ?? e}`);
+        }
+    }
     async function startInstance(nomadJobId) {
         const instanceBackedApp = await getInstanceBackedInstalledApp(nomadJobId);
         if (instanceBackedApp) {
+            // PR 3 sub-step 3d: switch to resolveConnections in runtime mode so
+            // missing required producers / ambiguous prefix candidates throw with
+            // structured codes (412 / 409 / 400). Read the live instance.json so
+            // UI bindings persisted via PUT /connections (PR 4) drive the
+            // resolution; fall back to a stub `{ connections: {} }` when the
+            // instance file isn't readable yet.
             let extraEnv = {};
             try {
-                const { resolveRequires } = await import("./app/app-manager.js");
-                extraEnv = resolveRequires(instanceBackedApp.spec);
+                const { resolveConnections, resolvedToLegacyEnv } = await import("./connection-resolver.js");
+                const legacyInstanceManager = await import("./instance-manager.js");
+                const meta = legacyInstanceManager.getInstance(nomadJobId);
+                const instance = { connections: meta?.connections ?? {} };
+                // Validate in runtime mode so missing required / ambiguous still throws
+                // with structured error codes (412 / 409 / 400) before we touch Nomad.
+                const { resolved } = resolveConnections(instanceBackedApp.spec, instance, "runtime");
+                // Render the full RUNTIME_HOOKS env (covers llm/search/browser/mcp)
+                // rather than just the default-category subset, so apply: openai-env
+                // consumers (e.g. OpenWebUI) self-heal across provider port changes.
+                const { renderRuntimeConnectionsEnv } = await import("./connection-apply.js");
+                const runtimeEnv = await renderRuntimeConnectionsEnv(instanceBackedApp.spec, { id: nomadJobId, connections: instance.connections });
+                extraEnv = { ...resolvedToLegacyEnv(resolved), ...runtimeEnv };
             }
             catch (e) {
-                return { ok: false, error: e.message };
+                return {
+                    ok: false,
+                    error: e.message,
+                    ...(e.code ? { code: e.code } : {}),
+                    ...(typeof e.statusCode === "number" ? { statusCode: e.statusCode } : {}),
+                };
             }
             const depCheck = await checkDependencies(instanceBackedApp.spec);
             if (!depCheck.ok) {
@@ -3189,10 +3557,17 @@ var UnifiedNomadJobs;
             return { ok: false, error: `App '${nomadJobId}' 必须通过 app-manager 启动` };
         }
         if (existsSync(instanceMetaPath(nomadJobId))) {
-            return instanceScheduler.startInstance(nomadJobId);
+            const result = await instanceScheduler.startInstance(nomadJobId);
+            if (result.ok)
+                await registerLegacyCapabilities(nomadJobId);
+            return result;
         }
         if (nomadJobId.startsWith(OPENCLAW_PREFIX)) {
-            return instanceScheduler.startInstance(nomadJobId.slice(OPENCLAW_PREFIX.length));
+            const inner = nomadJobId.slice(OPENCLAW_PREFIX.length);
+            const result = await instanceScheduler.startInstance(inner);
+            if (result.ok)
+                await registerLegacyCapabilities(inner);
+            return result;
         }
         if (!isAppJob(nomadJobId)) {
             return { ok: false, error: `Cannot start unmanaged job "${nomadJobId}"` };
@@ -3213,10 +3588,19 @@ var UnifiedNomadJobs;
             return { ok: false, error: `App '${nomadJobId}' 必须通过 app-manager 停止` };
         }
         if (existsSync(instanceMetaPath(nomadJobId))) {
-            return instanceScheduler.stopInstance(nomadJobId, purge);
+            const result = await instanceScheduler.stopInstance(nomadJobId, purge);
+            if (result.ok || result.error?.includes("not running") || result.error?.includes("not found")) {
+                await unregisterLegacyCapabilities(nomadJobId, purge);
+            }
+            return result;
         }
         if (nomadJobId.startsWith(OPENCLAW_PREFIX)) {
-            return instanceScheduler.stopInstance(nomadJobId.slice(OPENCLAW_PREFIX.length), purge);
+            const inner = nomadJobId.slice(OPENCLAW_PREFIX.length);
+            const result = await instanceScheduler.stopInstance(inner, purge);
+            if (result.ok || result.error?.includes("not running") || result.error?.includes("not found")) {
+                await unregisterLegacyCapabilities(inner, purge);
+            }
+            return result;
         }
         try {
             const resp = await nomadDelete(`/v1/job/${nomadJobId}?purge=${purge}`);
@@ -3239,10 +3623,17 @@ var UnifiedNomadJobs;
             return { ok: false, error: `App '${nomadJobId}' 必须通过 app-manager 重启` };
         }
         if (existsSync(instanceMetaPath(nomadJobId))) {
-            return instanceScheduler.restartInstance(nomadJobId);
+            const result = await instanceScheduler.restartInstance(nomadJobId);
+            if (result.ok)
+                await registerLegacyCapabilities(nomadJobId);
+            return result;
         }
         if (nomadJobId.startsWith(OPENCLAW_PREFIX)) {
-            return instanceScheduler.restartInstance(nomadJobId.slice(OPENCLAW_PREFIX.length));
+            const inner = nomadJobId.slice(OPENCLAW_PREFIX.length);
+            const result = await instanceScheduler.restartInstance(inner);
+            if (result.ok)
+                await registerLegacyCapabilities(inner);
+            return result;
         }
         if (!isAppJob(nomadJobId)) {
             return { ok: false, error: `Cannot restart unmanaged job "${nomadJobId}"` };
@@ -3346,4 +3737,6 @@ export const shouldAutoStartNomadJob = UnifiedNomadJobs.shouldAutoStart;
 export const startNomadJobInstance = UnifiedNomadJobs.startInstance;
 export const stopNomadJobInstance = UnifiedNomadJobs.stopInstance;
 export const restartNomadJobInstance = UnifiedNomadJobs.restartInstance;
+// @internal — exposed for Phase 10.4 unit testing only.
+export const __aggregateHealthStatusForTests = UnifiedNomadJobs.aggregateHealthStatus;
 //# sourceMappingURL=nomad-manager.js.map