jishushell 0.6.5 → 0.6.18

package/dist/services/app-common/status-refresh.js

@@ -0,0 +1,759 @@
+import { connect as netConnect } from "node:net";
+import { getCorePort, getInternalMcpToken } from "../../config.js";
+import { CanonicalRuntimePortRequiredError, getPrimaryHostPort, getHostForAppPort, isCanonicalRuntimePortRequiredError, } from "../instances/manager.js";
+import * as capabilityRegistry from "../capabilities/registry.js";
+import { resolveProvideEndpoint } from "./provide-resolver.js";
+import { advanceInstanceOperation, clearInstanceOperation, readCanonicalInstance, readCanonicalSpecForInstance, updateInstanceStatus, } from "./instance-store.js";
+const DEFAULT_REFRESH_MIN_INTERVAL_MS = 1_000;
+const WAIT_POLL_INTERVAL_MS = 1_000;
+const START_WAIT_TIMEOUT_MS = 120_000;
+const STOP_WAIT_TIMEOUT_MS = 60_000;
+const PROBE_TIMEOUT_MS = 5_000;
+const WHOLE_REFRESH_TIMEOUT_MS = 30_000;
+const inFlightRefreshes = new Map();
+const cachedRefreshes = new Map();
+function nowIso() {
+    return new Date().toISOString();
+}
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+function parseDurationMs(value) {
+    if (typeof value !== "string")
+        return null;
+    const trimmed = value.trim();
+    if (!trimmed)
+        return null;
+    const match = /^(\d+(?:\.\d+)?)(ms|s|m|h)?$/.exec(trimmed);
+    if (!match)
+        return null;
+    const amount = Number(match[1]);
+    if (!Number.isFinite(amount) || amount < 0)
+        return null;
+    const unit = match[2] ?? "ms";
+    const multiplier = unit === "h"
+        ? 3_600_000
+        : unit === "m"
+            ? 60_000
+            : unit === "s"
+                ? 1_000
+                : 1;
+    return Math.floor(amount * multiplier);
+}
+function isWithinStartupHealthGrace(instance, nowMs = Date.now()) {
+    if (instance.operation?.type !== "start" && instance.operation?.type !== "restart")
+        return false;
+    const graceMs = parseDurationMs(instance.runtime?.health?.start_period);
+    if (!graceMs || graceMs <= 0)
+        return false;
+    const startedAtMs = Date.parse(instance.operation.startedAt);
+    if (!Number.isFinite(startedAtMs))
+        return false;
+    return nowMs - startedAtMs < graceMs;
+}
+function buildObservedServiceSnapshot(instance) {
+    return {
+        status: mapObservedPhaseToServiceStatus(instance.status.phase),
+        pid: null,
+        uptime: null,
+        memory_mb: null,
+        cpu_percent: null,
+    };
+}
+function buildUnknownServiceSnapshot() {
+    return {
+        status: "unknown",
+        pid: null,
+        uptime: null,
+        memory_mb: null,
+        cpu_percent: null,
+    };
+}
+function buildPersistedSnapshot(instance) {
+    return {
+        instance,
+        rawService: buildUnknownServiceSnapshot(),
+        service: buildObservedServiceSnapshot(instance),
+    };
+}
+function pruneCachedRefresh(instanceId) {
+    cachedRefreshes.delete(instanceId);
+}
+async function withWholeRefreshTimeout(task, timeoutMs) {
+    let timer = null;
+    try {
+        return await Promise.race([
+            task,
+            new Promise((resolve) => {
+                timer = setTimeout(() => resolve({ timedOut: true }), timeoutMs);
+                timer.unref?.();
+            }),
+        ]);
+    }
+    finally {
+        if (timer)
+            clearTimeout(timer);
+    }
+}
+function mapObservedPhaseToServiceStatus(phase) {
+    switch (phase) {
+        case "ready":
+            return "running";
+        case "starting":
+            return "pending";
+        case "stopped":
+            return "stopped";
+        case "failed":
+            return "failed";
+        default:
+            return "unknown";
+    }
+}
+function toInstanceStatus(raw) {
+    return {
+        status: typeof raw?.status === "string" ? raw.status : "unknown",
+        pid: typeof raw?.pid === "number" ? raw.pid : null,
+        uptime: typeof raw?.uptime === "number" ? raw.uptime : null,
+        memory_mb: typeof raw?.memory_mb === "number" ? raw.memory_mb : null,
+        cpu_percent: typeof raw?.cpu_percent === "number" ? raw.cpu_percent : null,
+    };
+}
+async function loadRawServiceStatus(instanceId) {
+    const appService = await import("./service.js");
+    return await appService.getInstanceInstallRecordStatus(instanceId);
+}
+/**
+ * Re-project this instance's capability-registry entries from the
+ * current runtime status.
+ *
+ * Status observation projects runtime state into the derived capability
+ * registry. Observation is event-driven: lifecycle waits, explicit status
+ * APIs, and one-shot boot convergence invoke this path. Two boundary writers
+ * remain outside this function:
+ *   1. `delete-service.uninstallInstance` — deletes instance metadata; the
+ *      observer cannot iterate over a deleted instance to clean up its
+ *      stale registry entries.
+ *   2. `server.ts` boot-time `refreshCapabilityRegistry({force:true})`
+ *      — one-shot rebuild so reverse-proxy routes are populated immediately
+ *      after Core restart.
+ *
+ * Errors are swallowed: a failed sync is retried by the next lifecycle or
+ * API-triggered observation. Status reporting must not fail because the
+ * registry is briefly stale.
+ */
+async function reconcileCapabilityRegistryForInstance(instanceId, runtimeStatus) {
+    try {
+        const appService = await import("./service.js");
+        await appService.syncCapabilitiesForApp(instanceId, { runtimeStatus });
+    }
+    catch (error) {
+        if (isCanonicalRuntimePortRequiredError(error))
+            throw error;
+        console.warn(`[lifecycle-status] capability-sync failed for ${instanceId}: ${error?.message ?? error}`);
+    }
+}
+function capabilityStatusFromRuntime(status) {
+    if (status === "running")
+        return "running";
+    if (status === "stopped" || status === "dead" || status === "complete")
+        return "stopped";
+    return "unknown";
+}
+function observedStatusChanged(current, next) {
+    return current.phase !== next.phase
+        || current.reason !== next.reason
+        || current.message !== next.message;
+}
+function getTaskHealthSummary(rawStatus) {
+    let hasUnhealthy = false;
+    let hasUnknown = false;
+    let hasChecks = false;
+    const tasks = rawStatus?.tasks && typeof rawStatus.tasks === "object" ? Object.values(rawStatus.tasks) : [];
+    for (const task of tasks) {
+        const healthStatus = typeof task?.health_status === "string"
+            ? String(task.health_status)
+            : "";
+        if (!healthStatus)
+            continue;
+        hasChecks = true;
+        if (healthStatus === "unhealthy")
+            hasUnhealthy = true;
+        else if (healthStatus !== "healthy")
+            hasUnknown = true;
+    }
+    return { hasUnhealthy, hasUnknown, hasChecks };
+}
+function resolveRuntimeProbePort(instanceId, runtime, portRef) {
+    const ports = Array.isArray(runtime.ports) ? runtime.ports : [];
+    const matched = ports.find((entry) => Number(entry?.hostPort) === portRef || Number(entry?.containerPort) === portRef);
+    if (matched && Number.isInteger(matched.hostPort) && matched.hostPort > 0) {
+        return matched.hostPort;
+    }
+    throw new CanonicalRuntimePortRequiredError(instanceId);
+}
+function externalHttpProvides(spec) {
+    return (spec.provides ?? []).filter((provide) => {
+        if ((provide.visibility ?? "external") === "internal")
+            return false;
+        if (typeof provide.url === "string" && provide.url.trim())
+            return false;
+        const protocol = String(provide.protocol ?? "http").toLowerCase();
+        return protocol === "http" || protocol === "https";
+    });
+}
+function selectHttpProvide(spec) {
+    return externalHttpProvides(spec)[0] ?? null;
+}
+function selectCoreProxyProvide(spec) {
+    if (!spec)
+        return null;
+    const provides = externalHttpProvides(spec);
+    return provides.find((provide) => isUiProvide(provide)) ?? provides[0] ?? null;
+}
+function normalizeProbePath(path) {
+    const trimmed = path.trim();
+    if (!trimmed)
+        return "/";
+    return trimmed.startsWith("/") ? trimmed : `/${trimmed}`;
+}
+function coreProxyProbeSuffix(provide) {
+    const health = provide.health;
+    if (health
+        && (health.kind === undefined || health.kind === "http")
+        && typeof health.path === "string"
+        && health.path.trim()) {
+        return normalizeProbePath(health.path);
+    }
+    return "/";
+}
+function coreProxyProbeUrl(instanceId, capability, suffix) {
+    const normalizedSuffix = normalizeProbePath(suffix);
+    const wildcardSuffix = normalizedSuffix === "/" ? "" : normalizedSuffix.replace(/^\/+/, "");
+    return `http://127.0.0.1:${getCorePort()}/api/instances/${encodeURIComponent(instanceId)}/provides/${encodeURIComponent(capability)}/${wildcardSuffix}`;
+}
+/**
+ * Step 4 hard requirement: `status.phase=ready` may only be written when
+ * the user can actually reach the app *through Core*, not merely when the
+ * upstream `hostPort` happens to accept TCP/HTTP.
+ *
+ * The Core reverse proxies in `src/server.ts` (`/apps/:slug/*`) and
+ * `src/routes/instances.ts` (`/api/instances/:id/proxy/:capability/*`)
+ * both resolve against the capability registry and dispatch only to
+ * entries with `status === "running"` and non-internal visibility.
+ * The registry entry is only the routable precondition:
+ *   - the instance's spec declares at least one external provide, AND
+ *   - the registry currently lists at least one entry for this instance
+ *     with `status === "running"` and `visibility !== "internal"`.
+ *
+ * UI provides are preferred over API provides. This prevents an API auth
+ * boundary from proving ready while the user's actual Panel entry is broken.
+ * A spec with no local external HTTP provide is vacuously true.
+ *
+ * Then Core performs a real self HTTP request through the capability proxy
+ * using the internal token. If the selected provide declares
+ * `provides[].health.path`, that path is probed through the proxy; otherwise
+ * the capability base path is probed.
+ * This route exercises the same reverse proxy data path users hit from
+ * Panel. It must not be replaced with a direct upstream request: direct
+ * hostPort reachability is only one readiness rung, while `ready` means the
+ * Core proxy path succeeds too.
+ *
+ * Forbidden alternative (per architecture decision §10 Step 4):
+ *   - Treating `hostPort` TCP reachable as Core proxy reachable.
+ *   - Treating Nomad allocation running as Core proxy reachable.
+ */
+async function probeCoreProxy(instanceId, spec) {
+    const provide = selectCoreProxyProvide(spec);
+    if (!provide)
+        return { ready: true };
+    const entries = capabilityRegistry.listProvidersForInstance(instanceId);
+    const entry = entries.find((entry) => entry.capability === provide.capability &&
+        entry.status === "running" &&
+        (entry.visibility ?? "external") !== "internal" &&
+        (entry.protocol === "http" || entry.protocol === "https"));
+    if (!entry) {
+        return {
+            ready: false,
+            reason: "route-pending",
+            capability: provide.capability,
+        };
+    }
+    const url = coreProxyProbeUrl(instanceId, entry.capability, coreProxyProbeSuffix(provide));
+    try {
+        const response = await fetch(url, {
+            method: "GET",
+            headers: {
+                "X-Jishushell-Internal-Token": getInternalMcpToken(),
+                "X-Jishushell-Instance": instanceId,
+            },
+            signal: AbortSignal.timeout(PROBE_TIMEOUT_MS),
+        });
+        if (isCoreProxyProbeSuccess(entry, response.status)) {
+            return { ready: true, capability: entry.capability, status: response.status };
+        }
+        return {
+            ready: false,
+            reason: "response-not-ready",
+            capability: entry.capability,
+            status: response.status,
+        };
+    }
+    catch {
+        return {
+            ready: false,
+            reason: "request-failed",
+            capability: entry.capability,
+        };
+    }
+}
+function isUiCapability(entry) {
+    return entry.kind === "ui"
+        || entry.capability.endsWith("-ui")
+        || entry.capability.startsWith("web-")
+        || entry.capability.endsWith("-web");
+}
+function isCoreProxyProbeSuccess(entry, status) {
+    return isHttpProbeStatusReady(status, !isUiCapability(entry));
+}
+function isHttpProbeStatusReady(status, allowAuthBoundary) {
+    // User-facing UI readiness requires a page that can actually be opened, so
+    // only 2xx/3xx qualify. API capabilities may be alive but protected; for
+    // those, 401/403 means Core reached a real upstream auth boundary. 400/404
+    // are request/path errors and must never be treated as ready.
+    if (status >= 200 && status < 400)
+        return true;
+    return allowAuthBoundary && (status === 401 || status === 403);
+}
+async function buildReadinessProbe(instance, spec) {
+    const runtime = instance.runtime && typeof instance.runtime === "object" ? instance.runtime : {};
+    const runtimeHealth = runtime.health && typeof runtime.health === "object" ? runtime.health : null;
+    if (runtimeHealth?.type === "http" && Number.isInteger(runtimeHealth.port) && runtimeHealth.port > 0) {
+        const hostPort = resolveRuntimeProbePort(instance.id, runtime, Number(runtimeHealth.port));
+        const host = await getHostForAppPort(instance.id, hostPort);
+        return {
+            kind: "http",
+            host,
+            port: hostPort,
+            path: typeof runtimeHealth.path === "string" && runtimeHealth.path.trim() ? runtimeHealth.path.trim() : "/",
+        };
+    }
+    if (runtimeHealth?.type === "tcp" && Number.isInteger(runtimeHealth.port) && runtimeHealth.port > 0) {
+        const hostPort = resolveRuntimeProbePort(instance.id, runtime, Number(runtimeHealth.port));
+        const host = await getHostForAppPort(instance.id, hostPort);
+        return { kind: "tcp", host, port: hostPort };
+    }
+    if (spec) {
+        const provide = selectHttpProvide(spec);
+        if (provide) {
+            const endpoint = resolveProvideEndpoint(instance.id, spec, provide);
+            if (endpoint) {
+                const host = await getHostForAppPort(instance.id, endpoint.hostPort);
+                return {
+                    kind: "http",
+                    host,
+                    port: endpoint.hostPort,
+                    path: endpoint.path ?? "/",
+                    allowAuthBoundary: !isUiProvide(provide),
+                };
+            }
+        }
+    }
+    const ports = Array.isArray(runtime.ports) ? runtime.ports : [];
+    const primaryPort = ports.find((entry) => (entry?.visibility ?? "external") !== "internal"
+        && Number.isInteger(entry?.hostPort)
+        && entry.hostPort > 0) ?? ports.find((entry) => Number.isInteger(entry?.hostPort) && entry.hostPort > 0);
+    if (primaryPort) {
+        const hostPort = Number(primaryPort.hostPort);
+        const host = await getHostForAppPort(instance.id, hostPort);
+        return { kind: "tcp", host, port: hostPort };
+    }
+    return null;
+}
+async function probeTcp(host, port, timeoutMs = PROBE_TIMEOUT_MS) {
+    return await new Promise((resolve) => {
+        const socket = netConnect({ host, port });
+        let settled = false;
+        const finish = (value) => {
+            if (settled)
+                return;
+            settled = true;
+            socket.destroy();
+            resolve(value);
+        };
+        socket.setTimeout(timeoutMs);
+        socket.once("connect", () => finish(true));
+        socket.once("timeout", () => finish(false));
+        socket.once("error", () => finish(false));
+        socket.once("close", () => finish(false));
+    });
+}
+function isUiProvide(provide) {
+    return provide.embedded === "auto"
+        || provide.embedded === "proxy"
+        || provide.embedded === "direct"
+        || provide.kind === "ui"
+        || provide.capability.endsWith("-ui")
+        || provide.capability.startsWith("web-")
+        || provide.capability.endsWith("-web");
+}
+async function probeHttp(host, port, path, allowAuthBoundary = false, timeoutMs = PROBE_TIMEOUT_MS) {
+    const normalizedPath = path.startsWith("/") ? path : `/${path}`;
+    const url = `http://${host.includes(":") ? `[${host}]` : host}:${port}${normalizedPath}`;
+    try {
+        const response = await fetch(url, {
+            method: "GET",
+            headers: {
+                "X-Forwarded-For": "127.0.0.1",
+                "X-Real-IP": "127.0.0.1",
+            },
+            signal: AbortSignal.timeout(timeoutMs),
+        });
+        return isHttpProbeStatusReady(response.status, allowAuthBoundary);
+    }
+    catch {
+        return false;
+    }
+}
+async function isProbeReady(probe) {
+    if (!probe)
+        return true;
+    return probe.kind === "http"
+        ? await probeHttp(probe.host, probe.port, probe.path, probe.allowAuthBoundary === true)
+        : await probeTcp(probe.host, probe.port);
+}
+function coreProxyNotReadyStatus(probe) {
+    switch (probe.reason) {
+        case "route-pending":
+            return {
+                reason: "core-proxy-route-pending",
+                message: "Endpoint is up, but the Core proxy route has not been published yet.",
+            };
+        case "request-failed":
+            return {
+                reason: "core-proxy-request-failed",
+                message: "Endpoint is up, but the Core proxy probe could not reach the instance yet.",
+            };
+        case "response-not-ready":
+            return {
+                reason: "core-proxy-response-not-ready",
+                message: `Endpoint is up, but the Core proxy returned ${typeof probe.status === "number" ? `HTTP ${probe.status}` : "a non-ready response"}.`,
+            };
+        default:
+            return {
+                reason: "core-proxy-request-failed",
+                message: "Endpoint is up, but the Core proxy probe has not succeeded yet.",
+            };
+    }
+}
+function observeInstanceStatus(instance, rawService, readinessOk, coreProxyProbe) {
+    // The observer is the single source of truth for the user-facing
+    // `status.phase`. Each branch below answers exactly one question:
+    //   - Did the runtime fail outright? → failed.
+    //   - Is the runtime not yet runnable (pending/installing/stopped while
+    //     the user is starting)? → starting.
+    //   - Is the runtime up but not yet usable? Walk the readiness ladder
+    //     in this exact order:
+    //       1. health checks — Nomad-reported, cheapest signal
+    //       2. endpoint probe — confirms the published hostPort accepts a
+    //          connection (NOT equivalent to Core proxy reachable)
+    //       3. Core proxy reachable — confirms the user can reach the app
+    //          *through Core*; this is the Step 4 hard requirement
+    //   - Only when all three are clean do we write `ready`.
+    //
+    // Note: a later reconciler tick may legitimately move a `failed` row
+    // back into `starting`/`ready` if the runtime recovers. That is the
+    // intended "status reflects observation, not history" semantics — do
+    // NOT try to make `failed` sticky here.
+    const observedAt = nowIso();
+    const rawStatus = String(rawService.status ?? "unknown");
+    const { hasUnhealthy, hasUnknown, hasChecks } = getTaskHealthSummary(rawService);
+    const isWithinHealthGrace = isWithinStartupHealthGrace(instance);
+    if (rawStatus === "needs-migration") {
+        return {
+            phase: "failed",
+            reason: "needs-migration",
+            message: "Instance metadata is not canonical. Run repair or migration before using this instance.",
+            lastObservedAt: observedAt,
+        };
+    }
+    if (rawStatus === "failed") {
+        return {
+            phase: "failed",
+            reason: "runtime-failed",
+            ...(typeof rawService.error === "string" && rawService.error.trim() ? { message: rawService.error.trim() } : {}),
+            lastObservedAt: observedAt,
+        };
+    }
+    if (rawStatus === "stopped" || rawStatus === "dead" || rawStatus === "complete") {
+        if (instance.operation?.type === "start") {
+            return {
+                phase: "starting",
+                reason: "submit-pending",
+                message: "Start has been requested, but the runtime is not runnable yet.",
+                lastObservedAt: observedAt,
+            };
+        }
+        if (instance.operation?.type === "restart") {
+            if (isRestartStopPhase(instance.operation.phase)) {
+                return {
+                    phase: "stopped",
+                    reason: "restart-stop-complete",
+                    lastObservedAt: observedAt,
+                };
+            }
+            return {
+                phase: "starting",
+                reason: "submit-pending",
+                message: "Restart is in progress, but the runtime is not runnable yet.",
+                lastObservedAt: observedAt,
+            };
+        }
+        if (instance.desiredState === "running" && (!instance.operation || instance.operation.type !== "stop")) {
+            return {
+                phase: "failed",
+                reason: "desired-running-but-stopped",
+                message: "Desired state is running, but no live workload is present.",
+                lastObservedAt: observedAt,
+            };
+        }
+        return {
+            phase: "stopped",
+            reason: instance.desiredState === "stopped" ? "desired-stopped" : "runtime-stopped",
+            lastObservedAt: observedAt,
+        };
+    }
+    if (rawStatus === "pending" || rawStatus === "installing") {
+        return {
+            phase: "starting",
+            reason: "allocation-pending",
+            message: "Runtime allocation has not become runnable yet.",
+            lastObservedAt: observedAt,
+        };
+    }
+    if (rawStatus === "running") {
+        if (hasUnhealthy) {
+            if (isWithinHealthGrace) {
+                return {
+                    phase: "starting",
+                    reason: "health-pending",
+                    message: "Runtime is up, but health checks are still settling.",
+                    lastObservedAt: observedAt,
+                };
+            }
+            return {
+                phase: "failed",
+                reason: "health-unhealthy",
+                message: "Runtime reported unhealthy health checks.",
+                lastObservedAt: observedAt,
+            };
+        }
+        if (hasChecks && hasUnknown) {
+            return {
+                phase: "starting",
+                reason: "health-pending",
+                message: "Runtime is up, but health checks are still settling.",
+                lastObservedAt: observedAt,
+            };
+        }
+        if (!readinessOk) {
+            return {
+                phase: "starting",
+                reason: "endpoint-not-ready",
+                message: "Runtime is up, but the published endpoint is not reachable yet.",
+                lastObservedAt: observedAt,
+            };
+        }
+        if (!coreProxyProbe.ready) {
+            const coreProxyStatus = coreProxyNotReadyStatus(coreProxyProbe);
+            return {
+                phase: "starting",
+                reason: coreProxyStatus.reason,
+                message: coreProxyStatus.message,
+                lastObservedAt: observedAt,
+            };
+        }
+        return {
+            phase: "ready",
+            reason: "endpoint-ready",
+            lastObservedAt: observedAt,
+        };
+    }
+    return {
+        phase: "unknown",
+        reason: "runtime-unknown",
+        ...(typeof rawService.error === "string" && rawService.error.trim() ? { message: rawService.error.trim() } : {}),
+        lastObservedAt: observedAt,
+    };
+}
+function isRestartStopPhase(phase) {
+    return phase === "restart.waitStop"
+        || phase.startsWith("restart.stop")
+        || phase === "stop.waitStop";
+}
+function buildServiceSnapshot(rawService, observed) {
+    return {
+        ...rawService,
+        status: mapObservedPhaseToServiceStatus(observed.phase),
+    };
+}
+function maybeAdvanceStartOperation(instance, observed) {
+    if (!instance.operation)
+        return;
+    if (instance.operation.type !== "start" && instance.operation.type !== "restart")
+        return;
+    if (observed.phase !== "starting")
+        return;
+    if (instance.operation.type === "restart" && isRestartStopPhase(instance.operation.phase))
+        return;
+    const nextPhase = observed.reason === "allocation-pending"
+        ? `${instance.operation.type}.waitAllocation`
+        : `${instance.operation.type}.waitReady`;
+    if (instance.operation.phase !== nextPhase) {
+        advanceInstanceOperation(instance.id, nextPhase);
+    }
+}
+function maybeFinalizeOperation(instance, observed) {
+    if (!instance.operation)
+        return;
+    if (observed.phase === "ready" && (instance.operation.type === "start" || instance.operation.type === "restart")) {
+        clearInstanceOperation(instance.id);
+        return;
+    }
+    if (observed.phase === "stopped" && instance.operation.type === "stop") {
+        clearInstanceOperation(instance.id);
+    }
+}
+async function performRefresh(instanceId) {
+    const instance = readCanonicalInstance(instanceId);
+    if (!instance)
+        return null;
+    const spec = readCanonicalSpecForInstance(instanceId);
+    const rawStatusRecord = await loadRawServiceStatus(instanceId);
+    const rawService = toInstanceStatus(rawStatusRecord);
+    let observed;
+    try {
+        if (Array.isArray(instance.integrations) && instance.integrations.length > 0) {
+            getPrimaryHostPort(instanceId);
+        }
+        const probe = await buildReadinessProbe(instance, spec);
+        const readinessOk = await isProbeReady(probe);
+        // Project the derived capability registry from the same runtime snapshot
+        // this status observation already loaded. This avoids a second Nomad read
+        // and keeps the registry tied to explicit lifecycle/API observations.
+        //
+        // Why before the Core proxy probe: the registry rewrite below supplies
+        // the routable provider entry that the self HTTP probe goes through. If
+        // we observed in the other order the gate would lag by exactly one tick
+        // on every state transition.
+        await reconcileCapabilityRegistryForInstance(instanceId, capabilityStatusFromRuntime(rawStatusRecord.status));
+        const coreProxyProbe = await probeCoreProxy(instanceId, spec);
+        observed = observeInstanceStatus(instance, rawStatusRecord, readinessOk, coreProxyProbe);
+    }
+    catch (error) {
+        if (!isCanonicalRuntimePortRequiredError(error))
+            throw error;
+        observed = {
+            phase: "failed",
+            reason: "needs-migration",
+            message: error.message,
+            lastObservedAt: nowIso(),
+        };
+    }
+    const shouldClearLastError = observed.phase === "ready" && instance.lastError;
+    const statusChanged = observedStatusChanged(instance.status, observed);
+    const shouldPersistStatus = statusChanged || shouldClearLastError;
+    const persisted = shouldPersistStatus
+        ? updateInstanceStatus(instanceId, observed)
+        : { ...instance, status: observed };
+    if (!persisted) {
+        pruneCachedRefresh(instanceId);
+        return null;
+    }
+    maybeAdvanceStartOperation(persisted, observed);
+    maybeFinalizeOperation(persisted, observed);
+    const latestPersisted = readCanonicalInstance(instanceId);
+    const latest = latestPersisted && !shouldPersistStatus
+        ? { ...latestPersisted, status: observed }
+        : latestPersisted;
+    if (!latest) {
+        pruneCachedRefresh(instanceId);
+        return null;
+    }
+    const service = buildServiceSnapshot(rawService, observed);
+    return {
+        instance: latest,
+        rawService,
+        service,
+    };
+}
+export async function refreshInstanceStatus(instanceId, options = {}) {
+    const current = readCanonicalInstance(instanceId);
+    if (!current) {
+        pruneCachedRefresh(instanceId);
+        return null;
+    }
+    const minIntervalMs = options.force ? 0 : (options.minIntervalMs ?? DEFAULT_REFRESH_MIN_INTERVAL_MS);
+    const cached = cachedRefreshes.get(instanceId);
+    if (cached && Date.now() - cached.observedAt < minIntervalMs) {
+        return cached.snapshot;
+    }
+    const existing = inFlightRefreshes.get(instanceId);
+    if (existing)
+        return existing;
+    const task = (async () => {
+        try {
+            const result = await withWholeRefreshTimeout(performRefresh(instanceId), WHOLE_REFRESH_TIMEOUT_MS);
+            if (result.timedOut === true) {
+                console.warn(`[lifecycle-status] whole refresh timed out for ${instanceId} after ${WHOLE_REFRESH_TIMEOUT_MS}ms`);
+                const latest = readCanonicalInstance(instanceId);
+                if (!latest) {
+                    pruneCachedRefresh(instanceId);
+                    return null;
+                }
+                const snapshot = buildPersistedSnapshot(latest);
+                cachedRefreshes.set(instanceId, { snapshot, observedAt: Date.now() });
+                return snapshot;
+            }
+            const snapshot = result;
+            if (snapshot) {
+                cachedRefreshes.set(instanceId, { snapshot, observedAt: Date.now() });
+            }
+            else {
+                pruneCachedRefresh(instanceId);
+            }
+            return snapshot;
+        }
+        finally {
+            inFlightRefreshes.delete(instanceId);
+        }
+    })();
+    inFlightRefreshes.set(instanceId, task);
+    return await task;
+}
+export async function waitForObservedStatus(instanceId, target, options = { reason: "manual" }) {
+    const timeoutMs = options.timeoutMs ?? (target === "ready" ? START_WAIT_TIMEOUT_MS : STOP_WAIT_TIMEOUT_MS);
+    const deadline = Date.now() + timeoutMs;
+    while (Date.now() < deadline) {
+        const snapshot = await refreshInstanceStatus(instanceId, {
+            reason: options.reason,
+            force: true,
+            minIntervalMs: 0,
+        });
+        if (!snapshot)
+            return null;
+        if (snapshot.instance.status.phase === target)
+            return snapshot;
+        if (snapshot.instance.status.phase === "failed")
+            return snapshot;
+        await sleep(WAIT_POLL_INTERVAL_MS);
+    }
+    return await refreshInstanceStatus(instanceId, {
+        reason: options.reason,
+        force: true,
+        minIntervalMs: 0,
+    });
+}
+export const __observeInstanceStatusForTests = observeInstanceStatus;
+export const __coreProxyProbeSuffixForTests = coreProxyProbeSuffix;
+export const __coreProxyProbeUrlForTests = coreProxyProbeUrl;
+//# sourceMappingURL=status-refresh.js.map