@empire-builder-kit/nx 1.0.0-rc.44 → 1.0.0-rc.45

package/src/local-dev/dev-command-core.js

@@ -1,12 +1,59 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.FLEET_OPERATION_TARGET_MS = void 0;
+exports.fleetOperationProgressReporter = fleetOperationProgressReporter;
+exports.fleetOperationFailureDetail = fleetOperationFailureDetail;
+exports.foregroundUpSettlement = foregroundUpSettlement;
 exports.executeFleetUpCore = executeFleetUpCore;
 const node_crypto_1 = require("node:crypto");
 const environment_inspector_js_1 = require("./environment-inspector.js");
 const errors_js_1 = require("./errors.js");
 const fleet_launcher_js_1 = require("./fleet-launcher.js");
 const nx_workspace_js_1 = require("./nx-workspace.js");
-function foregroundWait(launch, command, signal, idempotencyKey, reportCleanupFailure) {
+/** Aspirational duration targets per Fleet command, taken from the measured
+ * two-Slice figures in docs/local-development.md (fresh up 334-442 s, down
+ * 242-345 s, restart about 70-85 s). Exceeding one prints a "slower than
+ * target" line; it never fails or stops observing an operation. */
+exports.FLEET_OPERATION_TARGET_MS = Object.freeze({
+    down: 350_000,
+    restart: 90_000,
+    up: 450_000,
+});
+function progressClock(milliseconds) {
+    const seconds = Math.max(0, Math.floor(milliseconds / 1_000));
+    return `${Math.floor(seconds / 60)}:${String(seconds % 60).padStart(2, '0')}`;
+}
+function describeSliceProgress(slice) {
+    const details = [
+        ...(slice.cleanup === 'failed' ? ['cleanup failed'] : []),
+        ...(slice.restartAttempts > 0
+            ? [`restart attempt ${slice.restartAttempts}`]
+            : []),
+    ];
+    return `${slice.name} ${slice.routeState}${details.length > 0 ? ` (${details.join(', ')})` : ''}`;
+}
+/** Progress lines for a watched Fleet operation; the first report past the
+ * command's target says so once, and observation continues. */
+function fleetOperationProgressReporter(command, report) {
+    const targetMs = exports.FLEET_OPERATION_TARGET_MS[command];
+    let exceeded = false;
+    return ({ elapsedMs, operation, slices }) => {
+        const summary = slices.length > 0
+            ? slices.map(describeSliceProgress).join(', ')
+            : `operation ${operation.state}`;
+        if (!exceeded && elapsedMs > targetMs) {
+            exceeded = true;
+            report(`[ebk dev ${command}] ${progressClock(elapsedMs)} slower than the ${progressClock(targetMs)} target for ${command}; still running: ${summary}`);
+            return;
+        }
+        report(`[ebk dev ${command}] ${progressClock(elapsedMs)} ${summary}`);
+    };
+}
+/** Human cause for a terminal failed operation record. */
+function fleetOperationFailureDetail(operation) {
+    return operation.failureMessage ? ` Cause: ${operation.failureMessage}` : '';
+}
+function foregroundWait(launch, command, signal, idempotencyKey, reportCleanupFailure, reportProgress) {
     if (!command.foreground || !launch.runtime)
         return undefined;
     const stop = async () => {
@@ -24,11 +71,15 @@ function foregroundWait(launch, command, signal, idempotencyKey, reportCleanupFa
                 }
             }
             if (down) {
-                const operation = await launch.client.waitForOperation(down.operationId, {
-                    timeoutMs: command.timeoutMs ?? 120_000,
-                });
+                // The supervisor lives in this process: a capped wait could only
+                // abandon its shutdown, so observe the down until it settles.
+                const operation = await launch.client.waitForOperationOutcome(down.operationId, reportProgress
+                    ? {
+                        onProgress: fleetOperationProgressReporter('down', reportProgress),
+                    }
+                    : {});
                 if (operation.state !== 'succeeded') {
-                    throw new errors_js_1.LocalDevContractError(operation.failureClassification ?? 'operation-conflict', 'Foreground Fleet down did not complete successfully; ownership was retained.');
+                    throw new errors_js_1.LocalDevContractError(operation.failureClassification ?? 'operation-conflict', `Foreground Fleet down did not complete successfully; ownership was retained.${fleetOperationFailureDetail(operation)}`);
                 }
             }
             await launch.client.shutdown(idempotencyKey());
@@ -46,6 +97,14 @@ function foregroundWait(launch, command, signal, idempotencyKey, reportCleanupFa
         signal.removeEventListener('abort', onAbort);
     });
 }
+const foregroundSettlements = new WeakMap();
+/** Owned-cleanup settlement attached to an error from a foreground `up` whose
+ * in-process supervisor can outlive the failure. Callers report the error
+ * first, then keep interrupt handling installed until this settles, exactly as
+ * they do with `afterWrite` after a successful foreground `up`. */
+function foregroundUpSettlement(error) {
+    return error instanceof Error ? foregroundSettlements.get(error) : undefined;
+}
 async function executeFleetUpCore(command, input, effects = {}) {
     const slices = (0, nx_workspace_js_1.selectWorkspaceSlices)(input.plan, command.selector);
     const inspect = effects.inspectEnvironment ?? environment_inspector_js_1.inspectDevelopmentEnvironment;
@@ -78,7 +137,8 @@ async function executeFleetUpCore(command, input, effects = {}) {
         }
         throw error;
     }
-    const afterWrite = foregroundWait(launch, command, input.signal, idempotencyKey, effects.reportCleanupFailure ?? (() => undefined));
+    const reportCleanupFailure = effects.reportCleanupFailure ?? (() => undefined);
+    const afterWrite = foregroundWait(launch, command, input.signal, idempotencyKey, reportCleanupFailure, effects.reportProgress);
     if (command.noWait) {
         return {
             ...(afterWrite ? { afterWrite } : {}),
@@ -88,21 +148,73 @@ async function executeFleetUpCore(command, input, effects = {}) {
             operationId: submission.operationId,
         };
     }
-    const operation = await launch.client.waitForOperation(submission.operationId, command.timeoutMs === undefined ? {} : { timeoutMs: command.timeoutMs });
+    if (command.foreground && command.timeoutMs !== undefined) {
+        effects.reportProgress?.('[ebk dev up] --timeout does not apply to a foreground up: its supervisor runs in this process, so the command watches until the outcome.');
+    }
+    let observed;
+    try {
+        // No default deadline: watch the supervised up until it settles. Only an
+        // explicit --timeout caps a detached wait, and in detached mode an
+        // interrupt ends observation; neither cancels the operation. A foreground
+        // up takes neither: its supervisor lives in this process, so a cap could
+        // only abandon it, and an interrupt instead drives the owned down above,
+        // which this wait then observes fail.
+        const operation = await launch.client.waitForOperationOutcome(submission.operationId, {
+            ...(command.foreground || command.timeoutMs === undefined
+                ? {}
+                : { timeoutMs: command.timeoutMs }),
+            ...(command.foreground ? {} : { signal: input.signal }),
+            ...(effects.reportProgress
+                ? {
+                    onProgress: fleetOperationProgressReporter('up', effects.reportProgress),
+                }
+                : {}),
+        });
+        observed =
+            operation.state === 'failed'
+                ? { operation }
+                : { operation, slices: (await launch.client.status()).slices };
+    }
+    catch (error) {
+        if (afterWrite && !input.signal.aborted) {
+            effects.reportProgress?.('[ebk dev up] The foreground Fleet supervisor keeps running in this process. Interrupt to run owned cleanup, or run ebk dev down --all from another terminal.');
+        }
+        throw holdForegroundSettlement(error, afterWrite);
+    }
+    const { operation } = observed;
     if (operation.state === 'failed') {
-        if (!launch.attached) {
-            await launch.client.shutdown(idempotencyKey()).catch(() => undefined);
+        // After a foreground interrupt the owned down and shutdown already run; a
+        // second shutdown here would only be refused while that down runs.
+        if (!launch.attached && !(afterWrite && input.signal.aborted)) {
+            await launch.client.shutdown(idempotencyKey()).catch((error) => {
+                reportCleanupFailure(`Fleet up failed and the supervisor it launched refused to shut down, so it keeps running and keeps its claims: ${error instanceof Error ? error.message : String(error)} ${afterWrite
+                    ? 'It runs in this process: interrupt to retry owned cleanup, or run ebk dev down --all from another terminal.'
+                    : 'Check ebk dev status, then run ebk dev down --all to retry cleanup.'}`);
+            });
         }
-        throw new errors_js_1.LocalDevContractError(operation.failureClassification ?? 'internal-error', `Fleet up failed after transactional rollback (${operation.failureClassification ?? 'internal-error'}).`);
+        throw holdForegroundSettlement(new errors_js_1.LocalDevContractError(operation.failureClassification ?? 'internal-error', `Fleet up failed after transactional rollback (${operation.failureClassification ?? 'internal-error'}).${fleetOperationFailureDetail(operation)}`), afterWrite);
     }
-    const state = await launch.client.status();
     return {
         ...(afterWrite ? { afterWrite } : {}),
         attached: launch.attached,
         fleetId: launch.fleetId,
         operation,
         operationId: submission.operationId,
-        slices: state.slices,
+        slices: observed.slices,
     };
 }
+/** A foreground supervisor lives in this process and can outlive a failed
+ * wait: a refused shutdown keeps it, and an interrupt's owned down may still
+ * run. Attach its settlement so the caller reports the failure, then holds
+ * interrupt handling until owned cleanup settles instead of leaving Node's
+ * default handler to kill the process. */
+function holdForegroundSettlement(error, afterWrite) {
+    if (!afterWrite)
+        return error;
+    const failure = error instanceof Error
+        ? error
+        : new errors_js_1.LocalDevContractError('internal-error', String(error));
+    foregroundSettlements.set(failure, afterWrite);
+    return failure;
+}
 //# sourceMappingURL=dev-command-core.js.map

package/src/local-dev/errors.d.ts

@@ -1,6 +1,24 @@
-export declare const LOCAL_DEV_FAILURE_CLASSIFICATIONS: readonly ["invalid-input", "personal-identity-unavailable", "fleet-in-use", "fleet-ownership-ambiguous", "local-context-missing", "local-context-invalid", "local-context-conflict", "unknown-slice", "invalid-adapter", "slice-active-elsewhere", "slice-claim-ambiguous", "not-running", "invalid-state", "configuration-mismatch", "protocol-mismatch", "environment-deficient", "prerequisite-unavailable", "foundation-unavailable", "operation-conflict", "readiness-timeout", "provisioning-timeout", "startup-interrupted", "cleanup-incomplete", "supervisor-unreachable", "stale-state", "stale-state-ambiguous", "log-query-invalid", "internal-error"];
+export declare const LOCAL_DEV_FAILURE_CLASSIFICATIONS: readonly ["invalid-input", "personal-identity-unavailable", "fleet-in-use", "fleet-ownership-ambiguous", "local-context-missing", "local-context-invalid", "local-context-conflict", "unknown-slice", "invalid-adapter", "slice-active-elsewhere", "slice-claim-ambiguous", "not-running", "invalid-state", "configuration-mismatch", "protocol-mismatch", "environment-deficient", "prerequisite-unavailable", "foundation-unavailable", "operation-conflict", "readiness-timeout", "readiness-stalled", "health-check-failing", "provisioning-timeout", "process-exited", "startup-interrupted", "observation-timeout", "observation-interrupted", "cleanup-incomplete", "supervisor-unreachable", "stale-state", "stale-state-ambiguous", "log-query-invalid", "internal-error"];
 export type LocalDevFailureClassification = (typeof LOCAL_DEV_FAILURE_CLASSIFICATIONS)[number];
 export declare class LocalDevContractError extends Error {
     readonly classification: LocalDevFailureClassification;
     constructor(classification: LocalDevFailureClassification, message: string);
 }
+/** What the client knew when it stopped watching an accepted operation that
+ * had not settled. The operation itself may still be running. */
+export interface FleetObservationDetail {
+    elapsedMs: number;
+    /** The last non-terminal state read; absent when no read succeeded. */
+    lastObservedState?: 'accepted' | 'running';
+    operationId: string;
+    /** The explicit cap, present only for `observation-timeout`. */
+    timeoutMs?: number;
+}
+/** Observation of an accepted operation ended without a terminal record: at an
+ * explicit cap, on interruption, or because the supervisor stopped answering.
+ * The CLI renders `detail` in the failure envelope, with the operation ID at
+ * the envelope level, so a script can keep following the same operation. */
+export declare class FleetObservationError extends LocalDevContractError {
+    readonly detail: FleetObservationDetail;
+    constructor(classification: 'observation-interrupted' | 'observation-timeout' | 'supervisor-unreachable', message: string, detail: FleetObservationDetail);
+}

package/src/local-dev/errors.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.LocalDevContractError = exports.LOCAL_DEV_FAILURE_CLASSIFICATIONS = void 0;
+exports.FleetObservationError = exports.LocalDevContractError = exports.LOCAL_DEV_FAILURE_CLASSIFICATIONS = void 0;
 exports.LOCAL_DEV_FAILURE_CLASSIFICATIONS = [
     'invalid-input',
     'personal-identity-unavailable',
@@ -22,8 +22,13 @@ exports.LOCAL_DEV_FAILURE_CLASSIFICATIONS = [
     'foundation-unavailable',
     'operation-conflict',
     'readiness-timeout',
+    'readiness-stalled',
+    'health-check-failing',
     'provisioning-timeout',
+    'process-exited',
     'startup-interrupted',
+    'observation-timeout',
+    'observation-interrupted',
     'cleanup-incomplete',
     'supervisor-unreachable',
     'stale-state',
@@ -40,4 +45,16 @@ class LocalDevContractError extends Error {
     }
 }
 exports.LocalDevContractError = LocalDevContractError;
+/** Observation of an accepted operation ended without a terminal record: at an
+ * explicit cap, on interruption, or because the supervisor stopped answering.
+ * The CLI renders `detail` in the failure envelope, with the operation ID at
+ * the envelope level, so a script can keep following the same operation. */
+class FleetObservationError extends LocalDevContractError {
+    detail;
+    constructor(classification, message, detail) {
+        super(classification, message);
+        this.detail = detail;
+    }
+}
+exports.FleetObservationError = FleetObservationError;
 //# sourceMappingURL=errors.js.map

package/src/local-dev/fleet-client.d.ts

@@ -1,19 +1,67 @@
 import { type FleetMutationCommand } from './control-contract.js';
+import { LocalDevContractError } from './errors.js';
 import { sendFleetControlRequest } from './fleet-ipc.js';
-import { type FleetOperationStateRecord, type FleetStateRecord } from './fleet-state.js';
+import { type FleetOperationStateRecord, type FleetSliceState, type FleetStateRecord } from './fleet-state.js';
 import { type PrivateStateSecurity } from './private-state.js';
 import type { FleetRuntimePaths } from './runtime-paths.js';
 export interface FleetClientEffects {
+    /** Monotonic-enough wall clock for observation bookkeeping (tests inject). */
+    now?: () => number;
     requestId?: () => string;
     send?: typeof sendFleetControlRequest;
+    /** Poll pause; resolves early when the signal aborts (tests inject). */
+    sleep?: (milliseconds: number, signal?: AbortSignal) => Promise<void>;
 }
+/** A supervisor whose authenticated status keeps answering with the same
+ * heartbeat for this long is not making progress. The supervisor writes its
+ * heartbeat every 5 s; Windows identity captures can block its loop longer.
+ * The window must also span enough consecutive status reads (see
+ * `waitForOperationOutcome`), so a host sleep that jumps the clock cannot
+ * trip it before the supervisor's next heartbeat. */
+export declare const FLEET_SUPERVISOR_HEARTBEAT_STALE_MS: Readonly<{
+    default: 30000;
+    win32: 60000;
+}>;
+/** Defaults for watching a supervised operation. None of them is an operation
+ * deadline: without an explicit cap the client waits for the real outcome. */
+export declare const FLEET_OPERATION_OBSERVATION: Readonly<{
+    pollMs: 200;
+    /** Progress is reported on every Slice change and at least this often. */
+    progressIntervalMs: 30000;
+    /** Status (heartbeat and Slice progress) is read at most this often. */
+    statusIntervalMs: 2000;
+    /** Consecutive unreachable polls tolerated before the wait fails. */
+    unreachableLimit: 6;
+}>;
+export interface FleetOperationProgress {
+    elapsedMs: number;
+    operation: FleetOperationStateRecord;
+    /** Live state of the operation's Slices from the latest status read. */
+    slices: FleetSliceState[];
+}
+export interface FleetOperationWaitOptions {
+    onProgress?: (progress: FleetOperationProgress) => void;
+    pollMs?: number;
+    /** Ends observation (never the operation) when aborted. */
+    signal?: AbortSignal;
+    /** Explicit observation cap. Absent means wait for the terminal state. */
+    timeoutMs?: number;
+}
+export interface FleetOperationObservationTuning {
+    progressIntervalMs?: number;
+    staleHeartbeatMs?: number;
+    statusIntervalMs?: number;
+    unreachableLimit?: number;
+}
+/** Terminal failure as an error that keeps the supervisor's persisted cause. */
+export declare function fleetOperationFailure(operation: FleetOperationStateRecord): LocalDevContractError;
 export interface FleetOperationSubmission {
     operationId: string;
     state: 'accepted';
 }
 export declare class FleetClient {
     #private;
-    constructor(paths: FleetRuntimePaths, security?: PrivateStateSecurity, effects?: FleetClientEffects);
+    constructor(paths: FleetRuntimePaths, security?: PrivateStateSecurity, effects?: FleetClientEffects, observation?: FleetOperationObservationTuning);
     ping(): Promise<{
         fleetId: string;
         protocolVersion: number;
@@ -25,9 +73,19 @@ export declare class FleetClient {
         slices: readonly string[];
     }, idempotencyKey?: `${string}-${string}-${string}-${string}-${string}`): Promise<FleetOperationSubmission>;
     operationStatus(operationId: string): Promise<FleetOperationStateRecord>;
-    waitForOperation(operationId: string, options?: {
-        pollMs?: number;
-        timeoutMs?: number;
-    }): Promise<FleetOperationStateRecord>;
+    /** Wait for a terminal operation record, `succeeded` or `failed`.
+     *
+     * There is no default deadline: an operation that is still progressing is
+     * observed until it settles. The wait ends early only on an explicit
+     * `timeoutMs` cap (`observation-timeout`), on `signal` abort
+     * (`observation-interrupted`), or when the supervisor itself stops making
+     * progress (`supervisor-unreachable`: repeated unreachable polls or a
+     * heartbeat that no longer advances). Ending observation never cancels the
+     * supervised operation.
+     */
+    waitForOperationOutcome(operationId: string, options?: FleetOperationWaitOptions): Promise<FleetOperationStateRecord>;
+    /** Throwing form: resolves only on `succeeded`; a terminal `failed` record
+     * becomes an error carrying its persisted classification and cause. */
+    waitForOperation(operationId: string, options?: FleetOperationWaitOptions): Promise<FleetOperationStateRecord>;
     shutdown(idempotencyKey?: `${string}-${string}-${string}-${string}-${string}`): Promise<void>;
 }

package/src/local-dev/fleet-client.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.FleetClient = void 0;
+exports.FleetClient = exports.FLEET_OPERATION_OBSERVATION = exports.FLEET_SUPERVISOR_HEARTBEAT_STALE_MS = void 0;
+exports.fleetOperationFailure = fleetOperationFailure;
 const node_crypto_1 = require("node:crypto");
 const control_contract_js_1 = require("./control-contract.js");
 const contract_validation_js_1 = require("./contract-validation.js");
@@ -8,6 +9,56 @@ const errors_js_1 = require("./errors.js");
 const fleet_ipc_js_1 = require("./fleet-ipc.js");
 const fleet_state_js_1 = require("./fleet-state.js");
 const private_state_js_1 = require("./private-state.js");
+/** A supervisor whose authenticated status keeps answering with the same
+ * heartbeat for this long is not making progress. The supervisor writes its
+ * heartbeat every 5 s; Windows identity captures can block its loop longer.
+ * The window must also span enough consecutive status reads (see
+ * `waitForOperationOutcome`), so a host sleep that jumps the clock cannot
+ * trip it before the supervisor's next heartbeat. */
+exports.FLEET_SUPERVISOR_HEARTBEAT_STALE_MS = Object.freeze({
+    default: 30_000,
+    win32: 60_000,
+});
+/** Defaults for watching a supervised operation. None of them is an operation
+ * deadline: without an explicit cap the client waits for the real outcome. */
+exports.FLEET_OPERATION_OBSERVATION = Object.freeze({
+    pollMs: 200,
+    /** Progress is reported on every Slice change and at least this often. */
+    progressIntervalMs: 30_000,
+    /** Status (heartbeat and Slice progress) is read at most this often. */
+    statusIntervalMs: 2_000,
+    /** Consecutive unreachable polls tolerated before the wait fails. */
+    unreachableLimit: 6,
+});
+function defaultSleep(milliseconds, signal) {
+    return new Promise((resolvePromise) => {
+        if (signal?.aborted) {
+            resolvePromise();
+            return;
+        }
+        const settle = () => {
+            clearTimeout(timer);
+            signal?.removeEventListener('abort', settle);
+            resolvePromise();
+        };
+        const timer = setTimeout(settle, milliseconds);
+        signal?.addEventListener('abort', settle, { once: true });
+    });
+}
+function formatObservedDuration(milliseconds) {
+    if (milliseconds < 1_000)
+        return `${milliseconds}ms`;
+    const seconds = Math.round(milliseconds / 1_000);
+    const minutes = Math.floor(seconds / 60);
+    return minutes === 0
+        ? `${seconds}s`
+        : `${minutes}m${String(seconds % 60).padStart(2, '0')}s`;
+}
+/** Terminal failure as an error that keeps the supervisor's persisted cause. */
+function fleetOperationFailure(operation) {
+    const classification = operation.failureClassification ?? 'internal-error';
+    return new errors_js_1.LocalDevContractError(classification, `Fleet ${operation.command} operation ${operation.id} failed (${classification})${operation.failureMessage ? `: ${operation.failureMessage}` : '.'}`);
+}
 function clientFailure(response) {
     if (!('error' in response)) {
         throw new errors_js_1.LocalDevContractError('invalid-state', 'Expected a Fleet control failure response.');
@@ -67,16 +118,33 @@ function validateOperation(value) {
 }
 class FleetClient {
     #capability;
+    #now;
+    #observation;
     #paths;
     #requestId;
     #send;
-    constructor(paths, security = {}, effects = {}) {
+    #sleep;
+    constructor(paths, security = {}, effects = {}, observation = {}) {
         this.#paths = paths;
         this.#capability = (0, private_state_js_1.readPrivateFile)(paths.capability, security)
             .toString('utf8')
             .trim();
+        this.#now = effects.now ?? (() => Date.now());
         this.#requestId = effects.requestId ?? (() => (0, node_crypto_1.randomUUID)());
         this.#send = effects.send ?? fleet_ipc_js_1.sendFleetControlRequest;
+        this.#sleep = effects.sleep ?? defaultSleep;
+        this.#observation = {
+            progressIntervalMs: observation.progressIntervalMs ??
+                exports.FLEET_OPERATION_OBSERVATION.progressIntervalMs,
+            staleHeartbeatMs: observation.staleHeartbeatMs ??
+                (process.platform === 'win32'
+                    ? exports.FLEET_SUPERVISOR_HEARTBEAT_STALE_MS.win32
+                    : exports.FLEET_SUPERVISOR_HEARTBEAT_STALE_MS.default),
+            statusIntervalMs: observation.statusIntervalMs ??
+                exports.FLEET_OPERATION_OBSERVATION.statusIntervalMs,
+            unreachableLimit: observation.unreachableLimit ??
+                exports.FLEET_OPERATION_OBSERVATION.unreachableLimit,
+        };
     }
     async #request(request) {
         const response = await this.#send(this.#paths.controlSocket, {
@@ -128,22 +196,128 @@ class FleetClient {
             payload: { operationId },
         }));
     }
-    async waitForOperation(operationId, options = {}) {
-        const pollMs = options.pollMs ?? 200;
-        const timeoutMs = options.timeoutMs ?? 120_000;
-        const deadline = Date.now() + timeoutMs;
-        while (Date.now() <= deadline) {
-            const operation = await this.operationStatus(operationId);
-            if (operation.state === 'succeeded')
+    /** Wait for a terminal operation record, `succeeded` or `failed`.
+     *
+     * There is no default deadline: an operation that is still progressing is
+     * observed until it settles. The wait ends early only on an explicit
+     * `timeoutMs` cap (`observation-timeout`), on `signal` abort
+     * (`observation-interrupted`), or when the supervisor itself stops making
+     * progress (`supervisor-unreachable`: repeated unreachable polls or a
+     * heartbeat that no longer advances). Ending observation never cancels the
+     * supervised operation.
+     */
+    async waitForOperationOutcome(operationId, options = {}) {
+        const pollMs = options.pollMs ?? exports.FLEET_OPERATION_OBSERVATION.pollMs;
+        const tuning = this.#observation;
+        const startedAt = this.#now();
+        const deadline = options.timeoutMs === undefined
+            ? undefined
+            : startedAt + options.timeoutMs;
+        let latest;
+        let unreachable = 0;
+        let nextStatusAt = startedAt + tuning.statusIntervalMs;
+        // A heartbeat is stale only when it stays unchanged both for the stale
+        // window by the clock and across enough consecutive status reads to cover
+        // that window at the status interval. After a host sleep the clock jumps
+        // at once, but the reads still have to accumulate, which gives the
+        // supervisor time for its next heartbeat.
+        const staleReadLimit = Math.max(1, Math.ceil(tuning.staleHeartbeatMs / Math.max(1, tuning.statusIntervalMs)));
+        let heartbeat;
+        let reportedSummary;
+        let reportedAt = startedAt;
+        const label = () => `Fleet ${latest ? `${latest.command} ` : ''}operation ${operationId}`;
+        const ended = (classification, message, elapsedMs = this.#now() - startedAt) => new errors_js_1.FleetObservationError(classification, message, {
+            elapsedMs,
+            ...(latest
+                ? {
+                    lastObservedState: latest.state,
+                }
+                : {}),
+            operationId,
+            ...(classification === 'observation-timeout'
+                ? { timeoutMs: options.timeoutMs }
+                : {}),
+        });
+        while (true) {
+            if (options.signal?.aborted) {
+                const elapsedMs = this.#now() - startedAt;
+                throw ended('observation-interrupted', `Stopped watching ${label()} after ${formatObservedDuration(elapsedMs)} on interruption; the supervised operation continues. Check ebk dev status and ebk dev logs before retrying.`, elapsedMs);
+            }
+            if (deadline !== undefined && this.#now() >= deadline) {
+                throw ended('observation-timeout', `Stopped watching ${label()} after the explicit ${formatObservedDuration(options.timeoutMs)} --timeout cap; it was last ${latest?.state ?? 'unobserved'} and continues in the supervisor. Check ebk dev status and ebk dev logs before retrying.`);
+            }
+            let operation;
+            try {
+                operation = await this.operationStatus(operationId);
+                unreachable = 0;
+            }
+            catch (error) {
+                if (!(error instanceof errors_js_1.LocalDevContractError) ||
+                    error.classification !== 'supervisor-unreachable')
+                    throw error;
+                unreachable += 1;
+                if (unreachable >= tuning.unreachableLimit) {
+                    throw ended('supervisor-unreachable', `${label()} could not be observed: the supervisor did not answer ${unreachable} consecutive polls (${error.message}). Check ebk dev status and ebk dev ps before retrying.`);
+                }
+                await this.#pause(Math.min(1_000 * 2 ** (unreachable - 1), 5_000), deadline, options.signal);
+                continue;
+            }
+            if (operation.state === 'succeeded' || operation.state === 'failed')
                 return operation;
-            if (operation.state === 'failed') {
-                throw new errors_js_1.LocalDevContractError(operation.failureClassification ?? 'internal-error', `Fleet ${operation.command} operation ${operation.id} failed.`);
+            latest = operation;
+            const now = this.#now();
+            if (now >= nextStatusAt) {
+                nextStatusAt = now + tuning.statusIntervalMs;
+                // Status is advisory: a refused or malformed read (for example while
+                // the supervisor is shutting down) never ends the wait by itself.
+                const state = await this.status().catch(() => undefined);
+                if (state) {
+                    const observedAt = this.#now();
+                    if (heartbeat?.value !== state.heartbeatAt) {
+                        heartbeat = { observedAt, staleReads: 0, value: state.heartbeatAt };
+                    }
+                    else {
+                        heartbeat.staleReads += 1;
+                        if (heartbeat.staleReads >= staleReadLimit &&
+                            observedAt - heartbeat.observedAt > tuning.staleHeartbeatMs) {
+                            throw ended('supervisor-unreachable', `${label()} could not be observed: the supervisor heartbeat has not advanced since ${state.heartbeatAt} (${formatObservedDuration(observedAt - heartbeat.observedAt)}, ${heartbeat.staleReads} consecutive status reads). Check ebk dev status and ebk dev ps before retrying.`);
+                        }
+                    }
+                    if (options.onProgress) {
+                        const slices = state.slices.filter(({ name }) => operation.slices.includes(name));
+                        const summary = JSON.stringify(slices.map(({ cleanup, name, restartAttempts, routeState }) => [
+                            name,
+                            routeState,
+                            cleanup,
+                            restartAttempts,
+                        ]));
+                        if (summary !== reportedSummary ||
+                            observedAt - reportedAt >= tuning.progressIntervalMs) {
+                            reportedSummary = summary;
+                            reportedAt = observedAt;
+                            options.onProgress({
+                                elapsedMs: observedAt - startedAt,
+                                operation,
+                                slices,
+                            });
+                        }
+                    }
+                }
             }
-            await new Promise((resolvePromise) => {
-                setTimeout(resolvePromise, pollMs);
-            });
+            await this.#pause(pollMs, deadline, options.signal);
         }
-        throw new errors_js_1.LocalDevContractError('readiness-timeout', `Fleet operation ${operationId} did not finish within ${timeoutMs}ms. The supervised operation may still be running; check ebk dev status and ebk dev logs before retrying.`);
+    }
+    /** Throwing form: resolves only on `succeeded`; a terminal `failed` record
+     * becomes an error carrying its persisted classification and cause. */
+    async waitForOperation(operationId, options = {}) {
+        const operation = await this.waitForOperationOutcome(operationId, options);
+        if (operation.state === 'failed')
+            throw fleetOperationFailure(operation);
+        return operation;
+    }
+    async #pause(milliseconds, deadline, signal) {
+        const remaining = deadline === undefined ? milliseconds : deadline - this.#now();
+        await this.#sleep(Math.max(0, Math.min(milliseconds, remaining)), signal);
     }
     async shutdown(idempotencyKey = (0, node_crypto_1.randomUUID)()) {
         await this.#request({