@empire-builder-kit/nx 1.0.0-rc.44 → 1.0.0-rc.46

package/src/local-dev/fleet-client.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.FleetClient = void 0;
+exports.FleetClient = exports.FLEET_OPERATION_OBSERVATION = exports.FLEET_SUPERVISOR_HEARTBEAT_STALE_MS = void 0;
+exports.fleetOperationFailure = fleetOperationFailure;
 const node_crypto_1 = require("node:crypto");
 const control_contract_js_1 = require("./control-contract.js");
 const contract_validation_js_1 = require("./contract-validation.js");
@@ -8,6 +9,56 @@ const errors_js_1 = require("./errors.js");
 const fleet_ipc_js_1 = require("./fleet-ipc.js");
 const fleet_state_js_1 = require("./fleet-state.js");
 const private_state_js_1 = require("./private-state.js");
+/** A supervisor whose authenticated status keeps answering with the same
+ * heartbeat for this long is not making progress. The supervisor writes its
+ * heartbeat every 5 s; Windows identity captures can block its loop longer.
+ * The window must also span enough consecutive status reads (see
+ * `waitForOperationOutcome`), so a host sleep that jumps the clock cannot
+ * trip it before the supervisor's next heartbeat. */
+exports.FLEET_SUPERVISOR_HEARTBEAT_STALE_MS = Object.freeze({
+    default: 30_000,
+    win32: 60_000,
+});
+/** Defaults for watching a supervised operation. None of them is an operation
+ * deadline: without an explicit cap the client waits for the real outcome. */
+exports.FLEET_OPERATION_OBSERVATION = Object.freeze({
+    pollMs: 200,
+    /** Progress is reported on every Slice change and at least this often. */
+    progressIntervalMs: 30_000,
+    /** Status (heartbeat and Slice progress) is read at most this often. */
+    statusIntervalMs: 2_000,
+    /** Consecutive unreachable polls tolerated before the wait fails. */
+    unreachableLimit: 6,
+});
+function defaultSleep(milliseconds, signal) {
+    return new Promise((resolvePromise) => {
+        if (signal?.aborted) {
+            resolvePromise();
+            return;
+        }
+        const settle = () => {
+            clearTimeout(timer);
+            signal?.removeEventListener('abort', settle);
+            resolvePromise();
+        };
+        const timer = setTimeout(settle, milliseconds);
+        signal?.addEventListener('abort', settle, { once: true });
+    });
+}
+function formatObservedDuration(milliseconds) {
+    if (milliseconds < 1_000)
+        return `${milliseconds}ms`;
+    const seconds = Math.round(milliseconds / 1_000);
+    const minutes = Math.floor(seconds / 60);
+    return minutes === 0
+        ? `${seconds}s`
+        : `${minutes}m${String(seconds % 60).padStart(2, '0')}s`;
+}
+/** Terminal failure as an error that keeps the supervisor's persisted cause. */
+function fleetOperationFailure(operation) {
+    const classification = operation.failureClassification ?? 'internal-error';
+    return new errors_js_1.LocalDevContractError(classification, `Fleet ${operation.command} operation ${operation.id} failed (${classification})${operation.failureMessage ? `: ${operation.failureMessage}` : '.'}`);
+}
 function clientFailure(response) {
     if (!('error' in response)) {
         throw new errors_js_1.LocalDevContractError('invalid-state', 'Expected a Fleet control failure response.');
@@ -67,16 +118,33 @@ function validateOperation(value) {
 }
 class FleetClient {
     #capability;
+    #now;
+    #observation;
     #paths;
     #requestId;
     #send;
-    constructor(paths, security = {}, effects = {}) {
+    #sleep;
+    constructor(paths, security = {}, effects = {}, observation = {}) {
         this.#paths = paths;
         this.#capability = (0, private_state_js_1.readPrivateFile)(paths.capability, security)
             .toString('utf8')
             .trim();
+        this.#now = effects.now ?? (() => Date.now());
         this.#requestId = effects.requestId ?? (() => (0, node_crypto_1.randomUUID)());
         this.#send = effects.send ?? fleet_ipc_js_1.sendFleetControlRequest;
+        this.#sleep = effects.sleep ?? defaultSleep;
+        this.#observation = {
+            progressIntervalMs: observation.progressIntervalMs ??
+                exports.FLEET_OPERATION_OBSERVATION.progressIntervalMs,
+            staleHeartbeatMs: observation.staleHeartbeatMs ??
+                (process.platform === 'win32'
+                    ? exports.FLEET_SUPERVISOR_HEARTBEAT_STALE_MS.win32
+                    : exports.FLEET_SUPERVISOR_HEARTBEAT_STALE_MS.default),
+            statusIntervalMs: observation.statusIntervalMs ??
+                exports.FLEET_OPERATION_OBSERVATION.statusIntervalMs,
+            unreachableLimit: observation.unreachableLimit ??
+                exports.FLEET_OPERATION_OBSERVATION.unreachableLimit,
+        };
     }
     async #request(request) {
         const response = await this.#send(this.#paths.controlSocket, {
@@ -128,22 +196,128 @@ class FleetClient {
             payload: { operationId },
         }));
     }
-    async waitForOperation(operationId, options = {}) {
-        const pollMs = options.pollMs ?? 200;
-        const timeoutMs = options.timeoutMs ?? 120_000;
-        const deadline = Date.now() + timeoutMs;
-        while (Date.now() <= deadline) {
-            const operation = await this.operationStatus(operationId);
-            if (operation.state === 'succeeded')
+    /** Wait for a terminal operation record, `succeeded` or `failed`.
+     *
+     * There is no default deadline: an operation that is still progressing is
+     * observed until it settles. The wait ends early only on an explicit
+     * `timeoutMs` cap (`observation-timeout`), on `signal` abort
+     * (`observation-interrupted`), or when the supervisor itself stops making
+     * progress (`supervisor-unreachable`: repeated unreachable polls or a
+     * heartbeat that no longer advances). Ending observation never cancels the
+     * supervised operation.
+     */
+    async waitForOperationOutcome(operationId, options = {}) {
+        const pollMs = options.pollMs ?? exports.FLEET_OPERATION_OBSERVATION.pollMs;
+        const tuning = this.#observation;
+        const startedAt = this.#now();
+        const deadline = options.timeoutMs === undefined
+            ? undefined
+            : startedAt + options.timeoutMs;
+        let latest;
+        let unreachable = 0;
+        let nextStatusAt = startedAt + tuning.statusIntervalMs;
+        // A heartbeat is stale only when it stays unchanged both for the stale
+        // window by the clock and across enough consecutive status reads to cover
+        // that window at the status interval. After a host sleep the clock jumps
+        // at once, but the reads still have to accumulate, which gives the
+        // supervisor time for its next heartbeat.
+        const staleReadLimit = Math.max(1, Math.ceil(tuning.staleHeartbeatMs / Math.max(1, tuning.statusIntervalMs)));
+        let heartbeat;
+        let reportedSummary;
+        let reportedAt = startedAt;
+        const label = () => `Fleet ${latest ? `${latest.command} ` : ''}operation ${operationId}`;
+        const ended = (classification, message, elapsedMs = this.#now() - startedAt) => new errors_js_1.FleetObservationError(classification, message, {
+            elapsedMs,
+            ...(latest
+                ? {
+                    lastObservedState: latest.state,
+                }
+                : {}),
+            operationId,
+            ...(classification === 'observation-timeout'
+                ? { timeoutMs: options.timeoutMs }
+                : {}),
+        });
+        while (true) {
+            if (options.signal?.aborted) {
+                const elapsedMs = this.#now() - startedAt;
+                throw ended('observation-interrupted', `Stopped watching ${label()} after ${formatObservedDuration(elapsedMs)} on interruption; the supervised operation continues. Check ebk dev status and ebk dev logs before retrying.`, elapsedMs);
+            }
+            if (deadline !== undefined && this.#now() >= deadline) {
+                throw ended('observation-timeout', `Stopped watching ${label()} after the explicit ${formatObservedDuration(options.timeoutMs)} --timeout cap; it was last ${latest?.state ?? 'unobserved'} and continues in the supervisor. Check ebk dev status and ebk dev logs before retrying.`);
+            }
+            let operation;
+            try {
+                operation = await this.operationStatus(operationId);
+                unreachable = 0;
+            }
+            catch (error) {
+                if (!(error instanceof errors_js_1.LocalDevContractError) ||
+                    error.classification !== 'supervisor-unreachable')
+                    throw error;
+                unreachable += 1;
+                if (unreachable >= tuning.unreachableLimit) {
+                    throw ended('supervisor-unreachable', `${label()} could not be observed: the supervisor did not answer ${unreachable} consecutive polls (${error.message}). Check ebk dev status and ebk dev ps before retrying.`);
+                }
+                await this.#pause(Math.min(1_000 * 2 ** (unreachable - 1), 5_000), deadline, options.signal);
+                continue;
+            }
+            if (operation.state === 'succeeded' || operation.state === 'failed')
                 return operation;
-            if (operation.state === 'failed') {
-                throw new errors_js_1.LocalDevContractError(operation.failureClassification ?? 'internal-error', `Fleet ${operation.command} operation ${operation.id} failed.`);
+            latest = operation;
+            const now = this.#now();
+            if (now >= nextStatusAt) {
+                nextStatusAt = now + tuning.statusIntervalMs;
+                // Status is advisory: a refused or malformed read (for example while
+                // the supervisor is shutting down) never ends the wait by itself.
+                const state = await this.status().catch(() => undefined);
+                if (state) {
+                    const observedAt = this.#now();
+                    if (heartbeat?.value !== state.heartbeatAt) {
+                        heartbeat = { observedAt, staleReads: 0, value: state.heartbeatAt };
+                    }
+                    else {
+                        heartbeat.staleReads += 1;
+                        if (heartbeat.staleReads >= staleReadLimit &&
+                            observedAt - heartbeat.observedAt > tuning.staleHeartbeatMs) {
+                            throw ended('supervisor-unreachable', `${label()} could not be observed: the supervisor heartbeat has not advanced since ${state.heartbeatAt} (${formatObservedDuration(observedAt - heartbeat.observedAt)}, ${heartbeat.staleReads} consecutive status reads). Check ebk dev status and ebk dev ps before retrying.`);
+                        }
+                    }
+                    if (options.onProgress) {
+                        const slices = state.slices.filter(({ name }) => operation.slices.includes(name));
+                        const summary = JSON.stringify(slices.map(({ cleanup, name, restartAttempts, routeState }) => [
+                            name,
+                            routeState,
+                            cleanup,
+                            restartAttempts,
+                        ]));
+                        if (summary !== reportedSummary ||
+                            observedAt - reportedAt >= tuning.progressIntervalMs) {
+                            reportedSummary = summary;
+                            reportedAt = observedAt;
+                            options.onProgress({
+                                elapsedMs: observedAt - startedAt,
+                                operation,
+                                slices,
+                            });
+                        }
+                    }
+                }
             }
-            await new Promise((resolvePromise) => {
-                setTimeout(resolvePromise, pollMs);
-            });
+            await this.#pause(pollMs, deadline, options.signal);
         }
-        throw new errors_js_1.LocalDevContractError('readiness-timeout', `Fleet operation ${operationId} did not finish within ${timeoutMs}ms. The supervised operation may still be running; check ebk dev status and ebk dev logs before retrying.`);
+    }
+    /** Throwing form: resolves only on `succeeded`; a terminal `failed` record
+     * becomes an error carrying its persisted classification and cause. */
+    async waitForOperation(operationId, options = {}) {
+        const operation = await this.waitForOperationOutcome(operationId, options);
+        if (operation.state === 'failed')
+            throw fleetOperationFailure(operation);
+        return operation;
+    }
+    async #pause(milliseconds, deadline, signal) {
+        const remaining = deadline === undefined ? milliseconds : deadline - this.#now();
+        await this.#sleep(Math.max(0, Math.min(milliseconds, remaining)), signal);
     }
     async shutdown(idempotencyKey = (0, node_crypto_1.randomUUID)()) {
         await this.#request({

package/src/local-dev/fleet-process.d.ts

@@ -9,6 +9,11 @@ export interface FleetProcessLaunch {
     attempt: number;
     child: ChildProcess;
 }
+/** How an owned process ended: an exit status or a terminating signal. */
+export interface FleetProcessExit {
+    code: number | null;
+    signal: string | null;
+}
 export interface FleetProcessEffects {
     /** Native provisioning may outlast HTTP readiness. This is an owned launch
      * notification, not an HTTP-health result. Non-native adapters omit it. */
@@ -17,15 +22,39 @@ export interface FleetProcessEffects {
     cleanup?: () => Promise<void>;
     delay?: (milliseconds: number, signal: AbortSignal) => Promise<void>;
     launch: (attempt: number) => FleetProcessLaunch;
-    probe?: (url: URL, signal: AbortSignal) => Promise<boolean>;
+    /** `true` is healthy, `'unhealthy'` is an HTTP response that is not OK,
+     * and `false` is no response (refused, reset, or unanswered). */
+    probe?: (url: URL, signal: AbortSignal) => Promise<FleetProbeResult>;
     prepareRestart?: (child: ChildProcess) => Promise<void>;
+    /** Stop the writer while the owned process keeps its pending cleanup
+     * authority. A coordinating runner stays alive and later runs declared
+     * cleanup itself when stopped. Without this effect, quiescence writes the
+     * restart intent and terminates the child. */
+    quiesce?: (child: ChildProcess) => Promise<void>;
     runnerGeneration?: () => string | undefined;
+    /** The Slice start target's own exit, when the launched child is a runner
+     * that recorded it. The runner exits cleanly after its child crashes, so
+     * its own status alone would hide the real failure. */
+    startTargetExit?: (child: ChildProcess) => FleetProcessExit | undefined;
     terminate?: (child: ChildProcess) => Promise<void>;
 }
+export type FleetProbeResult = boolean | 'unhealthy';
+/** Web readiness after launch is measured against a target, not enforced as
+ * a deadline: past it the controller logs once and keeps waiting while the
+ * Slice makes progress. */
+export declare const FLEET_READINESS_TARGET_MS = 120000;
+/** Readiness fails only when nothing has progressed for this long: no output
+ * and no state change while the health route gives no answer, or only
+ * unhealthy answers. Request logs a probe causes are not progress for an
+ * answering but unhealthy application, so that path ignores output. */
+export declare const FLEET_READINESS_STALL_MS = 120000;
+/** Recent output lines carried into a startup failure message. */
+export declare const FLEET_FAILURE_OUTPUT_LINES = 4;
 export interface FleetProcessControllerOptions {
     adapter: LocalDevAdapter;
-    healthTimeoutMs?: number;
     provisioningTimeoutMs?: number;
+    readinessStallMs?: number;
+    readinessTargetMs?: number;
     logStore?: FleetLogStore;
     maxRestartAttempts: number;
     name: string;
@@ -43,12 +72,22 @@ export declare function planFleetRestart(completedRestartAttempts: number, maxRe
 export declare function assertOwnedGroupAbsent(pid: number, deadline?: number): Promise<void>;
 export declare function terminateOwnedChild(child: ChildProcess, ownedIdentity?: FleetProcessIdentity): Promise<void>;
 export declare const FLEET_RUNNER_STOP_MESSAGE = "ebk-fleet-runner-stop";
+export declare const FLEET_RUNNER_QUIESCE_MESSAGE = "ebk-fleet-runner-quiesce";
+export declare const FLEET_RUNNER_QUIESCED_MESSAGE = "ebk-fleet-runner-quiesced";
 /** A runner coordinates detached children and native cleanup. Never force-kill
  * that coordinator on a client observation deadline. An explicit diagnostic
  * deadline retains live ownership and reports failure instead of orphaning a
  * writer (and falsely permitting stale-process recovery).
  */
 export declare function terminateCoordinatingFleetRunner(child: ChildProcess, runnerGeneration: string, timeoutMs?: number): Promise<void>;
+/** Ask the exact runner generation to stop its writer but keep running with
+ * pending cleanup authority. It acknowledges once its owned child group is
+ * absent, or at once when quiescence reaches it before its writer started.
+ * Like the stop, this has no client deadline: the runner bounds its own
+ * child termination. A clean exit also settles the request, because the
+ * runner then retained its authority in state for a cleanup retry.
+ */
+export declare function quiesceCoordinatingFleetRunner(child: ChildProcess, runnerGeneration: string): Promise<void>;
 export declare class FleetProcessController {
     #private;
     constructor(options: FleetProcessControllerOptions, effects: FleetProcessEffects);
@@ -62,6 +101,12 @@ export declare class FleetProcessController {
      * responsibility for native termination and exact cleanup before down runs. */
     interruptStartup(): void;
     restart(): Promise<void>;
+    /** Stop the owned writer without removing its cloud/dependency
+     * prerequisites. A Fleet transaction quiesces every selected Slice before
+     * reverse cleanup; a failed dependent cleanup must not leave a dependency
+     * writer running. A coordinating runner stays alive with pending cleanup
+     * authority, so its later cleanup output still reaches this Slice's log. */
+    quiesce(): Promise<void>;
     stop(options: {
         cleanup: boolean;
     }): Promise<void>;