@empire-builder-kit/nx 1.0.0-rc.44 → 1.0.0-rc.46

package/src/local-dev/fleet-process.js

@@ -1,15 +1,35 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.FleetProcessController = exports.FLEET_RUNNER_STOP_MESSAGE = void 0;
+exports.FleetProcessController = exports.FLEET_RUNNER_QUIESCED_MESSAGE = exports.FLEET_RUNNER_QUIESCE_MESSAGE = exports.FLEET_RUNNER_STOP_MESSAGE = exports.FLEET_FAILURE_OUTPUT_LINES = exports.FLEET_READINESS_STALL_MS = exports.FLEET_READINESS_TARGET_MS = void 0;
 exports.fleetRestartDelay = fleetRestartDelay;
 exports.planFleetRestart = planFleetRestart;
 exports.assertOwnedGroupAbsent = assertOwnedGroupAbsent;
 exports.terminateOwnedChild = terminateOwnedChild;
 exports.terminateCoordinatingFleetRunner = terminateCoordinatingFleetRunner;
+exports.quiesceCoordinatingFleetRunner = quiesceCoordinatingFleetRunner;
 const node_child_process_1 = require("node:child_process");
 const errors_js_1 = require("./errors.js");
 const process_group_js_1 = require("./process-group.js");
 const process_identity_js_1 = require("./process-identity.js");
+/** Web readiness after launch is measured against a target, not enforced as
+ * a deadline: past it the controller logs once and keeps waiting while the
+ * Slice makes progress. */
+exports.FLEET_READINESS_TARGET_MS = 120_000;
+/** Readiness fails only when nothing has progressed for this long: no output
+ * and no state change while the health route gives no answer, or only
+ * unhealthy answers. Request logs a probe causes are not progress for an
+ * answering but unhealthy application, so that path ignores output. */
+exports.FLEET_READINESS_STALL_MS = 120_000;
+/** Recent output lines carried into a startup failure message. */
+exports.FLEET_FAILURE_OUTPUT_LINES = 4;
+const FLEET_FAILURE_OUTPUT_LINE_LENGTH = 200;
+function describeExit(exit) {
+    if (!exit)
+        return 'status unknown';
+    return exit.signal
+        ? `signal ${exit.signal}`
+        : `status ${exit.code ?? 'unknown'}`;
+}
 function fleetRestartDelay(attempt) {
     if (!Number.isInteger(attempt) || attempt < 0) {
         throw new errors_js_1.LocalDevContractError('invalid-input', 'Fleet restart attempt must be a non-negative integer.');
@@ -104,6 +124,8 @@ async function terminateOwnedChild(child, ownedIdentity) {
     await assertOwnedGroupAbsent(child.pid, deadline);
 }
 exports.FLEET_RUNNER_STOP_MESSAGE = 'ebk-fleet-runner-stop';
+exports.FLEET_RUNNER_QUIESCE_MESSAGE = 'ebk-fleet-runner-quiesce';
+exports.FLEET_RUNNER_QUIESCED_MESSAGE = 'ebk-fleet-runner-quiesced';
 /** A runner coordinates detached children and native cleanup. Never force-kill
  * that coordinator on a client observation deadline. An explicit diagnostic
  * deadline retains live ownership and reports failure instead of orphaning a
@@ -151,13 +173,69 @@ async function terminateCoordinatingFleetRunner(child, runnerGeneration, timeout
         }
     });
 }
+/** Ask the exact runner generation to stop its writer but keep running with
+ * pending cleanup authority. It acknowledges once its owned child group is
+ * absent, or at once when quiescence reaches it before its writer started.
+ * Like the stop, this has no client deadline: the runner bounds its own
+ * child termination. A clean exit also settles the request, because the
+ * runner then retained its authority in state for a cleanup retry.
+ */
+async function quiesceCoordinatingFleetRunner(child, runnerGeneration) {
+    const failed = (exit) => new errors_js_1.LocalDevContractError('cleanup-incomplete', `Fleet runner exited with ${describeExit(exit)} while quiescing; its claim and cleanup authority remain retained.`);
+    if (child.exitCode !== null || child.signalCode !== null) {
+        if (child.exitCode === 0)
+            return;
+        throw failed({ code: child.exitCode, signal: child.signalCode });
+    }
+    if (!child.connected || !child.send) {
+        throw new errors_js_1.LocalDevContractError('cleanup-incomplete', 'Fleet runner control channel is unavailable; live ownership is retained.');
+    }
+    await new Promise((resolvePromise, reject) => {
+        const settle = (error) => {
+            child.removeListener('message', acknowledged);
+            child.removeListener('exit', exited);
+            child.removeListener('error', errored);
+            if (error)
+                reject(error);
+            else
+                resolvePromise();
+        };
+        const acknowledged = (value) => {
+            if (value &&
+                typeof value === 'object' &&
+                Object.keys(value).sort().join(',') === 'runnerGeneration,type' &&
+                'type' in value &&
+                value.type === exports.FLEET_RUNNER_QUIESCED_MESSAGE &&
+                'runnerGeneration' in value &&
+                value.runnerGeneration === runnerGeneration)
+                settle();
+        };
+        const exited = (code, signal) => code === 0 && signal === null
+            ? settle()
+            : settle(failed({ code, signal }));
+        const errored = (error) => settle(error);
+        child.on('message', acknowledged);
+        child.once('exit', exited);
+        child.once('error', errored);
+        try {
+            child.send?.({ type: exports.FLEET_RUNNER_QUIESCE_MESSAGE, runnerGeneration }, (error) => {
+                if (error)
+                    settle(error);
+            });
+        }
+        catch (error) {
+            settle(error instanceof Error ? error : new Error(String(error)));
+        }
+    });
+}
 async function defaultProbe(url, signal) {
     try {
         const response = await fetch(url, {
             redirect: 'manual',
             signal: AbortSignal.any([signal, AbortSignal.timeout(2_000)]),
         });
-        return response.ok;
+        await response.body?.cancel().catch(() => undefined);
+        return response.ok ? true : 'unhealthy';
     }
     catch {
         return false;
@@ -184,8 +262,15 @@ class FleetProcessController {
     #child;
     #cleanup;
     #effects;
-    #healthTimeoutMs;
+    #exits = new WeakMap();
+    #outputTail = [];
+    /** The live runner that acknowledged quiescence and still holds pending
+     * cleanup authority. */
+    #quiesced;
+    #lastProgressAt = 0;
     #provisioningTimeoutMs;
+    #readinessStallMs;
+    #readinessTargetMs;
     #lifetime = new AbortController();
     #startup = new AbortController();
     #logStore;
@@ -208,11 +293,15 @@ class FleetProcessController {
             options.adapter.adapterVersion !== 1 && options.adapter.lifecycle
                 ? 'pending'
                 : 'not-declared';
-        this.#healthTimeoutMs = options.healthTimeoutMs ?? 120_000;
         this.#provisioningTimeoutMs = options.provisioningTimeoutMs ?? 30 * 60_000;
+        this.#readinessStallMs =
+            options.readinessStallMs ?? exports.FLEET_READINESS_STALL_MS;
+        this.#readinessTargetMs =
+            options.readinessTargetMs ?? exports.FLEET_READINESS_TARGET_MS;
         for (const timeout of [
-            this.#healthTimeoutMs,
             this.#provisioningTimeoutMs,
+            this.#readinessStallMs,
+            this.#readinessTargetMs,
         ]) {
             if (!Number.isSafeInteger(timeout) ||
                 timeout < 1 ||
@@ -232,7 +321,9 @@ class FleetProcessController {
             launch: effects.launch,
             prepareRestart: effects.prepareRestart,
             probe: effects.probe ?? defaultProbe,
+            quiesce: effects.quiesce,
             runnerGeneration: effects.runnerGeneration,
+            startTargetExit: effects.startTargetExit,
             terminate: effects.terminate ?? terminateOwnedChild,
         };
     }
@@ -250,11 +341,52 @@ class FleetProcessController {
     }
     #transition(state) {
         this.#state = state;
+        this.#lastProgressAt = Date.now();
         this.#onStateChange?.(state, this.#restartAttempts);
     }
+    /** Keep the newest redacted log lines of the active attempt so a startup
+     * failure can name the real error instead of only a classification. */
+    #remember(attempt, envelopes) {
+        if (attempt !== this.#activeAttempt || !envelopes)
+            return;
+        for (const { message } of envelopes) {
+            const line = message.trim();
+            if (line.length === 0 || line === '[empty line]')
+                continue;
+            this.#outputTail.push(line.length > FLEET_FAILURE_OUTPUT_LINE_LENGTH
+                ? `${line.slice(0, FLEET_FAILURE_OUTPUT_LINE_LENGTH - 1)}\u2026`
+                : line);
+        }
+        this.#outputTail.splice(0, Math.max(0, this.#outputTail.length - exports.FLEET_FAILURE_OUTPUT_LINES));
+    }
     #capture(child, attempt) {
-        child.stdout?.on('data', (chunk) => this.#logStore?.writeChunk(this.#name, 'stdout', attempt, chunk));
-        child.stderr?.on('data', (chunk) => this.#logStore?.writeChunk(this.#name, 'stderr', attempt, chunk));
+        const observe = (stream) => (chunk) => {
+            if (attempt === this.#activeAttempt)
+                this.#lastProgressAt = Date.now();
+            this.#remember(attempt, this.#logStore?.writeChunk(this.#name, stream, attempt, chunk));
+        };
+        child.stdout?.on('data', observe('stdout'));
+        child.stderr?.on('data', observe('stderr'));
+    }
+    /** A structured warning in the Slice log; it is never a failure. */
+    #logWarning(message) {
+        this.#logStore?.writeLine(this.#name, 'stderr', this.#activeAttempt, JSON.stringify({ level: 'warn', message }));
+    }
+    #exitedBeforeReadiness(child) {
+        const runnerExit = child ? this.#exits.get(child) : undefined;
+        let startExit;
+        try {
+            startExit = child ? this.#effects.startTargetExit?.(child) : undefined;
+        }
+        catch {
+            startExit = undefined;
+        }
+        const output = this.#outputTail.length > 0
+            ? ` Last output: ${this.#outputTail.join(' | ')}.`
+            : '';
+        throw new errors_js_1.LocalDevContractError('process-exited', `${this.#name} exited before readiness (${startExit
+            ? `start target ${describeExit(startExit)}`
+            : describeExit(runnerExit)}).${output} Inspect ebk dev logs ${this.#name}; owned cleanup settles before a retry.`);
     }
     #launch() {
         this.#startup = new AbortController();
@@ -265,12 +397,14 @@ class FleetProcessController {
         const launched = this.#effects.launch(this.#activeAttempt);
         this.#child = launched.child;
         this.#activeAttempt = launched.attempt;
+        this.#outputTail = [];
         this.#capture(launched.child, launched.attempt);
         this.#transition('starting');
         launched.child.once('exit', (code, signal) => {
+            this.#exits.set(launched.child, { code, signal });
             if (this.#child === launched.child)
                 this.#child = undefined;
-            this.#logStore?.flush(this.#name, undefined, launched.attempt);
+            this.#remember(launched.attempt, this.#logStore?.flush(this.#name, undefined, launched.attempt));
             if (this.#stopping || this.#lifetime.signal.aborted)
                 return;
             this.#transition('unavailable');
@@ -295,13 +429,17 @@ class FleetProcessController {
             this.#startup.signal,
         ]);
         const provisioningDeadline = Date.now() + this.#provisioningTimeoutMs;
-        let deadline;
+        // Set once the web application has launched; HTTP readiness starts then.
+        let webStartedAt;
+        let unhealthySince;
+        let slowerThanTarget = false;
         const target = new URL(this.#adapter.healthPath, this.#upstream);
         while (!signal.aborted && !this.#stopping) {
             if (!child || this.#child !== child) {
-                throw new errors_js_1.LocalDevContractError('readiness-timeout', `${this.#name} exited before readiness.`);
+                // A crash is not a timeout: report the exit and the newest output.
+                this.#exitedBeforeReadiness(child);
             }
-            if (deadline === undefined) {
+            if (webStartedAt === undefined) {
                 const native = await this.#effects.nativeDeployment?.(signal);
                 if (native === 'failed') {
                     this.#transition('unavailable');
@@ -317,24 +455,45 @@ class FleetProcessController {
                     await this.#effects.delay(500, signal);
                     continue;
                 }
-                deadline = Date.now() + this.#healthTimeoutMs;
+                // The launch itself is progress: the stall clock starts here.
+                webStartedAt = Date.now();
+                this.#lastProgressAt = webStartedAt;
             }
-            if (Date.now() >= deadline)
-                break;
-            if (await this.#effects.probe(target, signal)) {
+            const probe = await this.#effects.probe(target, signal);
+            if (probe === true) {
                 if (this.#child !== child || this.#stopping || signal.aborted)
                     continue;
                 this.#restartAttempts = 0;
+                if (slowerThanTarget) {
+                    this.#logWarning(`${this.#name} became ready ${Math.round((Date.now() - webStartedAt) / 1_000)} s after web launch, slower than the ${Math.round(this.#readinessTargetMs / 1_000)} s readiness target.`);
+                }
                 this.#transition('ready');
                 return;
             }
+            const now = Date.now();
+            if (!slowerThanTarget && now - webStartedAt > this.#readinessTargetMs) {
+                slowerThanTarget = true;
+                this.#logWarning(`${this.#name} is slower than the ${Math.round(this.#readinessTargetMs / 1_000)} s readiness target; still waiting while it makes progress.`);
+            }
+            if (probe === 'unhealthy') {
+                unhealthySince ??= now;
+                if (now - unhealthySince > this.#readinessStallMs) {
+                    this.#transition('unavailable');
+                    throw new errors_js_1.LocalDevContractError('health-check-failing', `${this.#name} answered its health check ${this.#adapter.healthPath} only with unhealthy responses for ${Math.round((now - unhealthySince) / 1_000)} s. Inspect ebk dev logs ${this.#name}; owned cleanup settles before a retry.`);
+                }
+            }
+            else {
+                unhealthySince = undefined;
+                const stalledMs = now - this.#lastProgressAt;
+                if (stalledMs > this.#readinessStallMs) {
+                    this.#transition('unavailable');
+                    throw new errors_js_1.LocalDevContractError('readiness-stalled', `${this.#name} made no progress for ${Math.round(stalledMs / 1_000)} s: no output and no answer from its health check ${this.#adapter.healthPath}. Inspect ebk dev logs ${this.#name}; owned cleanup settles before a retry.`);
+                }
+            }
             await this.#effects.delay(500, signal);
         }
         this.#transition('unavailable');
-        if (signal.aborted || this.#stopping) {
-            throw new errors_js_1.LocalDevContractError('startup-interrupted', `${this.#name} startup was interrupted; owned cleanup must settle before retrying.`);
-        }
-        throw new errors_js_1.LocalDevContractError('readiness-timeout', `${this.#name} did not become ready within ${this.#healthTimeoutMs}ms.`);
+        throw new errors_js_1.LocalDevContractError('startup-interrupted', `${this.#name} startup was interrupted; owned cleanup must settle before retrying.`);
     }
     async #autoRestart() {
         const signal = AbortSignal.any([
@@ -358,7 +517,12 @@ class FleetProcessController {
             await this.#waitForReadiness();
         }
         catch (error) {
-            if (this.#child) {
+            // An interrupting quiesce or stop owns this runner. A bare stop here
+            // would run declared cleanup while peer writers may still be live.
+            if (this.#child &&
+                !this.#stopping &&
+                !this.#lifetime.signal.aborted &&
+                !this.#startup.signal.aborted) {
                 await this.#effects.terminate(this.#child);
             }
             if (!this.#restartTask &&
@@ -414,25 +578,78 @@ class FleetProcessController {
             throw error;
         }
     }
-    async stop(options) {
+    /** Stop the owned writer without removing its cloud/dependency
+     * prerequisites. A Fleet transaction quiesces every selected Slice before
+     * reverse cleanup; a failed dependent cleanup must not leave a dependency
+     * writer running. A coordinating runner stays alive with pending cleanup
+     * authority, so its later cleanup output still reaches this Slice's log. */
+    async quiesce() {
         if (this.#state === 'remote')
             return;
         this.#stopping = true;
         this.#startup.abort();
-        if (this.#child)
-            await this.#effects.terminate(this.#child);
-        await this.#restartTask;
-        this.#restartTask = undefined;
+        try {
+            const child = this.#child;
+            if (child && child !== this.#quiesced) {
+                if (this.#effects.quiesce) {
+                    await this.#effects.quiesce(child);
+                    if (this.#child === child)
+                        this.#quiesced = child;
+                }
+                else {
+                    // The same generation-bound intent used by explicit restart lets
+                    // the runner stop its native child and retain pending cleanup
+                    // authority. A signal alone would run cleanup before peer writers
+                    // were stopped.
+                    await this.#effects.prepareRestart?.(child);
+                    await this.#effects.terminate(child);
+                    if (this.#effects.prepareRestart &&
+                        (child.exitCode !== 0 || child.signalCode !== null)) {
+                        throw new errors_js_1.LocalDevContractError('cleanup-incomplete', `${this.#name} runner did not quiesce successfully; its claim and cleanup authority remain retained.`);
+                    }
+                }
+            }
+            await this.#restartTask;
+            this.#restartTask = undefined;
+        }
+        catch (error) {
+            // Status must not keep reporting pending authority as healthy.
+            this.#cleanup =
+                this.#adapter.adapterVersion !== 1 && this.#adapter.lifecycle
+                    ? 'failed'
+                    : 'not-declared';
+            throw error;
+        }
         this.#logStore?.flush(this.#name);
-        if (options.cleanup && this.#effects.cleanup) {
-            try {
+        // Unavailable is still selected. Quiescence never releases a claim or
+        // means cloud cleanup succeeded, and #stopping prevents automatic restart.
+        this.#transition('unavailable');
+    }
+    async stop(options) {
+        if (this.#state === 'remote')
+            return;
+        await this.quiesce();
+        try {
+            const child = this.#child;
+            if (child) {
+                // A quiesced runner still holds cleanup authority. Stopping it runs
+                // the declared cleanup in that runner, whose output this controller
+                // captures. A restart intent instead leaves cleanup pending.
+                if (!options.cleanup)
+                    await this.#effects.prepareRestart?.(child);
+                await this.#effects.terminate(child);
+            }
+            if (options.cleanup && this.#effects.cleanup) {
                 await this.#effects.cleanup();
                 this.#cleanup = 'succeeded';
             }
-            catch (error) {
-                this.#cleanup = 'failed';
-                throw error;
-            }
+        }
+        catch (error) {
+            this.#cleanup =
+                this.#adapter.adapterVersion !== 1 && this.#adapter.lifecycle
+                    ? 'failed'
+                    : 'not-declared';
+            throw error;
         }
         this.#transition('remote');
     }

package/src/local-dev/fleet-runtime.d.ts

@@ -1,10 +1,17 @@
+import type { spawn } from 'node:child_process';
 import { inspectLocalDevelopment } from '../executors/dev-doctor/dev-doctor.js';
+import { type LocalDevAdapter } from './contracts.js';
+import { FleetLogStore } from './fleet-logs.js';
+import type { FleetProcessIdentity } from './fleet-lock.js';
+import { type FleetProcessEffects } from './fleet-process.js';
 import { FleetSession } from './fleet-session.js';
 import { FleetSupervisor } from './fleet-supervisor.js';
 import { type FleetStartupRequest } from './fleet-startup.js';
 import { type WorkspaceFleetPlan } from './nx-workspace.js';
 import { allocateFleetPort } from './ports.js';
 import { type PersonalStageResolver } from './personal-stage.js';
+import { type FleetRuntimePaths } from './runtime-paths.js';
+import { type LaunchedFleetSliceRunner } from './slice-runner.js';
 export interface RunningFleetRuntime {
     fleetId: string;
     origin: string;
@@ -19,4 +26,34 @@ export interface FleetRuntimeDependencies {
     resolveRouter?: (request: FleetStartupRequest) => Promise<URL>;
 }
 export declare function resolveDevelopmentRouter(request: FleetStartupRequest): Promise<URL>;
+export interface FleetSliceRunnerEffectsInput {
+    adapter: LocalDevAdapter;
+    logStore: FleetLogStore;
+    name: string;
+    origin: URL;
+    parent: FleetProcessIdentity;
+    port: number;
+    projects: WorkspaceFleetPlan['projects'];
+    runtimePaths: FleetRuntimePaths;
+    sessionGeneration: string;
+    stage: string;
+    startupPath: string;
+    workspaceRealpath: string;
+}
+/** The runner daemon a Slice launches and retries cleanup with. Production
+ * omits both; real-runner specs substitute a fixture daemon. */
+export interface FleetSliceRunnerDaemon {
+    daemonPath?: string;
+    spawn?: typeof spawn;
+}
+export interface FleetSliceRunnerEffects {
+    /** The runner most recently launched for this Slice, if any. */
+    activeRunner: () => LaunchedFleetSliceRunner | undefined;
+    effects: FleetProcessEffects;
+}
+/** One Slice's Fleet process effects: a coordinating runner per launch,
+ * generation-bound restart and quiescence, and cleanup whose retry output
+ * joins the Slice log store. startFleetRuntime and the real-runner
+ * quiescence specs share this wiring, so a dropped effect fails those specs. */
+export declare function createFleetSliceRunnerEffects(input: FleetSliceRunnerEffectsInput, daemon?: FleetSliceRunnerDaemon): FleetSliceRunnerEffects;
 export declare function startFleetRuntime(startupPath: string, dependencies?: FleetRuntimeDependencies): Promise<RunningFleetRuntime>;