@empire-builder-kit/nx 1.0.0-rc.44 → 1.0.0-rc.45
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +3 -3
- package/src/blueprints/external-sandbox.d.ts +1 -1
- package/src/cli/main.d.ts +11 -5
- package/src/cli/main.js +88 -29
- package/src/container-images.d.ts +1 -1
- package/src/container-images.js +1 -1
- package/src/executors/deployment-plan/deployment-plan.js +5 -1
- package/src/executors/deployment-plan/schema.json +1 -1
- package/src/executors/fleet/fleet.js +4 -0
- package/src/executors/fleet/schema.json +1 -1
- package/src/executors/local-deploy/federated-credential-transport.d.ts +4 -3
- package/src/executors/local-deploy/federated-credential-transport.js +15 -5
- package/src/foundation/cost-estimate.d.ts +8 -3
- package/src/foundation/cost-estimate.js +42 -19
- package/src/foundation/index.d.ts +1 -0
- package/src/foundation/index.js +1 -0
- package/src/foundation/local-private-smoke-access.js +6 -0
- package/src/foundation/logging.d.ts +3 -1
- package/src/foundation/logging.js +8 -1
- package/src/foundation/observability.d.ts +42 -0
- package/src/foundation/observability.js +181 -0
- package/src/foundation/preflight.js +1 -1
- package/src/generators/function/function.js +5 -2
- package/src/generators/observability-infrastructure.d.ts +2 -0
- package/src/generators/observability-infrastructure.js +31 -0
- package/src/generators/observability-policy.d.ts +4 -0
- package/src/generators/observability-policy.js +23 -0
- package/src/generators/preset/dependency-overrides.d.ts +17 -6
- package/src/generators/preset/dependency-overrides.js +17 -6
- package/src/generators/preset/tool-versions.d.ts +2 -2
- package/src/generators/preset/tool-versions.js +2 -2
- package/src/generators/preset/workspace-files-adopter-skills.js +28 -17
- package/src/generators/preset/workspace-files-agent-development.d.ts +9 -0
- package/src/generators/preset/workspace-files-agent-development.js +42 -0
- package/src/generators/preset/workspace-files-thin-deploy.js +12 -3
- package/src/generators/preset/workspace-files-thin-docs.js +10 -22
- package/src/generators/preset/workspace-files.d.ts +1 -0
- package/src/generators/preset/workspace-files.js +89 -7
- package/src/generators/slice/built-in-plan-alarm-identity.d.ts +173 -0
- package/src/generators/slice/built-in-plan-alarm-identity.js +202 -0
- package/src/generators/slice/built-in-plan-api-contract-files.js +136 -0
- package/src/generators/slice/built-in-plan-database-lifecycle-files.js +1 -1
- package/src/generators/slice/built-in-plan-event-contract-literal.d.ts +6 -0
- package/src/generators/slice/built-in-plan-event-contract-literal.js +52 -0
- package/src/generators/slice/built-in-plan-infrastructure-files.js +304 -144
- package/src/generators/slice/built-in-plan-key-value-retention-infrastructure.js +2 -2
- package/src/generators/slice/built-in-plan-metadata-files.js +33 -1
- package/src/generators/slice/built-in-plan-operations-files.js +106 -6
- package/src/generators/slice/built-in-plan-project-files.js +1 -0
- package/src/generators/slice/built-in-plan-release-files.js +2 -43
- package/src/generators/slice/built-in-plan-reliability-files.js +222 -236
- package/src/generators/slice/built-in-plan-shared.d.ts +29 -3
- package/src/generators/slice/built-in-plan-shared.js +29 -2
- package/src/generators/slice/built-in-plan-web-files.js +18 -1
- package/src/generators/slice/built-in-plan.js +1 -0
- package/src/generators/slice/slice.js +2 -0
- package/src/generators/upgrade/upgrade.d.ts +4 -3
- package/src/generators/upgrade/upgrade.js +50 -26
- package/src/generators/workload/frozen-container.d.ts +1 -1
- package/src/generators/workload/workload.js +5 -1
- package/src/local-dev/dev-command-core.d.ts +24 -3
- package/src/local-dev/dev-command-core.js +124 -12
- package/src/local-dev/errors.d.ts +19 -1
- package/src/local-dev/errors.js +18 -1
- package/src/local-dev/fleet-client.d.ts +64 -6
- package/src/local-dev/fleet-client.js +189 -15
- package/src/local-dev/fleet-process.d.ts +47 -2
- package/src/local-dev/fleet-process.js +248 -31
- package/src/local-dev/fleet-runtime.d.ts +37 -0
- package/src/local-dev/fleet-runtime.js +127 -69
- package/src/local-dev/fleet-session.d.ts +1 -0
- package/src/local-dev/fleet-session.js +42 -3
- package/src/local-dev/fleet-state.d.ts +5 -0
- package/src/local-dev/fleet-state.js +11 -2
- package/src/local-dev/fleet-supervisor.d.ts +6 -0
- package/src/local-dev/fleet-supervisor.js +49 -1
- package/src/local-dev/slice-runner.d.ts +19 -1
- package/src/local-dev/slice-runner.js +290 -91
- package/src/repository-policy/deployment-plan.d.ts +1 -1
- package/src/repository-policy/deployment-plan.js +16 -1
- package/src/secrets/generated-aws-control-plane.d.ts +11 -1
- package/src/secrets/generated-aws-control-plane.js +92 -29
- package/src/secrets/operator.d.ts +3 -2
- package/src/secrets/operator.js +27 -5
- package/src/supply-chain/generated-files.js +9 -0
|
@@ -9,6 +9,11 @@ export interface FleetProcessLaunch {
|
|
|
9
9
|
attempt: number;
|
|
10
10
|
child: ChildProcess;
|
|
11
11
|
}
|
|
12
|
+
/** How an owned process ended: an exit status or a terminating signal. */
|
|
13
|
+
export interface FleetProcessExit {
|
|
14
|
+
code: number | null;
|
|
15
|
+
signal: string | null;
|
|
16
|
+
}
|
|
12
17
|
export interface FleetProcessEffects {
|
|
13
18
|
/** Native provisioning may outlast HTTP readiness. This is an owned launch
|
|
14
19
|
* notification, not an HTTP-health result. Non-native adapters omit it. */
|
|
@@ -17,15 +22,39 @@ export interface FleetProcessEffects {
|
|
|
17
22
|
cleanup?: () => Promise<void>;
|
|
18
23
|
delay?: (milliseconds: number, signal: AbortSignal) => Promise<void>;
|
|
19
24
|
launch: (attempt: number) => FleetProcessLaunch;
|
|
20
|
-
|
|
25
|
+
/** `true` is healthy, `'unhealthy'` is an HTTP response that is not OK,
|
|
26
|
+
* and `false` is no response (refused, reset, or unanswered). */
|
|
27
|
+
probe?: (url: URL, signal: AbortSignal) => Promise<FleetProbeResult>;
|
|
21
28
|
prepareRestart?: (child: ChildProcess) => Promise<void>;
|
|
29
|
+
/** Stop the writer while the owned process keeps its pending cleanup
|
|
30
|
+
* authority. A coordinating runner stays alive and later runs declared
|
|
31
|
+
* cleanup itself when stopped. Without this effect, quiescence writes the
|
|
32
|
+
* restart intent and terminates the child. */
|
|
33
|
+
quiesce?: (child: ChildProcess) => Promise<void>;
|
|
22
34
|
runnerGeneration?: () => string | undefined;
|
|
35
|
+
/** The Slice start target's own exit, when the launched child is a runner
|
|
36
|
+
* that recorded it. The runner exits cleanly after its child crashes, so
|
|
37
|
+
* its own status alone would hide the real failure. */
|
|
38
|
+
startTargetExit?: (child: ChildProcess) => FleetProcessExit | undefined;
|
|
23
39
|
terminate?: (child: ChildProcess) => Promise<void>;
|
|
24
40
|
}
|
|
41
|
+
export type FleetProbeResult = boolean | 'unhealthy';
|
|
42
|
+
/** Web readiness after launch is measured against a target, not enforced as
|
|
43
|
+
* a deadline: past it the controller logs once and keeps waiting while the
|
|
44
|
+
* Slice makes progress. */
|
|
45
|
+
export declare const FLEET_READINESS_TARGET_MS = 120000;
|
|
46
|
+
/** Readiness fails only when nothing has progressed for this long: no output
|
|
47
|
+
* and no state change while the health route gives no answer, or only
|
|
48
|
+
* unhealthy answers. Request logs a probe causes are not progress for an
|
|
49
|
+
* answering but unhealthy application, so that path ignores output. */
|
|
50
|
+
export declare const FLEET_READINESS_STALL_MS = 120000;
|
|
51
|
+
/** Recent output lines carried into a startup failure message. */
|
|
52
|
+
export declare const FLEET_FAILURE_OUTPUT_LINES = 4;
|
|
25
53
|
export interface FleetProcessControllerOptions {
|
|
26
54
|
adapter: LocalDevAdapter;
|
|
27
|
-
healthTimeoutMs?: number;
|
|
28
55
|
provisioningTimeoutMs?: number;
|
|
56
|
+
readinessStallMs?: number;
|
|
57
|
+
readinessTargetMs?: number;
|
|
29
58
|
logStore?: FleetLogStore;
|
|
30
59
|
maxRestartAttempts: number;
|
|
31
60
|
name: string;
|
|
@@ -43,12 +72,22 @@ export declare function planFleetRestart(completedRestartAttempts: number, maxRe
|
|
|
43
72
|
export declare function assertOwnedGroupAbsent(pid: number, deadline?: number): Promise<void>;
|
|
44
73
|
export declare function terminateOwnedChild(child: ChildProcess, ownedIdentity?: FleetProcessIdentity): Promise<void>;
|
|
45
74
|
export declare const FLEET_RUNNER_STOP_MESSAGE = "ebk-fleet-runner-stop";
|
|
75
|
+
export declare const FLEET_RUNNER_QUIESCE_MESSAGE = "ebk-fleet-runner-quiesce";
|
|
76
|
+
export declare const FLEET_RUNNER_QUIESCED_MESSAGE = "ebk-fleet-runner-quiesced";
|
|
46
77
|
/** A runner coordinates detached children and native cleanup. Never force-kill
|
|
47
78
|
* that coordinator on a client observation deadline. An explicit diagnostic
|
|
48
79
|
* deadline retains live ownership and reports failure instead of orphaning a
|
|
49
80
|
* writer (and falsely permitting stale-process recovery).
|
|
50
81
|
*/
|
|
51
82
|
export declare function terminateCoordinatingFleetRunner(child: ChildProcess, runnerGeneration: string, timeoutMs?: number): Promise<void>;
|
|
83
|
+
/** Ask the exact runner generation to stop its writer but keep running with
|
|
84
|
+
* pending cleanup authority. It acknowledges once its owned child group is
|
|
85
|
+
* absent, or at once when quiescence reaches it before its writer started.
|
|
86
|
+
* Like the stop, this has no client deadline: the runner bounds its own
|
|
87
|
+
* child termination. A clean exit also settles the request, because the
|
|
88
|
+
* runner then retained its authority in state for a cleanup retry.
|
|
89
|
+
*/
|
|
90
|
+
export declare function quiesceCoordinatingFleetRunner(child: ChildProcess, runnerGeneration: string): Promise<void>;
|
|
52
91
|
export declare class FleetProcessController {
|
|
53
92
|
#private;
|
|
54
93
|
constructor(options: FleetProcessControllerOptions, effects: FleetProcessEffects);
|
|
@@ -62,6 +101,12 @@ export declare class FleetProcessController {
|
|
|
62
101
|
* responsibility for native termination and exact cleanup before down runs. */
|
|
63
102
|
interruptStartup(): void;
|
|
64
103
|
restart(): Promise<void>;
|
|
104
|
+
/** Stop the owned writer without removing its cloud/dependency
|
|
105
|
+
* prerequisites. A Fleet transaction quiesces every selected Slice before
|
|
106
|
+
* reverse cleanup; a failed dependent cleanup must not leave a dependency
|
|
107
|
+
* writer running. A coordinating runner stays alive with pending cleanup
|
|
108
|
+
* authority, so its later cleanup output still reaches this Slice's log. */
|
|
109
|
+
quiesce(): Promise<void>;
|
|
65
110
|
stop(options: {
|
|
66
111
|
cleanup: boolean;
|
|
67
112
|
}): Promise<void>;
|
|
@@ -1,15 +1,35 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.FleetProcessController = exports.FLEET_RUNNER_STOP_MESSAGE = void 0;
|
|
3
|
+
exports.FleetProcessController = exports.FLEET_RUNNER_QUIESCED_MESSAGE = exports.FLEET_RUNNER_QUIESCE_MESSAGE = exports.FLEET_RUNNER_STOP_MESSAGE = exports.FLEET_FAILURE_OUTPUT_LINES = exports.FLEET_READINESS_STALL_MS = exports.FLEET_READINESS_TARGET_MS = void 0;
|
|
4
4
|
exports.fleetRestartDelay = fleetRestartDelay;
|
|
5
5
|
exports.planFleetRestart = planFleetRestart;
|
|
6
6
|
exports.assertOwnedGroupAbsent = assertOwnedGroupAbsent;
|
|
7
7
|
exports.terminateOwnedChild = terminateOwnedChild;
|
|
8
8
|
exports.terminateCoordinatingFleetRunner = terminateCoordinatingFleetRunner;
|
|
9
|
+
exports.quiesceCoordinatingFleetRunner = quiesceCoordinatingFleetRunner;
|
|
9
10
|
const node_child_process_1 = require("node:child_process");
|
|
10
11
|
const errors_js_1 = require("./errors.js");
|
|
11
12
|
const process_group_js_1 = require("./process-group.js");
|
|
12
13
|
const process_identity_js_1 = require("./process-identity.js");
|
|
14
|
+
/** Web readiness after launch is measured against a target, not enforced as
|
|
15
|
+
* a deadline: past it the controller logs once and keeps waiting while the
|
|
16
|
+
* Slice makes progress. */
|
|
17
|
+
exports.FLEET_READINESS_TARGET_MS = 120_000;
|
|
18
|
+
/** Readiness fails only when nothing has progressed for this long: no output
|
|
19
|
+
* and no state change while the health route gives no answer, or only
|
|
20
|
+
* unhealthy answers. Request logs a probe causes are not progress for an
|
|
21
|
+
* answering but unhealthy application, so that path ignores output. */
|
|
22
|
+
exports.FLEET_READINESS_STALL_MS = 120_000;
|
|
23
|
+
/** Recent output lines carried into a startup failure message. */
|
|
24
|
+
exports.FLEET_FAILURE_OUTPUT_LINES = 4;
|
|
25
|
+
const FLEET_FAILURE_OUTPUT_LINE_LENGTH = 200;
|
|
26
|
+
function describeExit(exit) {
|
|
27
|
+
if (!exit)
|
|
28
|
+
return 'status unknown';
|
|
29
|
+
return exit.signal
|
|
30
|
+
? `signal ${exit.signal}`
|
|
31
|
+
: `status ${exit.code ?? 'unknown'}`;
|
|
32
|
+
}
|
|
13
33
|
function fleetRestartDelay(attempt) {
|
|
14
34
|
if (!Number.isInteger(attempt) || attempt < 0) {
|
|
15
35
|
throw new errors_js_1.LocalDevContractError('invalid-input', 'Fleet restart attempt must be a non-negative integer.');
|
|
@@ -104,6 +124,8 @@ async function terminateOwnedChild(child, ownedIdentity) {
|
|
|
104
124
|
await assertOwnedGroupAbsent(child.pid, deadline);
|
|
105
125
|
}
|
|
106
126
|
exports.FLEET_RUNNER_STOP_MESSAGE = 'ebk-fleet-runner-stop';
|
|
127
|
+
exports.FLEET_RUNNER_QUIESCE_MESSAGE = 'ebk-fleet-runner-quiesce';
|
|
128
|
+
exports.FLEET_RUNNER_QUIESCED_MESSAGE = 'ebk-fleet-runner-quiesced';
|
|
107
129
|
/** A runner coordinates detached children and native cleanup. Never force-kill
|
|
108
130
|
* that coordinator on a client observation deadline. An explicit diagnostic
|
|
109
131
|
* deadline retains live ownership and reports failure instead of orphaning a
|
|
@@ -151,13 +173,69 @@ async function terminateCoordinatingFleetRunner(child, runnerGeneration, timeout
|
|
|
151
173
|
}
|
|
152
174
|
});
|
|
153
175
|
}
|
|
176
|
+
/** Ask the exact runner generation to stop its writer but keep running with
|
|
177
|
+
* pending cleanup authority. It acknowledges once its owned child group is
|
|
178
|
+
* absent, or at once when quiescence reaches it before its writer started.
|
|
179
|
+
* Like the stop, this has no client deadline: the runner bounds its own
|
|
180
|
+
* child termination. A clean exit also settles the request, because the
|
|
181
|
+
* runner then retained its authority in state for a cleanup retry.
|
|
182
|
+
*/
|
|
183
|
+
async function quiesceCoordinatingFleetRunner(child, runnerGeneration) {
|
|
184
|
+
const failed = (exit) => new errors_js_1.LocalDevContractError('cleanup-incomplete', `Fleet runner exited with ${describeExit(exit)} while quiescing; its claim and cleanup authority remain retained.`);
|
|
185
|
+
if (child.exitCode !== null || child.signalCode !== null) {
|
|
186
|
+
if (child.exitCode === 0)
|
|
187
|
+
return;
|
|
188
|
+
throw failed({ code: child.exitCode, signal: child.signalCode });
|
|
189
|
+
}
|
|
190
|
+
if (!child.connected || !child.send) {
|
|
191
|
+
throw new errors_js_1.LocalDevContractError('cleanup-incomplete', 'Fleet runner control channel is unavailable; live ownership is retained.');
|
|
192
|
+
}
|
|
193
|
+
await new Promise((resolvePromise, reject) => {
|
|
194
|
+
const settle = (error) => {
|
|
195
|
+
child.removeListener('message', acknowledged);
|
|
196
|
+
child.removeListener('exit', exited);
|
|
197
|
+
child.removeListener('error', errored);
|
|
198
|
+
if (error)
|
|
199
|
+
reject(error);
|
|
200
|
+
else
|
|
201
|
+
resolvePromise();
|
|
202
|
+
};
|
|
203
|
+
const acknowledged = (value) => {
|
|
204
|
+
if (value &&
|
|
205
|
+
typeof value === 'object' &&
|
|
206
|
+
Object.keys(value).sort().join(',') === 'runnerGeneration,type' &&
|
|
207
|
+
'type' in value &&
|
|
208
|
+
value.type === exports.FLEET_RUNNER_QUIESCED_MESSAGE &&
|
|
209
|
+
'runnerGeneration' in value &&
|
|
210
|
+
value.runnerGeneration === runnerGeneration)
|
|
211
|
+
settle();
|
|
212
|
+
};
|
|
213
|
+
const exited = (code, signal) => code === 0 && signal === null
|
|
214
|
+
? settle()
|
|
215
|
+
: settle(failed({ code, signal }));
|
|
216
|
+
const errored = (error) => settle(error);
|
|
217
|
+
child.on('message', acknowledged);
|
|
218
|
+
child.once('exit', exited);
|
|
219
|
+
child.once('error', errored);
|
|
220
|
+
try {
|
|
221
|
+
child.send?.({ type: exports.FLEET_RUNNER_QUIESCE_MESSAGE, runnerGeneration }, (error) => {
|
|
222
|
+
if (error)
|
|
223
|
+
settle(error);
|
|
224
|
+
});
|
|
225
|
+
}
|
|
226
|
+
catch (error) {
|
|
227
|
+
settle(error instanceof Error ? error : new Error(String(error)));
|
|
228
|
+
}
|
|
229
|
+
});
|
|
230
|
+
}
|
|
154
231
|
async function defaultProbe(url, signal) {
|
|
155
232
|
try {
|
|
156
233
|
const response = await fetch(url, {
|
|
157
234
|
redirect: 'manual',
|
|
158
235
|
signal: AbortSignal.any([signal, AbortSignal.timeout(2_000)]),
|
|
159
236
|
});
|
|
160
|
-
|
|
237
|
+
await response.body?.cancel().catch(() => undefined);
|
|
238
|
+
return response.ok ? true : 'unhealthy';
|
|
161
239
|
}
|
|
162
240
|
catch {
|
|
163
241
|
return false;
|
|
@@ -184,8 +262,15 @@ class FleetProcessController {
|
|
|
184
262
|
#child;
|
|
185
263
|
#cleanup;
|
|
186
264
|
#effects;
|
|
187
|
-
#
|
|
265
|
+
#exits = new WeakMap();
|
|
266
|
+
#outputTail = [];
|
|
267
|
+
/** The live runner that acknowledged quiescence and still holds pending
|
|
268
|
+
* cleanup authority. */
|
|
269
|
+
#quiesced;
|
|
270
|
+
#lastProgressAt = 0;
|
|
188
271
|
#provisioningTimeoutMs;
|
|
272
|
+
#readinessStallMs;
|
|
273
|
+
#readinessTargetMs;
|
|
189
274
|
#lifetime = new AbortController();
|
|
190
275
|
#startup = new AbortController();
|
|
191
276
|
#logStore;
|
|
@@ -208,11 +293,15 @@ class FleetProcessController {
|
|
|
208
293
|
options.adapter.adapterVersion !== 1 && options.adapter.lifecycle
|
|
209
294
|
? 'pending'
|
|
210
295
|
: 'not-declared';
|
|
211
|
-
this.#healthTimeoutMs = options.healthTimeoutMs ?? 120_000;
|
|
212
296
|
this.#provisioningTimeoutMs = options.provisioningTimeoutMs ?? 30 * 60_000;
|
|
297
|
+
this.#readinessStallMs =
|
|
298
|
+
options.readinessStallMs ?? exports.FLEET_READINESS_STALL_MS;
|
|
299
|
+
this.#readinessTargetMs =
|
|
300
|
+
options.readinessTargetMs ?? exports.FLEET_READINESS_TARGET_MS;
|
|
213
301
|
for (const timeout of [
|
|
214
|
-
this.#healthTimeoutMs,
|
|
215
302
|
this.#provisioningTimeoutMs,
|
|
303
|
+
this.#readinessStallMs,
|
|
304
|
+
this.#readinessTargetMs,
|
|
216
305
|
]) {
|
|
217
306
|
if (!Number.isSafeInteger(timeout) ||
|
|
218
307
|
timeout < 1 ||
|
|
@@ -232,7 +321,9 @@ class FleetProcessController {
|
|
|
232
321
|
launch: effects.launch,
|
|
233
322
|
prepareRestart: effects.prepareRestart,
|
|
234
323
|
probe: effects.probe ?? defaultProbe,
|
|
324
|
+
quiesce: effects.quiesce,
|
|
235
325
|
runnerGeneration: effects.runnerGeneration,
|
|
326
|
+
startTargetExit: effects.startTargetExit,
|
|
236
327
|
terminate: effects.terminate ?? terminateOwnedChild,
|
|
237
328
|
};
|
|
238
329
|
}
|
|
@@ -250,11 +341,52 @@ class FleetProcessController {
|
|
|
250
341
|
}
|
|
251
342
|
#transition(state) {
|
|
252
343
|
this.#state = state;
|
|
344
|
+
this.#lastProgressAt = Date.now();
|
|
253
345
|
this.#onStateChange?.(state, this.#restartAttempts);
|
|
254
346
|
}
|
|
347
|
+
/** Keep the newest redacted log lines of the active attempt so a startup
|
|
348
|
+
* failure can name the real error instead of only a classification. */
|
|
349
|
+
#remember(attempt, envelopes) {
|
|
350
|
+
if (attempt !== this.#activeAttempt || !envelopes)
|
|
351
|
+
return;
|
|
352
|
+
for (const { message } of envelopes) {
|
|
353
|
+
const line = message.trim();
|
|
354
|
+
if (line.length === 0 || line === '[empty line]')
|
|
355
|
+
continue;
|
|
356
|
+
this.#outputTail.push(line.length > FLEET_FAILURE_OUTPUT_LINE_LENGTH
|
|
357
|
+
? `${line.slice(0, FLEET_FAILURE_OUTPUT_LINE_LENGTH - 1)}\u2026`
|
|
358
|
+
: line);
|
|
359
|
+
}
|
|
360
|
+
this.#outputTail.splice(0, Math.max(0, this.#outputTail.length - exports.FLEET_FAILURE_OUTPUT_LINES));
|
|
361
|
+
}
|
|
255
362
|
#capture(child, attempt) {
|
|
256
|
-
|
|
257
|
-
|
|
363
|
+
const observe = (stream) => (chunk) => {
|
|
364
|
+
if (attempt === this.#activeAttempt)
|
|
365
|
+
this.#lastProgressAt = Date.now();
|
|
366
|
+
this.#remember(attempt, this.#logStore?.writeChunk(this.#name, stream, attempt, chunk));
|
|
367
|
+
};
|
|
368
|
+
child.stdout?.on('data', observe('stdout'));
|
|
369
|
+
child.stderr?.on('data', observe('stderr'));
|
|
370
|
+
}
|
|
371
|
+
/** A structured warning in the Slice log; it is never a failure. */
|
|
372
|
+
#logWarning(message) {
|
|
373
|
+
this.#logStore?.writeLine(this.#name, 'stderr', this.#activeAttempt, JSON.stringify({ level: 'warn', message }));
|
|
374
|
+
}
|
|
375
|
+
#exitedBeforeReadiness(child) {
|
|
376
|
+
const runnerExit = child ? this.#exits.get(child) : undefined;
|
|
377
|
+
let startExit;
|
|
378
|
+
try {
|
|
379
|
+
startExit = child ? this.#effects.startTargetExit?.(child) : undefined;
|
|
380
|
+
}
|
|
381
|
+
catch {
|
|
382
|
+
startExit = undefined;
|
|
383
|
+
}
|
|
384
|
+
const output = this.#outputTail.length > 0
|
|
385
|
+
? ` Last output: ${this.#outputTail.join(' | ')}.`
|
|
386
|
+
: '';
|
|
387
|
+
throw new errors_js_1.LocalDevContractError('process-exited', `${this.#name} exited before readiness (${startExit
|
|
388
|
+
? `start target ${describeExit(startExit)}`
|
|
389
|
+
: describeExit(runnerExit)}).${output} Inspect ebk dev logs ${this.#name}; owned cleanup settles before a retry.`);
|
|
258
390
|
}
|
|
259
391
|
#launch() {
|
|
260
392
|
this.#startup = new AbortController();
|
|
@@ -265,12 +397,14 @@ class FleetProcessController {
|
|
|
265
397
|
const launched = this.#effects.launch(this.#activeAttempt);
|
|
266
398
|
this.#child = launched.child;
|
|
267
399
|
this.#activeAttempt = launched.attempt;
|
|
400
|
+
this.#outputTail = [];
|
|
268
401
|
this.#capture(launched.child, launched.attempt);
|
|
269
402
|
this.#transition('starting');
|
|
270
403
|
launched.child.once('exit', (code, signal) => {
|
|
404
|
+
this.#exits.set(launched.child, { code, signal });
|
|
271
405
|
if (this.#child === launched.child)
|
|
272
406
|
this.#child = undefined;
|
|
273
|
-
this.#logStore?.flush(this.#name, undefined, launched.attempt);
|
|
407
|
+
this.#remember(launched.attempt, this.#logStore?.flush(this.#name, undefined, launched.attempt));
|
|
274
408
|
if (this.#stopping || this.#lifetime.signal.aborted)
|
|
275
409
|
return;
|
|
276
410
|
this.#transition('unavailable');
|
|
@@ -295,13 +429,17 @@ class FleetProcessController {
|
|
|
295
429
|
this.#startup.signal,
|
|
296
430
|
]);
|
|
297
431
|
const provisioningDeadline = Date.now() + this.#provisioningTimeoutMs;
|
|
298
|
-
|
|
432
|
+
// Set once the web application has launched; HTTP readiness starts then.
|
|
433
|
+
let webStartedAt;
|
|
434
|
+
let unhealthySince;
|
|
435
|
+
let slowerThanTarget = false;
|
|
299
436
|
const target = new URL(this.#adapter.healthPath, this.#upstream);
|
|
300
437
|
while (!signal.aborted && !this.#stopping) {
|
|
301
438
|
if (!child || this.#child !== child) {
|
|
302
|
-
|
|
439
|
+
// A crash is not a timeout: report the exit and the newest output.
|
|
440
|
+
this.#exitedBeforeReadiness(child);
|
|
303
441
|
}
|
|
304
|
-
if (
|
|
442
|
+
if (webStartedAt === undefined) {
|
|
305
443
|
const native = await this.#effects.nativeDeployment?.(signal);
|
|
306
444
|
if (native === 'failed') {
|
|
307
445
|
this.#transition('unavailable');
|
|
@@ -317,24 +455,45 @@ class FleetProcessController {
|
|
|
317
455
|
await this.#effects.delay(500, signal);
|
|
318
456
|
continue;
|
|
319
457
|
}
|
|
320
|
-
|
|
458
|
+
// The launch itself is progress: the stall clock starts here.
|
|
459
|
+
webStartedAt = Date.now();
|
|
460
|
+
this.#lastProgressAt = webStartedAt;
|
|
321
461
|
}
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
if (await this.#effects.probe(target, signal)) {
|
|
462
|
+
const probe = await this.#effects.probe(target, signal);
|
|
463
|
+
if (probe === true) {
|
|
325
464
|
if (this.#child !== child || this.#stopping || signal.aborted)
|
|
326
465
|
continue;
|
|
327
466
|
this.#restartAttempts = 0;
|
|
467
|
+
if (slowerThanTarget) {
|
|
468
|
+
this.#logWarning(`${this.#name} became ready ${Math.round((Date.now() - webStartedAt) / 1_000)} s after web launch, slower than the ${Math.round(this.#readinessTargetMs / 1_000)} s readiness target.`);
|
|
469
|
+
}
|
|
328
470
|
this.#transition('ready');
|
|
329
471
|
return;
|
|
330
472
|
}
|
|
473
|
+
const now = Date.now();
|
|
474
|
+
if (!slowerThanTarget && now - webStartedAt > this.#readinessTargetMs) {
|
|
475
|
+
slowerThanTarget = true;
|
|
476
|
+
this.#logWarning(`${this.#name} is slower than the ${Math.round(this.#readinessTargetMs / 1_000)} s readiness target; still waiting while it makes progress.`);
|
|
477
|
+
}
|
|
478
|
+
if (probe === 'unhealthy') {
|
|
479
|
+
unhealthySince ??= now;
|
|
480
|
+
if (now - unhealthySince > this.#readinessStallMs) {
|
|
481
|
+
this.#transition('unavailable');
|
|
482
|
+
throw new errors_js_1.LocalDevContractError('health-check-failing', `${this.#name} answered its health check ${this.#adapter.healthPath} only with unhealthy responses for ${Math.round((now - unhealthySince) / 1_000)} s. Inspect ebk dev logs ${this.#name}; owned cleanup settles before a retry.`);
|
|
483
|
+
}
|
|
484
|
+
}
|
|
485
|
+
else {
|
|
486
|
+
unhealthySince = undefined;
|
|
487
|
+
const stalledMs = now - this.#lastProgressAt;
|
|
488
|
+
if (stalledMs > this.#readinessStallMs) {
|
|
489
|
+
this.#transition('unavailable');
|
|
490
|
+
throw new errors_js_1.LocalDevContractError('readiness-stalled', `${this.#name} made no progress for ${Math.round(stalledMs / 1_000)} s: no output and no answer from its health check ${this.#adapter.healthPath}. Inspect ebk dev logs ${this.#name}; owned cleanup settles before a retry.`);
|
|
491
|
+
}
|
|
492
|
+
}
|
|
331
493
|
await this.#effects.delay(500, signal);
|
|
332
494
|
}
|
|
333
495
|
this.#transition('unavailable');
|
|
334
|
-
|
|
335
|
-
throw new errors_js_1.LocalDevContractError('startup-interrupted', `${this.#name} startup was interrupted; owned cleanup must settle before retrying.`);
|
|
336
|
-
}
|
|
337
|
-
throw new errors_js_1.LocalDevContractError('readiness-timeout', `${this.#name} did not become ready within ${this.#healthTimeoutMs}ms.`);
|
|
496
|
+
throw new errors_js_1.LocalDevContractError('startup-interrupted', `${this.#name} startup was interrupted; owned cleanup must settle before retrying.`);
|
|
338
497
|
}
|
|
339
498
|
async #autoRestart() {
|
|
340
499
|
const signal = AbortSignal.any([
|
|
@@ -358,7 +517,12 @@ class FleetProcessController {
|
|
|
358
517
|
await this.#waitForReadiness();
|
|
359
518
|
}
|
|
360
519
|
catch (error) {
|
|
361
|
-
|
|
520
|
+
// An interrupting quiesce or stop owns this runner. A bare stop here
|
|
521
|
+
// would run declared cleanup while peer writers may still be live.
|
|
522
|
+
if (this.#child &&
|
|
523
|
+
!this.#stopping &&
|
|
524
|
+
!this.#lifetime.signal.aborted &&
|
|
525
|
+
!this.#startup.signal.aborted) {
|
|
362
526
|
await this.#effects.terminate(this.#child);
|
|
363
527
|
}
|
|
364
528
|
if (!this.#restartTask &&
|
|
@@ -414,25 +578,78 @@ class FleetProcessController {
|
|
|
414
578
|
throw error;
|
|
415
579
|
}
|
|
416
580
|
}
|
|
417
|
-
|
|
581
|
+
/** Stop the owned writer without removing its cloud/dependency
|
|
582
|
+
* prerequisites. A Fleet transaction quiesces every selected Slice before
|
|
583
|
+
* reverse cleanup; a failed dependent cleanup must not leave a dependency
|
|
584
|
+
* writer running. A coordinating runner stays alive with pending cleanup
|
|
585
|
+
* authority, so its later cleanup output still reaches this Slice's log. */
|
|
586
|
+
async quiesce() {
|
|
418
587
|
if (this.#state === 'remote')
|
|
419
588
|
return;
|
|
420
589
|
this.#stopping = true;
|
|
421
590
|
this.#startup.abort();
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
591
|
+
try {
|
|
592
|
+
const child = this.#child;
|
|
593
|
+
if (child && child !== this.#quiesced) {
|
|
594
|
+
if (this.#effects.quiesce) {
|
|
595
|
+
await this.#effects.quiesce(child);
|
|
596
|
+
if (this.#child === child)
|
|
597
|
+
this.#quiesced = child;
|
|
598
|
+
}
|
|
599
|
+
else {
|
|
600
|
+
// The same generation-bound intent used by explicit restart lets
|
|
601
|
+
// the runner stop its native child and retain pending cleanup
|
|
602
|
+
// authority. A signal alone would run cleanup before peer writers
|
|
603
|
+
// were stopped.
|
|
604
|
+
await this.#effects.prepareRestart?.(child);
|
|
605
|
+
await this.#effects.terminate(child);
|
|
606
|
+
if (this.#effects.prepareRestart &&
|
|
607
|
+
(child.exitCode !== 0 || child.signalCode !== null)) {
|
|
608
|
+
throw new errors_js_1.LocalDevContractError('cleanup-incomplete', `${this.#name} runner did not quiesce successfully; its claim and cleanup authority remain retained.`);
|
|
609
|
+
}
|
|
610
|
+
}
|
|
611
|
+
}
|
|
612
|
+
await this.#restartTask;
|
|
613
|
+
this.#restartTask = undefined;
|
|
614
|
+
}
|
|
615
|
+
catch (error) {
|
|
616
|
+
// Status must not keep reporting pending authority as healthy.
|
|
617
|
+
this.#cleanup =
|
|
618
|
+
this.#adapter.adapterVersion !== 1 && this.#adapter.lifecycle
|
|
619
|
+
? 'failed'
|
|
620
|
+
: 'not-declared';
|
|
621
|
+
throw error;
|
|
622
|
+
}
|
|
426
623
|
this.#logStore?.flush(this.#name);
|
|
427
|
-
|
|
428
|
-
|
|
624
|
+
// Unavailable is still selected. Quiescence never releases a claim or
|
|
625
|
+
// means cloud cleanup succeeded, and #stopping prevents automatic restart.
|
|
626
|
+
this.#transition('unavailable');
|
|
627
|
+
}
|
|
628
|
+
async stop(options) {
|
|
629
|
+
if (this.#state === 'remote')
|
|
630
|
+
return;
|
|
631
|
+
await this.quiesce();
|
|
632
|
+
try {
|
|
633
|
+
const child = this.#child;
|
|
634
|
+
if (child) {
|
|
635
|
+
// A quiesced runner still holds cleanup authority. Stopping it runs
|
|
636
|
+
// the declared cleanup in that runner, whose output this controller
|
|
637
|
+
// captures. A restart intent instead leaves cleanup pending.
|
|
638
|
+
if (!options.cleanup)
|
|
639
|
+
await this.#effects.prepareRestart?.(child);
|
|
640
|
+
await this.#effects.terminate(child);
|
|
641
|
+
}
|
|
642
|
+
if (options.cleanup && this.#effects.cleanup) {
|
|
429
643
|
await this.#effects.cleanup();
|
|
430
644
|
this.#cleanup = 'succeeded';
|
|
431
645
|
}
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
646
|
+
}
|
|
647
|
+
catch (error) {
|
|
648
|
+
this.#cleanup =
|
|
649
|
+
this.#adapter.adapterVersion !== 1 && this.#adapter.lifecycle
|
|
650
|
+
? 'failed'
|
|
651
|
+
: 'not-declared';
|
|
652
|
+
throw error;
|
|
436
653
|
}
|
|
437
654
|
this.#transition('remote');
|
|
438
655
|
}
|
|
@@ -1,10 +1,17 @@
|
|
|
1
|
+
import type { spawn } from 'node:child_process';
|
|
1
2
|
import { inspectLocalDevelopment } from '../executors/dev-doctor/dev-doctor.js';
|
|
3
|
+
import { type LocalDevAdapter } from './contracts.js';
|
|
4
|
+
import { FleetLogStore } from './fleet-logs.js';
|
|
5
|
+
import type { FleetProcessIdentity } from './fleet-lock.js';
|
|
6
|
+
import { type FleetProcessEffects } from './fleet-process.js';
|
|
2
7
|
import { FleetSession } from './fleet-session.js';
|
|
3
8
|
import { FleetSupervisor } from './fleet-supervisor.js';
|
|
4
9
|
import { type FleetStartupRequest } from './fleet-startup.js';
|
|
5
10
|
import { type WorkspaceFleetPlan } from './nx-workspace.js';
|
|
6
11
|
import { allocateFleetPort } from './ports.js';
|
|
7
12
|
import { type PersonalStageResolver } from './personal-stage.js';
|
|
13
|
+
import { type FleetRuntimePaths } from './runtime-paths.js';
|
|
14
|
+
import { type LaunchedFleetSliceRunner } from './slice-runner.js';
|
|
8
15
|
export interface RunningFleetRuntime {
|
|
9
16
|
fleetId: string;
|
|
10
17
|
origin: string;
|
|
@@ -19,4 +26,34 @@ export interface FleetRuntimeDependencies {
|
|
|
19
26
|
resolveRouter?: (request: FleetStartupRequest) => Promise<URL>;
|
|
20
27
|
}
|
|
21
28
|
export declare function resolveDevelopmentRouter(request: FleetStartupRequest): Promise<URL>;
|
|
29
|
+
export interface FleetSliceRunnerEffectsInput {
|
|
30
|
+
adapter: LocalDevAdapter;
|
|
31
|
+
logStore: FleetLogStore;
|
|
32
|
+
name: string;
|
|
33
|
+
origin: URL;
|
|
34
|
+
parent: FleetProcessIdentity;
|
|
35
|
+
port: number;
|
|
36
|
+
projects: WorkspaceFleetPlan['projects'];
|
|
37
|
+
runtimePaths: FleetRuntimePaths;
|
|
38
|
+
sessionGeneration: string;
|
|
39
|
+
stage: string;
|
|
40
|
+
startupPath: string;
|
|
41
|
+
workspaceRealpath: string;
|
|
42
|
+
}
|
|
43
|
+
/** The runner daemon a Slice launches and retries cleanup with. Production
|
|
44
|
+
* omits both; real-runner specs substitute a fixture daemon. */
|
|
45
|
+
export interface FleetSliceRunnerDaemon {
|
|
46
|
+
daemonPath?: string;
|
|
47
|
+
spawn?: typeof spawn;
|
|
48
|
+
}
|
|
49
|
+
export interface FleetSliceRunnerEffects {
|
|
50
|
+
/** The runner most recently launched for this Slice, if any. */
|
|
51
|
+
activeRunner: () => LaunchedFleetSliceRunner | undefined;
|
|
52
|
+
effects: FleetProcessEffects;
|
|
53
|
+
}
|
|
54
|
+
/** One Slice's Fleet process effects: a coordinating runner per launch,
|
|
55
|
+
* generation-bound restart and quiescence, and cleanup whose retry output
|
|
56
|
+
* joins the Slice log store. startFleetRuntime and the real-runner
|
|
57
|
+
* quiescence specs share this wiring, so a dropped effect fails those specs. */
|
|
58
|
+
export declare function createFleetSliceRunnerEffects(input: FleetSliceRunnerEffectsInput, daemon?: FleetSliceRunnerDaemon): FleetSliceRunnerEffects;
|
|
22
59
|
export declare function startFleetRuntime(startupPath: string, dependencies?: FleetRuntimeDependencies): Promise<RunningFleetRuntime>;
|