@mostra/runtime 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +39 -2
- package/dist/bin.d.ts +1 -0
- package/dist/bin.js +236 -0
- package/dist/bin.js.map +1 -0
- package/dist/builder/child/main.d.ts +1 -0
- package/dist/builder/child/main.js +1408 -0
- package/dist/builder/child/main.js.map +1 -0
- package/dist/child-xrHmytks.js +49 -0
- package/dist/child-xrHmytks.js.map +1 -0
- package/dist/deps/child/install.d.ts +1 -0
- package/dist/deps/child/install.js +33 -0
- package/dist/deps/child/install.js.map +1 -0
- package/dist/index.d.ts +221 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +3 -0
- package/dist/instance-D7_UVdoN.js +15752 -0
- package/dist/instance-D7_UVdoN.js.map +1 -0
- package/dist/types-NjNy7gXX.js +57 -0
- package/dist/types-NjNy7gXX.js.map +1 -0
- package/package.json +87 -3
- package/src/actions/index.ts +622 -0
- package/src/addressing/index.ts +281 -0
- package/src/bin.ts +240 -0
- package/src/builder/child/dev.ts +383 -0
- package/src/builder/child/fields.ts +363 -0
- package/src/builder/child/main.ts +47 -0
- package/src/builder/child/plugins.ts +195 -0
- package/src/builder/child/scan.ts +459 -0
- package/src/builder/child/spa.ts +348 -0
- package/src/builder/coverage.ts +72 -0
- package/src/builder/index.ts +406 -0
- package/src/builder/types.ts +180 -0
- package/src/deps/child/install.ts +30 -0
- package/src/deps/index.ts +918 -0
- package/src/deps/installer.ts +321 -0
- package/src/deps/lock.ts +82 -0
- package/src/deps/scan.ts +266 -0
- package/src/deps/specifier.ts +64 -0
- package/src/deps/warm.ts +57 -0
- package/src/dev/index.ts +518 -0
- package/src/faults.ts +65 -0
- package/src/gate.ts +55 -0
- package/src/gateway/actions.ts +268 -0
- package/src/gateway/browser.ts +650 -0
- package/src/gateway/capabilities.ts +739 -0
- package/src/gateway/controls.ts +91 -0
- package/src/gateway/dev.ts +119 -0
- package/src/gateway/http.ts +229 -0
- package/src/gateway/index.ts +1209 -0
- package/src/gateway/integrity.ts +117 -0
- package/src/gateway/listener.ts +217 -0
- package/src/gateway/routes.ts +197 -0
- package/src/gateway/serve.ts +138 -0
- package/src/gc/index.ts +451 -0
- package/src/home/acquire.ts +276 -0
- package/src/home/index.ts +16 -0
- package/src/home/paths.ts +192 -0
- package/src/home/platform.ts +34 -0
- package/src/home/record.ts +76 -0
- package/src/home/settings.ts +113 -0
- package/src/index.ts +30 -0
- package/src/instance/index.ts +942 -0
- package/src/interaction/ephemeral.ts +300 -0
- package/src/interaction/events.ts +355 -0
- package/src/interaction/fields.ts +392 -0
- package/src/interaction/index.ts +1009 -0
- package/src/interaction/outbox.ts +493 -0
- package/src/interaction/policy.ts +26 -0
- package/src/interaction/results.ts +341 -0
- package/src/interaction/runs.ts +313 -0
- package/src/interaction/signals.ts +120 -0
- package/src/interaction/submissions.ts +585 -0
- package/src/interaction/subscriptions.ts +258 -0
- package/src/interaction/viewers.ts +173 -0
- package/src/interaction/wait.ts +201 -0
- package/src/ledger/index.ts +442 -0
- package/src/ledger/migrations.ts +583 -0
- package/src/ledger/requests.ts +105 -0
- package/src/ledger/types.ts +28 -0
- package/src/live/index.ts +573 -0
- package/src/log.ts +316 -0
- package/src/processes/child.ts +67 -0
- package/src/processes/environment.ts +64 -0
- package/src/processes/identity.ts +191 -0
- package/src/processes/index.ts +19 -0
- package/src/processes/manager.ts +543 -0
- package/src/protocol-server/client.ts +94 -0
- package/src/protocol-server/index.ts +91 -0
- package/src/protocol-server/server.ts +509 -0
- package/src/protocol-server/streams.ts +81 -0
- package/src/publish/checks.ts +92 -0
- package/src/publish/digest.ts +87 -0
- package/src/publish/index.ts +23 -0
- package/src/publish/manifest.ts +409 -0
- package/src/publish/methods.ts +96 -0
- package/src/publish/publisher.ts +913 -0
- package/src/publish/source.ts +272 -0
- package/src/publish/tar.ts +110 -0
- package/src/registry/index.ts +481 -0
- package/src/registry/store.ts +136 -0
- package/src/secrets/index.ts +223 -0
- package/src/service/idle.ts +64 -0
- package/src/service/index.ts +3 -0
- package/src/service/unit.ts +250 -0
- package/src/verify/fields.ts +125 -0
- package/src/verify/index.ts +913 -0
- package/src/verify/regression.ts +157 -0
- package/src/verify/runner.ts +230 -0
- package/src/verify/sandbox.ts +407 -0
- package/src/verify/typecheck.ts +371 -0
- package/src/workers/index.ts +17 -0
- package/src/workers/launcher.ts +260 -0
- package/src/workers/pool.ts +708 -0
|
@@ -0,0 +1,708 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The worker pool (RT-03, RT-04, D62, FAILURE_MODEL §2): one worker per active artifact version
|
|
3
|
+
* with server code, started on first need through the `WorkerLauncher`, stopped after its idle
|
|
4
|
+
* minutes, watched by a 10 s event-loop watchdog, restarted after a crash with exponential backoff
|
|
5
|
+
* (1 s to 60 s) and parked after more than 5 crashes in 10 minutes. A crash, an out-of-memory abort
|
|
6
|
+
* or a hang ends only that worker: the supervisor and every other worker keep serving.
|
|
7
|
+
*/
|
|
8
|
+
import {
|
|
9
|
+
type CallContext,
|
|
10
|
+
createDiagnostic,
|
|
11
|
+
DEFAULT_WORKER_LIMITS,
|
|
12
|
+
type Frame,
|
|
13
|
+
RpcError,
|
|
14
|
+
RpcPeer,
|
|
15
|
+
type RunProgress,
|
|
16
|
+
WORKER_METHODS,
|
|
17
|
+
WorkerBatchParams,
|
|
18
|
+
type WorkerCallOutcome,
|
|
19
|
+
WorkerCallOutcome as WorkerCallOutcomeSchema,
|
|
20
|
+
type WorkerLimits,
|
|
21
|
+
WorkerPingResult,
|
|
22
|
+
type WorkerStartResult,
|
|
23
|
+
WorkerStartResult as WorkerStartResultSchema,
|
|
24
|
+
type WorkerStatus,
|
|
25
|
+
type WorkerVersionRef,
|
|
26
|
+
} from "@mostra/protocol";
|
|
27
|
+
import type { Environment, Log } from "../log.ts";
|
|
28
|
+
import type { WorkerChannel, WorkerExit, WorkerLauncher } from "./launcher.ts";
|
|
29
|
+
|
|
30
|
+
/** What the pool needs to know about a version to start its worker. */
|
|
31
|
+
export interface WorkerVersion {
|
|
32
|
+
readonly artifactId: string;
|
|
33
|
+
readonly ref: WorkerVersionRef;
|
|
34
|
+
/** The server bundle, relative to the version (the render mode's `serverEntry`). */
|
|
35
|
+
readonly entry: string;
|
|
36
|
+
readonly limits: WorkerLimits;
|
|
37
|
+
readonly sourceDir?: string;
|
|
38
|
+
/** The IDs of the version's server functions, from its `registry.json` (the builder's scan). */
|
|
39
|
+
readonly functions: ReadonlySet<string>;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** A call's link back to the caller: its batches, progress and log lines as they come. */
|
|
43
|
+
export interface CallListener {
|
|
44
|
+
onBatch?(batch: readonly Frame[]): void;
|
|
45
|
+
onProgress?(progress: RunProgress): void;
|
|
46
|
+
onLog?(line: string): void;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** Who a call is for, as the bridge sees a worker's requests during it. */
|
|
50
|
+
export interface CallInfo {
|
|
51
|
+
readonly callId: string;
|
|
52
|
+
readonly context: CallContext;
|
|
53
|
+
readonly viewerId?: string;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** What a worker asks of the supervisor, and the live values it produces. */
|
|
57
|
+
export interface WorkerBridge {
|
|
58
|
+
/**
|
|
59
|
+
* Answers a worker's request (`server/secret`, `server/signal`, `server/emit`,
|
|
60
|
+
* `server/state.*`); `call` is the call it came from, when it names one.
|
|
61
|
+
*/
|
|
62
|
+
request(
|
|
63
|
+
worker: { readonly artifactId: string; readonly versionId: string },
|
|
64
|
+
method: string,
|
|
65
|
+
params: unknown,
|
|
66
|
+
call: CallInfo | undefined,
|
|
67
|
+
): Promise<unknown>;
|
|
68
|
+
/** A line a worker logged outside any call. */
|
|
69
|
+
log(worker: { readonly artifactId: string; readonly versionId: string }, line: string): void;
|
|
70
|
+
/** A live value or failure (`server/live.value`). */
|
|
71
|
+
liveValue(
|
|
72
|
+
worker: { readonly artifactId: string; readonly versionId: string },
|
|
73
|
+
params: unknown,
|
|
74
|
+
): void;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** A running worker, as the pool hands it out. */
|
|
78
|
+
export interface WorkerHandle {
|
|
79
|
+
readonly versionId: string;
|
|
80
|
+
readonly artifactId: string;
|
|
81
|
+
readonly started: WorkerStartResult;
|
|
82
|
+
/** Runs a server function; the outcome arrives after every batch was given to `listener`. */
|
|
83
|
+
call(
|
|
84
|
+
info: {
|
|
85
|
+
readonly fnId: string;
|
|
86
|
+
readonly args: unknown;
|
|
87
|
+
readonly context: CallContext;
|
|
88
|
+
readonly viewerId?: string;
|
|
89
|
+
},
|
|
90
|
+
listener: CallListener,
|
|
91
|
+
): { readonly callId: string; readonly outcome: Promise<WorkerCallOutcome> };
|
|
92
|
+
/** Aborts a call's `abortSignal`. */
|
|
93
|
+
cancel(callId: string, reason?: string): void;
|
|
94
|
+
/** Starts a live producer; it keeps the worker from idling until stopped. */
|
|
95
|
+
liveStart(key: string): Promise<void>;
|
|
96
|
+
liveStop(key: string): Promise<void>;
|
|
97
|
+
liveRefresh(key: string): void;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** Why a worker ended unexpectedly, and what happens next. */
|
|
101
|
+
export interface WorkerCrash {
|
|
102
|
+
readonly artifactId: string;
|
|
103
|
+
readonly versionId: string;
|
|
104
|
+
/** For people: "it ran out of memory (heap limit 1024 MiB)". */
|
|
105
|
+
readonly cause: string;
|
|
106
|
+
readonly kind: "crash" | "oom" | "hang";
|
|
107
|
+
readonly backoffMs: number;
|
|
108
|
+
readonly parked: boolean;
|
|
109
|
+
/** The calls that were running in it. */
|
|
110
|
+
readonly callIds: readonly string[];
|
|
111
|
+
/** The live producers that were running in it. */
|
|
112
|
+
readonly liveKeys: readonly string[];
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/** Options for {@link createWorkerPool}. */
|
|
116
|
+
export interface WorkerPoolOptions {
|
|
117
|
+
readonly launcher: WorkerLauncher;
|
|
118
|
+
readonly log: Log;
|
|
119
|
+
readonly bridge: WorkerBridge;
|
|
120
|
+
/** The environment every worker gets: the artifact allowlist, never secrets. */
|
|
121
|
+
readonly env: Environment;
|
|
122
|
+
/** Reports a crash, after the pool recorded it: append `worker.crashed`, end runs. */
|
|
123
|
+
readonly onCrash: (crash: WorkerCrash) => void;
|
|
124
|
+
/** How often the watchdog asks; default 1000 ms. */
|
|
125
|
+
readonly watchdogIntervalMs?: number;
|
|
126
|
+
/** Idle time in ms, overriding the versions' `idleMinutes` (tests). */
|
|
127
|
+
readonly idleMsOverride?: number;
|
|
128
|
+
/** The crash window and its limit (FAILURE_MODEL §2: 5 in 10 minutes). */
|
|
129
|
+
readonly parkAfter?: { readonly crashes: number; readonly windowMs: number };
|
|
130
|
+
/** The first backoff after a crash, doubling to 60 s (FAILURE_MODEL §2: 1 s; tests shorten it). */
|
|
131
|
+
readonly backoffMinMs?: number;
|
|
132
|
+
readonly now?: () => number;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/** The pool. */
|
|
136
|
+
export interface WorkerPool {
|
|
137
|
+
/**
|
|
138
|
+
* The version's worker, started if it isn't; waits out a backoff. Throws
|
|
139
|
+
* `MOSTRA_RUNTIME_WORKER_PARKED` for a parked version and `MOSTRA_RUNTIME_WORKER_CRASHED` when
|
|
140
|
+
* the worker couldn't start.
|
|
141
|
+
*/
|
|
142
|
+
acquire(version: WorkerVersion): Promise<WorkerHandle>;
|
|
143
|
+
/** The worker if it runs now, without starting one. */
|
|
144
|
+
running(versionId: string): WorkerHandle | undefined;
|
|
145
|
+
/**
|
|
146
|
+
* Stops a version's worker now if it runs nothing (no call, no producer), rather than after its
|
|
147
|
+
* idle time: for a worker started only to say what its live sources are.
|
|
148
|
+
*/
|
|
149
|
+
release(versionId: string): void;
|
|
150
|
+
/** Clears the park and crash history of every version of an artifact (a publish, `verify`). */
|
|
151
|
+
clear(artifactId: string): void;
|
|
152
|
+
/**
|
|
153
|
+
* Replaces a version's server bundle (`mostra dev` only, D140 item 12): stops its worker, busy
|
|
154
|
+
* or not, so the next call starts `version`, the same version ID with a new output digest, and
|
|
155
|
+
* clears its crash history.
|
|
156
|
+
*/
|
|
157
|
+
replace(version: WorkerVersion): Promise<void>;
|
|
158
|
+
/** Every version's worker state, for `instance/status`. */
|
|
159
|
+
status(): WorkerStatus[];
|
|
160
|
+
/** How many calls and producers run in all workers: the daemon stays while any do (D62). */
|
|
161
|
+
activity(): number;
|
|
162
|
+
/** Stops every worker. */
|
|
163
|
+
stopAll(): Promise<void>;
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
const CRASH_LIMIT = { crashes: 5, windowMs: 10 * 60_000 } as const;
|
|
167
|
+
const BACKOFF_MIN_MS = 1000;
|
|
168
|
+
const BACKOFF_MAX_MS = 60_000;
|
|
169
|
+
const START_TIMEOUT_MS = 60_000;
|
|
170
|
+
|
|
171
|
+
interface Slot {
|
|
172
|
+
version: WorkerVersion;
|
|
173
|
+
state: WorkerStatus["state"];
|
|
174
|
+
crashes: number[];
|
|
175
|
+
lastCause?: string;
|
|
176
|
+
retryAt?: number;
|
|
177
|
+
starting?: Promise<Running>;
|
|
178
|
+
current?: Running;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
interface Running {
|
|
182
|
+
readonly handle: WorkerHandle;
|
|
183
|
+
readonly channel: WorkerChannel;
|
|
184
|
+
readonly peer: RpcPeer;
|
|
185
|
+
readonly calls: Map<string, { info: CallInfo; listener: CallListener }>;
|
|
186
|
+
readonly live: Set<string>;
|
|
187
|
+
stopping: boolean;
|
|
188
|
+
/**
|
|
189
|
+
* Set until `worker/start` answers: a worker that ends before then failed to start, which
|
|
190
|
+
* `acquire` records once, with the start's message (Opus review 5).
|
|
191
|
+
*/
|
|
192
|
+
starting: boolean;
|
|
193
|
+
hung?: number;
|
|
194
|
+
/** The watchdog stopped it for not answering at all, rather than for a blocked event loop. */
|
|
195
|
+
silent?: boolean;
|
|
196
|
+
/** The heap in use at the last answered ping, in bytes: near the limit, a stall is memory. */
|
|
197
|
+
heapUsed?: number;
|
|
198
|
+
idleTimer?: ReturnType<typeof setTimeout>;
|
|
199
|
+
watchdog?: ReturnType<typeof setInterval>;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
let nextCall = 0;
|
|
203
|
+
|
|
204
|
+
/** Unanswered pings in a row, each as long as the watchdog's limit, before a worker is stopped. */
|
|
205
|
+
const PINGS_MISSED_BEFORE_STOP = 3;
|
|
206
|
+
|
|
207
|
+
/**
|
|
208
|
+
* The share of its heap limit a worker had in use at its last answered ping for a watchdog stop
|
|
209
|
+
* to count as running out of memory rather than hanging.
|
|
210
|
+
*/
|
|
211
|
+
const OOM_STALL_FRACTION = 0.8;
|
|
212
|
+
|
|
213
|
+
function crashed(message: string): RpcError {
|
|
214
|
+
return new RpcError(createDiagnostic("MOSTRA_RUNTIME_WORKER_CRASHED", { message }));
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
/** A worker that couldn't start, and how it ended. */
|
|
218
|
+
class StartFailure extends RpcError {
|
|
219
|
+
readonly kind: WorkerCrash["kind"];
|
|
220
|
+
constructor(message: string, kind: WorkerCrash["kind"]) {
|
|
221
|
+
super(createDiagnostic("MOSTRA_RUNTIME_WORKER_CRASHED", { message }));
|
|
222
|
+
this.kind = kind;
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/** Creates the pool. */
|
|
227
|
+
export function createWorkerPool(options: WorkerPoolOptions): WorkerPool {
|
|
228
|
+
const slots = new Map<string, Slot>();
|
|
229
|
+
const now = options.now ?? Date.now;
|
|
230
|
+
const park = options.parkAfter ?? CRASH_LIMIT;
|
|
231
|
+
const watchdogEvery = options.watchdogIntervalMs ?? 1000;
|
|
232
|
+
const backoffMin = options.backoffMinMs ?? BACKOFF_MIN_MS;
|
|
233
|
+
let stopped = false;
|
|
234
|
+
|
|
235
|
+
const idleMs = (version: WorkerVersion): number =>
|
|
236
|
+
options.idleMsOverride ?? version.limits.idleMinutes * 60_000;
|
|
237
|
+
|
|
238
|
+
const scheduleIdle = (slot: Slot, running: Running): void => {
|
|
239
|
+
if (running.idleTimer !== undefined) clearTimeout(running.idleTimer);
|
|
240
|
+
delete running.idleTimer;
|
|
241
|
+
if (running.calls.size > 0 || running.live.size > 0 || running.stopping) return;
|
|
242
|
+
running.idleTimer = setTimeout(() => {
|
|
243
|
+
if (running.calls.size > 0 || running.live.size > 0) return;
|
|
244
|
+
options.log.info("worker.idle_stop", { versionId: slot.version.ref.versionId });
|
|
245
|
+
void stopRunning(running);
|
|
246
|
+
}, idleMs(slot.version));
|
|
247
|
+
running.idleTimer.unref?.();
|
|
248
|
+
};
|
|
249
|
+
|
|
250
|
+
const stopRunning = async (running: Running): Promise<void> => {
|
|
251
|
+
running.stopping = true;
|
|
252
|
+
if (running.watchdog !== undefined) clearInterval(running.watchdog);
|
|
253
|
+
if (running.idleTimer !== undefined) clearTimeout(running.idleTimer);
|
|
254
|
+
await running.peer.request(WORKER_METHODS.stop, {}, { timeoutMs: 2000 }).catch(() => {});
|
|
255
|
+
const timer = setTimeout(() => running.channel.kill(), 3000);
|
|
256
|
+
await running.channel.ended;
|
|
257
|
+
clearTimeout(timer);
|
|
258
|
+
};
|
|
259
|
+
|
|
260
|
+
const classify = (
|
|
261
|
+
exit: WorkerExit,
|
|
262
|
+
running: Running,
|
|
263
|
+
limits: WorkerLimits,
|
|
264
|
+
): { kind: WorkerCrash["kind"]; cause: string } => {
|
|
265
|
+
const tail = exit.stderrTail.join("\n");
|
|
266
|
+
if (
|
|
267
|
+
/heap out of memory|Allocation failed|Reached heap limit/i.test(tail) ||
|
|
268
|
+
exit.signal === "SIGABRT" ||
|
|
269
|
+
exit.code === 134
|
|
270
|
+
) {
|
|
271
|
+
return { kind: "oom", cause: `it ran out of memory (heap limit ${limits.heapMb} MiB)` };
|
|
272
|
+
}
|
|
273
|
+
if (running.hung !== undefined) {
|
|
274
|
+
// Near its heap limit, V8 collects garbage over and over and the event loop stalls before
|
|
275
|
+
// the heap gives out: the watchdog stops it first, and the cause is memory (seen on Linux).
|
|
276
|
+
if ((running.heapUsed ?? 0) >= OOM_STALL_FRACTION * limits.heapMb * 1024 * 1024) {
|
|
277
|
+
return {
|
|
278
|
+
kind: "oom",
|
|
279
|
+
cause: `it ran out of memory (heap limit ${limits.heapMb} MiB): collecting garbage blocked its event loop for ${Math.round(running.hung / 1000)} s`,
|
|
280
|
+
};
|
|
281
|
+
}
|
|
282
|
+
return {
|
|
283
|
+
kind: "hang",
|
|
284
|
+
cause: running.silent
|
|
285
|
+
? `it answered nothing for ${Math.round(running.hung / 1000)} s (watchdog limit ${limits.watchdogMs / 1000} s)`
|
|
286
|
+
: `its event loop was blocked for ${Math.round(running.hung / 1000)} s (watchdog limit ${limits.watchdogMs / 1000} s)`,
|
|
287
|
+
};
|
|
288
|
+
}
|
|
289
|
+
// Node ends an uncaught exception's report with its version; the error line says what
|
|
290
|
+
// happened (`Error: …`, `TypeError: …`), so it is the cause, else the last line written.
|
|
291
|
+
const lines = exit.stderrTail.filter(
|
|
292
|
+
(line) => line.trim() !== "" && !/^Node\.js v\d/.test(line.trim()),
|
|
293
|
+
);
|
|
294
|
+
const last =
|
|
295
|
+
lines.findLast((line) => /^[A-Za-z_$][\w$.]*(?:Error|Exception)\b.*:/.test(line.trim())) ??
|
|
296
|
+
lines.findLast((line) => !/^\s+at\s/.test(line)) ??
|
|
297
|
+
lines.at(-1);
|
|
298
|
+
const how =
|
|
299
|
+
exit.signal !== null
|
|
300
|
+
? `it was ended by ${exit.signal}`
|
|
301
|
+
: `it exited with code ${exit.code ?? "?"}`;
|
|
302
|
+
return { kind: "crash", cause: last ? `${how}: ${last.slice(0, 300)}` : how };
|
|
303
|
+
};
|
|
304
|
+
|
|
305
|
+
/** Records a crash: the backoff or the park it leads to, the log and the caller's report. */
|
|
306
|
+
const recordCrash = (
|
|
307
|
+
slot: Slot,
|
|
308
|
+
kind: WorkerCrash["kind"],
|
|
309
|
+
cause: string,
|
|
310
|
+
callIds: readonly string[],
|
|
311
|
+
liveKeys: readonly string[],
|
|
312
|
+
): void => {
|
|
313
|
+
const at = now();
|
|
314
|
+
slot.crashes = [...slot.crashes.filter((when) => at - when < park.windowMs), at];
|
|
315
|
+
slot.lastCause = cause;
|
|
316
|
+
const parked = slot.crashes.length > park.crashes;
|
|
317
|
+
const backoffMs = parked
|
|
318
|
+
? 0
|
|
319
|
+
: Math.min(BACKOFF_MAX_MS, backoffMin * 2 ** (slot.crashes.length - 1));
|
|
320
|
+
slot.state = parked ? "parked" : "backoff";
|
|
321
|
+
if (parked) delete slot.retryAt;
|
|
322
|
+
else slot.retryAt = at + backoffMs;
|
|
323
|
+
options.log.warn("worker.crashed", {
|
|
324
|
+
versionId: slot.version.ref.versionId,
|
|
325
|
+
kind,
|
|
326
|
+
cause,
|
|
327
|
+
backoffMs,
|
|
328
|
+
parked,
|
|
329
|
+
});
|
|
330
|
+
options.onCrash({
|
|
331
|
+
artifactId: slot.version.artifactId,
|
|
332
|
+
versionId: slot.version.ref.versionId,
|
|
333
|
+
cause,
|
|
334
|
+
kind,
|
|
335
|
+
backoffMs,
|
|
336
|
+
parked,
|
|
337
|
+
callIds,
|
|
338
|
+
liveKeys,
|
|
339
|
+
});
|
|
340
|
+
};
|
|
341
|
+
|
|
342
|
+
const onEnded = (slot: Slot, running: Running, exit: WorkerExit): void => {
|
|
343
|
+
if (running.watchdog !== undefined) clearInterval(running.watchdog);
|
|
344
|
+
if (running.idleTimer !== undefined) clearTimeout(running.idleTimer);
|
|
345
|
+
running.peer.close();
|
|
346
|
+
if (slot.current === running) delete slot.current;
|
|
347
|
+
if (running.stopping || stopped) {
|
|
348
|
+
if (slot.state === "ready") slot.state = "stopped";
|
|
349
|
+
return;
|
|
350
|
+
}
|
|
351
|
+
// A worker that ends during its start failed to start: `acquire` records that, once.
|
|
352
|
+
if (running.starting) return;
|
|
353
|
+
const { kind, cause } = classify(exit, running, slot.version.limits);
|
|
354
|
+
recordCrash(slot, kind, cause, [...running.calls.keys()], [...running.live]);
|
|
355
|
+
};
|
|
356
|
+
|
|
357
|
+
const start = async (slot: Slot): Promise<Running> => {
|
|
358
|
+
const { version } = slot;
|
|
359
|
+
const who = { artifactId: version.artifactId, versionId: version.ref.versionId };
|
|
360
|
+
const channel = await options.launcher.launch({
|
|
361
|
+
artifactId: version.artifactId,
|
|
362
|
+
version: version.ref,
|
|
363
|
+
env: options.env,
|
|
364
|
+
limits: version.limits,
|
|
365
|
+
...(version.sourceDir ? { sourceDir: version.sourceDir } : {}),
|
|
366
|
+
});
|
|
367
|
+
const calls: Running["calls"] = new Map();
|
|
368
|
+
const live = new Set<string>();
|
|
369
|
+
const peer = new RpcPeer({
|
|
370
|
+
send: (message) => channel.send(message),
|
|
371
|
+
onRequest: (method, params) => {
|
|
372
|
+
const callId = (params as { callId?: unknown } | undefined)?.callId;
|
|
373
|
+
const call = typeof callId === "string" ? calls.get(callId)?.info : undefined;
|
|
374
|
+
return options.bridge.request(who, method, params, call);
|
|
375
|
+
},
|
|
376
|
+
onNotification: (method, params) => {
|
|
377
|
+
const callId = (params as { callId?: unknown } | undefined)?.callId;
|
|
378
|
+
const entry = typeof callId === "string" ? calls.get(callId) : undefined;
|
|
379
|
+
switch (method) {
|
|
380
|
+
case WORKER_METHODS.batch: {
|
|
381
|
+
const parsed = WorkerBatchParams.safeParse(params);
|
|
382
|
+
if (parsed.success) entry?.listener.onBatch?.(parsed.data.batch);
|
|
383
|
+
return;
|
|
384
|
+
}
|
|
385
|
+
case WORKER_METHODS.progress: {
|
|
386
|
+
const { callId: _call, ...progress } = params as RunProgress & { callId: string };
|
|
387
|
+
entry?.listener.onProgress?.(progress);
|
|
388
|
+
return;
|
|
389
|
+
}
|
|
390
|
+
case WORKER_METHODS.log: {
|
|
391
|
+
const line = String((params as { line?: unknown }).line ?? "");
|
|
392
|
+
if (entry) entry.listener.onLog?.(line);
|
|
393
|
+
else options.bridge.log(who, line);
|
|
394
|
+
return;
|
|
395
|
+
}
|
|
396
|
+
case WORKER_METHODS.liveValue:
|
|
397
|
+
options.bridge.liveValue(who, params);
|
|
398
|
+
return;
|
|
399
|
+
}
|
|
400
|
+
},
|
|
401
|
+
onInternalError: (error, method) =>
|
|
402
|
+
options.log.error("worker.channel_error", { versionId: who.versionId, method, error }),
|
|
403
|
+
});
|
|
404
|
+
channel.onMessage((message) => peer.receive(message));
|
|
405
|
+
|
|
406
|
+
const running: Running = {
|
|
407
|
+
channel,
|
|
408
|
+
peer,
|
|
409
|
+
calls,
|
|
410
|
+
live,
|
|
411
|
+
stopping: false,
|
|
412
|
+
starting: true,
|
|
413
|
+
handle: undefined as unknown as WorkerHandle,
|
|
414
|
+
};
|
|
415
|
+
void channel.ended.then((exit) => onEnded(slot, running, exit));
|
|
416
|
+
|
|
417
|
+
let started: WorkerStartResult;
|
|
418
|
+
try {
|
|
419
|
+
const raw = await peer.request(
|
|
420
|
+
WORKER_METHODS.start,
|
|
421
|
+
{
|
|
422
|
+
artifactId: version.artifactId,
|
|
423
|
+
version: version.ref,
|
|
424
|
+
entry: version.entry,
|
|
425
|
+
limits: version.limits,
|
|
426
|
+
},
|
|
427
|
+
{ timeoutMs: START_TIMEOUT_MS },
|
|
428
|
+
);
|
|
429
|
+
started = WorkerStartResultSchema.parse(raw);
|
|
430
|
+
running.starting = false;
|
|
431
|
+
} catch (error) {
|
|
432
|
+
running.stopping = true;
|
|
433
|
+
channel.kill();
|
|
434
|
+
const exit = await channel.ended;
|
|
435
|
+
const tail = exit.stderrTail
|
|
436
|
+
.filter((line) => line.trim() !== "")
|
|
437
|
+
.slice(-3)
|
|
438
|
+
.join(" / ");
|
|
439
|
+
// How it ended: a worker that ran out of memory while loading is an OOM, not a crash.
|
|
440
|
+
const { kind } = classify(exit, running, version.limits);
|
|
441
|
+
throw new StartFailure(
|
|
442
|
+
`The server code of ${version.ref.versionId} couldn't start: ${error instanceof RpcError ? error.diagnostic.message : String(error)}${tail ? ` (${tail})` : ""}`,
|
|
443
|
+
kind,
|
|
444
|
+
);
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
const handle: WorkerHandle = {
|
|
448
|
+
versionId: version.ref.versionId,
|
|
449
|
+
artifactId: version.artifactId,
|
|
450
|
+
started,
|
|
451
|
+
call(info, listener) {
|
|
452
|
+
nextCall += 1;
|
|
453
|
+
const callId = `call_${nextCall}`;
|
|
454
|
+
const callInfo: CallInfo = {
|
|
455
|
+
callId,
|
|
456
|
+
context: info.context,
|
|
457
|
+
...(info.viewerId === undefined ? {} : { viewerId: info.viewerId }),
|
|
458
|
+
};
|
|
459
|
+
calls.set(callId, { info: callInfo, listener });
|
|
460
|
+
scheduleIdle(slot, running);
|
|
461
|
+
const outcome = (async (): Promise<WorkerCallOutcome> => {
|
|
462
|
+
try {
|
|
463
|
+
const raw = await peer.request(WORKER_METHODS.call, {
|
|
464
|
+
callId,
|
|
465
|
+
fnId: info.fnId,
|
|
466
|
+
args: info.args as never,
|
|
467
|
+
context: info.context as never,
|
|
468
|
+
});
|
|
469
|
+
return WorkerCallOutcomeSchema.parse(raw);
|
|
470
|
+
} catch (error) {
|
|
471
|
+
if (
|
|
472
|
+
error instanceof RpcError &&
|
|
473
|
+
error.diagnostic.code === "MOSTRA_HOST_CONNECTION_CLOSED"
|
|
474
|
+
) {
|
|
475
|
+
throw crashed(
|
|
476
|
+
`The server code of ${version.ref.versionId} stopped during the call: ${slot.lastCause ?? "its worker ended"}.`,
|
|
477
|
+
);
|
|
478
|
+
}
|
|
479
|
+
throw error;
|
|
480
|
+
} finally {
|
|
481
|
+
calls.delete(callId);
|
|
482
|
+
scheduleIdle(slot, running);
|
|
483
|
+
}
|
|
484
|
+
})();
|
|
485
|
+
return { callId, outcome };
|
|
486
|
+
},
|
|
487
|
+
cancel(callId, reason) {
|
|
488
|
+
peer.notify(WORKER_METHODS.cancel, { callId, ...(reason ? { reason } : {}) });
|
|
489
|
+
},
|
|
490
|
+
async liveStart(key) {
|
|
491
|
+
live.add(key);
|
|
492
|
+
scheduleIdle(slot, running);
|
|
493
|
+
try {
|
|
494
|
+
await peer.request(WORKER_METHODS.liveStart, { key }, { timeoutMs: 30_000 });
|
|
495
|
+
} catch (error) {
|
|
496
|
+
live.delete(key);
|
|
497
|
+
scheduleIdle(slot, running);
|
|
498
|
+
throw error;
|
|
499
|
+
}
|
|
500
|
+
},
|
|
501
|
+
async liveStop(key) {
|
|
502
|
+
if (!live.delete(key)) return;
|
|
503
|
+
scheduleIdle(slot, running);
|
|
504
|
+
await peer.request(WORKER_METHODS.liveStop, { key }, { timeoutMs: 10_000 }).catch(() => {});
|
|
505
|
+
},
|
|
506
|
+
liveRefresh(key) {
|
|
507
|
+
peer.notify(WORKER_METHODS.liveRefresh, { key });
|
|
508
|
+
},
|
|
509
|
+
};
|
|
510
|
+
(running as { handle: WorkerHandle }).handle = handle;
|
|
511
|
+
|
|
512
|
+
// The watchdog (D62): a worker whose event loop stays blocked past its limit, or that stops
|
|
513
|
+
// answering, is stopped and reported as hung.
|
|
514
|
+
let asking = false;
|
|
515
|
+
// Pings missed in a row. The channel thread answers even while artifact code blocks the
|
|
516
|
+
// event loop, so silence means the channel is congested (a large result ahead of the answer,
|
|
517
|
+
// a slow link to a launcher elsewhere) or the process is gone; only a long silence stops it.
|
|
518
|
+
let missed = 0;
|
|
519
|
+
running.watchdog = setInterval(() => {
|
|
520
|
+
if (asking || running.stopping) return;
|
|
521
|
+
asking = true;
|
|
522
|
+
const limit = version.limits.watchdogMs;
|
|
523
|
+
peer
|
|
524
|
+
.request(WORKER_METHODS.ping, {}, { timeoutMs: limit })
|
|
525
|
+
.then((raw) => {
|
|
526
|
+
const ping = WorkerPingResult.parse(raw);
|
|
527
|
+
missed = 0;
|
|
528
|
+
running.heapUsed = ping.heapUsed;
|
|
529
|
+
if (!ping.waitingForSecret && ping.blockedMs > limit) {
|
|
530
|
+
running.hung = ping.blockedMs;
|
|
531
|
+
channel.kill();
|
|
532
|
+
}
|
|
533
|
+
})
|
|
534
|
+
.catch(() => {
|
|
535
|
+
if (running.stopping || peer.closed) return;
|
|
536
|
+
missed += 1;
|
|
537
|
+
if (missed < PINGS_MISSED_BEFORE_STOP) return;
|
|
538
|
+
running.hung = limit * missed;
|
|
539
|
+
running.silent = true;
|
|
540
|
+
channel.kill();
|
|
541
|
+
})
|
|
542
|
+
.finally(() => {
|
|
543
|
+
asking = false;
|
|
544
|
+
});
|
|
545
|
+
}, watchdogEvery);
|
|
546
|
+
running.watchdog.unref?.();
|
|
547
|
+
scheduleIdle(slot, running);
|
|
548
|
+
return running;
|
|
549
|
+
};
|
|
550
|
+
|
|
551
|
+
const slotOf = (version: WorkerVersion): Slot => {
|
|
552
|
+
let slot = slots.get(version.ref.versionId);
|
|
553
|
+
if (!slot) {
|
|
554
|
+
slot = { version, state: "stopped", crashes: [] };
|
|
555
|
+
slots.set(version.ref.versionId, slot);
|
|
556
|
+
}
|
|
557
|
+
return slot;
|
|
558
|
+
};
|
|
559
|
+
|
|
560
|
+
return {
|
|
561
|
+
async acquire(version) {
|
|
562
|
+
if (stopped) throw crashed("The instance is stopping.");
|
|
563
|
+
const slot = slotOf(version);
|
|
564
|
+
for (;;) {
|
|
565
|
+
if (stopped) throw crashed("The instance is stopping.");
|
|
566
|
+
if (slot.current && !slot.current.stopping) return slot.current.handle;
|
|
567
|
+
if (slot.current?.stopping) {
|
|
568
|
+
// An idle stop in progress: the version keeps one worker, so its replacement starts
|
|
569
|
+
// only once this one has gone (RT-03).
|
|
570
|
+
await slot.current.channel.ended;
|
|
571
|
+
await Promise.resolve();
|
|
572
|
+
if (slot.current?.stopping) delete slot.current;
|
|
573
|
+
continue;
|
|
574
|
+
}
|
|
575
|
+
if (slot.state === "parked") {
|
|
576
|
+
throw new RpcError(
|
|
577
|
+
createDiagnostic("MOSTRA_RUNTIME_WORKER_PARKED", {
|
|
578
|
+
message: `The server code of this artifact crashed ${slot.crashes.length} times in 10 minutes, so it is parked; the last cause: ${slot.lastCause ?? "unknown"}.`,
|
|
579
|
+
}),
|
|
580
|
+
);
|
|
581
|
+
}
|
|
582
|
+
if (slot.starting) {
|
|
583
|
+
await slot.starting.catch(() => {});
|
|
584
|
+
if (slot.current) continue;
|
|
585
|
+
throw crashed(
|
|
586
|
+
`The server code of ${version.ref.versionId} couldn't start: ${slot.lastCause ?? "it failed"}.`,
|
|
587
|
+
);
|
|
588
|
+
}
|
|
589
|
+
if (slot.retryAt !== undefined && slot.retryAt > now()) {
|
|
590
|
+
await new Promise((resolve) => setTimeout(resolve, (slot.retryAt ?? now()) - now()));
|
|
591
|
+
continue;
|
|
592
|
+
}
|
|
593
|
+
slot.state = "starting";
|
|
594
|
+
slot.starting = start(slot);
|
|
595
|
+
try {
|
|
596
|
+
const running = await slot.starting;
|
|
597
|
+
if (slot.current === undefined && !running.stopping) {
|
|
598
|
+
slot.current = running;
|
|
599
|
+
slot.state = "ready";
|
|
600
|
+
} else if (!running.stopping) {
|
|
601
|
+
// Never left running untracked: a worker the slot can't hold is stopped at once.
|
|
602
|
+
void stopRunning(running);
|
|
603
|
+
}
|
|
604
|
+
delete slot.retryAt;
|
|
605
|
+
} catch (error) {
|
|
606
|
+
// A worker that can't start crashed as surely as one that died later: it backs off,
|
|
607
|
+
// and a version that never starts is parked like one that keeps crashing.
|
|
608
|
+
recordCrash(
|
|
609
|
+
slot,
|
|
610
|
+
error instanceof StartFailure ? error.kind : "crash",
|
|
611
|
+
error instanceof RpcError ? error.diagnostic.message : String(error),
|
|
612
|
+
[],
|
|
613
|
+
[],
|
|
614
|
+
);
|
|
615
|
+
throw error;
|
|
616
|
+
} finally {
|
|
617
|
+
delete slot.starting;
|
|
618
|
+
}
|
|
619
|
+
}
|
|
620
|
+
},
|
|
621
|
+
running(versionId) {
|
|
622
|
+
const current = slots.get(versionId)?.current;
|
|
623
|
+
return current && !current.stopping ? current.handle : undefined;
|
|
624
|
+
},
|
|
625
|
+
release(versionId) {
|
|
626
|
+
const current = slots.get(versionId)?.current;
|
|
627
|
+
if (!current || current.stopping || current.calls.size > 0 || current.live.size > 0) return;
|
|
628
|
+
options.log.info("worker.idle_stop", { versionId, reason: "released" });
|
|
629
|
+
void stopRunning(current);
|
|
630
|
+
},
|
|
631
|
+
async replace(version) {
|
|
632
|
+
const slot = slots.get(version.ref.versionId);
|
|
633
|
+
if (!slot) return;
|
|
634
|
+
slot.version = version;
|
|
635
|
+
slot.crashes = [];
|
|
636
|
+
delete slot.retryAt;
|
|
637
|
+
if (slot.state === "parked" || slot.state === "backoff") slot.state = "stopped";
|
|
638
|
+
await slot.starting?.catch(() => {});
|
|
639
|
+
const current = slot.current;
|
|
640
|
+
if (current && !current.stopping) {
|
|
641
|
+
options.log.info("worker.replaced", { versionId: version.ref.versionId });
|
|
642
|
+
await stopRunning(current);
|
|
643
|
+
}
|
|
644
|
+
},
|
|
645
|
+
clear(artifactId) {
|
|
646
|
+
for (const slot of slots.values()) {
|
|
647
|
+
if (slot.version.artifactId !== artifactId) continue;
|
|
648
|
+
slot.crashes = [];
|
|
649
|
+
delete slot.retryAt;
|
|
650
|
+
if (slot.state === "parked" || slot.state === "backoff") slot.state = "stopped";
|
|
651
|
+
}
|
|
652
|
+
},
|
|
653
|
+
status() {
|
|
654
|
+
const at = now();
|
|
655
|
+
return [...slots.values()].map((slot) => {
|
|
656
|
+
const running = slot.current;
|
|
657
|
+
const recent = slot.crashes.filter((when) => at - when < park.windowMs).length;
|
|
658
|
+
return {
|
|
659
|
+
artifactId: slot.version.artifactId,
|
|
660
|
+
versionId: slot.version.ref.versionId,
|
|
661
|
+
state: slot.state,
|
|
662
|
+
...(running?.channel.pid !== undefined ? { pid: running.channel.pid } : {}),
|
|
663
|
+
crashes: recent,
|
|
664
|
+
...(slot.lastCause ? { lastCause: slot.lastCause } : {}),
|
|
665
|
+
...(slot.retryAt !== undefined && slot.state === "backoff"
|
|
666
|
+
? { retryAt: new Date(slot.retryAt).toISOString() }
|
|
667
|
+
: {}),
|
|
668
|
+
active: running ? running.calls.size + running.live.size : 0,
|
|
669
|
+
};
|
|
670
|
+
});
|
|
671
|
+
},
|
|
672
|
+
activity() {
|
|
673
|
+
let count = 0;
|
|
674
|
+
for (const slot of slots.values()) {
|
|
675
|
+
if (slot.current) count += slot.current.calls.size + slot.current.live.size;
|
|
676
|
+
}
|
|
677
|
+
return count;
|
|
678
|
+
},
|
|
679
|
+
async stopAll() {
|
|
680
|
+
stopped = true;
|
|
681
|
+
await Promise.all(
|
|
682
|
+
[...slots.values()].flatMap((slot) => (slot.current ? [stopRunning(slot.current)] : [])),
|
|
683
|
+
);
|
|
684
|
+
},
|
|
685
|
+
};
|
|
686
|
+
}
|
|
687
|
+
|
|
688
|
+
/** The limits of a version from its manifest's `limits` (D62), within sane bounds. */
|
|
689
|
+
export function workerLimits(manifest: {
|
|
690
|
+
readonly limits?: {
|
|
691
|
+
readonly heapMb?: number;
|
|
692
|
+
readonly watchdogMs?: number;
|
|
693
|
+
readonly idleMinutes?: number;
|
|
694
|
+
};
|
|
695
|
+
}): WorkerLimits {
|
|
696
|
+
const limits = manifest.limits ?? {};
|
|
697
|
+
const clamp = (value: number | undefined, fallback: number, min: number, max: number): number =>
|
|
698
|
+
typeof value === "number" && Number.isFinite(value)
|
|
699
|
+
? Math.min(max, Math.max(min, value))
|
|
700
|
+
: fallback;
|
|
701
|
+
return {
|
|
702
|
+
heapMb: Math.round(clamp(limits.heapMb, DEFAULT_WORKER_LIMITS.heapMb, 64, 65_536)),
|
|
703
|
+
watchdogMs: Math.round(
|
|
704
|
+
clamp(limits.watchdogMs, DEFAULT_WORKER_LIMITS.watchdogMs, 1000, 3_600_000),
|
|
705
|
+
),
|
|
706
|
+
idleMinutes: clamp(limits.idleMinutes, DEFAULT_WORKER_LIMITS.idleMinutes, 0.05, 7 * 24 * 60),
|
|
707
|
+
};
|
|
708
|
+
}
|