@genroc/eval-node 0.0.0-edge.e03ab964 → 0.0.0-edge.e1372854
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +114 -306
- package/dist/eval.js +9 -17
- package/dist/import.js +35 -51
- package/dist/realm.js +12 -17
- package/dist/worker.js +58 -73
- package/eval.ts +13 -23
- package/import.ts +43 -60
- package/package.json +1 -1
- package/realm.ts +12 -17
- package/worker.ts +61 -77
package/import.ts
CHANGED
|
@@ -1,11 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
// The
|
|
3
|
-
// with
|
|
4
|
-
//
|
|
5
|
-
//
|
|
6
|
-
// Two modes, one binary: "types" writes the declarations an editor needs and returns no
|
|
7
|
-
// code; "build" typechecks and bundles. A separate types hook would mean a second `tsc`
|
|
8
|
-
// over the same project.
|
|
2
|
+
// The typed-phase resolver: manifest on stdin, `{"values": [...]}` on stdout, diagnostic on stderr
|
|
3
|
+
// with a non-zero exit (specs/source-resolution.md). Two modes share one `tsc` pass: "generate"
|
|
4
|
+
// writes declarations only, "resolve" typechecks and bundles.
|
|
9
5
|
|
|
10
6
|
import { existsSync } from "node:fs";
|
|
11
7
|
import { access, writeFile } from "node:fs/promises";
|
|
@@ -29,9 +25,9 @@ type Site = {
|
|
|
29
25
|
child?: string;
|
|
30
26
|
// The slot as a path of keys and indices, so nothing has to unescape a JSON Pointer.
|
|
31
27
|
pointer: (string | number)[];
|
|
32
|
-
//
|
|
33
|
-
// this resolver's reading of
|
|
34
|
-
|
|
28
|
+
// The argument's words, split as sh splits single quotes. genctl does not read them as paths —
|
|
29
|
+
// that is this resolver's reading of them.
|
|
30
|
+
args: string[];
|
|
35
31
|
// What .genroc asked genctl to type, keyed by the name it chose. This resolver reads Input
|
|
36
32
|
// and Output; which ADDRESS each came from is the config's business, not ours.
|
|
37
33
|
types?: Record<string, Schema>;
|
|
@@ -49,8 +45,7 @@ type ManifestProcess = {
|
|
|
49
45
|
};
|
|
50
46
|
|
|
51
47
|
type Manifest = {
|
|
52
|
-
mode: "
|
|
53
|
-
root: string;
|
|
48
|
+
mode: "generate" | "resolve";
|
|
54
49
|
processes: ManifestProcess[];
|
|
55
50
|
};
|
|
56
51
|
|
|
@@ -174,7 +169,7 @@ function declarations(at: Located): string {
|
|
|
174
169
|
}
|
|
175
170
|
|
|
176
171
|
return [
|
|
177
|
-
"// Generated by genroc. Do not edit - regenerate with `genctl
|
|
172
|
+
"// Generated by genroc. Do not edit - regenerate with `genctl generate`.",
|
|
178
173
|
`// ${where.name} (${address(site.pointer)})`,
|
|
179
174
|
"",
|
|
180
175
|
...emitted,
|
|
@@ -194,9 +189,8 @@ function typesPathFor(scriptPath: string): string {
|
|
|
194
189
|
|
|
195
190
|
// ── typecheck ──────────────────────────────────────────────────────────────────
|
|
196
191
|
|
|
197
|
-
/** The
|
|
198
|
-
*
|
|
199
|
-
* the project root: above it is not this project. */
|
|
192
|
+
/** The tsconfig the author's editor reads, so editor and apply agree. Stops at the project
|
|
193
|
+
* root. */
|
|
200
194
|
async function nearestTsconfig(
|
|
201
195
|
from: string,
|
|
202
196
|
root: string,
|
|
@@ -233,9 +227,8 @@ async function typecheck(sites: Located[]): Promise<void> {
|
|
|
233
227
|
// `lib` DESCRIBES the realm and is written after `extends` so a base cannot widen it:
|
|
234
228
|
// a worker thread has no document, whatever an author's config claims.
|
|
235
229
|
lib: ["esnext", "webworker"],
|
|
236
|
-
// `types` is the author's
|
|
237
|
-
//
|
|
238
|
-
// base config there is nothing to opt in with, so the default stays none.
|
|
230
|
+
// `types` is the author's opt-in to node globals, which the realm has. With no base
|
|
231
|
+
// config there is nothing to opt in with, so it stays none.
|
|
239
232
|
...(base ? {} : { types: [] }),
|
|
240
233
|
},
|
|
241
234
|
files: group.flatMap((s) => [s.file, typesPathFor(s.file)]),
|
|
@@ -243,9 +236,8 @@ async function typecheck(sites: Located[]): Promise<void> {
|
|
|
243
236
|
// drag the author's whole tree in, to be checked under the worker lib.
|
|
244
237
|
include: [],
|
|
245
238
|
};
|
|
246
|
-
// Never written
|
|
247
|
-
//
|
|
248
|
-
// the root, and tsc refuses a config that extends its own path as circular.
|
|
239
|
+
// Never written; anchors `@types` lookup at the root. Not `tsconfig.json`: that is often the
|
|
240
|
+
// base it extends, and tsc refuses a circular extends.
|
|
249
241
|
const parsed = ts.parseJsonConfigFileContent(
|
|
250
242
|
config,
|
|
251
243
|
ts.sys,
|
|
@@ -303,11 +295,9 @@ const BUILTIN = new Set([
|
|
|
303
295
|
...builtinModules.map((m) => `node:${m}`),
|
|
304
296
|
]);
|
|
305
297
|
|
|
306
|
-
/** Resolves
|
|
307
|
-
*
|
|
308
|
-
*
|
|
309
|
-
* A package resolving to a `.d.ts` is declined — that is a type, not the implementation —
|
|
310
|
-
* which leaves node_modules to nodeResolve. */
|
|
298
|
+
/** Resolves through TYPESCRIPT under the typecheck's config, so a `paths` alias that compiles
|
|
299
|
+
* also bundles. A `.d.ts` resolution is declined (a type, not code), leaving node_modules to
|
|
300
|
+
* nodeResolve. */
|
|
311
301
|
function tsResolve(configPath: string | null): Plugin {
|
|
312
302
|
let options: ts.CompilerOptions = {};
|
|
313
303
|
if (configPath) {
|
|
@@ -337,14 +327,12 @@ function tsResolve(configPath: string | null): Plugin {
|
|
|
337
327
|
};
|
|
338
328
|
}
|
|
339
329
|
|
|
340
|
-
/**
|
|
341
|
-
*
|
|
342
|
-
* Bundling is entirely the importer's job, so a definition version pins its code forever. */
|
|
330
|
+
/** A self-contained ES module whose default export is the author's own, unwrapped. Bundling here
|
|
331
|
+
* means a definition version pins its code forever. */
|
|
343
332
|
async function bundle(at: Located): Promise<string> {
|
|
344
333
|
const site = at.site;
|
|
345
|
-
// Builtins are
|
|
346
|
-
//
|
|
347
|
-
// of a module that will not be there, which bundles clean and fails at runtime.
|
|
334
|
+
// Builtins are externalised; anything else unresolved is a REFUSAL, since rollup's default
|
|
335
|
+
// leaves an import that bundles clean and fails at runtime.
|
|
348
336
|
const built = await rollup({
|
|
349
337
|
input: at.file,
|
|
350
338
|
external: (id) => BUILTIN.has(id),
|
|
@@ -352,8 +340,7 @@ async function bundle(at: Located): Promise<string> {
|
|
|
352
340
|
tsResolve(await nearestTsconfig(dirname(at.file), root)),
|
|
353
341
|
nodeResolve({ extensions: [".ts", ".tsx", ".mjs", ".js", ".json"] }),
|
|
354
342
|
commonjs(),
|
|
355
|
-
//
|
|
356
|
-
// did natively; without it rollup hands the JSON to the JS parser.
|
|
343
|
+
// Inlines `.json` imports; without it rollup hands JSON to the JS parser.
|
|
357
344
|
json(),
|
|
358
345
|
transpile,
|
|
359
346
|
],
|
|
@@ -396,10 +383,8 @@ const root = process.cwd();
|
|
|
396
383
|
if (!manifest || !Array.isArray(manifest.processes))
|
|
397
384
|
die("stdin is not a genroc resolver manifest");
|
|
398
385
|
|
|
399
|
-
/**
|
|
400
|
-
*
|
|
401
|
-
* joining it to the definition's directory is this resolver's business: only it knows the
|
|
402
|
-
* argument is a path at all. */
|
|
386
|
+
/** Sites in manifest order, the order `code` must answer in. The argument is joined to the
|
|
387
|
+
* definition's directory here: only this resolver knows it is a path. */
|
|
403
388
|
type Located = { site: Site; where: ManifestProcess; file: string };
|
|
404
389
|
|
|
405
390
|
/** A pointer as the address it is, so a generated comment or an error can be pasted into
|
|
@@ -415,22 +400,22 @@ function address(pointer: (string | number)[]): string {
|
|
|
415
400
|
.replace(/^\./, "");
|
|
416
401
|
}
|
|
417
402
|
const located: Located[] = manifest.processes.flatMap((where) =>
|
|
418
|
-
where.sites.map((site) =>
|
|
419
|
-
site
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
403
|
+
where.sites.map((site) => {
|
|
404
|
+
const [script, ...rest] = site.args;
|
|
405
|
+
if (script === undefined || rest.length > 0) {
|
|
406
|
+
die(
|
|
407
|
+
`${join(where.dir, where.file)}: ${address(site.pointer)}: $import takes one script, ` +
|
|
408
|
+
`and is given ${site.args.length} arguments`,
|
|
409
|
+
);
|
|
410
|
+
}
|
|
411
|
+
return { site, where, file: resolve(where.dir, script) };
|
|
412
|
+
}),
|
|
423
413
|
);
|
|
424
414
|
|
|
425
|
-
//
|
|
426
|
-
//
|
|
427
|
-
// `child` is the shape the scaffold uses — a call to a process that forwards to the evaluator —
|
|
428
|
-
// and `external` is the same request made directly.
|
|
415
|
+
// The `code`-field contract is this resolver's, not genctl's, so it checks the directive landed
|
|
416
|
+
// there: in a child call (the scaffold's shape) or an external task's input.
|
|
429
417
|
for (const at of located) {
|
|
430
|
-
//
|
|
431
|
-
// it the directive sits is the pointer. A child call is the shape the scaffold generates — to a
|
|
432
|
-
// process that forwards to the evaluator — and an external task is the same request made
|
|
433
|
-
// directly.
|
|
418
|
+
// The action's kind is a field; where in it the directive sits is the pointer.
|
|
434
419
|
const kind = at.site.level === "action" ? (at.site.action ?? "") : "";
|
|
435
420
|
const slot = at.site.pointer.slice(-2).join(".");
|
|
436
421
|
if ((kind !== "child" && kind !== "external") || slot !== "input.code") {
|
|
@@ -442,13 +427,12 @@ for (const at of located) {
|
|
|
442
427
|
}
|
|
443
428
|
}
|
|
444
429
|
|
|
445
|
-
// genctl
|
|
446
|
-
// does is the one that must say when it is not there.
|
|
430
|
+
// genctl does not know the argument is a file, so this resolver reports a missing one.
|
|
447
431
|
for (const at of located) {
|
|
448
432
|
if (!existsSync(at.file)) {
|
|
449
433
|
die(
|
|
450
434
|
`${join(at.where.dir, at.where.file)}: ${address(at.site.pointer)}: ` +
|
|
451
|
-
`"$import: ${at.site.
|
|
435
|
+
`"$import: ${at.site.args[0]}" names no file (looked at ${at.file})`,
|
|
452
436
|
);
|
|
453
437
|
}
|
|
454
438
|
}
|
|
@@ -475,15 +459,14 @@ for (const at of byPath.values()) {
|
|
|
475
459
|
await writeFile(typesPathFor(at.file), declarations(at));
|
|
476
460
|
}
|
|
477
461
|
|
|
478
|
-
if (manifest.mode === "
|
|
479
|
-
console.log(JSON.stringify(manifest, null, 2));
|
|
462
|
+
if (manifest.mode === "generate") {
|
|
480
463
|
process.exit(0);
|
|
481
464
|
}
|
|
482
465
|
|
|
483
466
|
await typecheck([...byPath.values()]);
|
|
484
467
|
|
|
485
|
-
const
|
|
468
|
+
const values: string[] = [];
|
|
486
469
|
for (const at of located) {
|
|
487
|
-
|
|
470
|
+
values.push(await bundle(at));
|
|
488
471
|
}
|
|
489
|
-
process.stdout.write(JSON.stringify({
|
|
472
|
+
process.stdout.write(JSON.stringify({ values }));
|
package/package.json
CHANGED
package/realm.ts
CHANGED
|
@@ -1,18 +1,13 @@
|
|
|
1
|
-
// The evaluation realm
|
|
2
|
-
//
|
|
3
|
-
// eval.ts owns the budget and does the killing; nothing here knows about time.
|
|
4
|
-
//
|
|
5
|
-
// Everything that touches the script's VALUE lives on this side of the boundary — loading,
|
|
6
|
-
// classifying, serialising — because this is the only realm the value exists in.
|
|
1
|
+
// The evaluation realm: one Worker per execution, killable mid-loop by eval.ts, which owns the
|
|
2
|
+
// budget. Everything touching the script's VALUE lives here, the only realm it exists in.
|
|
7
3
|
|
|
8
4
|
import { registerHooks } from "node:module";
|
|
9
5
|
import { parentPort } from "node:worker_threads";
|
|
10
6
|
|
|
11
7
|
import type { EvalFailure, FailureKind, WorkerReply, WorkerRequest } from "./eval.ts";
|
|
12
8
|
|
|
13
|
-
//
|
|
14
|
-
//
|
|
15
|
-
// the way it does everywhere else.
|
|
9
|
+
// IMPORTED under our own URL rather than compiled from a string, so frames carry the author's
|
|
10
|
+
// line numbers and builtin imports resolve normally.
|
|
16
11
|
const SCRIPT_URL = "script:main";
|
|
17
12
|
const STACK_BYTES = 2_048;
|
|
18
13
|
|
|
@@ -40,7 +35,10 @@ function scriptStack(err: unknown): string | undefined {
|
|
|
40
35
|
|
|
41
36
|
function describe(err: unknown, kind: FailureKind): EvalFailure {
|
|
42
37
|
if (err instanceof Error) {
|
|
43
|
-
|
|
38
|
+
// A subclass inherits `name` from Error.prototype, so `class LimitExceeded extends Error {}`
|
|
39
|
+
// would report "Error"; a name the script set still wins.
|
|
40
|
+
const name = err.name === "Error" && err.constructor.name ? err.constructor.name : err.name;
|
|
41
|
+
return { kind, name, message: err.message, stack: scriptStack(err) };
|
|
44
42
|
}
|
|
45
43
|
// A script may throw a non-Error (`throw {code: "x"}`), so name/message must not assume one.
|
|
46
44
|
return { kind, name: "Thrown", message: safeText(err) };
|
|
@@ -54,9 +52,8 @@ function safeText(v: unknown): string {
|
|
|
54
52
|
}
|
|
55
53
|
}
|
|
56
54
|
|
|
57
|
-
/**
|
|
58
|
-
*
|
|
59
|
-
* which is the script throwing. */
|
|
55
|
+
/** Unparseable code or an unresolvable import is broken code; anything else the import throws is
|
|
56
|
+
* the module's top level running, i.e. the script throwing. */
|
|
60
57
|
function unloadable(err: unknown): boolean {
|
|
61
58
|
const code = (err as { code?: unknown } | null)?.code;
|
|
62
59
|
return err instanceof SyntaxError || (typeof code === "string" && code.startsWith("ERR_MODULE"));
|
|
@@ -99,10 +96,8 @@ async function run(req: WorkerRequest): Promise<WorkerReply> {
|
|
|
99
96
|
}
|
|
100
97
|
}
|
|
101
98
|
|
|
102
|
-
//
|
|
103
|
-
//
|
|
104
|
-
// write's callback fires once the queue ahead of it has drained, which makes the reply a barrier
|
|
105
|
-
// for `console` and a direct process.stdout.write alike: both are this stream.
|
|
99
|
+
// eval.ts terminates this thread the moment the reply lands, while stdio may still be in the
|
|
100
|
+
// pipe. An empty write's callback fires once the queue drains, making the reply a barrier.
|
|
106
101
|
function flush(stream: NodeJS.WriteStream): Promise<void> {
|
|
107
102
|
return new Promise((resolve) => stream.write("", () => resolve()));
|
|
108
103
|
}
|
package/worker.ts
CHANGED
|
@@ -1,42 +1,38 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
// The queue worker: claims parked `external` script tasks
|
|
3
|
-
//
|
|
4
|
-
// nothing about the queue, which is what keeps the containment strategy swappable.
|
|
5
|
-
//
|
|
6
|
-
// See README.md for the contract, and specs/external-task-queue.md for the queue itself.
|
|
2
|
+
// The queue worker: claims parked `external` script tasks, evaluates each in its own realm, and
|
|
3
|
+
// answers. The only genroc-facing half (README.md; specs/external-task-queue.md).
|
|
7
4
|
|
|
8
5
|
import { readFileSync } from "node:fs";
|
|
9
6
|
import { Cancelled, evaluate, type EvalRequest, type FailureKind } from "./eval.ts";
|
|
10
7
|
|
|
11
8
|
const SERVER = (process.env.GENROC_SERVER ?? "http://localhost:8448").replace(/\/$/, "");
|
|
12
9
|
const WORKER_ID = process.env.WORKER_ID ?? `evaluator-${process.pid}`;
|
|
13
|
-
//
|
|
14
|
-
//
|
|
15
|
-
//
|
|
16
|
-
// trust least. specs/api-auth.md §5.
|
|
17
|
-
//
|
|
18
|
-
// Sent as a header rather than in the URL because Node's fetch REFUSES a URL carrying
|
|
19
|
-
// credentials ("Request cannot be constructed from a URL that includes credentials"), so the
|
|
20
|
-
// basic-auth-in-the-URL trick that works for genctl is not available here.
|
|
21
|
-
// GENROC_TOKEN_FILE is the mounted-secret shape: a credential in a file rather than an
|
|
22
|
-
// environment variable, so it stays out of `docker inspect` and out of the process environment
|
|
23
|
-
// any child inherits. The inline variable wins when both are set.
|
|
10
|
+
// Needs only the `worker` permission (specs/api-auth.md §5): this credential sits on the machine
|
|
11
|
+
// you trust least. A header, because Node's fetch refuses credentials in a URL. The file form
|
|
12
|
+
// keeps it out of `docker inspect` and child environments; the inline variable wins.
|
|
24
13
|
const TOKEN =
|
|
25
14
|
process.env.GENROC_TOKEN ??
|
|
26
15
|
(process.env.GENROC_TOKEN_FILE ? readFileSync(process.env.GENROC_TOKEN_FILE, "utf8").trim() : "");
|
|
27
16
|
const authHeaders: Record<string, string> = TOKEN ? { authorization: `Bearer ${TOKEN}` } : {};
|
|
28
|
-
//
|
|
29
|
-
//
|
|
30
|
-
// evaluator accepted every one of them. Here it claims what it can run and no more, so a
|
|
31
|
-
// backlog is a queue rather than 200 threads fighting over a core.
|
|
17
|
+
// The worker claims only what it can run, so a backlog is a queue rather than threads fighting
|
|
18
|
+
// over a core.
|
|
32
19
|
const CONCURRENCY = Number(process.env.CONCURRENCY ?? 4);
|
|
33
20
|
const POLL_MS = Number(process.env.POLL_MS ?? 250);
|
|
34
21
|
// The visibility timeout. Short, and renewed while work is in flight: a worker that dies
|
|
35
22
|
// should return its task quickly rather than holding it for the whole budget.
|
|
36
23
|
const LEASE_MS = Number(process.env.LEASE_MS ?? 30_000);
|
|
37
24
|
const RENEW_MS = Math.max(1_000, Math.floor(LEASE_MS / 3));
|
|
38
|
-
|
|
39
|
-
|
|
25
|
+
// The answer has to land before the task's deadline fires, or a script that overran surfaces as
|
|
26
|
+
// external.timeout ("nobody answered") rather than as its own `timeout`.
|
|
27
|
+
const ANSWER_MARGIN_MS = 250;
|
|
28
|
+
// script-node's names; a renamed copy sets PROCESS. The claim requires both: this worker answers
|
|
29
|
+
// one task's contract, so it must never be handed another's.
|
|
30
|
+
const PROCESS_FILTER = process.env.PROCESS ?? "script-node";
|
|
31
|
+
const TASK_FILTER = process.env.TASK ?? "eval_node";
|
|
32
|
+
if (!PROCESS_FILTER || !TASK_FILTER) {
|
|
33
|
+
console.error("PROCESS and TASK must both name the queue this worker serves; neither may be empty.");
|
|
34
|
+
process.exit(1);
|
|
35
|
+
}
|
|
40
36
|
|
|
41
37
|
type ObjectEntry = { path: (string | number)[]; ref: string; size: number };
|
|
42
38
|
|
|
@@ -46,20 +42,18 @@ type QueueTask = {
|
|
|
46
42
|
task: string;
|
|
47
43
|
external_input: unknown;
|
|
48
44
|
objects?: ObjectEntry[];
|
|
49
|
-
|
|
45
|
+
deadline?: string;
|
|
50
46
|
};
|
|
51
47
|
|
|
52
|
-
//
|
|
53
|
-
//
|
|
54
|
-
// into each task. A ref is a content hash, so it is immutable — the cache never invalidates.
|
|
48
|
+
// Large values arrive as refs; a bundle is one object per definition version, fetched once.
|
|
49
|
+
// Refs are content hashes, so the cache never invalidates.
|
|
55
50
|
const objectCache = new Map<string, unknown>();
|
|
56
51
|
|
|
57
52
|
async function fetchObject(ref: string): Promise<unknown> {
|
|
58
53
|
const cached = objectCache.get(ref);
|
|
59
54
|
if (cached !== undefined) return cached;
|
|
60
|
-
// Left to throw
|
|
61
|
-
//
|
|
62
|
-
// releases the claim, so the task returns to the queue.
|
|
55
|
+
// Left to throw: a task whose input cannot be fetched must not run against a missing value.
|
|
56
|
+
// The caller releases the claim.
|
|
63
57
|
const res = await fetch(`${SERVER}/api/objects/${encodeURIComponent(ref)}`, { headers: authHeaders });
|
|
64
58
|
if (!res.ok) throw new Error(`fetch object ${ref}: HTTP ${res.status}`);
|
|
65
59
|
const { data } = (await res.json()) as { data: string };
|
|
@@ -97,10 +91,8 @@ async function resolveObjects(job: QueueTask): Promise<unknown> {
|
|
|
97
91
|
}
|
|
98
92
|
|
|
99
93
|
async function call(path: string, body: unknown): Promise<{ ok: boolean; status: number; data: any }> {
|
|
100
|
-
// A network error is a REPLY
|
|
101
|
-
//
|
|
102
|
-
// unhandled rejection here kills it for a condition the next poll would clear. Status 0
|
|
103
|
-
// says "never reached the server", which is distinct from anything genroc answers.
|
|
94
|
+
// A network error is a REPLY (status 0, "never reached the server"), not a throw: a worker
|
|
95
|
+
// outlives restarts and deploys, and an unhandled rejection would kill it.
|
|
104
96
|
let res: Response;
|
|
105
97
|
try {
|
|
106
98
|
res = await fetch(SERVER + path, {
|
|
@@ -121,32 +113,24 @@ async function call(path: string, body: unknown): Promise<{ ok: boolean; status:
|
|
|
121
113
|
return { ok: res.ok, status: res.status, data };
|
|
122
114
|
}
|
|
123
115
|
|
|
124
|
-
/**
|
|
125
|
-
*
|
|
126
|
-
|
|
127
|
-
function asEvalRequest(input: unknown): EvalRequest | string {
|
|
116
|
+
/** A task input that is not an EvalRequest is the definition's fault, reported as compile_error:
|
|
117
|
+
* the nearest permanent kind, since no retry fixes the definition. */
|
|
118
|
+
function asEvalRequest(input: unknown, timeoutMs: number | undefined): EvalRequest | string {
|
|
128
119
|
if (typeof input !== "object" || input === null) return "the task input is not an object";
|
|
129
120
|
const r = input as Record<string, unknown>;
|
|
130
121
|
if (typeof r.code !== "string") return "the task input has no `code` string";
|
|
131
|
-
return {
|
|
132
|
-
code: r.code,
|
|
133
|
-
input: r.input,
|
|
134
|
-
timeout_ms: typeof r.timeout_ms === "number" ? r.timeout_ms : undefined,
|
|
135
|
-
};
|
|
122
|
+
return { code: r.code, input: r.input, timeout_ms: timeoutMs };
|
|
136
123
|
}
|
|
137
124
|
|
|
138
|
-
/**
|
|
139
|
-
*
|
|
140
|
-
* heartbeat and has to be delivered inward from there. */
|
|
125
|
+
/** genroc cannot call us, so a cancellation arrives on our own heartbeat and reaches the
|
|
126
|
+
* evaluation through the controller. */
|
|
141
127
|
type Running = { job: QueueTask; abort: AbortController };
|
|
142
128
|
|
|
143
129
|
const inFlight = new Map<string, Running>();
|
|
144
130
|
let running = true;
|
|
145
131
|
|
|
146
|
-
//
|
|
147
|
-
//
|
|
148
|
-
// buries the one line that mattered. Announce the TRANSITIONS instead: going away, and coming
|
|
149
|
-
// back. Silence in between is the report that nothing changed.
|
|
132
|
+
// Announce reachability TRANSITIONS only: polling several times a second would otherwise emit
|
|
133
|
+
// thousands of lines during a restart.
|
|
150
134
|
let serverReachable = true;
|
|
151
135
|
|
|
152
136
|
async function claim(n: number): Promise<QueueTask[]> {
|
|
@@ -154,13 +138,12 @@ async function claim(n: number): Promise<QueueTask[]> {
|
|
|
154
138
|
worker_id: WORKER_ID,
|
|
155
139
|
limit: n,
|
|
156
140
|
lease_ms: LEASE_MS,
|
|
157
|
-
|
|
158
|
-
|
|
141
|
+
process: PROCESS_FILTER,
|
|
142
|
+
task: TASK_FILTER,
|
|
159
143
|
});
|
|
160
144
|
if (!ok) {
|
|
161
|
-
// A credential problem is not transient, and polling through it looks like a healthy
|
|
162
|
-
// worker
|
|
163
|
-
// instead, so a supervisor restarts it and the failure is visible where it happened.
|
|
145
|
+
// A credential problem is not transient, and polling through it looks like a healthy idle
|
|
146
|
+
// worker. Exit, so a supervisor restarts it visibly.
|
|
164
147
|
if (status === 401 || status === 403) {
|
|
165
148
|
console.error(
|
|
166
149
|
`claim rejected (${status}): ${JSON.stringify(data)}\n` +
|
|
@@ -169,9 +152,7 @@ async function claim(n: number): Promise<QueueTask[]> {
|
|
|
169
152
|
);
|
|
170
153
|
process.exit(1);
|
|
171
154
|
}
|
|
172
|
-
// status 0
|
|
173
|
-
// network blip. Not an error to act on — the next poll clears it — so it is reported once
|
|
174
|
-
// and then waited out.
|
|
155
|
+
// status 0 (see call): a restart or blip the next poll clears, so it is reported once.
|
|
175
156
|
if (status === 0) {
|
|
176
157
|
if (serverReachable) {
|
|
177
158
|
serverReachable = false;
|
|
@@ -197,10 +178,8 @@ async function release(token: string): Promise<void> {
|
|
|
197
178
|
if (!ok) console.error(`release failed: ${JSON.stringify(data)}`);
|
|
198
179
|
}
|
|
199
180
|
|
|
200
|
-
/**
|
|
201
|
-
*
|
|
202
|
-
* not fit `raises`), and guessing again would only pick a second wrong answer. Release it, so
|
|
203
|
-
* the task returns to the queue and an operator sees it waiting rather than silently gone. */
|
|
181
|
+
/** A refused outcome is not retried with another: the worker does not satisfy the declared
|
|
182
|
+
* contract. Released, so an operator sees the task waiting rather than gone. */
|
|
204
183
|
async function answer(token: string, outcome: Record<string, unknown>): Promise<void> {
|
|
205
184
|
const { ok, data } = await call("/api/external-tasks/resolve", { token, ...outcome });
|
|
206
185
|
if (ok) return;
|
|
@@ -209,6 +188,9 @@ async function answer(token: string, outcome: Record<string, unknown>): Promise<
|
|
|
209
188
|
}
|
|
210
189
|
|
|
211
190
|
async function run(job: QueueTask, signal: AbortSignal): Promise<void> {
|
|
191
|
+
// The task's own timeout is the script's budget, moved onto the monotonic clock so a wall-clock
|
|
192
|
+
// step mid-run cannot stretch it. Fetching the objects below spends the same deadline.
|
|
193
|
+
const due = job.deadline === undefined ? undefined : performance.now() + (Date.parse(job.deadline) - Date.now());
|
|
212
194
|
let resolved: unknown;
|
|
213
195
|
try {
|
|
214
196
|
resolved = await resolveObjects(job);
|
|
@@ -219,7 +201,16 @@ async function run(job: QueueTask, signal: AbortSignal): Promise<void> {
|
|
|
219
201
|
await release(job.token);
|
|
220
202
|
return;
|
|
221
203
|
}
|
|
222
|
-
|
|
204
|
+
let timeoutMs: number | undefined;
|
|
205
|
+
if (due !== undefined) {
|
|
206
|
+
timeoutMs = Math.floor(due - performance.now() - ANSWER_MARGIN_MS);
|
|
207
|
+
if (timeoutMs <= 0) {
|
|
208
|
+
// Too late to run anything: the deadline fires as external.timeout, which is the truth.
|
|
209
|
+
await release(job.token);
|
|
210
|
+
return;
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
const req = asEvalRequest(resolved, timeoutMs);
|
|
223
214
|
if (typeof req === "string") {
|
|
224
215
|
await answer(job.token, {
|
|
225
216
|
error: { code: "compile_error", message: req, data: { name: "BadTaskInput" } },
|
|
@@ -231,18 +222,15 @@ async function run(job: QueueTask, signal: AbortSignal): Promise<void> {
|
|
|
231
222
|
try {
|
|
232
223
|
result = await evaluate(req, signal);
|
|
233
224
|
} catch (err) {
|
|
234
|
-
// Cancelled is not a fault: the
|
|
235
|
-
//
|
|
236
|
-
// waiting out a lease nobody is serving.
|
|
225
|
+
// Cancelled is not a fault. Still released: the row is terminal so nothing re-claims it,
|
|
226
|
+
// but the release stops it waiting out a lease nobody serves.
|
|
237
227
|
if (err instanceof Cancelled) {
|
|
238
228
|
console.log(`cancelled: ${job.token}`);
|
|
239
229
|
await release(job.token);
|
|
240
230
|
return;
|
|
241
231
|
}
|
|
242
|
-
// The RUNNER faulted,
|
|
243
|
-
//
|
|
244
|
-
// it puts the task in front of a different worker instead of burning the definition's
|
|
245
|
-
// on_error budget on this one's bad day.
|
|
232
|
+
// The RUNNER faulted, the one retryable class. Releasing is how a queue spells "retryable",
|
|
233
|
+
// and it reaches another worker without spending the definition's on_error budget.
|
|
246
234
|
console.error(`evaluator fault on ${job.token}: ${err instanceof Error ? err.message : String(err)}`);
|
|
247
235
|
await release(job.token);
|
|
248
236
|
return;
|
|
@@ -278,17 +266,13 @@ async function renewLoop(): Promise<void> {
|
|
|
278
266
|
});
|
|
279
267
|
if (!ok) continue;
|
|
280
268
|
|
|
281
|
-
//
|
|
282
|
-
//
|
|
283
|
-
// keep burning a core on work an operator has already stopped. The release rides run()'s
|
|
284
|
-
// own catch, so nothing is released twice.
|
|
269
|
+
// Abort first rather than let a stopped script burn a core; the release rides run()'s
|
|
270
|
+
// catch, so nothing is released twice.
|
|
285
271
|
for (const token of (data?.cancelled ?? []) as string[]) {
|
|
286
272
|
inFlight.get(token)?.abort.abort();
|
|
287
273
|
}
|
|
288
|
-
// Lost:
|
|
289
|
-
//
|
|
290
|
-
// is too short for what these scripts actually take. Deliberately NOT released: the claim
|
|
291
|
-
// is the new holder's, and releasing would bump the epoch out from under it.
|
|
274
|
+
// Lost: another worker holds it now and our answer will be refused; logged as the sign that
|
|
275
|
+
// LEASE_MS is too short. NOT released: that would bump the epoch out from under the holder.
|
|
292
276
|
const lost = (data?.lost ?? []) as string[];
|
|
293
277
|
if (lost.length) {
|
|
294
278
|
console.error(`lost ${lost.length}/${tokens.length} claims; a lease lapsed under load`);
|