@genroc/eval-node 0.0.0-edge.4348678 → 0.0.0-edge.4394f2ae
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +114 -282
- package/dist/eval.js +27 -14
- package/dist/import.js +139 -130
- package/dist/realm.js +66 -80
- package/dist/worker.js +83 -73
- package/eval.ts +31 -20
- package/import.ts +228 -154
- package/package.json +1 -1
- package/realm.ts +61 -82
- package/worker.ts +92 -77
package/dist/realm.js
CHANGED
|
@@ -1,81 +1,45 @@
|
|
|
1
|
-
// The evaluation realm
|
|
2
|
-
//
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
1
|
+
// The evaluation realm: one Worker per execution, killable mid-loop by eval.ts, which owns the
|
|
2
|
+
// budget. Everything touching the script's VALUE lives here, the only realm it exists in.
|
|
3
|
+
var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExtension) || function (path, preserveJsx) {
|
|
4
|
+
if (typeof path === "string" && /^\.\.?\//.test(path)) {
|
|
5
|
+
return path.replace(/\.(tsx)$|((?:\.d)?)((?:\.[^./]+?)?)\.([cm]?)ts$/i, function (m, tsx, d, ext, cm) {
|
|
6
|
+
return tsx ? preserveJsx ? ".jsx" : ".js" : d && (!ext || !cm) ? m : (d + ext + "." + cm.toLowerCase() + "js");
|
|
7
|
+
});
|
|
8
|
+
}
|
|
9
|
+
return path;
|
|
10
|
+
};
|
|
11
|
+
import { registerHooks } from "node:module";
|
|
8
12
|
import { parentPort } from "node:worker_threads";
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
// inlines everything else — and a function built by the AsyncFunction constructor has no
|
|
13
|
-
// `require` in scope. Passing one in is what makes an import of a builtin work at runtime rather than at
|
|
14
|
-
// typecheck only. Resolution is anchored here, which is right: only builtins reach it.
|
|
15
|
-
const scriptRequire = createRequire(import.meta.url);
|
|
13
|
+
// IMPORTED under our own URL rather than compiled from a string, so frames carry the author's
|
|
14
|
+
// line numbers and builtin imports resolve normally.
|
|
15
|
+
const SCRIPT_URL = "script:main";
|
|
16
16
|
const STACK_BYTES = 2_048;
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
return 0;
|
|
28
|
-
}
|
|
29
|
-
catch (err) {
|
|
30
|
-
return reportedLine(err) - 1;
|
|
31
|
-
}
|
|
32
|
-
})();
|
|
33
|
-
// V8 marks a frame compiled by the AsyncFunction constructor with the site that CALLED the
|
|
34
|
-
// constructor, then the script's OWN position:
|
|
35
|
-
// at inner (eval at run (file:///…/worker.ts:107:10), <anonymous>:6:9)
|
|
36
|
-
// `eval at` is therefore what separates script frames from runner plumbing — matched without
|
|
37
|
-
// the function name, which is whatever encloses the `new AsyncFunction` below. The LAST such
|
|
38
|
-
// frame is the body's top level: frames interleave, since a script can throw inside a native
|
|
39
|
-
// callback.
|
|
40
|
-
const SCRIPT_FRAME = /\(eval at /;
|
|
41
|
-
// The trailing `<anonymous>:LINE:COL` — the script's position, after the host file's own.
|
|
42
|
-
const POSITION = /<anonymous>:(\d+):(\d+)\)?\s*$/;
|
|
43
|
-
// ` at name (` — absent on the top-level frame, which V8 names `eval`.
|
|
44
|
-
const FRAME_NAME = /^\s*at\s+(?:async\s+)?([^\s(]+)\s*\(/;
|
|
45
|
-
/** Line number of the throw as the engine reported it, or 1 if the stack is unreadable. */
|
|
46
|
-
function reportedLine(err) {
|
|
47
|
-
const stack = err instanceof Error && typeof err.stack === "string" ? err.stack : "";
|
|
48
|
-
const frame = stack.split("\n").find((l) => SCRIPT_FRAME.test(l)) ?? "";
|
|
49
|
-
const m = frame.match(POSITION);
|
|
50
|
-
return m ? Number(m[1]) : 1;
|
|
51
|
-
}
|
|
52
|
-
/** Renumbers each script frame to the line the AUTHOR wrote and drops the runner's own.
|
|
53
|
-
* Rewriting the whole location is also what keeps the runner's path out of a script's
|
|
54
|
-
* stack — V8 puts it inside every compiled frame. */
|
|
55
|
-
function scriptStack(err, offset) {
|
|
17
|
+
// The only channel a load hook has to the source. Written per request and read once — a realm
|
|
18
|
+
// evaluates one script and is then discarded, so no second execution can observe it.
|
|
19
|
+
let source = "";
|
|
20
|
+
registerHooks({
|
|
21
|
+
resolve: (specifier, context, next) => specifier === SCRIPT_URL ? { url: SCRIPT_URL, shortCircuit: true } : next(specifier, context),
|
|
22
|
+
load: (url, context, next) => url === SCRIPT_URL ? { format: "module", source, shortCircuit: true } : next(url, context),
|
|
23
|
+
});
|
|
24
|
+
/** Keeps the frames that are the script's own. Everything below the last of them is runner
|
|
25
|
+
* plumbing the author cannot act on, and V8 puts this file's path in it. */
|
|
26
|
+
function scriptStack(err) {
|
|
56
27
|
if (!(err instanceof Error) || typeof err.stack !== "string")
|
|
57
28
|
return undefined;
|
|
58
29
|
const lines = err.stack.split("\n");
|
|
59
|
-
let
|
|
30
|
+
let last = 0;
|
|
60
31
|
for (let i = 0; i < lines.length; i++)
|
|
61
|
-
if (
|
|
62
|
-
|
|
63
|
-
const
|
|
64
|
-
|
|
65
|
-
const pos = line.match(POSITION);
|
|
66
|
-
if (!pos)
|
|
67
|
-
return line; // the `Error: message` header and native frames, kept as-is
|
|
68
|
-
const name = line.match(FRAME_NAME)?.[1];
|
|
69
|
-
const at = name && name !== "eval" && name !== "anonymous" ? `at ${name} ` : "at ";
|
|
70
|
-
const indent = line.match(/^\s*/)[0];
|
|
71
|
-
return `${indent}${at}(script:${Math.max(1, Number(pos[1]) - offset)}:${pos[2]})`;
|
|
72
|
-
})
|
|
73
|
-
.join("\n");
|
|
74
|
-
return frames.length > STACK_BYTES ? frames.slice(0, STACK_BYTES) : frames;
|
|
32
|
+
if (lines[i].includes(SCRIPT_URL))
|
|
33
|
+
last = i;
|
|
34
|
+
const stack = lines.slice(0, last + 1).join("\n");
|
|
35
|
+
return stack.length > STACK_BYTES ? stack.slice(0, STACK_BYTES) : stack;
|
|
75
36
|
}
|
|
76
|
-
function describe(err, kind
|
|
37
|
+
function describe(err, kind) {
|
|
77
38
|
if (err instanceof Error) {
|
|
78
|
-
|
|
39
|
+
// A subclass inherits `name` from Error.prototype, so `class LimitExceeded extends Error {}`
|
|
40
|
+
// would report "Error"; a name the script set still wins.
|
|
41
|
+
const name = err.name === "Error" && err.constructor.name ? err.constructor.name : err.name;
|
|
42
|
+
return { kind, name, message: err.message, stack: scriptStack(err) };
|
|
79
43
|
}
|
|
80
44
|
// A script may throw a non-Error (`throw {code: "x"}`), so name/message must not assume one.
|
|
81
45
|
return { kind, name: "Thrown", message: safeText(err) };
|
|
@@ -88,23 +52,38 @@ function safeText(v) {
|
|
|
88
52
|
return String(v);
|
|
89
53
|
}
|
|
90
54
|
}
|
|
55
|
+
/** Unparseable code or an unresolvable import is broken code; anything else the import throws is
|
|
56
|
+
* the module's top level running, i.e. the script throwing. */
|
|
57
|
+
function unloadable(err) {
|
|
58
|
+
const code = err?.code;
|
|
59
|
+
return err instanceof SyntaxError || (typeof code === "string" && code.startsWith("ERR_MODULE"));
|
|
60
|
+
}
|
|
91
61
|
async function run(req) {
|
|
92
|
-
|
|
93
|
-
let
|
|
62
|
+
source = req.code;
|
|
63
|
+
let main;
|
|
94
64
|
try {
|
|
95
|
-
|
|
96
|
-
// never be hit. Repeated compilation is the price of the fresh global object.
|
|
97
|
-
fn = new AsyncFunction("input", "require", STRICT + req.code);
|
|
65
|
+
main = (await import(__rewriteRelativeImportExtension(SCRIPT_URL))).default;
|
|
98
66
|
}
|
|
99
67
|
catch (err) {
|
|
100
|
-
return { ok: false, failure: describe(err, "compile_error"
|
|
68
|
+
return { ok: false, failure: describe(err, unloadable(err) ? "compile_error" : "threw") };
|
|
69
|
+
}
|
|
70
|
+
if (typeof main !== "function") {
|
|
71
|
+
const got = main === undefined ? "no default export" : `a ${typeof main}`;
|
|
72
|
+
return {
|
|
73
|
+
ok: false,
|
|
74
|
+
failure: {
|
|
75
|
+
kind: "compile_error",
|
|
76
|
+
name: "NoDefaultExport",
|
|
77
|
+
message: `a script must export default a function; this one has ${got}`,
|
|
78
|
+
},
|
|
79
|
+
};
|
|
101
80
|
}
|
|
102
81
|
let value;
|
|
103
82
|
try {
|
|
104
|
-
value = await
|
|
83
|
+
value = await main(req.input);
|
|
105
84
|
}
|
|
106
85
|
catch (err) {
|
|
107
|
-
return { ok: false, failure: describe(err, "threw"
|
|
86
|
+
return { ok: false, failure: describe(err, "threw") };
|
|
108
87
|
}
|
|
109
88
|
try {
|
|
110
89
|
// undefined stringifies to undefined, not "undefined"; an empty body is how genroc
|
|
@@ -112,11 +91,18 @@ async function run(req) {
|
|
|
112
91
|
return { ok: true, body: value === undefined ? "" : JSON.stringify(value) ?? "" };
|
|
113
92
|
}
|
|
114
93
|
catch (err) {
|
|
115
|
-
return { ok: false, failure: describe(err, "nonserializable"
|
|
94
|
+
return { ok: false, failure: describe(err, "nonserializable") };
|
|
116
95
|
}
|
|
117
96
|
}
|
|
97
|
+
// eval.ts terminates this thread the moment the reply lands, while stdio may still be in the
|
|
98
|
+
// pipe. An empty write's callback fires once the queue drains, making the reply a barrier.
|
|
99
|
+
function flush(stream) {
|
|
100
|
+
return new Promise((resolve) => stream.write("", () => resolve()));
|
|
101
|
+
}
|
|
118
102
|
// Non-null because this module only ever runs as a worker entry point; a null port here
|
|
119
103
|
// would mean eval.ts loaded it as a plain module, which nothing does.
|
|
120
104
|
parentPort.on("message", async (req) => {
|
|
121
|
-
|
|
105
|
+
const reply = await run(req);
|
|
106
|
+
await Promise.all([flush(process.stdout), flush(process.stderr)]);
|
|
107
|
+
parentPort.postMessage(reply);
|
|
122
108
|
});
|
package/dist/worker.js
CHANGED
|
@@ -1,45 +1,44 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
// The queue worker: claims parked `external` script tasks
|
|
3
|
-
//
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
// See README.md for the contract, and specs/external-task-queue.md for the queue itself.
|
|
7
|
-
import { evaluate } from "./eval.js";
|
|
2
|
+
// The queue worker: claims parked `external` script tasks, evaluates each in its own realm, and
|
|
3
|
+
// answers. The only genroc-facing half (README.md; specs/external-task-queue.md).
|
|
4
|
+
import { readFileSync } from "node:fs";
|
|
5
|
+
import { Cancelled, evaluate } from "./eval.js";
|
|
8
6
|
const SERVER = (process.env.GENROC_SERVER ?? "http://localhost:8448").replace(/\/$/, "");
|
|
9
7
|
const WORKER_ID = process.env.WORKER_ID ?? `evaluator-${process.pid}`;
|
|
10
|
-
//
|
|
11
|
-
//
|
|
12
|
-
//
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
// Sent as a header rather than in the URL because Node's fetch REFUSES a URL carrying
|
|
16
|
-
// credentials ("Request cannot be constructed from a URL that includes credentials"), so the
|
|
17
|
-
// basic-auth-in-the-URL trick that works for genctl is not available here.
|
|
18
|
-
const TOKEN = process.env.GENROC_TOKEN ?? "";
|
|
8
|
+
// Needs only the `worker` permission (specs/api-auth.md §5): this credential sits on the machine
|
|
9
|
+
// you trust least. A header, because Node's fetch refuses credentials in a URL. The file form
|
|
10
|
+
// keeps it out of `docker inspect` and child environments; the inline variable wins.
|
|
11
|
+
const TOKEN = process.env.GENROC_TOKEN ??
|
|
12
|
+
(process.env.GENROC_TOKEN_FILE ? readFileSync(process.env.GENROC_TOKEN_FILE, "utf8").trim() : "");
|
|
19
13
|
const authHeaders = TOKEN ? { authorization: `Bearer ${TOKEN}` } : {};
|
|
20
|
-
//
|
|
21
|
-
//
|
|
22
|
-
// evaluator accepted every one of them. Here it claims what it can run and no more, so a
|
|
23
|
-
// backlog is a queue rather than 200 threads fighting over a core.
|
|
14
|
+
// The worker claims only what it can run, so a backlog is a queue rather than threads fighting
|
|
15
|
+
// over a core.
|
|
24
16
|
const CONCURRENCY = Number(process.env.CONCURRENCY ?? 4);
|
|
25
17
|
const POLL_MS = Number(process.env.POLL_MS ?? 250);
|
|
26
18
|
// The visibility timeout. Short, and renewed while work is in flight: a worker that dies
|
|
27
19
|
// should return its task quickly rather than holding it for the whole budget.
|
|
28
20
|
const LEASE_MS = Number(process.env.LEASE_MS ?? 30_000);
|
|
29
21
|
const RENEW_MS = Math.max(1_000, Math.floor(LEASE_MS / 3));
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
//
|
|
34
|
-
//
|
|
22
|
+
// The answer has to land before the task's deadline fires, or a script that overran surfaces as
|
|
23
|
+
// external.timeout ("nobody answered") rather than as its own `timeout`.
|
|
24
|
+
const ANSWER_MARGIN_MS = 250;
|
|
25
|
+
// script-node's names; a renamed copy sets PROCESS. The claim requires both: this worker answers
|
|
26
|
+
// one task's contract, so it must never be handed another's.
|
|
27
|
+
const PROCESS_FILTER = process.env.PROCESS ?? "script-node";
|
|
28
|
+
const TASK_FILTER = process.env.TASK ?? "eval_node";
|
|
29
|
+
if (!PROCESS_FILTER || !TASK_FILTER) {
|
|
30
|
+
console.error("PROCESS and TASK must both name the queue this worker serves; neither may be empty.");
|
|
31
|
+
process.exit(1);
|
|
32
|
+
}
|
|
33
|
+
// Large values arrive as refs; a bundle is one object per definition version, fetched once.
|
|
34
|
+
// Refs are content hashes, so the cache never invalidates.
|
|
35
35
|
const objectCache = new Map();
|
|
36
36
|
async function fetchObject(ref) {
|
|
37
37
|
const cached = objectCache.get(ref);
|
|
38
38
|
if (cached !== undefined)
|
|
39
39
|
return cached;
|
|
40
|
-
// Left to throw
|
|
41
|
-
//
|
|
42
|
-
// releases the claim, so the task returns to the queue.
|
|
40
|
+
// Left to throw: a task whose input cannot be fetched must not run against a missing value.
|
|
41
|
+
// The caller releases the claim.
|
|
43
42
|
const res = await fetch(`${SERVER}/api/objects/${encodeURIComponent(ref)}`, { headers: authHeaders });
|
|
44
43
|
if (!res.ok)
|
|
45
44
|
throw new Error(`fetch object ${ref}: HTTP ${res.status}`);
|
|
@@ -57,15 +56,15 @@ async function fetchObject(ref) {
|
|
|
57
56
|
/** Put each listed value back where its path says it belongs. Paths are arrays of keys, so this
|
|
58
57
|
* needs no parser: the whole reason they are not JSON Pointer strings. */
|
|
59
58
|
async function resolveObjects(job) {
|
|
60
|
-
let input = job.
|
|
59
|
+
let input = job.external_input;
|
|
61
60
|
for (const e of job.objects ?? []) {
|
|
62
61
|
const value = await fetchObject(e.ref);
|
|
63
62
|
if (e.path.length === 0) {
|
|
64
63
|
input = value;
|
|
65
64
|
continue;
|
|
66
65
|
}
|
|
67
|
-
// The path is rooted at the entry and starts with "
|
|
68
|
-
const rest = e.path[0] === "
|
|
66
|
+
// The path is rooted at the entry and starts with "external_input", the value being rebuilt.
|
|
67
|
+
const rest = e.path[0] === "external_input" ? e.path.slice(1) : e.path;
|
|
69
68
|
if (rest.length === 0) {
|
|
70
69
|
input = value;
|
|
71
70
|
continue;
|
|
@@ -79,10 +78,8 @@ async function resolveObjects(job) {
|
|
|
79
78
|
return input;
|
|
80
79
|
}
|
|
81
80
|
async function call(path, body) {
|
|
82
|
-
// A network error is a REPLY
|
|
83
|
-
//
|
|
84
|
-
// unhandled rejection here kills it for a condition the next poll would clear. Status 0
|
|
85
|
-
// says "never reached the server", which is distinct from anything genroc answers.
|
|
81
|
+
// A network error is a REPLY (status 0, "never reached the server"), not a throw: a worker
|
|
82
|
+
// outlives restarts and deploys, and an unhandled rejection would kill it.
|
|
86
83
|
let res;
|
|
87
84
|
try {
|
|
88
85
|
res = await fetch(SERVER + path, {
|
|
@@ -104,49 +101,39 @@ async function call(path, body) {
|
|
|
104
101
|
}
|
|
105
102
|
return { ok: res.ok, status: res.status, data };
|
|
106
103
|
}
|
|
107
|
-
/**
|
|
108
|
-
*
|
|
109
|
-
|
|
110
|
-
function asEvalRequest(input) {
|
|
104
|
+
/** A task input that is not an EvalRequest is the definition's fault, reported as compile_error:
|
|
105
|
+
* the nearest permanent kind, since no retry fixes the definition. */
|
|
106
|
+
function asEvalRequest(input, timeoutMs) {
|
|
111
107
|
if (typeof input !== "object" || input === null)
|
|
112
108
|
return "the task input is not an object";
|
|
113
109
|
const r = input;
|
|
114
110
|
if (typeof r.code !== "string")
|
|
115
111
|
return "the task input has no `code` string";
|
|
116
|
-
return {
|
|
117
|
-
code: r.code,
|
|
118
|
-
input: r.input,
|
|
119
|
-
timeout_ms: typeof r.timeout_ms === "number" ? r.timeout_ms : undefined,
|
|
120
|
-
};
|
|
112
|
+
return { code: r.code, input: r.input, timeout_ms: timeoutMs };
|
|
121
113
|
}
|
|
122
114
|
const inFlight = new Map();
|
|
123
115
|
let running = true;
|
|
124
|
-
//
|
|
125
|
-
//
|
|
126
|
-
// buries the one line that mattered. Announce the TRANSITIONS instead: going away, and coming
|
|
127
|
-
// back. Silence in between is the report that nothing changed.
|
|
116
|
+
// Announce reachability TRANSITIONS only: polling several times a second would otherwise emit
|
|
117
|
+
// thousands of lines during a restart.
|
|
128
118
|
let serverReachable = true;
|
|
129
119
|
async function claim(n) {
|
|
130
120
|
const { ok, status, data } = await call("/api/external-tasks/claim", {
|
|
131
121
|
worker_id: WORKER_ID,
|
|
132
122
|
limit: n,
|
|
133
123
|
lease_ms: LEASE_MS,
|
|
134
|
-
|
|
135
|
-
|
|
124
|
+
process: PROCESS_FILTER,
|
|
125
|
+
task: TASK_FILTER,
|
|
136
126
|
});
|
|
137
127
|
if (!ok) {
|
|
138
|
-
// A credential problem is not transient, and polling through it looks like a healthy
|
|
139
|
-
// worker
|
|
140
|
-
// instead, so a supervisor restarts it and the failure is visible where it happened.
|
|
128
|
+
// A credential problem is not transient, and polling through it looks like a healthy idle
|
|
129
|
+
// worker. Exit, so a supervisor restarts it visibly.
|
|
141
130
|
if (status === 401 || status === 403) {
|
|
142
131
|
console.error(`claim rejected (${status}): ${JSON.stringify(data)}\n` +
|
|
143
132
|
`The server requires authentication. Set GENROC_TOKEN to a token with the 'worker' ` +
|
|
144
133
|
`permission — mint one with: genctl token create --perms worker --label evaluator -q`);
|
|
145
134
|
process.exit(1);
|
|
146
135
|
}
|
|
147
|
-
// status 0
|
|
148
|
-
// network blip. Not an error to act on — the next poll clears it — so it is reported once
|
|
149
|
-
// and then waited out.
|
|
136
|
+
// status 0 (see call): a restart or blip the next poll clears, so it is reported once.
|
|
150
137
|
if (status === 0) {
|
|
151
138
|
if (serverReachable) {
|
|
152
139
|
serverReachable = false;
|
|
@@ -169,10 +156,8 @@ async function release(token) {
|
|
|
169
156
|
if (!ok)
|
|
170
157
|
console.error(`release failed: ${JSON.stringify(data)}`);
|
|
171
158
|
}
|
|
172
|
-
/**
|
|
173
|
-
*
|
|
174
|
-
* not fit `raises`), and guessing again would only pick a second wrong answer. Release it, so
|
|
175
|
-
* the task returns to the queue and an operator sees it waiting rather than silently gone. */
|
|
159
|
+
/** A refused outcome is not retried with another: the worker does not satisfy the declared
|
|
160
|
+
* contract. Released, so an operator sees the task waiting rather than gone. */
|
|
176
161
|
async function answer(token, outcome) {
|
|
177
162
|
const { ok, data } = await call("/api/external-tasks/resolve", { token, ...outcome });
|
|
178
163
|
if (ok)
|
|
@@ -180,7 +165,10 @@ async function answer(token, outcome) {
|
|
|
180
165
|
console.error(`genroc refused the outcome for ${token}: ${JSON.stringify(data)}`);
|
|
181
166
|
await release(token);
|
|
182
167
|
}
|
|
183
|
-
async function run(job) {
|
|
168
|
+
async function run(job, signal) {
|
|
169
|
+
// The task's own timeout is the script's budget, moved onto the monotonic clock so a wall-clock
|
|
170
|
+
// step mid-run cannot stretch it. Fetching the objects below spends the same deadline.
|
|
171
|
+
const due = job.deadline === undefined ? undefined : performance.now() + (Date.parse(job.deadline) - Date.now());
|
|
184
172
|
let resolved;
|
|
185
173
|
try {
|
|
186
174
|
resolved = await resolveObjects(job);
|
|
@@ -192,7 +180,16 @@ async function run(job) {
|
|
|
192
180
|
await release(job.token);
|
|
193
181
|
return;
|
|
194
182
|
}
|
|
195
|
-
|
|
183
|
+
let timeoutMs;
|
|
184
|
+
if (due !== undefined) {
|
|
185
|
+
timeoutMs = Math.floor(due - performance.now() - ANSWER_MARGIN_MS);
|
|
186
|
+
if (timeoutMs <= 0) {
|
|
187
|
+
// Too late to run anything: the deadline fires as external.timeout, which is the truth.
|
|
188
|
+
await release(job.token);
|
|
189
|
+
return;
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
const req = asEvalRequest(resolved, timeoutMs);
|
|
196
193
|
if (typeof req === "string") {
|
|
197
194
|
await answer(job.token, {
|
|
198
195
|
error: { code: "compile_error", message: req, data: { name: "BadTaskInput" } },
|
|
@@ -201,13 +198,18 @@ async function run(job) {
|
|
|
201
198
|
}
|
|
202
199
|
let result;
|
|
203
200
|
try {
|
|
204
|
-
result = await evaluate(req);
|
|
201
|
+
result = await evaluate(req, signal);
|
|
205
202
|
}
|
|
206
203
|
catch (err) {
|
|
207
|
-
//
|
|
208
|
-
//
|
|
209
|
-
|
|
210
|
-
|
|
204
|
+
// Cancelled is not a fault. Still released: the row is terminal so nothing re-claims it,
|
|
205
|
+
// but the release stops it waiting out a lease nobody serves.
|
|
206
|
+
if (err instanceof Cancelled) {
|
|
207
|
+
console.log(`cancelled: ${job.token}`);
|
|
208
|
+
await release(job.token);
|
|
209
|
+
return;
|
|
210
|
+
}
|
|
211
|
+
// The RUNNER faulted, the one retryable class. Releasing is how a queue spells "retryable",
|
|
212
|
+
// and it reaches another worker without spending the definition's on_error budget.
|
|
211
213
|
console.error(`evaluator fault on ${job.token}: ${err instanceof Error ? err.message : String(err)}`);
|
|
212
214
|
await release(job.token);
|
|
213
215
|
return;
|
|
@@ -240,11 +242,18 @@ async function renewLoop() {
|
|
|
240
242
|
tokens,
|
|
241
243
|
lease_ms: LEASE_MS,
|
|
242
244
|
});
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
//
|
|
246
|
-
|
|
247
|
-
|
|
245
|
+
if (!ok)
|
|
246
|
+
continue;
|
|
247
|
+
// Abort first rather than let a stopped script burn a core; the release rides run()'s
|
|
248
|
+
// catch, so nothing is released twice.
|
|
249
|
+
for (const token of (data?.cancelled ?? [])) {
|
|
250
|
+
inFlight.get(token)?.abort.abort();
|
|
251
|
+
}
|
|
252
|
+
// Lost: another worker holds it now and our answer will be refused; logged as the sign that
|
|
253
|
+
// LEASE_MS is too short. NOT released: that would bump the epoch out from under the holder.
|
|
254
|
+
const lost = (data?.lost ?? []);
|
|
255
|
+
if (lost.length) {
|
|
256
|
+
console.error(`lost ${lost.length}/${tokens.length} claims; a lease lapsed under load`);
|
|
248
257
|
}
|
|
249
258
|
}
|
|
250
259
|
}
|
|
@@ -253,8 +262,9 @@ async function pollLoop() {
|
|
|
253
262
|
const free = CONCURRENCY - inFlight.size;
|
|
254
263
|
const jobs = free > 0 ? await claim(free) : [];
|
|
255
264
|
for (const job of jobs) {
|
|
256
|
-
|
|
257
|
-
|
|
265
|
+
const abort = new AbortController();
|
|
266
|
+
inFlight.set(job.token, { job, abort });
|
|
267
|
+
void run(job, abort.signal).finally(() => inFlight.delete(job.token));
|
|
258
268
|
}
|
|
259
269
|
// Only idle when there was nothing to take: a full queue should be drained at the speed the
|
|
260
270
|
// realms allow, not at the poll interval.
|
package/eval.ts
CHANGED
|
@@ -1,22 +1,18 @@
|
|
|
1
|
-
//
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
// The containment is a Worker per execution (realm.ts). It is what makes the budget real:
|
|
6
|
-
// a synchronous busy loop never yields, so no in-process timer can interrupt it, and only a
|
|
7
|
-
// thread the host can kill bounds it.
|
|
1
|
+
// Imports a script module in its own realm (a Worker per execution, realm.ts), calls its default
|
|
2
|
+
// export, and classifies every outcome (README.md). Only a killable thread bounds a synchronous
|
|
3
|
+
// busy loop, which is what makes the budget real. Knows nothing of genroc; worker.ts does.
|
|
8
4
|
|
|
9
5
|
import { Worker } from "node:worker_threads";
|
|
10
6
|
|
|
11
7
|
export type EvalRequest = {
|
|
8
|
+
/** An ES module whose default export is the function to run; `input` is its argument. */
|
|
12
9
|
code: string;
|
|
13
10
|
input?: unknown;
|
|
14
11
|
timeout_ms?: number;
|
|
15
12
|
};
|
|
16
13
|
|
|
17
|
-
/** Every kind
|
|
18
|
-
*
|
|
19
|
-
* faults releases its claim and lets another worker take the task. */
|
|
14
|
+
/** Every kind is PERMANENT: a retry fails identically. A faulting runner is not an outcome; it
|
|
15
|
+
* releases its claim. */
|
|
20
16
|
export type FailureKind = "compile_error" | "threw" | "timeout" | "nonserializable" | "exited";
|
|
21
17
|
|
|
22
18
|
export type EvalFailure = {
|
|
@@ -26,9 +22,8 @@ export type EvalFailure = {
|
|
|
26
22
|
stack?: string;
|
|
27
23
|
};
|
|
28
24
|
|
|
29
|
-
/**
|
|
30
|
-
*
|
|
31
|
-
* crosses the worker boundary — structured clone would refuse a different set of values. */
|
|
25
|
+
/** JSON TEXT serialised in the realm, so a nonserializable return is a script fault, not a 500,
|
|
26
|
+
* and structured clone (which refuses a different set of values) never sees the value. */
|
|
32
27
|
export type EvalResult =
|
|
33
28
|
| { ok: true; body: string }
|
|
34
29
|
| { ok: false; failure: EvalFailure };
|
|
@@ -45,9 +40,8 @@ const REALM_URL = new URL(
|
|
|
45
40
|
import.meta.url,
|
|
46
41
|
);
|
|
47
42
|
|
|
48
|
-
/** Thrown, not returned: a realm that fails to start is the
|
|
49
|
-
* answers by releasing the claim
|
|
50
|
-
* return value. */
|
|
43
|
+
/** Thrown, not returned: a realm that fails to start is the runner faulting, which worker.ts
|
|
44
|
+
* answers by releasing the claim. */
|
|
51
45
|
class RealmFault extends Error {
|
|
52
46
|
constructor(message: string) {
|
|
53
47
|
super(message);
|
|
@@ -55,24 +49,41 @@ class RealmFault extends Error {
|
|
|
55
49
|
}
|
|
56
50
|
}
|
|
57
51
|
|
|
58
|
-
|
|
52
|
+
/** Thrown when `signal` aborts: the answer is no longer wanted, so worker.ts releases rather
|
|
53
|
+
* than answers. Not a RealmFault: nothing is wrong. */
|
|
54
|
+
export class Cancelled extends Error {
|
|
55
|
+
constructor() {
|
|
56
|
+
super("cancelled");
|
|
57
|
+
this.name = "Cancelled";
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export async function evaluate(req: EvalRequest, signal?: AbortSignal): Promise<EvalResult> {
|
|
59
62
|
const budget = typeof req.timeout_ms === "number" ? req.timeout_ms : DEFAULT_TIMEOUT_MS;
|
|
60
63
|
|
|
61
64
|
const worker = new Worker(REALM_URL);
|
|
62
65
|
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
66
|
+
let onAbort: (() => void) | undefined;
|
|
63
67
|
try {
|
|
64
68
|
return await new Promise<EvalResult>((resolve, reject) => {
|
|
65
69
|
timer = setTimeout(() => resolve(timedOut(budget)), budget);
|
|
70
|
+
// The abort is wired to the same promise as the budget, so both settle through the one
|
|
71
|
+
// finally below -- which is what guarantees the thread is gone before either returns.
|
|
72
|
+
if (signal) {
|
|
73
|
+
if (signal.aborted) reject(new Cancelled());
|
|
74
|
+
onAbort = () => reject(new Cancelled());
|
|
75
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
76
|
+
}
|
|
66
77
|
worker.once("message", (reply: WorkerReply) => resolve(reply));
|
|
67
|
-
// A script may
|
|
68
|
-
//
|
|
69
|
-
// this too, by which time the promise has settled and the first result stands.
|
|
78
|
+
// A script may `process.exit()`, which would otherwise look like a hang until the budget.
|
|
79
|
+
// Our own terminate() fires this too, after the promise has settled.
|
|
70
80
|
worker.once("exit", (code: number) => resolve(exited(code)));
|
|
71
81
|
worker.once("error", (err: Error) => reject(new RealmFault(errorText(err))));
|
|
72
82
|
worker.postMessage({ code: req.code, input: req.input } satisfies WorkerRequest);
|
|
73
83
|
});
|
|
74
84
|
} finally {
|
|
75
85
|
clearTimeout(timer);
|
|
86
|
+
if (signal && onAbort) signal.removeEventListener("abort", onAbort);
|
|
76
87
|
// Awaited, and the whole point: on the timeout path a thread is still burning a core, and
|
|
77
88
|
// resolving before it is gone would report an evaluation the machine is still running.
|
|
78
89
|
await worker.terminate();
|