@genroc/eval-node 0.0.0-edge.b8f0518 → 0.0.0-edge.bd59fe5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +58 -34
- package/dist/eval.js +25 -4
- package/dist/import.js +104 -59
- package/dist/realm.js +66 -75
- package/dist/worker.js +36 -11
- package/eval.ts +25 -4
- package/import.ts +189 -80
- package/package.json +1 -1
- package/realm.ts +61 -77
- package/worker.ts +43 -12
package/dist/import.js
CHANGED
|
@@ -7,9 +7,10 @@
|
|
|
7
7
|
// code; "build" typechecks and bundles. A separate types hook would mean a second `tsc`
|
|
8
8
|
// over the same project.
|
|
9
9
|
import { spawn } from "node:child_process";
|
|
10
|
+
import { existsSync } from "node:fs";
|
|
10
11
|
import { access, mkdir, writeFile } from "node:fs/promises";
|
|
11
12
|
import { builtinModules } from "node:module";
|
|
12
|
-
import { dirname, join, relative } from "node:path";
|
|
13
|
+
import { dirname, join, relative, resolve } from "node:path";
|
|
13
14
|
import { fileURLToPath } from "node:url";
|
|
14
15
|
import commonjs from "@rollup/plugin-commonjs";
|
|
15
16
|
import json from "@rollup/plugin-json";
|
|
@@ -86,7 +87,9 @@ function objectType(s, used) {
|
|
|
86
87
|
const required = new Set(s.required ?? []);
|
|
87
88
|
const lines = [];
|
|
88
89
|
for (const [key, sub] of Object.entries(props)) {
|
|
89
|
-
const doc = typeof sub.description === "string"
|
|
90
|
+
const doc = typeof sub.description === "string"
|
|
91
|
+
? ` /** ${sub.description} */\n`
|
|
92
|
+
: "";
|
|
90
93
|
lines.push(`${doc} ${propKey(key)}${required.has(key) ? "" : "?"}: ${tsType(sub, used)};`);
|
|
91
94
|
}
|
|
92
95
|
if (s.additionalProperties && typeof s.additionalProperties === "object") {
|
|
@@ -102,7 +105,7 @@ function union(parts) {
|
|
|
102
105
|
}
|
|
103
106
|
const IDENT = /^[A-Za-z_$][A-Za-z0-9_$]*$/;
|
|
104
107
|
const propKey = (k) => (IDENT.test(k) ? k : JSON.stringify(k));
|
|
105
|
-
const identifier = (n) =>
|
|
108
|
+
const identifier = (n) => IDENT.test(n) ? n : `Def_${n.replace(/[^A-Za-z0-9_$]/g, "_")}`;
|
|
106
109
|
function deref(s, defs) {
|
|
107
110
|
let cur = s;
|
|
108
111
|
for (let i = 0; cur && typeof cur.$ref === "string" && i < 16; i++) {
|
|
@@ -110,21 +113,14 @@ function deref(s, defs) {
|
|
|
110
113
|
}
|
|
111
114
|
return cur;
|
|
112
115
|
}
|
|
113
|
-
/** The manifest's `input` is the type of the whole ACTION input — for /eval that is
|
|
114
|
-
* `{code, input, timeout_ms, …}`, and only its `input` field is bound as the script's
|
|
115
|
-
* parameter. genroc cannot know that; this file owns the evaluator's wire contract, so
|
|
116
|
-
* the navigation belongs here rather than in genctl. */
|
|
117
|
-
function scriptInput(site, defs) {
|
|
118
|
-
const action = deref(site.input, defs);
|
|
119
|
-
const bound = action?.properties?.input;
|
|
120
|
-
return bound ?? site.input;
|
|
121
|
-
}
|
|
122
116
|
/** Emits one named type per reachable $def rather than inlining: a task output may
|
|
123
117
|
* reference itself (specs/recursive-type-inference.md) and inlining would not terminate. */
|
|
124
|
-
function declarations(
|
|
118
|
+
function declarations(at) {
|
|
119
|
+
const { site, where } = at;
|
|
120
|
+
const defs = where.$defs ?? {};
|
|
125
121
|
const used = new Set();
|
|
126
|
-
const input = tsType(
|
|
127
|
-
const output = tsType(site.
|
|
122
|
+
const input = tsType(site.types?.Input, used);
|
|
123
|
+
const output = tsType(site.types?.Output, used);
|
|
128
124
|
const emitted = [];
|
|
129
125
|
const done = new Set();
|
|
130
126
|
while (true) {
|
|
@@ -137,7 +133,7 @@ function declarations(site, defs) {
|
|
|
137
133
|
}
|
|
138
134
|
return [
|
|
139
135
|
"// Generated by genroc. Do not edit - regenerate with `genctl types`.",
|
|
140
|
-
`// ${
|
|
136
|
+
`// ${where.name} (${address(site.pointer)})`,
|
|
141
137
|
"",
|
|
142
138
|
...emitted,
|
|
143
139
|
emitted.length ? "" : "",
|
|
@@ -165,7 +161,7 @@ async function nearestTsconfig(from, root) {
|
|
|
165
161
|
return null;
|
|
166
162
|
}
|
|
167
163
|
}
|
|
168
|
-
async function typecheck(
|
|
164
|
+
async function typecheck(sites) {
|
|
169
165
|
// NOT `.genroc`: that is the project config FILE, and a directory of the same name cannot
|
|
170
166
|
// coexist with it. The suffix is what keeps the scratch area out of its way.
|
|
171
167
|
const dir = join(root, ".genroc-cache");
|
|
@@ -173,13 +169,13 @@ async function typecheck(root, sites) {
|
|
|
173
169
|
// One tsc per distinct base config: `extends` takes a single base, so merging two would
|
|
174
170
|
// check each script under the other author's options.
|
|
175
171
|
const groups = new Map();
|
|
176
|
-
for (const
|
|
177
|
-
const base = (await nearestTsconfig(dirname(
|
|
172
|
+
for (const at of sites) {
|
|
173
|
+
const base = (await nearestTsconfig(dirname(at.file), root)) ?? "";
|
|
178
174
|
const group = groups.get(base);
|
|
179
175
|
if (group)
|
|
180
|
-
group.push(
|
|
176
|
+
group.push(at);
|
|
181
177
|
else
|
|
182
|
-
groups.set(base, [
|
|
178
|
+
groups.set(base, [at]);
|
|
183
179
|
}
|
|
184
180
|
let n = 0;
|
|
185
181
|
for (const [base, group] of groups) {
|
|
@@ -200,7 +196,10 @@ async function typecheck(root, sites) {
|
|
|
200
196
|
// base config there is nothing to opt in with, so the default stays none.
|
|
201
197
|
...(base ? {} : { types: [] }),
|
|
202
198
|
},
|
|
203
|
-
files: group.flatMap((s) => [
|
|
199
|
+
files: group.flatMap((s) => [
|
|
200
|
+
relative(dir, s.file),
|
|
201
|
+
relative(dir, typesPathFor(s.file)),
|
|
202
|
+
]),
|
|
204
203
|
// `files` overrides the base's, but a base `include` survives beside it and would
|
|
205
204
|
// drag the author's whole tree in, to be checked under the worker lib.
|
|
206
205
|
include: [],
|
|
@@ -250,7 +249,10 @@ const transpile = {
|
|
|
250
249
|
return { code: out.outputText, map: out.sourceMapText ?? null };
|
|
251
250
|
},
|
|
252
251
|
};
|
|
253
|
-
const BUILTIN = new Set([
|
|
252
|
+
const BUILTIN = new Set([
|
|
253
|
+
...builtinModules,
|
|
254
|
+
...builtinModules.map((m) => `node:${m}`),
|
|
255
|
+
]);
|
|
254
256
|
/** Resolves imports through TYPESCRIPT, using the same config the typecheck ran under, so a
|
|
255
257
|
* `paths` alias that compiles also bundles. Reimplementing `paths` here would be a second
|
|
256
258
|
* resolver to keep in agreement with tsc; this one cannot disagree.
|
|
@@ -270,22 +272,25 @@ function tsResolve(configPath) {
|
|
|
270
272
|
const { resolvedModule } = ts.resolveModuleName(source, importer, options, ts.sys);
|
|
271
273
|
if (!resolvedModule || resolvedModule.isExternalLibraryImport)
|
|
272
274
|
return null;
|
|
273
|
-
return resolvedModule.resolvedFileName.endsWith(".d.ts")
|
|
275
|
+
return resolvedModule.resolvedFileName.endsWith(".d.ts")
|
|
276
|
+
? null
|
|
277
|
+
: resolvedModule.resolvedFileName;
|
|
274
278
|
},
|
|
275
279
|
};
|
|
276
280
|
}
|
|
277
|
-
/** Bundles to
|
|
278
|
-
*
|
|
279
|
-
*
|
|
280
|
-
async function bundle(
|
|
281
|
-
|
|
282
|
-
//
|
|
281
|
+
/** Bundles to a self-contained ES module, which is what the evaluator imports: the default
|
|
282
|
+
* export it calls is the author's own, so nothing wraps or rewrites the code between the two.
|
|
283
|
+
* Bundling is entirely the importer's job, so a definition version pins its code forever. */
|
|
284
|
+
async function bundle(at) {
|
|
285
|
+
const site = at.site;
|
|
286
|
+
// Builtins are EXTERNALISED as imports the realm resolves natively. Anything else
|
|
287
|
+
// unresolved is a REFUSAL, not an external: rollup's default is to leave it as an import
|
|
283
288
|
// of a module that will not be there, which bundles clean and fails at runtime.
|
|
284
289
|
const built = await rollup({
|
|
285
|
-
input:
|
|
290
|
+
input: at.file,
|
|
286
291
|
external: (id) => BUILTIN.has(id),
|
|
287
292
|
plugins: [
|
|
288
|
-
tsResolve(await nearestTsconfig(dirname(
|
|
293
|
+
tsResolve(await nearestTsconfig(dirname(at.file), root)),
|
|
289
294
|
nodeResolve({ extensions: [".ts", ".tsx", ".mjs", ".js", ".json"] }),
|
|
290
295
|
commonjs(),
|
|
291
296
|
// A `.json` import is a data file inlined at build time, which the previous bundler
|
|
@@ -295,22 +300,21 @@ async function bundle(site, root) {
|
|
|
295
300
|
],
|
|
296
301
|
onwarn(warning) {
|
|
297
302
|
if (warning.code === "UNRESOLVED_IMPORT") {
|
|
298
|
-
die(`${
|
|
303
|
+
die(`${at.file}: cannot resolve ${warning.exporter ?? "an import"} — is it installed?`);
|
|
299
304
|
}
|
|
300
305
|
},
|
|
301
|
-
}).catch((e) => die(`${
|
|
302
|
-
const { output } = await built.generate({
|
|
306
|
+
}).catch((e) => die(`${at.file}: ${e instanceof Error ? e.message : String(e)}`));
|
|
307
|
+
const { output } = await built.generate({
|
|
308
|
+
format: "es",
|
|
309
|
+
inlineDynamicImports: true,
|
|
310
|
+
});
|
|
303
311
|
await built.close();
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
` throw new Error(${JSON.stringify(`${site.path} has no default export function`)});`,
|
|
311
|
-
"}",
|
|
312
|
-
"return await __genroc_main(input);",
|
|
313
|
-
].join("\n");
|
|
312
|
+
// Refused here rather than in the realm: the evaluator can only report it against a running
|
|
313
|
+
// instance, and the file it names is on this machine.
|
|
314
|
+
if (!output[0].exports.includes("default")) {
|
|
315
|
+
die(`${at.file}: a script must \`export default\` the function to run`);
|
|
316
|
+
}
|
|
317
|
+
return output[0].code;
|
|
314
318
|
}
|
|
315
319
|
// ── main ───────────────────────────────────────────────────────────────────────
|
|
316
320
|
const stdin = await new Promise((resolve, reject) => {
|
|
@@ -321,32 +325,73 @@ const stdin = await new Promise((resolve, reject) => {
|
|
|
321
325
|
process.stdin.on("error", reject);
|
|
322
326
|
});
|
|
323
327
|
const manifest = JSON.parse(stdin);
|
|
324
|
-
|
|
328
|
+
// genctl runs a resolver with the project root as its cwd, so the manifest need not say so.
|
|
329
|
+
const root = process.cwd();
|
|
330
|
+
if (!manifest || !Array.isArray(manifest.processes))
|
|
325
331
|
die("stdin is not a genroc resolver manifest");
|
|
332
|
+
/** A pointer as the address it is, so a generated comment or an error can be pasted into
|
|
333
|
+
* `genctl schema`. A key no identifier can spell is bracketed, as the grammar spells it. */
|
|
334
|
+
function address(pointer) {
|
|
335
|
+
return pointer
|
|
336
|
+
.map((seg) => typeof seg === "string" && /^[A-Za-z_][A-Za-z0-9_]*$/.test(seg)
|
|
337
|
+
? `.${seg}`
|
|
338
|
+
: `[${JSON.stringify(seg)}]`)
|
|
339
|
+
.join("")
|
|
340
|
+
.replace(/^\./, "");
|
|
341
|
+
}
|
|
342
|
+
const located = manifest.processes.flatMap((where) => where.sites.map((site) => ({
|
|
343
|
+
site,
|
|
344
|
+
where,
|
|
345
|
+
file: resolve(where.dir, site.argument),
|
|
346
|
+
})));
|
|
347
|
+
// genctl is agnostic about what a script is for; the contract that an evaluation request carries
|
|
348
|
+
// its module in `code` is THIS resolver's, so it is the one that checks a directive landed there.
|
|
349
|
+
// `child` is the shape the scaffold uses — a call to a process that forwards to the evaluator —
|
|
350
|
+
// and `external` is the same request made directly.
|
|
351
|
+
for (const at of located) {
|
|
352
|
+
// Two halves, from the two places that carry them: the action's KIND is a field, and WHERE in
|
|
353
|
+
// it the directive sits is the pointer. A child call is the shape the scaffold generates — to a
|
|
354
|
+
// process that forwards to the evaluator — and an external task is the same request made
|
|
355
|
+
// directly.
|
|
356
|
+
const kind = at.site.level === "action" ? (at.site.action ?? "") : "";
|
|
357
|
+
const slot = at.site.pointer.slice(-2).join(".");
|
|
358
|
+
if ((kind !== "child" && kind !== "external") || slot !== "input.code") {
|
|
359
|
+
die(`${join(at.where.dir, at.where.file)}: ${address(at.site.pointer)}: an evaluated script ` +
|
|
360
|
+
"belongs in the `code` field of a child or external task's input, and this is " +
|
|
361
|
+
`${kind ? `a ${kind} task's ` : ""}\`${slot}\`.`);
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
// genctl no longer stats the argument — it does not know it is a file — so the resolver that
|
|
365
|
+
// does is the one that must say when it is not there.
|
|
366
|
+
for (const at of located) {
|
|
367
|
+
if (!existsSync(at.file)) {
|
|
368
|
+
die(`${join(at.where.dir, at.where.file)}: ${address(at.site.pointer)}: ` +
|
|
369
|
+
`"$import: ${at.site.argument}" names no file (looked at ${at.file})`);
|
|
370
|
+
}
|
|
371
|
+
}
|
|
326
372
|
// One script at two sites with different input types is a refusal, not a union: the union
|
|
327
373
|
// is sound and would typecheck a body that is wrong at one of the sites.
|
|
328
374
|
const byPath = new Map();
|
|
329
|
-
for (const
|
|
330
|
-
const seen = byPath.get(
|
|
331
|
-
const defsOf = (x) => (manifest.schemas[x.process]?.$defs ?? {});
|
|
375
|
+
for (const at of located) {
|
|
376
|
+
const seen = byPath.get(at.file);
|
|
332
377
|
if (seen &&
|
|
333
|
-
JSON.stringify(
|
|
334
|
-
|
|
335
|
-
|
|
378
|
+
JSON.stringify(seen.site.types) !== JSON.stringify(at.site.types)) {
|
|
379
|
+
die(`${at.file} is imported at ${address(seen.site.pointer)} and ` +
|
|
380
|
+
`${address(at.site.pointer)} with different input types.\n` +
|
|
336
381
|
"Split it into two scripts, or make the two call sites pass the same shape.");
|
|
337
382
|
}
|
|
338
|
-
byPath.set(
|
|
383
|
+
byPath.set(at.file, at);
|
|
339
384
|
}
|
|
340
|
-
for (const
|
|
341
|
-
|
|
342
|
-
await write(typesPathFor(site.path), declarations(site, defs));
|
|
385
|
+
for (const at of byPath.values()) {
|
|
386
|
+
await write(typesPathFor(at.file), declarations(at));
|
|
343
387
|
}
|
|
344
388
|
if (manifest.mode === "types") {
|
|
389
|
+
console.log(JSON.stringify(manifest, null, 2));
|
|
345
390
|
process.exit(0);
|
|
346
391
|
}
|
|
347
|
-
await typecheck(
|
|
392
|
+
await typecheck([...byPath.values()]);
|
|
348
393
|
const code = [];
|
|
349
|
-
for (const
|
|
350
|
-
code.push(await bundle(
|
|
394
|
+
for (const at of located) {
|
|
395
|
+
code.push(await bundle(at));
|
|
351
396
|
}
|
|
352
397
|
process.stdout.write(JSON.stringify({ code }));
|
package/dist/realm.js
CHANGED
|
@@ -2,80 +2,46 @@
|
|
|
2
2
|
// thread the host can kill mid-loop — the only thing that bounds a synchronous busy loop.
|
|
3
3
|
// eval.ts owns the budget and does the killing; nothing here knows about time.
|
|
4
4
|
//
|
|
5
|
-
// Everything that touches the script's VALUE lives on this side of the boundary —
|
|
5
|
+
// Everything that touches the script's VALUE lives on this side of the boundary — loading,
|
|
6
6
|
// classifying, serialising — because this is the only realm the value exists in.
|
|
7
|
-
|
|
7
|
+
var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExtension) || function (path, preserveJsx) {
|
|
8
|
+
if (typeof path === "string" && /^\.\.?\//.test(path)) {
|
|
9
|
+
return path.replace(/\.(tsx)$|((?:\.d)?)((?:\.[^./]+?)?)\.([cm]?)ts$/i, function (m, tsx, d, ext, cm) {
|
|
10
|
+
return tsx ? preserveJsx ? ".jsx" : ".js" : d && (!ext || !cm) ? m : (d + ext + "." + cm.toLowerCase() + "js");
|
|
11
|
+
});
|
|
12
|
+
}
|
|
13
|
+
return path;
|
|
14
|
+
};
|
|
15
|
+
import { registerHooks } from "node:module";
|
|
8
16
|
import { parentPort } from "node:worker_threads";
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
//
|
|
12
|
-
|
|
13
|
-
// `require` in scope. Passing one in is what makes an import of a builtin work at runtime rather than at
|
|
14
|
-
// typecheck only. Resolution is anchored here, which is right: only builtins reach it.
|
|
15
|
-
const scriptRequire = createRequire(import.meta.url);
|
|
17
|
+
// The script is IMPORTED as a module under a URL of our own, not compiled from a string: its
|
|
18
|
+
// frames then carry the author's own line numbers, and an `import` of a node builtin resolves
|
|
19
|
+
// the way it does everywhere else.
|
|
20
|
+
const SCRIPT_URL = "script:main";
|
|
16
21
|
const STACK_BYTES = 2_048;
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
return 0;
|
|
28
|
-
}
|
|
29
|
-
catch (err) {
|
|
30
|
-
return reportedLine(err) - 1;
|
|
31
|
-
}
|
|
32
|
-
})();
|
|
33
|
-
// V8 marks a frame compiled by the AsyncFunction constructor with the site that CALLED the
|
|
34
|
-
// constructor, then the script's OWN position:
|
|
35
|
-
// at inner (eval at run (file:///…/worker.ts:107:10), <anonymous>:6:9)
|
|
36
|
-
// `eval at` is therefore what separates script frames from runner plumbing — matched without
|
|
37
|
-
// the function name, which is whatever encloses the `new AsyncFunction` below. The LAST such
|
|
38
|
-
// frame is the body's top level: frames interleave, since a script can throw inside a native
|
|
39
|
-
// callback.
|
|
40
|
-
const SCRIPT_FRAME = /\(eval at /;
|
|
41
|
-
// The trailing `<anonymous>:LINE:COL` — the script's position, after the host file's own.
|
|
42
|
-
const POSITION = /<anonymous>:(\d+):(\d+)\)?\s*$/;
|
|
43
|
-
// ` at name (` — absent on the top-level frame, which V8 names `eval`.
|
|
44
|
-
const FRAME_NAME = /^\s*at\s+(?:async\s+)?([^\s(]+)\s*\(/;
|
|
45
|
-
/** Line number of the throw as the engine reported it, or 1 if the stack is unreadable. */
|
|
46
|
-
function reportedLine(err) {
|
|
47
|
-
const stack = err instanceof Error && typeof err.stack === "string" ? err.stack : "";
|
|
48
|
-
const frame = stack.split("\n").find((l) => SCRIPT_FRAME.test(l)) ?? "";
|
|
49
|
-
const m = frame.match(POSITION);
|
|
50
|
-
return m ? Number(m[1]) : 1;
|
|
51
|
-
}
|
|
52
|
-
/** Renumbers each script frame to the line the AUTHOR wrote and drops the runner's own.
|
|
53
|
-
* Rewriting the whole location is also what keeps the runner's path out of a script's
|
|
54
|
-
* stack — V8 puts it inside every compiled frame. */
|
|
55
|
-
function scriptStack(err, offset) {
|
|
22
|
+
// The only channel a load hook has to the source. Written per request and read once — a realm
|
|
23
|
+
// evaluates one script and is then discarded, so no second execution can observe it.
|
|
24
|
+
let source = "";
|
|
25
|
+
registerHooks({
|
|
26
|
+
resolve: (specifier, context, next) => specifier === SCRIPT_URL ? { url: SCRIPT_URL, shortCircuit: true } : next(specifier, context),
|
|
27
|
+
load: (url, context, next) => url === SCRIPT_URL ? { format: "module", source, shortCircuit: true } : next(url, context),
|
|
28
|
+
});
|
|
29
|
+
/** Keeps the frames that are the script's own. Everything below the last of them is runner
|
|
30
|
+
* plumbing the author cannot act on, and V8 puts this file's path in it. */
|
|
31
|
+
function scriptStack(err) {
|
|
56
32
|
if (!(err instanceof Error) || typeof err.stack !== "string")
|
|
57
33
|
return undefined;
|
|
58
34
|
const lines = err.stack.split("\n");
|
|
59
|
-
let
|
|
35
|
+
let last = 0;
|
|
60
36
|
for (let i = 0; i < lines.length; i++)
|
|
61
|
-
if (
|
|
62
|
-
|
|
63
|
-
const
|
|
64
|
-
|
|
65
|
-
const pos = line.match(POSITION);
|
|
66
|
-
if (!pos)
|
|
67
|
-
return line; // the `Error: message` header and native frames, kept as-is
|
|
68
|
-
const name = line.match(FRAME_NAME)?.[1];
|
|
69
|
-
const at = name && name !== "eval" && name !== "anonymous" ? `at ${name} ` : "at ";
|
|
70
|
-
const indent = line.match(/^\s*/)[0];
|
|
71
|
-
return `${indent}${at}(script:${Math.max(1, Number(pos[1]) - offset)}:${pos[2]})`;
|
|
72
|
-
})
|
|
73
|
-
.join("\n");
|
|
74
|
-
return frames.length > STACK_BYTES ? frames.slice(0, STACK_BYTES) : frames;
|
|
37
|
+
if (lines[i].includes(SCRIPT_URL))
|
|
38
|
+
last = i;
|
|
39
|
+
const stack = lines.slice(0, last + 1).join("\n");
|
|
40
|
+
return stack.length > STACK_BYTES ? stack.slice(0, STACK_BYTES) : stack;
|
|
75
41
|
}
|
|
76
|
-
function describe(err, kind
|
|
42
|
+
function describe(err, kind) {
|
|
77
43
|
if (err instanceof Error) {
|
|
78
|
-
return { kind, name: err.name, message: err.message, stack: scriptStack(err
|
|
44
|
+
return { kind, name: err.name, message: err.message, stack: scriptStack(err) };
|
|
79
45
|
}
|
|
80
46
|
// A script may throw a non-Error (`throw {code: "x"}`), so name/message must not assume one.
|
|
81
47
|
return { kind, name: "Thrown", message: safeText(err) };
|
|
@@ -88,23 +54,39 @@ function safeText(v) {
|
|
|
88
54
|
return String(v);
|
|
89
55
|
}
|
|
90
56
|
}
|
|
57
|
+
/** A module that will not parse, or names an import nothing resolves, is broken code — only
|
|
58
|
+
* editing it helps. Anything else thrown by the import is the module's top level running,
|
|
59
|
+
* which is the script throwing. */
|
|
60
|
+
function unloadable(err) {
|
|
61
|
+
const code = err?.code;
|
|
62
|
+
return err instanceof SyntaxError || (typeof code === "string" && code.startsWith("ERR_MODULE"));
|
|
63
|
+
}
|
|
91
64
|
async function run(req) {
|
|
92
|
-
|
|
93
|
-
let
|
|
65
|
+
source = req.code;
|
|
66
|
+
let main;
|
|
94
67
|
try {
|
|
95
|
-
|
|
96
|
-
// never be hit. Repeated compilation is the price of the fresh global object.
|
|
97
|
-
fn = new AsyncFunction("input", "require", STRICT + req.code);
|
|
68
|
+
main = (await import(__rewriteRelativeImportExtension(SCRIPT_URL))).default;
|
|
98
69
|
}
|
|
99
70
|
catch (err) {
|
|
100
|
-
return { ok: false, failure: describe(err, "compile_error"
|
|
71
|
+
return { ok: false, failure: describe(err, unloadable(err) ? "compile_error" : "threw") };
|
|
72
|
+
}
|
|
73
|
+
if (typeof main !== "function") {
|
|
74
|
+
const got = main === undefined ? "no default export" : `a ${typeof main}`;
|
|
75
|
+
return {
|
|
76
|
+
ok: false,
|
|
77
|
+
failure: {
|
|
78
|
+
kind: "compile_error",
|
|
79
|
+
name: "NoDefaultExport",
|
|
80
|
+
message: `a script must export default a function; this one has ${got}`,
|
|
81
|
+
},
|
|
82
|
+
};
|
|
101
83
|
}
|
|
102
84
|
let value;
|
|
103
85
|
try {
|
|
104
|
-
value = await
|
|
86
|
+
value = await main(req.input);
|
|
105
87
|
}
|
|
106
88
|
catch (err) {
|
|
107
|
-
return { ok: false, failure: describe(err, "threw"
|
|
89
|
+
return { ok: false, failure: describe(err, "threw") };
|
|
108
90
|
}
|
|
109
91
|
try {
|
|
110
92
|
// undefined stringifies to undefined, not "undefined"; an empty body is how genroc
|
|
@@ -112,11 +94,20 @@ async function run(req) {
|
|
|
112
94
|
return { ok: true, body: value === undefined ? "" : JSON.stringify(value) ?? "" };
|
|
113
95
|
}
|
|
114
96
|
catch (err) {
|
|
115
|
-
return { ok: false, failure: describe(err, "nonserializable"
|
|
97
|
+
return { ok: false, failure: describe(err, "nonserializable") };
|
|
116
98
|
}
|
|
117
99
|
}
|
|
100
|
+
// The realm's stdio is a pipe to the host thread, and eval.ts terminates this thread the moment
|
|
101
|
+
// the reply lands — so whatever a script wrote last is still in the pipe when it dies. An empty
|
|
102
|
+
// write's callback fires once the queue ahead of it has drained, which makes the reply a barrier
|
|
103
|
+
// for `console` and a direct process.stdout.write alike: both are this stream.
|
|
104
|
+
function flush(stream) {
|
|
105
|
+
return new Promise((resolve) => stream.write("", () => resolve()));
|
|
106
|
+
}
|
|
118
107
|
// Non-null because this module only ever runs as a worker entry point; a null port here
|
|
119
108
|
// would mean eval.ts loaded it as a plain module, which nothing does.
|
|
120
109
|
parentPort.on("message", async (req) => {
|
|
121
|
-
|
|
110
|
+
const reply = await run(req);
|
|
111
|
+
await Promise.all([flush(process.stdout), flush(process.stderr)]);
|
|
112
|
+
parentPort.postMessage(reply);
|
|
122
113
|
});
|
package/dist/worker.js
CHANGED
|
@@ -4,7 +4,8 @@
|
|
|
4
4
|
// nothing about the queue, which is what keeps the containment strategy swappable.
|
|
5
5
|
//
|
|
6
6
|
// See README.md for the contract, and specs/external-task-queue.md for the queue itself.
|
|
7
|
-
import {
|
|
7
|
+
import { readFileSync } from "node:fs";
|
|
8
|
+
import { Cancelled, evaluate } from "./eval.js";
|
|
8
9
|
const SERVER = (process.env.GENROC_SERVER ?? "http://localhost:8448").replace(/\/$/, "");
|
|
9
10
|
const WORKER_ID = process.env.WORKER_ID ?? `evaluator-${process.pid}`;
|
|
10
11
|
// The credential, when the server runs with --auth token. A worker needs exactly the `worker`
|
|
@@ -15,7 +16,11 @@ const WORKER_ID = process.env.WORKER_ID ?? `evaluator-${process.pid}`;
|
|
|
15
16
|
// Sent as a header rather than in the URL because Node's fetch REFUSES a URL carrying
|
|
16
17
|
// credentials ("Request cannot be constructed from a URL that includes credentials"), so the
|
|
17
18
|
// basic-auth-in-the-URL trick that works for genctl is not available here.
|
|
18
|
-
|
|
19
|
+
// GENROC_TOKEN_FILE is the mounted-secret shape: a credential in a file rather than an
|
|
20
|
+
// environment variable, so it stays out of `docker inspect` and out of the process environment
|
|
21
|
+
// any child inherits. The inline variable wins when both are set.
|
|
22
|
+
const TOKEN = process.env.GENROC_TOKEN ??
|
|
23
|
+
(process.env.GENROC_TOKEN_FILE ? readFileSync(process.env.GENROC_TOKEN_FILE, "utf8").trim() : "");
|
|
19
24
|
const authHeaders = TOKEN ? { authorization: `Bearer ${TOKEN}` } : {};
|
|
20
25
|
// Concurrency is the worker's to set, and that is the point of pulling: under the old fetch
|
|
21
26
|
// shape genroc decided how many scripts ran at once (--max-concurrent, default 200) and the
|
|
@@ -180,7 +185,7 @@ async function answer(token, outcome) {
|
|
|
180
185
|
console.error(`genroc refused the outcome for ${token}: ${JSON.stringify(data)}`);
|
|
181
186
|
await release(token);
|
|
182
187
|
}
|
|
183
|
-
async function run(job) {
|
|
188
|
+
async function run(job, signal) {
|
|
184
189
|
let resolved;
|
|
185
190
|
try {
|
|
186
191
|
resolved = await resolveObjects(job);
|
|
@@ -201,9 +206,17 @@ async function run(job) {
|
|
|
201
206
|
}
|
|
202
207
|
let result;
|
|
203
208
|
try {
|
|
204
|
-
result = await evaluate(req);
|
|
209
|
+
result = await evaluate(req, signal);
|
|
205
210
|
}
|
|
206
211
|
catch (err) {
|
|
212
|
+
// Cancelled is not a fault: the process was stopped while this ran. The claim still goes
|
|
213
|
+
// back — the row is terminal, so nothing re-claims it, and the release is what stops it
|
|
214
|
+
// waiting out a lease nobody is serving.
|
|
215
|
+
if (err instanceof Cancelled) {
|
|
216
|
+
console.log(`cancelled: ${job.token}`);
|
|
217
|
+
await release(job.token);
|
|
218
|
+
return;
|
|
219
|
+
}
|
|
207
220
|
// The RUNNER faulted, not the script — the one class where a retry can help. There is no
|
|
208
221
|
// error code for it on purpose: releasing the claim is how a queue spells "retryable", and
|
|
209
222
|
// it puts the task in front of a different worker instead of burning the definition's
|
|
@@ -240,11 +253,22 @@ async function renewLoop() {
|
|
|
240
253
|
tokens,
|
|
241
254
|
lease_ms: LEASE_MS,
|
|
242
255
|
});
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
//
|
|
246
|
-
|
|
247
|
-
|
|
256
|
+
if (!ok)
|
|
257
|
+
continue;
|
|
258
|
+
// Cancelled: the process was stopped, so the evaluation is abandoned and the claim handed
|
|
259
|
+
// back. Aborting first is the point of the list — waiting for the script to finish would
|
|
260
|
+
// keep burning a core on work an operator has already stopped. The release rides run()'s
|
|
261
|
+
// own catch, so nothing is released twice.
|
|
262
|
+
for (const token of (data?.cancelled ?? [])) {
|
|
263
|
+
inFlight.get(token)?.abort.abort();
|
|
264
|
+
}
|
|
265
|
+
// Lost: someone else holds this claim now. The work continues and its answer will be
|
|
266
|
+
// refused — that is not fixable here — but say so, because it is the signal that LEASE_MS
|
|
267
|
+
// is too short for what these scripts actually take. Deliberately NOT released: the claim
|
|
268
|
+
// is the new holder's, and releasing would bump the epoch out from under it.
|
|
269
|
+
const lost = (data?.lost ?? []);
|
|
270
|
+
if (lost.length) {
|
|
271
|
+
console.error(`lost ${lost.length}/${tokens.length} claims; a lease lapsed under load`);
|
|
248
272
|
}
|
|
249
273
|
}
|
|
250
274
|
}
|
|
@@ -253,8 +277,9 @@ async function pollLoop() {
|
|
|
253
277
|
const free = CONCURRENCY - inFlight.size;
|
|
254
278
|
const jobs = free > 0 ? await claim(free) : [];
|
|
255
279
|
for (const job of jobs) {
|
|
256
|
-
|
|
257
|
-
|
|
280
|
+
const abort = new AbortController();
|
|
281
|
+
inFlight.set(job.token, { job, abort });
|
|
282
|
+
void run(job, abort.signal).finally(() => inFlight.delete(job.token));
|
|
258
283
|
}
|
|
259
284
|
// Only idle when there was nothing to take: a full queue should be drained at the speed the
|
|
260
285
|
// realms allow, not at the poll interval.
|
package/eval.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
// Evaluation core:
|
|
2
|
-
// the failure kinds in README.md. Nothing here knows about
|
|
3
|
-
// thing that talks to the queue — so this stays testable and
|
|
1
|
+
// Evaluation core: import a script module in its OWN realm, call its default export, and
|
|
2
|
+
// classify every outcome into one of the failure kinds in README.md. Nothing here knows about
|
|
3
|
+
// genroc — worker.ts is the only thing that talks to the queue — so this stays testable and
|
|
4
|
+
// the containment stays swappable.
|
|
4
5
|
//
|
|
5
6
|
// The containment is a Worker per execution (realm.ts). It is what makes the budget real:
|
|
6
7
|
// a synchronous busy loop never yields, so no in-process timer can interrupt it, and only a
|
|
@@ -9,6 +10,7 @@
|
|
|
9
10
|
import { Worker } from "node:worker_threads";
|
|
10
11
|
|
|
11
12
|
export type EvalRequest = {
|
|
13
|
+
/** An ES module whose default export is the function to run; `input` is its argument. */
|
|
12
14
|
code: string;
|
|
13
15
|
input?: unknown;
|
|
14
16
|
timeout_ms?: number;
|
|
@@ -55,14 +57,32 @@ class RealmFault extends Error {
|
|
|
55
57
|
}
|
|
56
58
|
}
|
|
57
59
|
|
|
58
|
-
|
|
60
|
+
/** Thrown when `signal` aborts: the work was cancelled server-side, so there is no outcome to
|
|
61
|
+
* report and worker.ts must release rather than answer. Distinct from RealmFault because the
|
|
62
|
+
* runner did not fault -- nothing is wrong, the answer is simply no longer wanted. */
|
|
63
|
+
export class Cancelled extends Error {
|
|
64
|
+
constructor() {
|
|
65
|
+
super("cancelled");
|
|
66
|
+
this.name = "Cancelled";
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export async function evaluate(req: EvalRequest, signal?: AbortSignal): Promise<EvalResult> {
|
|
59
71
|
const budget = typeof req.timeout_ms === "number" ? req.timeout_ms : DEFAULT_TIMEOUT_MS;
|
|
60
72
|
|
|
61
73
|
const worker = new Worker(REALM_URL);
|
|
62
74
|
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
75
|
+
let onAbort: (() => void) | undefined;
|
|
63
76
|
try {
|
|
64
77
|
return await new Promise<EvalResult>((resolve, reject) => {
|
|
65
78
|
timer = setTimeout(() => resolve(timedOut(budget)), budget);
|
|
79
|
+
// The abort is wired to the same promise as the budget, so both settle through the one
|
|
80
|
+
// finally below -- which is what guarantees the thread is gone before either returns.
|
|
81
|
+
if (signal) {
|
|
82
|
+
if (signal.aborted) reject(new Cancelled());
|
|
83
|
+
onAbort = () => reject(new Cancelled());
|
|
84
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
85
|
+
}
|
|
66
86
|
worker.once("message", (reply: WorkerReply) => resolve(reply));
|
|
67
87
|
// A script may end its own realm (`process.exit()`), which is not a throw and would
|
|
68
88
|
// otherwise present as a hang until the budget expired. Our own terminate() raises
|
|
@@ -73,6 +93,7 @@ export async function evaluate(req: EvalRequest): Promise<EvalResult> {
|
|
|
73
93
|
});
|
|
74
94
|
} finally {
|
|
75
95
|
clearTimeout(timer);
|
|
96
|
+
if (signal && onAbort) signal.removeEventListener("abort", onAbort);
|
|
76
97
|
// Awaited, and the whole point: on the timeout path a thread is still burning a core, and
|
|
77
98
|
// resolving before it is gone would report an evaluation the machine is still running.
|
|
78
99
|
await worker.terminate();
|