@genroc/eval-node 0.0.0-edge.f33a7b2 → 0.0.0-edge.f4092c85

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/import.js CHANGED
@@ -1,17 +1,11 @@
1
1
  #!/usr/bin/env node
2
- // The code-phase resolver: manifest on stdin, `{"code": [...]}` on stdout, non-zero exit
3
- // with the diagnostic on stderr. genctl never parses TypeScript and this never parses YAML
4
- // — the manifest is the whole contract. See specs/source-resolution.md.
5
- //
6
- // Two modes, one binary: "types" writes the declarations an editor needs and returns no
7
- // code; "build" typechecks and bundles. A separate types hook would mean a second `tsc`
8
- // over the same project.
9
- import { spawn } from "node:child_process";
2
+ // The typed-phase resolver: manifest on stdin, `{"values": [...]}` on stdout, diagnostic on stderr
3
+ // with a non-zero exit (specs/source-resolution.md). Two modes share one `tsc` pass: "generate"
4
+ // writes declarations only, "resolve" typechecks and bundles.
10
5
  import { existsSync } from "node:fs";
11
- import { access, mkdir, writeFile } from "node:fs/promises";
6
+ import { access, writeFile } from "node:fs/promises";
12
7
  import { builtinModules } from "node:module";
13
- import { dirname, join, relative, resolve } from "node:path";
14
- import { fileURLToPath } from "node:url";
8
+ import { dirname, join, resolve } from "node:path";
15
9
  import commonjs from "@rollup/plugin-commonjs";
16
10
  import json from "@rollup/plugin-json";
17
11
  import { nodeResolve } from "@rollup/plugin-node-resolve";
@@ -30,11 +24,6 @@ async function exists(path) {
30
24
  return false;
31
25
  }
32
26
  }
33
- /** Creates the parent directory, which `.genroc-cache/` relies on: nothing else makes it. */
34
- async function write(path, content) {
35
- await mkdir(dirname(path), { recursive: true });
36
- await writeFile(path, content);
37
- }
38
27
  // ── JSON Schema → TypeScript ───────────────────────────────────────────────────
39
28
  /** Only genroc's keyword set is handled; anything else its strict decoder would have
40
29
  * refused before this ran (internal/schema, allowedKeywords). */
@@ -90,15 +79,17 @@ function objectType(s, used) {
90
79
  const doc = typeof sub.description === "string"
91
80
  ? ` /** ${sub.description} */\n`
92
81
  : "";
93
- lines.push(`${doc} ${propKey(key)}${required.has(key) ? "" : "?"}: ${tsType(sub, used)};`);
82
+ lines.push(`${doc} ${propKey(key)}${required.has(key) ? "" : "?"}: ${nest(tsType(sub, used))};`);
94
83
  }
95
84
  if (s.additionalProperties && typeof s.additionalProperties === "object") {
96
- lines.push(` [key: string]: ${tsType(s.additionalProperties, used)};`);
85
+ lines.push(` [key: string]: ${nest(tsType(s.additionalProperties, used))};`);
97
86
  }
98
87
  if (lines.length === 0)
99
88
  return "Record<string, unknown>";
100
89
  return `{\n${lines.join("\n")}\n}`;
101
90
  }
91
+ /** Every line but the first, which is already placed by the property that opens it. */
92
+ const nest = (t) => t.replace(/\n/g, "\n ");
102
93
  function union(parts) {
103
94
  const seen = [...new Set(parts)];
104
95
  return seen.length === 0 ? "unknown" : seen.join(" | ");
@@ -106,13 +97,6 @@ function union(parts) {
106
97
  const IDENT = /^[A-Za-z_$][A-Za-z0-9_$]*$/;
107
98
  const propKey = (k) => (IDENT.test(k) ? k : JSON.stringify(k));
108
99
  const identifier = (n) => IDENT.test(n) ? n : `Def_${n.replace(/[^A-Za-z0-9_$]/g, "_")}`;
109
- function deref(s, defs) {
110
- let cur = s;
111
- for (let i = 0; cur && typeof cur.$ref === "string" && i < 16; i++) {
112
- cur = defs[cur.$ref.replace(/^#\/\$defs\//, "")];
113
- }
114
- return cur;
115
- }
116
100
  /** Emits one named type per reachable $def rather than inlining: a task output may
117
101
  * reference itself (specs/recursive-type-inference.md) and inlining would not terminate. */
118
102
  function declarations(at) {
@@ -132,7 +116,7 @@ function declarations(at) {
132
116
  emitted.push(`export type ${identifier(next)} = ${body};`);
133
117
  }
134
118
  return [
135
- "// Generated by genroc. Do not edit - regenerate with `genctl types`.",
119
+ "// Generated by genroc. Do not edit - regenerate with `genctl generate`.",
136
120
  `// ${where.name} (${address(site.pointer)})`,
137
121
  "",
138
122
  ...emitted,
@@ -149,9 +133,8 @@ function typesPathFor(scriptPath) {
149
133
  return scriptPath.replace(/\.[^.\/]+$/, "") + ".genroc.d.ts";
150
134
  }
151
135
  // ── typecheck ──────────────────────────────────────────────────────────────────
152
- /** The nearest tsconfig above the script — the one the author's editor already reads. Two
153
- * different configs mean a red editor over a clean apply, or the reverse. The walk stops at
154
- * the project root: above it is not this project. */
136
+ /** The tsconfig the author's editor reads, so editor and apply agree. Stops at the project
137
+ * root. */
155
138
  async function nearestTsconfig(from, root) {
156
139
  for (let dir = from;; dir = dirname(dir)) {
157
140
  const candidate = join(dir, "tsconfig.json");
@@ -162,11 +145,7 @@ async function nearestTsconfig(from, root) {
162
145
  }
163
146
  }
164
147
  async function typecheck(sites) {
165
- // NOT `.genroc`: that is the project config FILE, and a directory of the same name cannot
166
- // coexist with it. The suffix is what keeps the scratch area out of its way.
167
- const dir = join(root, ".genroc-cache");
168
- await write(join(dir, ".gitignore"), "*\n");
169
- // One tsc per distinct base config: `extends` takes a single base, so merging two would
148
+ // One program per distinct base config: `extends` takes a single base, so merging two would
170
149
  // check each script under the other author's options.
171
150
  const groups = new Map();
172
151
  for (const at of sites) {
@@ -177,10 +156,10 @@ async function typecheck(sites) {
177
156
  else
178
157
  groups.set(base, [at]);
179
158
  }
180
- let n = 0;
159
+ const diagnostics = [];
181
160
  for (const [base, group] of groups) {
182
161
  const config = {
183
- ...(base ? { extends: relative(dir, base) } : {}),
162
+ ...(base ? { extends: base } : {}),
184
163
  compilerOptions: {
185
164
  noEmit: true,
186
165
  strict: true,
@@ -191,42 +170,36 @@ async function typecheck(sites) {
191
170
  // `lib` DESCRIBES the realm and is written after `extends` so a base cannot widen it:
192
171
  // a worker thread has no document, whatever an author's config claims.
193
172
  lib: ["esnext", "webworker"],
194
- // `types` is the author's, and it is how a script opts into the node globals —
195
- // the worker realm has them, so refusing the declarations would only lie. With no
196
- // base config there is nothing to opt in with, so the default stays none.
173
+ // `types` is the author's opt-in to node globals, which the realm has. With no base
174
+ // config there is nothing to opt in with, so it stays none.
197
175
  ...(base ? {} : { types: [] }),
198
176
  },
199
- files: group.flatMap((s) => [
200
- relative(dir, s.file),
201
- relative(dir, typesPathFor(s.file)),
202
- ]),
177
+ files: group.flatMap((s) => [s.file, typesPathFor(s.file)]),
203
178
  // `files` overrides the base's, but a base `include` survives beside it and would
204
179
  // drag the author's whole tree in, to be checked under the worker lib.
205
180
  include: [],
206
181
  };
207
- const configPath = join(dir, groups.size === 1 ? "tsconfig.json" : `tsconfig.${n++}.json`);
208
- await write(configPath, JSON.stringify(config, null, 2));
209
- await runTsc(root, configPath);
182
+ // Never written; anchors `@types` lookup at the root. Not `tsconfig.json`: that is often the
183
+ // base it extends, and tsc refuses a circular extends.
184
+ const parsed = ts.parseJsonConfigFileContent(config, ts.sys, root, undefined, join(root, "tsconfig.genroc.json"));
185
+ const program = ts.createProgram({
186
+ rootNames: parsed.fileNames,
187
+ options: parsed.options,
188
+ projectReferences: parsed.projectReferences,
189
+ configFileParsingDiagnostics: parsed.errors,
190
+ });
191
+ diagnostics.push(...parsed.errors, ...ts.getPreEmitDiagnostics(program));
210
192
  }
211
- }
212
- async function runTsc(root, configPath) {
213
- const tsc = fileURLToPath(import.meta.resolve("typescript/bin/tsc"));
214
- const proc = spawn(process.execPath, [tsc, "--noEmit", "-p", configPath], {
215
- cwd: root,
216
- stdio: ["ignore", "pipe", "pipe"],
217
- });
218
- let out = "";
219
- let err = "";
220
- proc.stdout.on("data", (c) => (out += c));
221
- proc.stderr.on("data", (c) => (err += c));
222
- const code = await new Promise((resolve, reject) => {
223
- proc.on("error", reject);
224
- proc.on("close", (c) => resolve(c ?? 1));
225
- });
226
- if (code !== 0) {
227
- // tsc reports on stdout; the exit code IS the type check, so this is the diagnostic
228
- // genctl surfaces and the reason a failed import never produces a string.
229
- die([out, err].filter(Boolean).join("\n").trimEnd());
193
+ if (diagnostics.length > 0) {
194
+ // The diagnostics are the whole type check: this is what genctl surfaces, and the reason a
195
+ // failed import never produces a string.
196
+ die(ts
197
+ .formatDiagnostics(diagnostics, {
198
+ getCanonicalFileName: (f) => f,
199
+ getCurrentDirectory: () => root,
200
+ getNewLine: () => "\n",
201
+ })
202
+ .trimEnd());
230
203
  }
231
204
  }
232
205
  // ── bundle ─────────────────────────────────────────────────────────────────────
@@ -253,11 +226,9 @@ const BUILTIN = new Set([
253
226
  ...builtinModules,
254
227
  ...builtinModules.map((m) => `node:${m}`),
255
228
  ]);
256
- /** Resolves imports through TYPESCRIPT, using the same config the typecheck ran under, so a
257
- * `paths` alias that compiles also bundles. Reimplementing `paths` here would be a second
258
- * resolver to keep in agreement with tsc; this one cannot disagree.
259
- * A package resolving to a `.d.ts` is declined — that is a type, not the implementation —
260
- * which leaves node_modules to nodeResolve. */
229
+ /** Resolves through TYPESCRIPT under the typecheck's config, so a `paths` alias that compiles
230
+ * also bundles. A `.d.ts` resolution is declined (a type, not code), leaving node_modules to
231
+ * nodeResolve. */
261
232
  function tsResolve(configPath) {
262
233
  let options = {};
263
234
  if (configPath) {
@@ -278,14 +249,12 @@ function tsResolve(configPath) {
278
249
  },
279
250
  };
280
251
  }
281
- /** Bundles to a self-contained ES module, which is what the evaluator imports: the default
282
- * export it calls is the author's own, so nothing wraps or rewrites the code between the two.
283
- * Bundling is entirely the importer's job, so a definition version pins its code forever. */
252
+ /** A self-contained ES module whose default export is the author's own, unwrapped. Bundling here
253
+ * means a definition version pins its code forever. */
284
254
  async function bundle(at) {
285
255
  const site = at.site;
286
- // Builtins are EXTERNALISED as imports the realm resolves natively. Anything else
287
- // unresolved is a REFUSAL, not an external: rollup's default is to leave it as an import
288
- // of a module that will not be there, which bundles clean and fails at runtime.
256
+ // Builtins are externalised; anything else unresolved is a REFUSAL, since rollup's default
257
+ // leaves an import that bundles clean and fails at runtime.
289
258
  const built = await rollup({
290
259
  input: at.file,
291
260
  external: (id) => BUILTIN.has(id),
@@ -293,8 +262,7 @@ async function bundle(at) {
293
262
  tsResolve(await nearestTsconfig(dirname(at.file), root)),
294
263
  nodeResolve({ extensions: [".ts", ".tsx", ".mjs", ".js", ".json"] }),
295
264
  commonjs(),
296
- // A `.json` import is a data file inlined at build time, which the previous bundler
297
- // did natively; without it rollup hands the JSON to the JS parser.
265
+ // Inlines `.json` imports; without it rollup hands JSON to the JS parser.
298
266
  json(),
299
267
  transpile,
300
268
  ],
@@ -339,20 +307,18 @@ function address(pointer) {
339
307
  .join("")
340
308
  .replace(/^\./, "");
341
309
  }
342
- const located = manifest.processes.flatMap((where) => where.sites.map((site) => ({
343
- site,
344
- where,
345
- file: resolve(where.dir, site.argument),
346
- })));
347
- // genctl is agnostic about what a script is for; the contract that an evaluation request carries
348
- // its module in `code` is THIS resolver's, so it is the one that checks a directive landed there.
349
- // `child` is the shape the scaffold uses — a call to a process that forwards to the evaluator —
350
- // and `external` is the same request made directly.
310
+ const located = manifest.processes.flatMap((where) => where.sites.map((site) => {
311
+ const [script, ...rest] = site.args;
312
+ if (script === undefined || rest.length > 0) {
313
+ die(`${join(where.dir, where.file)}: ${address(site.pointer)}: $import takes one script, ` +
314
+ `and is given ${site.args.length} arguments`);
315
+ }
316
+ return { site, where, file: resolve(where.dir, script) };
317
+ }));
318
+ // The `code`-field contract is this resolver's, not genctl's, so it checks the directive landed
319
+ // there: in a child call (the scaffold's shape) or an external task's input.
351
320
  for (const at of located) {
352
- // Two halves, from the two places that carry them: the action's KIND is a field, and WHERE in
353
- // it the directive sits is the pointer. A child call is the shape the scaffold generates — to a
354
- // process that forwards to the evaluator — and an external task is the same request made
355
- // directly.
321
+ // The action's kind is a field; where in it the directive sits is the pointer.
356
322
  const kind = at.site.level === "action" ? (at.site.action ?? "") : "";
357
323
  const slot = at.site.pointer.slice(-2).join(".");
358
324
  if ((kind !== "child" && kind !== "external") || slot !== "input.code") {
@@ -361,12 +327,11 @@ for (const at of located) {
361
327
  `${kind ? `a ${kind} task's ` : ""}\`${slot}\`.`);
362
328
  }
363
329
  }
364
- // genctl no longer stats the argument — it does not know it is a file — so the resolver that
365
- // does is the one that must say when it is not there.
330
+ // genctl does not know the argument is a file, so this resolver reports a missing one.
366
331
  for (const at of located) {
367
332
  if (!existsSync(at.file)) {
368
333
  die(`${join(at.where.dir, at.where.file)}: ${address(at.site.pointer)}: ` +
369
- `"$import: ${at.site.argument}" names no file (looked at ${at.file})`);
334
+ `"$import: ${at.site.args[0]}" names no file (looked at ${at.file})`);
370
335
  }
371
336
  }
372
337
  // One script at two sites with different input types is a refusal, not a union: the union
@@ -383,15 +348,14 @@ for (const at of located) {
383
348
  byPath.set(at.file, at);
384
349
  }
385
350
  for (const at of byPath.values()) {
386
- await write(typesPathFor(at.file), declarations(at));
351
+ await writeFile(typesPathFor(at.file), declarations(at));
387
352
  }
388
- if (manifest.mode === "types") {
389
- console.log(JSON.stringify(manifest, null, 2));
353
+ if (manifest.mode === "generate") {
390
354
  process.exit(0);
391
355
  }
392
356
  await typecheck([...byPath.values()]);
393
- const code = [];
357
+ const values = [];
394
358
  for (const at of located) {
395
- code.push(await bundle(at));
359
+ values.push(await bundle(at));
396
360
  }
397
- process.stdout.write(JSON.stringify({ code }));
361
+ process.stdout.write(JSON.stringify({ values }));
package/dist/realm.js CHANGED
@@ -1,9 +1,5 @@
1
- // The evaluation realm. One Worker per execution: a fresh global object per script, and a
2
- // thread the host can kill mid-loop — the only thing that bounds a synchronous busy loop.
3
- // eval.ts owns the budget and does the killing; nothing here knows about time.
4
- //
5
- // Everything that touches the script's VALUE lives on this side of the boundary — loading,
6
- // classifying, serialising — because this is the only realm the value exists in.
1
+ // The evaluation realm: one Worker per execution, killable mid-loop by eval.ts, which owns the
2
+ // budget. Everything touching the script's VALUE lives here, the only realm it exists in.
7
3
  var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExtension) || function (path, preserveJsx) {
8
4
  if (typeof path === "string" && /^\.\.?\//.test(path)) {
9
5
  return path.replace(/\.(tsx)$|((?:\.d)?)((?:\.[^./]+?)?)\.([cm]?)ts$/i, function (m, tsx, d, ext, cm) {
@@ -14,9 +10,8 @@ var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExte
14
10
  };
15
11
  import { registerHooks } from "node:module";
16
12
  import { parentPort } from "node:worker_threads";
17
- // The script is IMPORTED as a module under a URL of our own, not compiled from a string: its
18
- // frames then carry the author's own line numbers, and an `import` of a node builtin resolves
19
- // the way it does everywhere else.
13
+ // IMPORTED under our own URL rather than compiled from a string, so frames carry the author's
14
+ // line numbers and builtin imports resolve normally.
20
15
  const SCRIPT_URL = "script:main";
21
16
  const STACK_BYTES = 2_048;
22
17
  // The only channel a load hook has to the source. Written per request and read once — a realm
@@ -41,7 +36,10 @@ function scriptStack(err) {
41
36
  }
42
37
  function describe(err, kind) {
43
38
  if (err instanceof Error) {
44
- return { kind, name: err.name, message: err.message, stack: scriptStack(err) };
39
+ // A subclass inherits `name` from Error.prototype, so `class LimitExceeded extends Error {}`
40
+ // would report "Error"; a name the script set still wins.
41
+ const name = err.name === "Error" && err.constructor.name ? err.constructor.name : err.name;
42
+ return { kind, name, message: err.message, stack: scriptStack(err) };
45
43
  }
46
44
  // A script may throw a non-Error (`throw {code: "x"}`), so name/message must not assume one.
47
45
  return { kind, name: "Thrown", message: safeText(err) };
@@ -54,9 +52,8 @@ function safeText(v) {
54
52
  return String(v);
55
53
  }
56
54
  }
57
- /** A module that will not parse, or names an import nothing resolves, is broken code — only
58
- * editing it helps. Anything else thrown by the import is the module's top level running,
59
- * which is the script throwing. */
55
+ /** Unparseable code or an unresolvable import is broken code; anything else the import throws is
56
+ * the module's top level running, i.e. the script throwing. */
60
57
  function unloadable(err) {
61
58
  const code = err?.code;
62
59
  return err instanceof SyntaxError || (typeof code === "string" && code.startsWith("ERR_MODULE"));
@@ -97,10 +94,8 @@ async function run(req) {
97
94
  return { ok: false, failure: describe(err, "nonserializable") };
98
95
  }
99
96
  }
100
- // The realm's stdio is a pipe to the host thread, and eval.ts terminates this thread the moment
101
- // the reply lands — so whatever a script wrote last is still in the pipe when it dies. An empty
102
- // write's callback fires once the queue ahead of it has drained, which makes the reply a barrier
103
- // for `console` and a direct process.stdout.write alike: both are this stream.
97
+ // eval.ts terminates this thread the moment the reply lands, while stdio may still be in the
98
+ // pipe. An empty write's callback fires once the queue drains, making the reply a barrier.
104
99
  function flush(stream) {
105
100
  return new Promise((resolve) => stream.write("", () => resolve()));
106
101
  }
package/dist/worker.js CHANGED
@@ -1,31 +1,18 @@
1
1
  #!/usr/bin/env node
2
- // The queue worker: claims parked `external` script tasks from genroc, evaluates each in its
3
- // own realm, and answers. This is the whole genroc-facing half — eval.ts and realm.ts know
4
- // nothing about the queue, which is what keeps the containment strategy swappable.
5
- //
6
- // See README.md for the contract, and specs/external-task-queue.md for the queue itself.
2
+ // The queue worker: claims parked `external` script tasks, evaluates each in its own realm, and
3
+ // answers. The only genroc-facing half (README.md; specs/external-task-queue.md).
7
4
  import { readFileSync } from "node:fs";
8
- import { evaluate } from "./eval.js";
5
+ import { Cancelled, evaluate } from "./eval.js";
9
6
  const SERVER = (process.env.GENROC_SERVER ?? "http://localhost:8448").replace(/\/$/, "");
10
7
  const WORKER_ID = process.env.WORKER_ID ?? `evaluator-${process.pid}`;
11
- // The credential, when the server runs with --auth token. A worker needs exactly the `worker`
12
- // permission — the four queue verbs plus GET /api/objects — so mint it scoped rather than
13
- // handing a worker an admin token: this is the credential most likely to sit on a machine you
14
- // trust least. specs/api-auth.md §5.
15
- //
16
- // Sent as a header rather than in the URL because Node's fetch REFUSES a URL carrying
17
- // credentials ("Request cannot be constructed from a URL that includes credentials"), so the
18
- // basic-auth-in-the-URL trick that works for genctl is not available here.
19
- // GENROC_TOKEN_FILE is the mounted-secret shape: a credential in a file rather than an
20
- // environment variable, so it stays out of `docker inspect` and out of the process environment
21
- // any child inherits. The inline variable wins when both are set.
8
+ // Needs only the `worker` permission (specs/api-auth.md §5): this credential sits on the machine
9
+ // you trust least. A header, because Node's fetch refuses credentials in a URL. The file form
10
+ // keeps it out of `docker inspect` and child environments; the inline variable wins.
22
11
  const TOKEN = process.env.GENROC_TOKEN ??
23
12
  (process.env.GENROC_TOKEN_FILE ? readFileSync(process.env.GENROC_TOKEN_FILE, "utf8").trim() : "");
24
13
  const authHeaders = TOKEN ? { authorization: `Bearer ${TOKEN}` } : {};
25
- // Concurrency is the worker's to set, and that is the point of pulling: under the old fetch
26
- // shape genroc decided how many scripts ran at once (--max-concurrent, default 200) and the
27
- // evaluator accepted every one of them. Here it claims what it can run and no more, so a
28
- // backlog is a queue rather than 200 threads fighting over a core.
14
+ // The worker claims only what it can run, so a backlog is a queue rather than threads fighting
15
+ // over a core.
29
16
  const CONCURRENCY = Number(process.env.CONCURRENCY ?? 4);
30
17
  const POLL_MS = Number(process.env.POLL_MS ?? 250);
31
18
  // The visibility timeout. Short, and renewed while work is in flight: a worker that dies
@@ -34,17 +21,15 @@ const LEASE_MS = Number(process.env.LEASE_MS ?? 30_000);
34
21
  const RENEW_MS = Math.max(1_000, Math.floor(LEASE_MS / 3));
35
22
  const PROCESS_FILTER = process.env.PROCESS ?? "";
36
23
  const TASK_FILTER = process.env.TASK ?? "";
37
- // Values too large to ship inline are listed rather than carried, and a bundle is exactly that:
38
- // one object shared by every instance of a definition version, fetched once instead of copied
39
- // into each task. A ref is a content hash, so it is immutable — the cache never invalidates.
24
+ // Large values arrive as refs; a bundle is one object per definition version, fetched once.
25
+ // Refs are content hashes, so the cache never invalidates.
40
26
  const objectCache = new Map();
41
27
  async function fetchObject(ref) {
42
28
  const cached = objectCache.get(ref);
43
29
  if (cached !== undefined)
44
30
  return cached;
45
- // Left to throw on purpose: this runs while a task is IN FLIGHT, and a task whose input
46
- // cannot be fetched must fail rather than silently run against a missing value. The caller
47
- // releases the claim, so the task returns to the queue.
31
+ // Left to throw: a task whose input cannot be fetched must not run against a missing value.
32
+ // The caller releases the claim.
48
33
  const res = await fetch(`${SERVER}/api/objects/${encodeURIComponent(ref)}`, { headers: authHeaders });
49
34
  if (!res.ok)
50
35
  throw new Error(`fetch object ${ref}: HTTP ${res.status}`);
@@ -62,15 +47,15 @@ async function fetchObject(ref) {
62
47
  /** Put each listed value back where its path says it belongs. Paths are arrays of keys, so this
63
48
  * needs no parser: the whole reason they are not JSON Pointer strings. */
64
49
  async function resolveObjects(job) {
65
- let input = job.input;
50
+ let input = job.external_input;
66
51
  for (const e of job.objects ?? []) {
67
52
  const value = await fetchObject(e.ref);
68
53
  if (e.path.length === 0) {
69
54
  input = value;
70
55
  continue;
71
56
  }
72
- // The path is rooted at the entry and starts with "input", which is the value being rebuilt.
73
- const rest = e.path[0] === "input" ? e.path.slice(1) : e.path;
57
+ // The path is rooted at the entry and starts with "external_input", the value being rebuilt.
58
+ const rest = e.path[0] === "external_input" ? e.path.slice(1) : e.path;
74
59
  if (rest.length === 0) {
75
60
  input = value;
76
61
  continue;
@@ -84,10 +69,8 @@ async function resolveObjects(job) {
84
69
  return input;
85
70
  }
86
71
  async function call(path, body) {
87
- // A network error is a REPLY, not a throw. A worker outlives the server it polls — a
88
- // restart, a rolling deploy, a container coming up before genroc is listening — and an
89
- // unhandled rejection here kills it for a condition the next poll would clear. Status 0
90
- // says "never reached the server", which is distinct from anything genroc answers.
72
+ // A network error is a REPLY (status 0, "never reached the server"), not a throw: a worker
73
+ // outlives restarts and deploys, and an unhandled rejection would kill it.
91
74
  let res;
92
75
  try {
93
76
  res = await fetch(SERVER + path, {
@@ -109,9 +92,8 @@ async function call(path, body) {
109
92
  }
110
93
  return { ok: res.ok, status: res.status, data };
111
94
  }
112
- /** The task input IS an EvalRequest: `code` required, `input` and `timeout_ms` optional. A task
113
- * whose input is not that shape is the definition's fault, not the script's, and is reported
114
- * as a compile_error — the nearest permanent kind, since no retry can fix the definition. */
95
+ /** A task input that is not an EvalRequest is the definition's fault, reported as compile_error:
96
+ * the nearest permanent kind, since no retry fixes the definition. */
115
97
  function asEvalRequest(input) {
116
98
  if (typeof input !== "object" || input === null)
117
99
  return "the task input is not an object";
@@ -126,10 +108,8 @@ function asEvalRequest(input) {
126
108
  }
127
109
  const inFlight = new Map();
128
110
  let running = true;
129
- // Whether the last claim reached genroc. A worker polls several times a second, so an
130
- // unreachable server would otherwise emit a line per poll — thousands during a restart, which
131
- // buries the one line that mattered. Announce the TRANSITIONS instead: going away, and coming
132
- // back. Silence in between is the report that nothing changed.
111
+ // Announce reachability TRANSITIONS only: polling several times a second would otherwise emit
112
+ // thousands of lines during a restart.
133
113
  let serverReachable = true;
134
114
  async function claim(n) {
135
115
  const { ok, status, data } = await call("/api/external-tasks/claim", {
@@ -140,18 +120,15 @@ async function claim(n) {
140
120
  ...(TASK_FILTER ? { task: TASK_FILTER } : {}),
141
121
  });
142
122
  if (!ok) {
143
- // A credential problem is not transient, and polling through it looks like a healthy
144
- // worker that never picks anything up — the worst shape for an operator to debug. Exit
145
- // instead, so a supervisor restarts it and the failure is visible where it happened.
123
+ // A credential problem is not transient, and polling through it looks like a healthy idle
124
+ // worker. Exit, so a supervisor restarts it visibly.
146
125
  if (status === 401 || status === 403) {
147
126
  console.error(`claim rejected (${status}): ${JSON.stringify(data)}\n` +
148
127
  `The server requires authentication. Set GENROC_TOKEN to a token with the 'worker' ` +
149
128
  `permission — mint one with: genctl token create --perms worker --label evaluator -q`);
150
129
  process.exit(1);
151
130
  }
152
- // status 0 is "never reached the server" (see call): a restart, a rolling deploy, a
153
- // network blip. Not an error to act on — the next poll clears it — so it is reported once
154
- // and then waited out.
131
+ // status 0 (see call): a restart or blip the next poll clears, so it is reported once.
155
132
  if (status === 0) {
156
133
  if (serverReachable) {
157
134
  serverReachable = false;
@@ -174,10 +151,8 @@ async function release(token) {
174
151
  if (!ok)
175
152
  console.error(`release failed: ${JSON.stringify(data)}`);
176
153
  }
177
- /** answer submits the outcome. A refusal is NOT retried with a different one: the definition
178
- * declared a contract this worker does not satisfy (an undeclared code, a payload that does
179
- * not fit `raises`), and guessing again would only pick a second wrong answer. Release it, so
180
- * the task returns to the queue and an operator sees it waiting rather than silently gone. */
154
+ /** A refused outcome is not retried with another: the worker does not satisfy the declared
155
+ * contract. Released, so an operator sees the task waiting rather than gone. */
181
156
  async function answer(token, outcome) {
182
157
  const { ok, data } = await call("/api/external-tasks/resolve", { token, ...outcome });
183
158
  if (ok)
@@ -185,7 +160,7 @@ async function answer(token, outcome) {
185
160
  console.error(`genroc refused the outcome for ${token}: ${JSON.stringify(data)}`);
186
161
  await release(token);
187
162
  }
188
- async function run(job) {
163
+ async function run(job, signal) {
189
164
  let resolved;
190
165
  try {
191
166
  resolved = await resolveObjects(job);
@@ -206,13 +181,18 @@ async function run(job) {
206
181
  }
207
182
  let result;
208
183
  try {
209
- result = await evaluate(req);
184
+ result = await evaluate(req, signal);
210
185
  }
211
186
  catch (err) {
212
- // The RUNNER faulted, not the script — the one class where a retry can help. There is no
213
- // error code for it on purpose: releasing the claim is how a queue spells "retryable", and
214
- // it puts the task in front of a different worker instead of burning the definition's
215
- // on_error budget on this one's bad day.
187
+ // Cancelled is not a fault. Still released: the row is terminal so nothing re-claims it,
188
+ // but the release stops it waiting out a lease nobody serves.
189
+ if (err instanceof Cancelled) {
190
+ console.log(`cancelled: ${job.token}`);
191
+ await release(job.token);
192
+ return;
193
+ }
194
+ // The RUNNER faulted, the one retryable class. Releasing is how a queue spells "retryable",
195
+ // and it reaches another worker without spending the definition's on_error budget.
216
196
  console.error(`evaluator fault on ${job.token}: ${err instanceof Error ? err.message : String(err)}`);
217
197
  await release(job.token);
218
198
  return;
@@ -245,11 +225,18 @@ async function renewLoop() {
245
225
  tokens,
246
226
  lease_ms: LEASE_MS,
247
227
  });
248
- // A short count means a claim lapsed and was taken over. Nothing to do about it — the work
249
- // continues and its answer will be refused — but say so, because it is the signal that
250
- // LEASE_MS is too short for what these scripts actually take.
251
- if (ok && data?.renewed < tokens.length) {
252
- console.error(`renewed ${data.renewed}/${tokens.length} claims; a lease lapsed under load`);
228
+ if (!ok)
229
+ continue;
230
+ // Abort first rather than let a stopped script burn a core; the release rides run()'s
231
+ // catch, so nothing is released twice.
232
+ for (const token of (data?.cancelled ?? [])) {
233
+ inFlight.get(token)?.abort.abort();
234
+ }
235
+ // Lost: another worker holds it now and our answer will be refused; logged as the sign that
236
+ // LEASE_MS is too short. NOT released: that would bump the epoch out from under the holder.
237
+ const lost = (data?.lost ?? []);
238
+ if (lost.length) {
239
+ console.error(`lost ${lost.length}/${tokens.length} claims; a lease lapsed under load`);
253
240
  }
254
241
  }
255
242
  }
@@ -258,8 +245,9 @@ async function pollLoop() {
258
245
  const free = CONCURRENCY - inFlight.size;
259
246
  const jobs = free > 0 ? await claim(free) : [];
260
247
  for (const job of jobs) {
261
- inFlight.set(job.token, job);
262
- void run(job).finally(() => inFlight.delete(job.token));
248
+ const abort = new AbortController();
249
+ inFlight.set(job.token, { job, abort });
250
+ void run(job, abort.signal).finally(() => inFlight.delete(job.token));
263
251
  }
264
252
  // Only idle when there was nothing to take: a full queue should be drained at the speed the
265
253
  // realms allow, not at the poll interval.