@genroc/eval-node 0.0.0-edge.51cbf37 → 0.0.0-edge.51f2744b

package/realm.ts

@@ -1,89 +1,44 @@
-// The evaluation realm. One Worker per execution: a fresh global object per script, and a
-// thread the host can kill mid-loop — the only thing that bounds a synchronous busy loop.
-// eval.ts owns the budget and does the killing; nothing here knows about time.
-//
-// Everything that touches the script's VALUE lives on this side of the boundary — compiling,
-// classifying, serialising — because this is the only realm the value exists in.
+// The evaluation realm: one Worker per execution, killable mid-loop by eval.ts, which owns the
+// budget. Everything touching the script's VALUE lives here, the only realm it exists in.
 
-import { createRequire } from "node:module";
+import { registerHooks } from "node:module";
 import { parentPort } from "node:worker_threads";
 
 import type { EvalFailure, FailureKind, WorkerReply, WorkerRequest } from "./eval.ts";
 
-const AsyncFunction = async function () {}.constructor as new (
-  ...args: string[]
-) => (...args: unknown[]) => Promise<unknown>;
-
-const STRICT = '"use strict";\n';
-
-// Bundled `node:*` imports survive as `require` calls — the importer externalises builtins and
-// inlines everything else — and a function built by the AsyncFunction constructor has no
-// `require` in scope. Passing one in is what makes an import of a builtin work at runtime rather than at
-// typecheck only. Resolution is anchored here, which is right: only builtins reach it.
-const scriptRequire = createRequire(import.meta.url);
+// IMPORTED under our own URL rather than compiled from a string, so frames carry the author's
+// line numbers and builtin imports resolve normally.
+const SCRIPT_URL = "script:main";
 const STACK_BYTES = 2_048;
 
-/**
- * Line offset the AsyncFunction preamble adds, measured rather than assumed: the generated
- * wrapper's shape is engine-specific, and a hardcoded number silently misreports every
- * script's error location the day it changes.
- */
-const lineOffset: Promise<number> = (async () => {
-  // Same parameter list as a real compile: the preamble is what is being measured.
-  const probe = new AsyncFunction("input", "require", STRICT + "throw new Error('probe');");
-  try {
-    await probe();
-    return 0;
-  } catch (err) {
-    return reportedLine(err) - 1;
-  }
-})();
-
-// V8 marks a frame compiled by the AsyncFunction constructor with the site that CALLED the
-// constructor, then the script's OWN position:
-//   at inner (eval at run (file:///…/worker.ts:107:10), <anonymous>:6:9)
-// `eval at` is therefore what separates script frames from runner plumbing — matched without
-// the function name, which is whatever encloses the `new AsyncFunction` below. The LAST such
-// frame is the body's top level: frames interleave, since a script can throw inside a native
-// callback.
-const SCRIPT_FRAME = /\(eval at /;
-// The trailing `<anonymous>:LINE:COL` — the script's position, after the host file's own.
-const POSITION = /<anonymous>:(\d+):(\d+)\)?\s*$/;
-// `    at name (` — absent on the top-level frame, which V8 names `eval`.
-const FRAME_NAME = /^\s*at\s+(?:async\s+)?([^\s(]+)\s*\(/;
+// The only channel a load hook has to the source. Written per request and read once — a realm
+// evaluates one script and is then discarded, so no second execution can observe it.
+let source = "";
 
-/** Line number of the throw as the engine reported it, or 1 if the stack is unreadable. */
-function reportedLine(err: unknown): number {
-  const stack = err instanceof Error && typeof err.stack === "string" ? err.stack : "";
-  const frame = stack.split("\n").find((l) => SCRIPT_FRAME.test(l)) ?? "";
-  const m = frame.match(POSITION);
-  return m ? Number(m[1]) : 1;
-}
+registerHooks({
+  resolve: (specifier, context, next) =>
+    specifier === SCRIPT_URL ? { url: SCRIPT_URL, shortCircuit: true } : next(specifier, context),
+  load: (url, context, next) =>
+    url === SCRIPT_URL ? { format: "module", source, shortCircuit: true } : next(url, context),
+});
 
-/** Renumbers each script frame to the line the AUTHOR wrote and drops the runner's own.
- *  Rewriting the whole location is also what keeps the runner's path out of a script's
- *  stack — V8 puts it inside every compiled frame. */
-function scriptStack(err: unknown, offset: number): string | undefined {
+/** Keeps the frames that are the script's own. Everything below the last of them is runner
+ *  plumbing the author cannot act on, and V8 puts this file's path in it. */
+function scriptStack(err: unknown): string | undefined {
   if (!(err instanceof Error) || typeof err.stack !== "string") return undefined;
   const lines = err.stack.split("\n");
-  let boundary = -1;
-  for (let i = 0; i < lines.length; i++) if (SCRIPT_FRAME.test(lines[i]!)) boundary = i;
-  const frames = (boundary >= 0 ? lines.slice(0, boundary + 1) : lines.slice(0, 1))
-    .map((line) => {
-      const pos = line.match(POSITION);
-      if (!pos) return line; // the `Error: message` header and native frames, kept as-is
-      const name = line.match(FRAME_NAME)?.[1];
-      const at = name && name !== "eval" && name !== "anonymous" ? `at ${name} ` : "at ";
-      const indent = line.match(/^\s*/)![0];
-      return `${indent}${at}(script:${Math.max(1, Number(pos[1]) - offset)}:${pos[2]})`;
-    })
-    .join("\n");
-  return frames.length > STACK_BYTES ? frames.slice(0, STACK_BYTES) : frames;
+  let last = 0;
+  for (let i = 0; i < lines.length; i++) if (lines[i]!.includes(SCRIPT_URL)) last = i;
+  const stack = lines.slice(0, last + 1).join("\n");
+  return stack.length > STACK_BYTES ? stack.slice(0, STACK_BYTES) : stack;
 }
 
-function describe(err: unknown, kind: FailureKind, offset: number): EvalFailure {
+function describe(err: unknown, kind: FailureKind): EvalFailure {
   if (err instanceof Error) {
-    return { kind, name: err.name, message: err.message, stack: scriptStack(err, offset) };
+    // A subclass inherits `name` from Error.prototype, so `class LimitExceeded extends Error {}`
+    // would report "Error"; a name the script set still wins.
+    const name = err.name === "Error" && err.constructor.name ? err.constructor.name : err.name;
+    return { kind, name, message: err.message, stack: scriptStack(err) };
   }
   // A script may throw a non-Error (`throw {code: "x"}`), so name/message must not assume one.
   return { kind, name: "Thrown", message: safeText(err) };
@@ -97,23 +52,39 @@ function safeText(v: unknown): string {
   }
 }
 
+/** Unparseable code or an unresolvable import is broken code; anything else the import throws is
+ *  the module's top level running, i.e. the script throwing. */
+function unloadable(err: unknown): boolean {
+  const code = (err as { code?: unknown } | null)?.code;
+  return err instanceof SyntaxError || (typeof code === "string" && code.startsWith("ERR_MODULE"));
+}
+
 async function run(req: WorkerRequest): Promise<WorkerReply> {
-  const offset = await lineOffset;
+  source = req.code;
 
-  let fn: (...args: unknown[]) => Promise<unknown>;
+  let main: unknown;
   try {
-    // No compile cache: the realm is discarded after this execution, so a cache in it could
-    // never be hit. Repeated compilation is the price of the fresh global object.
-    fn = new AsyncFunction("input", "require", STRICT + req.code);
+    main = (await import(SCRIPT_URL)).default;
   } catch (err) {
-    return { ok: false, failure: describe(err, "compile_error", offset) };
+    return { ok: false, failure: describe(err, unloadable(err) ? "compile_error" : "threw") };
+  }
+  if (typeof main !== "function") {
+    const got = main === undefined ? "no default export" : `a ${typeof main}`;
+    return {
+      ok: false,
+      failure: {
+        kind: "compile_error",
+        name: "NoDefaultExport",
+        message: `a script must export default a function; this one has ${got}`,
+      },
+    };
   }
 
   let value: unknown;
   try {
-    value = await fn(req.input, scriptRequire);
+    value = await (main as (input: unknown) => unknown)(req.input);
   } catch (err) {
-    return { ok: false, failure: describe(err, "threw", offset) };
+    return { ok: false, failure: describe(err, "threw") };
   }
 
   try {
@@ -121,12 +92,20 @@ async function run(req: WorkerRequest): Promise<WorkerReply> {
     // spells null, which is the right reading of a script that returned nothing.
     return { ok: true, body: value === undefined ? "" : JSON.stringify(value) ?? "" };
   } catch (err) {
-    return { ok: false, failure: describe(err, "nonserializable", offset) };
+    return { ok: false, failure: describe(err, "nonserializable") };
   }
 }
 
+// eval.ts terminates this thread the moment the reply lands, while stdio may still be in the
+// pipe. An empty write's callback fires once the queue drains, making the reply a barrier.
+function flush(stream: NodeJS.WriteStream): Promise<void> {
+  return new Promise((resolve) => stream.write("", () => resolve()));
+}
+
 // Non-null because this module only ever runs as a worker entry point; a null port here
 // would mean eval.ts loaded it as a plain module, which nothing does.
 parentPort!.on("message", async (req: WorkerRequest) => {
-  parentPort!.postMessage(await run(req));
+  const reply = await run(req);
+  await Promise.all([flush(process.stdout), flush(process.stderr)]);
+  parentPort!.postMessage(reply);
 });

package/worker.ts

@@ -1,65 +1,59 @@
 #!/usr/bin/env node
-// The queue worker: claims parked `external` script tasks from genroc, evaluates each in its
-// own realm, and answers. This is the whole genroc-facing half — eval.ts and realm.ts know
-// nothing about the queue, which is what keeps the containment strategy swappable.
-//
-// See README.md for the contract, and specs/external-task-queue.md for the queue itself.
+// The queue worker: claims parked `external` script tasks, evaluates each in its own realm, and
+// answers. The only genroc-facing half (README.md; specs/external-task-queue.md).
 
 import { readFileSync } from "node:fs";
-import { evaluate, type EvalRequest, type FailureKind } from "./eval.ts";
+import { Cancelled, evaluate, type EvalRequest, type FailureKind } from "./eval.ts";
 
 const SERVER = (process.env.GENROC_SERVER ?? "http://localhost:8448").replace(/\/$/, "");
 const WORKER_ID = process.env.WORKER_ID ?? `evaluator-${process.pid}`;
-// The credential, when the server runs with --auth token. A worker needs exactly the `worker`
-// permission — the four queue verbs plus GET /api/objects — so mint it scoped rather than
-// handing a worker an admin token: this is the credential most likely to sit on a machine you
-// trust least. specs/api-auth.md §5.
-//
-// Sent as a header rather than in the URL because Node's fetch REFUSES a URL carrying
-// credentials ("Request cannot be constructed from a URL that includes credentials"), so the
-// basic-auth-in-the-URL trick that works for genctl is not available here.
-// GENROC_TOKEN_FILE is the mounted-secret shape: a credential in a file rather than an
-// environment variable, so it stays out of `docker inspect` and out of the process environment
-// any child inherits. The inline variable wins when both are set.
+// Needs only the `worker` permission (specs/api-auth.md §5): this credential sits on the machine
+// you trust least. A header, because Node's fetch refuses credentials in a URL. The file form
+// keeps it out of `docker inspect` and child environments; the inline variable wins.
 const TOKEN =
   process.env.GENROC_TOKEN ??
   (process.env.GENROC_TOKEN_FILE ? readFileSync(process.env.GENROC_TOKEN_FILE, "utf8").trim() : "");
 const authHeaders: Record<string, string> = TOKEN ? { authorization: `Bearer ${TOKEN}` } : {};
-// Concurrency is the worker's to set, and that is the point of pulling: under the old fetch
-// shape genroc decided how many scripts ran at once (--max-concurrent, default 200) and the
-// evaluator accepted every one of them. Here it claims what it can run and no more, so a
-// backlog is a queue rather than 200 threads fighting over a core.
+// The worker claims only what it can run, so a backlog is a queue rather than threads fighting
+// over a core.
 const CONCURRENCY = Number(process.env.CONCURRENCY ?? 4);
 const POLL_MS = Number(process.env.POLL_MS ?? 250);
 // The visibility timeout. Short, and renewed while work is in flight: a worker that dies
 // should return its task quickly rather than holding it for the whole budget.
 const LEASE_MS = Number(process.env.LEASE_MS ?? 30_000);
 const RENEW_MS = Math.max(1_000, Math.floor(LEASE_MS / 3));
-const PROCESS_FILTER = process.env.PROCESS ?? "";
-const TASK_FILTER = process.env.TASK ?? "";
+// The answer has to land before the task's deadline fires, or a script that overran surfaces as
+// external.timeout ("nobody answered") rather than as its own `timeout`.
+const ANSWER_MARGIN_MS = 250;
+// script-node's names; a renamed copy sets PROCESS. The claim requires both: this worker answers
+// one task's contract, so it must never be handed another's.
+const PROCESS_FILTER = process.env.PROCESS ?? "script-node";
+const TASK_FILTER = process.env.TASK ?? "eval_node";
+if (!PROCESS_FILTER || !TASK_FILTER) {
+  console.error("PROCESS and TASK must both name the queue this worker serves; neither may be empty.");
+  process.exit(1);
+}
 
 type ObjectEntry = { path: (string | number)[]; ref: string; size: number };
 
 type QueueTask = {
   token: string;
   process: string;
-  task_id: string;
-  input: unknown;
+  task: string;
+  external_input: unknown;
   objects?: ObjectEntry[];
-  raises?: Record<string, unknown>;
+  deadline?: string;
 };
 
-// Values too large to ship inline are listed rather than carried, and a bundle is exactly that:
-// one object shared by every instance of a definition version, fetched once instead of copied
-// into each task. A ref is a content hash, so it is immutable — the cache never invalidates.
+// Large values arrive as refs; a bundle is one object per definition version, fetched once.
+// Refs are content hashes, so the cache never invalidates.
 const objectCache = new Map<string, unknown>();
 
 async function fetchObject(ref: string): Promise<unknown> {
   const cached = objectCache.get(ref);
   if (cached !== undefined) return cached;
-  // Left to throw on purpose: this runs while a task is IN FLIGHT, and a task whose input
-  // cannot be fetched must fail rather than silently run against a missing value. The caller
-  // releases the claim, so the task returns to the queue.
+  // Left to throw: a task whose input cannot be fetched must not run against a missing value.
+  // The caller releases the claim.
   const res = await fetch(`${SERVER}/api/objects/${encodeURIComponent(ref)}`, { headers: authHeaders });
   if (!res.ok) throw new Error(`fetch object ${ref}: HTTP ${res.status}`);
   const { data } = (await res.json()) as { data: string };
@@ -76,15 +70,15 @@ async function fetchObject(ref: string): Promise<unknown> {
 /** Put each listed value back where its path says it belongs. Paths are arrays of keys, so this
  *  needs no parser: the whole reason they are not JSON Pointer strings. */
 async function resolveObjects(job: QueueTask): Promise<unknown> {
-  let input: any = job.input;
+  let input: any = job.external_input;
   for (const e of job.objects ?? []) {
     const value = await fetchObject(e.ref);
     if (e.path.length === 0) {
       input = value;
       continue;
     }
-    // The path is rooted at the entry and starts with "input", which is the value being rebuilt.
-    const rest = e.path[0] === "input" ? e.path.slice(1) : e.path;
+    // The path is rooted at the entry and starts with "external_input", the value being rebuilt.
+    const rest = e.path[0] === "external_input" ? e.path.slice(1) : e.path;
     if (rest.length === 0) {
       input = value;
       continue;
@@ -97,10 +91,8 @@ async function resolveObjects(job: QueueTask): Promise<unknown> {
 }
 
 async function call(path: string, body: unknown): Promise<{ ok: boolean; status: number; data: any }> {
-  // A network error is a REPLY, not a throw. A worker outlives the server it polls — a
-  // restart, a rolling deploy, a container coming up before genroc is listening — and an
-  // unhandled rejection here kills it for a condition the next poll would clear. Status 0
-  // says "never reached the server", which is distinct from anything genroc answers.
+  // A network error is a REPLY (status 0, "never reached the server"), not a throw: a worker
+  // outlives restarts and deploys, and an unhandled rejection would kill it.
   let res: Response;
   try {
     res = await fetch(SERVER + path, {
@@ -121,27 +113,24 @@ async function call(path: string, body: unknown): Promise<{ ok: boolean; status:
   return { ok: res.ok, status: res.status, data };
 }
 
-/** The task input IS an EvalRequest: `code` required, `input` and `timeout_ms` optional. A task
- *  whose input is not that shape is the definition's fault, not the script's, and is reported
- *  as a compile_error — the nearest permanent kind, since no retry can fix the definition. */
-function asEvalRequest(input: unknown): EvalRequest | string {
+/** A task input that is not an EvalRequest is the definition's fault, reported as compile_error:
+ *  the nearest permanent kind, since no retry fixes the definition. */
+function asEvalRequest(input: unknown, timeoutMs: number | undefined): EvalRequest | string {
   if (typeof input !== "object" || input === null) return "the task input is not an object";
   const r = input as Record<string, unknown>;
   if (typeof r.code !== "string") return "the task input has no `code` string";
-  return {
-    code: r.code,
-    input: r.input,
-    timeout_ms: typeof r.timeout_ms === "number" ? r.timeout_ms : undefined,
-  };
+  return { code: r.code, input: r.input, timeout_ms: timeoutMs };
 }
 
-const inFlight = new Map<string, QueueTask>();
+/** genroc cannot call us, so a cancellation arrives on our own heartbeat and reaches the
+ *  evaluation through the controller. */
+type Running = { job: QueueTask; abort: AbortController };
+
+const inFlight = new Map<string, Running>();
 let running = true;
 
-// Whether the last claim reached genroc. A worker polls several times a second, so an
-// unreachable server would otherwise emit a line per poll — thousands during a restart, which
-// buries the one line that mattered. Announce the TRANSITIONS instead: going away, and coming
-// back. Silence in between is the report that nothing changed.
+// Announce reachability TRANSITIONS only: polling several times a second would otherwise emit
+// thousands of lines during a restart.
 let serverReachable = true;
 
 async function claim(n: number): Promise<QueueTask[]> {
@@ -149,13 +138,12 @@ async function claim(n: number): Promise<QueueTask[]> {
     worker_id: WORKER_ID,
     limit: n,
     lease_ms: LEASE_MS,
-    ...(PROCESS_FILTER ? { process: PROCESS_FILTER } : {}),
-    ...(TASK_FILTER ? { task: TASK_FILTER } : {}),
+    process: PROCESS_FILTER,
+    task: TASK_FILTER,
   });
   if (!ok) {
-    // A credential problem is not transient, and polling through it looks like a healthy
-    // worker that never picks anything up — the worst shape for an operator to debug. Exit
-    // instead, so a supervisor restarts it and the failure is visible where it happened.
+    // A credential problem is not transient, and polling through it looks like a healthy idle
+    // worker. Exit, so a supervisor restarts it visibly.
     if (status === 401 || status === 403) {
       console.error(
         `claim rejected (${status}): ${JSON.stringify(data)}\n` +
@@ -164,9 +152,7 @@ async function claim(n: number): Promise<QueueTask[]> {
       );
       process.exit(1);
     }
-    // status 0 is "never reached the server" (see call): a restart, a rolling deploy, a
-    // network blip. Not an error to act on — the next poll clears it — so it is reported once
-    // and then waited out.
+    // status 0 (see call): a restart or blip the next poll clears, so it is reported once.
     if (status === 0) {
       if (serverReachable) {
         serverReachable = false;
@@ -192,10 +178,8 @@ async function release(token: string): Promise<void> {
   if (!ok) console.error(`release failed: ${JSON.stringify(data)}`);
 }
 
-/** answer submits the outcome. A refusal is NOT retried with a different one: the definition
- *  declared a contract this worker does not satisfy (an undeclared code, a payload that does
- *  not fit `raises`), and guessing again would only pick a second wrong answer. Release it, so
- *  the task returns to the queue and an operator sees it waiting rather than silently gone. */
+/** A refused outcome is not retried with another: the worker does not satisfy the declared
+ *  contract. Released, so an operator sees the task waiting rather than gone. */
 async function answer(token: string, outcome: Record<string, unknown>): Promise<void> {
   const { ok, data } = await call("/api/external-tasks/resolve", { token, ...outcome });
   if (ok) return;
@@ -203,7 +187,10 @@ async function answer(token: string, outcome: Record<string, unknown>): Promise<
   await release(token);
 }
 
-async function run(job: QueueTask): Promise<void> {
+async function run(job: QueueTask, signal: AbortSignal): Promise<void> {
+  // The task's own timeout is the script's budget, moved onto the monotonic clock so a wall-clock
+  // step mid-run cannot stretch it. Fetching the objects below spends the same deadline.
+  const due = job.deadline === undefined ? undefined : performance.now() + (Date.parse(job.deadline) - Date.now());
   let resolved: unknown;
   try {
     resolved = await resolveObjects(job);
@@ -214,7 +201,16 @@ async function run(job: QueueTask): Promise<void> {
     await release(job.token);
     return;
   }
-  const req = asEvalRequest(resolved);
+  let timeoutMs: number | undefined;
+  if (due !== undefined) {
+    timeoutMs = Math.floor(due - performance.now() - ANSWER_MARGIN_MS);
+    if (timeoutMs <= 0) {
+      // Too late to run anything: the deadline fires as external.timeout, which is the truth.
+      await release(job.token);
+      return;
+    }
+  }
+  const req = asEvalRequest(resolved, timeoutMs);
   if (typeof req === "string") {
     await answer(job.token, {
       error: { code: "compile_error", message: req, data: { name: "BadTaskInput" } },
@@ -224,12 +220,17 @@ async function run(job: QueueTask): Promise<void> {
 
   let result;
   try {
-    result = await evaluate(req);
+    result = await evaluate(req, signal);
   } catch (err) {
-    // The RUNNER faulted, not the script — the one class where a retry can help. There is no
-    // error code for it on purpose: releasing the claim is how a queue spells "retryable", and
-    // it puts the task in front of a different worker instead of burning the definition's
-    // on_error budget on this one's bad day.
+    // Cancelled is not a fault. Still released: the row is terminal so nothing re-claims it,
+    // but the release stops it waiting out a lease nobody serves.
+    if (err instanceof Cancelled) {
+      console.log(`cancelled: ${job.token}`);
+      await release(job.token);
+      return;
+    }
+    // The RUNNER faulted, the one retryable class. Releasing is how a queue spells "retryable",
+    // and it reaches another worker without spending the definition's on_error budget.
     console.error(`evaluator fault on ${job.token}: ${err instanceof Error ? err.message : String(err)}`);
     await release(job.token);
     return;
@@ -263,11 +264,18 @@ async function renewLoop(): Promise<void> {
       tokens,
       lease_ms: LEASE_MS,
     });
-    // A short count means a claim lapsed and was taken over. Nothing to do about it — the work
-    // continues and its answer will be refused — but say so, because it is the signal that
-    // LEASE_MS is too short for what these scripts actually take.
-    if (ok && data?.renewed < tokens.length) {
-      console.error(`renewed ${data.renewed}/${tokens.length} claims; a lease lapsed under load`);
+    if (!ok) continue;
+
+    // Abort first rather than let a stopped script burn a core; the release rides run()'s
+    // catch, so nothing is released twice.
+    for (const token of (data?.cancelled ?? []) as string[]) {
+      inFlight.get(token)?.abort.abort();
+    }
+    // Lost: another worker holds it now and our answer will be refused; logged as the sign that
+    // LEASE_MS is too short. NOT released: that would bump the epoch out from under the holder.
+    const lost = (data?.lost ?? []) as string[];
+    if (lost.length) {
+      console.error(`lost ${lost.length}/${tokens.length} claims; a lease lapsed under load`);
     }
   }
 }
@@ -277,8 +285,9 @@ async function pollLoop(): Promise<void> {
     const free = CONCURRENCY - inFlight.size;
     const jobs = free > 0 ? await claim(free) : [];
     for (const job of jobs) {
-      inFlight.set(job.token, job);
-      void run(job).finally(() => inFlight.delete(job.token));
+      const abort = new AbortController();
+      inFlight.set(job.token, { job, abort });
+      void run(job, abort.signal).finally(() => inFlight.delete(job.token));
     }
     // Only idle when there was nothing to take: a full queue should be drained at the speed the
     // realms allow, not at the poll interval.