@genroc/eval-node 0.0.0-edge.dc4ff88 → 0.0.0-edge.dcfec388

package/dist/worker.js

@@ -1,50 +1,38 @@
 #!/usr/bin/env node
-// The queue worker: claims parked `external` script tasks from genroc, evaluates each in its
-// own realm, and answers. This is the whole genroc-facing half — eval.ts and realm.ts know
-// nothing about the queue, which is what keeps the containment strategy swappable.
-//
-// See README.md for the contract, and specs/external-task-queue.md for the queue itself.
+// The queue worker: claims parked `external` script tasks, evaluates each in its own realm, and
+// answers. The only genroc-facing half (README.md; specs/external-task-queue.md).
 import { readFileSync } from "node:fs";
-import { evaluate } from "./eval.js";
+import { Cancelled, evaluate } from "./eval.js";
 const SERVER = (process.env.GENROC_SERVER ?? "http://localhost:8448").replace(/\/$/, "");
 const WORKER_ID = process.env.WORKER_ID ?? `evaluator-${process.pid}`;
-// The credential, when the server runs with --auth token. A worker needs exactly the `worker`
-// permission — the four queue verbs plus GET /api/objects — so mint it scoped rather than
-// handing a worker an admin token: this is the credential most likely to sit on a machine you
-// trust least. specs/api-auth.md §5.
-//
-// Sent as a header rather than in the URL because Node's fetch REFUSES a URL carrying
-// credentials ("Request cannot be constructed from a URL that includes credentials"), so the
-// basic-auth-in-the-URL trick that works for genctl is not available here.
-// GENROC_TOKEN_FILE is the mounted-secret shape: a credential in a file rather than an
-// environment variable, so it stays out of `docker inspect` and out of the process environment
-// any child inherits. The inline variable wins when both are set.
+// Needs only the `worker` permission (specs/api-auth.md §5): this credential sits on the machine
+// you trust least. A header, because Node's fetch refuses credentials in a URL. The file form
+// keeps it out of `docker inspect` and child environments; the inline variable wins.
 const TOKEN = process.env.GENROC_TOKEN ??
     (process.env.GENROC_TOKEN_FILE ? readFileSync(process.env.GENROC_TOKEN_FILE, "utf8").trim() : "");
 const authHeaders = TOKEN ? { authorization: `Bearer ${TOKEN}` } : {};
-// Concurrency is the worker's to set, and that is the point of pulling: under the old fetch
-// shape genroc decided how many scripts ran at once (--max-concurrent, default 200) and the
-// evaluator accepted every one of them. Here it claims what it can run and no more, so a
-// backlog is a queue rather than 200 threads fighting over a core.
+// The worker claims only what it can run, so a backlog is a queue rather than threads fighting
+// over a core.
 const CONCURRENCY = Number(process.env.CONCURRENCY ?? 4);
 const POLL_MS = Number(process.env.POLL_MS ?? 250);
 // The visibility timeout. Short, and renewed while work is in flight: a worker that dies
 // should return its task quickly rather than holding it for the whole budget.
 const LEASE_MS = Number(process.env.LEASE_MS ?? 30_000);
 const RENEW_MS = Math.max(1_000, Math.floor(LEASE_MS / 3));
+// The answer has to land before the task's deadline fires, or a script that overran surfaces as
+// external.timeout ("nobody answered") rather than as its own `timeout`.
+const ANSWER_MARGIN_MS = 250;
 const PROCESS_FILTER = process.env.PROCESS ?? "";
 const TASK_FILTER = process.env.TASK ?? "";
-// Values too large to ship inline are listed rather than carried, and a bundle is exactly that:
-// one object shared by every instance of a definition version, fetched once instead of copied
-// into each task. A ref is a content hash, so it is immutable — the cache never invalidates.
+// Large values arrive as refs; a bundle is one object per definition version, fetched once.
+// Refs are content hashes, so the cache never invalidates.
 const objectCache = new Map();
 async function fetchObject(ref) {
     const cached = objectCache.get(ref);
     if (cached !== undefined)
         return cached;
-    // Left to throw on purpose: this runs while a task is IN FLIGHT, and a task whose input
-    // cannot be fetched must fail rather than silently run against a missing value. The caller
-    // releases the claim, so the task returns to the queue.
+    // Left to throw: a task whose input cannot be fetched must not run against a missing value.
+    // The caller releases the claim.
     const res = await fetch(`${SERVER}/api/objects/${encodeURIComponent(ref)}`, { headers: authHeaders });
     if (!res.ok)
         throw new Error(`fetch object ${ref}: HTTP ${res.status}`);
@@ -62,15 +50,15 @@ async function fetchObject(ref) {
 /** Put each listed value back where its path says it belongs. Paths are arrays of keys, so this
  *  needs no parser: the whole reason they are not JSON Pointer strings. */
 async function resolveObjects(job) {
-    let input = job.input;
+    let input = job.external_input;
     for (const e of job.objects ?? []) {
         const value = await fetchObject(e.ref);
         if (e.path.length === 0) {
             input = value;
             continue;
         }
-        // The path is rooted at the entry and starts with "input", which is the value being rebuilt.
-        const rest = e.path[0] === "input" ? e.path.slice(1) : e.path;
+        // The path is rooted at the entry and starts with "external_input", the value being rebuilt.
+        const rest = e.path[0] === "external_input" ? e.path.slice(1) : e.path;
         if (rest.length === 0) {
             input = value;
             continue;
@@ -84,10 +72,8 @@ async function resolveObjects(job) {
     return input;
 }
 async function call(path, body) {
-    // A network error is a REPLY, not a throw. A worker outlives the server it polls — a
-    // restart, a rolling deploy, a container coming up before genroc is listening — and an
-    // unhandled rejection here kills it for a condition the next poll would clear. Status 0
-    // says "never reached the server", which is distinct from anything genroc answers.
+    // A network error is a REPLY (status 0, "never reached the server"), not a throw: a worker
+    // outlives restarts and deploys, and an unhandled rejection would kill it.
     let res;
     try {
         res = await fetch(SERVER + path, {
@@ -109,27 +95,20 @@ async function call(path, body) {
     }
     return { ok: res.ok, status: res.status, data };
 }
-/** The task input IS an EvalRequest: `code` required, `input` and `timeout_ms` optional. A task
- *  whose input is not that shape is the definition's fault, not the script's, and is reported
- *  as a compile_error — the nearest permanent kind, since no retry can fix the definition. */
-function asEvalRequest(input) {
+/** A task input that is not an EvalRequest is the definition's fault, reported as compile_error:
+ *  the nearest permanent kind, since no retry fixes the definition. */
+function asEvalRequest(input, timeoutMs) {
     if (typeof input !== "object" || input === null)
         return "the task input is not an object";
     const r = input;
     if (typeof r.code !== "string")
         return "the task input has no `code` string";
-    return {
-        code: r.code,
-        input: r.input,
-        timeout_ms: typeof r.timeout_ms === "number" ? r.timeout_ms : undefined,
-    };
+    return { code: r.code, input: r.input, timeout_ms: timeoutMs };
 }
 const inFlight = new Map();
 let running = true;
-// Whether the last claim reached genroc. A worker polls several times a second, so an
-// unreachable server would otherwise emit a line per poll — thousands during a restart, which
-// buries the one line that mattered. Announce the TRANSITIONS instead: going away, and coming
-// back. Silence in between is the report that nothing changed.
+// Announce reachability TRANSITIONS only: polling several times a second would otherwise emit
+// thousands of lines during a restart.
 let serverReachable = true;
 async function claim(n) {
     const { ok, status, data } = await call("/api/external-tasks/claim", {
@@ -140,18 +119,15 @@ async function claim(n) {
         ...(TASK_FILTER ? { task: TASK_FILTER } : {}),
     });
     if (!ok) {
-        // A credential problem is not transient, and polling through it looks like a healthy
-        // worker that never picks anything up — the worst shape for an operator to debug. Exit
-        // instead, so a supervisor restarts it and the failure is visible where it happened.
+        // A credential problem is not transient, and polling through it looks like a healthy idle
+        // worker. Exit, so a supervisor restarts it visibly.
         if (status === 401 || status === 403) {
             console.error(`claim rejected (${status}): ${JSON.stringify(data)}\n` +
                 `The server requires authentication. Set GENROC_TOKEN to a token with the 'worker' ` +
                 `permission — mint one with: genctl token create --perms worker --label evaluator -q`);
             process.exit(1);
         }
-        // status 0 is "never reached the server" (see call): a restart, a rolling deploy, a
-        // network blip. Not an error to act on — the next poll clears it — so it is reported once
-        // and then waited out.
+        // status 0 (see call): a restart or blip the next poll clears, so it is reported once.
         if (status === 0) {
             if (serverReachable) {
                 serverReachable = false;
@@ -174,10 +150,8 @@ async function release(token) {
     if (!ok)
         console.error(`release failed: ${JSON.stringify(data)}`);
 }
-/** answer submits the outcome. A refusal is NOT retried with a different one: the definition
- *  declared a contract this worker does not satisfy (an undeclared code, a payload that does
- *  not fit `raises`), and guessing again would only pick a second wrong answer. Release it, so
- *  the task returns to the queue and an operator sees it waiting rather than silently gone. */
+/** A refused outcome is not retried with another: the worker does not satisfy the declared
+ *  contract. Released, so an operator sees the task waiting rather than gone. */
 async function answer(token, outcome) {
     const { ok, data } = await call("/api/external-tasks/resolve", { token, ...outcome });
     if (ok)
@@ -185,7 +159,10 @@ async function answer(token, outcome) {
     console.error(`genroc refused the outcome for ${token}: ${JSON.stringify(data)}`);
     await release(token);
 }
-async function run(job) {
+async function run(job, signal) {
+    // The task's own timeout is the script's budget. Taken on arrival, because fetching the
+    // objects below spends the same deadline.
+    const due = job.deadline_in_ms === undefined ? undefined : performance.now() + job.deadline_in_ms;
     let resolved;
     try {
         resolved = await resolveObjects(job);
@@ -197,7 +174,16 @@ async function run(job) {
         await release(job.token);
         return;
     }
-    const req = asEvalRequest(resolved);
+    let timeoutMs;
+    if (due !== undefined) {
+        timeoutMs = Math.floor(due - performance.now() - ANSWER_MARGIN_MS);
+        if (timeoutMs <= 0) {
+            // Too late to run anything: the deadline fires as external.timeout, which is the truth.
+            await release(job.token);
+            return;
+        }
+    }
+    const req = asEvalRequest(resolved, timeoutMs);
     if (typeof req === "string") {
         await answer(job.token, {
             error: { code: "compile_error", message: req, data: { name: "BadTaskInput" } },
@@ -206,13 +192,18 @@ async function run(job) {
     }
     let result;
     try {
-        result = await evaluate(req);
+        result = await evaluate(req, signal);
     }
     catch (err) {
-        // The RUNNER faulted, not the script — the one class where a retry can help. There is no
-        // error code for it on purpose: releasing the claim is how a queue spells "retryable", and
-        // it puts the task in front of a different worker instead of burning the definition's
-        // on_error budget on this one's bad day.
+        // Cancelled is not a fault. Still released: the row is terminal so nothing re-claims it,
+        // but the release stops it waiting out a lease nobody serves.
+        if (err instanceof Cancelled) {
+            console.log(`cancelled: ${job.token}`);
+            await release(job.token);
+            return;
+        }
+        // The RUNNER faulted, the one retryable class. Releasing is how a queue spells "retryable",
+        // and it reaches another worker without spending the definition's on_error budget.
         console.error(`evaluator fault on ${job.token}: ${err instanceof Error ? err.message : String(err)}`);
         await release(job.token);
         return;
@@ -245,11 +236,18 @@ async function renewLoop() {
             tokens,
             lease_ms: LEASE_MS,
         });
-        // A short count means a claim lapsed and was taken over. Nothing to do about it — the work
-        // continues and its answer will be refused — but say so, because it is the signal that
-        // LEASE_MS is too short for what these scripts actually take.
-        if (ok && data?.renewed < tokens.length) {
-            console.error(`renewed ${data.renewed}/${tokens.length} claims; a lease lapsed under load`);
+        if (!ok)
+            continue;
+        // Abort first rather than let a stopped script burn a core; the release rides run()'s
+        // catch, so nothing is released twice.
+        for (const token of (data?.cancelled ?? [])) {
+            inFlight.get(token)?.abort.abort();
+        }
+        // Lost: another worker holds it now and our answer will be refused; logged as the sign that
+        // LEASE_MS is too short. NOT released: that would bump the epoch out from under the holder.
+        const lost = (data?.lost ?? []);
+        if (lost.length) {
+            console.error(`lost ${lost.length}/${tokens.length} claims; a lease lapsed under load`);
         }
     }
 }
@@ -258,8 +256,9 @@ async function pollLoop() {
         const free = CONCURRENCY - inFlight.size;
         const jobs = free > 0 ? await claim(free) : [];
         for (const job of jobs) {
-            inFlight.set(job.token, job);
-            void run(job).finally(() => inFlight.delete(job.token));
+            const abort = new AbortController();
+            inFlight.set(job.token, { job, abort });
+            void run(job, abort.signal).finally(() => inFlight.delete(job.token));
         }
         // Only idle when there was nothing to take: a full queue should be drained at the speed the
         // realms allow, not at the poll interval.

package/eval.ts

@@ -1,11 +1,6 @@
-// Evaluation core: import a script module in its OWN realm, call its default export, and
-// classify every outcome into one of the failure kinds in README.md. Nothing here knows about
-// genroc — worker.ts is the only thing that talks to the queue — so this stays testable and
-// the containment stays swappable.
-//
-// The containment is a Worker per execution (realm.ts). It is what makes the budget real:
-// a synchronous busy loop never yields, so no in-process timer can interrupt it, and only a
-// thread the host can kill bounds it.
+// Imports a script module in its own realm (a Worker per execution, realm.ts), calls its default
+// export, and classifies every outcome (README.md). Only a killable thread bounds a synchronous
+// busy loop, which is what makes the budget real. Knows nothing of genroc; worker.ts does.
 
 import { Worker } from "node:worker_threads";
 
@@ -16,9 +11,8 @@ export type EvalRequest = {
   timeout_ms?: number;
 };
 
-/** Every kind here is PERMANENT: a retry re-runs the same code on the same input and fails
- *  identically. The retryable class has no kind because it is not an outcome — a runner that
- *  faults releases its claim and lets another worker take the task. */
+/** Every kind is PERMANENT: a retry fails identically. A faulting runner is not an outcome; it
+ *  releases its claim. */
 export type FailureKind = "compile_error" | "threw" | "timeout" | "nonserializable" | "exited";
 
 export type EvalFailure = {
@@ -28,9 +22,8 @@ export type EvalFailure = {
   stack?: string;
 };
 
-/** `body` is JSON TEXT, not a value: serialising in the realm is what makes a nonserializable
- *  return a script fault rather than a 500 thrown out of the response path. It is also what
- *  crosses the worker boundary — structured clone would refuse a different set of values. */
+/** JSON TEXT serialised in the realm, so a nonserializable return is a script fault, not a 500,
+ *  and structured clone (which refuses a different set of values) never sees the value. */
 export type EvalResult =
   | { ok: true; body: string }
   | { ok: false; failure: EvalFailure };
@@ -47,9 +40,8 @@ const REALM_URL = new URL(
   import.meta.url,
 );
 
-/** Thrown, not returned: a realm that fails to start is the RUNNER faulting, which worker.ts
- *  answers by releasing the claim rather than by reporting an outcome. A script fault is a
- *  return value. */
+/** Thrown, not returned: a realm that fails to start is the runner faulting, which worker.ts
+ *  answers by releasing the claim. */
 class RealmFault extends Error {
   constructor(message: string) {
     super(message);
@@ -57,24 +49,41 @@ class RealmFault extends Error {
   }
 }
 
-export async function evaluate(req: EvalRequest): Promise<EvalResult> {
+/** Thrown when `signal` aborts: the answer is no longer wanted, so worker.ts releases rather
+ *  than answers. Not a RealmFault: nothing is wrong. */
+export class Cancelled extends Error {
+  constructor() {
+    super("cancelled");
+    this.name = "Cancelled";
+  }
+}
+
+export async function evaluate(req: EvalRequest, signal?: AbortSignal): Promise<EvalResult> {
   const budget = typeof req.timeout_ms === "number" ? req.timeout_ms : DEFAULT_TIMEOUT_MS;
 
   const worker = new Worker(REALM_URL);
   let timer: ReturnType<typeof setTimeout> | undefined;
+  let onAbort: (() => void) | undefined;
   try {
     return await new Promise<EvalResult>((resolve, reject) => {
       timer = setTimeout(() => resolve(timedOut(budget)), budget);
+      // The abort is wired to the same promise as the budget, so both settle through the one
+      // finally below -- which is what guarantees the thread is gone before either returns.
+      if (signal) {
+        if (signal.aborted) reject(new Cancelled());
+        onAbort = () => reject(new Cancelled());
+        signal.addEventListener("abort", onAbort, { once: true });
+      }
       worker.once("message", (reply: WorkerReply) => resolve(reply));
-      // A script may end its own realm (`process.exit()`), which is not a throw and would
-      // otherwise present as a hang until the budget expired. Our own terminate() raises
-      // this too, by which time the promise has settled and the first result stands.
+      // A script may `process.exit()`, which would otherwise look like a hang until the budget.
+      // Our own terminate() fires this too, after the promise has settled.
       worker.once("exit", (code: number) => resolve(exited(code)));
       worker.once("error", (err: Error) => reject(new RealmFault(errorText(err))));
       worker.postMessage({ code: req.code, input: req.input } satisfies WorkerRequest);
     });
   } finally {
     clearTimeout(timer);
+    if (signal && onAbort) signal.removeEventListener("abort", onAbort);
     // Awaited, and the whole point: on the timeout path a thread is still burning a core, and
     // resolving before it is gone would report an evaluation the machine is still running.
     await worker.terminate();