@genroc/eval-node 0.0.0-edge.f85ea85 → 0.0.0-edge.f97de45

package/dist/worker.js

@@ -0,0 +1,305 @@
+#!/usr/bin/env node
+// The queue worker: claims parked `external` script tasks from genroc, evaluates each in its
+// own realm, and answers. This is the whole genroc-facing half — eval.ts and realm.ts know
+// nothing about the queue, which is what keeps the containment strategy swappable.
+//
+// See README.md for the contract, and specs/external-task-queue.md for the queue itself.
+import { readFileSync } from "node:fs";
+import { Cancelled, evaluate } from "./eval.js";
+const SERVER = (process.env.GENROC_SERVER ?? "http://localhost:8448").replace(/\/$/, "");
+const WORKER_ID = process.env.WORKER_ID ?? `evaluator-${process.pid}`;
+// The credential, when the server runs with --auth token. A worker needs exactly the `worker`
+// permission — the four queue verbs plus GET /api/objects — so mint it scoped rather than
+// handing a worker an admin token: this is the credential most likely to sit on a machine you
+// trust least. specs/api-auth.md §5.
+//
+// Sent as a header rather than in the URL because Node's fetch REFUSES a URL carrying
+// credentials ("Request cannot be constructed from a URL that includes credentials"), so the
+// basic-auth-in-the-URL trick that works for genctl is not available here.
+// GENROC_TOKEN_FILE is the mounted-secret shape: a credential in a file rather than an
+// environment variable, so it stays out of `docker inspect` and out of the process environment
+// any child inherits. The inline variable wins when both are set.
+const TOKEN = process.env.GENROC_TOKEN ??
+    (process.env.GENROC_TOKEN_FILE ? readFileSync(process.env.GENROC_TOKEN_FILE, "utf8").trim() : "");
+const authHeaders = TOKEN ? { authorization: `Bearer ${TOKEN}` } : {};
+// Concurrency is the worker's to set, and that is the point of pulling: under the old fetch
+// shape genroc decided how many scripts ran at once (--max-concurrent, default 200) and the
+// evaluator accepted every one of them. Here it claims what it can run and no more, so a
+// backlog is a queue rather than 200 threads fighting over a core.
+const CONCURRENCY = Number(process.env.CONCURRENCY ?? 4);
+const POLL_MS = Number(process.env.POLL_MS ?? 250);
+// The visibility timeout. Short, and renewed while work is in flight: a worker that dies
+// should return its task quickly rather than holding it for the whole budget.
+const LEASE_MS = Number(process.env.LEASE_MS ?? 30_000);
+const RENEW_MS = Math.max(1_000, Math.floor(LEASE_MS / 3));
+const PROCESS_FILTER = process.env.PROCESS ?? "";
+const TASK_FILTER = process.env.TASK ?? "";
+// Values too large to ship inline are listed rather than carried, and a bundle is exactly that:
+// one object shared by every instance of a definition version, fetched once instead of copied
+// into each task. A ref is a content hash, so it is immutable — the cache never invalidates.
+const objectCache = new Map();
+async function fetchObject(ref) {
+    const cached = objectCache.get(ref);
+    if (cached !== undefined)
+        return cached;
+    // Left to throw on purpose: this runs while a task is IN FLIGHT, and a task whose input
+    // cannot be fetched must fail rather than silently run against a missing value. The caller
+    // releases the claim, so the task returns to the queue.
+    const res = await fetch(`${SERVER}/api/objects/${encodeURIComponent(ref)}`, { headers: authHeaders });
+    if (!res.ok)
+        throw new Error(`fetch object ${ref}: HTTP ${res.status}`);
+    const { data } = (await res.json());
+    let value;
+    try {
+        value = JSON.parse(data);
+    }
+    catch {
+        value = data;
+    }
+    objectCache.set(ref, value);
+    return value;
+}
+/** Put each listed value back where its path says it belongs. Paths are arrays of keys, so this
+ *  needs no parser: the whole reason they are not JSON Pointer strings. */
+async function resolveObjects(job) {
+    let input = job.input;
+    for (const e of job.objects ?? []) {
+        const value = await fetchObject(e.ref);
+        if (e.path.length === 0) {
+            input = value;
+            continue;
+        }
+        // The path is rooted at the entry and starts with "input", which is the value being rebuilt.
+        const rest = e.path[0] === "input" ? e.path.slice(1) : e.path;
+        if (rest.length === 0) {
+            input = value;
+            continue;
+        }
+        let cur = input;
+        for (let i = 0; i < rest.length - 1; i++)
+            cur = cur?.[rest[i]];
+        if (cur)
+            cur[rest[rest.length - 1]] = value;
+    }
+    return input;
+}
+async function call(path, body) {
+    // A network error is a REPLY, not a throw. A worker outlives the server it polls — a
+    // restart, a rolling deploy, a container coming up before genroc is listening — and an
+    // unhandled rejection here kills it for a condition the next poll would clear. Status 0
+    // says "never reached the server", which is distinct from anything genroc answers.
+    let res;
+    try {
+        res = await fetch(SERVER + path, {
+            method: "POST",
+            headers: { "content-type": "application/json", ...authHeaders },
+            body: JSON.stringify(body),
+        });
+    }
+    catch (err) {
+        return { ok: false, status: 0, data: { error: `${SERVER} unreachable: ${err.message}` } };
+    }
+    const text = await res.text();
+    let data = null;
+    try {
+        data = text ? JSON.parse(text) : null;
+    }
+    catch {
+        data = { error: text };
+    }
+    return { ok: res.ok, status: res.status, data };
+}
+/** The task input IS an EvalRequest: `code` required, `input` and `timeout_ms` optional. A task
+ *  whose input is not that shape is the definition's fault, not the script's, and is reported
+ *  as a compile_error — the nearest permanent kind, since no retry can fix the definition. */
+function asEvalRequest(input) {
+    if (typeof input !== "object" || input === null)
+        return "the task input is not an object";
+    const r = input;
+    if (typeof r.code !== "string")
+        return "the task input has no `code` string";
+    return {
+        code: r.code,
+        input: r.input,
+        timeout_ms: typeof r.timeout_ms === "number" ? r.timeout_ms : undefined,
+    };
+}
+const inFlight = new Map();
+let running = true;
+// Whether the last claim reached genroc. A worker polls several times a second, so an
+// unreachable server would otherwise emit a line per poll — thousands during a restart, which
+// buries the one line that mattered. Announce the TRANSITIONS instead: going away, and coming
+// back. Silence in between is the report that nothing changed.
+let serverReachable = true;
+async function claim(n) {
+    const { ok, status, data } = await call("/api/external-tasks/claim", {
+        worker_id: WORKER_ID,
+        limit: n,
+        lease_ms: LEASE_MS,
+        ...(PROCESS_FILTER ? { process: PROCESS_FILTER } : {}),
+        ...(TASK_FILTER ? { task: TASK_FILTER } : {}),
+    });
+    if (!ok) {
+        // A credential problem is not transient, and polling through it looks like a healthy
+        // worker that never picks anything up — the worst shape for an operator to debug. Exit
+        // instead, so a supervisor restarts it and the failure is visible where it happened.
+        if (status === 401 || status === 403) {
+            console.error(`claim rejected (${status}): ${JSON.stringify(data)}\n` +
+                `The server requires authentication. Set GENROC_TOKEN to a token with the 'worker' ` +
+                `permission — mint one with: genctl token create --perms worker --label evaluator -q`);
+            process.exit(1);
+        }
+        // status 0 is "never reached the server" (see call): a restart, a rolling deploy, a
+        // network blip. Not an error to act on — the next poll clears it — so it is reported once
+        // and then waited out.
+        if (status === 0) {
+            if (serverReachable) {
+                serverReachable = false;
+                console.error(`genroc at ${SERVER} is unreachable — ${data?.error ?? ""}. ` +
+                    `Still polling every ${POLL_MS}ms; work resumes when it comes back.`);
+            }
+            return [];
+        }
+        console.error(`claim failed: ${JSON.stringify(data)}`);
+        return [];
+    }
+    if (!serverReachable) {
+        serverReachable = true;
+        console.error(`genroc at ${SERVER} is reachable again — resuming.`);
+    }
+    return (data?.items ?? []);
+}
+async function release(token) {
+    const { ok, data } = await call("/api/external-tasks/release", { token });
+    if (!ok)
+        console.error(`release failed: ${JSON.stringify(data)}`);
+}
+/** answer submits the outcome. A refusal is NOT retried with a different one: the definition
+ *  declared a contract this worker does not satisfy (an undeclared code, a payload that does
+ *  not fit `raises`), and guessing again would only pick a second wrong answer. Release it, so
+ *  the task returns to the queue and an operator sees it waiting rather than silently gone. */
+async function answer(token, outcome) {
+    const { ok, data } = await call("/api/external-tasks/resolve", { token, ...outcome });
+    if (ok)
+        return;
+    console.error(`genroc refused the outcome for ${token}: ${JSON.stringify(data)}`);
+    await release(token);
+}
+async function run(job, signal) {
+    let resolved;
+    try {
+        resolved = await resolveObjects(job);
+    }
+    catch (err) {
+        // The values are there or they are not; this is the runner failing to read them, not the
+        // script failing, so hand the task back for someone else rather than reporting an outcome.
+        console.error(`resolving objects for ${job.token}: ${err instanceof Error ? err.message : String(err)}`);
+        await release(job.token);
+        return;
+    }
+    const req = asEvalRequest(resolved);
+    if (typeof req === "string") {
+        await answer(job.token, {
+            error: { code: "compile_error", message: req, data: { name: "BadTaskInput" } },
+        });
+        return;
+    }
+    let result;
+    try {
+        result = await evaluate(req, signal);
+    }
+    catch (err) {
+        // Cancelled is not a fault: the process was stopped while this ran. The claim still goes
+        // back — the row is terminal, so nothing re-claims it, and the release is what stops it
+        // waiting out a lease nobody is serving.
+        if (err instanceof Cancelled) {
+            console.log(`cancelled: ${job.token}`);
+            await release(job.token);
+            return;
+        }
+        // The RUNNER faulted, not the script — the one class where a retry can help. There is no
+        // error code for it on purpose: releasing the claim is how a queue spells "retryable", and
+        // it puts the task in front of a different worker instead of burning the definition's
+        // on_error budget on this one's bad day.
+        console.error(`evaluator fault on ${job.token}: ${err instanceof Error ? err.message : String(err)}`);
+        await release(job.token);
+        return;
+    }
+    if (result.ok) {
+        // `body` is JSON text produced inside the realm; an empty body is a script that returned
+        // nothing, which genroc reads as null.
+        await answer(job.token, { result: result.body === "" ? null : JSON.parse(result.body) });
+        return;
+    }
+    const f = result.failure;
+    await answer(job.token, {
+        error: {
+            // The failure KIND is the code, so an on_error rule branches on what went wrong without
+            // reading a payload. Every kind is permanent; see eval.ts.
+            code: f.kind,
+            message: f.message,
+            data: { name: f.name, ...(f.stack ? { stack: f.stack } : {}) },
+        },
+    });
+}
+async function renewLoop() {
+    while (running) {
+        await new Promise((r) => setTimeout(r, RENEW_MS));
+        const tokens = [...inFlight.keys()];
+        if (!tokens.length)
+            continue;
+        const { ok, data } = await call("/api/external-tasks/renew", {
+            worker_id: WORKER_ID,
+            tokens,
+            lease_ms: LEASE_MS,
+        });
+        if (!ok)
+            continue;
+        // Cancelled: the process was stopped, so the evaluation is abandoned and the claim handed
+        // back. Aborting first is the point of the list — waiting for the script to finish would
+        // keep burning a core on work an operator has already stopped. The release rides run()'s
+        // own catch, so nothing is released twice.
+        for (const token of (data?.cancelled ?? [])) {
+            inFlight.get(token)?.abort.abort();
+        }
+        // Lost: someone else holds this claim now. The work continues and its answer will be
+        // refused — that is not fixable here — but say so, because it is the signal that LEASE_MS
+        // is too short for what these scripts actually take. Deliberately NOT released: the claim
+        // is the new holder's, and releasing would bump the epoch out from under it.
+        const lost = (data?.lost ?? []);
+        if (lost.length) {
+            console.error(`lost ${lost.length}/${tokens.length} claims; a lease lapsed under load`);
+        }
+    }
+}
+async function pollLoop() {
+    while (running) {
+        const free = CONCURRENCY - inFlight.size;
+        const jobs = free > 0 ? await claim(free) : [];
+        for (const job of jobs) {
+            const abort = new AbortController();
+            inFlight.set(job.token, { job, abort });
+            void run(job, abort.signal).finally(() => inFlight.delete(job.token));
+        }
+        // Only idle when there was nothing to take: a full queue should be drained at the speed the
+        // realms allow, not at the poll interval.
+        if (jobs.length === 0)
+            await new Promise((r) => setTimeout(r, POLL_MS));
+    }
+}
+async function shutdown(signal) {
+    if (!running)
+        return;
+    running = false;
+    const tokens = [...inFlight.keys()];
+    console.log(`${signal}: releasing ${tokens.length} claim(s)`);
+    // Hand work back rather than letting it sit out its lease. The evaluations still running are
+    // abandoned, which is exactly what the release says: nobody answered.
+    await Promise.all(tokens.map(release));
+    process.exit(0);
+}
+process.on("SIGINT", () => void shutdown("SIGINT"));
+process.on("SIGTERM", () => void shutdown("SIGTERM"));
+console.log(`evaluator worker ${WORKER_ID} polling ${SERVER} (concurrency=${CONCURRENCY}, lease=${LEASE_MS}ms)`);
+void renewLoop();
+void pollLoop();

package/eval.ts

@@ -1,6 +1,7 @@
-// Evaluation core: run a code string in its OWN realm and classify every outcome into one of
-// the failure kinds in README.md. Nothing here knows about genroc — worker.ts is the only
-// thing that talks to the queue — so this stays testable and the containment stays swappable.
+// Evaluation core: import a script module in its OWN realm, call its default export, and
+// classify every outcome into one of the failure kinds in README.md. Nothing here knows about
+// genroc — worker.ts is the only thing that talks to the queue — so this stays testable and
+// the containment stays swappable.
 //
 // The containment is a Worker per execution (realm.ts). It is what makes the budget real:
 // a synchronous busy loop never yields, so no in-process timer can interrupt it, and only a
@@ -9,6 +10,7 @@
 import { Worker } from "node:worker_threads";
 
 export type EvalRequest = {
+  /** An ES module whose default export is the function to run; `input` is its argument. */
   code: string;
   input?: unknown;
   timeout_ms?: number;
@@ -38,7 +40,12 @@ export type WorkerRequest = { code: string; input?: unknown };
 export type WorkerReply = EvalResult;
 
 const DEFAULT_TIMEOUT_MS = 5_000;
-const REALM_URL = new URL("./realm.ts", import.meta.url);
+// Resolved from THIS file's own extension: run from a checkout it is `.ts`, and from the
+// published package `.js`, because Node will not strip types under node_modules.
+const REALM_URL = new URL(
+  import.meta.url.endsWith(".ts") ? "./realm.ts" : "./realm.js",
+  import.meta.url,
+);
 
 /** Thrown, not returned: a realm that fails to start is the RUNNER faulting, which worker.ts
  *  answers by releasing the claim rather than by reporting an outcome. A script fault is a
@@ -50,14 +57,32 @@ class RealmFault extends Error {
   }
 }
 
-export async function evaluate(req: EvalRequest): Promise<EvalResult> {
+/** Thrown when `signal` aborts: the work was cancelled server-side, so there is no outcome to
+ *  report and worker.ts must release rather than answer. Distinct from RealmFault because the
+ *  runner did not fault -- nothing is wrong, the answer is simply no longer wanted. */
+export class Cancelled extends Error {
+  constructor() {
+    super("cancelled");
+    this.name = "Cancelled";
+  }
+}
+
+export async function evaluate(req: EvalRequest, signal?: AbortSignal): Promise<EvalResult> {
   const budget = typeof req.timeout_ms === "number" ? req.timeout_ms : DEFAULT_TIMEOUT_MS;
 
   const worker = new Worker(REALM_URL);
   let timer: ReturnType<typeof setTimeout> | undefined;
+  let onAbort: (() => void) | undefined;
   try {
     return await new Promise<EvalResult>((resolve, reject) => {
       timer = setTimeout(() => resolve(timedOut(budget)), budget);
+      // The abort is wired to the same promise as the budget, so both settle through the one
+      // finally below -- which is what guarantees the thread is gone before either returns.
+      if (signal) {
+        if (signal.aborted) reject(new Cancelled());
+        onAbort = () => reject(new Cancelled());
+        signal.addEventListener("abort", onAbort, { once: true });
+      }
       worker.once("message", (reply: WorkerReply) => resolve(reply));
       // A script may end its own realm (`process.exit()`), which is not a throw and would
       // otherwise present as a hang until the budget expired. Our own terminate() raises
@@ -68,6 +93,7 @@ export async function evaluate(req: EvalRequest): Promise<EvalResult> {
     });
   } finally {
     clearTimeout(timer);
+    if (signal && onAbort) signal.removeEventListener("abort", onAbort);
     // Awaited, and the whole point: on the timeout path a thread is still burning a core, and
     // resolving before it is gone would report an evaluation the machine is still running.
     await worker.terminate();