@foldrun/core 0.2.0 → 0.4.0

package/dist/src/provider-health.d.ts

@@ -0,0 +1,55 @@
+import { type ProviderSpec } from "./store.ts";
+export type HealthVerdict = "ok" | "credential" | "not-found" | "busy" | "unreachable" | "provider-error";
+export interface ProviderCheck {
+    /** How this provider is written in the file: its preset name, or its host. */
+    provider: string;
+    /** The model id actually asked for — what a run would send. */
+    model: string;
+    verdict: HealthVerdict;
+    ok: boolean;
+    status: number | null;
+    ms: number;
+    /** One sentence a person can act on, in the provider's own words where
+     *  they said anything useful. */
+    detail: string;
+}
+/**
+ * One request at the provider, in the wire format it speaks, asking for a
+ * single token. Never throws: an unreachable host is a verdict, not an
+ * exception, because this runs on a tick.
+ */
+export declare function checkProvider(tenant: string, workspace: string | null, spec: ProviderSpec, model: string): Promise<ProviderCheck>;
+/** A provider block in use, and where it is declared — because "which file
+ *  do I edit" is the next question after "this key is dead". */
+export interface ProviderInUse {
+    spec: ProviderSpec;
+    /** The workspace whose vault resolves this block's secrets. Null for the
+     *  account's own block, which resolves against the account vault. */
+    workspace: string | null;
+    /** Human-readable origin: "account", "rank-desk", "rank-desk/writer". */
+    declaredIn: string;
+    /** The model id a run would send for the cheapest tier, which is what a
+     *  health check should spend. */
+    model: string;
+}
+/**
+ * Every distinct provider block this account would actually use, read from
+ * the same three places a run reads them: the account, each workspace, each
+ * agent.
+ *
+ * Deduplicated by endpoint and token, because ten agents sharing one gateway
+ * are one credential and checking it ten times spends ten times as much to
+ * learn the same fact. The origin recorded is the first one seen, which is
+ * the broadest — an account block covers everything under it.
+ */
+export declare function providersInUse(tenant: string): ProviderInUse[];
+/**
+ * Check every provider this account uses, one at a time.
+ *
+ * Serial on purpose: this is housekeeping on a tick, the whole point is that
+ * it is cheap, and firing twelve model requests at once to save eight
+ * seconds nobody is waiting for would be a strange trade.
+ */
+export declare function checkAccountProviders(tenant: string): Promise<(ProviderCheck & {
+    declaredIn: string;
+})[]>;

package/dist/src/provider-health.js

@@ -0,0 +1,232 @@
+// Is the key still good?
+//
+// A customer's own model key can be revoked, expire, run out of credit or
+// have its base URL moved, and today the platform finds out the way the
+// customer does: a desk fails at five in the morning. Twenty-seven of the
+// thirty provider presets have never had an agent run through them from
+// this runtime at all, so "it worked when we set it up" is doing a lot of
+// work in that sentence.
+//
+// This is deliberately NOT the tool-loop probe next door. `foldrun probe`
+// asks "can this model drive an agent", which is a question about a model
+// and is asked once. This asks "does this endpoint still take our key",
+// which is a question about a credential and has to be asked repeatedly —
+// so it has to be cheap. One request, one token of output, no SDK, no tool
+// loop, no agent. A fraction of a cent, and it answers the thing that
+// actually breaks.
+//
+// It reads the same provider block a run reads, resolved the same way, so a
+// pass here means a run would reach the same place with the same key.
+import fs from "node:fs";
+import path from "node:path";
+import matter from "gray-matter";
+import { getSecret } from "./secrets.js";
+import { noteSecretUse, healthKey } from "./secret-health.js";
+import { accountDir, workspaceDir, listWorkspaces, listAgents, parseProvider } from "./store.js";
+/** How long to wait. A provider slower than this is not one a step should
+ *  be waiting on either, and the check must never hold a tick. */
+const TIMEOUT_MS = 15_000;
+/** What each status means for a credential, and what a person does about
+ *  it. Kept as a table because the mapping IS the feature: "401" tells you
+ *  nothing you can act on, "the key is no longer accepted" tells you
+ *  everything. */
+function verdictFor(status) {
+    if (status >= 200 && status < 300)
+        return { verdict: "ok", detail: "the key was accepted and the model answered" };
+    if (status === 401 || status === 403) {
+        return { verdict: "credential", detail: "the key is no longer accepted — revoked, rotated, or wrong for this endpoint" };
+    }
+    if (status === 402)
+        return { verdict: "credential", detail: "the account behind this key is out of credit" };
+    if (status === 404) {
+        return { verdict: "not-found", detail: "the endpoint or the model id does not exist there — check base_url and models:" };
+    }
+    if (status === 429) {
+        // Not a health problem. A busy provider is a provider that is working,
+        // and reporting it as broken would train people to ignore the alert.
+        return { verdict: "busy", detail: "rate limited right now — the key is fine" };
+    }
+    if (status >= 500)
+        return { verdict: "provider-error", detail: "the provider is failing at its end" };
+    return { verdict: "provider-error", detail: `unexpected HTTP ${status}` };
+}
+/** `${SECRET}` in a token or header, filled from this account's vault. */
+function resolve(tenant, workspace, raw) {
+    return raw.replace(/\$\{([A-Z][A-Z0-9_]*)\}/g, (whole, name) => {
+        const hit = getSecret(tenant, name, workspace ?? undefined);
+        return hit ? hit.value : whole;
+    });
+}
+/**
+ * One request at the provider, in the wire format it speaks, asking for a
+ * single token. Never throws: an unreachable host is a verdict, not an
+ * exception, because this runs on a tick.
+ */
+export async function checkProvider(tenant, workspace, spec, model) {
+    const label = spec.name || (() => {
+        try {
+            return new URL(spec.baseUrl).host;
+        }
+        catch {
+            return spec.baseUrl || "provider";
+        }
+    })();
+    const began = Date.now();
+    const fail = (verdict, detail, status = null) => ({
+        provider: label,
+        model,
+        verdict,
+        ok: false,
+        status,
+        ms: Date.now() - began,
+        detail,
+    });
+    const token = resolve(tenant, workspace, spec.token);
+    if (!token || token.includes("${")) {
+        return fail("credential", `the secret named in provider.token is not in this account's vault (${spec.token})`);
+    }
+    if (!spec.baseUrl)
+        return fail("unreachable", "this provider block has no base_url");
+    // The three wire formats, each asking for the smallest possible answer.
+    const base = spec.baseUrl.replace(/\/+$/, "");
+    const anthropic = spec.format === "anthropic";
+    const url = anthropic
+        ? `${base}/v1/messages`
+        : spec.format === "responses"
+            ? `${base}/responses`
+            : `${base}/chat/completions`;
+    const body = anthropic
+        ? { model, max_tokens: 1, messages: [{ role: "user", content: "hi" }] }
+        : spec.format === "responses"
+            ? { model, max_output_tokens: 16, input: "hi" }
+            : { model, max_tokens: 1, messages: [{ role: "user", content: "hi" }] };
+    const headers = { "content-type": "application/json" };
+    // An Anthropic-format endpoint takes the key in whichever header its
+    // `auth:` says; everything else is a bearer token.
+    if (anthropic && spec.auth === "x-api-key") {
+        headers["x-api-key"] = token;
+        headers["anthropic-version"] = "2023-06-01";
+    }
+    else {
+        headers.authorization = `Bearer ${token}`;
+        if (anthropic)
+            headers["anthropic-version"] = "2023-06-01";
+    }
+    for (const [k, v] of Object.entries(spec.headers ?? {}))
+        headers[k.toLowerCase()] = resolve(tenant, workspace, v);
+    try {
+        const res = await fetch(url, {
+            method: "POST",
+            headers,
+            body: JSON.stringify(body),
+            signal: AbortSignal.timeout(TIMEOUT_MS),
+        });
+        const { verdict, detail } = verdictFor(res.status);
+        // The check is itself a use of the credential, and the most informative
+        // one there is: it was made for exactly this purpose.
+        const named = spec.token.match(/^\$\{([A-Z][A-Z0-9_]*)\}$/)?.[1];
+        if (named) {
+            let host = spec.baseUrl;
+            try {
+                host = new URL(spec.baseUrl).host;
+            }
+            catch {
+                // an unparseable base_url is its own problem, reported above
+            }
+            const hit = getSecret(tenant, named, workspace ?? undefined);
+            noteSecretUse(tenant, healthKey(named, hit?.scope === "workspace" ? "workspace" : "account", workspace), { host, status: res.status });
+        }
+        // The provider's own words when it refused. They are the difference
+        // between "fix your key" and "fix your model id", and they are only
+        // ever in the body.
+        let said = "";
+        if (!res.ok) {
+            said = (await res.text().catch(() => "")).replace(/\s+/g, " ").slice(0, 300);
+        }
+        return {
+            provider: label,
+            model,
+            verdict,
+            // Busy counts as healthy: the key worked, the provider is simply full.
+            ok: verdict === "ok" || verdict === "busy",
+            status: res.status,
+            ms: Date.now() - began,
+            detail: said ? `${detail} — ${said}` : detail,
+        };
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        return fail("unreachable", /timeout|aborted/i.test(message) ? `no answer within ${TIMEOUT_MS / 1000}s` : message);
+    }
+}
+/** The cheapest tier's id for a provider block: what it calls `fast`, or
+ *  our own fast model when it remaps nothing. */
+function cheapestModel(spec) {
+    return spec.models.fast || spec.models.default || spec.models.max || "claude-haiku-4-5-20251001";
+}
+function frontmatterOf(file) {
+    try {
+        return (matter(fs.readFileSync(file, "utf8")).data ?? {});
+    }
+    catch {
+        return {};
+    }
+}
+/**
+ * Every distinct provider block this account would actually use, read from
+ * the same three places a run reads them: the account, each workspace, each
+ * agent.
+ *
+ * Deduplicated by endpoint and token, because ten agents sharing one gateway
+ * are one credential and checking it ten times spends ten times as much to
+ * learn the same fact. The origin recorded is the first one seen, which is
+ * the broadest — an account block covers everything under it.
+ */
+export function providersInUse(tenant) {
+    const out = [];
+    const seen = new Set();
+    const add = (raw, workspace, declaredIn) => {
+        const spec = parseProvider(raw);
+        if (!spec || !spec.baseUrl)
+            return;
+        // The token is part of the identity: the same gateway with two different
+        // keys is two credentials, and either can die on its own.
+        const key = `${workspace ?? ""}|${spec.baseUrl}|${spec.token}|${cheapestModel(spec)}`;
+        if (seen.has(key))
+            return;
+        seen.add(key);
+        out.push({ spec, workspace, declaredIn, model: cheapestModel(spec) });
+    };
+    add(frontmatterOf(path.join(accountDir(tenant), "AGENTS.md")).provider, null, "account");
+    for (const ws of safeList(() => listWorkspaces(tenant))) {
+        const dir = workspaceDir(tenant, ws.name);
+        add(frontmatterOf(path.join(dir, "AGENTS.md")).provider, ws.name, ws.name);
+        for (const agent of safeList(() => listAgents(tenant, ws.name))) {
+            add(frontmatterOf(path.join(dir, "agents", agent.name, "agent.md")).provider, ws.name, `${ws.name}/${agent.name}`);
+        }
+    }
+    return out;
+}
+function safeList(fn) {
+    try {
+        return fn();
+    }
+    catch {
+        return [];
+    }
+}
+/**
+ * Check every provider this account uses, one at a time.
+ *
+ * Serial on purpose: this is housekeeping on a tick, the whole point is that
+ * it is cheap, and firing twelve model requests at once to save eight
+ * seconds nobody is waiting for would be a strange trade.
+ */
+export async function checkAccountProviders(tenant) {
+    const results = [];
+    for (const use of providersInUse(tenant)) {
+        const check = await checkProvider(tenant, use.workspace, use.spec, use.model);
+        results.push({ ...check, declaredIn: use.declaredIn });
+    }
+    return results;
+}

package/dist/src/run-container.d.ts

@@ -33,6 +33,7 @@ export interface ContainerStepInput {
     /** The step's spend ceiling and the model's per-token price — see
      *  step-exec.ts. Values, so the driver can enforce them in the pod. */
     budgetUsd?: number | null;
+    budgetNote?: string;
     price?: {
         input: number;
         output: number;
@@ -40,6 +41,9 @@ export interface ContainerStepInput {
     verify?: string;
     /** `output: json` — checked inside, where the reply is. */
     output?: "json";
+    /** `schema:` and `max_turns:`, resolved host-side — see ExecOptions. */
+    schema?: Record<string, unknown> | boolean;
+    maxTurns?: number;
     /** tools: [search] — the directories to search, as paths INSIDE the
      *  container (/workspace/…, /library/…). */
     search?: SearchRoot[];
@@ -78,6 +82,10 @@ export interface StepActuals {
 export interface ContainerStepOutcome {
     status: "completed" | "failed";
     result: string | null;
+    /** Why the sandbox ended when the step did not: what the cluster said
+     *  (OOMKilled, Evicted, the pod gone). Absent when the driver finished
+     *  and said so itself. A retry policy reads this. */
+    reason?: string | null;
     /** The model's final text block — see StepRecord.conclusion. Crosses the
      *  boundary with the rest of the outcome; the driver spreads it. */
     conclusion?: string | null;
@@ -120,7 +128,23 @@ export declare function allowedBack(rel: string): boolean;
  * the step did not touch it, so it is not ours to write back. Concurrent
  * edits survive, and a step that genuinely changed a file still wins.
  */
-export declare function applyContainerChanges(hostWs: string, containerWs: string, baseline?: string): string[];
+/**
+ * What a workspace copy looked like when it went in: the sha256 of every
+ * file the step could hand back. Small enough to ride into the sandbox
+ * beside the input, so a driver that inherits the step can tell what the
+ * step changed without the staging copy the first driver held in /tmp.
+ */
+export declare function hashTree(dir: string): Record<string, string>;
+/**
+ * Both sides changed a file the step was handed. When each is the handed-in
+ * content with lines added on the end — a ledger, history.md, a .jsonl — the
+ * answer is the base, then the host's additions, then the step's. Only the
+ * base's hash may be known, so the base is found as the longest shared
+ * leading run of whole lines whose hash matches. Anything else is null: a
+ * real conflict, which the caller must not settle by dropping one side.
+ */
+export declare function mergeAppends(was: Buffer | string | null, host: Buffer, next: Buffer): Buffer | null;
+export declare function applyContainerChanges(hostWs: string, containerWs: string, baseline?: string | Record<string, string>, note?: (message: string) => void, divert?: WriteDivert): string[];
 /**
  * Where the driver builds venvs and npm prefixes, as an absolute path *inside*
  * the runner. Fixed, and it has to be: a venv bakes its own absolute path into
@@ -154,7 +178,17 @@ export declare function runnerImageTag(): string;
  * has not been built before. `npm pack` of the installed core, so the image
  * runs the same bytes the server does.
  */
-export declare function ensureRunnerImage(): {
+/**
+ * `platform`: build for another architecture — "linux/arm64" on the amd64
+ * box, through QEMU — and name the result with the arch appended so it never
+ * shadows the native image in the daemon. Only the publisher asks for this;
+ * a step always runs the native image. Needs binfmt registered on the host
+ * (tonistiigi/binfmt); the publisher does that itself, every time, because
+ * the registration does not survive a reboot.
+ */
+export declare function ensureRunnerImage(opts?: {
+    platform?: string;
+}): {
     tag: string;
     log: string[];
 };
@@ -165,12 +199,14 @@ export declare function parseDriverLine(line: string): {
     call?: string;
     ms?: number;
     err?: boolean;
-} | {
+} | ({
     e: "done";
-} & ContainerStepOutcome | null;
+} & ContainerStepOutcome) | null;
 export interface RunInContainerArgs {
     workspaceRoot: string;
     libraryRoot: string;
+    /** The platform's gallery, staged under /library BEFORE the account library so the account's copies win. */
+    galleryRoot?: string;
     /** The account this step belongs to — the key for its dependency cache.
      *  Optional so an embedder that has no tenancy simply gets no cache. */
     tenant?: string;
@@ -185,7 +221,47 @@ export interface RunInContainerArgs {
     runId?: string;
     /** The reservation class — which limits this step's sandbox holds. */
     size?: "small" | "large" | "heavy";
+    /** Re-attach to a sandbox a previous driver started, instead of creating
+     *  one: its handle, and how many output lines that driver had applied.
+     *  The executor replays nothing before `consumed`, and reads the step's
+     *  files back from the sandbox's own copy-in snapshot. */
+    resume?: {
+        ref: string;
+        consumed: number;
+    } | null;
+    /** Called by an executor that can be resumed: when its sandbox exists
+     *  (consumed 0), and before each output line it applies. The runner
+     *  records these on the step, so the next driver knows where to attach. */
+    checkpoint?: (ref: string, consumed: number) => void;
+    /** A test run: where state/ and storage/ changes go instead of the
+     *  workspace, and who to tell about each. See test-mode.ts. */
+    divert?: WriteDivert;
+    /** The run is a test run. An isolated executor labels the sandbox so the
+     *  cluster can cut its network down to the egress proxy: a script tool
+     *  calls the network directly, not through the proxy where the test-mode
+     *  policy lives, so without this a test run's script could still send. */
+    test?: boolean;
+}
+/** What a test run does with a write it must not apply. */
+export interface WriteDivert {
+    /** The directory the diverted files are written under, by their
+     *  workspace-relative path: runs/<id>/test-writes/. */
+    to: string;
+    note: (rel: string, summary: string) => void;
 }
+/**
+ * What crosses into the container as its environment, and how.
+ *
+ * A docker --env-file is one `KEY=value` per line and cannot carry a
+ * newline. @file secrets are multi-line by nature (a PEM key, a cert), so
+ * they are staged as files NOW, into the copied-in workspace, and the env
+ * carries the container path — the one place a path is stable across the
+ * boundary. A plain secret whose value happens to hold a newline gets the
+ * same treatment, and the trace says so: it used to be dropped from the
+ * env file without a word, and the step failed later, elsewhere, on a
+ * variable that read as unset. Everything else goes through the env file.
+ */
+export declare function stageContainerEnv(env: Record<string, string>, wsIn: string, agentRel: string, emit: (type: "info" | "error", text: string) => void): Record<string, string>;
 /** The limits a size class reserves. Large is the install's configured
  *  limits unchanged, so existing installs bill and behave identically;
  *  small defaults to a quarter-ish slice and is env-tunable. */