@foldrun/core 0.3.0 → 0.5.0

package/dist/src/script-tools.d.ts

@@ -9,8 +9,9 @@ export interface ScriptSpec {
     args: Record<string, string>;
     interpreter?: string;
     /** Seconds this script may run — for the crawl that legitimately takes
-     *  five minutes. Defaults to 120, capped at 600: a limit an author can
-     *  raise is a budget; one they can remove is a hang. */
+     *  five minutes. Unset means no platform clock; there was a two-minute
+     *  default and a ten-minute cap here and both were removed (see the note
+     *  above `timeoutFor`). */
     timeout?: number;
     /** The script itself, when the tool is a single markdown file with its
      *  code in a fenced block. Materialised to a file at call time — the
@@ -26,12 +27,24 @@ export interface ScriptSpec {
      *  default, `materialised`, is a program that reads values from its
      *  environment and needs them real — a browser seeding cookies. */
     secrets?: "proxied" | "materialised";
+    /** `outward: true` — this program makes something happen outside (sends,
+     *  posts, orders). On a test run a step that grants it is handed no real
+     *  secret at all, whatever their names. */
+    outward?: boolean;
+    /** `test_mode: allow` — a reader that happens to use a send-capable
+     *  secret's name (a GBP reader on a GBP token): on a test run the step
+     *  still gets the real values. An `outward` tool in the same step wins. */
+    testMode?: "allow";
 }
 export declare function parseScripts(raw: unknown): ScriptSpec[];
 export declare function commandFor(spec: ScriptSpec, abs: string, overrides?: Record<string, string>): {
     cmd: string;
     args: string[];
 };
+export declare function runScript(cwd: string, spec: ScriptSpec, values: Record<string, string>, env: Record<string, string>, libraryScripts: string, interpreters: Record<string, string>, exec: ExecutionContext | null): Promise<{
+    code: number | null;
+    out: string;
+}>;
 export interface ExecutionContext {
     /** "docker" nests each script in its own container; "sandbox" spawns it
      *  directly inside the isolation the step already has. */

package/dist/src/script-tools.js

@@ -19,6 +19,9 @@ import { spawn } from "node:child_process";
 import { z } from "zod";
 import { createSdkMcpServer, tool } from "@anthropic-ai/claude-agent-sdk";
 import { runInContainer } from "./container.js";
+import { hostSafeEnv } from "./host-env.js";
+import { workspaceLinkEnv } from "./confine.js";
+import { platform } from "./platform.js";
 // A script's only clock is the `timeout:` its own tool.md declares. There
 // was a two-minute default and a ten-minute cap here; both were the
 // platform's opinion about how long a person's script may take, and the
@@ -55,11 +58,21 @@ export function parseScripts(raw) {
             args,
             interpreter: typeof e.interpreter === "string" ? e.interpreter : undefined,
             secrets: e.secrets === "proxied" ? "proxied" : undefined,
+            outward: e.outward === true ? true : undefined,
+            testMode: e.test_mode === "allow" ? "allow" : undefined,
             timeout: Number.isFinite(timeout) && timeout > 0 ? timeout : undefined,
         });
     }
     return out;
 }
+/** Interpreter names that mean a language rather than a binary, and the
+ *  extension whose prepared interpreter serves them. */
+const GENERIC_INTERPRETER = {
+    python: ".py",
+    python3: ".py",
+    "/usr/bin/python3": ".py",
+    "/usr/bin/env python3": ".py",
+};
 // Pick how to execute a file when no interpreter is declared.
 //
 // Exported because the tool tester has to make the same choice, and used to
@@ -67,8 +80,19 @@ export function parseScripts(raw) {
 // Test button spawned every .mjs tool as a program and reported EACCES for a
 // tool that runs perfectly. Two tables, one of them wrong.
 export function commandFor(spec, abs, overrides = {}) {
-    if (spec.interpreter)
+    if (spec.interpreter) {
+        // `interpreter: python3` names the language, not a particular binary, and
+        // it is how most tool files are written. Taken literally it ran the bare
+        // system python beside a venv built for this very tool, and the tool
+        // died on its first import ("No module named openpyxl") while the log
+        // above it said the runtime was cached. A generic python name is served
+        // by the prepared venv when there is one; a path or anything else is
+        // still obeyed as written.
+        const lang = GENERIC_INTERPRETER[spec.interpreter.trim()];
+        if (lang && overrides[lang])
+            return { cmd: overrides[lang], args: [abs] };
         return { cmd: spec.interpreter, args: [abs] };
+    }
     const ext = path.extname(abs).toLowerCase();
     // A prepared runtime (venv, npm prefix) wins over the host interpreter.
     if (overrides[ext])
@@ -96,7 +120,7 @@ function containerInterpreter(abs) {
     };
     return byExt[path.extname(abs).toLowerCase()] ?? abs;
 }
-function runScript(cwd, spec, values, env, libraryScripts, interpreters, exec) {
+export function runScript(cwd, spec, values, env, libraryScripts, interpreters, exec) {
     // An inline-code tool has no file yet: write it inside the agent's own
     // directory — already an allowed root — and run it like any other script.
     // Materialised per call rather than per build, so an edit to the tool's
@@ -188,9 +212,14 @@ function runScript(cwd, spec, values, env, libraryScripts, interpreters, exec) {
             }).then(resolve, (err) => resolve({ code: null, out: `container error: ${err instanceof Error ? err.message : String(err)}` }));
             return;
         }
+        // Not process.env: on a platform that is every tenant's root key and
+        // the datastore credentials. The child starts from the allowlisted
+        // base and gets what the agent declared, by name — see host-env.ts.
+        // In a run container `env` already carries the container's whole
+        // environment, which is the boundary, so nothing is lost there.
         const child = spawn(cmd, args, {
             cwd,
-            env: { ...process.env, ...env },
+            env: workspaceLinkEnv({ ...hostSafeEnv(), ...env }, cwd),
             timeout: timeoutFor(spec),
         });
         let out = "";
@@ -228,7 +257,18 @@ export function resolveRunPath(agentDir, run, libraryScripts) {
         return path.resolve(workspaceRoot, "tools", run.slice("workspace/tools/".length));
     }
     if (run.startsWith("account/tools/")) {
-        return path.resolve(libraryRoot, "tools", run.slice("account/tools/".length));
+        const rel = run.slice("account/tools/".length);
+        const own = path.resolve(libraryRoot, "tools", rel);
+        // A gallery tool nobody installed has no copy in the library. In a
+        // sandbox the gallery is laid down under the library, so the path above
+        // is it; a run on the host has no such merge, so it looks on the shelf.
+        const gallery = platform.galleryDir();
+        if (!fs.existsSync(own) && gallery) {
+            const shelf = path.resolve(gallery, "tools", rel);
+            if (fs.existsSync(shelf))
+                return shelf;
+        }
+        return own;
     }
     // Canonical prefixes, then the legacy spellings they replaced.
     for (const p of ["workspace/scripts/", "shared/"]) {

package/dist/src/secret-files.d.ts

@@ -1,12 +1,13 @@
 /**
- * Turn any @file/@ssh/@api values in `env` into files under
- * `<agentDir>/.secret-files`, rewriting each to its path. Returns the
- * directory written (for cleanup) and the transformed env. Plain values
- * pass through untouched.
+ * Turn any @file/@ssh/@api values in `env` into files under a fresh
+ * `<agentDir>/.secret-files/step-XXXXXX`, rewriting each to its path.
+ * Returns the directory written (for cleanup) and the transformed env.
+ * Plain values pass through untouched.
  */
 export declare function materializeFileSecrets(agentDir: string, env: Record<string, string>): {
     env: Record<string, string>;
     dir: string | null;
 };
-/** Best-effort removal of a materialised secret-files directory. */
+/** Best-effort removal of a materialised secret-files directory — this
+ *  step's own, and the shared parent once no other step is using it. */
 export declare function cleanupFileSecrets(dir: string | null): void;

package/dist/src/secret-files.js

@@ -9,9 +9,12 @@
 // `"$PROD_VM" 'uptime'` or `"$STRIPE" /v1/charges` and never learns whether
 // it was key or password, one header or five.
 //
-// Everything lands in a per-run scratch directory inside the agent's own
+// Everything lands in a per-step scratch directory inside the agent's own
 // tree (so it crosses into a container with the workspace) but dot-prefixed
-// and never archived or copied back — it holds live credentials.
+// and never archived or copied back — it holds live credentials. Per step,
+// not per agent: two steps of one agent run in parallel on the in-process
+// path (a fan-out), and with one shared directory the first to finish
+// deleted the keys the other was still using.
 import fs from "node:fs";
 import path from "node:path";
 import { isFileValue, fileContent, isSshValue, sshConfigOf, isApiValue, apiConfigOf, } from "./secrets.js";
@@ -25,17 +28,19 @@ function writeWrapper(dir, name, body) {
     return file;
 }
 /**
- * Turn any @file/@ssh/@api values in `env` into files under
- * `<agentDir>/.secret-files`, rewriting each to its path. Returns the
- * directory written (for cleanup) and the transformed env. Plain values
- * pass through untouched.
+ * Turn any @file/@ssh/@api values in `env` into files under a fresh
+ * `<agentDir>/.secret-files/step-XXXXXX`, rewriting each to its path.
+ * Returns the directory written (for cleanup) and the transformed env.
+ * Plain values pass through untouched.
  */
 export function materializeFileSecrets(agentDir, env) {
     const entries = Object.entries(env).filter(([, v]) => typeof v === "string" && (isFileValue(v) || isSshValue(v) || isApiValue(v)));
     if (entries.length === 0)
         return { env, dir: null };
-    const dir = path.join(agentDir, ".secret-files");
-    fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
+    const parent = path.join(agentDir, ".secret-files");
+    fs.mkdirSync(parent, { recursive: true, mode: 0o700 });
+    const dir = fs.mkdtempSync(path.join(parent, "step-"));
+    fs.chmodSync(dir, 0o700);
     const out = { ...env };
     for (const [name, value] of entries) {
         // Lowercased name as the filename keeps it predictable without leaking
@@ -85,8 +90,16 @@ export function materializeFileSecrets(agentDir, env) {
     }
     return { env: out, dir };
 }
-/** Best-effort removal of a materialised secret-files directory. */
+/** Best-effort removal of a materialised secret-files directory — this
+ *  step's own, and the shared parent once no other step is using it. */
 export function cleanupFileSecrets(dir) {
-    if (dir)
-        fs.rmSync(dir, { recursive: true, force: true });
+    if (!dir)
+        return;
+    fs.rmSync(dir, { recursive: true, force: true });
+    try {
+        fs.rmdirSync(path.dirname(dir));
+    }
+    catch {
+        // another step's directory is still in there
+    }
 }

package/dist/src/secret-health.d.ts

@@ -0,0 +1,83 @@
+/** How a credential is identified here. A name alone is not enough: the
+ *  account and each workspace can hold a secret by the same name, and they
+ *  are different credentials with different health. */
+export declare function healthKey(name: string, scope: "account" | "workspace", workspace?: string | null): string;
+export interface SecretUse {
+    /** ISO time of the most recent use. */
+    at: string;
+    /** The host it was sent to, so "which of my four keys is this" has an
+     *  answer without opening a run. */
+    host: string;
+    /** The response status, when there was one. */
+    status: number | null;
+    /** Refused means the credential itself was rejected: 401, 402 or 403.
+     *  A 500 is the other end failing, which says nothing about the key. */
+    outcome: "accepted" | "refused" | "error";
+}
+export interface SecretRecord {
+    last: SecretUse;
+    /** Consecutive refusals, newest first — one is a fluke, five is a dead
+     *  key. Reset by any acceptance. */
+    refusals: number;
+    /** The last acceptance, kept through a run of refusals so a page can say
+     *  "worked until Tuesday", which is the sentence that dates the breakage. */
+    lastAccepted: string | null;
+}
+export type HealthFile = Record<string, SecretRecord>;
+/** One use to fold into a record. */
+export interface HealthUpdate {
+    key: string;
+    use: {
+        host: string;
+        status: number | null;
+        at: string;
+    };
+}
+/**
+ * Where health is kept. A file per account by default — what the CLI and a
+ * single-box install use. The platform plugs in Postgres
+ * (setSecretHealthStore), where each use is one atomic upsert: two pods
+ * recording at once on a shared file was last-writer-wins, and a refusal
+ * could vanish.
+ */
+export interface SecretHealthStore {
+    apply(tenant: string, updates: HealthUpdate[]): Promise<void>;
+    read(tenant: string): Promise<HealthFile>;
+    forget(tenant: string, key: string): Promise<void>;
+}
+/** The platform's store (Postgres), or back to files with null. */
+export declare function setSecretHealthStore(next: SecretHealthStore | null): void;
+/** 401, 402 and 403 are the credential being refused. Everything else is
+ *  the request, the model or the provider — none of which is the key's
+ *  fault, and calling them a credential failure would send people to
+ *  rotate a key that was never the problem. */
+export declare function outcomeFor(status: number | null): SecretUse["outcome"];
+/**
+ * Record one use. `key` is a health key (see healthKey), never a bare name.
+ * Never throws and never blocks: the write is queued and happens off the
+ * caller's path, because this runs on every proxied request.
+ */
+export declare function noteSecretUse(tenant: string, key: string, use: {
+    host: string;
+    status: number | null;
+    at?: string;
+}): void;
+/** Wait for queued writes to land — for tests, and for a process that is
+ *  about to exit. */
+export declare function flushSecretHealth(): Promise<void>;
+/** Everything known about this account's credentials, by name. Names the
+ *  vault holds but nothing has used are simply absent, which is the honest
+ *  answer: unused is not the same as broken. */
+export declare function secretHealth(tenant: string): Promise<HealthFile>;
+/** The credentials worth saying something about: refused the last time they
+ *  were used. Sorted worst first — most consecutive refusals, then oldest
+ *  last-success, which is the order someone would work through them. */
+export declare function failingSecrets(tenant: string): Promise<{
+    name: string;
+    record: SecretRecord;
+}[]>;
+/** Drop what is known about a secret — for a delete, and for a rotation,
+ *  where the old key's refusals say nothing about the new one. */
+export declare function forgetSecretHealth(tenant: string, name: string): Promise<void>;
+/** The account's file, for the platform's one-time import into its store. */
+export declare function readSecretHealthFile(tenant: string): HealthFile;

package/dist/src/secret-health.js

@@ -0,0 +1,177 @@
+// Does this credential still work?
+//
+// The vault knows a secret's name, its scope and its shape. It knows nothing
+// about whether the thing on the other end still accepts it, and that is the
+// only question anyone actually has. A key that has quietly stopped working
+// looks identical to a key nobody has used yet, and both look identical to a
+// key that is fine — three very different situations rendered the same way.
+//
+// Two facts, recorded where they are already known and nowhere else:
+//
+//   when it was last used, and against which host
+//   what the other end said the last time — accepted, or refused
+//
+// The egress proxy sees both for every http tool and every model call,
+// because filling the credential is its whole job. Notifications and the
+// provider check report their own. Nothing new is instrumented, and a secret
+// materialised into a sandbox reports nothing, which the page says plainly
+// rather than guessing.
+//
+// Never the value, never a hash of the value, never the request body. A
+// name, a host, a status and a time.
+import fs from "node:fs";
+import path from "node:path";
+import crypto from "node:crypto";
+import { accountDir } from "./store.js";
+/** How a credential is identified here. A name alone is not enough: the
+ *  account and each workspace can hold a secret by the same name, and they
+ *  are different credentials with different health. */
+export function healthKey(name, scope, workspace) {
+    return scope === "workspace" && workspace ? `workspace:${workspace}:${name}` : `account:${name}`;
+}
+const healthFile = (tenant) => path.join(accountDir(tenant), "secret-health.json");
+function read(tenant) {
+    try {
+        return JSON.parse(fs.readFileSync(healthFile(tenant), "utf8"));
+    }
+    catch {
+        return {};
+    }
+}
+function write(tenant, data) {
+    const file = healthFile(tenant);
+    if (!fs.existsSync(path.dirname(file)))
+        return;
+    // pid alone is not unique across pods on a shared volume.
+    const tmp = `${file}.${process.pid}.${crypto.randomBytes(4).toString("hex")}`;
+    fs.writeFileSync(tmp, JSON.stringify(data, null, 2));
+    fs.renameSync(tmp, file);
+}
+const fileStore = {
+    async apply(tenant, updates) {
+        const data = read(tenant);
+        for (const { key, use } of updates)
+            apply(data, key, use);
+        write(tenant, data);
+    },
+    async read(tenant) {
+        return read(tenant);
+    },
+    async forget(tenant, key) {
+        const data = read(tenant);
+        if (!(key in data))
+            return;
+        delete data[key];
+        write(tenant, data);
+    },
+};
+let store = fileStore;
+/** The platform's store (Postgres), or back to files with null. */
+export function setSecretHealthStore(next) {
+    store = next ?? fileStore;
+}
+// Writes are queued and coalesced off the caller's path. The egress proxy
+// notes a use per secret per request, before it streams the response body,
+// and a synchronous read-modify-write of a JSON file there is latency on
+// every proxied call — "never blocks" has to be true, not aspirational.
+// One writer per process; the queue drains in order.
+const pending = new Map();
+let draining = false;
+function drain() {
+    if (draining)
+        return;
+    draining = true;
+    setImmediate(async () => {
+        try {
+            for (const [tenant, updates] of pending) {
+                pending.delete(tenant);
+                try {
+                    await store.apply(tenant, updates);
+                }
+                catch {
+                    // best effort, always
+                }
+            }
+        }
+        finally {
+            draining = false;
+            if (pending.size)
+                drain();
+        }
+    });
+}
+function apply(data, name, use) {
+    const outcome = outcomeFor(use.status);
+    const at = use.at ?? new Date().toISOString();
+    const prev = data[name];
+    data[name] = {
+        last: { at, host: use.host, status: use.status, outcome },
+        // Only a refusal counts against a key. An error at the other end is
+        // the other end's problem and must not accumulate into a false alarm.
+        refusals: outcome === "refused" ? (prev?.refusals ?? 0) + 1 : outcome === "accepted" ? 0 : prev?.refusals ?? 0,
+        lastAccepted: outcome === "accepted" ? at : prev?.lastAccepted ?? null,
+    };
+}
+/** 401, 402 and 403 are the credential being refused. Everything else is
+ *  the request, the model or the provider — none of which is the key's
+ *  fault, and calling them a credential failure would send people to
+ *  rotate a key that was never the problem. */
+export function outcomeFor(status) {
+    if (status === null)
+        return "error";
+    if (status === 401 || status === 402 || status === 403)
+        return "refused";
+    if (status >= 200 && status < 400)
+        return "accepted";
+    return "error";
+}
+/**
+ * Record one use. `key` is a health key (see healthKey), never a bare name.
+ * Never throws and never blocks: the write is queued and happens off the
+ * caller's path, because this runs on every proxied request.
+ */
+export function noteSecretUse(tenant, key, use) {
+    if (!key)
+        return;
+    const list = pending.get(tenant) ?? [];
+    list.push({ key, use: { ...use, at: use.at ?? new Date().toISOString() } });
+    pending.set(tenant, list);
+    drain();
+}
+/** Wait for queued writes to land — for tests, and for a process that is
+ *  about to exit. */
+export function flushSecretHealth() {
+    return new Promise((resolve) => {
+        const check = () => (pending.size === 0 && !draining ? resolve() : setImmediate(check));
+        check();
+    });
+}
+/** Everything known about this account's credentials, by name. Names the
+ *  vault holds but nothing has used are simply absent, which is the honest
+ *  answer: unused is not the same as broken. */
+export async function secretHealth(tenant) {
+    return store.read(tenant);
+}
+/** The credentials worth saying something about: refused the last time they
+ *  were used. Sorted worst first — most consecutive refusals, then oldest
+ *  last-success, which is the order someone would work through them. */
+export async function failingSecrets(tenant) {
+    return Object.entries(await store.read(tenant))
+        .filter(([, r]) => r.last.outcome === "refused")
+        .map(([name, record]) => ({ name, record }))
+        .sort((a, b) => b.record.refusals - a.record.refusals || (a.record.lastAccepted ?? "").localeCompare(b.record.lastAccepted ?? ""));
+}
+/** Drop what is known about a secret — for a delete, and for a rotation,
+ *  where the old key's refusals say nothing about the new one. */
+export async function forgetSecretHealth(tenant, name) {
+    try {
+        await store.forget(tenant, name);
+    }
+    catch {
+        // best effort
+    }
+}
+/** The account's file, for the platform's one-time import into its store. */
+export function readSecretHealthFile(tenant) {
+    return read(tenant);
+}

package/dist/src/secrets.d.ts

@@ -1,9 +1,39 @@
 /** Which store a secret came from. */
 export type SecretScope = "account" | "workspace";
+/**
+ * The install's root secret, as text: FOLDRUN_SECRET_KEY when set, else the
+ * contents of `.secret-key` under the data root — created here, the first
+ * time anything asks, so there is never a moment when a derived token has
+ * nothing real to hang off. The vault, hook tokens, approval links and git
+ * push secrets all derive from this one value; rotating it invalidates every
+ * one of them together. Exported for webhook.ts, which used to fall back to
+ * a fixed dev string until the vault happened to create this file — so hook
+ * URLs were computable by anyone who read the source, and then changed the
+ * first time a secret was stored.
+ */
+export declare function installKeyMaterial(): string;
 /** How the stored value is used at run time — "oauth2"/"service-account"
  *  are refreshed into live tokens, "file"/"ssh"/"api" are materialised into
  *  the run's scratch. Plain values have no kind. */
 export type SecretKind = "oauth2" | "service-account" | "file" | "ssh" | "api";
+interface StoredSecret {
+    iv: string;
+    tag: string;
+    data: string;
+    updatedAt: string;
+    kind?: SecretKind;
+    /** Which key opens this. Absent means the install key, which is what every
+     *  record written before per-account keys existed is encrypted with — so an
+     *  old vault stays readable without a migration having to run first. */
+    k?: "tenant";
+}
+type SecretsFile = Record<string, StoredSecret>;
+/**
+ * Change one vault file safely: `change` gets what is stored now and returns
+ * the new contents, or null to leave the file untouched. Returns what is
+ * stored after.
+ */
+export declare function mutateVaultFile(file: string, change: (all: SecretsFile) => SecretsFile | null): SecretsFile;
 export declare const SECRET_NAME: RegExp;
 export declare function setSecret(tenant: string, name: string, value: string, workspace?: string, kind?: SecretKind): void;
 /** Encrypt one value for an account — the same envelope secrets use, exported
@@ -102,7 +132,13 @@ export declare function isOAuth2Value(value: string): boolean;
  * refresh throws with the secret's name and the provider's reason — a run
  * should fail saying "GOOGLE_ADS_TOKEN: invalid_grant", not 401 somewhere.
  */
-export declare function materializeSecrets(env: Record<string, string>): Promise<Record<string, string>>;
+export declare function materializeSecrets(env: Record<string, string>, 
+/** Whose secrets these are, when the caller knows: lets the platform keep
+ *  each OAuth connection's health and quote a reconnect link on failure. */
+ctx?: {
+    tenant: string;
+    workspace?: string;
+}): Promise<Record<string, string>>;
 export interface ServiceAccountConfig {
     token_url: string;
     issuer: string;
@@ -145,3 +181,4 @@ export declare function rewrapTenantSecrets(tenant: string): {
     moved: number;
     unreadable: string[];
 };
+export {};