@foldrun/core 0.2.0 → 0.4.0

package/src/script-tools.ts

@@ -20,6 +20,7 @@ import { spawn } from "node:child_process";
 import { z } from "zod";
 import { createSdkMcpServer, tool } from "@anthropic-ai/claude-agent-sdk";
 import { runInContainer } from "./container.ts";
+import { hostSafeEnv } from "./host-env.ts";
 
 // A script's only clock is the `timeout:` its own tool.md declares. There
 // was a two-minute default and a ten-minute cap here; both were the
@@ -55,6 +56,14 @@ export interface ScriptSpec {
    *  default, `materialised`, is a program that reads values from its
    *  environment and needs them real — a browser seeding cookies. */
   secrets?: "proxied" | "materialised";
+  /** `outward: true` — this program makes something happen outside (sends,
+   *  posts, orders). On a test run a step that grants it is handed no real
+   *  secret at all, whatever their names. */
+  outward?: boolean;
+  /** `test_mode: allow` — a reader that happens to use a send-capable
+   *  secret's name (a GBP reader on a GBP token): on a test run the step
+   *  still gets the real values. An `outward` tool in the same step wins. */
+  testMode?: "allow";
 }
 
 export function parseScripts(raw: unknown): ScriptSpec[] {
@@ -82,6 +91,8 @@ export function parseScripts(raw: unknown): ScriptSpec[] {
       args,
       interpreter: typeof e.interpreter === "string" ? e.interpreter : undefined,
       secrets: e.secrets === "proxied" ? "proxied" : undefined,
+      outward: e.outward === true ? true : undefined,
+      testMode: e.test_mode === "allow" ? "allow" : undefined,
       timeout: Number.isFinite(timeout) && timeout > 0 ? timeout : undefined,
     });
   }
@@ -128,7 +139,7 @@ function containerInterpreter(abs: string): string {
   return byExt[path.extname(abs).toLowerCase()] ?? abs;
 }
 
-function runScript(
+export function runScript(
   cwd: string,
   spec: ScriptSpec,
   values: Record<string, string>,
@@ -229,9 +240,14 @@ function runScript(
       return;
     }
 
+    // Not process.env: on a platform that is every tenant's root key and
+    // the datastore credentials. The child starts from the allowlisted
+    // base and gets what the agent declared, by name — see host-env.ts.
+    // In a run container `env` already carries the container's whole
+    // environment, which is the boundary, so nothing is lost there.
     const child = spawn(cmd, args, {
       cwd,
-      env: { ...process.env, ...env },
+      env: { ...hostSafeEnv(), ...env },
       timeout: timeoutFor(spec),
     });
     let out = "";

package/src/secret-health.ts

@@ -0,0 +1,185 @@
+// Does this credential still work?
+//
+// The vault knows a secret's name, its scope and its shape. It knows nothing
+// about whether the thing on the other end still accepts it, and that is the
+// only question anyone actually has. A key that has quietly stopped working
+// looks identical to a key nobody has used yet, and both look identical to a
+// key that is fine — three very different situations rendered the same way.
+//
+// Two facts, recorded where they are already known and nowhere else:
+//
+//   when it was last used, and against which host
+//   what the other end said the last time — accepted, or refused
+//
+// The egress proxy sees both for every http tool and every model call,
+// because filling the credential is its whole job. Notifications and the
+// provider check report their own. Nothing new is instrumented, and a secret
+// materialised into a sandbox reports nothing, which the page says plainly
+// rather than guessing.
+//
+// Never the value, never a hash of the value, never the request body. A
+// name, a host, a status and a time.
+
+import fs from "node:fs";
+import path from "node:path";
+import crypto from "node:crypto";
+import { accountDir } from "./store.ts";
+
+/** How a credential is identified here. A name alone is not enough: the
+ *  account and each workspace can hold a secret by the same name, and they
+ *  are different credentials with different health. */
+export function healthKey(name: string, scope: "account" | "workspace", workspace?: string | null): string {
+  return scope === "workspace" && workspace ? `workspace:${workspace}:${name}` : `account:${name}`;
+}
+
+export interface SecretUse {
+  /** ISO time of the most recent use. */
+  at: string;
+  /** The host it was sent to, so "which of my four keys is this" has an
+   *  answer without opening a run. */
+  host: string;
+  /** The response status, when there was one. */
+  status: number | null;
+  /** Refused means the credential itself was rejected: 401, 402 or 403.
+   *  A 500 is the other end failing, which says nothing about the key. */
+  outcome: "accepted" | "refused" | "error";
+}
+
+export interface SecretRecord {
+  last: SecretUse;
+  /** Consecutive refusals, newest first — one is a fluke, five is a dead
+   *  key. Reset by any acceptance. */
+  refusals: number;
+  /** The last acceptance, kept through a run of refusals so a page can say
+   *  "worked until Tuesday", which is the sentence that dates the breakage. */
+  lastAccepted: string | null;
+}
+
+type HealthFile = Record<string, SecretRecord>;
+
+const healthFile = (tenant: string) => path.join(accountDir(tenant), "secret-health.json");
+
+function read(tenant: string): HealthFile {
+  try {
+    return JSON.parse(fs.readFileSync(healthFile(tenant), "utf8")) as HealthFile;
+  } catch {
+    return {};
+  }
+}
+
+function write(tenant: string, data: HealthFile): void {
+  const file = healthFile(tenant);
+  if (!fs.existsSync(path.dirname(file))) return;
+  // pid alone is not unique across pods on a shared volume.
+  const tmp = `${file}.${process.pid}.${crypto.randomBytes(4).toString("hex")}`;
+  fs.writeFileSync(tmp, JSON.stringify(data, null, 2));
+  fs.renameSync(tmp, file);
+}
+
+// Writes are queued and coalesced off the caller's path. The egress proxy
+// notes a use per secret per request, before it streams the response body,
+// and a synchronous read-modify-write of a JSON file there is latency on
+// every proxied call — "never blocks" has to be true, not aspirational.
+// One writer per process; the queue drains in order.
+const pending = new Map<string, { name: string; use: { host: string; status: number | null; at?: string } }[]>();
+let draining = false;
+
+function drain(): void {
+  if (draining) return;
+  draining = true;
+  setImmediate(() => {
+    try {
+      for (const [tenant, uses] of pending) {
+        pending.delete(tenant);
+        const data = read(tenant);
+        for (const { name, use } of uses) apply(data, name, use);
+        write(tenant, data);
+      }
+    } catch {
+      // best effort, always
+    } finally {
+      draining = false;
+      if (pending.size) drain();
+    }
+  });
+}
+
+function apply(data: HealthFile, name: string, use: { host: string; status: number | null; at?: string }): void {
+  const outcome = outcomeFor(use.status);
+  const at = use.at ?? new Date().toISOString();
+  const prev = data[name];
+  data[name] = {
+    last: { at, host: use.host, status: use.status, outcome },
+    // Only a refusal counts against a key. An error at the other end is
+    // the other end's problem and must not accumulate into a false alarm.
+    refusals: outcome === "refused" ? (prev?.refusals ?? 0) + 1 : outcome === "accepted" ? 0 : prev?.refusals ?? 0,
+    lastAccepted: outcome === "accepted" ? at : prev?.lastAccepted ?? null,
+  };
+}
+
+/** 401, 402 and 403 are the credential being refused. Everything else is
+ *  the request, the model or the provider — none of which is the key's
+ *  fault, and calling them a credential failure would send people to
+ *  rotate a key that was never the problem. */
+export function outcomeFor(status: number | null): SecretUse["outcome"] {
+  if (status === null) return "error";
+  if (status === 401 || status === 402 || status === 403) return "refused";
+  if (status >= 200 && status < 400) return "accepted";
+  return "error";
+}
+
+/**
+ * Record one use. `key` is a health key (see healthKey), never a bare name.
+ * Never throws and never blocks: the write is queued and happens off the
+ * caller's path, because this runs on every proxied request.
+ */
+export function noteSecretUse(
+  tenant: string,
+  key: string,
+  use: { host: string; status: number | null; at?: string },
+): void {
+  if (!key) return;
+  const list = pending.get(tenant) ?? [];
+  list.push({ name: key, use: { ...use, at: use.at ?? new Date().toISOString() } });
+  pending.set(tenant, list);
+  drain();
+}
+
+/** Wait for queued writes to land — for tests, and for a process that is
+ *  about to exit. */
+export function flushSecretHealth(): Promise<void> {
+  return new Promise((resolve) => {
+    const check = () => (pending.size === 0 && !draining ? resolve() : setImmediate(check));
+    check();
+  });
+}
+
+/** Everything known about this account's credentials, by name. Names the
+ *  vault holds but nothing has used are simply absent, which is the honest
+ *  answer: unused is not the same as broken. */
+export function secretHealth(tenant: string): HealthFile {
+  return read(tenant);
+}
+
+/** The credentials worth saying something about: refused the last time they
+ *  were used. Sorted worst first — most consecutive refusals, then oldest
+ *  last-success, which is the order someone would work through them. */
+export function failingSecrets(tenant: string): { name: string; record: SecretRecord }[] {
+  return Object.entries(read(tenant))
+    .filter(([, r]) => r.last.outcome === "refused")
+    .map(([name, record]) => ({ name, record }))
+    .sort((a, b) => b.record.refusals - a.record.refusals || (a.record.lastAccepted ?? "").localeCompare(b.record.lastAccepted ?? ""));
+}
+
+/** Drop what is known about a secret — for a delete, and for a rotation,
+ *  where the old key's refusals say nothing about the new one. */
+export function forgetSecretHealth(tenant: string, name: string): void {
+  try {
+    const data = read(tenant);
+    if (!(name in data)) return;
+    delete data[name];
+    write(tenant, data);
+  } catch {
+    // best effort
+  }
+}

package/src/secrets.ts

@@ -24,25 +24,52 @@ import path from "node:path";
 import { dataRoot } from "./paths.ts";
 import { platform } from "./platform.ts";
 
+/**
+ * GCM's auth tag, pinned to its full 16 bytes at every cipher call. Without
+ * `authTagLength`, Node accepts whatever length `setAuthTag` is handed — 4, 8,
+ * 12, 13, 14, 15 or 16 bytes — so a stored record whose tag had been cut to
+ * four bytes would still "verify" on 32 bits of authentication instead of
+ * 128. The tag comes from the record on disk; the record is the thing an
+ * attacker with file access would edit. One option makes a short tag an
+ * error rather than a weaker check.
+ */
+const GCM = { authTagLength: 16 } as const;
+
 // The account's own data key, when the platform holds one; the install key
 // otherwise. See platform.ts — locally there is no account key.
 const tenantKey = (tenant: string) => platform.tenantKey(tenant);
 import crypto from "node:crypto";
 import { assertSafeName } from "./store.ts";
+import { trimSlashes } from "./paths.ts";
 
 const keyFile = () => path.join(dataRoot(), ".secret-key");
 
 /** Which store a secret came from. */
 export type SecretScope = "account" | "workspace";
 
-function masterKey(): Buffer {
+/**
+ * The install's root secret, as text: FOLDRUN_SECRET_KEY when set, else the
+ * contents of `.secret-key` under the data root — created here, the first
+ * time anything asks, so there is never a moment when a derived token has
+ * nothing real to hang off. The vault, hook tokens, approval links and git
+ * push secrets all derive from this one value; rotating it invalidates every
+ * one of them together. Exported for webhook.ts, which used to fall back to
+ * a fixed dev string until the vault happened to create this file — so hook
+ * URLs were computable by anyone who read the source, and then changed the
+ * first time a secret was stored.
+ */
+export function installKeyMaterial(): string {
   const fromEnv = process.env.FOLDRUN_SECRET_KEY;
-  if (fromEnv) return crypto.createHash("sha256").update(fromEnv).digest();
+  if (fromEnv) return fromEnv;
   fs.mkdirSync(dataRoot(), { recursive: true });
   if (!fs.existsSync(keyFile())) {
     fs.writeFileSync(keyFile(), crypto.randomBytes(32).toString("hex"), { mode: 0o600 });
   }
-  return crypto.createHash("sha256").update(fs.readFileSync(keyFile(), "utf8")).digest();
+  return fs.readFileSync(keyFile(), "utf8");
+}
+
+function masterKey(): Buffer {
+  return crypto.createHash("sha256").update(installKeyMaterial()).digest();
 }
 
 // `workspace` undefined means the account store.
@@ -135,7 +162,7 @@ export function encryptValue(
 ): { iv: string; tag: string; data: string; k?: "tenant" } {
   const { key, mark } = keyFor(tenant);
   const iv = crypto.randomBytes(12);
-  const cipher = crypto.createCipheriv("aes-256-gcm", key, iv);
+  const cipher = crypto.createCipheriv("aes-256-gcm", key, iv, GCM);
   const enc = Buffer.concat([cipher.update(value, "utf8"), cipher.final()]);
   return {
     iv: iv.toString("base64"),
@@ -180,6 +207,7 @@ function decrypt(tenant: string, rec: StoredSecret | undefined): string | null {
       "aes-256-gcm",
       key,
       Buffer.from(rec.iv, "base64"),
+      GCM,
     );
     decipher.setAuthTag(Buffer.from(rec.tag, "base64"));
     return Buffer.concat([
@@ -357,7 +385,7 @@ export function rotateMasterKey(
       }
       let plain: string;
       try {
-        const d = crypto.createDecipheriv("aes-256-gcm", oldKey, Buffer.from(rec.iv, "base64"));
+        const d = crypto.createDecipheriv("aes-256-gcm", oldKey, Buffer.from(rec.iv, "base64"), GCM);
         d.setAuthTag(Buffer.from(rec.tag, "base64"));
         plain = Buffer.concat([d.update(Buffer.from(rec.data, "base64")), d.final()]).toString("utf8");
       } catch {
@@ -366,7 +394,7 @@ export function rotateMasterKey(
         continue;
       }
       const iv = crypto.randomBytes(12);
-      const c = crypto.createCipheriv("aes-256-gcm", newKey, iv);
+      const c = crypto.createCipheriv("aes-256-gcm", newKey, iv, GCM);
       const enc = Buffer.concat([c.update(plain, "utf8"), c.final()]);
       next[name] = {
         iv: iv.toString("base64"),
@@ -498,6 +526,9 @@ async function exchange(config: OAuth2Config, cacheKey: string): Promise<string>
  */
 export async function materializeSecrets(
   env: Record<string, string>,
+  /** Whose secrets these are, when the caller knows: lets the platform keep
+   *  each OAuth connection's health and quote a reconnect link on failure. */
+  ctx?: { tenant: string; workspace?: string },
 ): Promise<Record<string, string>> {
   const out: Record<string, string> = { ...env };
   for (const [name, value] of Object.entries(env)) {
@@ -518,8 +549,11 @@ export async function materializeSecrets(
         const fp = crypto.createHash("sha256").update(grantMaterial).digest("hex").slice(0, 16);
         out[name] = await exchange(config, `${name}:${config.client_id}:${fp}`);
       } catch (err) {
-        throw new Error(`secret ${name}: ${err instanceof Error ? err.message : String(err)}`);
+        const message = err instanceof Error ? err.message : String(err);
+        const link = ctx ? await platform.noteOAuthRefresh(ctx, name, message).catch(() => undefined) : undefined;
+        throw new Error(`secret ${name}: ${message}${link ? ` — reconnect it in one click: ${link}` : ""}`);
       }
+      if (ctx) await platform.noteOAuthRefresh(ctx, name, null).catch(() => undefined);
     } else if (value.startsWith(SERVICE_ACCOUNT_PREFIX)) {
       let config: ServiceAccountConfig;
       try {
@@ -570,7 +604,7 @@ export function setServiceAccountSecret(
   for (const field of ["token_url", "issuer", "private_key", "scope"] as const) {
     if (!config[field]?.trim()) throw new Error(`service account secret needs ${field}`);
   }
-  if (!/BEGIN [A-Z ]*PRIVATE KEY/.test(config.private_key)) {
+  if (!/BEGIN [A-Z ]{0,40}PRIVATE KEY/.test(config.private_key)) {
     throw new Error("private_key must be a PEM key (from the service account JSON)");
   }
   setSecret(tenant, name, SERVICE_ACCOUNT_PREFIX + JSON.stringify(config), workspace, "service-account");
@@ -692,7 +726,7 @@ export function setSshSecret(tenant: string, name: string, config: SshConfig, wo
   const hasKey = Boolean(config.private_key?.trim());
   const hasPassword = Boolean(config.password);
   if (hasKey === hasPassword) throw new Error("ssh secret needs a private key or a password (not both)");
-  if (hasKey && !/BEGIN [A-Z ]*PRIVATE KEY/.test(config.private_key!)) {
+  if (hasKey && !/BEGIN [A-Z ]{0,40}PRIVATE KEY/.test(config.private_key!)) {
     throw new Error("private_key must be a PEM/OpenSSH private key");
   }
   if (hasPassword && /[\n\r]/.test(config.password!)) throw new Error("ssh password cannot contain newlines");
@@ -737,7 +771,7 @@ export function setApiSecret(tenant: string, name: string, config: ApiConfig, wo
   if (base && !/^https?:\/\//.test(base)) throw new Error("base_url must be http(s)");
   setSecret(
     tenant, name,
-    API_PREFIX + JSON.stringify({ ...(base ? { base_url: base.replace(/\/+$/, "") } : {}), headers: config.headers }),
+    API_PREFIX + JSON.stringify({ ...(base ? { base_url: trimSlashes(base) } : {}), headers: config.headers }),
     workspace, "api",
   );
 }

package/src/starter.ts

@@ -319,6 +319,7 @@ timezone: Australia/Sydney
 model: default           # a default for every step here
 effort: low
 overlap: skip            # skip | queue — what a second run does while one is live
+priority: normal         # high | normal | low — where its runs stand in the queue
 ---
 
 1. [[researcher]] — Find three sources on {{topic}}.
@@ -354,7 +355,10 @@ Indented under a step, any of:
 | \`preview: draft/*.mdx\` | what the approval box shows — paths under \`storage/\`, globs allowed |
 | \`delegate: [[a]], [[b]]\` | hand the step to whichever fits, up to 5 |
 | \`output: json\` | the reply ends with one JSON value; the next step gets it as data, \`each: items\` fans out over it |
+| \`schema: {...}\` | with \`output: json\` — the JSON Schema the value must satisfy; inline, a YAML block under the key, or a file path |
 | \`each: items\` | run once per element of the previous step's JSON array |
+| \`parallel: 3\` | how many \`each:\` instances run at once — default all |
+| \`max_turns: 20\` | the most model turns before the step is stopped, beside \`budget:\` and \`timeout:\` |
 
 ## Writing an agent