@foldrun/core 0.4.0 → 0.6.0

package/dist/src/notify.js

@@ -10,7 +10,7 @@
 //
 //   notify:
 //     url: ${SLACK_WEBHOOK_URL}       # a secret name, or a literal URL
-//     events: [failed, awaiting-approval, completed]
+//     events: [failed, awaiting-approval, completed, blocked]
 //
 // Or, for people who live in an inbox rather than a channel:
 //
@@ -32,26 +32,34 @@
 // every success gets muted, which un-pings the failures too.
 import crypto from "node:crypto";
 import { readAgentsMd } from "./runner.js";
-import { accountDir, workspaceDir, runCost, readFlow } from "./store.js";
+import { accountDir, workspaceDir, runCost, readFlow, runVerdict } from "./store.js";
 import { getSecret } from "./secrets.js";
 import { publicUrl } from "./webhook.js";
 import { approveLinkPath, approveLinkTtlMs } from "./approvals.js";
 import { noteSecretUse, healthKey } from "./secret-health.js";
+import { fetchUntrusted } from "./untrusted-fetch.js";
+import { platform } from "./platform.js";
+import { sendMail } from "./mail.js";
 /**
- * The platform's own mail: an invite, a low balance.
+ * The platform's own mail: an invite, a password reset, a low balance.
  *
- * These are from foldrun, about the platform, to the account's owner — so
- * they go through the platform's Resend key and sender (FOLDRUN_RESEND_API_KEY,
- * FOLDRUN_EMAIL_FROM), never through anything the customer configured.
- *
- * Without a platform key — the CLI on a laptop, a test — the account's own
- * key and EMAIL_FROM are the fallback.
+ * An account that configured its own mail — RESEND_API_KEY and EMAIL_FROM
+ * on the account — gets ALL of its mail from that sender, these included:
+ * a business that sends from its own domain does not want a second, foreign
+ * address turning up in its people's inboxes (2026-09-17, an owner found
+ * foldrun.io mail beside their own and asked for none of it). Only an
+ * account with no sender of its own falls to the platform's key and sender
+ * (FOLDRUN_RESEND_API_KEY, FOLDRUN_EMAIL_FROM), so invites still work
+ * before anyone has set mail up.
  */
 export function platformMail(tenant) {
+    const own = accountMail(tenant);
+    if (own && getSecret(tenant, "EMAIL_FROM")?.value?.trim())
+        return own;
     const key = process.env.FOLDRUN_RESEND_API_KEY;
     if (key)
         return { key, from: process.env.FOLDRUN_EMAIL_FROM || "foldrun <hello@foldrun.io>" };
-    return accountMail(tenant);
+    return own;
 }
 /** The account's own Resend key and sender, or null when it has none. */
 export function accountMail(tenant) {
@@ -142,6 +150,85 @@ export function signatureHeaders(tenant, workspace, config, body) {
     const plain = crypto.createHmac("sha256", secret.value).update(body).digest("hex");
     return { "x-foldrun-timestamp": timestamp, "x-foldrun-signature": `sha256=${mac}`, "x-signature": plain };
 }
+// ------------------------------------------------------------ deliveries
+//
+// A webhook used to be one POST with a timeout: a receiver that was down
+// for the minute a run failed never heard about it, and nothing anywhere
+// said so. Now every send is a DELIVERY with an id, attempted here and —
+// on a platform — retried from the worker on the backoff below, each
+// attempt recorded (webhook-deliveries.ts in the platform).
+/** When each attempt is made, counted from the one before: at once, then
+ *  1m, 5m, 30m, 2h and 6h. Six attempts over about eight and a half hours,
+ *  then the delivery is given up and stays in the log to be redelivered. */
+export const WEBHOOK_BACKOFF_MS = [0, 60_000, 5 * 60_000, 30 * 60_000, 2 * 3600_000, 6 * 3600_000];
+/** When attempt `n + 1` is due after attempt `n` (1-based) failed at
+ *  `at`, or null when `n` was the last. */
+export function nextAttemptAt(n, at) {
+    return n < WEBHOOK_BACKOFF_MS.length ? at + WEBHOOK_BACKOFF_MS[n] : null;
+}
+/** The delivery id: sent as X-Foldrun-Delivery on every attempt of one
+ *  delivery, redeliveries included, so a receiver dedupes on it. */
+export function newDeliveryId() {
+    return `dlv_${crypto.randomUUID().replace(/-/g, "")}`;
+}
+/** The URL with ${SECRET}s resolved, and how it may be shown: a Slack hook
+ *  URL is itself a credential, so it is never echoed resolved. */
+export function webhookTarget(tenant, workspace, config = notifyConfig(tenant, workspace)) {
+    const raw = config?.url ?? "";
+    const url = raw.replace(/\$\{([A-Z][A-Z0-9_]*)\}/g, (whole, name) => {
+        const hit = getSecret(tenant, name, workspace);
+        return hit ? hit.value : whole;
+    });
+    const shown = raw.includes("${") ? raw : raw ? `${raw.slice(0, 40)}${raw.length > 40 ? "…" : ""}` : "none";
+    return { url: !url || url.includes("${") ? null : url, shown };
+}
+/**
+ * One attempt at one delivery, against the workspace's notify: as it reads
+ * NOW — a URL fixed between attempts is the one the retry goes to. Signed
+ * afresh each time (the timestamp is inside the signature, so a stale
+ * signature is one a receiver should refuse). Never throws.
+ */
+export async function attemptWebhook(tenant, workspace, d) {
+    const at = new Date().toISOString();
+    const started = Date.now();
+    const config = notifyConfig(tenant, workspace);
+    const base = { n: d.attempt, at, ...(d.manual ? { manual: true } : {}) };
+    if (!config?.url) {
+        return { ...base, ok: false, statusCode: null, durationMs: 0, response: null, error: "this workspace's notify: has no url any more" };
+    }
+    const { url } = webhookTarget(tenant, workspace, config);
+    if (!url) {
+        return { ...base, ok: false, statusCode: null, durationMs: 0, response: null, error: "the secret named in notify.url is not in this account's vault" };
+    }
+    try {
+        const res = await fetchUntrusted(url, {
+            method: "POST",
+            headers: {
+                "content-type": "application/json",
+                "user-agent": "foldrun-webhooks/1",
+                "x-foldrun-delivery": d.id,
+                "x-foldrun-event": d.event,
+                "x-foldrun-attempt": String(d.attempt),
+                ...signatureHeaders(tenant, workspace, config, d.body),
+            },
+            body: d.body,
+            signal: AbortSignal.timeout(Number(process.env.FOLDRUN_WEBHOOK_TIMEOUT_MS) || 8000),
+        });
+        const text = (await res.text().catch(() => "")).slice(0, 500);
+        return {
+            ...base,
+            ok: res.ok,
+            statusCode: res.status,
+            durationMs: Date.now() - started,
+            response: text || null,
+            error: res.ok ? null : `HTTP ${res.status}`,
+        };
+    }
+    catch (err) {
+        const why = err instanceof Error ? (err.name === "TimeoutError" ? "timed out" : err.message) : String(err);
+        return { ...base, ok: false, statusCode: null, durationMs: Date.now() - started, response: null, error: why.slice(0, 500) };
+    }
+}
 /**
  * The approve/reject links for a run waiting on a person, or null when the
  * install cannot say where it lives. A notification that only says "waiting
@@ -232,33 +319,21 @@ export async function sendPlainNotification(tenant, workspace, msg) {
             const mail = notifyMail(tenant);
             if (!mail)
                 return false;
-            const res = await fetch("https://api.resend.com/emails", {
-                method: "POST",
-                headers: { authorization: `Bearer ${mail.key}`, "content-type": "application/json" },
-                body: JSON.stringify({
-                    from: mail.from,
-                    to: config.email,
-                    subject: `${msg.headline}`.slice(0, 160),
-                    text: `${msg.headline}\n\n${msg.detail}\n\nworkspace: ${workspace}\n${msg.flow ? `flow: ${msg.flow}\n` : ""}${msg.runId ? `run: ${msg.runId}\n` : ""}`,
-                }),
-                signal: AbortSignal.timeout(8000),
+            const r = await sendMail({
+                tenant,
+                mail,
+                to: config.email,
+                category: "run-alerts",
+                workspace,
+                subject: `${msg.headline}`.slice(0, 160),
+                text: `${msg.headline}\n\n${msg.detail}\n\nworkspace: ${workspace}\n${msg.flow ? `flow: ${msg.flow}\n` : ""}${msg.runId ? `run: ${msg.runId}\n` : ""}`,
             });
-            return res.ok;
+            return r.ok;
         }
-        const url = (config.url ?? "").replace(/\$\{([A-Z][A-Z0-9_]*)\}/g, (whole, name) => {
-            const hit = getSecret(tenant, name, workspace);
-            return hit ? hit.value : whole;
-        });
-        if (!url || url.includes("${"))
+        if (!webhookTarget(tenant, workspace, config).url)
             return false;
-        const payload = JSON.stringify(body);
-        const res = await fetch(url, {
-            method: "POST",
-            headers: { "content-type": "application/json", ...signatureHeaders(tenant, workspace, config, payload) },
-            body: payload,
-            signal: AbortSignal.timeout(5000),
-        });
-        return res.ok;
+        const r = await platform.deliverWebhook(tenant, workspace, { event: msg.event, body: JSON.stringify(body), retry: true });
+        return r.ok;
     }
     catch (err) {
         console.error(`[foldrun] notify (${msg.event}): ${tenant}/${workspace} →`, err instanceof Error ? err.message : err);
@@ -289,7 +364,7 @@ export async function sendTestNotification(tenant, workspace) {
         };
     }
     const sent = new Date().toISOString();
-    const headline = "\u2713 foldrun test notification";
+    const headline = "\u2713 test notification";
     const detail = `Sent from ${workspace} at ${sent}. If you are reading this, failures and approval gates will reach you here too.`;
     if (config.email) {
         const mail = notifyMail(tenant);
@@ -300,59 +375,56 @@ export async function sendTestNotification(tenant, workspace) {
                 detail: "email is configured but there is no mail credential — set RESEND_API_KEY (and EMAIL_FROM) on the account, or FOLDRUN_RESEND_API_KEY on the platform",
             };
         }
-        try {
-            const res = await fetch("https://api.resend.com/emails", {
-                method: "POST",
-                headers: { authorization: `Bearer ${mail.key}`, "content-type": "application/json" },
-                body: JSON.stringify({ from: mail.from, to: config.email, subject: headline, text: `${headline}\n\n${detail}\n` }),
-                signal: AbortSignal.timeout(8000),
-            });
-            noteMailUse(tenant, res.status);
-            const body = (await res.text().catch(() => "")).replace(/\s+/g, " ").slice(0, 400);
-            return res.ok
-                ? { ok: true, destination: `${config.email} (from ${mail.from})`, detail: "accepted by Resend for delivery" }
-                : { ok: false, destination: config.email, detail: `HTTP ${res.status} from Resend — ${body}` };
-        }
-        catch (err) {
-            return { ok: false, destination: config.email, detail: err instanceof Error ? err.message : String(err) };
+        const r = await sendMail({ tenant, mail, to: config.email, category: "run-alerts", workspace, subject: headline, text: `${headline}\n\n${detail}\n` });
+        if (r.status !== undefined)
+            noteMailUse(tenant, r.status);
+        if (r.suppressed.length && !r.sent.length && !r.error) {
+            return {
+                ok: false,
+                destination: config.email,
+                detail: `not sent: ${r.suppressed.join(", ")} turned run alerts off — turn them back on in Profile → Notifications (or by \`foldrun notifications set run-alerts on\`)`,
+            };
         }
+        return r.ok
+            ? { ok: true, destination: `${config.email} (from ${mail.from})`, detail: `accepted by Resend for delivery${r.suppressed.length ? ` — not to ${r.suppressed.join(", ")}, who turned run alerts off` : ""}` }
+            : { ok: false, destination: config.email, detail: `${r.status ? `HTTP ${r.status} from Resend — ` : ""}${r.error ?? "not sent"}` };
     }
-    const raw = config.url ?? "";
-    const url = raw.replace(/\$\{([A-Z][A-Z0-9_]*)\}/g, (whole, name) => {
-        const hit = getSecret(tenant, name, workspace);
-        return hit ? hit.value : whole;
-    });
     // The destination is echoed back with the secret still unresolved: a
     // Slack webhook URL is itself a credential, and a page that prints it is
     // a page that leaks it into a screenshot.
-    const shown = raw.includes("${") ? raw : `${url.slice(0, 40)}…`;
-    if (!url || url.includes("${")) {
+    const { url, shown } = webhookTarget(tenant, workspace, config);
+    if (!url) {
         return { ok: false, destination: shown, detail: `the secret named in notify.url is not in this account's vault` };
     }
-    try {
-        const payload = JSON.stringify({ text: `${headline} — ${detail}`, workspace, status: "test", summary: detail });
-        const res = await fetch(url, {
-            method: "POST",
-            headers: { "content-type": "application/json", ...signatureHeaders(tenant, workspace, config, payload) },
-            body: payload,
-            signal: AbortSignal.timeout(8000),
-        });
-        const body = (await res.text().catch(() => "")).replace(/\s+/g, " ").slice(0, 400);
-        return res.ok
-            ? {
-                ok: true,
-                destination: shown,
-                detail: config.signingSecret ? "accepted, signed with " + config.signingSecret : "accepted",
-            }
-            : { ok: false, destination: shown, detail: `HTTP ${res.status} — ${body}` };
-    }
-    catch (err) {
-        return { ok: false, destination: shown, detail: err instanceof Error ? err.message : String(err) };
-    }
+    const payload = JSON.stringify({ text: `${headline} — ${detail}`, workspace, status: "test", summary: detail });
+    // One attempt, recorded in the delivery log like any other — and the way
+    // back for an endpoint that was switched off after days of failures: a
+    // test that is accepted turns it on again.
+    const r = await platform.deliverWebhook(tenant, workspace, { event: "test", body: payload, retry: false });
+    return r.ok
+        ? {
+            ok: true,
+            destination: shown,
+            detail: `${config.signingSecret ? "accepted, signed with " + config.signingSecret : "accepted"} (delivery ${r.deliveryId})`,
+        }
+        : { ok: false, destination: shown, detail: `${r.statusCode ? `HTTP ${r.statusCode} — ` : ""}${r.detail.replace(/\s+/g, " ").slice(0, 400)}` };
 }
 export async function sendRunNotification(tenant, workspace, run) {
     const config = notifyConfig(tenant, workspace);
-    if (!config || !config.events.includes(run.status))
+    if (!config)
+        return false;
+    // A completed run whose work refused itself — the verdict is BLOCKED — is
+    // its own event, `blocked`. It follows `failed` the way the plain alerts
+    // do: whoever hears about failures hears about it, because "nothing was
+    // published" is the news a person needs whether a step crashed or an
+    // agent stopped itself. Naming `blocked` turns it on alone; a list
+    // without either stays silent.
+    const blocked = run.status === "completed" && runVerdict(run) === "BLOCKED";
+    if (blocked) {
+        if (!config.events.includes("blocked") && !config.events.includes("failed"))
+            return false;
+    }
+    else if (!config.events.includes(run.status))
         return false;
     // A test is not news. An eval case or an adhoc single-agent run is started
     // by a person who is watching it, and a desk that emails "completed" for
@@ -374,13 +446,15 @@ export async function sendRunNotification(tenant, workspace, run) {
     // says what the run is waiting for, so nobody hunts for a button that
     // the outside world is meant to press.
     const onEvent = waitingSteps.length > 0 && waitingSteps.every((s) => s.waitFor === "event");
-    const headline = run.status === "completed"
-        ? `✓ ${run.flow} completed`
-        : run.status === "failed"
-            ? `✗ ${run.flow} failed${failed.length ? ` at ${failed.join(", ")}` : ""}`
-            : onEvent
-                ? `⏳ ${run.flow} is waiting for an external event${waiting.length ? ` (${waiting.join(", ")})` : ""}`
-                : `⏸ ${run.flow} is waiting for your approval${waiting.length ? ` (${waiting.join(", ")})` : ""}`;
+    const headline = blocked
+        ? `⛔ ${run.flow} blocked`
+        : run.status === "completed"
+            ? `✓ ${run.flow} completed`
+            : run.status === "failed"
+                ? `✗ ${run.flow} failed${failed.length ? ` at ${failed.join(", ")}` : ""}`
+                : onEvent
+                    ? `⏳ ${run.flow} is waiting for an external event${waiting.length ? ` (${waiting.join(", ")})` : ""}`
+                    : `⏸ ${run.flow} is waiting for your approval${waiting.length ? ` (${waiting.join(", ")})` : ""}`;
     // What the run concluded, if it concluded anything. This is the whole point
     // of the message: "✓ health completed · $0.42" tells you a thing ran and
     // what it cost, and nothing at all about what it found. A scheduled desk
@@ -408,6 +482,7 @@ export async function sendRunNotification(tenant, workspace, run) {
         runId: run.id,
         flow: run.flow,
         status: run.status,
+        verdict: runVerdict(run),
         summary,
         costUsd: runCost(run),
         startedAt: run.startedAt,
@@ -421,60 +496,53 @@ export async function sendRunNotification(tenant, workspace, run) {
                 console.error(`[foldrun] notify: email configured but no mail credential — set RESEND_API_KEY (and EMAIL_FROM) on the account ${tenant}, or FOLDRUN_RESEND_API_KEY on the platform`);
                 return false;
             }
-            const res = await fetch("https://api.resend.com/emails", {
-                method: "POST",
-                headers: { authorization: `Bearer ${mail.key}`, "content-type": "application/json" },
-                body: JSON.stringify({
-                    from: mail.from,
-                    to: config.email,
-                    // The subject is the headline and what the run concluded, and
-                    // nothing else. Ids and costs belong in the body: a subject line is
-                    // read in a list, where the only useful question it can answer is
-                    // whether this one needs opening.
-                    subject: summary ? `${headline} — ${summary}`.slice(0, 160) : headline,
-                    text: `${headline}\n` +
-                        (summary ? `\n${summary}\n` : soFar ? `\n${soFar}\n` : "") +
-                        (asks.length ? `\n${asks.map((a) => `Question: ${a}`).join("\n")}\n` : "") +
-                        `\nworkspace: ${workspace}\nrun: ${run.id}\nflow: ${run.flow}\n` +
-                        `cost: $${runCost(run).toFixed(4)}\nstarted: ${run.startedAt}\nfinished: ${run.finishedAt ?? "-"}\n` +
-                        (links
-                            ? links.links.length === 1
-                                ? `\nApprove: ${links.approveUrl}\nReject: ${links.rejectUrl}\n`
-                                : links.links.map((l) => `\nStep ${l.step + 1} (${l.agent})\nApprove: ${l.approveUrl}\nReject: ${l.rejectUrl}\n`).join("")
-                            : ""),
-                }),
-                signal: AbortSignal.timeout(8000),
+            // A gate is its own category: a person may want failures and not
+            // the approval requests (they decide from the dashboard), or the
+            // other way round.
+            const category = run.status === "awaiting-approval" && !onEvent ? "approvals" : "run-alerts";
+            const r = await sendMail({
+                tenant,
+                mail,
+                to: config.email,
+                category,
+                workspace,
+                // The subject is the headline and what the run concluded, and
+                // nothing else. Ids and costs belong in the body: a subject line is
+                // read in a list, where the only useful question it can answer is
+                // whether this one needs opening.
+                subject: summary ? `${headline} — ${summary}`.slice(0, 160) : headline,
+                text: `${headline}\n` +
+                    (summary ? `\n${summary}\n` : soFar ? `\n${soFar}\n` : "") +
+                    (asks.length ? `\n${asks.map((a) => `Question: ${a}`).join("\n")}\n` : "") +
+                    `\nworkspace: ${workspace}\nrun: ${run.id}\nflow: ${run.flow}\n` +
+                    `cost: $${runCost(run).toFixed(4)}\nstarted: ${run.startedAt}\nfinished: ${run.finishedAt ?? "-"}\n` +
+                    (links
+                        ? links.links.length === 1
+                            ? `\nApprove: ${links.approveUrl}\nReject: ${links.rejectUrl}\n`
+                            : links.links.map((l) => `\nStep ${l.step + 1} (${l.agent})\nApprove: ${l.approveUrl}\nReject: ${l.rejectUrl}\n`).join("")
+                        : ""),
             });
-            noteMailUse(tenant, res.status);
-            if (!res.ok) {
+            if (r.status !== undefined)
+                noteMailUse(tenant, r.status);
+            if (!r.ok) {
                 // Resend's body says WHY — "domain not verified", "can only send to
                 // your own address" — and a status alone sent someone to the wrong
                 // dashboard for twenty minutes on 2026-09-06.
-                const why = (await res.text().catch(() => "")).replace(/\s+/g, " ").slice(0, 300);
-                console.error(`[foldrun] notify email: ${tenant}/${workspace} → HTTP ${res.status} ${why}`);
+                console.error(`[foldrun] notify email: ${tenant}/${workspace} → ${r.status ? `HTTP ${r.status} ` : ""}${r.error ?? ""}`);
                 return false;
             }
             return true;
         }
         // ${SECRET} so the Slack URL — itself a credential — can live in the
         // vault instead of in a file that gets committed.
-        const url = (config.url ?? "").replace(/\$\{([A-Z][A-Z0-9_]*)\}/g, (whole, name) => {
-            const hit = getSecret(tenant, name, workspace);
-            return hit ? hit.value : whole;
-        });
-        if (!url || url.includes("${")) {
+        if (!webhookTarget(tenant, workspace, config).url) {
             console.error(`[foldrun] notify: secret in URL not set for ${tenant}/${workspace}`);
             return false;
         }
-        const payload = JSON.stringify(body);
-        const res = await fetch(url, {
-            method: "POST",
-            headers: { "content-type": "application/json", ...signatureHeaders(tenant, workspace, config, payload) },
-            body: payload,
-            signal: AbortSignal.timeout(5000),
-        });
-        if (!res.ok) {
-            console.error(`[foldrun] notify: ${tenant}/${workspace} → HTTP ${res.status}`);
+        const event = blocked ? "blocked" : run.status;
+        const r = await platform.deliverWebhook(tenant, workspace, { event, body: JSON.stringify(body), retry: true });
+        if (!r.ok) {
+            console.error(`[foldrun] notify: ${tenant}/${workspace} → ${r.statusCode ? `HTTP ${r.statusCode}` : r.detail}${r.retrying ? " (will retry)" : ""}`);
             return false;
         }
         return true;

package/dist/src/openapi.js

@@ -22,6 +22,7 @@ import path from "node:path";
 import crypto from "node:crypto";
 import { createRequire } from "node:module";
 import { dataRoot } from "./paths.js";
+import { fetchUntrusted } from "./untrusted-fetch.js";
 /** The most operations one API turns into tools. Past this a tool list
  *  stops helping the model choose and starts crowding the prompt. */
 export const MAX_OPERATIONS = 60;
@@ -289,7 +290,7 @@ export async function prefetchOpenApi(tenant, sources) {
         if (cached && Date.now() - Date.parse(cached.fetchedAt) < CACHE_TTL_MS)
             continue;
         try {
-            const res = await fetch(source, {
+            const res = await fetchUntrusted(source, {
                 signal: AbortSignal.timeout(10_000),
                 headers: { accept: "application/json, application/yaml, text/yaml, */*" },
             });
@@ -325,15 +326,16 @@ export function attachOperations(specs, tenant, workspaceRoot) {
         if (!spec.openapi)
             return spec;
         const { doc, error } = loadOpenApiDocument(tenant, workspaceRoot, spec.openapi);
+        const closed = spec.operations?.length ? " — its operations: allowlist fails closed, so it has no tools this run" : "";
         if (error) {
-            warnings.push(`api ${spec.name}: openapi ${error}`);
+            warnings.push(`api ${spec.name}: openapi ${error}${closed}`);
             return spec;
         }
         const parsed = parseOpenApi(doc, { operations: spec.operations, methods: spec.methods });
         for (const w of parsed.warnings)
             warnings.push(`api ${spec.name}: ${w}`);
         if (!parsed.operations.length) {
-            warnings.push(`api ${spec.name}: the OpenAPI document yielded no operations (methods: ${spec.methods.join(", ")})`);
+            warnings.push(`api ${spec.name}: the OpenAPI document yielded no operations (methods: ${spec.methods.join(", ")})${closed}`);
             return spec;
         }
         return { ...spec, resolvedOperations: parsed.operations };