@foldrun/core 2.3.0 → 2.5.0

package/src/compare.ts

@@ -0,0 +1,146 @@
+// Two runs of one flow side by side (FOL-72 step 6): "why did this get
+// worse" read off the records — which files changed between the revisions
+// they ran, and step by step what differs: status, the instruction, the
+// model, time, tokens, model calls, the tools it used and the errors it hit.
+// Pure; the caller hands in the changed files (history.ts changedPathsBetween).
+
+import type { RunRecord, StepRecord } from "./store.ts";
+
+export interface StepSide {
+  status: string;
+  instruction: string;
+  model?: string;
+  secs: number | null;
+  tokensIn: number;
+  tokensOut: number;
+  modelCalls: number;
+  /** Tool name → how many calls. */
+  tools: Record<string, number>;
+  toolErrors: number;
+  /** The first errors it wrote, each one line. */
+  errors: string[];
+  /** The first line of what it concluded. */
+  said: string | null;
+}
+
+export interface StepDiff {
+  /** 1-based, as the trace numbers steps. */
+  step: number;
+  agent: string;
+  base: StepSide | null;
+  run: StepSide | null;
+  /** What differs, in words, most telling first; [] when nothing does. */
+  changes: string[];
+}
+
+export interface RunComparison {
+  base: { id: string; status: string; startedAt: string; revision: string | null };
+  run: { id: string; status: string; startedAt: string; revision: string | null };
+  revisionChanged: boolean;
+  /** Files changed between the two revisions; null when that cannot be said. */
+  changedFiles: string[] | null;
+  steps: StepDiff[];
+}
+
+const firstLine = (s: string | null | undefined, n = 200) => {
+  const l = (s ?? "").split("\n").map((x) => x.trim()).find(Boolean);
+  return l ? l.slice(0, n) : null;
+};
+
+function side(s: StepRecord): StepSide {
+  const tools: Record<string, number> = {};
+  let toolErrors = 0;
+  for (const e of s.events) {
+    if (e.type !== "tool") continue;
+    // A pair is one call: count the call, and its completion's error.
+    if (e.ms == null) tools[e.text] = (tools[e.text] ?? 0) + 1;
+    else if (e.err) toolErrors++;
+  }
+  const start = s.startedAt ? Date.parse(s.startedAt) : NaN;
+  const end = s.finishedAt ? Date.parse(s.finishedAt) : NaN;
+  return {
+    status: s.status,
+    instruction: s.instruction,
+    ...(s.model ? { model: s.model } : {}),
+    secs: Number.isFinite(start) && Number.isFinite(end) ? Math.max(0, Math.round((end - start) / 1000)) : null,
+    tokensIn: s.tokens?.input ?? 0,
+    tokensOut: s.tokens?.output ?? 0,
+    modelCalls: s.calls?.length ?? s.turns ?? 0,
+    tools,
+    toolErrors,
+    errors: s.events.filter((e) => e.type === "error").slice(0, 3).map((e) => firstLine(e.text) ?? ""),
+    said: firstLine(s.conclusion ?? s.result),
+  };
+}
+
+const k = (n: number) => (n >= 1000 ? `${(n / 1000).toFixed(1)}k` : String(n));
+
+/** Much more or much less: by a quarter and by a real amount. */
+const moved = (a: number, b: number, floor: number) => Math.abs(b - a) >= floor && (a === 0 ? b > 0 : Math.abs(b - a) / a >= 0.25);
+
+function changes(a: StepSide | null, b: StepSide | null): string[] {
+  if (!a && b) return ["new step: not in the base run"];
+  if (a && !b) return ["not in this run"];
+  if (!a || !b) return [];
+  const out: string[] = [];
+  if (a.status !== b.status) out.push(`status: ${a.status} → ${b.status}`);
+  const newErrors = b.errors.filter((e) => !a.errors.includes(e));
+  if (newErrors.length) out.push(`new error: ${newErrors[0]}`);
+  if (a.instruction.trim() !== b.instruction.trim()) out.push("instruction changed");
+  if ((a.model ?? "") !== (b.model ?? "")) out.push(`model: ${a.model ?? "default"} → ${b.model ?? "default"}`);
+  if (b.toolErrors > a.toolErrors) out.push(`tool errors: ${a.toolErrors} → ${b.toolErrors}`);
+  const names = new Set([...Object.keys(a.tools), ...Object.keys(b.tools)]);
+  const toolDiff = [...names]
+    .map((n) => ({ n, d: (b.tools[n] ?? 0) - (a.tools[n] ?? 0) }))
+    .filter((x) => x.d !== 0)
+    .sort((x, y) => Math.abs(y.d) - Math.abs(x.d))
+    .slice(0, 4)
+    .map((x) => `${x.d > 0 ? "+" : "−"}${x.n} ×${Math.abs(x.d)}`);
+  if (toolDiff.length) out.push(`tools: ${toolDiff.join(", ")}`);
+  if (a.secs !== null && b.secs !== null && moved(a.secs, b.secs, 10)) out.push(`took ${a.secs}s → ${b.secs}s`);
+  if (moved(a.tokensIn, b.tokensIn, 5000)) out.push(`tokens in ${k(a.tokensIn)} → ${k(b.tokensIn)}`);
+  if (moved(a.modelCalls, b.modelCalls, 3)) out.push(`model calls ${a.modelCalls} → ${b.modelCalls}`);
+  if (a.said !== b.said && b.said) out.push(`said: ${b.said}`);
+  return out;
+}
+
+/** Steps lined up: the flow's own steps by position where the agents agree,
+ *  fan-out instances by agent and order. */
+function align(a: StepRecord[], b: StepRecord[]): [StepRecord | null, StepRecord | null][] {
+  const pairs: [StepRecord | null, StepRecord | null][] = [];
+  const used = new Set<number>();
+  b.forEach((s, i) => {
+    let j = a[i] && a[i]!.agent === s.agent && !used.has(i) ? i : -1;
+    if (j < 0) j = a.findIndex((x, m) => !used.has(m) && x.agent === s.agent);
+    if (j >= 0) used.add(j);
+    pairs.push([j >= 0 ? a[j]! : null, s]);
+  });
+  a.forEach((s, m) => !used.has(m) && pairs.push([s, null]));
+  return pairs;
+}
+
+export function compareRuns(base: RunRecord, run: RunRecord, changedFiles: string[] | null = null): RunComparison {
+  const head = (r: RunRecord) => ({ id: r.id, status: r.status, startedAt: r.startedAt, revision: r.revision ?? null });
+  return {
+    base: head(base),
+    run: head(run),
+    revisionChanged: (base.revision ?? null) !== (run.revision ?? null),
+    changedFiles,
+    steps: align(base.steps, run.steps).map(([a, b], i) => {
+      const sa = a ? side(a) : null;
+      const sb = b ? side(b) : null;
+      return { step: b ? run.steps.indexOf(b) + 1 : i + 1, agent: (b ?? a)!.agent, base: sa, run: sb, changes: changes(sa, sb) };
+    }),
+  };
+}
+
+/** The run to compare a run with when none is named: the newest earlier run
+ *  of the same flow that finished well (completed, not BLOCKED or BAD), or
+ *  null. `outcomeOf` is the caller's, so this stays free of the verdict reader. */
+export function lastGoodBefore<R extends { id: string; flow: string; startedAt: string }>(runs: R[], run: { id: string; flow: string; startedAt: string }, good: (r: R) => boolean): R | null {
+  return (
+    runs
+      .filter((r) => r.flow === run.flow && r.id !== run.id && r.startedAt < run.startedAt && good(r))
+      .sort((a, b) => b.startedAt.localeCompare(a.startedAt))[0] ?? null
+  );
+}

package/src/completions.ts

@@ -77,6 +77,7 @@ const TOOL_GROUPS: Completion[] = [
   { label: "code", hint: "Bash — runs anything in the sandbox" },
   { label: "web", hint: "search, fetch, browse, crawl, map, extract, answer, monitor — providers under web:" },
   { label: "ask", hint: "ask_person — ask the person running the desk mid-step, and wait for the answer" },
+  { label: "peers", hint: "peers_list + peers_message — the colleagues running beside it in its parallel group, and a message to one of them" },
   { label: "install", hint: "install apt, pip or npm packages mid-step — every agent has it; disallowedTools: [install] removes it" },
 ];
 

package/src/deploy.ts

@@ -33,20 +33,26 @@ import {
   assertSafeName,
   type DeployFile,
   notifyWorkspaceChanged,
+  accountDir,
+  frontmatterErrorText,
 } from "./store.ts";
+import { readAgentsMdKeys } from "./agents-md.ts";
+import { libraryTools } from "./library.ts";
 import { lintFlow } from "./flow-lint.ts";
 import { refNames } from "./refs.ts";
-import matter from "gray-matter";
+import matter from "./matter.ts";
 import { conformanceIssues } from "./okf.ts";
 import { timezoneProblem } from "./clock.ts";
 import { webProblems, webWarnings, runnerEngines } from "./providers.ts";
 import { readLimits } from "./limits.ts";
+import { parseAlerts } from "./alerts.ts";
+import { parseTelemetry } from "./otel-config.ts";
 import { networkProblems } from "./network.ts";
 import { actionProblems } from "./web-actions.ts";
 import { languageProblem } from "./language.ts";
 import { localeProblems } from "./locale.ts";
 import { entriesNoFollow } from "./paths.ts";
-import { filterIgnored, findingFingerprint, ruleForWhere, scanFiles, type ScanFinding, type ScanOptions } from "./scan.ts";
+import { filterIgnored, findingFingerprint, ruleForWhere, scanFiles, scanRule, type ScanFinding, type ScanOptions } from "./scan.ts";
 
 /** Directories a deploy never reads out of a source tree. */
 // `.cache` is what a run's tools leave in an agent's folder (HOME is that
@@ -121,9 +127,60 @@ export interface DeployResult extends DeployPlan {
 // agree on what counts as an agent.
 export const SUBAGENT_IMPORT_DIRS = [".agents/agents", ".claude/agents"] as const;
 
-export function readTree(dir: string): DeployFile[] {
+/** What agents write as they run, not what anyone authored: state/ and
+ *  memory/, the workspace's or one agent's. */
+const DATA_PATH = /^(?:agents\/[^/]+\/)?(?:state|memory)\//;
+
+/** Bounds for a reader that only looks at the text (the checks, the
+ *  assistant): on 20 Oct 2026 the daily rescan read one desk's 1.2 GB of
+ *  state/ into a 2 GB heap and crash-looped the worker. A deploy sets none. */
+export interface ReadTreeOptions {
+  /** Data (DATA_PATH) larger than this is left out, unread. */
+  maxDataFileBytes?: number;
+  /** Data past this many bytes in all, or this many files, is left out. */
+  maxDataBytes?: number;
+  maxDataFiles?: number;
+  /** Everything else past this many bytes in all: readTree throws
+   *  TreeTooLarge rather than return a part of the definitions, which a
+   *  check would read as missing agents and flows. */
+  maxSourceBytes?: number;
+  /** Each data file left out, and its size. */
+  onSkip?: (path: string, bytes: number) => void;
+}
+
+export class TreeTooLarge extends Error {
+  /** What an HTTP route answers with it. */
+  readonly status = 413;
+}
+
+export function readTree(dir: string, opts: ReadTreeOptions = {}): DeployFile[] {
   const root = path.resolve(dir);
   const out: DeployFile[] = [];
+  let dataBytes = 0;
+  let dataFiles = 0;
+  let sourceBytes = 0;
+  /** May this file be read? Data over its budget is left out; source over
+   *  its budget ends the read. */
+  const admit = (rel: string, bytes: number): boolean => {
+    if (DATA_PATH.test(rel)) {
+      const over =
+        (opts.maxDataFileBytes !== undefined && bytes > opts.maxDataFileBytes) ||
+        (opts.maxDataBytes !== undefined && dataBytes + bytes > opts.maxDataBytes) ||
+        (opts.maxDataFiles !== undefined && dataFiles >= opts.maxDataFiles);
+      if (over) {
+        opts.onSkip?.(rel, bytes);
+        return false;
+      }
+      dataBytes += bytes;
+      dataFiles++;
+      return true;
+    }
+    sourceBytes += bytes;
+    if (opts.maxSourceBytes !== undefined && sourceBytes > opts.maxSourceBytes) {
+      throw new TreeTooLarge(`the workspace's files other than state/ and memory/ are over ${Math.round(opts.maxSourceBytes / 1048576)} MiB — not read`);
+    }
+    return true;
+  };
 
   const walk = (abs: string, rel: string) => {
     let entries: string[];
@@ -148,6 +205,7 @@ export function readTree(dir: string): DeployFile[] {
       if (stat.isDirectory()) {
         walk(full, next);
       } else if (next === "AGENTS.md" || next === "project.md" || IN_WORKSPACE_DIR.test(next)) {
+        if (!admit(next, stat.size)) continue;
         out.push({ path: next, content: fs.readFileSync(full, "utf8") });
       }
     }
@@ -188,9 +246,12 @@ export function readTree(dir: string): DeployFile[] {
       if (nativeAgent.has(target) || out.some((fEntry) => fEntry.path === target)) continue;
       const full = path.join(root, importDir, entry);
       try {
-        if (fs.lstatSync(full).isSymbolicLink()) continue;
+        const st = fs.lstatSync(full);
+        if (st.isSymbolicLink()) continue;
+        if (!admit(target, st.size)) continue;
         out.push({ path: target, content: fs.readFileSync(full, "utf8") });
-      } catch {
+      } catch (err) {
+        if (err instanceof TreeTooLarge) throw err;
         // an unreadable import is skipped, not fatal
       }
     }
@@ -238,6 +299,90 @@ export function unopenedFrontmatter(raw: string): string | null {
  * advice (an agent that could leak, an unpinned package) is for `foldrun
  * check` and the Checks page, not for every deploy's output.
  */
+/**
+ * What `deploy_block:` in AGENTS.md refuses (FOL-58). `none`, the default:
+ * the scanner's findings are warnings and the deploy goes ahead, as it always
+ * did. `secrets`: a key or password written into a file is refused.
+ * `security`: every security finding is — secrets, hidden or injected text,
+ * the lethal trifecta, pipe-to-shell, an unpinned MCP server, a step that
+ * acts outside with no check or approval. A
+ * `foldrun-ignore` with a reason still accepts one.
+ */
+export const DEPLOY_BLOCK_MODES = ["none", "secrets", "security"] as const;
+export type DeployBlockMode = (typeof DEPLOY_BLOCK_MODES)[number];
+
+const blockMode = (v: unknown): DeployBlockMode | null =>
+  typeof v === "string" && (DEPLOY_BLOCK_MODES as readonly string[]).includes(v.trim()) ? (v.trim() as DeployBlockMode) : null;
+
+/** The workspace's own AGENTS.md, else the account's, else none. */
+export function deployBlockMode(files: DeployFile[], accountKeys: Record<string, unknown>): DeployBlockMode {
+  // project.md is the older name readAgentsMdKeys still reads (Codex review).
+  const own = files.find((f) => f.path === "AGENTS.md") ?? files.find((f) => f.path === "project.md");
+  let fromWorkspace: DeployBlockMode | null = null;
+  if (own) {
+    try {
+      fromWorkspace = blockMode((matter(own.content).data as Record<string, unknown>).deploy_block);
+    } catch {
+      // an AGENTS.md that does not parse is reported elsewhere
+    }
+  }
+  return fromWorkspace ?? blockMode(accountKeys.deploy_block) ?? "none";
+}
+
+function blockersFor(tenant: string, given: DeployFile[]): DeployIssue[] {
+  // The tree as it will be written: `./AGENTS.md` is saved as AGENTS.md, so
+  // it is read as that here too (Codex review).
+  // Two entries for one path: the save keeps the last, so this does too.
+  const byPath = new Map<string, DeployFile>();
+  for (const f of given) {
+    // path.normalize, as saveWorkspace writes it (a backslash is a filename
+    // character here, not a separator — Codex review).
+    const p = path.normalize(f.path);
+    byPath.set(p, { ...f, path: p });
+  }
+  const files = [...byPath.values()];
+  const account = readAgentsMdKeys(accountDir(tenant));
+  const mode = deployBlockMode(files, account);
+  if (mode === "none") return [];
+  // Nearest wins: a workspace tool of the same name replaces the library's,
+  // and the scanner adds the workspace's own outward tools itself. Named as
+  // the runtime names them — the definition's name:, else the file's.
+  const ownTools = new Set<string>();
+  for (const f of files) {
+    const m = /^tools\/([^/]+?)(?:\/tool)?\.md$/.exec(f.path);
+    if (!m) continue;
+    let data: Record<string, unknown> = {};
+    try {
+      data = matter(f.content).data as Record<string, unknown>;
+    } catch {
+      continue; // unreadable: the runtime skips it and keeps the library's
+    }
+    // Only a definition the runtime accepts replaces the library's; an
+    // invalid one is skipped there, and the library tool stays (Codex review).
+    const def = parseToolDef(data, m[1], f.content);
+    if (def) ownTools.add(def.name ?? m[1]);
+  }
+  const outwardTools = Object.entries(libraryTools(tenant))
+    .filter(([n, d]) => (d as { outward?: boolean }).outward === true && !ownTools.has(n))
+    .map(([n]) => n);
+  return deployBlockers(files, mode, { accountNetwork: account.network, outwardTools });
+}
+
+/** The findings `mode` turns into refusals, as deploy issues. */
+export function deployBlockers(files: DeployFile[], mode: DeployBlockMode, options: ScanOptions = {}): DeployIssue[] {
+  if (mode === "none") return [];
+  // Every finding checkFiles reports — the scanner's and the structural
+  // ones, flow/outward-ungated among them (Codex review) — with the same
+  // ignores honoured, kept by its rule's category.
+  const categories = new Set(mode === "secrets" ? ["secret"] : ["secret", "content", "security", "supply-chain"]);
+  return checkFiles(files, options)
+    .filter((f) => f.level !== "info" && categories.has(scanRule(f.rule)?.category ?? ""))
+    .map((f) => ({
+      where: `${f.file}${f.line ? `:${f.line}` : ""}`,
+      message: `${f.message} [${f.rule}] — refused: deploy_block: ${mode} in AGENTS.md`,
+    }));
+}
+
 export function deployWarnings(files: DeployFile[]): DeployIssue[] {
   const scanned = scanFiles(files, { only: ["secret", "content"] })
     .filter((f) => f.level !== "info")
@@ -261,19 +406,20 @@ export function checkFiles(files: DeployFile[], options: ScanOptions = {}): Scan
   };
   const own = [
     ...deployIssues(files).map((i) => finding(i, "error")),
-    ...structuralWarnings(files).map((i) => finding(i, "warn", i.kind)),
+    ...structuralWarnings(files, options.outwardTools).map((i) => finding(i, "warn", i.kind)),
   ];
   return [...filterIgnored(own, files), ...scanFiles(files, options)].sort(
     (a, b) => a.file.localeCompare(b.file) || (a.line ?? 0) - (b.line ?? 0) || a.rule.localeCompare(b.rule),
   );
 }
 
-function structuralWarnings(files: DeployFile[]): (DeployIssue & { kind?: string })[] {
+function structuralWarnings(files: DeployFile[], extraOutward: Iterable<string> = []): (DeployIssue & { kind?: string })[] {
   const front = (content: string): Record<string, unknown> => {
     try { return matter(content).data as Record<string, unknown>; } catch { return {}; }
   };
-  // Workspace tools that change something outside the workspace.
-  const outwardTools = new Set<string>();
+  // Tools that change something outside the workspace: the workspace's own,
+  // and the library's the caller resolved (nearest wins already applied).
+  const outwardTools = new Set<string>(extraOutward);
   for (const f of files) {
     const m = f.path.match(/^tools\/([^/]+?)(?:\/tool)?\.md$/);
     if (!m) continue;
@@ -282,6 +428,8 @@ function structuralWarnings(files: DeployFile[]): (DeployIssue & { kind?: string
     // `outward: true` is read off the frontmatter, the same test the runtime
     // and the CLI use; parseToolDef does not fold it onto the def.
     if (def && data.outward === true) outwardTools.add(def.name ?? m[1]);
+    // A workspace tool that is not outward replaces a library one that was.
+    else if (def) outwardTools.delete(def.name ?? m[1]);
   }
   // Agents that grant one of them.
   const outwardAgents: string[] = [];
@@ -342,6 +490,24 @@ export function deployIssues(files: DeployFile[]): DeployIssue[] {
     if (m) agents.add(m[1]);
   }
 
+  // Frontmatter whose YAML does not parse is a file the runtime cannot read:
+  // an agent with no tools and the default model, a flow that never fires,
+  // a tool that is skipped. Refused where it was written. Until 20 Oct 2026
+  // nothing said so — gray-matter's cache let a second parse pass — and one
+  // such agent file kept the worker from booting for every account.
+  // Definition files only: a skill is read leniently (an unquoted colon is
+  // quoted and retried, runner.ts safeMatter), and the markdown beside a
+  // definition — references, a tool's README — is prose, not frontmatter.
+  for (const f of files) {
+    if (!/^(AGENTS\.md|agents\/[^/]+\/agent\.md|flows\/[^/]+\.md|tools\/[^/]+\.md|tools\/[^/]+\/tool\.md|evals\/[^/]+\.md|hooks\/[^/]+\/hook\.md|\.claude\/agents\/[^/]+\.md)$/.test(f.path)) continue;
+    if (!/^\uFEFF?---\r?\n/.test(f.content)) continue;
+    try {
+      matter(f.content);
+    } catch (err) {
+      at(f.path, frontmatterErrorText(err));
+    }
+  }
+
   // `timezone:` is read at four levels and a value nothing can parse falls
   // silently through to the level above — so it is refused where it was
   // written, not discovered as a date that is a day out.
@@ -370,6 +536,11 @@ export function deployIssues(files: DeployFile[]): DeployIssue[] {
     // `network:` that cannot be read would leave the step on the level
     // above it — wider or narrower than the author wrote.
     for (const w of networkProblems(front.network)) at(f.path, w);
+    // `alerts:` that cannot be read would never alert (FOL-72).
+    for (const w of parseAlerts(front.alerts).problems) at(f.path, w);
+    // `telemetry:` that cannot be read would never send — and a token
+    // written into it would ship in the file.
+    for (const w of parseTelemetry(front.telemetry).problems) at(f.path, w);
     const lang = languageProblem(front.language);
     if (lang) at(f.path, lang);
     for (const w of localeProblems(front)) at(f.path, w);
@@ -509,7 +680,10 @@ export function planDeploy(tenant: string, workspace: string, files: DeployFile[
     added: added.sort(),
     updated: updated.sort(),
     removed: removed.sort(),
-    issues: deployIssues(files),
+    // deploy_block: (FOL-58) turns the scanner's findings into refusals —
+    // scanned with what the Checks page passes too: the account's network:
+    // and the library's outward tools (Codex review).
+    issues: [...deployIssues(files), ...blockersFor(tenant, files)],
     warnings: deployWarnings(files),
     blockedBy: runsInFlight(tenant, workspace),
   };

package/src/egress.ts

@@ -43,6 +43,7 @@
 // platform's (it needs a long-lived process and a Service); the pod side is
 // here in core, because it is what the runner image runs.
 
+import type { PeerInfo } from "./operator.ts";
 import { trimSlashes } from "./paths.ts";
 import type { TestEffect } from "./test-mode.ts";
 import type { NetworkLevel } from "./network.ts";
@@ -150,13 +151,22 @@ export type EgressLogLine = string | { text: string; effect: TestEffect };
  * means "no proxy here" and the runner falls back to materialising, exactly
  * as it always did.
  */
+
 export interface EgressHooks {
   /** `test` — the run is a test run: the proxy applies the outward-write
    *  policy in test-mode.ts to every request on this lease. Carried on the
    *  lease, not the request, because the pod is the party not trusted to
    *  say so. */
-  /** `step` — the step's index in run.steps; keys that step's operator inbox. */
-  lease(args: { tenant: string; runId: string; grant: EgressGrant; test?: boolean; step?: number }): Promise<EgressLease | null>;
+  /** `step` — the step's index in run.steps; keys that step's operator inbox.
+   *  `agent` and `peers` — this step's agent and the other steps of its
+   *  parallel group (tools: [peers]): the proxy serves the roster and
+   *  addresses a colleague's message to the right inbox. */
+  lease(args: { tenant: string; runId: string; grant: EgressGrant; test?: boolean; step?: number; agent?: string; peers?: PeerInfo[] }): Promise<EgressLease | null>;
+  /** The step is over — every attempt, however it ended (tools: [peers]).
+   *  A lease comes and goes per attempt, so it cannot say this: a sibling
+   *  between retries, or waiting for a slot, holds none and is not done.
+   *  Optional: a platform without it keeps every sibling reachable. */
+  stepEnded?(args: { tenant: string; runId: string; step: number }): Promise<void>;
 }
 
 /** Strip a header set of the proxy's own and hop-by-hop headers. */

package/src/evals.ts

@@ -31,18 +31,20 @@
 // they pass — no point paying a model to grade output already known to be
 // wrong.
 
-import { accountModelEnv } from "./model-env.ts";
+import { accountModelEnv, withModelEnv } from "./model-env.ts";
+import type { TranslatorSpec } from "./translator.ts";
 import { sdkEnv } from "./step-exec.ts";
 import fs from "node:fs";
 import path from "node:path";
 import { deployedCommit } from "./deploy.ts";
 import { latestRevisionId } from "./history.ts";
-import matter from "gray-matter";
+import matter from "./matter.ts";
 import { spawn } from "node:child_process";
 import { query } from "@anthropic-ai/claude-agent-sdk";
 import {
   workspaceDir,
   readRun,
+  runVerdict,
   writeWorkspaceFile,
   resolveModel,
   resolveEffort,
@@ -265,16 +267,26 @@ export function promoteRunToEval(
   tenant: string,
   workspace: string,
   runId: string,
-  opts: { evalName?: string; caseName?: string; expect?: string[]; by?: string } = {},
+  /** `should`: what a run that went wrong should have done, in a sentence —
+   *  the judge's claim for the case (FOL-72 step 10). */
+  opts: { evalName?: string; caseName?: string; expect?: string[]; should?: string; by?: string } = {},
 ): { file: string; created: boolean; caseName: string } {
   const run = readRun(tenant, workspace, runId);
   if (!run) throw refuse(`run ${runId} not found`, 404);
-  if (run.status !== "completed") {
-    throw refuse(`only a completed run can become an eval — this one is ${run.status}`, 409);
+  if (run.status !== "completed" && run.status !== "failed") {
+    throw refuse(`only a finished run can become an eval — this one is ${run.status}`, 409);
+  }
+  // A run that went wrong is the test worth having most — but its own answer
+  // is the wrong one to hold the next run to, so it needs to be told what
+  // was right (FOL-72 step 10).
+  const should = oneLine(opts.should ?? "");
+  const wentWrong = run.status === "failed" || ["BAD", "BLOCKED"].includes(runVerdict(run) ?? "");
+  if (wentWrong && !should && !(opts.expect && opts.expect.length)) {
+    throw refuse("this run went wrong, so its answer is not the one to keep — say what it should have done (`should`, a sentence for the judge) or give `expect` lines", 409);
   }
   const produced = run.steps.filter((s) => (s.result ?? "").trim());
   const last = produced[produced.length - 1];
-  if (!last) throw refuse("this run produced no result to hold the next one to", 409);
+  if (!last && !should && !(opts.expect && opts.expect.length)) throw refuse("this run produced no result to hold the next one to", 409);
   if (run.steps.length === 0 || !run.steps[0].instruction?.trim()) {
     throw refuse("this run has no task on record", 409);
   }
@@ -309,8 +321,10 @@ export function promoteRunToEval(
       }
       return `${m[1]}: ${m[2]}`;
     });
+  } else if (should) {
+    expect = [`judge: ${should.slice(0, 400)}`];
   } else {
-    const conclusion = oneLine(run.summary?.trim() || (last.result ?? "")).slice(0, 200);
+    const conclusion = oneLine(run.summary?.trim() || (last!.result ?? "")).slice(0, 200);
     expect = [`judge: reaches the same conclusion as the recorded run: ${conclusion}`];
   }
 
@@ -515,7 +529,7 @@ async function checkAssertion(
   agentDir: string,
   model: string,
   effort: Effort | null,
-  judgeEnv: () => Record<string, string | undefined> = () => sdkEnv(undefined),
+  judgeEnv: () => { env: Record<string, string | undefined>; translator?: TranslatorSpec | null } = () => ({ env: sdkEnv(undefined) }),
 ): Promise<AssertionResult> {
   const hay = output.toLowerCase();
   switch (assertion.type) {
@@ -550,13 +564,13 @@ async function checkAssertion(
     case "judge": {
       // The judge runs on the account's own model key, never quietly on the
       // platform's: a customer's eval is not the operator's bill.
-      let env: Record<string, string | undefined>;
+      let m: ReturnType<typeof judgeEnv>;
       try {
-        env = judgeEnv();
+        m = judgeEnv();
       } catch (err) {
         return { assertion, passed: false, detail: `judge not run — ${err instanceof Error ? err.message : String(err)}` };
       }
-      const verdict = await judge(assertion.value, output, model, effort, env);
+      const verdict = await withModelEnv(m, (env) => judge(assertion.value, output, model, effort, env));
       const passed = /^\s*PASS\b/i.test(verdict);
       return { assertion, passed, detail: verdict.slice(0, 300) || "judge returned nothing" };
     }
@@ -639,13 +653,7 @@ export async function runEval(
       const judges = testCase.expect.filter((a) => a.type === "judge");
       for (const a of cheap) assertions.push(await checkAssertion(a, output, agentDir, info.model, info.effort));
       if (assertions.every((r) => r.passed)) {
-        const judgeEnv = () => {
-          const m = accountModelEnv(tenant, workspace, () => sdkEnv(undefined));
-          if (m.format !== "anthropic") {
-            throw new Error(`this account's provider speaks ${m.format}; a judge needs an Anthropic-format provider`);
-          }
-          return m.env;
-        };
+        const judgeEnv = () => accountModelEnv(tenant, workspace, () => sdkEnv(undefined));
         for (const a of judges) assertions.push(await checkAssertion(a, output, agentDir, info.model, info.effort, judgeEnv));
       } else {
         for (const a of judges) {

package/src/frontmatter-fields.ts

@@ -50,6 +50,7 @@ const LOCALE = { validated: true } as const;
 const AGENT: FieldSpec[] = [
   { key: "name", type: "string", help: "kebab-case; the folder is the identity" },
   { key: "description", type: "string", help: "what it does and when — other agents and people read it" },
+  { key: "owner", type: "string", help: "the person accountable for it — an email or a name; else the workspace's, else the account's" },
   { key: "model", type: "string", help: "fast | default | max, an alias, or a full model id" },
   { key: "effort", type: "string", values: EFFORT_LEVELS, help: "how long it thinks" },
   { key: "size", type: "string", values: ["small", "large", "heavy"], help: "the sandbox it rents; anything else becomes large" },
@@ -86,6 +87,7 @@ const AGENTS_MD: FieldSpec[] = [
   { key: "mdagent_version", type: ["string", "number"], legacy: true, help: "older spelling of foldrun_version" },
   { key: "name", type: "string", help: "the workspace's name" },
   { key: "description", type: "string", help: "shown in the workspace list" },
+  { key: "owner", type: "string", help: "the person accountable for every agent and flow here that names none" },
   { key: "provider", type: "object", help: "the model endpoint every agent here uses unless it names its own" },
   { key: "runtime", type: "object", help: "packages for agents that declare none" },
   { key: "web", type: "object", validated: true, help: "web.browse defaults" },
@@ -102,18 +104,25 @@ const AGENTS_MD: FieldSpec[] = [
   { key: "budget", type: ["number", "string"], help: "USD per month, or N/day | N/week | N/month" },
   { key: "priority", type: "string", values: ["high", "normal", "low"], help: "account: where its runs stand in the queue" },
   { key: "concurrency", type: ["number", "string"], help: "account: runs at once" },
+  { key: "alerts", type: "object", validated: true, help: "thresholds that tell notify: — failure_rate (30%), min_runs, credits_day, run_minutes, stale_days, cost_spike, runs_per_hour, new_host" },
+  { key: "capture", type: "string", values: ["full", "metadata"], help: "what runs record of each tool call: full (its input and output, secrets masked) or metadata (name, size and time only)" },
+  { key: "network_enforce", type: "boolean", help: "true: a site off a step's network: list is refused, not only named in the trace. The account's or the workspace's true is enough" },
+  { key: "telemetry", type: "object", validated: true, help: "send each run as an OpenTelemetry trace — otlp: <https …/v1/traces>, headers: { authorization: ${SECRET} }" },
+  { key: "deploy_block", type: "string", values: ["none", "secrets", "security"], help: "refuse a deploy on scanner findings: secrets (a key written in) or security (any security warning)" },
   { key: "schedule", type: ["string", "object"], validated: true, legacy: true, help: "refused: only a flow has a clock" },
 ];
 
 const FLOW: FieldSpec[] = [
   { key: "name", type: "string", help: "the flow's name" },
   { key: "description", type: "string", help: "what it is for" },
+  { key: "owner", type: "string", help: "the person accountable for it — an email or a name; else the workspace's, else the account's" },
   { key: "trigger", type: "string", values: FLOW_TRIGGERS, help: "what starts it; any other value never fires" },
   { key: "schedule", type: "string", help: "cron, when trigger: schedule" },
   { key: "timezone", type: "string", validated: true, help: "the zone its schedule reads in" },
   { key: "model", type: "string", help: "every step's model unless the step says" },
   { key: "effort", type: "string", values: EFFORT_LEVELS, help: "every step's effort unless the step says" },
   { key: "overlap", type: "string", validated: true, help: "skip | queue" },
+  { key: "issues", type: "string", values: ["stay-open", "auto-close"], help: "what a good run does to this flow's open issues: stay-open (the default — a person resolves them) or auto-close" },
   { key: "priority", type: "string", validated: true, help: "high | normal | low" },
   { key: "catchup", type: "string", validated: true, help: "last | none" },
   { key: "on", type: "string", validated: true, help: "trigger: flow — completed | failed | blocked | any" },

package/src/frontmatter-schema.ts

@@ -6,6 +6,7 @@ import { FRONTMATTER, type FieldType, type SchemaKind } from "./frontmatter-fiel
 import { resolveEffort, parseDuration, parseInstant, parseRate, STEP_OPTION_KEYS } from "./store.ts";
 import { budgetProblem } from "./budget.ts";
 import { cronProblem } from "./cron.ts";
+import { modelSecretCollision } from "./model-credential.ts";
 
 export { FRONTMATTER, schemaKindOf, type FieldSpec, type FieldType, type SchemaKind } from "./frontmatter-fields.ts";
 
@@ -31,6 +32,7 @@ const CHECKS: Partial<Record<SchemaKind, Record<string, (value: unknown) => stri
     apis: apisProblem,
     budget: (v) => budgetProblem(v, ["run"]),
     ask: (v) => askProblem(v),
+    secrets: modelSecretCollision,
   },
   "agents-md": {
     budget: (v) => budgetProblem(v, ["day", "week", "month"]),
@@ -54,6 +56,7 @@ const CHECKS: Partial<Record<SchemaKind, Record<string, (value: unknown) => stri
     transport: (v) => (["http", "script", "mcp"].includes(String(v).toLowerCase()) ? null : `"${String(v)}" is not a transport — http, script or mcp`),
     methods: methodsProblem,
     rate: (v) => (parseRate(v) ? null : `"${shown(v)}" is not a rate — N/s, N/minute, N/hour, or a number per second, above 0`),
+    secrets: modelSecretCollision,
   },
   "eval": {
     effort: (v) => (resolveEffort(v) ? null : `"${String(v)}" is not an effort — low, medium, high, xhigh or max; the judge runs at none`),