@foldrun/core 2.4.0 → 2.6.0

package/src/frontmatter-text.ts

@@ -0,0 +1,109 @@
+// Frontmatter edited as text, where it is written: at the top or inside one
+// of the four job blocks. No node imports — the flow canvas (flow-patterns.ts)
+// runs in the browser and edits frontmatter with it too.
+
+/** A block header on its own line — an anchor and a trailing `# comment`
+ *  allowed. */
+const JOB_BLOCK = /^["']?(governance|guardrail|observability|evaluation)["']?\s*:\s*(&\S+\s*)?(#.*)?$/;
+/** Any other block header: an inline map, an alias, a tag — a form these
+ *  writers do not edit. */
+const JOB_OTHER = /^["']?(governance|guardrail|observability|evaluation)["']?\s*:\s*\S/;
+/** An inline map closed on its own line: `guardrail: {network: [a.com]}`. */
+const JOB_ONE_LINE = /^["']?(governance|guardrail|observability|evaluation)["']?\s*:\s*\{.*\}\s*(#.*)?$/;
+
+const indentOf = (l: string) => l.match(/^[ \t]*/)?.[0].length ?? 0;
+
+export interface KeySpan {
+  /** The key's line. */
+  at: number;
+  /** One past its last line. */
+  end: number;
+  /** What its line starts with. */
+  indent: string;
+  /** The block header's line, when it is inside one. */
+  block: number | null;
+}
+
+/** Thrown for a key a writer cannot edit in place: one that is, or may be,
+ *  inside a block not written one key per line (`guardrail: {…}`, an alias).
+ *  Rewriting it would mean re-serialising the person's YAML, and adding the
+ *  key at the top beside it would be a config/conflict — so the writer says
+ *  what to write instead. */
+export class InlineBlockError extends Error {
+  status = 409;
+  constructor(key: string, block: string) {
+    super(`\`${key}\` may be inside \`${block}:\`, which is not written one key per line — write \`${block}:\` one key per line to change it here, or edit the file`);
+  }
+}
+
+/** A comment or blank line: neither ends a block nor is one of its keys. */
+const filler = (l: string) => l.trim() === "" || /^\s*#/.test(l);
+
+/**
+ * Where a key is written in frontmatter lines: at the top, or a direct child
+ * of one of the four job blocks (`guardrail:` and so on). Every writer that
+ * edits a key as text reads it here, so a file written in blocks is edited
+ * where it is — never given a second, flat copy that check would refuse as
+ * config/conflict. A key inside an inline block map throws InlineBlockError.
+ */
+export function keySpan(lines: string[], key: string): KeySpan | null {
+  const k = key.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+  const span = (at: number, indent: string, block: number | null): KeySpan => {
+    let end = at + 1;
+    // The key continues while lines are indented deeper, are an item of its
+    // list written at its own indent (YAML allows `key:\n- a`), or are blank
+    // lines and comments inside it.
+    while (
+      end < lines.length &&
+      (filler(lines[end]) ||
+        indentOf(lines[end]) > indent.length ||
+        (indentOf(lines[end]) === indent.length && /^[ \t]*-(\s|$)/.test(lines[end])))
+    ) end++;
+    // Blank lines and comments at its foot, no deeper than the key, belong
+    // to what follows.
+    while (end > at + 1 && filler(lines[end - 1]) && (lines[end - 1].trim() === "" || indentOf(lines[end - 1]) <= indent.length)) end--;
+    return { at, end, indent, block };
+  };
+  // A key may be quoted: `"network": none`.
+  const named = `["']?${k}["']?\\s*:`;
+  const top = lines.findIndex((l) => new RegExp(`^${named}`).test(l));
+  if (top !== -1) return span(top, "", null);
+  for (let h = 0; h < lines.length; h++) {
+    const other = !JOB_BLOCK.test(lines[h]) && JOB_OTHER.exec(lines[h]);
+    if (other) {
+      // One closed line can be read: refuse only if the key is in it. Any
+      // other form (a map over several lines, an alias) cannot be, so any
+      // key not found at the top is refused rather than added beside it.
+      if (!JOB_ONE_LINE.test(lines[h]) || new RegExp(`[{,]\\s*${named}`).test(lines[h])) throw new InlineBlockError(key, other[1]!);
+      continue;
+    }
+    if (!JOB_BLOCK.test(lines[h])) continue;
+    // Its children sit at the indent of its first indented line; a deeper
+    // line of the same name belongs to one of them, not to the block.
+    // Comments and blank lines, indented or not, do not end it.
+    let child = -1;
+    let merged = false;
+    for (let n = h + 1; n < lines.length && (/^[ \t]/.test(lines[n]) || filler(lines[n])); n++) {
+      if (filler(lines[n])) continue;
+      if (child === -1) child = indentOf(lines[n]);
+      if (indentOf(lines[n]) !== child) continue;
+      const m = new RegExp(`^([ \t]+)${named}`).exec(lines[n]);
+      if (m) return span(n, m[1]!, h);
+      if (/^[ \t]+<<\s*:/.test(lines[n])) merged = true;
+    }
+    // `<<: *policy` may bring the key in from elsewhere: not one this text
+    // edit can see, so it is refused rather than added beside it.
+    if (merged) throw new InlineBlockError(key, JOB_BLOCK.exec(lines[h])![1]!);
+  }
+  return null;
+}
+
+/** A block header left with nothing under it is removed with its last key.
+ *  Blank lines and comments between it and its next key do not count as
+ *  nothing: the header stays while any indented key follows them. */
+export function dropEmptyBlock(lines: string[], header: number): string[] {
+  let n = header + 1;
+  while (n < lines.length && filler(lines[n])) n++;
+  const next = lines[n];
+  return next === undefined || !/^[ \t]+\S/.test(next) ? [...lines.slice(0, header), ...lines.slice(header + 1)] : lines;
+}

package/src/history.ts

@@ -402,6 +402,19 @@ export function readRevision(tenant: string, scope: string, id: string, forPath?
   return forPath ? { ...rev, files: rev.files.filter((f) => f.path === forPath) } : rev;
 }
 
+/** The files changed after revision `from` up to and including `to`, in a
+ *  scope — what differs between the versions two runs ran (FOL-72). Null
+ *  when either is unknown or `to` is not after `from`. */
+export function changedPathsBetween(tenant: string, scope: string, from: string | null, to: string | null): string[] | null {
+  if (!from || !to) return null;
+  if (from === to) return [];
+  const revs = listRevisions(tenant, scope, { limit: 500 }); // newest first
+  const ti = revs.findIndex((r) => r.id === to || r.commit === to);
+  const fi = revs.findIndex((r) => r.id === from || r.commit === from);
+  if (ti < 0 || fi < 0 || ti > fi) return null;
+  return [...new Set(revs.slice(ti, fi).flatMap((r) => r.paths))].sort();
+}
+
 /** The most recent revision id for a scope — what a change is attributed
  *  to when nothing more specific (a deployed commit) is on record. */
 export function latestRevisionId(tenant: string, scope: string): string | null {

package/src/hooks.ts

@@ -31,7 +31,7 @@ import fs from "node:fs";
 import path from "node:path";
 import os from "node:os";
 import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
-import matter from "gray-matter";
+import matter from "./matter.ts";
 
 export const HOOK_EVENTS = ["tool.call", "tool.result", "step.start", "step.end"] as const;
 export type HookEvent = (typeof HOOK_EVENTS)[number];

package/src/insights.ts

@@ -0,0 +1,174 @@
+// Insights (FOL-72 step 8): the problems that recur without failing anything,
+// found in the runs with no setup — each with its evidence, what it costs and
+// what to do. The operator's "3 problems, here is why" instead of a list of
+// runs. Pure: the API, the Observe page and `foldrun insights` read the same.
+//
+//   cache      an agent re-reads a big prompt uncached, step after step
+//   big-result a tool's result is large and every later model call sends it again
+//   retry-loop a tool errors again and again inside one step
+//   timeout    steps that use most of their `timeout:` — the next bigger input fails
+//
+// Costs are estimates in credits from the steps' own token costs (a credit
+// is a cent of USD), said as "about".
+
+import type { RunRecord, StepRecord } from "./store.ts";
+
+export type InsightKind = "cache" | "big-result" | "retry-loop" | "timeout";
+
+export interface Insight {
+  id: string;
+  kind: InsightKind;
+  workspace: string;
+  agent: string;
+  tool?: string;
+  title: string;
+  /** Why it happens, in a sentence. */
+  why: string;
+  fix: string;
+  /** About how many credits it cost in the window; null when not money. */
+  credits: number | null;
+  /** How many steps showed it. */
+  steps: number;
+  /** The newest runs it was seen in, at most 5. */
+  evidence: { workspace: string; runId: string; step: number }[];
+}
+
+type Run = RunRecord & { workspace: string };
+
+/** A cached input token costs about a tenth of a fresh one. */
+const CACHE_SAVING = 0.9;
+/** Characters to tokens, roughly, for text in a conversation. */
+const CHARS_PER_TOKEN = 4;
+const BIG_RESULT = 20_000;
+
+const stepRefs = (hits: { run: Run; index: number }[]) =>
+  [...hits].sort((a, b) => b.run.startedAt.localeCompare(a.run.startedAt)).slice(0, 5).map((h) => ({ workspace: h.run.workspace, runId: h.run.id, step: h.index + 1 }));
+
+const credits = (usd: number) => Math.round(usd * 100);
+
+export function findInsights(runs: Run[], opts: { minSteps?: number } = {}): Insight[] {
+  const min = opts.minSteps ?? 3;
+  const steps: { run: Run; index: number; s: StepRecord }[] = [];
+  for (const run of runs) run.steps.forEach((s, index) => {
+    if (s.status === "completed" || s.status === "failed") steps.push({ run, index, s });
+  });
+  const out: Insight[] = [];
+  const byAgent = new Map<string, typeof steps>();
+  for (const x of steps) {
+    const k = `${x.run.workspace}\u0000${x.s.agent}`;
+    (byAgent.get(k) ?? byAgent.set(k, []).get(k)!).push(x);
+  }
+
+  for (const [k, list] of byAgent) {
+    const [workspace, agent] = k.split("\u0000") as [string, string];
+
+    // cache: big inputs, little of it from the cache, on most of its steps.
+    const measured = list.filter((x) => x.s.tokens && x.s.tokens.cacheRead !== undefined && x.s.tokens.input >= 20_000);
+    const poor = measured.filter((x) => (x.s.tokens!.cacheRead ?? 0) / x.s.tokens!.input < 0.3);
+    if (poor.length >= min && poor.length >= measured.length / 2) {
+      const usd = poor.reduce((n, x) => {
+        const t = x.s.tokens!;
+        const uncached = Math.max(0, t.input - (t.cacheRead ?? 0)) / Math.max(1, t.input);
+        return n + (x.s.costUsd ?? 0) * uncached * CACHE_SAVING * 0.5;
+      }, 0);
+      const rate = Math.round((100 * poor.reduce((n, x) => n + (x.s.tokens!.cacheRead ?? 0), 0)) / Math.max(1, poor.reduce((n, x) => n + x.s.tokens!.input, 0)));
+      out.push({
+        id: `cache/${workspace}/${agent}`,
+        kind: "cache",
+        workspace,
+        agent,
+        title: `${agent} reads its prompt mostly uncached (${rate}% from the cache)`,
+        why: "Each call sends a large prompt that the model's cache rarely serves: something near its start changes from call to call (a date, a run id, a file listing), or the steps are spaced further apart than the cache lasts.",
+        fix: "Keep what changes (dates, ids, the task) at the end of the agent's prompt and the large fixed part first; run its steps closer together.",
+        credits: credits(usd),
+        steps: poor.length,
+        evidence: stepRefs(poor),
+      });
+    }
+
+    // big-result: a tool's result over 20k characters with calls after it,
+    // each of which sends it again.
+    const big = new Map<string, { hits: typeof list; usd: number; chars: number }>();
+    for (const x of list) {
+      const callsAfter = (at: string) => (x.s.calls ?? []).filter((c) => c.at > at).length;
+      const turns = x.s.calls?.length ?? x.s.turns ?? 0;
+      for (const e of x.s.events) {
+        if (e.type !== "tool" || e.ms == null || (e.bytes ?? 0) < BIG_RESULT) continue;
+        const after = x.s.calls ? callsAfter(e.t) : Math.max(0, turns - 1);
+        if (after < 2) continue;
+        const tokensResent = ((e.bytes ?? 0) / CHARS_PER_TOKEN) * after;
+        const input = Math.max(1, x.s.tokens?.input ?? 0);
+        const share = Math.min(1, tokensResent / input);
+        const b = big.get(e.text) ?? { hits: [], usd: 0, chars: 0 };
+        if (!b.hits.includes(x)) b.hits.push(x);
+        b.usd += (x.s.costUsd ?? 0) * share;
+        b.chars = Math.max(b.chars, e.bytes ?? 0);
+        big.set(e.text, b);
+      }
+    }
+    for (const [tool, b] of big) {
+      if (b.hits.length < min) continue;
+      out.push({
+        id: `big-result/${workspace}/${agent}/${tool}`,
+        kind: "big-result",
+        workspace,
+        agent,
+        tool,
+        title: `${tool} hands ${agent} results up to ${Math.round(b.chars / 1000)}k characters, sent again on every later call`,
+        why: "A tool result stays in the conversation, so every model call after it pays for it again.",
+        fix: `Have ${tool} return less (filter, a summary, or write the bulk to a file and return the path), or split the step so the big read ends it.`,
+        credits: credits(b.usd),
+        steps: b.hits.length,
+        evidence: stepRefs(b.hits),
+      });
+    }
+
+    // retry-loop: one tool erroring three or more times in a step.
+    const loops = new Map<string, typeof list>();
+    for (const x of list) {
+      const errs = new Map<string, number>();
+      for (const e of x.s.events) if (e.type === "tool" && e.ms != null && e.err) errs.set(e.text, (errs.get(e.text) ?? 0) + 1);
+      for (const [tool, n] of errs) if (n >= 3) (loops.get(tool) ?? loops.set(tool, []).get(tool)!).push(x);
+    }
+    for (const [tool, hits] of loops) {
+      if (hits.length < Math.min(min, 2)) continue;
+      out.push({
+        id: `retry-loop/${workspace}/${agent}/${tool}`,
+        kind: "retry-loop",
+        workspace,
+        agent,
+        tool,
+        title: `${agent} keeps retrying ${tool} after it errors`,
+        why: "The tool fails the same way and the agent tries again instead of stopping — every attempt is a model call and a tool call.",
+        fix: `Read one of the errors (the trace's input / output): fix the tool or its input, or tell ${agent} in its prompt to stop after one failure and say BLOCKED with why.`,
+        credits: credits(hits.reduce((n, x) => n + (x.s.costUsd ?? 0) * 0.3, 0)),
+        steps: hits.length,
+        evidence: stepRefs(hits),
+      });
+    }
+
+    // timeout: steps that took 80% or more of their timeout.
+    const tight = list.filter((x) => {
+      if (!x.s.timeout || !x.s.startedAt || !x.s.finishedAt) return false;
+      const secs = (Date.parse(x.s.finishedAt) - Date.parse(x.s.startedAt)) / 1000;
+      return secs >= 0.8 * x.s.timeout;
+    });
+    if (tight.length >= Math.min(min, 2)) {
+      const limit = tight[0]!.s.timeout!;
+      out.push({
+        id: `timeout/${workspace}/${agent}`,
+        kind: "timeout",
+        workspace,
+        agent,
+        title: `${agent} uses most of its ${limit}s timeout`,
+        why: "Steps finish just inside their limit; a slightly bigger input or a slow site and they are cut off.",
+        fix: "Raise timeout: on the step, or give the agent less to do per step (each: to fan out, a smaller batch).",
+        credits: null,
+        steps: tight.length,
+        evidence: stepRefs(tight),
+      });
+    }
+  }
+  const order: Record<InsightKind, number> = { "retry-loop": 0, timeout: 1, "big-result": 2, cache: 3 };
+  return out.sort((a, b) => (b.credits ?? -1) - (a.credits ?? -1) || order[a.kind] - order[b.kind]);
+}