@foldrun/core 0.2.0

package/src/context-tools.ts

@@ -0,0 +1,435 @@
+// Two tools over what an agent already has and cannot easily reach at scale:
+// its documents, and its own past.
+//
+//   tools: [search]    search_files — ranked full-text search over knowledge,
+//                      memory, state and storage, all three scopes
+//   tools: [history]   recall_runs / read_run — what this workspace's earlier
+//                      runs concluded, and what their steps said
+//   tools: [desks]     recall_desk_runs / read_desk_run — the same, for every
+//                      OTHER workspace in the account
+//
+// Both take values, not stores: a list of directories, a list of run
+// digests. That is what lets the same servers be rebuilt inside a run
+// container, where the vault, the run journal and the account do not exist
+// — the host gathers, the boundary is crossed as JSON, the container serves.
+//
+// Search is lexical (BM25 over tokens), not embeddings. It runs anywhere,
+// needs no vendor, and nothing leaves the box — which is the property the
+// search-on-our-box work already paid for. Behind the tool is an inverted
+// index built once per step and refreshed by mtime: a call tokenises only
+// the files that changed since the last one (the agent writing to state/
+// mid-step is the common case) and scores only the postings of the query's
+// terms, so the tenth search over ten thousand files costs what the first
+// one over ten did. Nothing is persisted: a step's index dies with the
+// step, so it can never be stale across a deploy or a hand edit.
+
+import fs from "node:fs";
+import path from "node:path";
+import { z } from "zod";
+import { createSdkMcpServer, tool } from "@anthropic-ai/claude-agent-sdk";
+import type { RunRecord } from "./store.ts";
+
+// ------------------------------------------------------------------ search
+
+export interface SearchRoot {
+  /** How the agent should name the place: "knowledge/", "../../memory/". */
+  label: string;
+  /** The directory, absolute. */
+  dir: string;
+}
+
+const TEXT_EXT = new Set([".md", ".txt", ".csv", ".json", ".yaml", ".yml", ".html", ".tsv"]);
+const MAX_FILE_BYTES = 512_000;
+const MAX_FILES = 5_000;
+
+interface Doc {
+  label: string;
+  rel: string;
+  text: string;
+  /** Frontmatter name/title/description, weighted higher than the body. */
+  head: string;
+}
+
+/** A file as the index holds it: tokenised once, refreshed when its mtime moves. */
+interface IndexedDoc extends Doc {
+  abs: string;
+  size: number;
+  mtimeMs: number;
+  tf: Map<string, number>;
+  length: number;
+}
+
+function docFrom(root: SearchRoot, abs: string, st: fs.Stats): IndexedDoc | null {
+  try {
+    const text = fs.readFileSync(abs, "utf8");
+    const front = text.match(/^---\n([\s\S]*?)\n---/)?.[1] ?? "";
+    const head = [...front.matchAll(/^(?:name|title|description):\s*(.+)$/gm)].map((m) => m[1]).join(" ");
+    const body = tokens(head + " " + head + " " + head + " " + text);
+    const tf = new Map<string, number>();
+    for (const t of body) tf.set(t, (tf.get(t) ?? 0) + 1);
+    return { label: root.label, rel: path.relative(root.dir, abs).replaceAll("\\", "/"), text, head, abs, size: st.size, mtimeMs: st.mtimeMs, tf, length: body.length };
+  } catch {
+    return null; // unreadable: not a hit
+  }
+}
+
+/** Every indexable file under the roots, with the stat the index compares against. */
+function listFiles(roots: SearchRoot[]): { root: SearchRoot; abs: string; st: fs.Stats }[] {
+  const out: { root: SearchRoot; abs: string; st: fs.Stats }[] = [];
+  for (const root of roots) {
+    if (!fs.existsSync(root.dir)) continue;
+    const walk = (dir: string) => {
+      let entries: fs.Dirent[];
+      try {
+        entries = fs.readdirSync(dir, { withFileTypes: true });
+      } catch {
+        return;
+      }
+      for (const e of entries) {
+        if (out.length >= MAX_FILES) return;
+        const abs = path.join(dir, e.name);
+        if (e.isDirectory()) {
+          if (e.name.startsWith(".") || e.name === "node_modules") continue;
+          walk(abs);
+          continue;
+        }
+        if (!e.isFile() || !TEXT_EXT.has(path.extname(e.name).toLowerCase())) continue;
+        // OKF's generated files describe the bundle, not a fact in it.
+        if (e.name === "index.md" || e.name === "log.md") continue;
+        try {
+          const st = fs.statSync(abs);
+          if (st.size > MAX_FILE_BYTES) continue;
+          out.push({ root, abs, st });
+        } catch {
+          // vanished between readdir and stat
+        }
+      }
+    };
+    walk(root.dir);
+  }
+  return out;
+}
+
+/**
+ * The inverted index over a set of roots, scored BM25's way — term
+ * frequency saturating, rare terms weighing more, long documents not
+ * winning by bulk — with the frontmatter head counted three times, because
+ * a memory file's name and description are what its author thought it was
+ * about. `refresh()` walks the tree (a
+ * stat per file — cheap), re-reads only files whose size or mtime moved,
+ * drops the ones that went, and rebuilds the postings from the per-file
+ * token maps it already holds. `search()` touches only the query terms'
+ * postings.
+ */
+export class SearchIndex {
+  private docs = new Map<string, IndexedDoc>();
+  private postings = new Map<string, { doc: IndexedDoc; tf: number }[]>();
+  private avg = 1;
+  /** How many files the last refresh read from disk — what a test watches. */
+  lastRead = 0;
+
+  private roots: SearchRoot[];
+
+  constructor(roots: SearchRoot[]) {
+    this.roots = roots;
+  }
+
+  refresh(): this {
+    const seen = new Set<string>();
+    let changed = false;
+    this.lastRead = 0;
+    for (const { root, abs, st } of listFiles(this.roots)) {
+      seen.add(abs);
+      const have = this.docs.get(abs);
+      if (have && have.size === st.size && have.mtimeMs === st.mtimeMs) continue;
+      const doc = docFrom(root, abs, st);
+      this.lastRead++;
+      if (doc) this.docs.set(abs, doc);
+      else this.docs.delete(abs);
+      changed = true;
+    }
+    for (const abs of [...this.docs.keys()]) {
+      if (!seen.has(abs)) {
+        this.docs.delete(abs);
+        changed = true;
+      }
+    }
+    if (changed) this.rebuild();
+    return this;
+  }
+
+  private rebuild(): void {
+    this.postings = new Map();
+    let total = 0;
+    for (const doc of this.docs.values()) {
+      total += doc.length;
+      for (const [term, tf] of doc.tf) {
+        let list = this.postings.get(term);
+        if (!list) this.postings.set(term, (list = []));
+        list.push({ doc, tf });
+      }
+    }
+    this.avg = total / this.docs.size || 1;
+  }
+
+  get size(): number {
+    return this.docs.size;
+  }
+
+  search(query: string, limit = 10): SearchHit[] {
+    const q = [...new Set(tokens(query))];
+    if (q.length === 0 || this.docs.size === 0) return [];
+    const n = this.docs.size;
+    const k1 = 1.4;
+    const bParam = 0.75;
+    const scores = new Map<IndexedDoc, number>();
+    for (const term of q) {
+      const list = this.postings.get(term);
+      if (!list) continue;
+      const idf = Math.log(1 + (n - list.length + 0.5) / (list.length + 0.5));
+      for (const { doc, tf } of list) {
+        const part = idf * ((tf * (k1 + 1)) / (tf + k1 * (1 - bParam + bParam * (doc.length / this.avg))));
+        scores.set(doc, (scores.get(doc) ?? 0) + part);
+      }
+    }
+    return [...scores.entries()]
+      .sort((a, b) => b[1] - a[1])
+      .slice(0, Math.min(Math.max(1, limit), 30))
+      .map(([d, score]) => ({ path: `${d.label}${d.rel}`, score: Math.round(score * 100) / 100, snippet: snippetFor(d.text, q) }));
+  }
+}
+
+function tokens(text: string): string[] {
+  return text
+    .toLowerCase()
+    .split(/[^a-z0-9$@._-]+/)
+    .map((t) => t.replace(/^[._-]+|[._-]+$/g, ""))
+    .filter((t) => t.length > 1);
+}
+
+export interface SearchHit {
+  path: string;
+  score: number;
+  snippet: string;
+}
+
+/** The line that mentions the most query terms, trimmed — enough to tell
+ *  whether to open the file, not a substitute for opening it. */
+function snippetFor(text: string, q: string[]): string {
+  let best = "";
+  let bestN = 0;
+  for (const line of text.split("\n")) {
+    const l = line.toLowerCase();
+    const n = q.filter((t) => l.includes(t)).length;
+    if (n > bestN && line.trim()) {
+      bestN = n;
+      best = line.trim();
+    }
+  }
+  return best.length > 200 ? `${best.slice(0, 197)}…` : best;
+}
+
+export function searchRoots(roots: SearchRoot[], query: string, limit = 10): SearchHit[] {
+  return new SearchIndex(roots).refresh().search(query, limit);
+}
+
+export interface ContextToolResult {
+  server: ReturnType<typeof createSdkMcpServer> | null;
+  toolNames: string[];
+  promptLines: string[];
+}
+
+export function buildSearchTools(roots: SearchRoot[]): ContextToolResult {
+  if (roots.length === 0) return { server: null, toolNames: [], promptLines: [] };
+  // One index per server — per step. Built on the first call, not at
+  // start-up, so a step that never searches never pays for it.
+  const index = new SearchIndex(roots);
+  const search = tool(
+    "search_files",
+    `Full-text search over this agent's knowledge, memory, state and stored files (all scopes: own, workspace, account). ` +
+      `Returns the best-matching paths with a one-line snippet each — open a hit with Read to use it. ` +
+      `Use it instead of guessing which of many files holds a fact.`,
+    {
+      query: z.string().describe("Words to look for — names, ids, phrases"),
+      limit: z.number().int().min(1).max(30).optional().describe("How many hits (default 10)"),
+    },
+    async (args) => {
+      const hits = index.refresh().search(args.query, args.limit ?? 10);
+      const text = hits.length
+        ? hits.map((h) => `${h.path}  (${h.score})\n    ${h.snippet}`).join("\n")
+        : "no files mention that";
+      return { content: [{ type: "text" as const, text }] };
+    },
+  );
+  return {
+    server: createSdkMcpServer({ name: "foldrun_search", version: "1.0.0", tools: [search] }),
+    toolNames: ["mcp__foldrun_search__search_files"],
+    promptLines: [
+      `- **search_files(query)** — ranked search across ${roots.map((r) => r.label).join(", ")}. Search first, then Read the hit.`,
+    ],
+  };
+}
+
+// ----------------------------------------------------------------- history
+
+export interface RunDigest {
+  /** Which workspace it ran in. Absent for this workspace's own history,
+   *  where naming it every line would be noise. */
+  workspace?: string;
+  id: string;
+  flow: string;
+  status: string;
+  startedAt: string;
+  finishedAt: string | null;
+  summary: string | null;
+  costUsd: number;
+  tags: string[];
+  steps: { agent: string; status: string; result: string | null }[];
+}
+
+const STEP_RESULT_CHARS = 6_000;
+
+/** The digest of a workspace's recent runs — host-side, from the records. */
+export function digestRuns(runs: RunRecord[], limit = 30, exceptRunId?: string): RunDigest[] {
+  return runs
+    .filter((r) => r.id !== exceptRunId && (r.status === "completed" || r.status === "failed"))
+    .sort((a, b) => (b.startedAt > a.startedAt ? 1 : -1))
+    .slice(0, limit)
+    .map((r) => ({
+      id: r.id,
+      flow: r.flow,
+      status: r.status,
+      startedAt: r.startedAt,
+      finishedAt: r.finishedAt,
+      summary: r.summary ?? null,
+      costUsd: r.steps.reduce((n, s) => n + (s.costUsd ?? 0), 0),
+      tags: r.tags ?? [],
+      steps: r.steps
+        .filter((s) => s.status === "completed" || s.status === "failed")
+        .map((s) => ({
+          agent: s.agent,
+          status: s.status,
+          result: s.result ? (s.result.length > STEP_RESULT_CHARS ? `${s.result.slice(0, STEP_RESULT_CHARS)}…` : s.result) : null,
+        })),
+    }));
+}
+
+export function buildHistoryTools(digest: RunDigest[]): ContextToolResult {
+  const recall = tool(
+    "recall_runs",
+    `What this workspace's earlier runs concluded — newest first, one line each (id, flow, when, status, cost, summary). ` +
+      `Filter by flow, status or a word. Use it to avoid redoing yesterday's work, to continue where a run left off, ` +
+      `or to see what a scheduled flow found last time. Then read_run(id) for the detail.`,
+    {
+      limit: z.number().int().min(1).max(50).optional().describe("How many (default 10)"),
+      flow: z.string().optional().describe("Only runs of this flow"),
+      status: z.enum(["completed", "failed"]).optional(),
+      query: z.string().optional().describe("Only runs whose summary or results mention this"),
+    },
+    async (args) => {
+      const q = (args.query ?? "").toLowerCase();
+      const rows = digest
+        .filter((r) => !args.flow || r.flow === args.flow)
+        .filter((r) => !args.status || r.status === args.status)
+        .filter(
+          (r) =>
+            !q ||
+            (r.summary ?? "").toLowerCase().includes(q) ||
+            r.steps.some((s) => (s.result ?? "").toLowerCase().includes(q)),
+        )
+        .slice(0, args.limit ?? 10)
+        .map((r) => `${r.id}  ${r.flow}  ${r.startedAt.slice(0, 16)}  ${r.status}  $${r.costUsd.toFixed(3)}  ${r.summary ?? "(no summary)"}`);
+      return { content: [{ type: "text" as const, text: rows.length ? rows.join("\n") : "no earlier runs match" }] };
+    },
+  );
+  const read = tool(
+    "read_run",
+    "One earlier run in full: each step's agent, status and what it replied (long replies trimmed).",
+    { id: z.string().describe("A run id from recall_runs") },
+    async (args) => {
+      const r = digest.find((d) => d.id === args.id);
+      if (!r) return { content: [{ type: "text" as const, text: `no run ${args.id} in the recent history` }], isError: true };
+      const text =
+        `${r.flow} · ${r.status} · started ${r.startedAt}${r.finishedAt ? ` · finished ${r.finishedAt}` : ""} · $${r.costUsd.toFixed(4)}\n` +
+        (r.summary ? `summary: ${r.summary}\n` : "") +
+        r.steps.map((s) => `\n## ${s.agent} (${s.status})\n${s.result ?? "(no reply)"}`).join("\n");
+      return { content: [{ type: "text" as const, text }] };
+    },
+  );
+  return {
+    server: createSdkMcpServer({ name: "foldrun_history", version: "1.0.0", tools: [recall, read] }),
+    toolNames: ["mcp__foldrun_history__recall_runs", "mcp__foldrun_history__read_run"],
+    promptLines: [
+      `- **recall_runs()** / **read_run(id)** — this workspace's last ${digest.length} finished runs and what they concluded. ` +
+        `Check before repeating work a recent run already did.`,
+    ],
+  };
+}
+
+/**
+ * The same two verbs across the account's OTHER workspaces.
+ *
+ * A digest agent has to read what six desks concluded this week, and it
+ * cannot: a run pod's network policy denies the platform's own API, and the
+ * account library is mounted read-only, so there is no path from inside a
+ * sandbox to another workspace's files. This is that path, built the way
+ * `history` already is — the host gathers the records, the values cross the
+ * boundary as JSON, and the container serves them. Nothing new is reachable
+ * from a run; what crosses is what the host decided to send.
+ *
+ * Read-only and summary-shaped on purpose: run records, not files. An agent
+ * that needs another desk's *file* is asking for something this platform
+ * deliberately does not have.
+ */
+export function buildDeskTools(digest: RunDigest[]): ContextToolResult {
+  const workspaces = [...new Set(digest.map((d) => d.workspace ?? "?"))].sort();
+  const recall = tool(
+    "recall_desk_runs",
+    `What the account's OTHER workspaces' runs concluded — newest first, one line each ` +
+      `(workspace, id, flow, when, status, cost, summary). The summary is the run's headline: ` +
+      `since every desk's last step opens with its verdict, this is how the week reads at a glance. ` +
+      `Then read_desk_run(id) for a run's steps in full.`,
+    {
+      workspace: z.string().optional().describe("Only this workspace"),
+      flow: z.string().optional().describe("Only runs of this flow"),
+      status: z.enum(["completed", "failed"]).optional(),
+      since: z.string().optional().describe("ISO date; only runs started on or after it"),
+      limit: z.number().int().min(1).max(200).optional().describe("How many (default 30)"),
+    },
+    async (args) => {
+      const rows = digest
+        .filter((r) => !args.workspace || r.workspace === args.workspace)
+        .filter((r) => !args.flow || r.flow === args.flow)
+        .filter((r) => !args.status || r.status === args.status)
+        .filter((r) => !args.since || r.startedAt >= args.since)
+        .slice(0, args.limit ?? 30)
+        .map(
+          (r) =>
+            `${(r.workspace ?? "?").padEnd(12)} ${r.id}  ${r.flow}  ${r.startedAt.slice(0, 16)}  ${r.status}  $${r.costUsd.toFixed(3)}  ${r.summary ?? "(no summary)"}`,
+        );
+      return { content: [{ type: "text" as const, text: rows.length ? rows.join("\n") : "no runs match" }] };
+    },
+  );
+  const read = tool(
+    "read_desk_run",
+    "One run from another workspace in full: each step's agent, status and what it replied (long replies trimmed).",
+    { id: z.string().describe("A run id from recall_desk_runs") },
+    async (args) => {
+      const r = digest.find((d) => d.id === args.id);
+      if (!r) return { content: [{ type: "text" as const, text: `no run ${args.id} in the account's recent history` }], isError: true };
+      const text =
+        `${r.workspace ?? "?"} · ${r.flow} · ${r.status} · started ${r.startedAt}${r.finishedAt ? ` · finished ${r.finishedAt}` : ""} · $${r.costUsd.toFixed(4)}\n` +
+        (r.summary ? `summary: ${r.summary}\n` : "") +
+        r.steps.map((s) => `\n## ${s.agent} (${s.status})\n${s.result ?? "(no reply)"}`).join("\n");
+      return { content: [{ type: "text" as const, text }] };
+    },
+  );
+  return {
+    server: createSdkMcpServer({ name: "foldrun_desks", version: "1.0.0", tools: [recall, read] }),
+    toolNames: ["mcp__foldrun_desks__recall_desk_runs", "mcp__foldrun_desks__read_desk_run"],
+    promptLines: [
+      `- **recall_desk_runs()** / **read_desk_run(id)** — what the account's other workspaces ` +
+        `(${workspaces.join(", ") || "none yet"}) concluded, newest first. Their run records, not their files.`,
+    ],
+  };
+}