@100xprompt/brain 0.0.0-stage → 0.1.1

package/src/memory/store.ts

@@ -0,0 +1,270 @@
+// The brain's own memory store: Chitta (permission-aware graph + vector memory) at `<home>/.brain/memory/context.db`.
+//
+// Chitta is optional and not a dependency: the SDK is loaded lazily from the first place it is found (`$BRAIN_CHITTA`,
+// next to brain's own install, `~/node_modules/@100xprompt/chitta`, a global bun/npm install, then $BRAIN_CHITTA_DIRS) and
+// `status()` says why when it is not.
+// The person's personal Chitta store (CONTEXT_DB, ~/.local/share/100xprompt) is never opened: the path is always explicit.
+//
+// Project scoping, chosen: ONE CHITTA PRINCIPAL PER PROJECT (`chitta.agent("brain-<hash of project id>")`). Every record is
+// private to its principal, so the engine's own ACL keeps projects apart in search, graph expansion and facts; a memory
+// from project A cannot come back for project B even by a bug in this file. (The alternative, filtering after recall, would
+// leak across projects the day a filter was forgotten, and would let one project's records crowd out another's top-k.)
+// A small side table `brain_items` in the same SQLite file keeps the full text and the kind/source/project of every item:
+// Chitta chunks long records into passages, and recall must hand back whole items, so Chitta only ranks and the table answers.
+import { existsSync, mkdirSync, statSync } from "node:fs"
+import { homedir } from "node:os"
+import { dirname, join } from "node:path"
+import { pathToFileURL } from "node:url"
+import { createHash } from "node:crypto"
+import { openRemoteMemory } from "../org/client.ts"
+import { readOrgConfig } from "../org/config.ts"
+import { redact } from "./redact.ts"
+import { approxTokens, type Memory, type MemoryItem, type MemoryKind, type MemoryStatus, type ProjectKey, type Recalled } from "./types.ts"
+
+export interface MemoryCounts {
+  total: number
+  byKind: Partial<Record<MemoryKind, number>>
+}
+
+/** `Memory` plus what the CLI and the MCP server's status need. */
+export interface BrainMemory extends Memory {
+  /** Item counts for one project, or the whole store. */
+  counts(project?: ProjectKey): Promise<MemoryCounts>
+  /** Projects with memories: id, label, items. */
+  projects(): Promise<Array<{ id: string; label: string; items: number }>>
+  /** Closes the database (long-running servers may keep it open; tests and one-shot commands close it). */
+  close(): void
+}
+
+export const memoryDbPath = (home: string): string => join(home, ".brain", "memory", "context.db")
+
+const sha = (s: string, n = 12): string => createHash("sha1").update(s).digest("hex").slice(0, n)
+const principalOf = (project: ProjectKey): string => `brain-${sha(project.id)}`
+const recordId = (project: ProjectKey, item: MemoryItem): string => `${sha(project.id)}~${sha(item.source, 8)}~${item.id}`
+
+// ---- locating the SDK ---------------------------------------------------------------------------------------------
+
+/** Where Chitta's SDK entry might be, in order; an explicit `$BRAIN_CHITTA` is the only candidate when it is set. */
+export function chittaCandidates(env = process.env, home = homedir()): string[] {
+  const entry = (dir: string): string[] => [join(dir, "dist", "sdk.js"), join(dir, "src", "sdk.ts")]
+  const explicit = env.BRAIN_CHITTA
+  if (explicit) return /\.(?:js|ts|mjs)$/.test(explicit) ? [explicit] : entry(explicit)
+  const dirs = [
+    // Beside this brain: installed in the same node_modules (a global bun or npm install, or a project's).
+    join(import.meta.dir, "..", "..", "..", "chitta"),
+    join(home, "node_modules", "@100xprompt", "chitta"),
+    ...(env.BUN_INSTALL ? [join(env.BUN_INSTALL, "install", "global", "node_modules", "@100xprompt", "chitta")] : []),
+    join(home, ".bun", "install", "global", "node_modules", "@100xprompt", "chitta"),
+    "/opt/homebrew/lib/node_modules/@100xprompt/chitta",
+    "/usr/local/lib/node_modules/@100xprompt/chitta",
+    join(home, ".npm-global", "lib", "node_modules", "@100xprompt", "chitta"),
+    // More places, searched last (a Chitta checkout, for example): BRAIN_CHITTA_DIRS, separated like PATH.
+    ...(env.BRAIN_CHITTA_DIRS ?? "").split(process.platform === "win32" ? ";" : ":").filter(Boolean),
+  ]
+  return dirs.flatMap(entry)
+}
+
+interface ChittaLike {
+  ctx: { store: { db: SqlDb; close(): void }; deleteRecord(actingUserId: string, recordId: string): void }
+  agent(id: string): { userId: string; remember(text: string, opts: Record<string, unknown>): Promise<{ id: string }>; recall(query: string, opts?: { limit?: number }): Promise<Array<{ text: string; score: number; recordId?: string }>> }
+  close(): void
+}
+interface SqlDb {
+  exec(sql: string): void
+  query(sql: string): { all(...p: unknown[]): any[]; get(...p: unknown[]): any; run(...p: unknown[]): unknown }
+}
+
+async function loadChittaClass(): Promise<{ Chitta: new (o: Record<string, unknown>) => ChittaLike; from: string } | { why: string }> {
+  const tried: string[] = []
+  for (const file of chittaCandidates()) {
+    if (!existsSync(file)) continue
+    try {
+      const mod = await import(pathToFileURL(file).href)
+      if (typeof mod.Chitta === "function") return { Chitta: mod.Chitta, from: file }
+      tried.push(`${file} (no Chitta export)`)
+    } catch (e) {
+      tried.push(`${file} (${(e as Error).message.split("\n")[0]})`)
+    }
+  }
+  return { why: tried.length ? `Chitta could not be loaded: ${tried.join("; ")}` : "Chitta is not installed (set BRAIN_CHITTA to its folder, or install @100xprompt/chitta)" }
+}
+
+// ---- busy handling ------------------------------------------------------------------------------------------------
+
+const BUSY = /SQLITE_BUSY|database is locked|database table is locked/i
+const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms))
+
+/** Runs `fn`, retrying while another process holds the database (several tools each run their own `brain mcp`). */
+async function whenIdle<T>(fn: () => T | Promise<T>, tries = 12): Promise<T> {
+  for (let i = 0; ; i++) {
+    try {
+      return await fn()
+    } catch (e) {
+      if (i >= tries || !BUSY.test(String((e as Error)?.message ?? e))) throw e
+      await sleep(Math.min(40 * 2 ** Math.min(i, 5), 600) + Math.random() * 40)
+    }
+  }
+}
+
+// ---- the store ----------------------------------------------------------------------------------------------------
+
+/** The memory the brain uses: the organisation's server when this machine is logged in to one (<home>/.brain/org.json, unless
+ *  BRAIN_ORG=off), else the local store below, exactly as before. */
+export function openMemory(home: string): BrainMemory {
+  if (process.env.BRAIN_ORG !== "off") {
+    const cfg = readOrgConfig(home)
+    if (cfg) return openRemoteMemory(cfg)
+  }
+  return openLocalMemory(home)
+}
+
+/** The local store only: `<home>/.brain/memory/context.db`. */
+export function openLocalMemory(home: string): BrainMemory {
+  const path = memoryDbPath(home)
+  const hash = process.env.BRAIN_MEMORY_EMBEDDINGS === "hash" || process.env.NODE_ENV === "test"
+  let handle: Promise<{ chitta: ChittaLike; db: SqlDb } | { why: string }> | undefined
+
+  const open = (): Promise<{ chitta: ChittaLike; db: SqlDb } | { why: string }> =>
+    (handle ??= (async () => {
+      const loaded = await loadChittaClass()
+      if ("why" in loaded) return loaded
+      try {
+        mkdirSync(dirname(path), { recursive: true, mode: 0o700 })
+        const chitta = await whenIdle(() => new loaded.Chitta({ path, embeddings: hash ? "hash" : "auto", rerank: false }))
+        const db = chitta.ctx.store.db
+        try {
+          db.exec("PRAGMA busy_timeout = 5000")
+        } catch {}
+        await whenIdle(() =>
+          db.exec(
+            `CREATE TABLE IF NOT EXISTS brain_items (rid TEXT PRIMARY KEY, project TEXT NOT NULL, label TEXT NOT NULL, source TEXT NOT NULL, kind TEXT NOT NULL, name TEXT NOT NULL, text TEXT NOT NULL, at TEXT);
+             CREATE INDEX IF NOT EXISTS brain_items_project ON brain_items(project, source);`,
+          ),
+        )
+        return { chitta, db }
+      } catch (e) {
+        return { why: `memory store could not be opened at ${path}: ${(e as Error).message.split("\n")[0]}` }
+      }
+    })())
+
+  const ready = async () => {
+    const h = await open()
+    return "why" in h ? undefined : h
+  }
+
+  return {
+    async status(): Promise<MemoryStatus> {
+      const h = await open()
+      if ("why" in h) return { available: false, why: h.why }
+      const items = (h.db.query("SELECT count(*) c FROM brain_items").get() as { c: number }).c
+      return { available: true, store: path, items }
+    },
+
+    async ingest(project, items) {
+      const h = await ready()
+      if (!h) throw new Error("memory is not available")
+      const agent = h.chitta.agent(principalOf(project))
+      let stored = 0
+      for (const item of items) {
+        // Everything is redacted on the way in, even when the caller already did: this is the last gate before disk.
+        const text = redact(item.text)
+        const name = redact(item.name)
+        const rid = recordId(project, item)
+        await whenIdle(() =>
+          agent.remember(text, {
+            id: rid,
+            name: `${item.kind}: ${name}`,
+            entities: item.entities?.map((e) => ({ name: redact(e.name), ...(e.type ? { type: e.type } : {}) })),
+            relations: item.relations?.map((r) => ({ from: redact(r.from), to: redact(r.to), type: r.type })),
+            procedures: item.procedures?.map((p) => ({ ...(p.trigger ? { trigger: redact(p.trigger) } : {}), action: redact(p.action) })),
+            occurredAt: item.occurredAt,
+          }),
+        )
+        await whenIdle(() =>
+          h.db
+            .query("INSERT OR REPLACE INTO brain_items (rid, project, label, source, kind, name, text, at) VALUES (?,?,?,?,?,?,?,?)")
+            .run(rid, project.id, project.label, item.source, item.kind, name, text, item.occurredAt ?? null),
+        )
+        stored++
+      }
+      return { stored }
+    },
+
+    async recall(project, query, opts = {}) {
+      const h = await ready()
+      if (!h || !query.trim()) return []
+      const limit = opts.limit ?? 8
+      const maxTokens = opts.maxTokens ?? 1500
+      const agent = h.chitta.agent(principalOf(project))
+      const hits = await whenIdle(() => agent.recall(query, { limit: Math.max(limit * 4, 20) }))
+      const seen = new Set<string>()
+      const out: Recalled[] = []
+      let tokens = 0
+      for (const hit of hits) {
+        if (!hit.recordId || seen.has(hit.recordId) || !(hit.score > 0)) continue
+        seen.add(hit.recordId)
+        const row = h.db.query("SELECT kind, name, source, text FROM brain_items WHERE rid = ? AND project = ?").get(hit.recordId, project.id) as
+          | { kind: MemoryKind; name: string; source: string; text: string }
+          | undefined
+        if (!row) continue // not ours, or deleted: never returned
+        const cost = approxTokens(row.text)
+        if (out.length && tokens + cost > maxTokens) continue
+        out.push({ text: redact(row.text), kind: row.kind, name: row.name, score: hit.score, source: row.source })
+        tokens += cost
+        if (out.length >= limit) break
+      }
+      return out
+    },
+
+    async forget(project, opts = {}) {
+      const h = await ready()
+      if (!h) return 0
+      const agent = h.chitta.agent(principalOf(project))
+      const rows = (opts.source
+        ? h.db.query("SELECT rid FROM brain_items WHERE project = ? AND source = ?").all(project.id, opts.source)
+        : h.db.query("SELECT rid FROM brain_items WHERE project = ?").all(project.id)) as Array<{ rid: string }>
+      for (const { rid } of rows) {
+        await whenIdle(() => h.chitta.ctx.deleteRecord(agent.userId, rid))
+        await whenIdle(() => h.db.query("DELETE FROM brain_items WHERE rid = ?").run(rid))
+      }
+      return rows.length
+    },
+
+    async counts(project) {
+      const h = await ready()
+      if (!h) return { total: 0, byKind: {} }
+      const rows = (project
+        ? h.db.query("SELECT kind, count(*) c FROM brain_items WHERE project = ? GROUP BY kind").all(project.id)
+        : h.db.query("SELECT kind, count(*) c FROM brain_items GROUP BY kind").all()) as Array<{ kind: MemoryKind; c: number }>
+      const byKind: MemoryCounts["byKind"] = {}
+      let total = 0
+      for (const r of rows) {
+        byKind[r.kind] = r.c
+        total += r.c
+      }
+      return { total, byKind }
+    },
+
+    async projects() {
+      const h = await ready()
+      if (!h) return []
+      return (h.db.query("SELECT project id, max(label) label, count(*) items FROM brain_items GROUP BY project ORDER BY items DESC").all() as Array<{ id: string; label: string; items: number }>)
+    },
+
+    close() {
+      if (!handle) return
+      void handle.then((h) => {
+        if (!("why" in h)) try { h.chitta.close() } catch {}
+      })
+    },
+  }
+}
+
+/** The size of the store on disk in bytes (0 when it does not exist), for status output. */
+export function storeBytes(home: string): number {
+  try {
+    return statSync(memoryDbPath(home)).size
+  } catch {
+    return 0
+  }
+}

package/src/memory/sync.ts

@@ -0,0 +1,166 @@
+// Keeps a project's memory up to date by itself: the sessions of this project that memory has not seen (or that grew since)
+// are distilled and ingested. It runs in the background when a session starts in any tool (the session-start hook starts it),
+// and by hand as `brain memory sync`. Idempotent: item ids are stable and a ledger records what was done, so running it
+// twice does nothing the second time. Bounded: a few sessions and a few seconds per run; the rest waits for the next run.
+import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs"
+import { join } from "node:path"
+import { gitInfo } from "../fsutil.ts"
+import type { Session } from "../session/types.ts"
+import { distill } from "./distill.ts"
+import { projectKey } from "./project.ts"
+import { openMemory } from "./store.ts"
+import type { Memory } from "./types.ts"
+
+export interface SessionSource {
+  /** Portable id (`<tool>:<id>`). */
+  id: string
+  cwd?: string
+  updatedAt?: string
+  read(): Promise<Session>
+}
+
+export interface SyncOptions {
+  home: string
+  /** Only sessions of the project this folder belongs to (default: every project). */
+  cwd?: string
+  /** At most this many sessions per run. Default 8. */
+  limit?: number
+  /** Stop starting new sessions after this long. Default 8,000 ms. */
+  maxMs?: number
+}
+
+export interface SyncResult {
+  ingested: number
+  items: number
+  /** Sessions already in the ledger at their current time. */
+  upToDate: number
+  /** Sessions left for a later run (over the limit or the time budget). */
+  deferred: number
+  unavailable?: string
+  /** Another sync holds the lock: nothing was done. */
+  busy?: boolean
+  errors: string[]
+  ms: number
+}
+
+type Ledger = Record<string, { u: string; n: number }>
+
+const dirOf = (home: string) => join(home, ".brain", "memory")
+const ledgerFile = (home: string) => join(dirOf(home), "ledger.json")
+const lockDir = (home: string) => join(dirOf(home), "sync.lock")
+/** A lock older than this is left over from a run that died. */
+const STALE_LOCK_MS = 2 * 60_000
+
+function readLedger(home: string): Ledger {
+  try {
+    const j = JSON.parse(readFileSync(ledgerFile(home), "utf8"))
+    return j && typeof j === "object" && !Array.isArray(j) ? j : {}
+  } catch {
+    return {}
+  }
+}
+
+function writeLedger(home: string, l: Ledger) {
+  mkdirSync(dirOf(home), { recursive: true })
+  const tmp = `${ledgerFile(home)}.tmp-${process.pid}`
+  writeFileSync(tmp, JSON.stringify(l))
+  renameSync(tmp, ledgerFile(home))
+}
+
+/** Takes the lock (an atomic mkdir), clearing one left by a dead run. False when another run holds it. */
+function lock(home: string): boolean {
+  mkdirSync(dirOf(home), { recursive: true })
+  const dir = lockDir(home)
+  for (let attempt = 0; attempt < 2; attempt++) {
+    try {
+      mkdirSync(dir)
+      return true
+    } catch {
+      try {
+        if (Date.now() - statSync(dir).mtimeMs < STALE_LOCK_MS) return false
+        rmSync(dir, { recursive: true, force: true })
+      } catch {
+        return false
+      }
+    }
+  }
+  return false
+}
+
+const unlock = (home: string) => rmSync(lockDir(home), { recursive: true, force: true })
+
+/** Whether a session that ran in `cwd` belongs to the project of `here` (the same folder, one inside it, or one above it in one repository). */
+function inProject(cwd: string | undefined, here: string, home: string): boolean {
+  if (!cwd) return false
+  if (cwd === here || cwd.startsWith(`${here}/`)) return true
+  if (!here.startsWith(`${cwd}/`) || cwd === home || home.startsWith(`${cwd}/`)) return false
+  const a = gitInfo(cwd).gitRemote
+  return !!a && a === gitInfo(here).gitRemote
+}
+
+async function defaultSessions(home: string): Promise<SessionSource[]> {
+  const { READERS } = await import("../adapters/index.ts")
+  const lists = await Promise.all(READERS.map((r) => r.list(home).then((refs) => refs.map((ref) => ({ r, ref }))).catch(() => [])))
+  return lists
+    .flat()
+    .map(({ r, ref }) => ({ id: `${ref.tool}:${ref.sourceId}`, ...(ref.cwd ? { cwd: ref.cwd } : {}), ...(ref.updatedAt ? { updatedAt: ref.updatedAt } : {}), read: () => r.read(ref, home) }))
+}
+
+export async function syncMemory(o: SyncOptions, deps: { memory?: Memory; sessions?: () => Promise<SessionSource[]>; now?: () => number } = {}): Promise<SyncResult> {
+  const now = deps.now ?? Date.now
+  const t0 = now()
+  const result: SyncResult = { ingested: 0, items: 0, upToDate: 0, deferred: 0, errors: [], ms: 0 }
+  const done = (): SyncResult => ({ ...result, ms: now() - t0 })
+
+  const memory = deps.memory ?? openMemory(o.home)
+  const status = await memory.status().catch((e: Error) => ({ available: false as const, why: e.message }))
+  if (!status.available) return { ...done(), unavailable: status.why ?? "unavailable" }
+  if (!lock(o.home)) return { ...done(), busy: true }
+  try {
+    const here = o.cwd
+    const all = (await (deps.sessions ?? (() => defaultSessions(o.home)))()).filter((s) => !here || inProject(s.cwd, here, o.home))
+    all.sort((a, b) => (b.updatedAt ?? "").localeCompare(a.updatedAt ?? ""))
+    const ledger = readLedger(o.home)
+    const todo = all.filter((s) => (ledger[s.id]?.u ?? "") !== (s.updatedAt ?? "none"))
+    result.upToDate = all.length - todo.length
+    const limit = o.limit ?? 8
+    const budget = o.maxMs ?? 8_000
+    for (const [i, s] of todo.entries()) {
+      if (result.ingested >= limit || now() - t0 > budget) {
+        result.deferred = todo.length - i
+        break
+      }
+      try {
+        const session = await s.read()
+        const items = distill(session)
+        if (items.length) {
+          const key = projectKey({ ...session.header.project, ...(session.header.project.cwd ? gitInfo(session.header.project.cwd) : {}) })
+          await memory.ingest(key, items)
+        }
+        ledger[s.id] = { u: s.updatedAt ?? "none", n: items.length }
+        result.ingested++
+        result.items += items.length
+      } catch (e) {
+        // One unreadable session must not stop the rest; it is retried next time (it is not in the ledger).
+        result.errors.push(`${s.id}: ${(e as Error).message}`)
+      }
+    }
+    if (result.ingested) writeLedger(o.home, ledger)
+  } finally {
+    unlock(o.home)
+  }
+  return done()
+}
+
+/** Starts a sync in a separate process and returns at once: a hook must never wait for it. */
+export function syncInBackground(home: string, cwd: string, cli: string) {
+  if (process.env.BRAIN_NO_MEMORY === "1") return
+  try {
+    // Only where memory has been set up on this machine: a person who never used it does not get a store created by a hook.
+    if (!existsSync(join(home, ".brain", "memory"))) return
+    const child = Bun.spawn([process.execPath, cli, "memory", "sync", "--here", "--quiet"], { cwd, stdin: "ignore", stdout: "ignore", stderr: "ignore", env: { ...process.env, BRAIN_HOME: home } })
+    child.unref()
+  } catch {
+    // Best effort.
+  }
+}

package/src/memory/tidy.ts

@@ -0,0 +1,193 @@
+// tidy: lossless-in-meaning cleaning of a tool result that STAYS in the conversation (the original text of anything replaced
+// by a digest is kept byte for byte in the blob store; this step never sees it). Pure, deterministic, no model, never grows a text.
+//
+// What it does (and what it deliberately does not):
+//   - strips ANSI escape sequences (CSI and OSC);
+//   - resolves carriage-return progress lines the way a terminal shows them (the last state of a `\r`-overwritten line);
+//   - turns CRLF into LF, trims trailing spaces/tabs per line, collapses 3+ blank lines to one blank line;
+//   - collapses 3+ identical consecutive lines into the line plus `[previous line repeated N more times]`, only when that is shorter;
+//   - minifies a text that is ONE JSON document, or JSON lines that each parse, by removing only the whitespace outside strings:
+//     every lexeme (1.0, 1e2, big integers, escapes, key order, duplicate keys) stays byte for byte, and the result is checked to
+//     parse to the same value, else the text is left alone.
+// NOT done: stripping a `cat -n` style line-number prefix. The numbers carry meaning (a model cites them, an edit tool is told them)
+// and the strip cannot be made reversible, so it is skipped by design.
+import { STUB_PREFIX } from "./types.ts"
+import type { Session, Turn } from "../session/types.ts"
+
+export interface Tidied {
+  text: string
+  /** Characters saved (never negative). */
+  saved: number
+}
+
+/** Longer texts are left alone: bounded time matters more than the last few percent. */
+const MAX_CHARS = 10_000_000
+const MAX_JSON_VERIFY = 5_000_000
+
+// CSI: ESC [ params intermediates final.  OSC: ESC ] ... BEL or ESC \.  Neither class can contain ESC, so scans are linear.
+const ANSI = /\x1b\[[0-?]*[ -/]*[@-~]|\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)/g
+
+function trimEnd(line: string): string {
+  let e = line.length
+  while (e > 0) {
+    const c = line.charCodeAt(e - 1)
+    if (c === 32 || c === 9) e--
+    else break
+  }
+  return e === line.length ? line : line.slice(0, e)
+}
+
+/** The state of a line a terminal would show after all its `\r`-separated writes (later writes overwrite from column 0). */
+function overlay(line: string): string {
+  let cur: string[] = []
+  for (const seg of line.split("\r")) {
+    if (!seg) continue
+    const s = Array.from(seg)
+    cur = s.length >= cur.length ? s : [...s, ...cur.slice(s.length)]
+  }
+  return cur.join("")
+}
+
+const isWs = (c: number) => c === 32 || c === 9 || c === 10 || c === 13
+
+/** Removes the whitespace outside strings from text that is already known to be valid JSON. Linear, no recursion. */
+function stripJsonWhitespace(s: string): string {
+  const parts: string[] = []
+  let start = 0
+  let inStr = false
+  for (let i = 0; i < s.length; i++) {
+    const c = s.charCodeAt(i)
+    if (inStr) {
+      if (c === 92) i++
+      else if (c === 34) inStr = false
+    } else if (c === 34) inStr = true
+    else if (isWs(c)) {
+      if (i > start) parts.push(s.slice(start, i))
+      start = i + 1
+    }
+  }
+  if (start < s.length) parts.push(s.slice(start))
+  return parts.join("")
+}
+
+/**
+ * For text that is one JSON container document: its minified form (or the trimmed text itself when nothing can be removed or the
+ * minified form cannot be proven to parse to the same value). undefined when the text is not one JSON container document.
+ */
+function minifyDoc(s: string): string | undefined {
+  const t = s.trim()
+  if (t.length < 2) return undefined
+  const a = t[0]
+  const z = t[t.length - 1]
+  if (!((a === "{" && z === "}") || (a === "[" && z === "]"))) return undefined
+  let v: unknown
+  try {
+    v = JSON.parse(t)
+  } catch {
+    return undefined
+  }
+  const min = stripJsonWhitespace(t)
+  if (min.length >= t.length) return t
+  try {
+    if (t.length <= MAX_JSON_VERIFY && JSON.stringify(JSON.parse(min)) !== JSON.stringify(v)) return t
+  } catch {
+    return t
+  }
+  return min
+}
+
+/** Minified JSON text (possibly unchanged) when the text is one JSON document or JSON lines that all parse; else undefined. */
+function minifyJson(s: string): string | undefined {
+  const whole = minifyDoc(s)
+  if (whole !== undefined) return whole
+  const out: string[] = []
+  let docs = 0
+  for (const line of s.split("\n")) {
+    if (!line.trim()) {
+      out.push("")
+      continue
+    }
+    const m = minifyDoc(line)
+    if (m === undefined) return undefined
+    out.push(m)
+    docs++
+  }
+  return docs >= 2 ? out.join("\n") : undefined
+}
+
+function markerFor(n: number): string {
+  return `[previous line repeated ${n} more times]`
+}
+
+function collapseLines(lines: string[]): string[] {
+  // 1. blank runs of 3+ become one blank line
+  const a: string[] = []
+  for (let i = 0; i < lines.length; ) {
+    if (lines[i] === "") {
+      let j = i
+      while (j < lines.length && lines[j] === "") j++
+      const n = j - i
+      if (n >= 3) a.push("")
+      else for (let k = 0; k < n; k++) a.push("")
+      i = j
+    } else a.push(lines[i]!), i++
+  }
+  // 2. runs of 3+ identical lines become the line plus a marker, only when shorter
+  const out: string[] = []
+  for (let i = 0; i < a.length; ) {
+    let j = i + 1
+    while (j < a.length && a[j] === a[i]) j++
+    const n = j - i
+    if (n >= 3 && a[i] !== "") {
+      const marker = markerFor(n - 1)
+      if (marker.length < (n - 1) * (a[i]!.length + 1)) {
+        out.push(a[i]!, marker)
+        i = j
+        continue
+      }
+    }
+    for (let k = i; k < j; k++) out.push(a[k]!)
+    i = j
+  }
+  return out
+}
+
+function pass(text: string): string {
+  let s = text
+  if (s.includes("\x1b")) s = s.replace(ANSI, "")
+  if (s.includes("\r")) {
+    s = s.replace(/\r\n/g, "\n")
+    if (s.includes("\r")) s = s.split("\n").map((l) => (l.includes("\r") ? overlay(l) : l)).join("\n")
+  }
+  const lines = s.split("\n").map(trimEnd)
+  s = lines.join("\n")
+  const json = minifyJson(s)
+  if (json !== undefined) return json
+  return collapseLines(lines).join("\n")
+}
+
+/** Tidy one tool result's text. Never longer than the input; unchanged when the text is a brain stub, holds one, or looks binary. */
+export function tidyOutput(text: string): Tidied {
+  if (typeof text !== "string" || text.length < 4 || text.length > MAX_CHARS) return { text, saved: 0 }
+  if (text.includes(STUB_PREFIX) || text.includes("\0")) return { text, saved: 0 }
+  let cur = text
+  for (let i = 0; i < 6; i++) {
+    const next = pass(cur)
+    if (next === cur || next.length > cur.length) break
+    cur = next
+  }
+  return cur.length < text.length ? { text: cur, saved: text.length - cur.length } : { text, saved: 0 }
+}
+
+/** Tidies every tool result of a session (the input is not modified; untouched turns are shared). `tidied` is characters saved. */
+export function tidySession(session: Session): { session: Session; tidied: number } {
+  let tidied = 0
+  const turns = session.turns.map((t): Turn => {
+    if (t.kind !== "tool_result") return t
+    const r = tidyOutput(t.output)
+    if (!r.saved) return t
+    tidied += r.saved
+    return { ...t, output: r.text }
+  })
+  return tidied ? { session: { ...session, header: { ...session.header }, turns }, tidied } : { session, tidied: 0 }
+}