@100xprompt/brain 0.0.0-stage → 0.1.1

package/src/memory/audit.ts

@@ -0,0 +1,514 @@
+// `brain tokens audit [--dir <folder>]`: where a session's tokens are spent BEFORE the first message, on every request.
+// Static and read-only: it reads text files, parses configs as data and never starts, runs or evaluates anything it finds
+// (an MCP server's `command` is never even looked at: only the server names are read). The one process it runs is
+// `git ls-files` (with the fsmonitor hook switched off), and only to list tracked files; without git it walks the folder.
+// Tokens are chars / 4 like everywhere else in the brain. A finding is a fact plus a one-line fix, worst first.
+
+import { closeSync, lstatSync, openSync, readFileSync, readSync, readdirSync, realpathSync, statSync } from "node:fs"
+import { dirname, isAbsolute, join, relative, resolve } from "node:path"
+import { LEAN_AUTO_TOKENS } from "./prepare.ts"
+import { approxTokens } from "./types.ts"
+
+export const FILE_WARN_TOKENS = 2_000
+export const TOTAL_WARN_TOKENS = 6_000
+export const LARGE_FILE_BYTES = 100 * 1024
+/** A config or instruction file bigger than this is not read into memory (its size still counts). */
+const MAX_TEXT_BYTES = 4 * 1024 * 1024
+/** ~/.claude.json holds every project's history and gets big. */
+const MAX_CONFIG_BYTES = 64 * 1024 * 1024
+
+export interface InstructionFile {
+  path: string
+  scope: "project" | "user"
+  chars: number
+  tokens: number
+  /** Files pulled in by `@path` lines (Claude memory files only, one level). */
+  includes: Array<{ path: string; chars: number; tokens: number }>
+  /** tokens of the file plus its includes. */
+  totalTokens: number
+  /** Set when the file could not be read; it then counts as zero. */
+  unreadable?: string
+}
+
+export interface McpConfig {
+  tool: string
+  path: string
+  servers: string[]
+  error?: string
+}
+
+export interface LargeFile {
+  path: string
+  bytes: number
+  tokens: number
+}
+
+export interface Finding {
+  severity: 1 | 2 | 3
+  area: "instructions" | "mcp" | "ignore" | "files" | "brain"
+  message: string
+  fix: string
+  /** Tokens at stake, when there is a number. */
+  tokens?: number
+}
+
+export interface AuditReport {
+  dir: string
+  gitTracked: boolean
+  instructions: InstructionFile[]
+  instructionTokens: number
+  mcp: McpConfig[]
+  /** Most servers any one tool has configured (a session uses one tool). */
+  mcpServersMax: number
+  ignore: {
+    gitignore: boolean
+    agentIgnores: string[]
+    nodeModulesPresent: boolean
+    nodeModulesCovered: boolean
+    buildDirsUncovered: string[]
+    lockfilesUncovered: string[]
+  }
+  large: LargeFile[]
+  brain: { installedIn: string[]; leanThresholdTokens: number }
+  findings: Finding[]
+  /** Instruction-file tokens: what is certainly paid on every request. MCP tool definitions come on top and are not measured. */
+  reloadedTokens: number
+}
+
+// ---- reading ------------------------------------------------------------------------------------------------------
+
+type Read = { ok: true; size: number; text?: string } | { ok: false; why: string }
+
+/** A regular file's text (or, over `max`, only its size). Never throws. */
+function readText(path: string, max = MAX_TEXT_BYTES): Read | undefined {
+  let st
+  try {
+    st = statSync(path)
+  } catch {
+    return undefined
+  }
+  if (!st.isFile()) return { ok: false, why: "not a regular file" }
+  if (st.size > max) return { ok: true, size: st.size }
+  try {
+    const text = readFileSync(path, "utf8")
+    return { ok: true, size: st.size, text }
+  } catch (e) {
+    return { ok: false, why: (e as NodeJS.ErrnoException).code ?? "unreadable" }
+  }
+}
+
+const real = (p: string): string => {
+  try {
+    return realpathSync(p)
+  } catch {
+    return p
+  }
+}
+
+const tokensOf = (r: { size: number; text?: string }): { chars: number; tokens: number } =>
+  r.text !== undefined ? { chars: r.text.length, tokens: approxTokens(r.text) } : { chars: r.size, tokens: Math.ceil(r.size / 4) }
+
+// ---- instruction files --------------------------------------------------------------------------------------------
+
+/** `@path` tokens outside code fences and inline code, as Claude memory files use them to include other files. */
+export function includeTargets(text: string): string[] {
+  const out: string[] = []
+  let fenced = false
+  for (const raw of text.split(/\r?\n/)) {
+    if (/^\s*(```|~~~)/.test(raw)) {
+      fenced = !fenced
+      continue
+    }
+    if (fenced) continue
+    const line = raw.replace(/`[^`]*`/g, " ")
+    for (const m of line.matchAll(/(?:^|\s)@(\S+)/g)) {
+      const p = m[1]!.replace(/[.,;:!?)\]}'"]+$/, "")
+      if (p && (p.startsWith("~/") || p.startsWith("/") || p.startsWith("./") || p.startsWith("../") || /[/.]/.test(p)) && !out.includes(p)) out.push(p)
+    }
+  }
+  return out
+}
+
+const CLAUDE_MEMORY = /(^|\/)CLAUDE(\.local)?\.md$/
+
+function instructionFile(path: string, scope: "project" | "user", home: string): InstructionFile | undefined {
+  const r = readText(path)
+  if (!r) return undefined
+  if (!r.ok) return { path, scope, chars: 0, tokens: 0, includes: [], totalTokens: 0, unreadable: r.why }
+  const own = tokensOf(r)
+  const includes: InstructionFile["includes"] = []
+  if (r.text !== undefined && CLAUDE_MEMORY.test(path)) {
+    const seen = new Set<string>([real(path)])
+    for (const t of includeTargets(r.text)) {
+      const target = t.startsWith("~/") ? join(home, t.slice(2)) : isAbsolute(t) ? t : resolve(dirname(path), t)
+      const key = real(target)
+      if (seen.has(key)) continue
+      const inc = readText(target)
+      if (!inc || !inc.ok) continue // an @word that is not a readable file is just text
+      seen.add(key)
+      // One level only: what the included file includes is not followed.
+      includes.push({ path: target, ...tokensOf(inc) })
+    }
+  }
+  const totalTokens = own.tokens + includes.reduce((n, i) => n + i.tokens, 0)
+  return { path, scope, chars: own.chars, tokens: own.tokens, includes, totalTokens }
+}
+
+function findInstructions(dir: string, home: string): InstructionFile[] {
+  const candidates: Array<[string, "project" | "user"]> = [
+    ...["CLAUDE.md", "CLAUDE.local.md", ".claude/CLAUDE.md", "AGENTS.md", ".cursorrules", "GEMINI.md", ".github/copilot-instructions.md"].map((p): [string, "project"] => [join(dir, p), "project"]),
+    [join(home, ".claude", "CLAUDE.md"), "user"],
+    [join(home, ".codex", "AGENTS.md"), "user"],
+    [join(home, ".gemini", "GEMINI.md"), "user"],
+  ]
+  const seen = new Set<string>()
+  const out: InstructionFile[] = []
+  for (const [path, scope] of candidates) {
+    const key = real(path)
+    if (seen.has(key)) continue // a symlink to a file already counted (AGENTS.md -> CLAUDE.md) is one file
+    const f = instructionFile(path, scope, home)
+    if (!f) continue
+    seen.add(key)
+    out.push(f)
+  }
+  return out
+}
+
+// ---- MCP servers (names only) -------------------------------------------------------------------------------------
+
+const isObject = (v: unknown): v is Record<string, unknown> => !!v && typeof v === "object" && !Array.isArray(v)
+
+/** Names of the enabled servers in a `{name: {...}}` table. Only keys and the enabled flag are looked at. */
+function serverNames(table: unknown): string[] {
+  if (!isObject(table)) return []
+  return Object.keys(table).filter((k) => {
+    const v = table[k]
+    return !(isObject(v) && (v.disabled === true || v.enabled === false))
+  })
+}
+
+function mcpConfigs(dir: string, home: string): McpConfig[] {
+  const out: McpConfig[] = []
+  const json = (tool: string, path: string, pick: (c: Record<string, unknown>) => string[]) => {
+    const r = readText(path, MAX_CONFIG_BYTES)
+    if (!r) return
+    if (!r.ok || r.text === undefined) return void out.push({ tool, path, servers: [], error: r.ok ? "too large to read" : r.why })
+    try {
+      const c: unknown = JSON.parse(r.text)
+      if (!isObject(c)) return void out.push({ tool, path, servers: [], error: "not a JSON object" })
+      out.push({ tool, path, servers: [...new Set(pick(c))] })
+    } catch {
+      out.push({ tool, path, servers: [], error: "not plain JSON" })
+    }
+  }
+  json("project", join(dir, ".mcp.json"), (c) => serverNames(c.mcpServers ?? c.servers))
+  json("claude-code", join(home, ".claude.json"), (c) => {
+    const names = serverNames(c.mcpServers)
+    const projects = isObject(c.projects) ? c.projects : {}
+    for (const key of new Set([dir, real(dir)])) {
+      const p = projects[key]
+      if (isObject(p)) names.push(...serverNames(p.mcpServers))
+    }
+    return names
+  })
+  json("cursor", join(home, ".cursor", "mcp.json"), (c) => serverNames(c.mcpServers))
+  json("gemini", join(home, ".gemini", "settings.json"), (c) => serverNames(c.mcpServers))
+  const toml = join(home, ".codex", "config.toml")
+  const r = readText(toml, MAX_CONFIG_BYTES)
+  if (r) {
+    if (!r.ok || r.text === undefined) out.push({ tool: "codex", path: toml, servers: [], error: r.ok ? "too large to read" : r.why })
+    else {
+      try {
+        const c = Bun.TOML.parse(r.text) as Record<string, unknown>
+        out.push({ tool: "codex", path: toml, servers: serverNames(c.mcp_servers) })
+      } catch {
+        out.push({ tool: "codex", path: toml, servers: [], error: "not valid TOML" })
+      }
+    }
+  }
+  return out
+}
+
+// ---- ignore files -------------------------------------------------------------------------------------------------
+
+const AGENT_IGNORES = [".claudeignore", ".cursorignore", ".aiderignore"]
+const BUILD_DIRS = ["dist", "build", "out", ".next", ".nuxt", ".output", "target", "coverage", ".turbo", "__pycache__"]
+const LOCKFILES = ["package-lock.json", "yarn.lock", "pnpm-lock.yaml", "bun.lock", "bun.lockb", "Cargo.lock", "poetry.lock", "Pipfile.lock", "composer.lock", "Gemfile.lock", "go.sum", "uv.lock"]
+
+function ignorePatterns(path: string): string[] | undefined {
+  const r = readText(path, 1024 * 1024)
+  if (!r || !r.ok || r.text === undefined) return undefined
+  return r.text.split(/\r?\n/).map((l) => l.trim()).filter((l) => l && !l.startsWith("#") && !l.startsWith("!"))
+}
+
+/** Whether a top-level name is covered by an ignore pattern (a plain name, a trailing-slash dir, a `**` prefix, or a simple glob). */
+export function covers(patterns: string[], name: string): boolean {
+  for (const raw of patterns) {
+    const p = raw.replace(/^\/+/, "").replace(/^(\*\*\/)+/, "").replace(/\/+$/, "").replace(/\/\*\*$/, "")
+    if (!p || p.includes("/")) continue
+    if (p === name) return true
+    if (/[*?]/.test(p)) {
+      const re = new RegExp(`^${p.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*").replace(/\?/g, ".")}$`)
+      if (re.test(name)) return true
+    }
+  }
+  return false
+}
+
+const isDirPath = (p: string): boolean => {
+  try {
+    return statSync(p).isDirectory()
+  } catch {
+    return false
+  }
+}
+const isFilePath = (p: string): boolean => {
+  try {
+    return statSync(p).isFile()
+  } catch {
+    return false
+  }
+}
+
+function ignoreState(dir: string): AuditReport["ignore"] {
+  const git = ignorePatterns(join(dir, ".gitignore"))
+  const agent = AGENT_IGNORES.filter((f) => isFilePath(join(dir, f)))
+  const all = [...(git ?? []), ...agent.flatMap((f) => ignorePatterns(join(dir, f)) ?? [])]
+  const nodeModulesPresent = isDirPath(join(dir, "node_modules"))
+  return {
+    gitignore: git !== undefined,
+    agentIgnores: agent,
+    nodeModulesPresent,
+    nodeModulesCovered: covers(all, "node_modules"),
+    buildDirsUncovered: BUILD_DIRS.filter((d) => isDirPath(join(dir, d)) && !covers(all, d)),
+    lockfilesUncovered: LOCKFILES.filter((f) => isFilePath(join(dir, f)) && !covers(all, f)),
+  }
+}
+
+// ---- large tracked files ------------------------------------------------------------------------------------------
+
+function gitFiles(dir: string): string[] | undefined {
+  try {
+    // fsmonitor is a config-defined command: switch it off so listing files can never run anything from the repo's config.
+    const p = Bun.spawnSync(["git", "-c", "core.fsmonitor=false", "-C", dir, "ls-files", "-z"], { stdout: "pipe", stderr: "ignore", env: { ...process.env, GIT_OPTIONAL_LOCKS: "0" } })
+    if (p.exitCode !== 0) return undefined
+    // Paths are relative to `dir` already (ls-files lists the current directory when given -C).
+    return p.stdout.toString().split("\0").filter(Boolean)
+  } catch {
+    return undefined
+  }
+}
+
+const SKIP_DIRS = new Set(["node_modules", ...BUILD_DIRS])
+
+function walk(dir: string): string[] {
+  const out: string[] = []
+  const go = (d: string, depth: number) => {
+    if (depth > 8 || out.length >= 50_000) return
+    let entries
+    try {
+      entries = readdirSync(d, { withFileTypes: true })
+    } catch {
+      return
+    }
+    for (const e of entries) {
+      if (out.length >= 50_000) return
+      if (e.isDirectory()) {
+        if (!e.name.startsWith(".") && !SKIP_DIRS.has(e.name)) go(join(d, e.name), depth + 1)
+      } else if (e.isFile()) out.push(relative(dir, join(d, e.name)))
+    }
+  }
+  go(dir, 0)
+  return out
+}
+
+function isText(path: string): boolean {
+  let fd: number | undefined
+  try {
+    fd = openSync(path, "r")
+    const buf = Buffer.alloc(8192)
+    const n = readSync(fd, buf, 0, buf.length, 0)
+    return !buf.subarray(0, n).includes(0)
+  } catch {
+    return false
+  } finally {
+    if (fd !== undefined) closeSync(fd)
+  }
+}
+
+function largeFiles(dir: string, files: string[]): LargeFile[] {
+  const sized: LargeFile[] = []
+  for (const f of files) {
+    try {
+      const st = lstatSync(join(dir, f))
+      if (st.isFile() && st.size > LARGE_FILE_BYTES) sized.push({ path: f, bytes: st.size, tokens: Math.ceil(st.size / 4) })
+    } catch {
+      // Listed by git but gone from the working tree.
+    }
+  }
+  sized.sort((a, b) => b.bytes - a.bytes || (a.path < b.path ? -1 : 1))
+  const out: LargeFile[] = []
+  for (const s of sized) {
+    if (out.length === 5) break
+    if (isText(join(dir, s.path))) out.push(s)
+  }
+  return out
+}
+
+// ---- the audit ----------------------------------------------------------------------------------------------------
+
+const n = (x: number) => x.toLocaleString("en-US")
+
+export function auditTokens(dirIn: string, home: string): AuditReport {
+  const dir = resolve(dirIn)
+  const findings: Finding[] = []
+
+  const instructions = findInstructions(dir, home)
+  const instructionTokens = instructions.reduce((s, f) => s + f.totalTokens, 0)
+  for (const f of instructions) {
+    if (f.unreadable) findings.push({ severity: 1, area: "instructions", message: `${f.path} could not be read (${f.unreadable}), so it is not counted`, fix: "fix its permissions or remove it if it is not an instruction file" })
+    else if (f.totalTokens > FILE_WARN_TOKENS)
+      findings.push({
+        severity: 3,
+        area: "instructions",
+        message: `${f.path} reloads every request and is ${n(f.totalTokens)} tokens${f.includes.length ? ` (${n(f.tokens)} itself + ${f.includes.length} included file${f.includes.length === 1 ? "" : "s"})` : ""}, over ${n(FILE_WARN_TOKENS)}`,
+        fix: "keep only what every task needs; move the rest to files the model reads on demand",
+        tokens: f.totalTokens,
+      })
+  }
+  if (instructionTokens > TOTAL_WARN_TOKENS)
+    findings.push({
+      severity: 2,
+      area: "instructions",
+      message: `instruction files total ${n(instructionTokens)} tokens, over ${n(TOTAL_WARN_TOKENS)} (counted across all tools; one tool loads only its own files)`,
+      fix: "delete duplicated rules between CLAUDE.md, AGENTS.md and the user-level files",
+      tokens: instructionTokens,
+    })
+
+  const mcp = mcpConfigs(dir, home)
+  for (const c of mcp) if (c.error) findings.push({ severity: 1, area: "mcp", message: `${c.path} could not be read (${c.error}); its servers are not counted`, fix: "fix the file or ignore this note" })
+  const perTool = new Map<string, Set<string>>()
+  for (const c of mcp) {
+    const tool = c.tool === "project" ? "project" : c.tool
+    const set = perTool.get(tool) ?? new Set<string>()
+    for (const s of c.servers) if (s !== "brain") set.add(s) // the brain's own three tool definitions are small and fixed: not a finding
+    perTool.set(tool, set)
+  }
+  // The project's .mcp.json is read by Claude Code (and others): count it with each tool that has servers of its own.
+  const project = perTool.get("project") ?? new Set<string>()
+  let mcpServersMax = project.size
+  for (const [tool, set] of perTool) if (tool !== "project") mcpServersMax = Math.max(mcpServersMax, new Set([...set, ...project]).size)
+  const allNames = [...new Set(mcp.flatMap((c) => c.servers))].filter((s) => s !== "brain")
+  if (mcpServersMax > 0)
+    findings.push({
+      severity: mcpServersMax >= 6 ? 3 : mcpServersMax >= 3 ? 2 : 1,
+      area: "mcp",
+      message: `${mcpServersMax} MCP server${mcpServersMax === 1 ? "" : "s"} configured for one tool (${allNames.join(", ")}): each server's tool definitions are paid on every request unless deferred (the size was not measured: the servers were not started)`,
+      fix: "remove servers this project does not use, or turn on your tool's tool-search/deferred loading so definitions load only when needed",
+    })
+
+  const ignore = ignoreState(dir)
+  const tracked = gitFiles(dir)
+  const files = tracked ?? walk(dir)
+  const large = largeFiles(dir, files)
+  if (ignore.nodeModulesPresent && !ignore.nodeModulesCovered)
+    findings.push({ severity: 3, area: "ignore", message: "node_modules is present and not covered by .gitignore or an agent ignore file", fix: "add `node_modules/` to .gitignore" })
+  if (!ignore.gitignore && (ignore.buildDirsUncovered.length || ignore.nodeModulesPresent))
+    findings.push({ severity: 2, area: "ignore", message: "no .gitignore, and the folder has dependency or build output", fix: "add a .gitignore listing node_modules/ and build output" })
+  if (ignore.buildDirsUncovered.length)
+    findings.push({ severity: 2, area: "ignore", message: `build output not ignored: ${ignore.buildDirsUncovered.join(", ")}`, fix: `add ${ignore.buildDirsUncovered.map((d) => `${d}/`).join(" ")} to .gitignore` })
+  if (ignore.lockfilesUncovered.length)
+    findings.push({ severity: 1, area: "ignore", message: `lockfiles an agent may read (${ignore.lockfilesUncovered.join(", ")}) are not excluded from its context`, fix: "list them in .cursorignore / .aiderignore (Claude Code: a permissions.deny Read rule); keep them in git" })
+  if (!ignore.agentIgnores.length && (ignore.lockfilesUncovered.length || large.length))
+    findings.push({ severity: 1, area: "ignore", message: "no agent ignore file (.claudeignore, .cursorignore or .aiderignore)", fix: "add one listing lockfiles, generated files and large data files" })
+  if (large.length) {
+    const sum = large.reduce((s, f) => s + f.tokens, 0)
+    findings.push({
+      severity: 2,
+      area: "files",
+      message: `${large.length} large text file${large.length === 1 ? "" : "s"} over ${LARGE_FILE_BYTES / 1024} KB an agent may read whole (${large.map((f) => `${f.path} ${n(f.tokens)} tokens`).join(", ")})`,
+      fix: "exclude them in an agent ignore file or move generated data out of the repo",
+      tokens: sum,
+    })
+  }
+
+  const installedIn = mcp.filter((c) => c.servers.includes("brain")).map((c) => c.tool)
+  if (!installedIn.length) findings.push({ severity: 1, area: "brain", message: "the brain MCP server is not installed in any tool, so digests of big tool output cannot be expanded", fix: "run `brain mcp install`" })
+
+  findings.sort((a, b) => b.severity - a.severity || (b.tokens ?? 0) - (a.tokens ?? 0) || (a.message < b.message ? -1 : a.message > b.message ? 1 : 0))
+  return {
+    dir,
+    gitTracked: tracked !== undefined,
+    instructions,
+    instructionTokens,
+    mcp,
+    mcpServersMax,
+    ignore,
+    large,
+    brain: { installedIn, leanThresholdTokens: LEAN_AUTO_TOKENS },
+    findings,
+    reloadedTokens: instructionTokens,
+  }
+}
+
+export function formatAudit(r: AuditReport): string[] {
+  const out: string[] = [`brain tokens audit: ${r.dir}`, ""]
+  const label = ["", "low", "medium", "high"]
+  if (!r.findings.length) out.push("findings: none")
+  else {
+    out.push(`findings (${r.findings.length}, worst first):`)
+    r.findings.forEach((f, i) => {
+      out.push(`${i + 1}. [${label[f.severity]}] ${f.message}`)
+      out.push(`   fix: ${f.fix}`)
+    })
+  }
+  out.push("", `instruction files (reload every request): ${r.instructions.length}, ${n(r.instructionTokens)} tokens`)
+  for (const f of r.instructions) {
+    out.push(`  ${f.scope === "user" ? "user   " : "project"} ${f.path}  ${f.unreadable ? `unreadable (${f.unreadable})` : `${n(f.tokens)} tokens`}`)
+    for (const i of f.includes) out.push(`          @ ${i.path}  ${n(i.tokens)} tokens`)
+  }
+  const servers = r.mcp.filter((c) => c.servers.length)
+  out.push("", `mcp servers: ${servers.reduce((s, c) => s + c.servers.length, 0)} across ${servers.length} config${servers.length === 1 ? "" : "s"} (names only, nothing started)`)
+  for (const c of servers) out.push(`  ${c.tool}: ${c.servers.join(", ")}`)
+  const ig = r.ignore
+  out.push("", `ignore: .gitignore ${ig.gitignore ? "present" : "missing"}; agent ignore files: ${ig.agentIgnores.join(", ") || "none"}`)
+  out.push(`large files (${r.gitTracked ? "git-tracked" : "not a git repo: folder walk"}, text over ${LARGE_FILE_BYTES / 1024} KB): ${r.large.length ? "" : "none"}`.trimEnd())
+  for (const f of r.large) out.push(`  ${f.path}  ${n(f.tokens)} tokens`)
+  out.push("", `brain: MCP server ${r.brain.installedIn.length ? `installed (${r.brain.installedIn.join(", ")})` : "not installed"}; lean is automatic from ${n(r.brain.leanThresholdTokens)} tokens`)
+  out.push("", `tokens reloaded on every request: about ${n(r.reloadedTokens)} (instruction files)${r.mcpServersMax ? `, plus the tool definitions of ${r.mcpServersMax} MCP server${r.mcpServersMax === 1 ? "" : "s"} (not measured)` : ""}`)
+  return out
+}
+
+/** `brain tokens audit [--dir <folder>]`. Exit 0 when the audit ran, 1 on a usage error or a missing folder. */
+export async function tokensCommand(args: string[], home: string, io: { out(line: string): void } = { out: (l) => console.log(l) }): Promise<number> {
+  const [sub, ...rest] = args
+  if (sub === "lookups") {
+    const { formatLookupRates, lookupRates } = await import("./lookups.ts")
+    for (const l of formatLookupRates(lookupRates(home))) io.out(l)
+    return 0
+  }
+  if (sub !== "audit" && sub !== undefined) {
+    io.out(`unknown subcommand "${sub}"; usage: brain tokens audit [--dir <folder>] | brain tokens lookups`)
+    return 1
+  }
+  let dir = process.cwd()
+  for (let i = 0; i < rest.length; i++) {
+    if (rest[i] === "--dir") {
+      const v = rest[++i]
+      if (!v) {
+        io.out("--dir needs a folder")
+        return 1
+      }
+      dir = v
+    } else {
+      io.out(`unexpected argument "${rest[i]}"; usage: brain tokens audit [--dir <folder>]`)
+      return 1
+    }
+  }
+  if (!isDirPath(resolve(dir))) {
+    io.out(`not a folder: ${dir}`)
+    return 1
+  }
+  for (const line of formatAudit(auditTokens(dir, home))) io.out(line)
+  return 0
+}

package/src/memory/blobs.ts

@@ -0,0 +1,94 @@
+// The blob store: full texts that externalize() replaced with digests, kept on the person's disk so they can come back
+// on demand. `<home>/.brain/blobs/<id>.txt`, dir 0700, files 0600. The id is the first 16 hex of the sha256 of the text,
+// so the same text is stored once and no index is needed. Never ingested into Chitta; stored as they are.
+import { createHash } from "node:crypto"
+import { chmodSync, existsSync, mkdirSync, readdirSync, readFileSync, renameSync, statSync, unlinkSync, utimesSync, writeFileSync } from "node:fs"
+import { join } from "node:path"
+import type { BlobRef, BlobStore } from "./types.ts"
+
+const ID = /^[0-9a-f]{16}$/
+
+export const blobId = (text: string): string => createHash("sha256").update(text).digest("hex").slice(0, 16)
+
+export function openBlobStore(home: string): BlobStore {
+  const dir = join(home, ".brain", "blobs")
+  const ensure = () => {
+    mkdirSync(dir, { recursive: true, mode: 0o700 })
+    try {
+      chmodSync(dir, 0o700)
+    } catch {}
+  }
+  return {
+    dir,
+    put(text: string, meta?: { tool?: string; project?: string }): BlobRef {
+      const id = blobId(text)
+      const file = join(dir, `${id}.txt`)
+      // The tool kind, for the lookup-rate log (lookups.ts). A sidecar, so blobs stay plain text files.
+      const tag = () => {
+        if (!meta?.tool || !/^[a-z]{1,20}$/.test(meta.tool)) return
+        const m = join(dir, `${id}.kind`)
+        try {
+          if (!existsSync(m)) writeFileSync(m, meta.tool, { mode: 0o600 })
+        } catch {}
+      }
+      if (existsSync(file)) {
+        tag()
+        // Used again: refresh the mtime so prune() keeps what sessions still point at.
+        try {
+          const now = new Date()
+          utimesSync(file, now, now)
+        } catch {}
+        return { id, chars: text.length }
+      }
+      ensure()
+      const tmp = join(dir, `.${id}.${process.pid}.tmp`)
+      writeFileSync(tmp, text, { mode: 0o600 })
+      try {
+        chmodSync(tmp, 0o600)
+      } catch {}
+      renameSync(tmp, file)
+      tag()
+      return { id, chars: text.length }
+    },
+    kindOf(id: string): string | undefined {
+      if (!ID.test(id)) return undefined
+      try {
+        return readFileSync(join(dir, `${id}.kind`), "utf8").trim() || undefined
+      } catch {
+        return undefined
+      }
+    },
+    get(id: string): string | undefined {
+      if (!ID.test(id)) return undefined // also blocks path traversal
+      try {
+        return readFileSync(join(dir, `${id}.txt`), "utf8")
+      } catch {
+        return undefined
+      }
+    },
+    prune(olderThanMs: number): number {
+      let n = 0
+      let names: string[]
+      try {
+        names = readdirSync(dir)
+      } catch {
+        return 0
+      }
+      const cutoff = Date.now() - olderThanMs
+      for (const name of names) {
+        if (!/^[0-9a-f]{16}\.txt$/.test(name)) continue
+        const p = join(dir, name)
+        try {
+          if (statSync(p).mtimeMs < cutoff) {
+            unlinkSync(p)
+            try {
+              unlinkSync(p.replace(/\.txt$/, ".kind"))
+            } catch {}
+            n++
+          }
+        } catch {}
+      }
+      return n
+    },
+  }
+}