@100xprompt/brain 0.0.0-stage → 0.1.0

package/src/memory/externalize.ts

@@ -0,0 +1,435 @@
+// externalize: replace big tool calls/results/patches with a one-line digest and a pointer; the full text lives in the blob
+// store and comes back with `brain_expand`. Pure over the session (the input is not modified; untouched turns are shared by
+// reference), deterministic, no model. Stubs keep id/callId/ok/toolKind/name, so call/result pairing never breaks.
+import type { Session, ToolKind, Turn } from "../session/types.ts"
+import { asText } from "../session/util.ts"
+import { hasSecret, redact } from "./redact.ts"
+import { tidyOutput } from "./tidy.ts"
+import { previewOf } from "./preview.ts"
+import { STUB_PREFIX, approxTokens, type BlobRef, type BlobStore, type ExternalizeOptions, type ExternalizeResult } from "./types.ts"
+
+/** ExternalizeOptions plus what the tidy and budget steps add (types.ts is not edited here; fold these in there when convenient). */
+export interface ExternalizeOpts extends ExternalizeOptions {
+  /** Tidy the tool results that stay in the conversation (lossless in meaning, see tidy.ts). Default true. */
+  tidy?: boolean
+  /** The newest tool turns are protected only while their total stays at or under this many tokens (besides `keepRecent`). */
+  keepRecentTokens?: number
+  /** Stop replacing (oldest first) once the session is estimated at or under this many tokens. */
+  stopAtTokens?: number
+  /** Add a preview (outline, head/tail, error lines, first matches) under each result's stub. Default true. */
+  preview?: boolean
+}
+
+/** ExternalizeResult plus `tidied`: characters saved by tidy on results that stayed. */
+export interface ExternalizeOutcome extends ExternalizeResult {
+  tidied: number
+  /** Replaced tool results by tool kind (for the lookup-rate log). */
+  byKind: Partial<Record<ToolKind, number>>
+}
+
+type Call = Extract<Turn, { kind: "tool_call" }>
+type Result = Extract<Turn, { kind: "tool_result" }>
+
+const ID_IN_STUB = /^\[brain: output stored[^\]]*?\bid=([0-9a-f]{16})\b/
+
+/** True when the text is (or starts with) a stub written by externalize. */
+export const isStub = (text: string): boolean => typeof text === "string" && text.trimStart().startsWith(STUB_PREFIX)
+
+/** The blob id inside a stub, or undefined when the text is not a stub. */
+export function expandStub(text: string): string | undefined {
+  if (typeof text !== "string") return undefined
+  return ID_IN_STUB.exec(text.trimStart())?.[1]
+}
+
+const clipLine = (s: string, n: number): string => {
+  const t = s.replace(/\s+/g, " ").trim()
+  return t.length > n ? `${t.slice(0, n - 1)}…` : t
+}
+const lines = (s: string): string[] => s.split("\n")
+const nonEmpty = (s: string): string[] => lines(s).filter((l) => l.trim())
+const firstMeaningful = (s: string): string => nonEmpty(s).find((l) => /[\p{L}\p{N}]/u.test(l)) ?? ""
+
+function field(input: unknown, ...keys: string[]): string | undefined {
+  if (!input || typeof input !== "object") return undefined
+  const o = input as Record<string, unknown>
+  for (const k of keys) {
+    const v = o[k]
+    if (typeof v === "string" && v) return v
+    if (Array.isArray(v) && v.every((x) => typeof x === "string") && v.length) return v.join(" ")
+  }
+  return undefined
+}
+const pathOf = (input: unknown) => field(input, "file_path", "path", "filePath", "target_file", "file", "filename", "notebook_path")
+const commandOf = (input: unknown) => (typeof input === "string" ? input : field(input, "command", "cmd", "script"))
+
+/** A one-line summary of a test run when a common runner format is found (bun, jest, pytest, go, cargo). */
+export function testSummary(out: string): string | undefined {
+  const bunPass = /^\s*(\d+) pass\b/m.exec(out)
+  const bunFail = /^\s*(\d+) fail\b/m.exec(out)
+  if (bunPass || bunFail) return `${bunPass?.[1] ?? 0} pass, ${bunFail?.[1] ?? 0} fail`
+  const jest = /^Tests:\s+(.+)$/m.exec(out)
+  if (jest) return clipLine(jest[1]!, 80)
+  const py = /^=+ (.*\b(?:passed|failed|error)[^=]*?) =+\s*$/m.exec(out)
+  if (py) return clipLine(py[1]!, 80)
+  const cargo = [...out.matchAll(/^test result: (\w+)\. (\d+) passed; (\d+) failed/gm)]
+  if (cargo.length) {
+    const p = cargo.reduce((a, m) => a + Number(m[2]), 0)
+    const f = cargo.reduce((a, m) => a + Number(m[3]), 0)
+    return `${p} passed, ${f} failed`
+  }
+  const goOk = (out.match(/^ok\s+\S+/gm) ?? []).length
+  const goFail = (out.match(/^(?:FAIL\s+\S+|--- FAIL)/gm) ?? []).length
+  if (goOk || goFail) return `go: ${goOk} ok, ${goFail} fail`
+  return undefined
+}
+
+function exitStatus(out: string, ok: boolean): string {
+  const m = /(?:Exit code|Process exited with code|exit code|exited with code)[:\s]+(-?\d+)/i.exec(out)
+  if (m) return `exit ${m[1]}`
+  return ok ? "exit 0" : "failed"
+}
+
+const sizeOf = (n: number) => (n >= 10_000 ? `${Math.round(n / 1000)}K chars` : `${n} chars`)
+
+/**
+ * A short, deterministic description of a tool call or result. `call` is the call a result answers (for the path or
+ * command, which the result alone does not carry). Never contains text matching hasSecret.
+ */
+export function digestOf(turn: Turn, kindHint?: ToolKind, call?: Call): string {
+  const raw = digestRaw(turn, kindHint, call)
+  let d = redact(raw)
+  if (hasSecret(d)) d = "[redacted]"
+  return d
+}
+
+function digestRaw(turn: Turn, kindHint: ToolKind | undefined, call: Call | undefined): string {
+  if (turn.kind === "file_edit") {
+    const p = turn.patch ?? ""
+    const first = lines(p).find((l) => /^[+-](?![+-])/.test(l))
+    return `edit ${turn.path} (${sizeOf(p.length)})${first ? `, first change: ${clipLine(first, 80)}` : ""}`
+  }
+  if (turn.kind === "tool_call") {
+    const kind = kindHint ?? turn.toolKind
+    const p = pathOf(turn.input)
+    const size = sizeOf(asText(turn.input).length)
+    if (kind === "edit" || kind === "write") {
+      const body = field(turn.input, "new_string", "content", "file_text", "new_str", "text", "patch", "diff", "contents") ?? ""
+      const first = firstMeaningful(body)
+      return `${kind} ${p ?? turn.name} (${size})${first ? `, first line: ${clipLine(first, 80)}` : ""}`
+    }
+    if (kind === "shell") return `${turn.name}: ${clipLine(firstMeaningfulOr(commandOf(turn.input) ?? asText(turn.input)), 100)} (${size})`
+    return `${turn.name}${p ? ` ${p}` : ""} (${size})`
+  }
+  if (turn.kind !== "tool_result") return clipLine(firstMeaningful(asText((turn as { text?: string }).text)), 100)
+
+  const kind = kindHint ?? call?.toolKind ?? "other"
+  const out = turn.output
+  const n = nonEmpty(out).length
+  const size = sizeOf(out.length)
+  switch (kind) {
+    case "read": {
+      const p = call ? pathOf(call.input) : undefined
+      const first = firstMeaningful(out)
+      return `read${p ? ` ${p}` : ""}: ${n} lines${first ? `, starts: ${clipLine(first, 80)}` : ""}`
+    }
+    case "shell": {
+      const cmd = call ? commandOf(call.input) : undefined
+      const t = testSummary(out)
+      return `${cmd ? `$ ${clipLine(firstMeaningfulOr(cmd), 100)}: ` : ""}${exitStatus(out, turn.ok)}, ${n} lines${t ? `, tests: ${t}` : ""}`
+    }
+    case "search": {
+      const m = nonEmpty(out)
+      return `${m.length} matches${m[0] ? `: ${clipLine(m[0], 70)}` : ""}${m[1] ? ` | ${clipLine(m[1], 70)}` : ""}`
+    }
+    case "list":
+      return `${n} entries`
+    case "web": {
+      const url = (call && field(call.input, "url", "uri", "query")) || /https?:\/\/\S+/.exec(out)?.[0]
+      return `web${url ? ` ${clipLine(url, 100)}` : ""}: ${size}`
+    }
+    default: {
+      const first = firstMeaningful(out)
+      return `${first ? clipLine(first, 100) : "output"} (${size})`
+    }
+  }
+}
+const firstMeaningfulOr = (s: string): string => nonEmpty(s)[0] ?? s
+
+function stubText(chars: number, id: string, digest: string): string {
+  return `${STUB_PREFIX}, ${chars} chars, id=${id}: ${clipLine(digest, 200)} | brain_expand id=${id} (grep= for matching lines)]`
+}
+
+/** Characters a turn carries (the text the measurements count). */
+function charsOf(t: Turn): number {
+  switch (t.kind) {
+    case "user":
+    case "assistant":
+    case "reasoning":
+      return t.text.length
+    case "tool_call":
+      return asText(t.input).length
+    case "tool_result":
+      return t.output.length
+    case "file_edit":
+      return (t.patch ?? "").length
+    case "compaction":
+      return t.summary.length
+    default:
+      return 0
+  }
+}
+
+const FIELD_MIN = 300
+
+/** Replaces long strings anywhere inside a tool input with stubs; returns the new input and how many were replaced. */
+function stubInput(input: unknown, call: Call, store: BlobStore, depth = 0): { value: unknown; n: number } {
+  if (typeof input === "string") {
+    if (input.length <= FIELD_MIN || isStub(input)) return { value: input, n: 0 }
+    const ref = store.put(input)
+    return { value: stubText(input.length, ref.id, digestOf({ ...call, input }, call.toolKind)), n: 1 }
+  }
+  if (depth > 6 || !input || typeof input !== "object") return { value: input, n: 0 }
+  let n = 0
+  if (Array.isArray(input)) {
+    const out = input.map((v) => {
+      const r = stubInput(v, call, store, depth + 1)
+      n += r.n
+      return r.value
+    })
+    return { value: n ? out : input, n }
+  }
+  const out: Record<string, unknown> = {}
+  for (const [k, v] of Object.entries(input as Record<string, unknown>)) {
+    if (typeof v === "string" && v.length > FIELD_MIN && !isStub(v)) {
+      const ref = store.put(v)
+      const first = firstMeaningful(v)
+      out[k] = stubText(v.length, ref.id, `${call.name} ${pathOf(input) ?? ""} ${k}${first ? `: ${clipLine(first, 70)}` : ""}`.replace(/\s+/g, " "))
+      n++
+    } else {
+      const r = stubInput(v, call, store, depth + 1)
+      n += r.n
+      out[k] = r.value
+    }
+  }
+  return { value: n ? out : input, n }
+}
+
+export function externalize(session: Session, store: BlobStore, opts: ExternalizeOpts = {}): ExternalizeOutcome {
+  const minChars = opts.minChars ?? 1500
+  const keepRecent = opts.keepRecent ?? 8
+  const doTidy = opts.tidy !== false
+  const doPreview = opts.preview !== false
+  const byKind: Partial<Record<ToolKind, number>> = {}
+
+  // Units: a call with its result and file edits; other file edits stand alone. The newest keepRecent units are protected.
+  const unitOf = new Map<number, number>()
+  const byCall = new Map<string, number>()
+  const calls = new Map<string, Call>()
+  let units = 0
+  session.turns.forEach((t, i) => {
+    if (t.kind === "tool_call") {
+      calls.set(t.id, t)
+      byCall.set(t.id, units)
+      unitOf.set(i, units++)
+    } else if (t.kind === "tool_result" || (t.kind === "file_edit" && t.callId)) {
+      const u = byCall.get(t.kind === "tool_result" ? t.callId : t.callId!)
+      unitOf.set(i, u ?? units++)
+    } else if (t.kind === "file_edit") unitOf.set(i, units++)
+  })
+  let protectedFrom = units - keepRecent
+  if (opts.keepRecentTokens !== undefined) {
+    // Also a token cap: walk the newest units back while their total stays within it.
+    const unitChars = new Array<number>(units).fill(0)
+    session.turns.forEach((t, i) => {
+      const u = unitOf.get(i)
+      if (u !== undefined) unitChars[u]! += charsOf(t)
+    })
+    let acc = 0
+    let from = units
+    for (let u = units - 1; u >= Math.max(0, units - keepRecent); u--) {
+      acc += unitChars[u]!
+      if (acc / 4 > opts.keepRecentTokens) break
+      from = u
+    }
+    protectedFrom = Math.max(protectedFrom, from)
+  }
+
+  // Tidy first (on a view; the blob always gets the original text): what stays in the conversation is the tidied text.
+  const tidyOf = new Map<number, { text: string; saved: number }>()
+  let before = 0
+  let tidySaved = 0
+  session.turns.forEach((t, i) => {
+    before += charsOf(t)
+    if (doTidy && t.kind === "tool_result" && !isStub(t.output)) {
+      const r = tidyOutput(t.output)
+      if (r.saved) {
+        tidyOf.set(i, r)
+        tidySaved += r.saved
+      }
+    }
+  })
+  const stopAt = opts.stopAtTokens !== undefined ? opts.stopAtTokens * 4 : undefined
+  let projected = before - tidySaved
+
+  let after = 0
+  let replaced = 0
+  let tidied = 0
+  const turns = session.turns.map((t, i): Turn => {
+    const td = tidyOf.get(i)
+    const cur: Turn = td && t.kind === "tool_result" ? { ...t, output: td.text } : t
+    const b = charsOf(cur)
+    const u = unitOf.get(i)
+    const done = stopAt !== undefined && projected <= stopAt
+    if (u === undefined || u >= protectedFrom || done) {
+      after += b
+      if (td) tidied += td.saved
+      return cur
+    }
+    let next = cur
+    if (t.kind === "tool_result") {
+      if (t.output.length > minChars && !isStub(t.output)) {
+        const call = calls.get(t.callId)
+        const kind: ToolKind = call?.toolKind ?? "other"
+        const ref = store.put(t.output, { tool: kind })
+        const stub = stubText(t.output.length, ref.id, digestOf(t, undefined, call))
+        let text = stub
+        // A preview is worth keeping only while the stub stays well under the output it replaces (at most half of it);
+        // otherwise (short outputs at the deeper --budget levels) the plain stub is used, as before previews existed.
+        const pv = doPreview ? previewOf(t.output, kind, { ok: t.ok, ...(call && pathOf(call.input) ? { path: pathOf(call.input)! } : {}) }) : ""
+        if (pv && stub.length + 1 + pv.length <= b / 2) {
+          text = `${stub}\n${pv}`
+        } else if (!t.ok) {
+          const err = nonEmpty(t.output)
+            .slice(0, 4)
+            .map((l) => {
+              const r = clipLine(redact(l), 200)
+              return hasSecret(r) ? "[line withheld: secret]" : r
+            })
+            .join("\n")
+          text = `${stub}\n${err}`
+        }
+        if (text.length < b) {
+          next = { ...t, output: text }
+          replaced++
+          byKind[kind] = (byKind[kind] ?? 0) + 1
+        }
+      }
+    } else if (t.kind === "tool_call") {
+      if (b > minChars) {
+        const r = stubInput(t.input, t, store)
+        if (r.n) {
+          next = { ...t, input: r.value }
+          replaced++
+        }
+      }
+    } else if (t.kind === "file_edit") {
+      const p = t.patch
+      if (p && p.length > minChars && !isStub(p)) {
+        const ref = store.put(p)
+        const stub = stubText(p.length, ref.id, digestOf(t))
+        if (stub.length < p.length) {
+          next = { ...t, patch: stub }
+          replaced++
+        }
+      }
+    }
+    if (next === cur && td) tidied += td.saved
+    const nb = charsOf(next)
+    projected -= b - nb
+    after += nb
+    return next
+  })
+  return { session: { ...session, header: { ...session.header }, turns }, replaced, before, after, tidied, byKind }
+}
+
+/** Blobs counted but not kept: trial runs and dry runs. */
+export function countingBlobs(): BlobStore {
+  const m = new Map<string, string>()
+  return {
+    dir: "(not stored)",
+    put(text: string): BlobRef {
+      const id = Bun.hash(text).toString(16).padStart(16, "0").slice(0, 16)
+      m.set(id, text)
+      return { id, chars: text.length }
+    },
+    get: (id) => m.get(id),
+    prune: () => 0,
+  }
+}
+
+/** The tokens in a session's text-bearing parts (chars / 4): the same estimate the lean notes and the measurements use. */
+export function estimateTokens(s: Session): number {
+  let chars = 0
+  for (const t of s.turns) {
+    switch (t.kind) {
+      case "user":
+      case "assistant":
+      case "reasoning":
+        chars += t.text.length
+        break
+      case "tool_call":
+        chars += JSON.stringify(t.input ?? "").length
+        break
+      case "tool_result":
+        chars += t.output.length
+        break
+      case "file_edit":
+        chars += t.patch?.length ?? 0
+        break
+      case "compaction":
+        chars += t.summary.length
+        break
+    }
+  }
+  return Math.ceil(chars / 4)
+}
+
+/** Escalation levels (the graduated idea: mask a little, then more, before giving up detail). Level 0 means nothing needed masking. */
+export const BUDGET_LEVELS: ReadonlyArray<{ minChars: number; keepRecent: number }> = [
+  { minChars: 1500, keepRecent: 8 },
+  { minChars: 800, keepRecent: 8 },
+  { minChars: 800, keepRecent: 4 },
+  { minChars: 300, keepRecent: 2 },
+]
+/** Share of the target the newest tool turns may take while they stay protected. */
+export const KEEP_RECENT_SHARE = 0.2
+
+export interface BudgetOptions {
+  targetTokens: number
+  tidy?: boolean
+}
+export interface BudgetOutcome extends ExternalizeOutcome {
+  /** 0: the tidied session already fits; 1..4: the first level at which it fits (4 when none does). */
+  level: number
+  /** Whether the session ended at or under the target. */
+  reached: boolean
+  /** Estimated tokens after (chars / 4, as `estimateTokens`). */
+  tokens: number
+}
+
+/**
+ * Masks only as much as the target needs: tries level 1 (today's defaults), then 2, 3, 4 until the whole level brings the session
+ * to the target, then applies that level for real, oldest first, stopping as soon as the session fits. Trial runs store nothing.
+ */
+export function externalizeToBudget(session: Session, store: BlobStore, o: BudgetOptions): BudgetOutcome {
+  const target = Math.max(0, Math.floor(o.targetTokens))
+  const cap = Math.round(target * KEEP_RECENT_SHARE)
+  const common = { keepRecentTokens: cap, ...(o.tidy === false ? { tidy: false } : {}) }
+  let level = 0
+  const trialTokens = (idx: number) => approxTokens("x".repeat(externalize(session, countingBlobs(), { ...BUDGET_LEVELS[idx]!, ...common }).after))
+  const base = externalize(session, countingBlobs(), { ...BUDGET_LEVELS[0]!, ...common, stopAtTokens: target })
+  if (base.replaced === 0 && approxTokens("x".repeat(base.after)) <= target) {
+    level = 0
+  } else {
+    for (let i = 0; i < BUDGET_LEVELS.length; i++) {
+      level = i + 1
+      if (trialTokens(i) <= target) {
+        break
+      }
+    }
+  }
+  const opts: ExternalizeOpts = { ...BUDGET_LEVELS[Math.max(0, level - 1)]!, ...common, stopAtTokens: target }
+  const r = externalize(session, store, opts)
+  const tokens = estimateTokens(r.session)
+  return { ...r, level, reached: tokens <= target, tokens }
+}

package/src/memory/lookups.ts

@@ -0,0 +1,70 @@
+// The lookup-rate log: which kinds of tool output get replaced by stubs (at conversion) and which the model then asks back with
+// brain_expand. Each lookup costs a whole extra request (docs/live-savings/), so this is the number that says whether a stub's
+// preview is good enough for that kind of output. Local only: `<home>/.brain/logs/lookups.jsonl` (0600), ids and kinds, never text.
+import { appendFileSync, chmodSync, existsSync, mkdirSync, readFileSync } from "node:fs"
+import { join } from "node:path"
+
+export const lookupLog = (home: string) => join(home, ".brain", "logs", "lookups.jsonl")
+
+export type LookupEvent =
+  | { event: "stubs"; at: string; session: string; byKind: Record<string, number> }
+  | { event: "expand"; at: string; id: string; kind: string; how: "grep" | "page" | "window" }
+
+export function logLookup(home: string, e: LookupEvent): void {
+  try {
+    const f = lookupLog(home)
+    mkdirSync(join(home, ".brain", "logs"), { recursive: true, mode: 0o700 })
+    const fresh = !existsSync(f)
+    appendFileSync(f, `${JSON.stringify(e)}\n`, { mode: 0o600 })
+    if (fresh) chmodSync(f, 0o600)
+  } catch {
+    // A log never breaks a conversion or a tool call.
+  }
+}
+
+export interface LookupRow {
+  kind: string
+  stubbed: number
+  expanded: number
+  /** expanded / stubbed (an output expanded twice counts twice). */
+  rate: number
+}
+
+/** Stubs and expansions per tool kind, most expanded first. */
+export function lookupRates(home: string): LookupRow[] {
+  let text = ""
+  try {
+    text = readFileSync(lookupLog(home), "utf8")
+  } catch {
+    return []
+  }
+  const stubbed = new Map<string, number>()
+  const expanded = new Map<string, number>()
+  for (const line of text.split("\n")) {
+    if (!line.trim()) continue
+    let e: LookupEvent
+    try {
+      e = JSON.parse(line)
+    } catch {
+      continue
+    }
+    if (e.event === "stubs") for (const [k, n] of Object.entries(e.byKind ?? {})) stubbed.set(k, (stubbed.get(k) ?? 0) + (Number(n) || 0))
+    else if (e.event === "expand") expanded.set(e.kind || "unknown", (expanded.get(e.kind || "unknown") ?? 0) + 1)
+  }
+  const kinds = new Set([...stubbed.keys(), ...expanded.keys()])
+  return [...kinds]
+    .map((kind) => {
+      const s = stubbed.get(kind) ?? 0
+      const x = expanded.get(kind) ?? 0
+      return { kind, stubbed: s, expanded: x, rate: s ? x / s : x ? Infinity : 0 }
+    })
+    .sort((a, b) => b.expanded - a.expanded || b.stubbed - a.stubbed)
+}
+
+export function formatLookupRates(rows: LookupRow[]): string[] {
+  if (!rows.length) return ["no lean conversions or expansions logged yet (`brain session continue --lean` writes them)"]
+  const out = ["kind       stubbed  expanded  rate"]
+  for (const r of rows) out.push(`${r.kind.padEnd(10)} ${String(r.stubbed).padStart(7)} ${String(r.expanded).padStart(9)}  ${Number.isFinite(r.rate) ? `${Math.round(100 * r.rate)}%` : "n/a"}`)
+  out.push("", "A kind with a high rate needs a better stub preview (src/memory/preview.ts): each expansion is one more request.")
+  return out
+}