@100xprompt/brain 0.0.0-stage → 0.1.0

package/src/adapters/gemini-cli/records.ts

@@ -0,0 +1,521 @@
+import { readFileSync } from "node:fs"
+import type { Attachment, SessionHeader, Turn, Usage } from "../../session/types.ts"
+import { asText, clip, iso, noteModel, toolKindOf, type lossCounter } from "../../session/util.ts"
+
+// Facts about Gemini CLI's session records, shared by its reader. Checked against the installed 0.33.1
+// (whole-file chats/session-*.json and checkpoints/*.json on this Mac) and against upstream main
+// (packages/core/src/services/chatRecordingService.ts and chatRecordingTypes.ts), which writes the
+// append-only .jsonl. See docs/sessions/2026-11-08-session-formats.md, entry 4.
+
+export const TOOL = "gemini-cli"
+export const LABEL = "Gemini CLI"
+
+type Lost = ReturnType<typeof lossCounter>
+
+/**
+ * The note brain puts first in a continued session's first user message, as its own text part: the model
+ * reads it, the screen shows the message's displayContent without it. Readers drop it and record the source.
+ */
+export const continuedNote = (label: string, id: string, when: string) =>
+  `[This session continues ${label} session ${id}, carried over by brain session on ${when}. The conversation below is its history; what could not carry over is left out. Check files before relying on details, since they may have changed.]`
+export const CONTINUED_NOTE = /^\[This session continues .+? session (\S+), carried over by brain session/
+
+/**
+ * Gemini CLI's compression replaces the history with its summary as a user message (the model writes it
+ * inside <state_snapshot>) and a fixed model acknowledgement; current versions persist that in the session.
+ */
+export const SNAPSHOT_OPEN = "<state_snapshot>"
+export const SNAPSHOT_ACK = "Got it. Thanks for the additional context!"
+/** A portable compaction's summary as the user message Gemini CLI's compression writes. */
+export const snapshotOf = (label: string, summary: string) =>
+  `${SNAPSHOT_OPEN}\n[Summary of the earlier conversation in ${label}, which replaced the turns before it]\n\n${summary}\n</state_snapshot>`
+/** The summary inside a <state_snapshot> message, or undefined when the text is not one. */
+export const snapshotText = (text: string) => {
+  const t = text.trim()
+  if (!t.startsWith(SNAPSHOT_OPEN)) return undefined
+  // The header snapshotOf adds is brain's, not part of the summary.
+  return t.slice(SNAPSHOT_OPEN.length).replace(/<\/state_snapshot>\s*$/, "").trim().replace(/^\[Summary of the earlier conversation in .+?, which replaced the turns before it\]\n\n/, "")
+}
+
+/** A Part from @google/genai, as Gemini CLI stores it in messages, tool results and checkpoints. */
+export interface Part {
+  text?: string
+  /** A thought summary part (readable). */
+  thought?: boolean
+  /** Gemini 3 signs function calls and thoughts; signatures never cross to another model. */
+  thoughtSignature?: string
+  functionCall?: { id?: string; name?: string; args?: unknown }
+  functionResponse?: { id?: string; name?: string; response?: unknown }
+  inlineData?: { mimeType?: string; data?: string }
+  fileData?: { mimeType?: string; fileUri?: string }
+}
+
+/** PartListUnion: a string, one Part, or a list of either. */
+export type PartList = string | Part | Array<string | Part>
+
+export interface ToolCallRecord {
+  id: string
+  name: string
+  args?: unknown
+  result?: PartList | null
+  status?: string
+  timestamp?: string
+  resultDisplay?: unknown
+}
+
+export interface MessageRecord {
+  id: string
+  timestamp?: string
+  /** user | gemini | info | error | warning */
+  type: string
+  content?: PartList
+  toolCalls?: ToolCallRecord[]
+  thoughts?: Array<{ subject?: string; description?: string; timestamp?: string }>
+  tokens?: { input?: number; output?: number; cached?: number; thoughts?: number } | null
+  model?: string
+}
+
+export interface Conversation {
+  sessionId: string
+  projectHash?: string
+  startTime?: string
+  lastUpdated?: string
+  /** A model-written summary of the session (newer versions), used as its title. */
+  summary?: string
+  kind?: string
+  messages: MessageRecord[]
+}
+
+export interface Loaded {
+  conv: Conversation
+  /** Record kinds the replay did not know, by name. */
+  unknown: Map<string, number>
+  /** Lines that were not JSON (a half-written last line, say). */
+  bad: number
+}
+
+const isObj = (v: unknown): v is Record<string, any> => typeof v === "object" && v !== null && !Array.isArray(v)
+
+function asConversation(v: unknown): Conversation | undefined {
+  if (!isObj(v) || typeof v.sessionId !== "string" || !Array.isArray(v.messages)) return undefined
+  return { ...(v as Conversation), messages: v.messages.filter((m: unknown) => isObj(m) && typeof m.id === "string") }
+}
+
+/**
+ * A session file, whichever way it was written: the older whole-file JSON (one ConversationRecord,
+ * rewritten every turn) or the current JSONL log, replayed. Undefined when it is neither.
+ */
+export function loadConversation(path: string): Loaded | undefined {
+  let text: string
+  try {
+    text = readFileSync(path, "utf8")
+  } catch {
+    return undefined
+  }
+  const whole = () => {
+    try {
+      const conv = asConversation(JSON.parse(text))
+      return conv ? { conv, unknown: new Map<string, number>(), bad: 0 } : undefined
+    } catch {
+      return undefined
+    }
+  }
+  // Gemini falls back the same way: a .jsonl whose replay finds no metadata is tried as legacy JSON.
+  return path.endsWith(".json") ? whole() ?? replay(text) : replay(text) ?? whole()
+}
+
+/**
+ * Replays a JSONL session the way chatRecordingService's loader does: the first line is metadata; a
+ * message line adds a message, or replaces an earlier one with the same id in place (that is how tool
+ * results and token counts are added after the fact); {$set} merges metadata (a `messages` array there is
+ * a legacy full-history checkpoint that replaces everything); {$patch} edits messages by id, removes
+ * some, or reorders them; {$rewindTo: id} drops that message and everything after it.
+ */
+export function replay(text: string): Loaded | undefined {
+  let meta: Record<string, any> = {}
+  const msgs = new Map<string, MessageRecord>()
+  const unknown = new Map<string, number>()
+  let bad = 0
+  const addAll = (list: unknown) => {
+    if (Array.isArray(list)) for (const m of list) if (isObj(m) && typeof m.id === "string") msgs.set(m.id, m as MessageRecord)
+  }
+  const patchOne = (p: Record<string, any>) => {
+    const m = msgs.get(p.id)
+    if (!m) return
+    if (p.content !== undefined) m.content = p.content
+    if (Array.isArray(p.toolCalls) && m.toolCalls) {
+      for (const tp of p.toolCalls) {
+        const tc = m.toolCalls.find((t) => t.id === tp?.id)
+        if (tc && "result" in tp) tc.result = tp.result
+      }
+    }
+  }
+  for (const line of text.split("\n")) {
+    if (!line.trim()) continue
+    let r: unknown
+    try {
+      r = JSON.parse(line)
+    } catch {
+      bad++
+      continue
+    }
+    if (!isObj(r)) {
+      bad++
+      continue
+    }
+    if (typeof r.$rewindTo === "string") {
+      let found = false
+      for (const id of [...msgs.keys()]) {
+        if (id === r.$rewindTo) found = true
+        if (found) msgs.delete(id)
+      }
+      if (!found) msgs.clear()
+    } else if (isObj(r.$patch)) {
+      const p = r.$patch
+      if (typeof p.id === "string") patchOne(p)
+      if (Array.isArray(p.updates)) for (const u of p.updates) if (isObj(u) && typeof u.id === "string") patchOne(u)
+      if (Array.isArray(p.removeIds)) for (const id of p.removeIds) if (typeof id === "string") msgs.delete(id)
+      if (Array.isArray(p.orderIds)) {
+        // Messages named in orderIds move to the end in that order; the rest keep their places before them.
+        const ordered = p.orderIds.filter((id: unknown) => typeof id === "string" && msgs.has(id)) as string[]
+        const keep = new Set(ordered)
+        const entries = [...[...msgs].filter(([id]) => !keep.has(id)), ...ordered.map((id) => [id, msgs.get(id)!] as const)]
+        msgs.clear()
+        for (const [id, m] of entries) msgs.set(id, m)
+      }
+    } else if (typeof r.id === "string") {
+      msgs.set(r.id, r as MessageRecord)
+    } else if (isObj(r.$set)) {
+      const { messages, ...rest } = r.$set
+      if (Array.isArray(messages)) {
+        msgs.clear()
+        addAll(messages)
+      }
+      meta = { ...meta, ...rest }
+    } else if (typeof r.sessionId === "string" && typeof r.projectHash === "string") {
+      const { messages, ...rest } = r
+      meta = { ...meta, ...rest }
+      addAll(messages)
+    } else {
+      const name = Object.keys(r)[0] ?? "empty"
+      unknown.set(name, (unknown.get(name) ?? 0) + 1)
+    }
+  }
+  if (typeof meta.sessionId !== "string") return undefined
+  return { conv: { ...(meta as Omit<Conversation, "messages">), sessionId: meta.sessionId, messages: [...msgs.values()] }, unknown, bad }
+}
+
+/** The parts of a PartListUnion as a list. */
+export function toParts(c: PartList | null | undefined): Part[] {
+  if (c === undefined || c === null) return []
+  if (typeof c === "string") return [{ text: c }]
+  if (Array.isArray(c)) return c.map((p) => (typeof p === "string" ? { text: p } : p)).filter(isObj)
+  return isObj(c) ? [c] : []
+}
+
+/** Readable text of a functionResponse.response: tools answer {output}, {error}, or both. */
+function responseText(r: unknown): { text: string; ok: boolean } {
+  if (isObj(r) && ("output" in r || "error" in r)) {
+    const text = [asText(r.output), asText(r.error)].filter(Boolean).join("\n")
+    return { text, ok: !r.error }
+  }
+  return { text: asText(r), ok: true }
+}
+
+/** Text of a tool result (its functionResponse parts plus any text parts), whether it failed, and binary parts. */
+export function resultText(result: PartList | null | undefined): { text: string; ok: boolean; binary: number } {
+  const out: string[] = []
+  let ok = true
+  let binary = 0
+  for (const p of toParts(result)) {
+    if (p.functionResponse) {
+      const r = responseText(p.functionResponse.response)
+      out.push(r.text)
+      ok &&= r.ok
+    } else if (typeof p.text === "string") out.push(p.text)
+    else if (p.inlineData || p.fileData) binary++
+  }
+  return { text: out.filter(Boolean).join("\n"), ok, binary }
+}
+
+/**
+ * Gemini's per-message token counts. Its `input` includes the cached part and its `output` leaves out
+ * thinking; the portable record keeps cached input apart and counts thinking as output, as Claude's and
+ * OpenAI's counts (and so the Claude Code and Codex readers) do.
+ */
+function usageOf(t: MessageRecord["tokens"]): Usage | undefined {
+  if (!t) return undefined
+  const cached = typeof t.cached === "number" && t.cached > 0 ? t.cached : 0
+  const u: Usage = {}
+  if (typeof t.input === "number") u.input = Math.max(0, t.input - cached)
+  if (typeof t.output === "number") u.output = t.output + (typeof t.thoughts === "number" ? t.thoughts : 0)
+  if (cached) u.cacheRead = cached
+  return Object.keys(u).length ? u : undefined
+}
+
+/** Keeps tool call ids unique within a session (a rewound turn can reuse one) and names calls that have none. */
+export function idMaker() {
+  const seen = new Set<string>()
+  let n = 0
+  return (id: string | undefined): string => {
+    const base = id || `gemini_call_${++n}`
+    let out = base
+    for (let i = 2; seen.has(out); i++) out = `${base}#${i}`
+    seen.add(out)
+    return out
+  }
+}
+
+type CallTurn = Extract<Turn, { kind: "tool_call" }>
+
+/**
+ * Pairs function calls with their responses across the API history kept in checkpoints, where a
+ * functionCall part often has no id and only the functionResponse carries one: by id when both have it,
+ * otherwise the oldest pending call with the same name, otherwise the oldest pending call.
+ */
+export function pairer(newId: ReturnType<typeof idMaker>) {
+  // `src` is the call's own id as recorded; the turn's id can differ when idMaker had to make it unique.
+  const pending: Array<{ turn: CallTurn; src?: string; name: string }> = []
+  const answered = new Set<string>()
+  return {
+    call(fc: NonNullable<Part["functionCall"]>, at: string | undefined): CallTurn {
+      const name = fc.name ?? "unknown"
+      const turn: CallTurn = { kind: "tool_call", ...(at ? { at } : {}), id: newId(fc.id), name, toolKind: toolKindOf(name), input: fc.args ?? {} }
+      pending.push({ turn, ...(fc.id ? { src: fc.id } : {}), name })
+      return turn
+    },
+    /** The id of the call this response answers, or undefined when no call is waiting for it. */
+    respond(fr: NonNullable<Part["functionResponse"]>): string | undefined {
+      let i = fr.id ? pending.findIndex((p) => p.src === fr.id) : -1
+      if (i < 0) i = pending.findIndex((p) => !p.src && p.name === fr.name)
+      if (i < 0) i = pending.findIndex((p) => !p.src)
+      if (i < 0) return undefined
+      const [p] = pending.splice(i, 1)
+      // A call without an id takes its response's id, so the pair reads naturally.
+      if (!p!.src && fr.id) p!.turn.id = newId(fr.id)
+      if (fr.id) answered.add(fr.id)
+      return p!.turn.id
+    },
+    /** The call recorded with this id, while it still waits for its result. */
+    waiting: (id: string | undefined) => (id ? pending.find((p) => p.src === id)?.turn : undefined),
+    /** Marks a call answered by a result found elsewhere (the tool call record itself). */
+    settle(turn: CallTurn, id: string | undefined) {
+      const i = pending.findIndex((p) => p.turn === turn)
+      if (i >= 0) pending.splice(i, 1)
+      if (id) answered.add(id)
+    },
+    /** Whether a response with this id was paired already and no call with it waits: a second copy of it. */
+    repeated: (id: string | undefined) => Boolean(id && answered.has(id) && !pending.some((p) => p.src === id)),
+    /** Calls still waiting, in order. */
+    open: () => pending.map((p) => p.turn),
+  }
+}
+
+export interface Ctx {
+  h: SessionHeader
+  lost: Lost
+  newId: ReturnType<typeof idMaker>
+  pair: ReturnType<typeof pairer>
+  /** File edits waiting for their call's result, which a later user message carries, by call id. */
+  edits: Map<string, Turn>
+}
+
+export function newCtx(h: SessionHeader, lost: Lost): Ctx {
+  const newId = idMaker()
+  return { h, lost, newId, pair: pairer(newId), edits: new Map() }
+}
+
+/**
+ * The turns in one message's content. Text becomes a user or assistant turn; thought parts become
+ * reasoning; functionCall and functionResponse parts become tool calls and results paired by `ctx.pair`.
+ */
+export function partsToTurns(content: PartList | undefined, role: "user" | "model", at: string | undefined, ctx: Ctx, model?: string, usage?: Usage): Turn[] {
+  const out: Turn[] = []
+  let text = ""
+  let attachments: Attachment[] = []
+  const flush = () => {
+    if (role === "user" && (text.trim() || attachments.length)) out.push({ kind: "user", ...(at ? { at } : {}), text, ...(attachments.length ? { attachments } : {}) })
+    if (role === "model" && text.trim()) out.push({ kind: "assistant", ...(at ? { at } : {}), text, ...(model ? { model } : {}) })
+    text = ""
+    attachments = []
+  }
+  for (const p of toParts(content)) {
+    if (p.thoughtSignature) ctx.lost.add("thought signatures")
+    if (typeof p.text === "string") {
+      if (p.thought) {
+        flush()
+        if (p.text.trim()) out.push({ kind: "reasoning", ...(at ? { at } : {}), text: p.text })
+      } else text += p.text
+    } else if (p.functionCall) {
+      flush()
+      out.push(ctx.pair.call(p.functionCall, at))
+    } else if (p.functionResponse) {
+      flush()
+      // Newer versions can record a response twice; Gemini CLI keeps the first one per round, and so do we.
+      if (ctx.pair.repeated(p.functionResponse.id)) continue
+      const callId = ctx.pair.respond(p.functionResponse)
+      if (!callId) {
+        ctx.lost.add("tool results without a call")
+        continue
+      }
+      const r = responseText(p.functionResponse.response)
+      const c = clip(r.text)
+      out.push({ kind: "tool_result", ...(at ? { at } : {}), callId, ok: r.ok, output: c.text, ...(c.truncated ? { truncated: true } : {}) })
+      const edit = ctx.edits.get(callId)
+      if (edit) {
+        out.push(edit)
+        ctx.edits.delete(callId)
+      }
+    } else if (p.fileData?.fileUri) {
+      attachments.push({ kind: "file", path: p.fileData.fileUri, ...(p.fileData.mimeType ? { mime: p.fileData.mimeType } : {}) })
+    } else if (p.inlineData) {
+      ctx.lost.add("inline images and files")
+    } else if (!p.thoughtSignature) {
+      ctx.lost.add(`unknown part types: ${Object.keys(p)[0] ?? "empty"}`)
+    }
+  }
+  flush()
+  if (usage) {
+    const last = [...out].reverse().find((t) => t.kind === "assistant") as Extract<Turn, { kind: "assistant" }> | undefined
+    if (last) last.usage = usage
+  }
+  return out
+}
+
+/** A file edit's unified diff, which edits keep in their display (FileDiff). */
+function editOf(tc: ToolCallRecord, at: string | undefined, callId: string): Turn | undefined {
+  const d = tc.resultDisplay
+  if (!isObj(d) || typeof d.fileDiff !== "string") return undefined
+  const path = typeof d.filePath === "string" ? d.filePath : typeof d.fileName === "string" ? d.fileName : undefined
+  return path ? { kind: "file_edit", ...(at ? { at } : {}), path, patch: d.fileDiff, callId } : undefined
+}
+
+/**
+ * A tool call recorded in a gemini message, with its result (stored inside the call) and the edit it made.
+ * Newer versions also keep the call in the message's content, where it has become a tool_call already, and
+ * record its response in the user message after it (`answeredLater`); then this adds neither again, the
+ * way Gemini CLI's own resume (convertSessionToClientHistory) does.
+ */
+function toolCallTurns(tc: ToolCallRecord, fallbackAt: string | undefined, ctx: Ctx, answeredLater: boolean): Turn[] {
+  const at = iso(tc.timestamp) ?? fallbackAt
+  const name = typeof tc.name === "string" ? tc.name : "unknown"
+  const out: Turn[] = []
+  let call = ctx.pair.waiting(tc.id)
+  if (!call) {
+    call = ctx.pair.call({ ...(tc.id ? { id: tc.id } : {}), name, args: tc.args ?? {} }, at)
+    out.push(call)
+  }
+  const edit = editOf(tc, at, call.id)
+  if (answeredLater) {
+    if (edit) ctx.edits.set(call.id, edit)
+    return out
+  }
+  ctx.pair.settle(call, tc.id)
+  const hasResult = tc.result !== undefined && tc.result !== null
+  const r = resultText(tc.result)
+  if (r.binary) ctx.lost.add("binary tool output")
+  if (!hasResult) ctx.lost.add("tool results not recorded")
+  const display = typeof tc.resultDisplay === "string" ? tc.resultDisplay : ""
+  const text = hasResult ? r.text : display || `(no result recorded; the call's status was ${tc.status ?? "unknown"})`
+  const c = clip(text)
+  // A {$patch} adds the result without touching the status, which can still say "executing".
+  const ok = hasResult && r.ok && tc.status !== "error" && tc.status !== "cancelled"
+  out.push({ kind: "tool_result", ...(at ? { at } : {}), callId: call.id, ok, output: c.text, ...(c.truncated ? { truncated: true } : {}) })
+  if (edit) out.push(edit)
+  return out
+}
+
+/** A thought summary as one readable text. */
+const thoughtText = (t: { subject?: string; description?: string }) => [t.subject, t.description].filter((x) => typeof x === "string" && x.trim()).join("\n")
+
+/**
+ * User messages Gemini CLI itself leaves out when it resumes a session (isIgnoredUserContent): commands,
+ * and the session and hook context it injects again on its own.
+ */
+export function ignoredUserContent(content: PartList | undefined): boolean {
+  const parts = toParts(content)
+  if (parts.some((p) => p.functionResponse)) return false
+  const text = parts.map((p) => (typeof p.text === "string" && !p.thought ? p.text : "")).join("").trim()
+  return /^(\/|\?|<session_context>|<hook_context>)/.test(text)
+}
+
+/** The ids of the tool responses in a user message; "" for a response without one. */
+const responseIds = (m: MessageRecord) => (m.type === "user" ? toParts(m.content).flatMap((p) => (p.functionResponse ? [p.functionResponse.id ?? ""] : [])) : [])
+
+/** The portable turns for a recorded conversation's messages. */
+export function messagesToTurns(messages: MessageRecord[], ctx: Ctx): Turn[] {
+  const turns: Turn[] = []
+  let afterSnapshot = false
+  messages.forEach((m, i) => {
+    const at = iso(m.timestamp)
+    const snapshot = afterSnapshot
+    afterSnapshot = false
+    switch (m.type) {
+      case "user": {
+        if (ignoredUserContent(m.content)) {
+          ctx.lost.add("Gemini CLI commands and injected context")
+          break
+        }
+        let parts = toParts(m.content)
+        const note = parts[0]?.text?.match(CONTINUED_NOTE)
+        if (note) {
+          ctx.h.parent = { id: note[1]!, kind: "continued" }
+          parts = parts.slice(1)
+        }
+        const summary = !parts.some((p) => p.functionResponse) ? snapshotText(parts.map((p) => (typeof p.text === "string" && !p.thought ? p.text : "")).join("")) : undefined
+        if (summary !== undefined) {
+          turns.push({ kind: "compaction", ...(at ? { at } : {}), summary })
+          afterSnapshot = true
+        } else turns.push(...partsToTurns(parts, "user", at, ctx))
+        break
+      }
+      case "gemini": {
+        // The acknowledgement Gemini CLI's compression adds after its summary is part of the compaction.
+        if (snapshot && !m.toolCalls?.length && toParts(m.content).map((p) => p.text ?? "").join("").trim() === SNAPSHOT_ACK) break
+        // Newer versions keep the model's own parts in content (thoughts, calls); thoughts[] repeats them.
+        if (!toParts(m.content).some((p) => p.thought)) {
+          for (const t of m.thoughts ?? []) {
+            const text = thoughtText(t)
+            if (text) turns.push({ kind: "reasoning", ...((iso(t.timestamp) ?? at) ? { at: iso(t.timestamp) ?? at } : {}), text })
+          }
+        }
+        noteModel(ctx.h, m.model)
+        turns.push(...partsToTurns(m.content, "model", at, ctx, m.model, usageOf(m.tokens)))
+        // Responses the user messages up to the next gemini message carry, as Gemini CLI's resume finds them.
+        const later = new Set<string>()
+        for (let j = i + 1; j < messages.length && messages[j]!.type !== "gemini"; j++) for (const id of responseIds(messages[j]!)) later.add(id)
+        for (const tc of m.toolCalls ?? []) if (isObj(tc)) turns.push(...toolCallTurns(tc, at, ctx, tc.id ? later.has(tc.id) : later.has("")))
+        break
+      }
+      // UI notices (cancelled requests, quota messages, API errors): what the person saw, not the conversation.
+      case "info":
+      case "error":
+      case "warning":
+        ctx.lost.add(`${m.type} notices`)
+        break
+      default:
+        ctx.lost.add(`unknown record types: ${String(m.type)}`)
+    }
+  })
+  return turns
+}
+
+/**
+ * Results for calls that never got one (a cancelled call, or a checkpoint taken just before its tool ran),
+ * placed right after their calls, so every call stays paired. The output says that nothing was recorded.
+ */
+export function closeOpenCalls(turns: Turn[], open: CallTurn[], ctx: Ctx, why: (call: CallTurn) => string): Turn[] {
+  if (!open.length) return turns
+  ctx.lost.add("tool results not recorded", open.length)
+  const waiting = new Set(open)
+  const out: Turn[] = []
+  for (const t of turns) {
+    out.push(t)
+    if (t.kind !== "tool_call" || !waiting.has(t)) continue
+    out.push({ kind: "tool_result", ...(t.at ? { at: t.at } : {}), callId: t.id, ok: false, output: `(no result recorded: ${why(t)})` })
+    const edit = ctx.edits.get(t.id)
+    if (edit) out.push(edit)
+    ctx.edits.delete(t.id)
+  }
+  return out
+}