@100xprompt/brain 0.0.0-stage → 0.1.1

package/src/adapters/copilot-cli/reader.ts

@@ -0,0 +1,186 @@
+import { closeSync, openSync, readFileSync, readSync, statSync } from "node:fs"
+import { join } from "node:path"
+import { exists, gitInfo, isDir, ls, mtime } from "../../fsutil.ts"
+import type { Project, Session, Turn } from "../../session/types.ts"
+import { header, iso, lossCounter } from "../../session/util.ts"
+import type { Reader, SessionRef } from "../types.ts"
+import { applyRewinds, type Event, eventsToTurns, isPrompt, LABEL, readJournal, TESTED_EVENT_VERSION, TOOL, workspaceYaml } from "./records.ts"
+
+// The Copilot CLI keeps each session in ~/.copilot/session-state/<id>/: events.jsonl (the journal every
+// resume replays; rotated archives beside it as events.N.jsonl) and workspace.yaml (cwd, git, summary,
+// timestamps). Early versions wrote a flat session-state/<id>.jsonl. The format comes from the Copilot SDK's
+// event schema and from sessions Copilot CLI 1.0.107 wrote itself (records.ts).
+
+const oneLine = (s: string) => s.replace(/\s+/g, " ").trim()
+const titleFrom = (s: unknown) => (typeof s === "string" && oneLine(s) ? oneLine(s).slice(0, 120) : undefined)
+
+function readText(p: string): string | undefined {
+  try {
+    return readFileSync(p, "utf8")
+  } catch {
+    return undefined
+  }
+}
+
+/** The whole JSON lines among bytes [start, start+length) of a journal; a line cut by the window is skipped. */
+function linesAt(path: string, start: number, length: number): Event[] {
+  const fd = openSync(path, "r")
+  try {
+    const buf = Buffer.alloc(length)
+    const n = readSync(fd, buf, 0, length, Math.max(0, start))
+    const out: Event[] = []
+    for (const line of buf.subarray(0, n).toString("utf8").split("\n")) {
+      try {
+        const v = JSON.parse(line)
+        if (v && typeof v === "object") out.push(v)
+      } catch {
+        // cut by the window, or half-written
+      }
+    }
+    return out
+  } finally {
+    closeSync(fd)
+  }
+}
+
+interface Found {
+  journal: string
+  /** The session folder (current layout), where workspace.yaml lives. */
+  dir?: string
+  stem: string
+}
+
+/** Every session under session-state: <id>/events.jsonl, or a flat <id>.jsonl. */
+function sessions(home: string): Found[] {
+  const root = join(home, ".copilot", "session-state")
+  const out: Found[] = []
+  for (const name of ls(root)) {
+    const p = join(root, name)
+    if (isDir(p)) {
+      if (exists(join(p, "events.jsonl"))) out.push({ journal: join(p, "events.jsonl"), dir: p, stem: name })
+    } else if (name.endsWith(".jsonl")) out.push({ journal: p, stem: name.slice(0, -6) })
+  }
+  return out
+}
+
+const WINDOW = 64 * 1024
+
+export const reader: Reader = {
+  tool: TOOL,
+  label: LABEL,
+
+  async list(home) {
+    const refs: SessionRef[] = []
+    for (const f of sessions(home)) {
+      let size = 0
+      try {
+        size = statSync(f.journal).size
+      } catch {
+        continue
+      }
+      // The start of the journal has session.start and the first message; the end has the latest title.
+      const head = linesAt(f.journal, 0, WINDOW)
+      const tail = size > WINDOW ? linesAt(f.journal, size - WINDOW, WINDOW) : head
+      const ws = f.dir ? workspaceYaml(readText(join(f.dir, "workspace.yaml"))) : {}
+      const start = head.find((e) => e.type === "session.start")?.data
+      const firstUser = head.find((e) => e.type === "user.message" && !e.agentId && isPrompt(e.data?.source))?.data
+      // A session that never got a message is not worth continuing; the CLI makes one on every launch.
+      if (!firstUser && size <= WINDOW) continue
+      const title = titleFrom([...tail].reverse().find((e) => e.type === "session.title_changed")?.data?.title) ?? titleFrom(ws.summary) ?? titleFrom(ws.name) ?? titleFrom(firstUser?.content)
+      const cwd = ws.cwd || (typeof start?.context?.cwd === "string" ? start.context.cwd : undefined)
+      const updatedAt = iso(ws.updated_at) ?? mtime(f.journal)
+      const sourceId = typeof start?.sessionId === "string" ? start.sessionId : ws.id || f.stem
+      refs.push({ tool: TOOL, sourceId, path: f.journal, ...(title ? { title } : {}), ...(cwd ? { cwd } : {}), ...(updatedAt ? { updatedAt } : {}), ...(f.dir ? { extra: { dir: f.dir } } : {}) })
+    }
+    return refs.sort((a, b) => (b.updatedAt ?? "").localeCompare(a.updatedAt ?? ""))
+  },
+
+  async read(ref) {
+    if (!exists(ref.path)) throw new Error(`${ref.path} does not exist`)
+    const journal = readJournal(ref.path)
+    const { events, removed } = applyRewinds(journal.events.filter((e) => !e.ephemeral))
+    if (!events.length) throw new Error(`${ref.path} has no Copilot CLI events`)
+    const lost = lossCounter()
+    if (journal.bad) lost.add("unreadable lines", journal.bad)
+    if (journal.missingArchives) lost.add("rotated journal archives missing (older history)", journal.missingArchives)
+    if (removed) lost.add("events removed by a rewind", removed)
+
+    const start = events.find((e) => e.type === "session.start")?.data
+    if (start && start.version !== TESTED_EVENT_VERSION) lost.add(`unverified event schema version ${String(start.version)}`)
+    const sessionId = typeof start?.sessionId === "string" ? start.sessionId : ref.sourceId
+    // The working folder can change (session.resume, session.context_changed); the latest is where work goes on.
+    const contexts = events.flatMap((e) => (e.type === "session.start" || e.type === "session.resume" ? [e.data?.context] : e.type === "session.context_changed" ? [e.data] : [])).filter((c) => c && typeof c.cwd === "string")
+    const context = contexts[contexts.length - 1]
+    const ws = ref.extra?.dir ? workspaceYaml(readText(join(String(ref.extra.dir), "workspace.yaml"))) : {}
+    const cwd: string | undefined = context?.cwd ?? (ws.cwd || ref.cwd)
+    const local = gitInfo(cwd)
+    const project: Project = {
+      ...(cwd ? { cwd } : {}),
+      ...local,
+      ...(typeof context?.branch === "string" ? { gitBranch: context.branch } : {}),
+      ...(typeof context?.headCommit === "string" ? { gitCommit: context.headCommit } : {}),
+    }
+    if (!project.gitRemote && typeof context?.repository === "string") project.gitRemote = `https://${typeof context.repositoryHost === "string" ? context.repositoryHost : "github.com"}/${context.repository}`
+    const titled = events.filter((e) => e.type === "session.title_changed").pop()?.data?.title
+    const firstUser = events.find((e) => e.type === "user.message" && !e.agentId && isPrompt(e.data?.source))?.data?.content
+    const title = titleFrom(titled) ?? titleFrom(ws.summary) ?? titleFrom(ws.name) ?? titleFrom(firstUser)
+    const startedAt = iso(start?.startTime) ?? iso(events[0]?.timestamp)
+    const updatedAt = iso(events[events.length - 1]?.timestamp) ?? mtime(ref.path)
+    const h = header(TOOL, sessionId, {
+      ...(typeof start?.copilotVersion === "string" ? { toolVersion: start.copilotVersion } : {}),
+      sourcePath: ref.path,
+      ...(title ? { title } : {}),
+      project,
+      ...(startedAt ? { startedAt } : {}),
+      ...(updatedAt ? { updatedAt } : {}),
+    })
+
+    // Subagents write into the same journal under their own agentId; each becomes a child session, shown
+    // in the parent after the tool call that started it and that call's result.
+    // Older events name only the spawning call (parentToolCallId, now deprecated), not the agent.
+    const agentOf = (e: Event) => (typeof e.agentId === "string" && e.agentId ? e.agentId : typeof e.data?.parentToolCallId === "string" ? `task-${e.data.parentToolCallId}` : undefined)
+    const byAgent = new Map<string, Event[]>()
+    for (const e of events) {
+      const a = agentOf(e)
+      if (!a) continue
+      if (!byAgent.has(a)) byAgent.set(a, [])
+      byAgent.get(a)!.push(e)
+    }
+    const started = events.filter((e) => e.type === "subagent.started" && typeof e.data?.toolCallId === "string")
+    const turns: Turn[] = eventsToTurns(events.filter((e) => !agentOf(e)), h, lost)
+    const children: Session[] = []
+    for (const [agentId, own] of byAgent) {
+      const callId =
+        started.find((e) => e.agentId === agentId)?.data?.toolCallId ??
+        own.map((e) => e.data?.parentToolCallId).find((v) => typeof v === "string")
+      const info = started.find((e) => e.data?.toolCallId === callId)?.data
+      const name = titleFrom(info?.agentDisplayName)
+      const from = iso(own[0]?.timestamp)
+      const to = iso(own[own.length - 1]?.timestamp)
+      const ch = header(TOOL, `${sessionId}/${agentId}`, {
+        ...(h.toolVersion ? { toolVersion: h.toolVersion } : {}),
+        sourcePath: ref.path,
+        ...(name ? { title: name } : {}),
+        project,
+        ...(from ? { startedAt: from } : {}),
+        ...(to ? { updatedAt: to } : {}),
+        parent: { id: h.id, kind: "subagent" },
+      })
+      const childLost = lossCounter()
+      const child: Session = { header: ch, turns: eventsToTurns(own, ch, childLost) }
+      childLost.flush(ch)
+      children.push(child)
+      const description = titleFrom(info?.agentDescription) ?? name
+      const t: Turn = { kind: "subagent", ...(ch.startedAt ? { at: ch.startedAt } : {}), sessionId: ch.id, ...(typeof callId === "string" ? { callId } : {}), ...(description ? { description } : {}) }
+      let i = -1
+      turns.forEach((x, j) => {
+        if ((x.kind === "tool_call" && x.id === callId) || (x.kind === "tool_result" && x.callId === callId)) i = j
+      })
+      if (i >= 0) turns.splice(i + 1, 0, t)
+      else turns.push(t)
+    }
+    lost.flush(h)
+    return children.length ? { header: h, turns, children } : { header: h, turns }
+  },
+}
+

package/src/adapters/copilot-cli/records.ts

@@ -0,0 +1,241 @@
+import { existsSync } from "node:fs"
+import { dirname, join } from "node:path"
+import { readJsonl } from "../../fsutil.ts"
+import type { Attachment, SessionHeader, Turn } from "../../session/types.ts"
+import { asText, clip, iso, noteModel, toolKindOf, type lossCounter } from "../../session/util.ts"
+
+// Facts about the Copilot CLI's session journal, shared by its reader and writer. They come from the
+// Copilot SDK's generated event schema (github/copilot-sdk, nodejs/src/generated/session-events.ts, and
+// docs/features/session-persistence.md) and from journals Copilot CLI 1.0.107 wrote itself.
+// See docs/sessions/2026-11-08-session-formats.md, entry 34.
+
+export const TOOL = "copilot-cli"
+export const LABEL = "Copilot CLI"
+
+/** session.start's `version`, the event schema version, as the SDK and Copilot CLI 1.0.107 write it. */
+export const TESTED_EVENT_VERSION = 1
+
+type Lost = ReturnType<typeof lossCounter>
+type Json = Record<string, any>
+
+/** One journal line: {id, parentId, timestamp, type, data}, plus agentId for subagent events. */
+export interface Event {
+  id?: string
+  parentId?: string | null
+  timestamp?: string
+  type?: string
+  data?: Json
+  /** The subagent instance that emitted it; absent for the main agent. */
+  agentId?: string
+  /** Streaming deltas; the CLI does not persist them, but a reader skips them if it meets one. */
+  ephemeral?: boolean
+  /** Part of a rotated journal's replay prefix rather than original history. */
+  checkpoint?: boolean
+}
+
+const isObj = (v: unknown): v is Json => typeof v === "object" && v !== null && !Array.isArray(v)
+
+/**
+ * A session's events in order, archives included. With size rotation on, the CLI moves the journal to
+ * events.N.jsonl and starts a new events.jsonl with a session.journal_link to it, then `checkpoint: true`
+ * events that replay the state so far. The full history is the predecessor's events followed by this
+ * file's events after that replay prefix; when the predecessor is gone, the prefix is all that is left.
+ */
+export function readJournal(path: string): { events: Event[]; bad: number; missingArchives: number } {
+  let bad = 0
+  let missingArchives = 0
+  const seen = new Set<string>()
+  const read = (file: string): Event[] => {
+    seen.add(file)
+    const r = readJsonl(file)
+    bad += r.bad
+    const rows = r.rows.filter(isObj) as Event[]
+    const link = rows[0]?.type === "session.journal_link" ? rows[0].data : undefined
+    if (!link) return rows
+    const prev = typeof link.previousFile === "string" && !link.previousFile.includes("/") ? join(dirname(file), link.previousFile) : undefined
+    if (!prev || seen.has(prev) || !existsSync(prev)) {
+      missingArchives++
+      return rows.slice(1)
+    }
+    let i = 1
+    while (rows[i]?.checkpoint === true) i++
+    return [...read(prev), ...rows.slice(i)]
+  }
+  return { events: read(path), bad, missingArchives }
+}
+
+/**
+ * Events with rewinds applied: session.snapshot_rewind removes `eventIds` when it lists them, otherwise
+ * `upToEventId` and every event after it.
+ */
+export function applyRewinds(events: Event[]): { events: Event[]; removed: number } {
+  let out: Event[] = []
+  let removed = 0
+  for (const e of events) {
+    if (e.type !== "session.snapshot_rewind") {
+      out.push(e)
+      continue
+    }
+    const before = out.length
+    const ids = e.data?.eventIds
+    if (Array.isArray(ids) && ids.length) {
+      const drop = new Set(ids)
+      out = out.filter((x) => !drop.has(x.id))
+    } else {
+      const i = out.findIndex((x) => x.id === e.data?.upToEventId)
+      if (i >= 0) out.length = i
+    }
+    removed += before - out.length
+  }
+  return { events: out, removed }
+}
+
+// Event families that carry no conversation: session bookkeeping, streaming, hooks, permissions, MCP,
+// billing and UI. They are dropped without being counted; any other unhandled type is counted as unknown.
+const NOT_CONVERSATION =
+  /^(session\.|hook\.|permission\.|mcp[._]|model\.|skill\.|elicitation\.|sampling\.|command\.|commands\.|capabilities\.|auto_mode|exit_plan_mode\.|user_input\.|workflow\.|tool_search\.|ui\.|pending_messages\.|instruction|system\.|external_tool\.|human_response\.|session_limits|new_inbox|abort$|agent_(idle|completed)$|subagent\.|assistant\.(turn_start|turn_end|turn_retry|idle|intent|message_start|message_delta|reasoning_delta|streaming_delta|tool_call_delta|server_tool_progress|fusion_)|tool\.(execution_progress|execution_partial_result|user_requested|shell_output))/
+
+type CallTurn = Extract<Turn, { kind: "tool_call" }>
+
+/**
+ * Whether a user.message is a prompt rather than injected context. Copilot CLI 1.0.107's timeline shows a
+ * message with no source, "user", or a command-, schedule- or autopilot- source; "agent-" is another
+ * agent's prompt (a subagent's task). Others ("system" context, "skill-" injections) are not anyone's words.
+ */
+export const isPrompt = (source: unknown) => typeof source !== "string" || source === "user" || /^(command|schedule|autopilot|agent)-/.test(source)
+
+function attachmentsOf(list: unknown, lost: Lost): Attachment[] {
+  const out: Attachment[] = []
+  for (const a of Array.isArray(list) ? list : []) {
+    if (!isObj(a)) continue
+    const name = typeof a.displayName === "string" ? { name: a.displayName } : {}
+    if ((a.type === "file" || a.type === "directory") && typeof a.path === "string")
+      // Copilot 1.0.70 records a pasted or @mentioned image as a "file" attachment with an image mimeType (plus assetId, byteLength).
+      out.push({ kind: typeof a.mimeType === "string" && a.mimeType.startsWith("image/") ? "image" : "file", path: a.path, ...name, ...(typeof a.mimeType === "string" ? { mime: a.mimeType } : {}) })
+    else if (a.type === "selection" && typeof a.filePath === "string") out.push({ kind: "file", path: a.filePath, ...name })
+    else if (a.type === "blob") {
+      out.push({ kind: typeof a.mimeType === "string" && a.mimeType.startsWith("image/") ? "image" : "file", ...name, ...(typeof a.mimeType === "string" ? { mime: a.mimeType } : {}) })
+      lost.add("inline attachment bytes")
+    } else if (typeof a.type === "string" && a.type.startsWith("github")) lost.add("GitHub reference attachments")
+  }
+  return out
+}
+
+/** A tool result's text as the model got it, and whether the tool succeeded. */
+function resultOf(d: Json, lost: Lost): { text: string; ok: boolean } {
+  const r = isObj(d.result) ? d.result : {}
+  const text = typeof r.content === "string" ? r.content : typeof r.detailedContent === "string" ? r.detailedContent : asText(r.content ?? "")
+  const binary = (Array.isArray(r.binaryResultsForLlm) ? r.binaryResultsForLlm.length : 0) + (Array.isArray(r.contents) ? r.contents.filter((c: Json) => isObj(c) && c.type !== "text" && c.type !== "terminal").length : 0)
+  if (binary) lost.add("binary tool output", binary)
+  const error = isObj(d.error) && typeof d.error.message === "string" ? d.error.message : undefined
+  return { text: [error, text].filter(Boolean).join("\n"), ok: d.success !== false && !error }
+}
+
+/**
+ * The turns in one agent's events (the main agent's, or one subagent's). Tool calls come from the
+ * assistant message's toolRequests, or from tool.execution_start for calls the model did not make (a
+ * command the person ran); results come from tool.execution_complete, both keyed by toolCallId.
+ */
+export function eventsToTurns(events: Event[], h: SessionHeader, lost: Lost): Turn[] {
+  const turns: Turn[] = []
+  const calls = new Map<string, CallTurn>()
+  const started = new Map<string, Json>()
+  const answered = new Set<string>()
+  let reasoning = new Set<string>()
+  let last: Extract<Turn, { kind: "assistant" }> | undefined
+  const call = (id: unknown, name: unknown, input: unknown, at: string | undefined) => {
+    if (typeof id !== "string" || !id || calls.has(id)) return
+    const n = typeof name === "string" && name ? name : "unknown"
+    const t: CallTurn = { kind: "tool_call", ...(at ? { at } : {}), id, name: n, toolKind: toolKindOf(n), input: input ?? {} }
+    calls.set(id, t)
+    turns.push(t)
+  }
+  for (const e of events) {
+    if (e.ephemeral) continue
+    const d = isObj(e.data) ? e.data : {}
+    const at = iso(e.timestamp)
+    switch (e.type) {
+      case "user.message": {
+        if (!isPrompt(d.source)) break
+        const text = typeof d.content === "string" ? d.content : typeof d.transformedContent === "string" ? d.transformedContent : ""
+        const attachments = attachmentsOf(d.attachments, lost)
+        if (text.trim() || attachments.length) turns.push({ kind: "user", ...(at ? { at } : {}), text, ...(attachments.length ? { attachments } : {}) })
+        break
+      }
+      case "assistant.reasoning":
+        if (typeof d.content === "string" && d.content.trim()) {
+          turns.push({ kind: "reasoning", ...(at ? { at } : {}), text: d.content })
+          reasoning.add(d.content)
+        }
+        break
+      case "assistant.message": {
+        const model = typeof d.model === "string" ? d.model : undefined
+        noteModel(h, model)
+        // reasoningOpaque (Anthropic) and encryptedContent (OpenAI) are bound to the provider that made them.
+        if (d.reasoningOpaque || d.encryptedContent) lost.add("signed or encrypted reasoning")
+        // reasoningText repeats the assistant.reasoning event of the same call, when there was one.
+        if (typeof d.reasoningText === "string" && d.reasoningText.trim() && !reasoning.has(d.reasoningText)) turns.push({ kind: "reasoning", ...(at ? { at } : {}), text: d.reasoningText })
+        reasoning = new Set()
+        if (typeof d.content === "string" && d.content.trim()) {
+          last = { kind: "assistant", ...(at ? { at } : {}), text: d.content, ...(model ? { model } : {}) }
+          turns.push(last)
+        }
+        for (const r of Array.isArray(d.toolRequests) ? d.toolRequests : []) if (isObj(r)) call(r.toolCallId, r.name, r.arguments, at)
+        break
+      }
+      case "tool.execution_start":
+        if (typeof d.toolCallId === "string") started.set(d.toolCallId, d)
+        call(d.toolCallId, d.toolName, d.arguments, at)
+        break
+      case "tool.execution_complete": {
+        if (typeof d.toolCallId !== "string" || answered.has(d.toolCallId)) break
+        const s = started.get(d.toolCallId)
+        call(d.toolCallId, s?.toolName, s?.arguments, at)
+        const r = resultOf(d, lost)
+        const c = clip(r.text)
+        turns.push({ kind: "tool_result", ...(at ? { at } : {}), callId: d.toolCallId, ok: r.ok, output: c.text, ...(c.truncated ? { truncated: true } : {}) })
+        answered.add(d.toolCallId)
+        break
+      }
+      case "assistant.usage":
+        if (last && !last.usage) {
+          const u = { input: d.inputTokens, output: d.outputTokens, cacheRead: d.cacheReadTokens, cacheWrite: d.cacheWriteTokens }
+          const usage = Object.fromEntries(Object.entries(u).filter(([, v]) => typeof v === "number"))
+          if (Object.keys(usage).length) last.usage = usage
+        }
+        noteModel(h, typeof d.model === "string" ? d.model : undefined)
+        break
+      case "session.compaction_complete":
+        if (d.success !== false && typeof d.summaryContent === "string" && d.summaryContent.trim()) turns.push({ kind: "compaction", ...(at ? { at } : {}), summary: d.summaryContent })
+        break
+      default:
+        if (typeof e.type !== "string" || !NOT_CONVERSATION.test(e.type)) lost.add(`unknown record types: ${String(e.type)}`)
+    }
+  }
+  // Calls the journal never answered (interrupted, or still running when it was copied) get a result
+  // saying so, right after the call, so every call stays paired.
+  const open = [...calls.values()].filter((c) => !answered.has(c.id))
+  if (!open.length) return turns
+  lost.add("tool results not recorded", open.length)
+  const waiting = new Set(open)
+  return turns.flatMap((t) => (t.kind === "tool_call" && waiting.has(t) ? [t, { kind: "tool_result" as const, ...(t.at ? { at: t.at } : {}), callId: t.id, ok: false, output: "(no result recorded)" }] : [t]))
+}
+
+/** Top-level `key: value` lines of workspace.yaml (id, cwd, branch, summary, created_at, updated_at…). */
+export function workspaceYaml(text: string | undefined): Record<string, string> {
+  const out: Record<string, string> = {}
+  for (const line of (text ?? "").split("\n")) {
+    const m = line.match(/^([A-Za-z_][\w-]*):\s*(.*?)\s*$/)
+    if (!m || !m[2] || m[2] === "|" || m[2] === ">") continue
+    let v = m[2]
+    if (v.startsWith('"') && v.endsWith('"')) {
+      try {
+        v = JSON.parse(v)
+      } catch {
+        v = v.slice(1, -1)
+      }
+    } else if (v.startsWith("'") && v.endsWith("'")) v = v.slice(1, -1).replace(/''/g, "'")
+    out[m[1]!] = v
+  }
+  return out
+}