@100xprompt/brain 0.0.0-stage → 0.1.1

package/src/adapters/letta/reader.ts

@@ -0,0 +1,177 @@
+import { join } from "node:path"
+import { exists, gitInfo, readJson, readJsonl } from "../../fsutil.ts"
+import type { Session, Turn, Usage } from "../../session/types.ts"
+import { clip, header, iso, lossCounter, noteModel } from "../../session/util.ts"
+import type { Reader, SessionRef } from "../types.ts"
+import { CONTINUED_NOTE, LABEL, MANIFEST, REMINDER, TOOL, conversationDirs, lettaToolKind } from "./records.ts"
+
+// Letta Code's local-backend conversations (records.ts has the layout). Every conversation that has a
+// transcript is listed, the agents' default conversations included; hidden ones (subagents, workflow
+// workers) are not. A message Letta rewrote is read as its last snapshot, where it first appeared. A
+// compaction turn goes where the model's context starts over (before the first message it kept), with the
+// turns it replaced kept in front of it. Server-side ("cloud") Letta conversations are not on disk.
+
+type Json = Record<string, any>
+type Lost = ReturnType<typeof lossCounter>
+
+/** A user message's words: its text parts without the system reminders Letta (or brain) put there. */
+function userText(content: unknown, lost: Lost): { text: string; reminders: string[] } {
+  const parts: string[] = []
+  const reminders: string[] = []
+  for (const c of typeof content === "string" ? [{ type: "text", text: content }] : Array.isArray(content) ? content : []) {
+    if (c?.type === "text" && typeof c.text === "string") {
+      reminders.push(...(c.text.match(REMINDER) ?? []))
+      const t = c.text.replace(REMINDER, "").trim()
+      if (t) parts.push(t)
+    } else if (c?.type === "image") lost.add("images")
+  }
+  return { text: parts.join("\n\n"), reminders }
+}
+
+const usageOf = (u: Json | undefined): Usage | undefined => {
+  if (!u || typeof u !== "object") return undefined
+  const out: Usage = {}
+  if (u.input) out.input = u.input
+  if (u.output) out.output = u.output
+  if (u.cacheRead) out.cacheRead = u.cacheRead
+  if (u.cacheWrite) out.cacheWrite = u.cacheWrite
+  return Object.keys(out).length ? out : undefined
+}
+
+/** A packed summary's words: Letta stores the summary itself on the compaction entry and in metadata. */
+const summaryOf = (entry: Json) =>
+  typeof entry.summary === "string" ? entry.summary : typeof entry.message?.metadata?.compaction?.summary === "string" ? entry.message.metadata.compaction.summary : ""
+
+export const lettaReader: Reader = {
+  tool: TOOL,
+  label: LABEL,
+  async list(home) {
+    const refs: SessionRef[] = []
+    for (const { dir, key, record } of conversationDirs(home)) {
+      if (!record?.id || record.hidden === true || record.agent_free === true || record.is_subagent === true) continue
+      const transcript = join(dir, "messages.jsonl")
+      if (!exists(transcript)) continue
+      const head = readJsonl(transcript).rows[0]
+      const cwd = head?.type === "session" && typeof head.cwd === "string" ? head.cwd : undefined
+      const updated = iso(record.last_message_at) ?? iso(record.updated_at)
+      refs.push({
+        tool: TOOL,
+        sourceId: key,
+        path: dir,
+        title: typeof record.summary === "string" && record.summary ? record.summary : record.id === "default" ? `Default conversation of ${record.agent_id}` : String(record.id),
+        ...(cwd ? { cwd } : {}),
+        ...(updated ? { updatedAt: updated } : {}),
+        extra: { agentId: record.agent_id, conversationId: record.id },
+      })
+    }
+    return refs.sort((a, b) => (b.updatedAt ?? "").localeCompare(a.updatedAt ?? ""))
+  },
+  async read(ref, home) {
+    const dir = ref.path
+    const record = readJson<Json>(join(dir, "conversation.json"))
+    if (!record) throw new Error(`${dir} has no conversation.json`)
+    const manifest = readJson<Json>(join(dir, "manifest.json"))
+    if (manifest?.message_format !== MANIFEST.message_format)
+      throw new Error(`${dir} is in Letta's transcript format ${String(manifest?.message_format ?? "without a manifest")}; run \`letta local-backend migrate-transcripts\` first`)
+    const { rows, bad } = readJsonl(join(dir, "messages.jsonl"))
+    return parseTranscript(rows, { key: ref.sourceId, dir, record, home, bad })
+  },
+}
+
+/** A conversation's transcript rows as a portable session. */
+export function parseTranscript(rows: Json[], ctx: { key: string; dir: string; record: Json; home: string; bad?: number }): Session {
+  const head = rows[0]?.type === "session" ? rows[0] : undefined
+  const cwd = typeof head?.cwd === "string" && head.cwd ? head.cwd : undefined
+  const h = header(TOOL, ctx.key, {
+    sourcePath: join(ctx.dir, "messages.jsonl"),
+    ...(typeof ctx.record.summary === "string" && ctx.record.summary ? { title: ctx.record.summary } : {}),
+    project: cwd ? { cwd, ...gitInfo(cwd, ctx.home) } : {},
+    ...(iso(ctx.record.created_at) ?? iso(head?.timestamp) ? { startedAt: iso(ctx.record.created_at) ?? iso(head?.timestamp) } : {}),
+    ...(iso(ctx.record.last_message_at) ?? iso(ctx.record.updated_at) ? { updatedAt: iso(ctx.record.last_message_at) ?? iso(ctx.record.updated_at) } : {}),
+  })
+  const lost = lossCounter()
+  if (ctx.bad) lost.add("unreadable transcript lines", ctx.bad)
+
+  // The last snapshot of each message, kept where it first appeared; compactions placed before what they kept.
+  const latest = new Map<string, Json>()
+  const order: Json[] = []
+  const entryIndex = new Map<string, number>()
+  for (const r of rows) {
+    if ((r.type !== "message" && r.type !== "compaction") || typeof r.id !== "string" || !r.message || typeof r.message.id !== "string") continue
+    if (r.type === "message") {
+      if (!latest.has(r.message.id)) {
+        entryIndex.set(r.id, order.length)
+        order.push(r)
+      } else entryIndex.set(r.id, order.findIndex((o) => o.type === "message" && o.message.id === r.message.id))
+      latest.set(r.message.id, r.message)
+    } else order.push(r)
+  }
+  const before = new Map<number, Json[]>()
+  const placed = new Set<Json>()
+  for (const r of order) {
+    if (r.type !== "compaction" || typeof r.firstKeptEntryId !== "string") continue
+    const k = entryIndex.get(r.firstKeptEntryId)
+    if (k === undefined || order[k] === r) continue
+    before.set(k, [...(before.get(k) ?? []), r])
+    placed.add(r)
+  }
+
+  const turns: Turn[] = []
+  const compaction = (r: Json) => {
+    const summary = summaryOf(r)
+    if (summary.trim()) turns.push({ kind: "compaction", ...(iso(r.timestamp) ? { at: iso(r.timestamp)! } : {}), summary })
+  }
+  const summaries = new Set(order.filter((r) => r.type === "compaction").map((r) => r.message.id as string))
+  for (const [i, r] of order.entries()) {
+    for (const c of before.get(i) ?? []) compaction(c)
+    if (r.type === "compaction") {
+      if (!placed.has(r)) compaction(r)
+      continue
+    }
+    const m = latest.get(r.message.id)!
+    if (summaries.has(m.id)) continue // the compaction entry carries it
+    const at = iso(m.metadata?.created_at) ?? iso(r.timestamp)
+    const when = at ? { at } : {}
+    if (m.role === "user") {
+      if (m.metadata?.compaction) {
+        compaction({ summary: m.metadata.compaction.summary, timestamp: at })
+        continue
+      }
+      const { text, reminders } = userText(m.content, lost)
+      for (const rem of reminders) {
+        const continued = rem.match(CONTINUED_NOTE)
+        if (continued?.[1] && !h.parent) h.parent = { id: continued[1], kind: "continued" }
+      }
+      if (text) turns.push({ kind: "user", ...when, text })
+    } else if (m.role === "assistant") {
+      const model = typeof m.model === "string" && m.model ? m.model : undefined
+      noteModel(h, model)
+      const usage = usageOf(m.usage)
+      let usageGiven = false
+      for (const c of Array.isArray(m.content) ? m.content : []) {
+        if (c?.type === "text" && typeof c.text === "string") {
+          if (!c.text.trim()) continue
+          turns.push({ kind: "assistant", ...when, text: c.text, ...(model ? { model } : {}), ...(usage && !usageGiven ? { usage } : {}) })
+          usageGiven = true
+        } else if (c?.type === "thinking") {
+          if (c.redacted) lost.add("redacted reasoning")
+          else if (typeof c.thinking === "string" && c.thinking.trim()) turns.push({ kind: "reasoning", ...when, text: c.thinking })
+        } else if (c?.type === "toolCall" && typeof c.id === "string") {
+          const name = String(c.name ?? "tool")
+          turns.push({ kind: "tool_call", ...when, id: c.id, name, toolKind: lettaToolKind(name), input: c.arguments ?? {} })
+        } else lost.add("unknown assistant content")
+      }
+      if (m.stopReason === "error" || m.stopReason === "aborted") lost.add("failed or aborted model responses")
+    } else if (m.role === "toolResult") {
+      const parts: string[] = []
+      for (const c of Array.isArray(m.content) ? m.content : []) {
+        if (c?.type === "text" && typeof c.text === "string") parts.push(c.text)
+        else if (c?.type === "image") lost.add("images in tool results")
+      }
+      const out = clip(parts.join("\n"))
+      turns.push({ kind: "tool_result", ...when, callId: String(m.toolCallId), ok: m.isError !== true, output: out.text, ...(out.truncated ? { truncated: true } : {}) })
+    } else lost.add("unknown message roles")
+  }
+  lost.flush(h)
+  return { header: h, turns }
+}

package/src/adapters/letta/records.ts

@@ -0,0 +1,119 @@
+import { readdirSync, statSync } from "node:fs"
+import { join } from "node:path"
+import { readJson } from "../../fsutil.ts"
+import type { ToolKind } from "../../session/types.ts"
+import { toolKindOf } from "../../session/util.ts"
+
+// What Letta Code's local backend stores, as far as this adapter relies on it. Checked against letta-code's
+// source at 0.34.44 (letta-ai/letta-code 9176d8c, 2026-11-06): src/backend/local/local-store.ts (LocalStore:
+// conversation records, ids, fork, compaction), local-transcript.ts (the session-entry JSONL and its manifest),
+// local-message.ts (pi-ai messages with an id and metadata), local-conversation-record.ts,
+// local-conversation-list.ts and src/utils/local-backend-paths.ts. Catalogue entry 16.
+//
+// Letta keeps its default ("cloud") agents and conversations on a Letta server, never on disk. Only the
+// local backend (`letta --backend local`, agent ids `agent-local-*`) is a store a writer can reach:
+//
+//   ~/.letta/lc-local-backend/agents/<base64url(agentId)>.json        the agent (system prompt, model, tags)
+//   ~/.letta/lc-local-backend/memfs/<agentId>/memory/                 its memory, a git repo
+//   ~/.letta/lc-local-backend/conversations/<base64url(key)>/         one conversation, key below
+//     conversation.json   {id, agent_id, summary (the title /resume shows), in_context_message_ids, times}
+//     manifest.json       {schema_version 2, message_format "pi-session-entry-jsonl", provider_stack "pi-ai"}
+//     messages.jsonl      a {type:"session", version:3} header, then message and compaction entries on one
+//                         parentId chain; each holds a pi-ai message (user, assistant, toolResult) with an id
+//     system-prompt.json  the compiled system prompt, written by Letta on the first turn
+//
+// The key is "default:<agentId>" for an agent's default conversation and "conversation:<id>" for the others;
+// new conversation ids are local-conv-<n> and message ids ui-msg-<n>. Only the messages whose ids are in
+// in_context_message_ids go to the model; a compaction entry's message is the summary that replaces the rest.
+
+export const TOOL = "letta-code"
+export const LABEL = "Letta Code"
+
+/** Letta Code releases whose local store was checked in source; the writer was run against 0.34.44. */
+export const TESTED_VERSIONS = ["0.34.44"]
+
+export const MANIFEST = { schema_version: 2, message_format: "pi-session-entry-jsonl", provider_stack: "pi-ai" } as const
+export const SESSION_HEADER_VERSION = 3
+
+export const storageDir = (home: string) => join(home, ".letta", "lc-local-backend")
+export const conversationsDir = (home: string) => join(storageDir(home), "conversations")
+export const agentsDir = (home: string) => join(storageDir(home), "agents")
+
+/** LocalStore's encodePathSegment. */
+export const encodeSegment = (v: string) => Buffer.from(v).toString("base64url")
+export const decodeSegment = (v: string) => Buffer.from(v, "base64url").toString("utf8")
+
+/** localConversationKey. */
+export const conversationKey = (conversationId: string, agentId: string) =>
+  conversationId === "default" ? `default:${agentId}` : `conversation:${conversationId}`
+
+export const isLocalAgentId = (id: unknown): id is string => typeof id === "string" && id.startsWith("agent-local-")
+
+/** Letta's own tools (src/tools/toolset-catalog.ts), where the name alone does not say what they do. */
+const KINDS: Record<string, ToolKind> = {
+  Bash: "shell",
+  exec_command: "shell",
+  write_stdin: "shell",
+  Monitor: "shell",
+  Read: "read",
+  ViewImage: "read",
+  Edit: "edit",
+  ApplyPatch: "edit",
+  Write: "write",
+  Task: "task",
+  SendAgentMessage: "task",
+  TaskCreate: "todo",
+  TaskGet: "todo",
+  TaskList: "todo",
+  TaskUpdate: "todo",
+  UpdatePlan: "todo",
+}
+export const lettaToolKind = (name: string): ToolKind => KINDS[name] ?? toolKindOf(name)
+
+/** Letta hides these from the person (backfill.ts removeSystemContextBlocks) but sends them to the model. */
+export const REMINDER = /<system-reminder>[\s\S]*?<\/system-reminder>|<system-alert>[\s\S]*?<\/system-alert>/g
+
+/** The note saying where a conversation came from, carried in a system reminder so it is not the person's words. */
+export const continuedNote = (label: string, id: string, when: string) =>
+  `<system-reminder>\nThis conversation continues ${label} session ${id}, carried over by brain session on ${when}. The messages before this one are its history; reasoning and anything listed as lost did not carry over. Check files before relying on details, since they may have changed.\n</system-reminder>`
+export const CONTINUED_NOTE = /This conversation continues .+? session (\S+), carried over by brain session/
+
+/** The local agents on disk, newest first: id, name, and whether Letta hides it (subagents, workers). */
+export function localAgents(home: string): Array<{ id: string; name: string; hidden: boolean; mtimeMs: number }> {
+  const dir = agentsDir(home)
+  let names: string[]
+  try {
+    names = readdirSync(dir).filter((n) => n.endsWith(".json") && !n.startsWith("._"))
+  } catch {
+    return []
+  }
+  const out: Array<{ id: string; name: string; hidden: boolean; mtimeMs: number }> = []
+  for (const n of names) {
+    const rec = readJson<Record<string, unknown>>(join(dir, n))
+    if (!rec || !isLocalAgentId(rec.id)) continue
+    let mtimeMs = 0
+    try {
+      mtimeMs = statSync(join(dir, n)).mtimeMs
+    } catch {}
+    out.push({ id: rec.id, name: typeof rec.name === "string" ? rec.name : rec.id, hidden: rec.hidden === true, mtimeMs })
+  }
+  return out.sort((a, b) => b.mtimeMs - a.mtimeMs)
+}
+
+/** Every conversation folder: its decoded key and its record, when the record parses. */
+export function conversationDirs(home: string): Array<{ dir: string; key: string; record?: Record<string, any> }> {
+  const root = conversationsDir(home)
+  let names: string[]
+  try {
+    names = readdirSync(root)
+  } catch {
+    return []
+  }
+  return names
+    .filter((n) => !n.startsWith("."))
+    .map((n) => ({ dir: join(root, n), key: decodeSegment(n), record: readJson<Record<string, any>>(join(root, n, "conversation.json")) }))
+}
+
+/** The number after `prefix` in `id`, 0 when there is none (LocalStore's numericSuffix). */
+export const numericSuffix = (id: unknown, prefix: string) =>
+  typeof id === "string" && id.startsWith(prefix) ? Number.parseInt(id.slice(prefix.length), 10) || 0 : 0