@100xprompt/brain 0.0.0-stage → 0.1.0

package/src/adapters/letta/writer.ts

@@ -0,0 +1,413 @@
+import { randomUUID } from "node:crypto"
+import { mkdirSync, readFileSync, realpathSync, rmSync, statSync, writeFileSync } from "node:fs"
+import { extname, join } from "node:path"
+import { toolLabel } from "../../brief.ts"
+import { exists, isDir, readJson, versionOf } from "../../fsutil.ts"
+import type { Session, Turn } from "../../session/types.ts"
+import { countTurns, firstUserText, unpaired } from "../../session/util.ts"
+import type { Readiness, WriteOptions, WriteResult, Writer } from "../types.ts"
+import { lettaReader } from "./reader.ts"
+import {
+  LABEL,
+  MANIFEST,
+  SESSION_HEADER_VERSION,
+  TESTED_VERSIONS,
+  TOOL,
+  continuedNote,
+  conversationDirs,
+  conversationKey,
+  conversationsDir,
+  encodeSegment,
+  isLocalAgentId,
+  localAgents,
+  numericSuffix,
+  storageDir,
+} from "./records.ts"
+
+// Continues a portable session as a new conversation of one of the person's local Letta agents, written
+// the way LocalStore writes a forked conversation: a folder under lc-local-backend/conversations with
+// conversation.json (the title as its summary, every message in context), manifest.json and messages.jsonl
+// (the session header, then one entry per message on a parentId chain). `letta --agent <id> --conversation
+// <id>` opens it with the earlier turns on screen, and /resume lists it under that agent. Letta's server
+// ("cloud") agents keep their conversations on the server, which only takes new messages by running the
+// agent, so they are not a target here.
+//
+// Messages are pi-ai messages, as Letta stores them: one assistant message per model response (its readable
+// reasoning as thinking without a signature, its text, its toolCall blocks), then one toolResult message per
+// call, right after it, since Letta drops a result that does not follow its call. A call without a result is
+// closed as an error. Assistant messages name the source tool as their provider and "brain-session" as their
+// api, so pi-ai treats them as another model's turns (their reasoning goes to the model as text, their call
+// ids are normalised). A compaction is Letta's compaction entry: the summary becomes a user message packed
+// as a system alert, and only it and what follows stay in context. The note saying where the conversation
+// came from is a <system-reminder> in the first user message in context, which Letta keeps off screen.
+//
+// The agent is the one Letta last used for the local store (the project's settings, then the global ones),
+// else the newest visible local agent. The agent's memory stays its own; only the conversation is new.
+
+const API = "brain-session"
+const IMAGE_TYPES: Record<string, string> = { ".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg", ".gif": "image/gif", ".webp": "image/webp" }
+const IMAGE_LIMIT = 5 * 1024 * 1024
+const ZERO_USAGE = { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 0, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } }
+
+type Json = Record<string, any>
+type Call = Extract<Turn, { kind: "tool_call" }>
+type Result = Extract<Turn, { kind: "tool_result" }>
+
+const shellQuote = (s: string) => (/^[\w./@%+=:,-]+$/.test(s) ? s : `'${s.replace(/'/g, `'\\''`)}'`)
+
+function realCwd(cwd: string): string {
+  try {
+    return realpathSync(cwd)
+  } catch {
+    return cwd
+  }
+}
+
+/** Tool names and call ids the providers Letta talks to accept (Anthropic's ^[a-zA-Z0-9_-]{1,64}$). */
+const toolName = (name: string) => name.replace(/[^a-zA-Z0-9_-]/g, "_").slice(0, 64) || "tool"
+const safeId = (id: string) => id.replace(/[^a-zA-Z0-9_-]/g, "_").slice(0, 64) || "call"
+
+/** Tool input as an object, as pi-ai's toolCall.arguments must be; a raw string or array is wrapped. */
+const objectInput = (input: unknown): Json => (input && typeof input === "object" && !Array.isArray(input) ? (input as Json) : { input: input ?? null })
+
+/** Letta's packageLocalSummaryMessage for an "all" compaction. */
+const packedSummary = (summary: string, time: string) =>
+  JSON.stringify({
+    type: "system_alert",
+    message: `Note: prior messages have been hidden from view due to conversation memory constraints.\nThe following is a summary of the previous messages:\n ${summary}`,
+    time,
+  })
+
+/** The local agent to file the conversation under: the last one Letta used here, else the newest visible one. */
+export function pickAgent(home: string, cwd: string): string | undefined {
+  const agents = localAgents(home)
+  const known = new Set(agents.map((a) => a.id))
+  const last = (settings: Json | undefined): string | undefined => {
+    if (!settings) return undefined
+    for (const [server, s] of Object.entries((settings.sessionsByServer ?? {}) as Record<string, Json>))
+      if (server.startsWith("local:") && known.has(s?.agentId)) return s.agentId
+    for (const id of [settings.lastSession?.agentId, settings.lastAgent]) if (isLocalAgentId(id) && known.has(id)) return id
+    return undefined
+  }
+  return (
+    last(readJson<Json>(join(cwd, ".letta", "settings.local.json"))) ??
+    last(readJson<Json>(join(home, ".letta", "settings.json"))) ??
+    agents.find((a) => !a.hidden)?.id
+  )
+}
+
+export interface ConversationPlan {
+  /** conversation.json */
+  record: Json
+  /** messages.jsonl rows: the session header, then message and compaction entries. */
+  rows: Json[]
+  title: string
+  lost: string[]
+  /** What a read-back must find. */
+  expect: { users: number; assistants: number; calls: number }
+}
+
+/** The files of new conversation `conversationId` of `agentId` continuing `session`. Pure apart from reading attached images. */
+export function planConversation(
+  session: Session,
+  opts: { agentId: string; conversationId: string; firstMessageSeq: number; now: Date; cwd: string },
+): ConversationPlan {
+  const drops = new Map<string, number>()
+  const drop = (what: string, n = 1) => drops.set(what, (drops.get(what) ?? 0) + n)
+  const label = toolLabel(session.header.tool)
+  const expect = { users: 0, assistants: 0, calls: 0 }
+  const nowIso = opts.now.toISOString()
+
+  let seq = opts.firstMessageSeq
+  const messageId = () => `ui-msg-${seq++}`
+  // Timestamps never go backwards, and fall back to just before now for turns that carry none.
+  const known = session.turns.map((t) => (t.at ? Date.parse(t.at) : Number.NaN)).find(Number.isFinite)
+  let clock = (known ?? opts.now.getTime()) - 1
+  const stamp = (at?: string) => {
+    const t = at ? Date.parse(at) : Number.NaN
+    clock = Number.isFinite(t) && t > clock ? t : clock + 1
+    return clock
+  }
+  const meta = (ms: number) => ({ created_at: new Date(ms).toISOString(), updated_at: new Date(ms).toISOString(), agent_id: opts.agentId, conversation_id: opts.conversationId })
+
+  // Calls are answered right after the message holding them; results are looked up by call id.
+  const results = new Map<string, Result>()
+  for (const t of session.turns) if (t.kind === "tool_result" && !results.has(t.callId)) results.set(t.callId, t)
+  const ids = new Map<string, string>()
+  const used = new Set<string>()
+  let remapped = 0
+  const idFor = (id: string) => {
+    let v = ids.get(id)
+    if (v) return v
+    v = safeId(id)
+    for (let n = 2; used.has(v); n++) v = `${safeId(id)}_${n}`
+    ids.set(id, v)
+    used.add(v)
+    if (v !== id) remapped++
+    return v
+  }
+
+  // Items in order: messages, and compaction markers (the summary message they carry).
+  type Item = { kind: "message"; message: Json } | { kind: "compaction"; summary: string; message: Json }
+  const items: Item[] = []
+  let cur: { msg: Json; calls: Call[] } | undefined
+  const flush = () => {
+    if (!cur) return
+    const { msg, calls } = cur
+    cur = undefined
+    if (!msg.content.length) return
+    msg.stopReason = calls.length ? "toolUse" : "stop"
+    items.push({ kind: "message", message: msg })
+    for (const call of calls) {
+      const r = results.get(call.id)
+      const ts = stamp(r?.at)
+      const output = r ? (r.fullPath ? `${r.output}\n\n[Full output: ${r.fullPath}]` : r.output) : "No result was recorded for this call; the session moved on before it finished."
+      if (!r) drop("tool calls without a result (closed as errors)")
+      items.push({
+        kind: "message",
+        message: {
+          id: messageId(),
+          role: "toolResult",
+          toolCallId: idFor(call.id),
+          toolName: toolName(call.name),
+          content: [{ type: "text", text: output.trim() ? output : "(no output)" }],
+          isError: r ? !r.ok : true,
+          timestamp: ts,
+          metadata: meta(ts),
+        },
+      })
+    }
+  }
+  const assistant = (at: string | undefined, model: string | undefined) => {
+    if (!cur) {
+      const ts = stamp(at)
+      cur = {
+        msg: { id: messageId(), role: "assistant", content: [], api: API, provider: session.header.tool, model: model ?? session.header.models.at(-1) ?? "unknown", usage: ZERO_USAGE, stopReason: "stop", timestamp: ts, metadata: meta(ts) },
+        calls: [],
+      }
+    }
+    return cur
+  }
+  const hasCalls = () => !!cur?.calls.length
+  const seen = new Set<string>()
+
+  for (const t of session.turns) {
+    switch (t.kind) {
+      case "user": {
+        flush()
+        const content: Json[] = []
+        let text = t.text
+        for (const a of t.attachments ?? []) {
+          if (!a.path) continue // bytes the reader could not keep; it already counted them
+          if (a.kind === "file") {
+            text += `\n\n[Attached file: ${a.path}]`
+            continue
+          }
+          const mime = a.mime ?? IMAGE_TYPES[extname(a.path).toLowerCase()]
+          let size = Number.POSITIVE_INFINITY
+          try {
+            size = statSync(a.path).size
+          } catch {}
+          if (mime && Object.values(IMAGE_TYPES).includes(mime) && size <= IMAGE_LIMIT) content.push({ type: "image", data: readFileSync(a.path).toString("base64"), mimeType: mime })
+          else drop("images Letta cannot hold")
+        }
+        if (!text.trim() && !content.length) break
+        if (text.trim()) content.unshift({ type: "text", text })
+        const ts = stamp(t.at)
+        items.push({ kind: "message", message: { id: messageId(), role: "user", content, timestamp: ts, metadata: meta(ts) } })
+        if (text.trim()) expect.users++
+        break
+      }
+      case "assistant": {
+        // Text after a call is a new response: it was written after the call's result came back.
+        if (hasCalls()) flush()
+        if (!t.text.trim()) break
+        assistant(t.at, t.model).msg.content.push({ type: "text", text: t.text })
+        expect.assistants++
+        break
+      }
+      case "reasoning":
+        if (hasCalls()) flush()
+        if (t.text.trim()) assistant(t.at, undefined).msg.content.push({ type: "thinking", thinking: t.text })
+        break
+      case "tool_call": {
+        if (seen.has(t.id)) {
+          drop("repeated tool call ids")
+          break
+        }
+        seen.add(t.id)
+        const name = toolName(t.name)
+        if (name !== t.name) drop("tool names changed to fit Letta")
+        const a = assistant(t.at, undefined)
+        a.msg.content.push({ type: "toolCall", id: idFor(t.id), name, arguments: objectInput(t.input) })
+        a.calls.push(t)
+        expect.calls++
+        break
+      }
+      case "tool_result":
+        if (!seen.has(t.callId)) drop("tool results without a call")
+        break
+      case "compaction": {
+        flush()
+        const ts = stamp(t.at)
+        const summary = `[Summary of the earlier conversation in ${label}, which replaced the turns before it]\n\n${t.summary}`
+        items.push({
+          kind: "compaction",
+          summary,
+          message: { id: messageId(), role: "user", content: [{ type: "text", text: packedSummary(summary, new Date(ts).toISOString()) }], timestamp: ts, metadata: { ...meta(ts), compaction: { summary } } },
+        })
+        break
+      }
+      case "file_edit":
+        if (!t.callId || !seen.has(t.callId)) drop("file edits outside a tool call")
+        break
+      case "subagent":
+        drop("subagent sessions (their results stay in the parent's tool results)")
+        break
+    }
+  }
+  flush()
+  for (const t of session.turns) if (t.kind === "tool_result" && results.get(t.callId) !== t) drop("repeated tool results")
+  if (!items.some((i) => i.kind === "message")) throw new Error("nothing Letta can show: the session has no messages or tool calls")
+
+  // Context starts at the last compaction; the note goes in the first user message there (or opens it).
+  const last = items.map((i) => i.kind).lastIndexOf("compaction")
+  const note = { type: "text", text: continuedNote(label, session.header.id, nowIso.slice(0, 10)) }
+  const firstUser = items.findIndex((i, k) => k > last && i.kind === "message" && i.message.role === "user")
+  if (firstUser >= 0) (items[firstUser] as { message: Json }).message.content.push(note)
+  else {
+    const ts = stamp(undefined)
+    items.splice(last + 1, 0, { kind: "message", message: { id: messageId(), role: "user", content: [note], timestamp: ts, metadata: meta(ts) } })
+  }
+
+  // Letta shows each part of a message 1 ms after the one before it and sorts by time, so every message
+  // starts after the last part of the one before it.
+  let next = 0
+  for (const { message: m } of items) {
+    m.timestamp = Math.max(m.timestamp, next)
+    m.metadata.created_at = m.metadata.updated_at = new Date(m.timestamp).toISOString()
+    next = m.timestamp + Math.max(1, m.role === "assistant" ? m.content.length : 1)
+  }
+
+  // The transcript: header, then entries on one parentId chain with 8-hex ids, as LocalStore appends them.
+  const rows: Json[] = [{ type: "session", version: SESSION_HEADER_VERSION, id: opts.conversationId, timestamp: nowIso, cwd: opts.cwd }]
+  const usedEntries = new Set<string>()
+  const entryId = () => {
+    let v = randomUUID().slice(0, 8)
+    while (usedEntries.has(v)) v = randomUUID().slice(0, 8)
+    usedEntries.add(v)
+    return v
+  }
+  let parentId: string | null = null
+  for (const item of items) {
+    const id = entryId()
+    const timestamp = new Date(item.message.timestamp).toISOString()
+    // A compaction keeps nothing from before it, so its firstKeptEntryId is null.
+    rows.push(
+      item.kind === "message"
+        ? { type: "message", id, parentId, timestamp, message: item.message }
+        : { type: "compaction", id, parentId, timestamp, summary: item.summary, firstKeptEntryId: null, tokensBefore: 0, message: item.message },
+    )
+    parentId = id
+  }
+
+  const inContext = items.slice(last >= 0 ? last : 0).map((i) => i.message.id as string)
+  const lastAt = new Date(items.at(-1)!.message.timestamp).toISOString()
+  const firstText = firstUserText(session)?.replace(/\s+/g, " ").trim().slice(0, 60)
+  const title = `${session.header.title ?? firstText ?? "Session"} (from ${label})`
+  const record: Json = {
+    id: opts.conversationId,
+    agent_id: opts.agentId,
+    archived: false,
+    archived_at: null,
+    created_at: nowIso,
+    updated_at: nowIso,
+    last_message_at: lastAt,
+    summary: title,
+    in_context_message_ids: inContext,
+  }
+  const lost = [...drops].map(([what, n]) => `${what} (${n})`)
+  if (remapped) lost.push(`tool call ids, changed to fit Letta (${remapped})`)
+  return { record, rows, title, lost, expect }
+}
+
+/** The next free local-conv-<n> and ui-msg-<n>, past every conversation already on disk. */
+export function nextIds(home: string): { conversationId: string; firstMessageSeq: number } {
+  let conv = 0
+  let msg = 0
+  for (const { key, record } of conversationDirs(home)) {
+    if (key.startsWith("conversation:")) conv = Math.max(conv, numericSuffix(key.slice("conversation:".length), "local-conv-"))
+    for (const id of Array.isArray(record?.in_context_message_ids) ? record.in_context_message_ids : []) msg = Math.max(msg, numericSuffix(id, "ui-msg-"))
+  }
+  for (let n = conv + 1; ; n++) {
+    const conversationId = `local-conv-${n}`
+    if (!exists(join(conversationsDir(home), encodeSegment(conversationKey(conversationId, ""))))) return { conversationId, firstMessageSeq: msg + 1 }
+  }
+}
+
+const semver = (v: string | undefined) => v?.match(/(\d+\.\d+\.\d+)/)?.[1]
+
+/** The Letta writer; `version` is for tests, which must not depend on what is installed. */
+export function lettaWriter(opts: { version?: string } = {}): Writer {
+  const installed = () => opts.version ?? semver(versionOf("letta"))
+  return {
+    tool: TOOL,
+    label: LABEL,
+    way: "native",
+
+    async ready(w: WriteOptions): Promise<Readiness> {
+      const v = installed()
+      if (!v) return { ok: false, why: "Letta Code is not installed (no `letta` on the PATH)" }
+      if (!TESTED_VERSIONS.includes(v)) return { ok: false, why: `Letta Code ${v} is not a version this writer was checked against (${TESTED_VERSIONS.join(", ")})` }
+      if (!isDir(storageDir(w.home)))
+        return { ok: false, why: "Letta has no local store here: its agents live on the Letta server, which only adds messages by running the agent. Create a local agent with `letta --backend local` first" }
+      if (!pickAgent(w.home, w.cwd)) return { ok: false, why: "Letta's local store has no agent to file the conversation under; create one with `letta --backend local` first" }
+      if (!isDir(w.cwd)) return { ok: false, why: `${w.cwd} is not a folder` }
+      return { ok: true }
+    },
+
+    async write(session: Session, w: WriteOptions): Promise<WriteResult> {
+      const now = new Date()
+      const cwd = realCwd(w.cwd)
+      const agentId = pickAgent(w.home, cwd)
+      if (!agentId) throw new Error("Letta's local store has no agent to file the conversation under; create one with `letta --backend local` first")
+      const { conversationId, firstMessageSeq } = nextIds(w.home)
+      const plan = planConversation(session, { agentId, conversationId, firstMessageSeq, now, cwd })
+      const dir = join(conversationsDir(w.home), encodeSegment(conversationKey(conversationId, agentId)))
+      const files = [join(dir, "conversation.json"), join(dir, "manifest.json"), join(dir, "messages.jsonl")]
+      if (!w.dryRun) {
+        mkdirSync(dir, { recursive: true })
+        writeFileSync(files[1]!, `${JSON.stringify({ ...MANIFEST, created_at: now.toISOString() }, null, 2)}\n`)
+        writeFileSync(files[2]!, `${plan.rows.map((r) => JSON.stringify(r)).join("\n")}\n`)
+        // The record last: Letta lists a conversation by its record, so it never sees a half-written one.
+        writeFileSync(files[0]!, `${JSON.stringify(plan.record, null, 2)}\n`)
+        let problem: string | undefined
+        try {
+          const back = await lettaReader.read({ tool: TOOL, sourceId: conversationKey(conversationId, agentId), path: dir }, w.home)
+          const n = countTurns(back)
+          const pairs = unpaired(back)
+          if (n.user !== plan.expect.users || n.assistant !== plan.expect.assistants || n.tool_call !== plan.expect.calls || pairs.calls.length || pairs.results.length)
+            problem = `users ${n.user}/${plan.expect.users}, replies ${n.assistant}/${plan.expect.assistants}, tool calls ${n.tool_call}/${plan.expect.calls}, unpaired ${pairs.calls.length + pairs.results.length}`
+        } catch (e) {
+          problem = e instanceof Error ? e.message : String(e)
+        }
+        if (problem) {
+          rmSync(dir, { recursive: true, force: true })
+          throw new Error(`the new Letta conversation did not read back as written (${problem}); it was removed`)
+        }
+      }
+      return {
+        tool: TOOL,
+        way: "native",
+        sessionId: conversationId,
+        written: files,
+        open: `cd ${shellQuote(cwd)} && letta --agent ${agentId} --conversation ${conversationId}`,
+        note: `Filed under the local agent ${agentId}; /resume lists it as "${plan.title}".`,
+        lost: plan.lost,
+        dryRun: !!w.dryRun,
+      }
+    },
+  }
+}
+
+export const lettaNativeWriter = lettaWriter()

package/src/adapters/mastracode/index.ts

@@ -0,0 +1,8 @@
+import type { Reader, Writer } from "../types.ts"
+import { mastracodeReader } from "./reader.ts"
+import { mastracodeWriter } from "./writer.ts"
+
+// Mastra Code (the mastracode CLI): reads every thread of its local mastra.db, and continues any session as a
+// new native thread in that store (with MastraCode closed, after a backup). Ported from session-migrate (MIT).
+export const readers: Reader[] = [mastracodeReader]
+export const writers: Writer[] = [mastracodeWriter]