lyra-core 0.1.0

package/src/memory/index-sync.ts

@@ -0,0 +1,99 @@
+import { readFile, stat } from 'node:fs/promises'
+import { join } from 'node:path'
+import matter from 'gray-matter'
+import { addEntryLine, readIndexFile, writeIndexFile } from './index-file'
+
+/**
+ * v0.23.0: MEMORY.md historically only listed user-partition emergent topic
+ * files; the agent-partition meta-files (identity, persona) were anchored to
+ * the iNFT but invisible to brain enumeration via the index. Result: the
+ * brain's `memory.read name=identity` would only succeed via the slug fallback
+ * branch; `memory.list` would report agent[] files but the brain wouldn't
+ * see them in narrative MEMORY.md prose.
+ *
+ * This module adds synthetic top-of-index entries for the canonical
+ * agent-partition + the user/profile.md anchor whenever those files exist
+ * on disk. Idempotent (matches by file path). Runs at boot-restore and at
+ * every sync.doFlush() so the index stays current.
+ */
+export interface SyntheticIndexFile {
+  /** Path relative to memoryDir, e.g. `agent/identity.md`. */
+  file: string
+  /** Title used when frontmatter `name` is absent. */
+  fallbackTitle: string
+}
+
+export const STANDARD_SYNTHETIC_INDEX_FILES: readonly SyntheticIndexFile[] = [
+  { file: 'agent/identity.md', fallbackTitle: 'identity' },
+  { file: 'agent/persona.md', fallbackTitle: 'persona' },
+  { file: 'user/profile.md', fallbackTitle: 'profile' },
+]
+
+export interface SyntheticIndexResult {
+  added: string[]
+  skipped: string[]
+}
+
+export async function ensureSyntheticIndexEntries(
+  memoryDir: string,
+  files: readonly SyntheticIndexFile[] = STANDARD_SYNTHETIC_INDEX_FILES,
+): Promise<SyntheticIndexResult> {
+  const indexPath = join(memoryDir, 'MEMORY.md')
+  let index: Awaited<ReturnType<typeof readIndexFile>>
+  try {
+    index = await readIndexFile(indexPath)
+  } catch {
+    // Index file missing or unreadable — skip silently. seedStarterMemoryFiles
+    // creates it at init; existing agents that pre-date that path may need a
+    // one-time backfill via migration.
+    return { added: [], skipped: files.map(f => f.file) }
+  }
+
+  const added: string[] = []
+  const skipped: string[] = []
+
+  for (const f of files) {
+    if (index.entries.has(f.file)) {
+      skipped.push(f.file)
+      continue
+    }
+    const fsPath = join(memoryDir, f.file)
+    if (!(await fileExists(fsPath))) {
+      skipped.push(f.file)
+      continue
+    }
+    let title = f.fallbackTitle
+    let description: string | null = null
+    try {
+      const content = await readFile(fsPath, 'utf8')
+      const head = content.length > 4096 ? content.slice(0, 4096) : content
+      const parsed = matter(head)
+      const fm = parsed.data as { name?: string; description?: string }
+      if (fm.name && typeof fm.name === 'string') title = fm.name
+      if (fm.description && typeof fm.description === 'string') description = fm.description
+    } catch {
+      // bad frontmatter — fall back to filename
+    }
+    index = addEntryLine(index, {
+      file: f.file,
+      title,
+      hook: description ?? title,
+    })
+    added.push(f.file)
+  }
+
+  if (added.length > 0) {
+    await writeIndexFile(indexPath, index)
+  }
+
+  return { added, skipped }
+}
+
+async function fileExists(path: string): Promise<boolean> {
+  try {
+    const s = await stat(path)
+    return s.isFile() && s.size > 0
+  } catch {
+    return false
+  }
+}

package/src/memory/index.ts

@@ -0,0 +1,58 @@
+export type {
+  MemoryType,
+  MemoryPartition,
+  MemoryFrontmatter,
+  MemoryTopic,
+  MemoryIndexEntry,
+  MemoryIndex,
+} from './types'
+export { MEMORY_TYPES } from './types'
+export { parseTopic, stringifyTopic } from './parser'
+export { scanForThreats, type ThreatScanResult } from './scan'
+export { applyEdit, EditError, type EditOp, type EditAction } from './edit'
+export {
+  parseIndex,
+  stringifyIndex,
+  readIndexFile,
+  writeIndexFile,
+  addEntryLine,
+  removeEntryLine,
+  INDEX_LINE_LIMIT,
+  INDEX_BYTE_LIMIT,
+} from './index-file'
+export { readTopic, writeTopic, topicPath } from './topic'
+export { makeMemorySaveTool, type MemorySaveArgs } from './save-tool'
+export { makeMemoryReadTool, type MemoryReadArgs } from './read-tool'
+export {
+  makeMemoryListTool,
+  type MemoryListArgs,
+  type MemoryListAgentFile,
+} from './list-tool'
+export {
+  ensureSyntheticIndexEntries,
+  STANDARD_SYNTHETIC_INDEX_FILES,
+  type SyntheticIndexFile,
+  type SyntheticIndexResult,
+} from './index-sync'
+export {
+  MEMORY_BLOB_VERSION,
+  deriveMemoryKey,
+  encryptMemoryBytes,
+  decryptMemoryBytes,
+} from './encryption'
+export { readOrNull } from './fs-util'
+export {
+  PACK_BLOB_VERSION,
+  encodePackBlob,
+  decodePackBlob,
+  isV2Envelope,
+  type PackBlob,
+  type EncodePackOpts,
+} from './pack-blob'
+export {
+  gatherAgentPack,
+  gatherUserPack,
+  writeAgentPack,
+  writeUserPack,
+  type GatherResult,
+} from './pack-gather'

package/src/memory/list-tool.ts

@@ -0,0 +1,105 @@
+import { readFile, readdir, stat } from 'node:fs/promises'
+import { join } from 'node:path'
+import matter from 'gray-matter'
+import { z } from 'zod'
+import { agentPaths } from '../paths'
+import type { ToolDef } from '../tools/types'
+
+/**
+ * `memory.list` — enumerate every memory file the agent has stored locally.
+ *
+ * Returns two sections:
+ *   - `agent[]`: files under `memory/agent/` (identity, persona, learned-*)
+ *   - `user[]`: files under `memory/user/` (feedback, project, reference, profile)
+ *
+ * Use when the operator asks to enumerate what the agent knows. `memory.read`
+ * fetches individual file bodies; this tool just lists what's available.
+ */
+const listSchema = z.object({})
+
+export type MemoryListArgs = z.infer<typeof listSchema>
+
+export interface MemoryListAgentFile {
+  file: string
+  title: string
+  description: string | null
+  bytes: number
+}
+
+export interface MakeMemoryListToolArgs {
+  agentId: string
+  agentDir?: string
+}
+
+export function makeMemoryListTool(opts: MakeMemoryListToolArgs): ToolDef<MemoryListArgs> {
+  const memDir = opts.agentDir
+    ? join(opts.agentDir, 'memory')
+    : agentPaths.agent(opts.agentId).memoryDir
+  return {
+    name: 'memory.list',
+    description:
+      "Enumerate every memory file the agent has stored (agent + user partitions). Call when the operator asks 'show me all your memory' / 'what do you remember' / 'list everything you have stored'. Returns two sections: agent (identity, persona, learned-*) and user (feedback, project, reference, profile).",
+    schema: listSchema,
+    handler: async () => {
+      const [agentFiles, userFiles] = await Promise.all([
+        listPartition(memDir, 'agent'),
+        listPartition(memDir, 'user'),
+      ])
+      return {
+        ok: true,
+        data: {
+          agent: agentFiles,
+          user: userFiles,
+        },
+      }
+    },
+  }
+}
+
+async function listPartition(
+  memDir: string,
+  partition: 'agent' | 'user',
+): Promise<MemoryListAgentFile[]> {
+  const dir = join(memDir, partition)
+  let names: string[]
+  try {
+    names = await readdir(dir)
+  } catch {
+    return []
+  }
+  const results = await Promise.all(
+    names
+      .filter(n => n.endsWith('.md'))
+      .map(async name => {
+        const filePath = join(dir, name)
+        try {
+          const [statResult, content] = await Promise.all([
+            stat(filePath),
+            readFile(filePath, 'utf8'),
+          ])
+          if (!statResult.isFile()) return null
+          // gray-matter on first 4KB is enough for frontmatter parse.
+          const head = content.length > 4096 ? content.slice(0, 4096) : content
+          let title = name.replace(/\.md$/, '')
+          let description: string | null = null
+          try {
+            const parsed = matter(head)
+            const fm = parsed.data as { name?: string; description?: string }
+            if (fm.name && typeof fm.name === 'string') title = fm.name
+            if (fm.description && typeof fm.description === 'string') description = fm.description
+          } catch {
+            // Bad frontmatter — fall back to filename.
+          }
+          return {
+            file: `${partition}/${name}`,
+            title,
+            description,
+            bytes: statResult.size,
+          } satisfies MemoryListAgentFile
+        } catch {
+          return null
+        }
+      }),
+  )
+  return results.filter((r): r is MemoryListAgentFile => r !== null)
+}

package/src/memory/pack-blob.test.ts

@@ -0,0 +1,90 @@
+import { describe, expect, test } from 'bun:test'
+import { PACK_BLOB_VERSION, decodePackBlob, encodePackBlob, isV2Envelope } from './pack-blob'
+
+describe('pack-blob v2 envelope', () => {
+  test('encode + decode round-trips single root', () => {
+    const bytes = encodePackBlob({ root: '# MEMORY.md\n\nentries' })
+    expect(isV2Envelope(bytes)).toBe(true)
+    const blob = decodePackBlob(bytes)
+    expect(blob.v).toBe(PACK_BLOB_VERSION)
+    expect(blob.root).toBe('# MEMORY.md\n\nentries')
+    expect(blob.files).toEqual({})
+  })
+
+  test('encode + decode round-trips with files map', () => {
+    const bytes = encodePackBlob({
+      root: '# profile',
+      files: {
+        'operator-preferences.md': '# Operator Preferences\n\ndark mode',
+        'hackathon-deadline.md': 'May 16 2026',
+      },
+    })
+    const blob = decodePackBlob(bytes)
+    expect(blob.root).toBe('# profile')
+    expect(blob.files['operator-preferences.md']).toContain('dark mode')
+    expect(blob.files['hackathon-deadline.md']).toBe('May 16 2026')
+  })
+
+  test('encode rejects unsafe filenames', () => {
+    expect(() => encodePackBlob({ root: 'x', files: { '../etc/passwd.md': 'hax' } })).toThrow(
+      /unsafe filename/,
+    )
+    expect(() => encodePackBlob({ root: 'x', files: { 'no-dot-md': 'x' } })).toThrow(
+      /unsafe filename/,
+    )
+    expect(() => encodePackBlob({ root: 'x', files: { 'UPPERCASE.md': 'x' } })).toThrow(
+      /unsafe filename/,
+    )
+    expect(() => encodePackBlob({ root: 'x', files: { 'has spaces.md': 'x' } })).toThrow(
+      /unsafe filename/,
+    )
+    expect(() => encodePackBlob({ root: 'x', files: { '.md': 'x' } })).toThrow(/unsafe filename/)
+  })
+
+  test('isV2Envelope rejects legacy markdown', () => {
+    const md = new TextEncoder().encode('# Heading\n\ncontent')
+    expect(isV2Envelope(md)).toBe(false)
+  })
+
+  test('isV2Envelope rejects v1 JSON without v:2', () => {
+    const v1 = new TextEncoder().encode('{"version":1,"data":"x"}')
+    expect(isV2Envelope(v1)).toBe(false)
+  })
+
+  test('isV2Envelope rejects garbage', () => {
+    const junk = new Uint8Array([0xff, 0x00, 0xab, 0xcd])
+    expect(isV2Envelope(junk)).toBe(false)
+    expect(isV2Envelope(new Uint8Array(0))).toBe(false)
+  })
+
+  test('isV2Envelope tolerates leading whitespace', () => {
+    const bytes = new TextEncoder().encode('   {"v":2,"root":"x","files":{}}')
+    expect(isV2Envelope(bytes)).toBe(true)
+  })
+
+  test('decode rejects mismatched version', () => {
+    const bytes = new TextEncoder().encode('{"v":999,"root":"x"}')
+    expect(() => decodePackBlob(bytes)).toThrow(/expected v=2/)
+  })
+
+  test('decode tolerates unsafe filenames by dropping them', () => {
+    const bytes = new TextEncoder().encode(
+      '{"v":2,"root":"x","files":{"ok.md":"y","../bad.md":"hax"}}',
+    )
+    const blob = decodePackBlob(bytes)
+    expect(blob.files['ok.md']).toBe('y')
+    expect(blob.files['../bad.md']).toBeUndefined()
+  })
+
+  test('handles large blobs (~100 small files)', () => {
+    const files: Record<string, string> = {}
+    for (let i = 0; i < 100; i++) {
+      files[`learned-${i}.md`] = `fact ${i}\n`.repeat(50)
+    }
+    const bytes = encodePackBlob({ root: '# MEMORY', files })
+    expect(bytes.length).toBeGreaterThan(1000)
+    const blob = decodePackBlob(bytes)
+    expect(Object.keys(blob.files).length).toBe(100)
+    expect(blob.files['learned-50.md']).toContain('fact 50')
+  })
+})

package/src/memory/pack-blob.ts

@@ -0,0 +1,120 @@
+/**
+ * v0.24.0: versioned envelope format for the memory-index (slot 0) and
+ * profile (slot 3) blobs. The legacy v1 layout stored a single markdown
+ * file's raw text. The v2 envelope wraps the root file's content plus
+ * an arbitrary map of additional sibling files so the harness can anchor
+ * the whole partition with one slot per partition.
+ *
+ * Why: pre-v0.24.0, only the 6 hard-coded `RESTORE_TARGETS` files survived
+ * reprovision. Every other `agent/*.md` and `user/*.md` was local-only
+ * scratchpad, and MEMORY.md retained dangling references after a fresh
+ * sandbox boot. The iNFT contract caps slots at 6 per token and is
+ * immutable, so adding new slots is impossible without re-minting every
+ * existing agent. The fix: extend the encoding of slots 0 + 3 without
+ * touching the contract.
+ *
+ * Envelope shape (plaintext, pre-encryption):
+ *
+ *   { "v": 2,
+ *     "root": "<markdown body of the canonical file>",
+ *     "files": { "<filename>.md": "<markdown body>", ... } }
+ *
+ * - For slot 0 (memory-index, agent key): `root` is MEMORY.md text.
+ *   `files` keys are paths under `memory/agent/` (e.g. `learned-foo.md`).
+ *   `identity.md` and `persona.md` are NOT packed here — they keep their
+ *   own slots (1, 2).
+ * - For slot 3 (profile, operator PROFILE key): `root` is profile.md text.
+ *   `files` keys are paths under `memory/user/` (e.g.
+ *   `operator-preferences.md`). `profile.md` itself is NOT a `files` key
+ *   (its content is in `root`).
+ *
+ * Backwards compat:
+ *
+ * - `isV2Envelope(bytes)`: cheap byte sniff — first non-whitespace char
+ *   must be `{` AND the parsed object must have `"v": 2`.
+ * - Decoders fall through to legacy v1 (raw markdown) when sniff fails.
+ * - Encoders default to v2; pass `legacy: true` for the old single-file
+ *   format if a caller specifically needs v1 wire-compat.
+ *
+ * Filename sanitization:
+ *
+ * Keys in `files` must match `^[a-z0-9][a-z0-9._-]{0,63}\.md$` to keep
+ * the pack format predictable. The brain's slug generator (`toSlug` in
+ * `save-tool.ts`) already produces filenames that match. Unsafe names
+ * (path traversal, absolute paths, weird chars) are rejected at encode
+ * time.
+ */
+
+const SAFE_NAME = /^[a-z0-9][a-z0-9._-]{0,63}\.md$/
+
+export const PACK_BLOB_VERSION = 2 as const
+
+export interface PackBlob {
+  v: typeof PACK_BLOB_VERSION
+  /** Root file content (MEMORY.md for slot 0, profile.md for slot 3). */
+  root: string
+  /** Additional packed files keyed by filename. May be empty. */
+  files: Record<string, string>
+}
+
+export interface EncodePackOpts {
+  root: string
+  files?: Record<string, string>
+}
+
+/** Encode a pack blob to UTF-8 bytes ready for AES-GCM encryption. */
+export function encodePackBlob(opts: EncodePackOpts): Uint8Array {
+  const files: Record<string, string> = {}
+  for (const [name, content] of Object.entries(opts.files ?? {})) {
+    if (!SAFE_NAME.test(name)) {
+      throw new Error(`pack-blob: unsafe filename ${JSON.stringify(name)}`)
+    }
+    files[name] = content
+  }
+  const blob: PackBlob = { v: PACK_BLOB_VERSION, root: opts.root, files }
+  return new TextEncoder().encode(JSON.stringify(blob))
+}
+
+/**
+ * Returns true if `bytes` looks like a v2 envelope (starts with `{` and
+ * parses to `{ v: 2 }`). Cheap enough to call on every restore.
+ */
+export function isV2Envelope(bytes: Uint8Array): boolean {
+  if (bytes.length < 8) return false
+  for (let i = 0; i < bytes.length && i < 16; i++) {
+    const b = bytes[i] as number
+    if (b === 0x7b /* { */) break
+    if (b === 0x20 || b === 0x09 || b === 0x0a || b === 0x0d) continue
+    return false
+  }
+  try {
+    const text = new TextDecoder().decode(bytes)
+    const parsed = JSON.parse(text) as { v?: unknown }
+    return parsed && typeof parsed === 'object' && parsed.v === PACK_BLOB_VERSION
+  } catch {
+    return false
+  }
+}
+
+/**
+ * Decode a v2 envelope. Throws if `bytes` is not a valid v2 envelope.
+ * Caller is expected to have run `isV2Envelope` first when handling
+ * legacy/v2 mixed input.
+ */
+export function decodePackBlob(bytes: Uint8Array): PackBlob {
+  const text = new TextDecoder().decode(bytes)
+  const parsed = JSON.parse(text) as Partial<PackBlob>
+  if (parsed.v !== PACK_BLOB_VERSION) {
+    throw new Error(`pack-blob: expected v=${PACK_BLOB_VERSION}, got ${parsed.v}`)
+  }
+  if (typeof parsed.root !== 'string') {
+    throw new Error('pack-blob: missing root field')
+  }
+  const files: Record<string, string> = {}
+  for (const [name, content] of Object.entries(parsed.files ?? {})) {
+    if (typeof content !== 'string') continue
+    if (!SAFE_NAME.test(name)) continue
+    files[name] = content
+  }
+  return { v: PACK_BLOB_VERSION, root: parsed.root, files }
+}

package/src/memory/pack-gather.test.ts

@@ -0,0 +1,110 @@
+import { afterEach, beforeEach, expect, test } from 'bun:test'
+import { mkdtempSync, rmSync } from 'node:fs'
+import { mkdir, readFile, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { decodePackBlob, encodePackBlob } from './pack-blob'
+import { gatherAgentPack, gatherUserPack, writeAgentPack, writeUserPack } from './pack-gather'
+
+let tmp: string
+
+beforeEach(() => {
+  tmp = mkdtempSync(join(tmpdir(), 'pack-gather-'))
+})
+
+afterEach(() => {
+  rmSync(tmp, { recursive: true, force: true })
+})
+
+async function seed(rel: string, content: string): Promise<void> {
+  const full = join(tmp, rel)
+  await mkdir(join(full, '..'), { recursive: true })
+  await writeFile(full, content)
+}
+
+test('gatherAgentPack returns root + non-excluded files', async () => {
+  await seed('MEMORY.md', '# memory index')
+  await seed('agent/identity.md', 'identity body')
+  await seed('agent/persona.md', 'persona body')
+  await seed('agent/learned-foo.md', 'foo content')
+  await seed('agent/learned-bar.md', 'bar content')
+
+  const res = await gatherAgentPack(tmp)
+  expect(res.root).toBe('# memory index')
+  expect(res.files['learned-foo.md']).toBe('foo content')
+  expect(res.files['learned-bar.md']).toBe('bar content')
+  // Identity + persona excluded — they have their own slots
+  expect(res.files['identity.md']).toBeUndefined()
+  expect(res.files['persona.md']).toBeUndefined()
+})
+
+test('gatherUserPack returns profile.md as root + other user files', async () => {
+  await seed('user/profile.md', '# user profile')
+  await seed('user/operator-preferences.md', 'dark mode')
+  await seed('user/0g-hackathon.md', 'deadline')
+  const res = await gatherUserPack(tmp)
+  expect(res.root).toBe('# user profile')
+  expect(res.files['operator-preferences.md']).toBe('dark mode')
+  expect(res.files['0g-hackathon.md']).toBe('deadline')
+  // profile.md is the root, not in files
+  expect(res.files['profile.md']).toBeUndefined()
+})
+
+test('gather handles missing partition dir', async () => {
+  await seed('MEMORY.md', 'just an index')
+  const res = await gatherAgentPack(tmp)
+  expect(res.root).toBe('just an index')
+  expect(res.files).toEqual({})
+})
+
+test('gather handles missing root file', async () => {
+  await seed('agent/learned-x.md', 'content')
+  const res = await gatherAgentPack(tmp)
+  expect(res.root).toBe('')
+  expect(res.files['learned-x.md']).toBe('content')
+})
+
+test('gather skips non-md files + empty .md', async () => {
+  await seed('agent/learned-x.md', 'content')
+  await seed('agent/junk.txt', 'not md')
+  await seed('agent/empty.md', '')
+  const res = await gatherAgentPack(tmp)
+  expect(res.files['learned-x.md']).toBe('content')
+  expect(res.files['junk.txt']).toBeUndefined()
+  expect(res.files['empty.md']).toBeUndefined()
+})
+
+test('writeAgentPack reverses gatherAgentPack', async () => {
+  await seed('MEMORY.md', 'index v1')
+  await seed('agent/learned-a.md', 'a')
+  await seed('agent/learned-b.md', 'b')
+  const gathered = await gatherAgentPack(tmp)
+  const bytes = encodePackBlob(gathered)
+  const decoded = decodePackBlob(bytes)
+
+  const dst = mkdtempSync(join(tmpdir(), 'pack-write-'))
+  try {
+    await writeAgentPack(dst, decoded)
+    expect(await readFile(join(dst, 'MEMORY.md'), 'utf8')).toBe('index v1')
+    expect(await readFile(join(dst, 'agent', 'learned-a.md'), 'utf8')).toBe('a')
+    expect(await readFile(join(dst, 'agent', 'learned-b.md'), 'utf8')).toBe('b')
+  } finally {
+    rmSync(dst, { recursive: true, force: true })
+  }
+})
+
+test('writeUserPack creates user dir + profile + sibling files', async () => {
+  const dst = mkdtempSync(join(tmpdir(), 'pack-write-'))
+  try {
+    await writeUserPack(dst, {
+      v: 2,
+      root: '# profile',
+      files: { 'operator-preferences.md': 'prefs', 'hack-2026.md': 'deadline' },
+    })
+    expect(await readFile(join(dst, 'user', 'profile.md'), 'utf8')).toBe('# profile')
+    expect(await readFile(join(dst, 'user', 'operator-preferences.md'), 'utf8')).toBe('prefs')
+    expect(await readFile(join(dst, 'user', 'hack-2026.md'), 'utf8')).toBe('deadline')
+  } finally {
+    rmSync(dst, { recursive: true, force: true })
+  }
+})