nebula-ai-core 0.1.0

package/src/memory/edit.ts

@@ -0,0 +1,53 @@
+export type EditAction = 'add' | 'replace' | 'remove'
+
+export interface EditOp {
+  action: EditAction
+  /** Required for replace/remove. Substring to match in the existing body. */
+  oldText?: string
+  /** Required for add/replace. Text to insert. */
+  newText?: string
+}
+
+export class EditError extends Error {
+  constructor(
+    message: string,
+    readonly op: EditOp,
+  ) {
+    super(message)
+    this.name = 'EditError'
+  }
+}
+
+/**
+ * Apply a single edit op to `body`. Substring-based matching (hermes pattern),
+ * no IDs or line numbers. Returns the new body.
+ */
+export function applyEdit(body: string, op: EditOp): string {
+  switch (op.action) {
+    case 'add': {
+      if (op.newText === undefined) throw new EditError('add requires newText', op)
+      return body.length === 0 ? op.newText : `${body.trimEnd()}\n\n${op.newText}\n`
+    }
+    case 'replace': {
+      if (op.oldText === undefined || op.newText === undefined) {
+        throw new EditError('replace requires oldText AND newText', op)
+      }
+      const idx = locateUnique(body, op.oldText, op)
+      return body.slice(0, idx) + op.newText + body.slice(idx + op.oldText.length)
+    }
+    case 'remove': {
+      if (op.oldText === undefined) throw new EditError('remove requires oldText', op)
+      const idx = locateUnique(body, op.oldText, op)
+      return body.slice(0, idx) + body.slice(idx + op.oldText.length)
+    }
+  }
+}
+
+function locateUnique(body: string, needle: string, op: EditOp): number {
+  const idx = body.indexOf(needle)
+  if (idx < 0) throw new EditError('oldText not found in body', op)
+  if (body.indexOf(needle, idx + 1) >= 0) {
+    throw new EditError('oldText is not unique in body', op)
+  }
+  return idx
+}

package/src/memory/encryption.ts

@@ -0,0 +1,88 @@
+import { createCipheriv, createDecipheriv, hkdfSync, randomBytes } from 'node:crypto'
+import { gunzipSync, gzipSync } from 'node:zlib'
+import type { Hex } from 'viem'
+import { hexToBytes } from 'viem'
+
+/**
+ * Phase 6.7 memory file encryption.
+ *
+ * Key derivation: HKDF-SHA256(ikm = agent privkey bytes, info = "nebula-memory-aead-v1")
+ *   → 32-byte AES-256-GCM key.
+ *
+ * Why agent privkey (not operator wallet)? Memory writes happen mid-chat —
+ * thousands of times in a long conversation. Asking the operator wallet to
+ * sign per write would be miserable for WC users. The agent privkey is
+ * already in RAM during the chat session (decrypted via operator at session
+ * start), so deriving a memory key from it is silent and fast.
+ *
+ * Recovery: anyone who can decrypt the keystore can derive this key, so the
+ * security envelope is the same as the keystore's.
+ *
+ * Format: v(1) || iv(12) || tag(16) || ciphertext  (raw bytes, no JSON wrap).
+ *   v=1: plaintext encrypted directly (legacy).
+ *   v=2: plaintext gzip-compressed first then encrypted. Decryption gunzips
+ *        after AES-GCM. Used for the activity-log slot where JSON content
+ *        compresses 5-10x and the Mantle Storage upload is the bottleneck.
+ *
+ * Both versions are backwards-compatible: decryptMemoryBytes dispatches on
+ * the leading version byte and reads either layout.
+ */
+export const MEMORY_BLOB_VERSION = 1 as const
+export const MEMORY_BLOB_VERSION_GZIP = 2 as const
+
+const HKDF_INFO = Buffer.from('nebula-memory-aead-v1', 'utf8')
+const KEY_LEN = 32
+const IV_LEN = 12
+const TAG_LEN = 16
+
+export function deriveMemoryKey(agentPrivkey: Hex): Buffer {
+  const ikm = Buffer.from(hexToBytes(agentPrivkey))
+  return Buffer.from(hkdfSync('sha256', ikm, Buffer.alloc(0), HKDF_INFO, KEY_LEN))
+}
+
+export interface EncryptOpts {
+  /**
+   * Gzip the plaintext before encrypting. Reduces blob size 5-10x on JSON-
+   * heavy content like the activity log. Costs a few ms of CPU per upload
+   * — fine because the network upload it saves is much slower. Default
+   * false to preserve byte-for-byte compatibility with v=1 callers.
+   */
+  compress?: boolean
+}
+
+export function encryptMemoryBytes(
+  plaintext: Uint8Array,
+  key: Buffer,
+  opts: EncryptOpts = {},
+): Uint8Array {
+  if (key.length !== KEY_LEN) throw new Error(`memory key must be ${KEY_LEN} bytes`)
+  const iv = randomBytes(IV_LEN)
+  const cipher = createCipheriv('aes-256-gcm', key, iv)
+  const payload = opts.compress ? gzipSync(plaintext) : plaintext
+  const ct = Buffer.concat([cipher.update(payload), cipher.final()])
+  const tag = cipher.getAuthTag()
+  const version = opts.compress ? MEMORY_BLOB_VERSION_GZIP : MEMORY_BLOB_VERSION
+  return new Uint8Array(Buffer.concat([Buffer.from([version]), iv, tag, ct]))
+}
+
+export function decryptMemoryBytes(blob: Uint8Array, key: Buffer): Uint8Array {
+  if (key.length !== KEY_LEN) throw new Error(`memory key must be ${KEY_LEN} bytes`)
+  const buf = Buffer.from(blob)
+  if (buf.length < 1 + IV_LEN + TAG_LEN) {
+    throw new Error(`memory blob too short: ${buf.length} bytes`)
+  }
+  const version = buf[0]
+  if (version !== MEMORY_BLOB_VERSION && version !== MEMORY_BLOB_VERSION_GZIP) {
+    throw new Error(`unsupported memory blob version: ${version}`)
+  }
+  const iv = buf.subarray(1, 1 + IV_LEN)
+  const tag = buf.subarray(1 + IV_LEN, 1 + IV_LEN + TAG_LEN)
+  const ct = buf.subarray(1 + IV_LEN + TAG_LEN)
+  const decipher = createDecipheriv('aes-256-gcm', key, iv)
+  decipher.setAuthTag(tag)
+  const decrypted = Buffer.concat([decipher.update(ct), decipher.final()])
+  if (version === MEMORY_BLOB_VERSION_GZIP) {
+    return new Uint8Array(gunzipSync(decrypted))
+  }
+  return new Uint8Array(decrypted)
+}

package/src/memory/fs-util.ts

@@ -0,0 +1,15 @@
+import { readFile } from 'node:fs/promises'
+
+/**
+ * Read a file as bytes; return null on ENOENT, rethrow other errors.
+ * Used wherever an "absent file is fine, anything else is a bug" semantic
+ * is needed (memory sync, activity log sync, on-chain diff vs local).
+ */
+export async function readOrNull(path: string): Promise<Uint8Array | null> {
+  try {
+    return new Uint8Array(await readFile(path))
+  } catch (e) {
+    if ((e as NodeJS.ErrnoException).code === 'ENOENT') return null
+    throw e
+  }
+}

package/src/memory/index-file.ts

@@ -0,0 +1,74 @@
+import { readFile, rename, writeFile } from 'node:fs/promises'
+import type { MemoryIndex, MemoryIndexEntry } from './types'
+
+/** Max enforced by Claude Code conventions — loaded into every prompt. */
+export const INDEX_LINE_LIMIT = 200
+export const INDEX_BYTE_LIMIT = 25 * 1024
+
+const ENTRY_RE = /^-\s*\[([^\]]+)\]\(([^)]+)\)(?:\s*[-—]\s*(.*))?$/
+
+export function parseIndex(raw: string): MemoryIndex {
+  const lines = raw.split('\n')
+  const entries = new Map<string, MemoryIndexEntry>()
+  for (const line of lines) {
+    const m = line.match(ENTRY_RE)
+    if (m?.[1] && m[2]) {
+      const title = m[1]
+      const file = m[2]
+      const hook = m[3] ?? ''
+      entries.set(file, { file, title, hook: hook.trim() })
+    }
+  }
+  return { lines, entries }
+}
+
+export function stringifyIndex(index: MemoryIndex): string {
+  const joined = index.lines.join('\n')
+  return joined.endsWith('\n') ? joined : `${joined}\n`
+}
+
+export async function readIndexFile(path: string): Promise<MemoryIndex> {
+  const raw = await readFile(path, 'utf8').catch(e => {
+    if ((e as NodeJS.ErrnoException).code === 'ENOENT') return ''
+    throw e
+  })
+  return parseIndex(raw)
+}
+
+export async function writeIndexFile(path: string, index: MemoryIndex): Promise<void> {
+  const content = stringifyIndex(index)
+  if (content.length > INDEX_BYTE_LIMIT) {
+    throw new Error(`MEMORY.md exceeds ${INDEX_BYTE_LIMIT}-byte cap (got ${content.length})`)
+  }
+  if (index.lines.length > INDEX_LINE_LIMIT) {
+    throw new Error(`MEMORY.md exceeds ${INDEX_LINE_LIMIT}-line cap (got ${index.lines.length})`)
+  }
+  const tmp = `${path}.tmp-${process.pid}-${Date.now().toString(36)}`
+  await writeFile(tmp, content, 'utf8')
+  await rename(tmp, path)
+}
+
+export function addEntryLine(index: MemoryIndex, entry: MemoryIndexEntry): MemoryIndex {
+  if (index.entries.has(entry.file)) return index
+  const line = `- [${entry.title}](${entry.file}) — ${entry.hook}`
+  const next: MemoryIndex = {
+    lines: [...index.lines, line],
+    entries: new Map(index.entries),
+  }
+  next.entries.set(entry.file, entry)
+  return next
+}
+
+export function removeEntryLine(index: MemoryIndex, file: string): MemoryIndex {
+  if (!index.entries.has(file)) return index
+  const filtered = index.lines.filter(line => {
+    const m = line.match(ENTRY_RE)
+    return !(m && m[2] === file)
+  })
+  const next: MemoryIndex = {
+    lines: filtered,
+    entries: new Map(index.entries),
+  }
+  next.entries.delete(file)
+  return next
+}

package/src/memory/index-sync.ts

@@ -0,0 +1,99 @@
+import { readFile, stat } from 'node:fs/promises'
+import { join } from 'node:path'
+import matter from 'gray-matter'
+import { addEntryLine, readIndexFile, writeIndexFile } from './index-file'
+
+/**
+ * v0.23.0: MEMORY.md historically only listed user-partition emergent topic
+ * files; the agent-partition meta-files (identity, persona) were anchored to
+ * the iNFT but invisible to brain enumeration via the index. Result: the
+ * brain's `memory.read name=identity` would only succeed via the slug fallback
+ * branch; `memory.list` would report agent[] files but the brain wouldn't
+ * see them in narrative MEMORY.md prose.
+ *
+ * This module adds synthetic top-of-index entries for the canonical
+ * agent-partition + the user/profile.md anchor whenever those files exist
+ * on disk. Idempotent (matches by file path). Runs at boot-restore and at
+ * every sync.doFlush() so the index stays current.
+ */
+export interface SyntheticIndexFile {
+  /** Path relative to memoryDir, e.g. `agent/identity.md`. */
+  file: string
+  /** Title used when frontmatter `name` is absent. */
+  fallbackTitle: string
+}
+
+export const STANDARD_SYNTHETIC_INDEX_FILES: readonly SyntheticIndexFile[] = [
+  { file: 'agent/identity.md', fallbackTitle: 'identity' },
+  { file: 'agent/persona.md', fallbackTitle: 'persona' },
+  { file: 'user/profile.md', fallbackTitle: 'profile' },
+]
+
+export interface SyntheticIndexResult {
+  added: string[]
+  skipped: string[]
+}
+
+export async function ensureSyntheticIndexEntries(
+  memoryDir: string,
+  files: readonly SyntheticIndexFile[] = STANDARD_SYNTHETIC_INDEX_FILES,
+): Promise<SyntheticIndexResult> {
+  const indexPath = join(memoryDir, 'MEMORY.md')
+  let index: Awaited<ReturnType<typeof readIndexFile>>
+  try {
+    index = await readIndexFile(indexPath)
+  } catch {
+    // Index file missing or unreadable — skip silently. seedStarterMemoryFiles
+    // creates it at init; existing agents that pre-date that path may need a
+    // one-time backfill via migration.
+    return { added: [], skipped: files.map(f => f.file) }
+  }
+
+  const added: string[] = []
+  const skipped: string[] = []
+
+  for (const f of files) {
+    if (index.entries.has(f.file)) {
+      skipped.push(f.file)
+      continue
+    }
+    const fsPath = join(memoryDir, f.file)
+    if (!(await fileExists(fsPath))) {
+      skipped.push(f.file)
+      continue
+    }
+    let title = f.fallbackTitle
+    let description: string | null = null
+    try {
+      const content = await readFile(fsPath, 'utf8')
+      const head = content.length > 4096 ? content.slice(0, 4096) : content
+      const parsed = matter(head)
+      const fm = parsed.data as { name?: string; description?: string }
+      if (fm.name && typeof fm.name === 'string') title = fm.name
+      if (fm.description && typeof fm.description === 'string') description = fm.description
+    } catch {
+      // bad frontmatter — fall back to filename
+    }
+    index = addEntryLine(index, {
+      file: f.file,
+      title,
+      hook: description ?? title,
+    })
+    added.push(f.file)
+  }
+
+  if (added.length > 0) {
+    await writeIndexFile(indexPath, index)
+  }
+
+  return { added, skipped }
+}
+
+async function fileExists(path: string): Promise<boolean> {
+  try {
+    const s = await stat(path)
+    return s.isFile() && s.size > 0
+  } catch {
+    return false
+  }
+}

package/src/memory/index.ts

@@ -0,0 +1,58 @@
+export type {
+  MemoryType,
+  MemoryPartition,
+  MemoryFrontmatter,
+  MemoryTopic,
+  MemoryIndexEntry,
+  MemoryIndex,
+} from './types'
+export { MEMORY_TYPES } from './types'
+export { parseTopic, stringifyTopic } from './parser'
+export { scanForThreats, type ThreatScanResult } from './scan'
+export { applyEdit, EditError, type EditOp, type EditAction } from './edit'
+export {
+  parseIndex,
+  stringifyIndex,
+  readIndexFile,
+  writeIndexFile,
+  addEntryLine,
+  removeEntryLine,
+  INDEX_LINE_LIMIT,
+  INDEX_BYTE_LIMIT,
+} from './index-file'
+export { readTopic, writeTopic, topicPath } from './topic'
+export { makeMemorySaveTool, type MemorySaveArgs } from './save-tool'
+export { makeMemoryReadTool, type MemoryReadArgs } from './read-tool'
+export {
+  makeMemoryListTool,
+  type MemoryListArgs,
+  type MemoryListAgentFile,
+} from './list-tool'
+export {
+  ensureSyntheticIndexEntries,
+  STANDARD_SYNTHETIC_INDEX_FILES,
+  type SyntheticIndexFile,
+  type SyntheticIndexResult,
+} from './index-sync'
+export {
+  MEMORY_BLOB_VERSION,
+  deriveMemoryKey,
+  encryptMemoryBytes,
+  decryptMemoryBytes,
+} from './encryption'
+export { readOrNull } from './fs-util'
+export {
+  PACK_BLOB_VERSION,
+  encodePackBlob,
+  decodePackBlob,
+  isV2Envelope,
+  type PackBlob,
+  type EncodePackOpts,
+} from './pack-blob'
+export {
+  gatherAgentPack,
+  gatherUserPack,
+  writeAgentPack,
+  writeUserPack,
+  type GatherResult,
+} from './pack-gather'

package/src/memory/list-tool.ts

@@ -0,0 +1,105 @@
+import { readFile, readdir, stat } from 'node:fs/promises'
+import { join } from 'node:path'
+import matter from 'gray-matter'
+import { z } from 'zod'
+import { agentPaths } from '../paths'
+import type { ToolDef } from '../tools/types'
+
+/**
+ * `memory.list` — enumerate every memory file the agent has stored locally.
+ *
+ * Returns two sections:
+ *   - `agent[]`: files under `memory/agent/` (identity, persona, learned-*)
+ *   - `user[]`: files under `memory/user/` (feedback, project, reference, profile)
+ *
+ * Use when the operator asks to enumerate what the agent knows. `memory.read`
+ * fetches individual file bodies; this tool just lists what's available.
+ */
+const listSchema = z.object({})
+
+export type MemoryListArgs = z.infer<typeof listSchema>
+
+export interface MemoryListAgentFile {
+  file: string
+  title: string
+  description: string | null
+  bytes: number
+}
+
+export interface MakeMemoryListToolArgs {
+  agentId: string
+  agentDir?: string
+}
+
+export function makeMemoryListTool(opts: MakeMemoryListToolArgs): ToolDef<MemoryListArgs> {
+  const memDir = opts.agentDir
+    ? join(opts.agentDir, 'memory')
+    : agentPaths.agent(opts.agentId).memoryDir
+  return {
+    name: 'memory.list',
+    description:
+      "Enumerate every memory file the agent has stored (agent + user partitions). Call when the operator asks 'show me all your memory' / 'what do you remember' / 'list everything you have stored'. Returns two sections: agent (identity, persona, learned-*) and user (feedback, project, reference, profile).",
+    schema: listSchema,
+    handler: async () => {
+      const [agentFiles, userFiles] = await Promise.all([
+        listPartition(memDir, 'agent'),
+        listPartition(memDir, 'user'),
+      ])
+      return {
+        ok: true,
+        data: {
+          agent: agentFiles,
+          user: userFiles,
+        },
+      }
+    },
+  }
+}
+
+async function listPartition(
+  memDir: string,
+  partition: 'agent' | 'user',
+): Promise<MemoryListAgentFile[]> {
+  const dir = join(memDir, partition)
+  let names: string[]
+  try {
+    names = await readdir(dir)
+  } catch {
+    return []
+  }
+  const results = await Promise.all(
+    names
+      .filter(n => n.endsWith('.md'))
+      .map(async name => {
+        const filePath = join(dir, name)
+        try {
+          const [statResult, content] = await Promise.all([
+            stat(filePath),
+            readFile(filePath, 'utf8'),
+          ])
+          if (!statResult.isFile()) return null
+          // gray-matter on first 4KB is enough for frontmatter parse.
+          const head = content.length > 4096 ? content.slice(0, 4096) : content
+          let title = name.replace(/\.md$/, '')
+          let description: string | null = null
+          try {
+            const parsed = matter(head)
+            const fm = parsed.data as { name?: string; description?: string }
+            if (fm.name && typeof fm.name === 'string') title = fm.name
+            if (fm.description && typeof fm.description === 'string') description = fm.description
+          } catch {
+            // Bad frontmatter — fall back to filename.
+          }
+          return {
+            file: `${partition}/${name}`,
+            title,
+            description,
+            bytes: statResult.size,
+          } satisfies MemoryListAgentFile
+        } catch {
+          return null
+        }
+      }),
+  )
+  return results.filter((r): r is MemoryListAgentFile => r !== null)
+}

package/src/memory/pack-blob.ts

@@ -0,0 +1,120 @@
+/**
+ * v0.24.0: versioned envelope format for the memory-index (slot 0) and
+ * profile (slot 3) blobs. The legacy v1 layout stored a single markdown
+ * file's raw text. The v2 envelope wraps the root file's content plus
+ * an arbitrary map of additional sibling files so the harness can anchor
+ * the whole partition with one slot per partition.
+ *
+ * Why: pre-v0.24.0, only the 6 hard-coded `RESTORE_TARGETS` files survived
+ * reprovision. Every other `agent/*.md` and `user/*.md` was local-only
+ * scratchpad, and MEMORY.md retained dangling references after a fresh
+ * sandbox boot. The iNFT contract caps slots at 6 per token and is
+ * immutable, so adding new slots is impossible without re-minting every
+ * existing agent. The fix: extend the encoding of slots 0 + 3 without
+ * touching the contract.
+ *
+ * Envelope shape (plaintext, pre-encryption):
+ *
+ *   { "v": 2,
+ *     "root": "<markdown body of the canonical file>",
+ *     "files": { "<filename>.md": "<markdown body>", ... } }
+ *
+ * - For slot 0 (memory-index, agent key): `root` is MEMORY.md text.
+ *   `files` keys are paths under `memory/agent/` (e.g. `learned-foo.md`).
+ *   `identity.md` and `persona.md` are NOT packed here — they keep their
+ *   own slots (1, 2).
+ * - For slot 3 (profile, operator PROFILE key): `root` is profile.md text.
+ *   `files` keys are paths under `memory/user/` (e.g.
+ *   `operator-preferences.md`). `profile.md` itself is NOT a `files` key
+ *   (its content is in `root`).
+ *
+ * Backwards compat:
+ *
+ * - `isV2Envelope(bytes)`: cheap byte sniff — first non-whitespace char
+ *   must be `{` AND the parsed object must have `"v": 2`.
+ * - Decoders fall through to legacy v1 (raw markdown) when sniff fails.
+ * - Encoders default to v2; pass `legacy: true` for the old single-file
+ *   format if a caller specifically needs v1 wire-compat.
+ *
+ * Filename sanitization:
+ *
+ * Keys in `files` must match `^[a-z0-9][a-z0-9._-]{0,63}\.md$` to keep
+ * the pack format predictable. The brain's slug generator (`toSlug` in
+ * `save-tool.ts`) already produces filenames that match. Unsafe names
+ * (path traversal, absolute paths, weird chars) are rejected at encode
+ * time.
+ */
+
+const SAFE_NAME = /^[a-z0-9][a-z0-9._-]{0,63}\.md$/
+
+export const PACK_BLOB_VERSION = 2 as const
+
+export interface PackBlob {
+  v: typeof PACK_BLOB_VERSION
+  /** Root file content (MEMORY.md for slot 0, profile.md for slot 3). */
+  root: string
+  /** Additional packed files keyed by filename. May be empty. */
+  files: Record<string, string>
+}
+
+export interface EncodePackOpts {
+  root: string
+  files?: Record<string, string>
+}
+
+/** Encode a pack blob to UTF-8 bytes ready for AES-GCM encryption. */
+export function encodePackBlob(opts: EncodePackOpts): Uint8Array {
+  const files: Record<string, string> = {}
+  for (const [name, content] of Object.entries(opts.files ?? {})) {
+    if (!SAFE_NAME.test(name)) {
+      throw new Error(`pack-blob: unsafe filename ${JSON.stringify(name)}`)
+    }
+    files[name] = content
+  }
+  const blob: PackBlob = { v: PACK_BLOB_VERSION, root: opts.root, files }
+  return new TextEncoder().encode(JSON.stringify(blob))
+}
+
+/**
+ * Returns true if `bytes` looks like a v2 envelope (starts with `{` and
+ * parses to `{ v: 2 }`). Cheap enough to call on every restore.
+ */
+export function isV2Envelope(bytes: Uint8Array): boolean {
+  if (bytes.length < 8) return false
+  for (let i = 0; i < bytes.length && i < 16; i++) {
+    const b = bytes[i] as number
+    if (b === 0x7b /* { */) break
+    if (b === 0x20 || b === 0x09 || b === 0x0a || b === 0x0d) continue
+    return false
+  }
+  try {
+    const text = new TextDecoder().decode(bytes)
+    const parsed = JSON.parse(text) as { v?: unknown }
+    return parsed && typeof parsed === 'object' && parsed.v === PACK_BLOB_VERSION
+  } catch {
+    return false
+  }
+}
+
+/**
+ * Decode a v2 envelope. Throws if `bytes` is not a valid v2 envelope.
+ * Caller is expected to have run `isV2Envelope` first when handling
+ * legacy/v2 mixed input.
+ */
+export function decodePackBlob(bytes: Uint8Array): PackBlob {
+  const text = new TextDecoder().decode(bytes)
+  const parsed = JSON.parse(text) as Partial<PackBlob>
+  if (parsed.v !== PACK_BLOB_VERSION) {
+    throw new Error(`pack-blob: expected v=${PACK_BLOB_VERSION}, got ${parsed.v}`)
+  }
+  if (typeof parsed.root !== 'string') {
+    throw new Error('pack-blob: missing root field')
+  }
+  const files: Record<string, string> = {}
+  for (const [name, content] of Object.entries(parsed.files ?? {})) {
+    if (typeof content !== 'string') continue
+    if (!SAFE_NAME.test(name)) continue
+    files[name] = content
+  }
+  return { v: PACK_BLOB_VERSION, root: parsed.root, files }
+}