@100xprompt/brain 0.0.0-stage → 0.1.0

package/src/org/crypto.ts

@@ -0,0 +1,537 @@
+// brain org: encryption at rest.
+//
+// WHAT IT IS. The whole organisation database (Chitta's tables AND the `org_*` tables: one file) is encrypted by libSQL's built-in
+// whole-file AES-256 (the same driver Chitta itself switches to when `CONTEXT_DB_KEY` is set). Pages, the write-ahead log (-wal)
+// and the shared-memory index (-shm) are all written through the cipher: there is no application-level encryption of single
+// columns here and nothing is left readable on purpose. The proof is in test/org-crypto.test.ts: it reads the raw bytes of the
+// three files and looks for item text, user ids and audit text.
+//
+// WHAT IT IS NOT. It protects the file at rest (a copied disk, a stolen backup, a snapshot, another account on the machine that
+// can read the file but not the key). It does not protect against someone who can read the server process's memory or the key
+// itself (`BRAIN_ORG_KEY`, the key file): while the server runs the key and decrypted pages are in its memory. The key is never
+// stored in the database and never printed or logged by this code.
+//
+// HOW THE KEY GETS IN. `BRAIN_ORG_KEY` (an environment variable) or `--key-file <path>` (a file holding the key: the usual way with
+// Docker/Kubernetes secrets and a Secret Manager mount). Never a command-line argument (it would show in `ps`). A key is
+// 32-512 characters from `A-Za-z0-9+/=_.~-` (the generated one is 64 hex characters = 256 random bits).
+//
+// HOW CHITTA IS KEPT IN STEP. Chitta decides at construction time whether to open its store with libSQL by reading the
+// `CONTEXT_DB_KEY` environment variable. `withKeyEnv` sets it only for the duration of that synchronous constructor call and then
+// puts the previous value back, so the key is not left in the process environment (child processes never inherit it).
+//
+// ONE CONNECTION, TWO DRIVERS. `OrgDb` runs its queries on the connection Chitta opened. libSQL's handle is almost, but not quite,
+// bun:sqlite's: no `transaction()`, named parameters bound without their `$`, and rows come back with an extra `_metadata`
+// property. `adapt` closes exactly those gaps, so the same SQL (including the visibility predicate) runs unchanged on both.
+//
+// LIMITS (the honest list): libSQL must be installed where Chitta (or brain) can resolve it (`bun add libsql` in Chitta's folder;
+// it is a dev dependency of Chitta, not shipped by default). While encrypted, Chitta cannot load the sqlite-vec extension: it
+// uses libSQL's native vector index when it works and a brute-force scan otherwise (correct, slower on very large stores), and
+// decryption costs CPU on every page read. Metadata that is not in the file is not hidden: the file's size, its modification
+// times and the names of the files are visible to anyone who can list the directory. Converting a database (`encrypt`, `rekey`)
+// is a logical copy; the previous file is kept next to it as `*.bak-<time>` (plain text for `encrypt`!) until you remove it.
+import { Database } from "bun:sqlite"
+import { randomBytes } from "node:crypto"
+import { chmodSync, closeSync, existsSync, openSync, readFileSync, readSync, renameSync, rmSync, statSync, unlinkSync, writeFileSync, writeSync } from "node:fs"
+import { createRequire } from "node:module"
+import { dirname } from "node:path"
+import { chittaCandidates } from "../memory/store.ts"
+
+export const SQLITE_MAGIC = "SQLite format 3\u0000"
+
+/** A problem with the encryption key or with opening an encrypted file: the message is safe to show (it never holds the key). */
+export class OrgKeyError extends Error {
+  constructor(message: string) {
+    super(message)
+    this.name = "OrgKeyError"
+  }
+}
+
+export type DbState = "missing" | "empty" | "plain" | "encrypted"
+
+/** What the file's first bytes say: a plain SQLite database starts with "SQLite format 3\0"; an encrypted one starts with
+ *  cipher text (indistinguishable from random). An empty file is a database about to be created. */
+export function dbState(path: string): DbState {
+  if (!existsSync(path)) return "missing"
+  const fd = openSync(path, "r")
+  try {
+    const buf = Buffer.alloc(16)
+    const n = readSync(fd, buf, 0, 16, 0)
+    if (n === 0) return "empty"
+    return n === 16 && buf.toString("latin1") === SQLITE_MAGIC ? "plain" : "encrypted"
+  } finally {
+    closeSync(fd)
+  }
+}
+
+// ---- keys -------------------------------------------------------------------------------------------------------------------
+
+const KEY_RE = /^[A-Za-z0-9+/=_.~-]{32,512}$/
+
+/** Trims and checks a key; the error never contains the key. */
+export function validateKey(raw: string, where = "the key"): string {
+  const k = raw.trim()
+  if (!KEY_RE.test(k)) throw new OrgKeyError(`${where} is not usable: it must be 32 to 512 characters from A-Z a-z 0-9 + / = _ . ~ - (a generated key is 64 hex characters: use \`brain org admin init --encrypt\`)`)
+  return k
+}
+
+/** 256 random bits, as 64 hex characters. */
+export const generateKey = (): string => randomBytes(32).toString("hex")
+
+/** Writes a new key file (mode 0600, refuses to overwrite). */
+export function writeKeyFile(path: string, key: string): void {
+  writeFileSync(path, `${key}\n`, { mode: 0o600, flag: "wx" })
+  try {
+    chmodSync(path, 0o600)
+  } catch {}
+}
+
+/** Reads a key file. A file others can read is reported through `warning` (secret mounts are often 0444: a warning, not a refusal). */
+export function readKeyFile(path: string): { key: string; warning?: string } {
+  let text: string
+  try {
+    text = readFileSync(path, "utf8")
+  } catch {
+    throw new OrgKeyError(`cannot read the key file ${path}`)
+  }
+  const key = validateKey(text, `the key in ${path}`)
+  let warning: string | undefined
+  try {
+    if (statSync(path).mode & 0o044) warning = `the key file ${path} is readable by other users: restrict it (chmod 600)`
+  } catch {}
+  return { key, ...(warning ? { warning } : {}) }
+}
+
+export interface KeySource {
+  /** The key itself (for programs: tests, an embedding application). */
+  key?: string
+  keyFile?: string
+  env?: Record<string, string | undefined>
+}
+/** The key from, in order: `key`, `keyFile`, `BRAIN_ORG_KEY`. A key file that cannot be read is an error, never a silent fallback. */
+export function resolveKey(s: KeySource = {}): { key?: string; from?: "option" | "file" | "env"; warning?: string } {
+  if (s.key !== undefined) return { key: validateKey(s.key, "the key"), from: "option" }
+  if (s.keyFile) {
+    const r = readKeyFile(s.keyFile)
+    return { key: r.key, from: "file", ...(r.warning ? { warning: r.warning } : {}) }
+  }
+  const e = (s.env ?? process.env).BRAIN_ORG_KEY
+  if (e !== undefined && e.trim() !== "") return { key: validateKey(e, "BRAIN_ORG_KEY"), from: "env" }
+  return {}
+}
+
+// ---- libSQL -----------------------------------------------------------------------------------------------------------------
+
+interface RawStmt {
+  all(...p: unknown[]): unknown[]
+  get(...p: unknown[]): unknown
+  run(...p: unknown[]): unknown
+}
+export interface RawHandle {
+  prepare(sql: string): RawStmt
+  exec(sql: string): void
+  close(): void
+}
+type RawCtor = new (path: string, opts?: { encryptionKey?: string }) => RawHandle
+
+const NO_LIBSQL = (tried: string[]): OrgKeyError =>
+  new OrgKeyError(
+    `encryption needs the \`libsql\` package and none was found${tried.length ? ` (looked next to ${tried.join(", ")})` : ""}. Install it where Chitta lives: \`cd <chitta folder> && bun add libsql\` (BRAIN_CHITTA points to that folder), or in brain: \`bun add libsql\`.`,
+  )
+
+let libsqlCache: RawCtor | undefined
+const requireFrom = (file: string): ((id: string) => unknown) => createRequire(file)
+
+/** The libSQL constructor, found next to brain or next to any Chitta that brain would load. */
+export function loadLibsql(): RawCtor {
+  if (libsqlCache) return libsqlCache
+  const tried: string[] = []
+  const places = [import.meta.url, ...chittaCandidates().filter(existsSync)]
+  for (const p of places) {
+    try {
+      const mod = requireFrom(p)("libsql") as { default?: RawCtor } & RawCtor
+      libsqlCache = (mod.default ?? mod) as RawCtor
+      return libsqlCache
+    } catch {
+      tried.push(p === import.meta.url ? "brain" : dirname(p))
+    }
+  }
+  throw NO_LIBSQL(tried)
+}
+
+/** The first Chitta entry file (in brain's usual search order) whose folder can resolve `libsql`: the one to load for an encrypted
+ *  database (Chitta's own switch to libSQL resolves the package from its own folder). Undefined when none can. */
+export function chittaWithLibsql(): string | undefined {
+  for (const file of chittaCandidates()) {
+    if (!existsSync(file)) continue
+    try {
+      requireFrom(file)("libsql")
+      return file
+    } catch {}
+  }
+  return undefined
+}
+
+const isPlainObject = (x: unknown): x is Record<string, unknown> => typeof x === "object" && x !== null && Object.getPrototypeOf(x) === Object.prototype
+
+function bindArgs(p: unknown[]): unknown[] {
+  const conv = (v: unknown) => (typeof v === "boolean" ? (v ? 1 : 0) : v)
+  // bun:sqlite binds `{ $name: v }` to `$name`; libSQL wants the key without the sigil (and silently binds NULL otherwise).
+  if (p.length === 1 && isPlainObject(p[0])) return [Object.fromEntries(Object.entries(p[0]).map(([k, v]) => [/^[$@:]/.test(k) ? k.slice(1) : k, conv(v)]))]
+  return p.map(conv)
+}
+function clean<T>(row: T): T {
+  if (row && typeof row === "object" && "_metadata" in (row as object)) delete (row as Record<string, unknown>)._metadata
+  return row
+}
+
+/** Makes a libSQL-shaped handle behave like the bun:sqlite `Database` the organisation code was written against. */
+export function adapt(raw: { query?: (sql: string) => RawStmt; prepare?: (sql: string) => RawStmt; exec(sql: string): void; close(): void }): Database {
+  const prep = (sql: string): RawStmt => (raw.query ? raw.query(sql) : raw.prepare!(sql))
+  const cache = new Map<string, RawStmt>()
+  const stmt = (sql: string): RawStmt => {
+    let s = cache.get(sql)
+    if (!s) {
+      if (cache.size > 300) cache.clear()
+      s = prep(sql)
+      cache.set(sql, s)
+    }
+    return s
+  }
+  let depth = 0
+  let seq = 0
+  const handle = {
+    query(sql: string) {
+      const s = stmt(sql)
+      return {
+        all: (...p: unknown[]) => (s.all(...bindArgs(p)) as unknown[]).map(clean),
+        get: (...p: unknown[]) => clean(s.get(...bindArgs(p)) ?? null),
+        run: (...p: unknown[]) => s.run(...bindArgs(p)),
+      }
+    },
+    exec: (sql: string) => raw.exec(sql),
+    close: () => raw.close(),
+    /** `db.transaction(fn)` returns a function that runs `fn` in a transaction (a savepoint when already inside one). */
+    transaction<A extends unknown[], R>(fn: (...a: A) => R): (...a: A) => R {
+      return (...a: A): R => {
+        let sp: string | undefined = depth > 0 ? `brain_sp_${++seq}` : undefined
+        if (!sp) {
+          try {
+            raw.exec("BEGIN IMMEDIATE")
+          } catch (e) {
+            if (!/within a transaction/i.test(String((e as Error)?.message))) throw e
+            sp = `brain_sp_${++seq}` // Chitta holds a transaction open on this connection: join it
+            raw.exec(`SAVEPOINT ${sp}`)
+          }
+        } else raw.exec(`SAVEPOINT ${sp}`)
+        depth++
+        try {
+          const r = fn(...a)
+          raw.exec(sp ? `RELEASE ${sp}` : "COMMIT")
+          return r
+        } catch (e) {
+          try {
+            raw.exec(sp ? `ROLLBACK TO ${sp}; RELEASE ${sp}` : "ROLLBACK")
+          } catch {}
+          throw e
+        } finally {
+          depth--
+        }
+      }
+    },
+  }
+  return handle as unknown as Database
+}
+
+/** The handle as is when it already has `transaction` (bun:sqlite), otherwise adapted. */
+export const ensureTransactions = (db: Database): Database => (typeof (db as { transaction?: unknown }).transaction === "function" ? db : adapt(db as never))
+
+const WRONG_KEY = /not a database|file is encrypted|decrypt|malformed|SQLITE_NOTADB|SQLITE_CORRUPT/i
+
+/** Opens the database file with its own connection: plain with bun:sqlite, encrypted with libSQL. Refuses a key on a plain file and
+ *  no key on an encrypted one, each with a message that says what to do. Wrong key: "does not open". */
+export function openSqlite(path: string, key?: string): Database {
+  const state = dbState(path)
+  if (!key) {
+    if (state === "encrypted") throw new OrgKeyError(`${path} is encrypted: give the key with BRAIN_ORG_KEY or --key-file <path>`)
+    return new Database(path, { create: true })
+  }
+  if (state === "plain") throw new OrgKeyError(`${path} is not encrypted but a key was given: convert it with \`brain org admin encrypt --db ${path}\`, or serve it without a key`)
+  const L = loadLibsql()
+  let raw: RawHandle | undefined
+  try {
+    raw = new L(path, { encryptionKey: key })
+    raw.prepare("SELECT count(*) AS c FROM sqlite_master").get()
+  } catch (e) {
+    try {
+      raw?.close()
+    } catch {}
+    if (WRONG_KEY.test(String((e as Error)?.message ?? e))) throw new OrgKeyError(`the key does not open ${path} (wrong key, or the file is damaged)`)
+    throw new OrgKeyError(`cannot open ${path} with the key: ${String((e as Error)?.message ?? e).split("\n")[0]}`)
+  }
+  return adapt(raw as never)
+}
+
+/** Runs `fn` (synchronously) with CONTEXT_DB_KEY set to `key` (or removed when none), then restores the previous value. Chitta reads
+ *  this variable once, when it opens its store; the key must not stay in the environment afterwards. */
+export function withKeyEnv<T>(key: string | undefined, fn: () => T): T {
+  const prev = process.env.CONTEXT_DB_KEY
+  if (key) process.env.CONTEXT_DB_KEY = key
+  else delete process.env.CONTEXT_DB_KEY
+  try {
+    return fn()
+  } finally {
+    if (prev === undefined) delete process.env.CONTEXT_DB_KEY
+    else process.env.CONTEXT_DB_KEY = prev
+  }
+}
+
+// ---- one server per file ----------------------------------------------------------------------------------------------------
+
+const lockPath = (db: string): string => `${db}.lock`
+
+/** Marks the database as served by this process. Not a mutex (a second server is not refused: that is not this file's job); it lets
+ *  `encrypt`, `rekey` and `restore` refuse to replace a file a live server has open. Returns a function that releases it. */
+export function claimServerLock(db: string): () => void {
+  const token = `${process.pid}:${randomBytes(8).toString("hex")}`
+  try {
+    writeFileSync(lockPath(db), `${token}\n`, { mode: 0o600 })
+  } catch {
+    return () => {}
+  }
+  return () => {
+    try {
+      if (readFileSync(lockPath(db), "utf8").trim() === token) unlinkSync(lockPath(db))
+    } catch {}
+  }
+}
+
+/** The pid of a live process serving this database, if there is one (a stale lock of a dead process is ignored). */
+export function liveServer(db: string): number | undefined {
+  try {
+    const pid = Number(readFileSync(lockPath(db), "utf8").split(":")[0])
+    if (!Number.isInteger(pid) || pid <= 0) return undefined
+    process.kill(pid, 0)
+    return pid
+  } catch (e) {
+    return (e as NodeJS.ErrnoException)?.code === "EPERM" ? -1 : undefined
+  }
+}
+
+// ---- converting a database: encrypt, rekey ----------------------------------------------------------------------------------
+
+const qIdent = (s: string): string => `"${s.replaceAll('"', '""')}"`
+
+export interface ConvertResult {
+  /** The previous file, kept next to the database. For `encrypt` it holds PLAIN TEXT until you remove it. */
+  backup: string
+  tables: Record<string, number>
+}
+
+/** Copies every ordinary table (and its indexes, views, triggers) of the database at `path` into a new file encrypted with `newKey`,
+ *  verifies the row counts and an integrity check on the new file, and swaps it into place (the old file becomes `<path>.bak-<time>`).
+ *  `oldKey` is the current key, or undefined for a plain database. Works for Chitta's tables and for every `org_*` table, including
+ *  ones added later: tables are discovered, not listed. Virtual tables (full-text, vector) and their shadow tables are not copied:
+ *  Chitta rebuilds the full-text index when it next opens the store, and this function rebuilds the vector index it needs. */
+export function convertDatabase(o: { path: string; oldKey?: string; newKey: string; now?: () => number }): ConvertResult {
+  const { path } = o
+  const state = dbState(path)
+  if (state === "missing" || state === "empty") throw new OrgKeyError(`no database at ${path}`)
+  if (state === "plain" && o.oldKey) throw new OrgKeyError(`${path} is not encrypted: do not pass a current key`)
+  if (state === "encrypted" && !o.oldKey) throw new OrgKeyError(`${path} is encrypted: give the current key (BRAIN_ORG_KEY or --key-file)`)
+  const live = liveServer(path)
+  if (live !== undefined) throw new OrgKeyError(`a server (pid ${live > 0 ? live : "unknown"}) is running on ${path}: stop it first, then convert`)
+  const newKey = validateKey(o.newKey, "the new key")
+  const L = loadLibsql()
+  const stamp = (o.now ?? Date.now)()
+  const tmp = `${path}.converting-${process.pid}.tmp`
+  const backup = `${path}.bak-${stamp}`
+  for (const f of [tmp, `${tmp}-wal`, `${tmp}-shm`]) rmSync(f, { force: true })
+  closeSync(openSync(tmp, "wx", 0o600))
+
+  const counts: Record<string, number> = {}
+  let src: RawHandle | undefined
+  try {
+    try {
+      src = o.oldKey ? new L(path, { encryptionKey: o.oldKey }) : new L(path)
+      src.prepare("SELECT count(*) FROM sqlite_master").get()
+    } catch (e) {
+      try {
+        src?.close()
+      } catch {}
+      src = undefined
+      throw new OrgKeyError(WRONG_KEY.test(String((e as Error)?.message)) ? `the current key does not open ${path} (wrong key, or the file is damaged)` : `cannot open ${path}: ${String((e as Error)?.message).split("\n")[0]}`)
+    }
+    const S = src
+    const rows = (sql: string, ...p: unknown[]): Array<Record<string, unknown>> => (S.prepare(sql).all(...p) as Array<Record<string, unknown>>).map((r) => clean(r))
+    try {
+      S.exec("PRAGMA busy_timeout = 3000")
+      S.exec("PRAGMA wal_checkpoint(TRUNCATE)")
+    } catch {}
+    S.prepare("ATTACH DATABASE ? AS dst KEY ?").run(tmp, newKey)
+    const master = rows("SELECT type, name, tbl_name, sql FROM main.sqlite_master WHERE sql IS NOT NULL AND name NOT LIKE 'sqlite_%'")
+    const virtual = master.filter((m) => m.type === "table" && /^\s*CREATE\s+VIRTUAL/i.test(String(m.sql))).map((m) => String(m.name))
+    const skipTable = (name: string): boolean => virtual.some((v) => name === v || name.startsWith(`${v}_`)) || name.startsWith("libsql_") || /_idx_shadow$/.test(name) || name === "vec_native"
+    const tables = master.filter((m) => m.type === "table" && !virtual.includes(String(m.name)) && !skipTable(String(m.name)))
+    const copied = new Set(tables.map((t) => String(t.name)))
+    const qualify = (sql: string, kind: "TABLE" | "INDEX" | "VIEW" | "TRIGGER"): string =>
+      sql.replace(new RegExp(`^\\s*CREATE\\s+(UNIQUE\\s+)?${kind}\\s+(IF\\s+NOT\\s+EXISTS\\s+)?`, "i"), (_m, u) => `CREATE ${u ?? ""}${kind} dst.`)
+    S.exec("BEGIN")
+    try {
+      for (const t of tables) S.exec(qualify(String(t.sql), "TABLE"))
+      for (const t of tables) {
+        const name = qIdent(String(t.name))
+        S.exec(`INSERT INTO dst.${name} SELECT * FROM main.${name}`)
+        counts[String(t.name)] = Number((S.prepare(`SELECT count(*) AS c FROM main.${name}`).get() as { c: number }).c)
+      }
+      for (const m of master) {
+        if (m.type === "index" && copied.has(String(m.tbl_name)) && !/libsql_vector_idx/i.test(String(m.sql))) S.exec(qualify(String(m.sql), "INDEX"))
+        else if ((m.type === "view" || m.type === "trigger") && !virtual.includes(String(m.tbl_name))) S.exec(qualify(String(m.sql), m.type === "view" ? "VIEW" : "TRIGGER"))
+      }
+      try {
+        S.exec("DELETE FROM dst.sqlite_sequence")
+        S.exec("INSERT INTO dst.sqlite_sequence SELECT * FROM main.sqlite_sequence")
+      } catch {} // no AUTOINCREMENT tables
+      S.exec("COMMIT")
+    } catch (e) {
+      try {
+        S.exec("ROLLBACK")
+      } catch {}
+      throw e
+    }
+    S.exec("DETACH DATABASE dst")
+    S.close()
+    src = undefined
+
+    // Second step, on the new file alone: verify, and rebuild the vector index Chitta keeps for an encrypted store.
+    const D = new L(tmp, { encryptionKey: newKey })
+    try {
+      if (String((D.prepare("PRAGMA integrity_check").get() as { integrity_check?: string })?.integrity_check ?? "") !== "ok") throw new OrgKeyError("the converted database failed its integrity check; nothing was changed")
+      for (const [name, n] of Object.entries(counts)) {
+        const c = Number((D.prepare(`SELECT count(*) AS c FROM ${qIdent(name)}`).get() as { c: number }).c)
+        if (c !== n) throw new OrgKeyError(`the converted database holds ${c} rows in ${name} instead of ${n}; nothing was changed`)
+      }
+      rebuildNativeVectors(D)
+      for (const pragma of ["PRAGMA journal_mode = WAL", "PRAGMA wal_checkpoint(TRUNCATE)"]) {
+        try {
+          D.exec(pragma) // best effort: closing the connection checkpoints anyway; a log left behind travels with the file below
+        } catch {}
+      }
+    } finally {
+      D.close()
+    }
+  } catch (e) {
+    try {
+      src?.close()
+    } catch {}
+    for (const f of [tmp, `${tmp}-wal`, `${tmp}-shm`]) rmSync(f, { force: true })
+    throw e
+  }
+  // swap: the old file stays as the backup; a stale -wal/-shm of the old file must not be read against the new one
+  renameSync(path, backup)
+  try {
+    if (statSync(`${path}-wal`).size > 0) renameSync(`${path}-wal`, `${backup}-wal`) // the old file's log stays with the old file
+  } catch {}
+  rmSync(`${path}-wal`, { force: true })
+  rmSync(`${path}-shm`, { force: true })
+  rmSync(`${tmp}-shm`, { force: true }) // the shared-memory index is rebuilt on open; a log (if one is left) must stay with its file
+  try {
+    renameSync(`${tmp}-wal`, `${path}-wal`)
+  } catch {}
+  renameSync(tmp, path)
+  try {
+    chmodSync(path, 0o600)
+    chmodSync(backup, 0o600)
+  } catch {}
+  return { backup, tables: counts }
+}
+
+/** Chitta's native vector index for an encrypted store (table `vec_native` + a DiskANN index over it) must hold EVERY chunk, or a
+ *  search that uses it would silently miss the others. Rebuilt here from the chunks' stored embeddings, with Chitta's own DDL. */
+/**
+ * libSQL keeps one row per vector index in `libsql_vector_meta_shadow` (a WITHOUT ROWID table keyed by the index name). `VACUUM INTO`
+ * re-creates each vector index in the copy, which writes that row, and then copies the source's rows of the table as well: the copy
+ * holds the name twice and fails SQLite's integrity check ("row not in PRIMARY KEY order"), so every backup of a store with
+ * vectors was refused. Found on the first real deployment. Keep one row per index name, rewritten in key order.
+ */
+export function dedupeVectorMeta(D: RawHandle): void {
+  if (!D.prepare("SELECT 1 AS x FROM sqlite_master WHERE type = 'table' AND name = 'libsql_vector_meta_shadow'").get()) return
+  const d = D.prepare("SELECT count(*) - count(DISTINCT name) AS d FROM libsql_vector_meta_shadow").get() as { d: number }
+  if (!Number(d.d)) return
+  D.exec("BEGIN")
+  try {
+    D.exec("CREATE TEMP TABLE brain_vector_meta AS SELECT name, metadata FROM libsql_vector_meta_shadow GROUP BY name")
+    D.exec("DELETE FROM libsql_vector_meta_shadow")
+    D.exec("INSERT INTO libsql_vector_meta_shadow (name, metadata) SELECT name, metadata FROM brain_vector_meta ORDER BY name")
+    D.exec("DROP TABLE brain_vector_meta")
+    D.exec("COMMIT")
+  } catch (e) {
+    try {
+      D.exec("ROLLBACK")
+    } catch {}
+    throw e
+  }
+}
+
+function rebuildNativeVectors(D: RawHandle): void {
+  try {
+    const first = D.prepare("SELECT embedding FROM chunks WHERE embedding IS NOT NULL LIMIT 1").get() as { embedding?: ArrayBuffer | Uint8Array | string } | undefined
+    if (!first?.embedding) return
+    const bytes = typeof first.embedding === "string" ? null : first.embedding.byteLength
+    if (bytes === null) return // legacy JSON embeddings: Chitta rewrites them itself; brute force serves until then
+    const dim = bytes / 4
+    D.exec("DROP INDEX IF EXISTS vec_native_idx")
+    D.exec("DROP TABLE IF EXISTS vec_native")
+    D.exec(`CREATE TABLE vec_native (rowid INTEGER PRIMARY KEY, embedding F32_BLOB(${dim}))`)
+    const ins = D.prepare("INSERT INTO vec_native (rowid, embedding) VALUES (?, vector32(?))")
+    const all = D.prepare("SELECT rowid AS r, embedding AS e FROM chunks WHERE embedding IS NOT NULL").all() as Array<{ r: number; e: ArrayBuffer | Uint8Array }>
+    D.exec("BEGIN")
+    for (const c of all) {
+      const u8 = c.e instanceof Uint8Array ? c.e : new Uint8Array(c.e)
+      if (u8.byteLength !== dim * 4) continue
+      const f = new Float32Array(u8.slice().buffer)
+      ins.run(Math.trunc(c.r), JSON.stringify(Array.from(f)))
+    }
+    D.exec("COMMIT")
+    D.exec("CREATE INDEX vec_native_idx ON vec_native (libsql_vector_idx(embedding, 'metric=cosine'))")
+  } catch {
+    try {
+      D.exec("ROLLBACK")
+    } catch {}
+    try {
+      D.exec("DROP INDEX IF EXISTS vec_native_idx")
+      D.exec("DROP TABLE IF EXISTS vec_native")
+    } catch {} // brute force serves (Chitta falls back by itself)
+  }
+}
+
+/** `VACUUM INTO` copies libSQL's vector index tables in a state its own integrity check rejects ("row not in PRIMARY KEY order for
+ *  libsql_vector_meta_shadow"). The copy is rebuilt here, from the chunks, so a backup restores to an index that is consistent. */
+export function rebuildVectorIndexAt(path: string, key: string): void {
+  const L = loadLibsql()
+  const D = new L(path, { encryptionKey: key })
+  try {
+    dedupeVectorMeta(D)
+    if (D.prepare("SELECT 1 AS x FROM sqlite_master WHERE name = 'vec_native'").get()) rebuildNativeVectors(D)
+  } finally {
+    D.close()
+  }
+}
+
+/** Best-effort removal of a file that held plain text: overwritten with zeros, then unlinked. On SSDs, copy-on-write and journaling
+ *  file systems this does NOT guarantee the old bytes are gone: delete the disk or snapshot if that matters. */
+export function wipeFile(path: string): void {
+  try {
+    const size = statSync(path).size
+    const fd = openSync(path, "r+")
+    try {
+      const zero = Buffer.alloc(1 << 20)
+      for (let off = 0; off < size; off += zero.length) {
+        const n = Math.min(zero.length, size - off)
+        writeSync(fd, zero, 0, n, off)
+      }
+    } finally {
+      closeSync(fd)
+    }
+  } catch {}
+  rmSync(path, { force: true })
+}