@100xprompt/brain 0.0.0-stage → 0.1.1

package/src/memory/types.ts

@@ -0,0 +1,118 @@
+// brain memory: spend fewer tokens when a session moves tools, restarts or grows.
+//
+// Measured on nine real sessions (562K tokens): tool results 49%, tool calls 31%, file edits 7%, assistant prose 7%,
+// the person's messages 3%, compaction summaries 3%. So what you and the model said is about a tenth of a long session;
+// the rest is tool input and output. Two mechanisms, both zero-model:
+//
+//   1. externalize: a big tool call/result is replaced by a one-line digest and a pointer; the full text goes to a blob
+//      store on disk and comes back on demand (`brain_expand`). Standalone: needs no Chitta.
+//   2. memory: what the session established (decisions, constraints, preferences, files, failed approaches, commands) is
+//      distilled into small items and stored in the brain's OWN Chitta store, per project; a new session in any tool is
+//      started with a budgeted handful of them and can ask for more (`brain_recall`). Chitta is optional: without it
+//      memory is unavailable and everything else still works.
+//
+// Hard rules: the brain never writes to the person's personal Chitta store (CONTEXT_DB / ~/.local/share/100xprompt);
+// its store is `<home>/.brain/memory/context.db`. Nothing here calls a model. Secrets are filtered before anything is stored.
+
+import type { Project, Session } from "../session/types.ts"
+
+/** Rough token count (chars / 4), the same estimate the measurements use. */
+export const approxTokens = (text: string): number => Math.ceil(text.length / 4)
+
+// ---- externalize --------------------------------------------------------------------------------------------------
+
+export interface BlobRef {
+  /** Content hash (short, stable): the same text always gets the same id. */
+  id: string
+  chars: number
+}
+
+/** Full texts that were replaced by digests in a session, on disk under `<home>/.brain/blobs/`. */
+export interface BlobStore {
+  dir: string
+  /** Stores the text (idempotent: same text, same id) and returns its reference. */
+  put(text: string, meta?: { tool?: string; project?: string }): BlobRef
+  /** The full text, or undefined when it was never stored or has been removed. */
+  get(id: string): string | undefined
+  /** The tool kind recorded when the blob was stored (read, shell, search, ...), when known. */
+  kindOf?(id: string): string | undefined
+  /** Removes blobs not used since `olderThanMs`; returns how many were removed. */
+  prune(olderThanMs: number): number
+}
+
+export interface ExternalizeOptions {
+  /** Tool calls/results shorter than this stay as they are. Default 1500 characters. */
+  minChars?: number
+  /** The newest this-many tool turns are never replaced (the model is most likely to need them). Default 8. */
+  keepRecent?: number
+}
+
+export interface ExternalizeResult {
+  /** A copy of the session with big tool calls/results replaced by digest stubs. The input is not modified. */
+  session: Session
+  /** How many turns were replaced. */
+  replaced: number
+  /** Characters before and after, over the whole session. */
+  before: number
+  after: number
+}
+
+/** The stub text written in place of a big tool result: starts with this so a reader can tell it is one. */
+export const STUB_PREFIX = "[brain: output stored"
+
+// ---- memory -------------------------------------------------------------------------------------------------------
+
+export type MemoryKind = "decision" | "constraint" | "preference" | "failed" | "file" | "command" | "summary" | "state"
+
+/** A project: its remote when it has one (so a clone anywhere shares memory), else its folder. Stable across tools. */
+export interface ProjectKey {
+  id: string
+  label: string
+}
+
+export interface MemoryItem {
+  /** Stable: re-distilling the same session gives the same ids, so re-ingesting updates rather than duplicates. */
+  id: string
+  kind: MemoryKind
+  /** A short title (one line). */
+  name: string
+  /** The memory itself, self-contained, at most a few hundred characters. */
+  text: string
+  /** Which session it came from (portable id), for provenance. */
+  source: string
+  occurredAt?: string
+  entities?: Array<{ name: string; type?: string }>
+  relations?: Array<{ from: string; to: string; type: string }>
+  procedures?: Array<{ trigger?: string; action: string }>
+}
+
+export interface Recalled {
+  text: string
+  kind?: MemoryKind
+  name?: string
+  score: number
+  source?: string
+  /** Set by the organisation memory (src/org): who may see the item and whose it is. */
+  scope?: "private" | "project" | "team" | "org"
+  owner?: string
+}
+
+export interface MemoryStatus {
+  available: boolean
+  /** Why it is unavailable, when it is ("Chitta is not installed", ...). */
+  why?: string
+  store?: string
+  items?: number
+}
+
+export interface Memory {
+  status(): Promise<MemoryStatus>
+  /** `opts.scope`/`opts.team` are used by the organisation memory (src/org); the local memory ignores them (everything local is private to the person). */
+  ingest(project: ProjectKey, items: MemoryItem[], opts?: { scope?: "private" | "project" | "team" | "org"; team?: string }): Promise<{ stored: number }>
+  /** The best matches for a question within a project, at most `limit` items and `maxTokens` tokens in total. */
+  recall(project: ProjectKey, query: string, opts?: { limit?: number; maxTokens?: number }): Promise<Recalled[]>
+  /** Removes a project's memories (all of them, or those from one session). Returns how many. */
+  forget(project: ProjectKey, opts?: { source?: string }): Promise<number>
+}
+
+export type { Project, Session }

package/src/org/admin-ops.ts

@@ -0,0 +1,96 @@
+// brain org admin: the offline operations on the database file: `init --encrypt`, `encrypt`, `rekey`, `backup`, `restore`. They run on the
+// server's machine, need no login, and never print a key: they print where a key file is, and that is all. cli.ts routes to here.
+import { existsSync } from "node:fs"
+import { resolve } from "node:path"
+import { backupDatabase, restoreDatabase } from "./backup.ts"
+import { OrgKeyError, convertDatabase, dbState, generateKey, readKeyFile, resolveKey, wipeFile, writeKeyFile } from "./crypto.ts"
+
+type Out = (line: string) => void
+interface Args {
+  flags: Record<string, string>
+  bools: Set<string>
+}
+
+/** The key for the encrypted database `init --encrypt` / `encrypt` is about to create: an existing key file is used as it is, a missing one
+ *  is created with 256 random bits (mode 0600, never overwriting), and without `--key-file` the key comes from BRAIN_ORG_KEY, or a new
+ *  file `<db>.key` is made. Says where the key is, never what it is. */
+export function keyForNewDatabase(db: string, keyFileFlag: string | undefined, out: Out): string {
+  if (keyFileFlag) {
+    const f = resolve(keyFileFlag)
+    if (existsSync(f)) {
+      const r = readKeyFile(f)
+      if (r.warning) out(`warning: ${r.warning}`)
+      out(`using the existing key file ${f}`)
+      return r.key
+    }
+    const key = generateKey()
+    writeKeyFile(f, key)
+    out(`a new random 256-bit key was written to ${f} (mode 0600). Keep a copy somewhere safe and separate: without this key the database cannot be read, by anyone, ever.`)
+    return key
+  }
+  const env = resolveKey()
+  if (env.key) {
+    out("using the key from BRAIN_ORG_KEY")
+    return env.key
+  }
+  return keyForNewDatabase(db, `${db}.key`, out)
+}
+
+/** The current key of an existing database: `--key-file`, else BRAIN_ORG_KEY; none for a plain database. */
+function currentKey(a: Args, out: Out): string | undefined {
+  const r = resolveKey({ keyFile: a.flags["key-file"] ? resolve(a.flags["key-file"]) : undefined })
+  if (r.warning) out(`warning: ${r.warning}`)
+  return r.key
+}
+
+export async function adminOps(verb: "encrypt" | "rekey" | "backup" | "restore", a: Args, out: Out): Promise<number> {
+  try {
+    const f = a.flags
+    if (verb === "encrypt") {
+      if (!f.db) return (out("usage: brain org admin encrypt --db <path> [--key-file <path>] [--remove-backup]"), 2)
+      const db = resolve(f.db)
+      if (dbState(db) === "missing") return (out(`no database at ${db}`), 1)
+      if (dbState(db) === "encrypted") return (out(`${db} is already encrypted. To change its key use: brain org admin rekey --db ${db} --key-file <current key file> --new-key-file <new key file>`), 1)
+      const key = keyForNewDatabase(db, f["key-file"], out)
+      const r = convertDatabase({ path: db, newKey: key })
+      out(`${db} is now encrypted (${Object.keys(r.tables).length} tables copied, row counts and integrity verified).`)
+      if (a.bools.has("remove-backup")) {
+        wipeFile(r.backup)
+        out(`the plain-text copy ${r.backup} was overwritten and removed (best effort: not a guarantee on SSDs or copy-on-write file systems).`)
+      } else {
+        out(`the previous, UNENCRYPTED file is kept as ${r.backup}: once you have started the server and checked it works, remove it (brain org admin encrypt --remove-backup does that at conversion time), and remember copies of it elsewhere (snapshots, earlier backups) are still plain text.`)
+      }
+      out("start the server with the key: BRAIN_ORG_KEY=... or --key-file <path>")
+      return 0
+    }
+    if (verb === "rekey") {
+      if (!f.db || !f["new-key-file"]) return (out("usage: brain org admin rekey --db <path> --key-file <current key file> --new-key-file <new key file>   (the current key may also be in BRAIN_ORG_KEY; the new file is created if missing)"), 2)
+      const db = resolve(f.db)
+      if (dbState(db) === "missing") return (out(`no database at ${db}`), 1)
+      if (dbState(db) !== "encrypted") return (out(`${db} is not encrypted: use \`brain org admin encrypt --db ${db}\``), 1)
+      const old = currentKey(a, out)
+      if (!old) return (out("give the current key with --key-file or BRAIN_ORG_KEY"), 2)
+      if (f["key-file"] && resolve(f["key-file"]) === resolve(f["new-key-file"])) return (out("--new-key-file must be a different file from --key-file"), 2)
+      const key = keyForNewDatabase(db, f["new-key-file"], out)
+      if (key === old) return (out("the new key is the same as the current one: nothing to do"), 1)
+      const r = convertDatabase({ path: db, oldKey: old, newKey: key })
+      out(`${db} is now encrypted with the new key (${Object.keys(r.tables).length} tables copied, row counts and integrity verified).`)
+      out(`the previous file (still encrypted with the OLD key) is kept as ${r.backup}. Backups made earlier also need the old key. Remove them once you no longer need them, and put the new key where the server reads it before starting it.`)
+      return 0
+    }
+    if (verb === "backup") {
+      if (!f.db || !f.out) return (out("usage: brain org admin backup --db <path> --out <file> [--key-file <path>]   (an encrypted database is backed up encrypted with the same key)"), 2)
+      const r = backupDatabase({ db: f.db, out: f.out, key: currentKey(a, out) })
+      out(`backup written to ${r.out} (${r.bytes} bytes, ${r.items} items, ${r.encrypted ? "encrypted with the database's key" : "NOT encrypted: protect it like the database"}), mode 0600, verified.`)
+      return 0
+    }
+    // restore
+    if (!f.from || !f.db) return (out("usage: brain org admin restore --from <backup file> --db <path> [--key-file <path>] [--force]   (stop the server first)"), 2)
+    const r = restoreDatabase({ from: f.from, db: f.db, key: currentKey(a, out), force: a.bools.has("force") })
+    out(`restored ${r.items} items into ${r.db}${r.movedAside ? `; the previous database was moved to ${r.movedAside}` : ""}. Start the server with the key the backup was made with.`)
+    out("everything after the backup is gone from this database: tokens created or revoked since, and items added or forgotten since.")
+    return 0
+  } catch (e) {
+    return (out(`error: ${e instanceof OrgKeyError ? e.message : String((e as Error)?.message ?? e).split("\n")[0]}`), 1)
+  }
+}

package/src/org/auth.ts

@@ -0,0 +1,112 @@
+// brain org: authentication primitives for the organisation server. Pure functions and two small limiters; no I/O except
+// randomness. Nothing here ever logs, returns or stores a token: a token is generated here, handed to the caller ONCE, and
+// from then on only its SHA-256 is kept.
+//
+// Token format:  brn_<16 hex id>_<43 base64url chars = 256 random bits>
+//   * the id is a public handle (listing, revoking, rate-limit key, audit); knowing it gives no access;
+//   * the secret is 256 bits from the OS CSPRNG, so the stored SHA-256 (no salt, no stretching needed for a high-entropy
+//     secret) cannot be brute forced;
+//   * the server looks the row up by id, then compares SHA-256(candidate) with the stored hash in CONSTANT TIME. For an
+//     unknown id it still performs a comparison against a dummy hash so "no such id" and "wrong secret" cost the same.
+import { createHash, randomBytes, timingSafeEqual } from "node:crypto"
+import type { OrgDb, UserRow } from "./db.ts"
+
+export const TOKEN_PREFIX = "brn_"
+const TOKEN_RE = /^brn_([0-9a-f]{16})_([A-Za-z0-9_-]{43})$/
+
+export const sha256Hex = (s: string): string => createHash("sha256").update(s).digest("hex")
+
+export interface NewToken {
+  id: string
+  /** The secret string to give to the user. Shown once; never stored. */
+  token: string
+  /** What is stored. */
+  hash: string
+}
+
+export function generateToken(): NewToken {
+  const id = randomBytes(8).toString("hex")
+  const secret = randomBytes(32).toString("base64url")
+  const token = `${TOKEN_PREFIX}${id}_${secret}`
+  return { id, token, hash: sha256Hex(token) }
+}
+
+/** The id of a well-formed token, or undefined. */
+export function parseToken(token: string): { id: string } | undefined {
+  const m = TOKEN_RE.exec(token)
+  return m ? { id: m[1]! } : undefined
+}
+
+/** Constant-time equality of two hex SHA-256 strings (both are always 64 chars; anything else is simply unequal). */
+export function hashesEqual(a: string, b: string): boolean {
+  if (a.length !== 64 || b.length !== 64) return false
+  return timingSafeEqual(Buffer.from(a, "utf8"), Buffer.from(b, "utf8"))
+}
+
+const DUMMY_HASH = sha256Hex("brain-org-dummy")
+
+export type AuthFailure = "missing" | "malformed" | "unknown" | "bad_secret" | "revoked" | "expired" | "disabled"
+export type AuthResult = { ok: true; user: UserRow; tokenId: string } | { ok: false; reason: AuthFailure; actor?: string }
+
+/** `Authorization: Bearer <token>` -> the active user it belongs to, or why not. The caller answers every failure with the same 401. */
+export function authenticate(db: OrgDb, header: string | null, nowMs: number): AuthResult {
+  if (!header) return { ok: false, reason: "missing" }
+  const m = /^Bearer ([^\s]+)$/i.exec(header.trim())
+  if (!m) return { ok: false, reason: "malformed" }
+  const token = m[1]!
+  const parsed = parseToken(token)
+  if (!parsed) return { ok: false, reason: "malformed" }
+  const row = db.token(parsed.id)
+  const candidate = sha256Hex(token)
+  if (!row) {
+    hashesEqual(candidate, DUMMY_HASH) // keep the cost of the unknown-id path equal to the wrong-secret path
+    return { ok: false, reason: "unknown" }
+  }
+  if (!hashesEqual(candidate, row.hash)) return { ok: false, reason: "bad_secret" }
+  if (row.revoked_at != null) return { ok: false, reason: "revoked", actor: row.user_id }
+  if (row.expires_at != null && row.expires_at <= nowMs) return { ok: false, reason: "expired", actor: row.user_id }
+  const user = db.user(row.user_id)
+  if (!user || user.disabled) return { ok: false, reason: "disabled", actor: row.user_id }
+  // last-used is bookkeeping for admins; throttled so a busy client does not write on every request
+  if (row.last_used_at == null || nowMs - row.last_used_at > 60_000) db.touchToken(row.id, nowMs)
+  return { ok: true, user, tokenId: row.id }
+}
+
+/** Fixed-window counter per key (a token id, an IP address). Memory is bounded: expired windows are dropped when the map grows. */
+export class WindowLimiter {
+  private windows = new Map<string, { start: number; n: number }>()
+  constructor(
+    private readonly limit: number,
+    private readonly windowMs = 60_000,
+    private readonly maxKeys = 10_000,
+  ) {}
+
+  /** Counts one event; returns false when the key is over its limit for the current window. */
+  hit(key: string, nowMs: number): boolean {
+    const w = this.windows.get(key)
+    if (!w || nowMs - w.start >= this.windowMs) {
+      if (this.windows.size >= this.maxKeys) this.sweep(nowMs)
+      if (this.windows.size >= this.maxKeys) this.windows.clear() // under attack with many keys: forget rather than grow
+      this.windows.set(key, { start: nowMs, n: 1 })
+      return true
+    }
+    w.n++
+    return w.n <= this.limit
+  }
+
+  /** Seconds until the key's window ends. */
+  retryAfter(key: string, nowMs: number): number {
+    const w = this.windows.get(key)
+    return w ? Math.max(1, Math.ceil((w.start + this.windowMs - nowMs) / 1000)) : 1
+  }
+
+  /** True when the key is already over its limit (does not count). */
+  blocked(key: string, nowMs: number): boolean {
+    const w = this.windows.get(key)
+    return !!w && nowMs - w.start < this.windowMs && w.n > this.limit
+  }
+
+  private sweep(nowMs: number) {
+    for (const [k, w] of this.windows) if (nowMs - w.start >= this.windowMs) this.windows.delete(k)
+  }
+}

package/src/org/backup.ts

@@ -0,0 +1,115 @@
+// brain org: backup and restore of the organisation database (one file: Chitta's tables and the `org_*` tables together).
+//
+// BACKUP is `VACUUM INTO`: SQLite writes a consistent snapshot of the database as of one moment, through the same connection layer
+// that reads it, so it is safe while the server is running (readers do not block the writer in WAL mode; the copy sees the
+// state at the moment it started) and it includes whatever is still only in the -wal. From an encrypted database the copy is
+// written encrypted with the SAME key (checked in test/org-crypto.test.ts by reading the copy's bytes); from a plain one it is plain.
+// It refuses to overwrite anything (the output file is created exclusively, mode 0600) and verifies the copy before reporting success.
+//
+// RESTORE (`brain org admin restore`) puts a backup in place as the database: stop the server first (it refuses while a live server's
+// lock file exists), it refuses to replace an existing database unless `--force` (the old file is then moved aside, never deleted),
+// it removes a stale -wal/-shm of the old file (a leftover log applied to a different file would corrupt it), and it checks the backup
+// opens with the key before touching anything. Plain `cp` of the backup file back to the database path, with the server stopped and the
+// old -wal/-shm removed, is equivalent; `restore` just does the checks. The key that opens a backup is the key the database had when
+// the backup was made: after `admin rekey`, older backups still need the OLD key, so keep it until they are gone.
+// A restore brings back everything as of the backup: tokens created or revoked, and items added or forgotten, after it are lost.
+import { chmodSync, closeSync, copyFileSync, existsSync, openSync, renameSync, rmSync, statSync } from "node:fs"
+import { dirname, resolve } from "node:path"
+import { OrgKeyError, dbState, liveServer, openSqlite, rebuildVectorIndexAt, validateKey } from "./crypto.ts"
+
+export interface BackupResult {
+  out: string
+  bytes: number
+  encrypted: boolean
+  items: number
+}
+
+/** Checks that `path` opens (with `key` when encrypted), passes SQLite's integrity check and holds an organisation. */
+function verify(path: string, key: string | undefined): { items: number; org: string } {
+  const db = openSqlite(path, key)
+  try {
+    const ok = (db.query("PRAGMA integrity_check").get() as { integrity_check?: string } | null)?.integrity_check
+    if (ok !== "ok") throw new OrgKeyError(`${path} failed SQLite's integrity check`)
+    const org = db.query("SELECT value FROM org_meta WHERE key = 'org_id'").get() as { value: string } | null
+    if (!org) throw new OrgKeyError(`${path} holds no organisation`)
+    const items = (db.query("SELECT count(*) AS c FROM org_items").get() as { c: number }).c
+    return { items, org: org.value }
+  } finally {
+    db.close()
+  }
+}
+
+const sqlStr = (s: string): string => `'${s.replaceAll("'", "''")}'`
+
+export function backupDatabase(o: { db: string; out: string; key?: string }): BackupResult {
+  const db = resolve(o.db)
+  const out = resolve(o.out)
+  const state = dbState(db)
+  if (state === "missing" || state === "empty") throw new OrgKeyError(`no database at ${db}`)
+  if (state === "encrypted" && !o.key) throw new OrgKeyError(`${db} is encrypted: give the key with BRAIN_ORG_KEY or --key-file <path> (the backup stays encrypted with it)`)
+  if (state === "plain" && o.key) throw new OrgKeyError(`${db} is not encrypted but a key was given`)
+  if (o.key) validateKey(o.key)
+  if (existsSync(out)) throw new OrgKeyError(`${out} already exists; a backup never overwrites a file: pick another name`)
+  if (!existsSync(dirname(out))) throw new OrgKeyError(`the folder ${dirname(out)} does not exist`)
+  const src = openSqlite(db, o.key)
+  // VACUUM INTO refuses a target that exists (so it can never overwrite) and creates it itself: a restrictive umask makes it private from the first byte
+  const oldMask = process.umask(0o077)
+  try {
+    try {
+      src.exec("PRAGMA busy_timeout = 5000")
+    } catch {}
+    src.exec(`VACUUM INTO ${sqlStr(out)}`)
+  } catch (e) {
+    if (!/already exists/i.test(String((e as Error)?.message))) rmSync(out, { force: true }) // never delete a file that was not ours
+    throw e instanceof OrgKeyError ? e : new OrgKeyError(`the backup failed: ${String((e as Error)?.message ?? e).split("\n")[0]}`)
+  } finally {
+    process.umask(oldMask)
+    src.close()
+  }
+  try {
+    chmodSync(out, 0o600)
+    if (o.key) rebuildVectorIndexAt(out, o.key)
+    const v = verify(out, o.key)
+    return { out, bytes: statSync(out).size, encrypted: state === "encrypted", items: v.items }
+  } catch (e) {
+    rmSync(out, { force: true })
+    throw e
+  }
+}
+
+export interface RestoreResult {
+  db: string
+  items: number
+  /** The database that was moved aside (only with --force and an existing database). */
+  movedAside?: string
+}
+
+export function restoreDatabase(o: { from: string; db: string; key?: string; force?: boolean; now?: () => number }): RestoreResult {
+  const from = resolve(o.from)
+  const db = resolve(o.db)
+  if (!existsSync(from)) throw new OrgKeyError(`no backup at ${from}`)
+  if (from === db) throw new OrgKeyError("the backup and the database are the same file")
+  const state = dbState(from)
+  if (state === "empty" || state === "missing") throw new OrgKeyError(`${from} is empty`)
+  if (state === "encrypted" && !o.key) throw new OrgKeyError(`${from} is encrypted: give the key it was made with (BRAIN_ORG_KEY or --key-file)`)
+  if (state === "plain" && o.key) throw new OrgKeyError(`${from} is not encrypted but a key was given`)
+  const live = liveServer(db)
+  if (live !== undefined) throw new OrgKeyError(`a server (pid ${live > 0 ? live : "unknown"}) is running on ${db}: stop it first, then restore`)
+  if (existsSync(db) && !o.force) throw new OrgKeyError(`${db} already exists; restoring would replace it. Move it away, or pass --force to have it moved aside for you`)
+  const v = verify(from, o.key) // before anything is touched
+  if (!existsSync(dirname(db))) throw new OrgKeyError(`the folder ${dirname(db)} does not exist`)
+  const tmp = `${db}.restoring-${process.pid}.tmp`
+  rmSync(tmp, { force: true })
+  closeSync(openSync(tmp, "wx", 0o600))
+  copyFileSync(from, tmp)
+  chmodSync(tmp, 0o600)
+  let movedAside: string | undefined
+  if (existsSync(db)) {
+    movedAside = `${db}.replaced-${(o.now ?? Date.now)()}`
+    renameSync(db, movedAside)
+    for (const s of ["-wal", "-shm"]) if (existsSync(`${db}${s}`)) renameSync(`${db}${s}`, `${movedAside}${s}`) // the log goes with its file
+  }
+  for (const s of ["-wal", "-shm"]) rmSync(`${db}${s}`, { force: true }) // a stale log must never be applied to the restored file
+  renameSync(tmp, db)
+  return { db, items: v.items, ...(movedAside ? { movedAside } : {}) }
+}