beatrina 0.8.6

package/host/planes/ai-store.mjs

@@ -0,0 +1,327 @@
+// planes/ai-store.mjs — the FIFTH allow-list: AI conversations.
+//
+// A transcription of spike/ai-store.R. Page-only in BOTH senses — AGENT_REFUSED
+// because an agent must not read the user's conversations, and
+// PAGE_ONLY_CLASSES because AGENT_REFUSED alone is an honour system.
+//
+// WHY NOT localStorage, WHICH IS WHERE THIS LIVED. Every `file://` page shares
+// ONE origin. A conversation holds full prompts — the user's question, the chunk
+// sources it carried, the model's prose — so keeping them in that origin means
+// any local HTML file the user ever opens can read every question they have
+// asked about their data. Same argument that moved console history here, one
+// notch stronger: a transcript is what you typed; a conversation is what you
+// typed PLUS what the notebook told the model about your data.
+//
+// WHY THE HOST AND NOT THE WORKER. The file ops' and the journal's reason:
+// writing a few kilobytes is not evaluation, and it must never queue behind a
+// forty-second model fit — which is exactly when the AI is being used.
+//
+// ── the on-disk shape ──────────────────────────────────────────────────────
+//
+//   <root>/<document key, ':' -> '__'>/<conversation id>.json
+//
+// One file per conversation, replaced atomically, rather than the journal's
+// append-only NDJSON. The journal is append-only because it has a contiguity
+// invariant to defend; a conversation has none — it is small, rewritten in
+// place, and an interrupted write must leave the PREVIOUS one intact rather
+// than a half-grown one. Write a sibling, rename over.
+//
+// WHAT IS NOT PROMISED: there is no fsync, so a power cut between the write and
+// the operating system's flush can lose the newest turn. The previous state
+// survives, because the rename either happened or did not.
+
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import { privateModeHeld, stateDir } from "../user-dirs.mjs";
+import { PAGE_ONLY_CLASSES } from "../server.mjs";
+
+export const AI_STORE_OPS = Object.freeze(["ai_list", "ai_read", "ai_write", "ai_delete", "ai_purge", "ai_keys"]);
+
+// The SAME key shape the journal uses, deliberately: a conversation belongs to
+// the document it was had about, and two stores disagreeing about what a
+// document is called is how a conversation ends up filed under a document that
+// no longer exists. `.` is in the character class (a conversation id may carry
+// one), so `.` and `..` MATCH these patterns as written — refused explicitly
+// instead. Neither is exploitable as it stands, because the directory welds the
+// key to a prefix and the path appends `.json` to the id; depending on a suffix
+// for a traversal defence is the kind of accident that survives exactly until
+// someone changes the extension.
+const AI_DOTS_RE = /(^|\/)\.{1,2}($|\/)/;
+const AI_KEY_RE = /^[a-z]+:[A-Za-z0-9._~-]{1,200}$/;
+const AI_ID_RE = /^[A-Za-z0-9._~-]{1,120}$/;
+
+// Caps, refused rather than truncated. A truncated conversation is one that
+// silently lies about what was said.
+const AI_MAX_BYTES = 2 * 1024 * 1024;       // one conversation
+const AI_MAX_PER_DOCUMENT = 200;            // conversations kept per document
+const AI_MAX_REPLY_BYTES = 6e6;             // under the 8 MB frame cap, with room
+
+const envOf = (name, unset = "") => (process.env[name] == null ? unset : String(process.env[name]));
+const bytesOf = (text) => Buffer.byteLength(text, "utf8");
+const withCommas = (n) => String(n).replace(/\B(?=(\d{3})+(?!\d))/g, ",");
+
+export function createAiConversationStore() {
+  const enabled = envOf("CARMAR_NO_AI_STORE") !== "1";
+  const root = () => envOf("CARMAR_AI_STORE_DIR") || path.join(stateDir(), "ai");
+  const validKey = (key) => typeof key === "string" && AI_KEY_RE.test(key) && !AI_DOTS_RE.test(key.replace(/^[a-z]+:/, ""));
+  const validId = (id) => typeof id === "string" && AI_ID_RE.test(id) && !AI_DOTS_RE.test(id);
+  const dirOf = (key) => path.join(root(), key.replace(":", "__"));
+  const pathOf = (key, id) => path.join(dirOf(key), `${id}.json`);
+
+  /**
+   * The second lock after the regex, and it is not redundant. The regex forbids
+   * `/` and `..`, so a traversal cannot be spelled. It cannot see a SYMLINK,
+   * which is a path that resolves out of the store while looking like a name.
+   *
+   * `realpath` only resolves a path that EXISTS; one about to be created comes
+   * back with its symlinks still in it. So both sides of a containment test
+   * must be resolved the SAME way or they cannot be compared — on macOS the
+   * store lives under /var, which IS a symlink to /private/var, and a root that
+   * happened to exist while the file did not made every legitimate write look
+   * like an escape. Resolve what exists and re-attach the rest.
+   */
+  const resolve = (p) => {
+    const keep = [];
+    let probe = path.resolve(p);
+    while (!fs.existsSync(probe)) {
+      const parent = path.dirname(probe);
+      if (parent === probe) return path.resolve(p);
+      keep.unshift(path.basename(probe));
+      probe = parent;
+    }
+    let full;
+    try { full = fs.realpathSync(probe); } catch { full = probe; }
+    return keep.length ? path.join(full, ...keep) : full;
+  };
+  const contained = (p) => {
+    const base = resolve(root());
+    const full = resolve(p);
+    return full === base || full.startsWith(base + path.sep);
+  };
+
+  const ensureDir = (p) => {
+    if (!fs.existsSync(p)) {
+      fs.mkdirSync(p, { recursive: true });
+      if (!fs.existsSync(p)) throw new Error("could not create the conversation directory");
+    }
+    fs.chmodSync(p, 0o700);
+    return p;
+  };
+  const err = (type, id, code, message, extra = {}) => ({ type, id, error: message, code, ...extra });
+
+  const writePrivate = (file, text) => {
+    fs.writeFileSync(file, Buffer.from(text, "utf8"));
+    fs.chmodSync(file, 0o600);
+    if (!privateModeHeld(file)) throw new Error("could not make the conversation file private");
+  };
+  /** Atomic replace within the same directory: write a sibling, rename over. */
+  const replace = (file, text) => {
+    const temp = path.join(path.dirname(file), `.carmar-ai-${crypto.randomBytes(8).toString("hex")}`);
+    writePrivate(temp, text);
+    try { fs.renameSync(temp, file); } catch {
+      try { fs.unlinkSync(temp); } catch { /* gone */ }
+      throw new Error("could not replace the conversation file atomically; the previous one was left unchanged");
+    }
+  };
+
+  /**
+   * Read one conversation's raw JSON TEXT, never a parsed object. The page
+   * wrote this JSON and the page is the only thing that reads it, so parsing it
+   * here would buy nothing and cost the fidelity relay_frame's lesson names. It
+   * stays TEXT the whole way.
+   */
+  const readText = (file) => {
+    if (!fs.existsSync(file)) return null;
+    let size;
+    try { size = fs.statSync(file).size; } catch { return null; }
+    if (!Number.isFinite(size) || size === 0) return null;
+    const raw = fs.readFileSync(file);
+    if (raw.includes(0)) return null;                 // an embedded NUL is corruption
+    return raw.toString("utf8");
+  };
+
+  /**
+   * Everything the store knows about one document's conversations. `updated`
+   * comes from the FILE's mtime rather than from a field inside it: the page's
+   * own clock is the page's, and a listing that sorts by a number the page
+   * chose can be reordered by a page with a wrong clock.
+   */
+  const listDir = (key) => {
+    const dir = dirOf(key);
+    if (!fs.existsSync(dir) || !contained(dir)) return [];
+    const files = fs.readdirSync(dir).filter((n) => n.endsWith(".json")).map((n) => path.join(dir, n));
+    if (!files.length) return [];
+    const rows = files.map((f) => {
+      const st = fs.statSync(f);
+      return { id: path.basename(f).replace(/\.json$/, ""), key, bytes: st.size, updated: st.mtimeMs };
+    });
+    return rows.sort((a, b) => b.updated - a.updated);
+  };
+
+  /** Every document key the store holds, for an unfiltered listing. */
+  const keys = () => {
+    const r = root();
+    if (!fs.existsSync(r)) return [];
+    return fs.readdirSync(r, { withFileTypes: true }).filter((d) => d.isDirectory())
+      .map((d) => d.name.replace("__", ":"))
+      .filter(validKey)
+      // A directory inside the store that resolves OUT of it is not this
+      // store's, and every other op already refuses it. Listing it anyway would
+      // put a name in front of the user that nothing else will open.
+      .filter((k) => contained(dirOf(k)));
+  };
+
+  /**
+   * Drop the oldest conversations past the per-document cap. Oldest by FILE
+   * mtime, and only ever the tail: the cap exists so a store does not grow
+   * without bound, not to decide which conversation matters.
+   */
+  const trim = (key) => {
+    const rows = listDir(key);
+    if (rows.length <= AI_MAX_PER_DOCUMENT) return 0;
+    let removed = 0;
+    for (const row of rows.slice(AI_MAX_PER_DOCUMENT)) {
+      const p = pathOf(key, row.id);
+      if (contained(p) && fs.existsSync(p)) { fs.unlinkSync(p); removed += 1; }
+    }
+    return removed;
+  };
+
+  const answer = (cmd) => {
+    const { type, id } = cmd;
+    if (!enabled) {
+      return err(type, id, "disabled", "Conversation storage is switched off on this machine (CARMAR_NO_AI_STORE=1).");
+    }
+    // Every op but `ai_keys` names a document, and the key is checked BEFORE
+    // the file system is touched — the journal's rule, for the same reason.
+    if (type === "ai_keys") return { type, id, keys: keys() };
+
+    const key = cmd.key;
+    if (!validKey(key)) return err(type, id, "bad_key", "That is not a document key this store recognises.");
+
+    if (type === "ai_list") return { type, id, key, conversations: listDir(key) };
+
+    if (type === "ai_read") {
+      if (!validId(cmd.conversation)) return err(type, id, "bad_id", "That is not a conversation id.");
+      const p = pathOf(key, cmd.conversation);
+      if (!contained(p)) return err(type, id, "refused", "That path is outside the conversation store.");
+      const text = readText(p);
+      if (text == null) return err(type, id, "missing", "That conversation is not in the store.", { conversation: cmd.conversation });
+      if (bytesOf(text) > AI_MAX_REPLY_BYTES) {
+        return err(type, id, "too_large", "That conversation is too large to send in one frame.",
+          { conversation: cmd.conversation, bytes: bytesOf(text) });
+      }
+      return { type, id, key, conversation: cmd.conversation, record: text };
+    }
+
+    if (type === "ai_write") {
+      if (!validId(cmd.conversation)) return err(type, id, "bad_id", "That is not a conversation id.");
+      const text = cmd.record;
+      if (typeof text !== "string") {
+        return err(type, id, "bad_record", "A conversation travels as one JSON string, not as an object.");
+      }
+      const bytes = bytesOf(text);
+      if (bytes > AI_MAX_BYTES) {
+        // Refused, not truncated: a truncated conversation is one that silently
+        // lies about what was said.
+        return err(type, id, "too_large",
+          `That conversation is ${withCommas(bytes)} bytes; the limit is ${withCommas(AI_MAX_BYTES)}. Nothing was written.`,
+          { conversation: cmd.conversation, bytes, max: AI_MAX_BYTES });
+      }
+      // It must PARSE before it is kept. The page is the only writer, but a
+      // store whose files are not valid JSON is a store that fails at read
+      // time, on the day someone needs it, with no way back.
+      try { JSON.parse(text); } catch {
+        return err(type, id, "bad_record", "That conversation is not valid JSON; nothing was written.");
+      }
+      const p = pathOf(key, cmd.conversation);
+      if (!contained(p)) return err(type, id, "refused", "That path is outside the conversation store.");
+      try { ensureDir(dirOf(key)); replace(p, text); }
+      catch (e) { return err(type, id, "write_failed", e && e.message ? e.message : String(e)); }
+      return { type, id, key, conversation: cmd.conversation, bytes, trimmed: trim(key) };
+    }
+
+    if (type === "ai_delete") {
+      if (!validId(cmd.conversation)) return err(type, id, "bad_id", "That is not a conversation id.");
+      const p = pathOf(key, cmd.conversation);
+      if (!contained(p)) return err(type, id, "refused", "That path is outside the conversation store.");
+      const existed = fs.existsSync(p);
+      if (existed) fs.unlinkSync(p);
+      return { type, id, key, conversation: cmd.conversation, deleted: existed };
+    }
+
+    if (type === "ai_purge") {
+      // Every conversation for one document, gone. The page states the scope in
+      // words before it asks.
+      const dir = dirOf(key);
+      if (!contained(dir)) return err(type, id, "refused", "That path is outside the conversation store.");
+      const rows = listDir(key);
+      // `only` NAMES the conversations to remove. One directory holds two kinds
+      // of record — chats and agent sessions — and nothing in the file tells
+      // them apart, so a directory-wide unlink answering "Clear chat history"
+      // took the agent sessions with it. The caller knows which are its own; it
+      // says so. Each id is validated and confined on its own: a list from the
+      // wire is a list of requests, not a list of facts.
+      const only = cmd.only;
+      if (only != null) {
+        const ids = Array.isArray(only) ? only.map(String) : [String(only)];
+        let removed = 0;
+        for (const one of ids) {
+          if (!validId(one)) continue;
+          const p = pathOf(key, one);
+          if (contained(p) && fs.existsSync(p)) { fs.unlinkSync(p); removed += 1; }
+        }
+        return { type, id, key, deleted: removed };
+      }
+      if (fs.existsSync(dir)) fs.rmSync(dir, { recursive: true, force: true });
+      return { type, id, key, deleted: rows.length };
+    }
+
+    return err(type, id, "unknown_op", `The conversation store has no op named ${type}.`);
+  };
+
+  /**
+   * The guard the journal has, and which this allow-list did not inherit when
+   * it was built by analogy. It matters MORE here: the send is best-effort, so
+   * a condition raised while BUILDING the frame means NO FRAME IS SENT AT ALL —
+   * the page's promise never settles and the pane waits out its timeout. A
+   * refusal is a reply; silence is a hang.
+   */
+  const answerGuarded = (cmd) => {
+    try {
+      const frame = answer(cmd);
+      if (!frame || typeof frame !== "object") return err(cmd.type, cmd.id, "failed", "The store produced no reply.");
+      return frame;
+    } catch (e) { return err(cmd.type, cmd.id, "failed", e && e.message ? e.message : String(e)); }
+  };
+
+  return { enabled, root, dirOf, pathOf, validKey, validId, contained, listDir, keys, answer, answerGuarded };
+}
+
+export function createPlane({ audit } = {}) {
+  const store = createAiConversationStore();
+  return {
+    name: "ai-store",
+    audits: true,          // the audit line carries the document key; see server.mjs
+    ops: AI_STORE_OPS,
+    agentRefused: AI_STORE_OPS,
+    commands: AI_STORE_OPS,
+    store,
+    handle(cmd, rec, ctx) {
+      if (typeof cmd.id !== "string" || !cmd.id.length) return undefined;
+      if (!PAGE_ONLY_CLASSES.includes(rec.class)) {
+        audit?.("ai-store-refused", { detail: cmd.type, class: rec.class || "unknown" });
+        rec.ws.send(ctx.enc({ type: cmd.type, id: cmd.id, code: "refused",
+          error: "AI conversations are available to the notebook page only." }));
+        return undefined;
+      }
+      // The key is logged, never the record: an audit line naming what was
+      // asked would put the conversation in the log the store exists to keep it
+      // out of.
+      audit?.(cmd.type, { id: cmd.id, key: typeof cmd.key === "string" ? cmd.key : "" });
+      rec.ws.send(ctx.enc(store.answerGuarded(cmd)));
+      return undefined;
+    },
+  };
+}