beatrina 0.8.6

package/failsafe/ai-store.R

@@ -0,0 +1,373 @@
+# ai-store.R — AI conversations, answered by the SUPERVISOR.
+#
+# THE FIFTH ALLOW-LIST (docs/ai-plan.md D1). `SUPERVISOR_AI_OPS` in serve.R,
+# answered here, page-only in BOTH senses — `AGENT_REFUSED` because an agent
+# must not read the user's conversations, and `PAGE_ONLY_CLASSES` because
+# `AGENT_REFUSED` alone is an honour system (a client that simply never
+# declared itself would sail through it, which is exactly the hole `ai-key`
+# had until 0.60.89).
+#
+# WHY NOT localStorage, WHICH IS WHERE THIS LIVED.
+# Every `file://` page shares ONE origin. A conversation holds full prompts —
+# the user's question, the chunk sources it carried, the model's prose — so
+# keeping them in that origin means any local HTML file the user ever opens can
+# read every question they have asked about their data. That is the same
+# argument that already moved console history here (CLAUDE.md, "Console history
+# is kernel state, not browser state"), and it is stronger here: a console
+# transcript is what you typed, a conversation is what you typed plus what the
+# notebook told the model about your data.
+#
+# WHY THE SUPERVISOR AND NOT THE WORKER. The same reason as the file ops and
+# the journal: writing a few kilobytes is not evaluation, and it must never
+# queue behind a forty-second model fit. Every function here is PURE in
+# fileio.R's sense — takes a request, touches disk, RETURNS the frame — and
+# nothing in this file evaluates, parses or sources anything a user wrote.
+#
+# ── the on-disk shape ──────────────────────────────────────────────────────
+#
+#   <root>/ai/<document key, ':' -> '__'>/
+#     <conversation id>.json     the whole conversation, rewritten per turn
+#
+# One file per conversation, replaced atomically, rather than the journal's
+# append-only NDJSON. The journal is append-only because it has a contiguity
+# invariant to defend and unbounded history to keep; a conversation has neither
+# — it is small, it is rewritten in place, and an interrupted write must leave
+# the PREVIOUS conversation intact rather than a half-grown one. Write a
+# sibling, rename over: the rename is the only atomic primitive this needs.
+#
+# WHAT IS NOT PROMISED, in words: R has no fsync, so a power cut between the
+# write and the operating system's flush can lose the newest turn. The previous
+# state survives, because the rename either happened or did not.
+
+ai_store_root <- function() {
+  root <- Sys.getenv("CARMAR_AI_STORE_DIR", "")
+  if (!nzchar(root)) {
+    state <- Sys.getenv("CARMAR_STATE", "")
+    if (!nzchar(state)) state <- tools::R_user_dir("carmar", "data")
+    root <- file.path(state, "ai")
+  }
+  root
+}
+
+AI_STORE_ENABLED <- !identical(Sys.getenv("CARMAR_NO_AI_STORE", ""), "1")
+
+# The SAME key shape the journal uses, deliberately: a conversation belongs to
+# the document it was had about, and two stores disagreeing about what a
+# document is called is how a conversation ends up filed under a document that
+# no longer exists. `lib/history/durable-document.js` mints both.
+# `.` is in the character class (a conversation id may carry one), so `.` and
+# `..` MATCH these patterns as written — refused explicitly instead. Neither is
+# exploitable as it stands, because `ai_dir` welds the key to a prefix and
+# `ai_path` appends `.json` to the id, so `..` can only ever name a file called
+# `...json`. Depending on a suffix for a traversal defence is the kind of
+# accident that survives exactly until someone changes the extension.
+AI_DOTS_RE <- "(^|/)\\.{1,2}($|/)"
+AI_KEY_RE <- "^[a-z]+:[A-Za-z0-9._~-]{1,200}$"
+AI_ID_RE  <- "^[A-Za-z0-9._~-]{1,120}$"
+
+# Caps, refused rather than truncated. A truncated conversation is a conversation
+# that silently lies about what was said.
+AI_MAX_BYTES <- 2 * 1024 * 1024          # one conversation
+AI_MAX_PER_DOCUMENT <- 200               # conversations kept per document
+AI_MAX_REPLY_BYTES <- 6e6                # under the 8 MB frame cap, with room
+
+ai_valid_key <- function(key) is.character(key) && length(key) == 1L && !is.na(key) &&
+  grepl(AI_KEY_RE, key) && !grepl(AI_DOTS_RE, sub("^[a-z]+:", "", key))
+ai_valid_id  <- function(id)  is.character(id)  && length(id)  == 1L && !is.na(id)  &&
+  grepl(AI_ID_RE, id) && !grepl(AI_DOTS_RE, id)
+
+ai_dir <- function(key) file.path(ai_store_root(), sub(":", "__", key, fixed = TRUE))
+ai_path <- function(key, id) file.path(ai_dir(key), paste0(id, ".json"))
+
+#' The second lock after the regex, and it is not redundant.
+#'
+#' The regex forbids `/` and `..`, so a traversal cannot be spelled. It cannot
+#' see a SYMLINK, which is a path that resolves out of the store while looking
+#' like a name. `plugin_contained()` in spike/plugins.R makes the same check for
+#' the same reason; resolved paths are compared, not the strings handed in.
+#' Resolve what EXISTS of a path and re-attach the rest.
+#'
+#' `normalizePath` only resolves a path that exists; one about to be created
+#' comes back with its symlinks still in it. So the two sides of a containment
+#' test must be resolved the SAME way or they cannot be compared — on macOS the
+#' store lives under /var, which IS a symlink to /private/var, and a root that
+#' happened to exist while the file did not (or the reverse) made every
+#' legitimate write look like an escape. Both sides go through here.
+ai_resolve <- function(path) {
+  keep <- character(); probe <- path
+  while (!file.exists(probe)) {
+    parent <- dirname(probe)
+    if (identical(parent, probe)) return(normalizePath(path, mustWork = FALSE))
+    keep <- c(basename(probe), keep)
+    probe <- parent
+  }
+  full <- normalizePath(probe, mustWork = FALSE)
+  if (length(keep)) full <- do.call(file.path, as.list(c(full, keep)))
+  full
+}
+
+ai_contained <- function(path) {
+  root <- ai_resolve(ai_store_root())
+  full <- ai_resolve(path)
+  identical(full, root) || startsWith(full, paste0(root, .Platform$file.sep))
+}
+
+ai_ensure_dir <- function(path) {
+  if (!dir.exists(path)) {
+    dir.create(path, recursive = TRUE, showWarnings = FALSE)
+    if (!dir.exists(path)) stop("could not create the conversation directory")
+  }
+  Sys.chmod(path, "0700")
+  invisible(path)
+}
+
+ai_error <- function(type, id, code, message, ...) {
+  list(type = type, id = id, error = message, code = code, ...)
+}
+
+ai_write_private <- function(path, text) {
+  con <- file(path, open = "wb")
+  on.exit(close(con), add = TRUE)
+  writeBin(charToRaw(enc2utf8(text)), con)
+  close(con); on.exit()
+  if (!isTRUE(Sys.chmod(path, "0600"))) stop("could not make the conversation file private")
+  invisible(TRUE)
+}
+
+#' Atomic replace within the same directory: write a sibling, rename over.
+ai_replace <- function(path, text) {
+  temp <- tempfile(pattern = ".carmar-ai-", tmpdir = dirname(path))
+  ai_write_private(temp, text)
+  if (!isTRUE(suppressWarnings(file.rename(temp, path)))) {
+    unlink(temp)
+    stop("could not replace the conversation file atomically; the previous one was left unchanged")
+  }
+  invisible(TRUE)
+}
+
+#' Read one conversation's raw JSON TEXT, never a parsed object.
+#'
+#' The page wrote this JSON and the page is the only thing that reads it, so
+#' parsing it here would buy nothing and cost everything relay_frame's lesson
+#' names: `toJSON(auto_unbox = TRUE)` collapses a one-element array, and a
+#' conversation is full of one-element arrays (one turn, one tool call, one
+#' context item). It stays TEXT the whole way — see `ai_frame`.
+ai_read_text <- function(path) {
+  if (!file.exists(path)) return(NULL)
+  size <- file.info(path)$size
+  if (is.na(size) || size == 0) return(NULL)
+  raw <- readBin(path, "raw", n = size)
+  text <- tryCatch(rawToChar(raw), error = function(e) NULL)   # an embedded NUL is corruption
+  if (is.null(text)) return(NULL)
+  Encoding(text) <- "UTF-8"
+  text
+}
+
+#' Everything the store knows about one document's conversations.
+#'
+#' `updated` comes from the FILE's mtime rather than from a field inside it:
+#' the page's own clock is the page's, and a listing that sorts by a number the
+#' page chose can be reordered by a page with a wrong clock.
+ai_list_dir <- function(key) {
+  dir <- ai_dir(key)
+  if (!dir.exists(dir) || !ai_contained(dir)) return(list())
+  files <- list.files(dir, pattern = "\\.json$", full.names = TRUE)
+  if (!length(files)) return(list())
+  info <- file.info(files)
+  out <- lapply(seq_along(files), function(i) list(
+    id = sub("\\.json$", "", basename(files[[i]])),
+    key = key,
+    bytes = as.numeric(info$size[[i]]),
+    updated = as.numeric(info$mtime[[i]]) * 1000
+  ))
+  out[order(vapply(out, function(x) -x$updated, numeric(1)))]
+}
+
+#' Every document key the store holds, for an unfiltered listing.
+ai_keys <- function() {
+  root <- ai_store_root()
+  if (!dir.exists(root)) return(character())
+  dirs <- list.dirs(root, full.names = FALSE, recursive = FALSE)
+  keys <- sub("__", ":", dirs, fixed = TRUE)
+  keys <- keys[vapply(keys, ai_valid_key, logical(1))]
+  # A directory inside the store that resolves OUT of it is not this store's,
+  # and every other op already refuses it. Listing it anyway would put a name
+  # in front of the user that nothing else will open.
+  keys[vapply(keys, function(k) ai_contained(ai_dir(k)), logical(1))]
+}
+
+#' Drop the oldest conversations past the per-document cap.
+#'
+#' Oldest by FILE mtime, and only ever the tail: the cap exists so a store does
+#' not grow without bound, not to decide which conversation matters.
+ai_trim <- function(key) {
+  rows <- ai_list_dir(key)
+  if (length(rows) <= AI_MAX_PER_DOCUMENT) return(0L)
+  doomed <- rows[(AI_MAX_PER_DOCUMENT + 1L):length(rows)]
+  removed <- 0L
+  for (row in doomed) {
+    path <- ai_path(key, row$id)
+    if (ai_contained(path) && file.exists(path)) { unlink(path); removed <- removed + 1L }
+  }
+  removed
+}
+
+# ── the ops ────────────────────────────────────────────────────────────────
+
+# The guard `answer_file_op` (serve.R) and `answer_journal_op` (serve.R) both
+# have, and which this fifth allow-list did not inherit when it was built by
+# analogy with them. It matters MORE here, not less: the send at serve.R is
+# `try(rec$ws$send(...), silent = TRUE)`, so a condition raised while building
+# the frame means NO FRAME IS SENT AT ALL — the page's promise never settles
+# and the pane waits out its timeout instead of showing an error. A refusal is
+# a reply; silence is a hang. `readBin` on a file a concurrent trim has just
+# removed is one way in.
+answer_ai_store_op_guarded <- function(cmd) {
+  frame <- tryCatch(answer_ai_store_op(cmd),
+                    error = function(e) ai_error(cmd$type, cmd$id, "failed", conditionMessage(e)))
+  if (!is.list(frame)) frame <- ai_error(cmd$type, cmd$id, "failed", "The store produced no reply.")
+  frame
+}
+
+answer_ai_store_op <- function(cmd) {
+  type <- cmd$type; id <- cmd$id
+  if (!AI_STORE_ENABLED) {
+    return(ai_error(type, id, "disabled",
+                    "Conversation storage is switched off on this machine (CARMAR_NO_AI_STORE=1)."))
+  }
+
+  # Every op but `ai_keys` names a document, and the key is checked BEFORE the
+  # file system is touched — the journal's rule, for the same reason.
+  if (identical(type, "ai_keys")) {
+    keys <- ai_keys()
+    return(list(type = type, id = id, keys = I(as.character(keys))))
+  }
+
+  key <- cmd$key
+  if (!ai_valid_key(key)) {
+    return(ai_error(type, id, "bad_key", "That is not a document key this store recognises."))
+  }
+
+  if (identical(type, "ai_list")) {
+    rows <- ai_list_dir(key)
+    return(list(type = type, id = id, key = key,
+                conversations = if (length(rows)) rows else I(list())))
+  }
+
+  if (identical(type, "ai_read")) {
+    if (!ai_valid_id(cmd$conversation)) {
+      return(ai_error(type, id, "bad_id", "That is not a conversation id."))
+    }
+    path <- ai_path(key, cmd$conversation)
+    if (!ai_contained(path)) {
+      return(ai_error(type, id, "refused", "That path is outside the conversation store."))
+    }
+    text <- ai_read_text(path)
+    if (is.null(text)) {
+      return(ai_error(type, id, "missing", "That conversation is not in the store.",
+                      conversation = cmd$conversation))
+    }
+    if (nchar(text, type = "bytes") > AI_MAX_REPLY_BYTES) {
+      return(ai_error(type, id, "too_large", "That conversation is too large to send in one frame.",
+                      conversation = cmd$conversation, bytes = nchar(text, type = "bytes")))
+    }
+    return(list(type = type, id = id, key = key, conversation = cmd$conversation,
+                record = text))
+  }
+
+  if (identical(type, "ai_write")) {
+    if (!ai_valid_id(cmd$conversation)) {
+      return(ai_error(type, id, "bad_id", "That is not a conversation id."))
+    }
+    text <- cmd$record
+    if (!is.character(text) || length(text) != 1L || is.na(text)) {
+      return(ai_error(type, id, "bad_record",
+                      "A conversation travels as one JSON string, not as an object."))
+    }
+    bytes <- nchar(text, type = "bytes")
+    if (bytes > AI_MAX_BYTES) {
+      # Refused, not truncated: a truncated conversation is one that silently
+      # lies about what was said.
+      return(ai_error(type, id, "too_large",
+                      sprintf("That conversation is %s bytes; the limit is %s. Nothing was written.",
+                              format(bytes, big.mark = ","), format(AI_MAX_BYTES, big.mark = ",")),
+                      conversation = cmd$conversation, bytes = bytes, max = AI_MAX_BYTES))
+    }
+    # It must PARSE before it is kept. The page is the only writer, but a store
+    # whose files are not valid JSON is a store that fails at read time, on the
+    # day someone needs it, with no way back.
+    if (is.null(tryCatch(jsonlite::fromJSON(text, simplifyVector = FALSE), error = function(e) NULL))) {
+      return(ai_error(type, id, "bad_record", "That conversation is not valid JSON; nothing was written."))
+    }
+    path <- ai_path(key, cmd$conversation)
+    if (!ai_contained(path)) {
+      return(ai_error(type, id, "refused", "That path is outside the conversation store."))
+    }
+    out <- tryCatch({
+      ai_ensure_dir(ai_dir(key))
+      ai_replace(path, text)
+      TRUE
+    }, error = function(e) conditionMessage(e))
+    if (!isTRUE(out)) return(ai_error(type, id, "write_failed", out))
+    trimmed <- ai_trim(key)
+    return(list(type = type, id = id, key = key, conversation = cmd$conversation,
+                bytes = bytes, trimmed = trimmed))
+  }
+
+  if (identical(type, "ai_delete")) {
+    if (!ai_valid_id(cmd$conversation)) {
+      return(ai_error(type, id, "bad_id", "That is not a conversation id."))
+    }
+    path <- ai_path(key, cmd$conversation)
+    if (!ai_contained(path)) {
+      return(ai_error(type, id, "refused", "That path is outside the conversation store."))
+    }
+    existed <- file.exists(path)
+    if (existed) unlink(path)
+    return(list(type = type, id = id, key = key, conversation = cmd$conversation,
+                deleted = existed))
+  }
+
+  if (identical(type, "ai_purge")) {
+    # Every conversation for one document, gone. The page states the scope in
+    # words before it asks (Settings ▸ Recovery's rule: a confirmation that does
+    # not say what it deletes is not a confirmation).
+    dir <- ai_dir(key)
+    if (!ai_contained(dir)) {
+      return(ai_error(type, id, "refused", "That path is outside the conversation store."))
+    }
+    rows <- ai_list_dir(key)
+    # `only` NAMES the conversations to remove. One directory holds two kinds of
+    # record — chats and agent sessions — and nothing in the file tells them
+    # apart, so a directory-wide unlink answering "Clear chat history" took the
+    # agent sessions with it. The caller knows which are its own; it says so.
+    # Each id is validated and confined on its own, exactly as ai_delete does:
+    # a list from the wire is a list of requests, not a list of facts.
+    only <- cmd$only
+    if (!is.null(only)) {
+      ids <- as.character(only)
+      removed <- sum(vapply(ids, function(one) {
+        if (!ai_valid_id(one)) return(FALSE)
+        path <- ai_path(key, one)
+        if (ai_contained(path) && file.exists(path)) { unlink(path); TRUE } else FALSE
+      }, logical(1)))
+      return(list(type = type, id = id, key = key, deleted = as.integer(removed)))
+    }
+    if (dir.exists(dir)) unlink(dir, recursive = TRUE)
+    return(list(type = type, id = id, key = key, deleted = length(rows)))
+  }
+
+  ai_error(type, id, "unknown_op", sprintf("The conversation store has no op named %s.", type))
+}
+
+#' The record travels as a JSON STRING, and that is the whole encoding rule.
+#'
+#' `toJSON(auto_unbox = TRUE)` would collapse `turns: [one]` to an object and
+#' `images: ["x"]` to `"x"` — relay_frame's lesson, and the journal's. The way
+#' past it is not to hand jsonlite the structure at all: the conversation stays
+#' a character scalar from the page, through the file, back to the page, and
+#' jsonlite only ever quotes and escapes it. The page parses it itself, exactly
+#' as it already does for a journal checkpoint.
+ai_frame <- function(answer) {
+  jsonlite::toJSON(answer, auto_unbox = TRUE, null = "null", na = "null", digits = NA)
+}