chamba 0.3.0 → 0.4.0

package/templates/webterm/artifacts.js

@@ -0,0 +1,583 @@
+// artifacts.js - The web pane's files: the pages a session published, and the feedback submitted on them.
+//
+// The pane is a directory, not a protocol. A page is a standalone HTML file written into the conversation's
+// own directory; the interface lists what is there and serves it. That is what makes the history survive a
+// closed tab, a stopped container and a resumed conversation, and it is what lets anything that can write a
+// file publish - the helper is a convenience over `cp`, not a gatekeeper.
+//
+// So this file owns the directory and nothing else: where a conversation's pages live, what a page is called,
+// how big the whole thing may get, what a page is titled, which ones are new, and how a page is read back
+// without ever leaving the root. The server owns who is asking and what to do about it.
+//
+// Two rules run through all of it. Nothing is ever deleted - not by age, not on session close, not when the
+// cap is reached; past the cap a publish is refused and what exists stays. And nothing outside the root is
+// ever read: names are checked against a narrow pattern, symlinks are refused rather than followed, and the
+// resolved path is checked to still be inside the root before a byte is read.
+
+import {
+    closeSync,
+    existsSync,
+    lstatSync,
+    mkdirSync,
+    openSync,
+    readdirSync,
+    readFileSync,
+    readSync,
+    realpathSync,
+    renameSync,
+    rmdirSync,
+    statSync,
+    writeFileSync,
+} from "node:fs";
+import { join } from "node:path";
+
+// A conversation key is a directory name, so it is checked like one: it may not be "." or "..", may not hold
+// a separator, and must start with something ordinary. Agent conversation ids (uuids, "ses_" ids) pass; the
+// server's own provisional keys are minted to match.
+const KEY_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/;
+
+// A page is one HTML file directly inside a conversation's directory. Same reasoning as the key: no
+// separators, no leading dot, nothing that could name a parent.
+const PAGE_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}\.html$/;
+
+// Where the feedback for a conversation's pages is filed, inside that conversation's own directory.
+const FEEDBACK_DIR = "feedback";
+
+// How much of a page is read looking for its title. A title that is not in the first few kilobytes is not a
+// title anyone wrote for the head of the document.
+const TITLE_SCAN_BYTES = 8 * 1024;
+
+// Longest title the pane will carry, in characters. It rides into a chip, into a log line, and into the
+// nudge typed at an agent, so it is cut here once rather than by each of them.
+const MAX_TITLE_LENGTH = 80;
+
+// Longest slug in a filename. Long enough to stay readable in `ls`, short enough that the name is a name.
+const MAX_SLUG_LENGTH = 48;
+
+/** True when a value can name a conversation's directory. */
+export function isPaneKey(value) {
+    return typeof value === "string" && KEY_PATTERN.test(value) && value !== "." && value !== "..";
+}
+
+/** True when a value can name a page inside one. */
+export function isPageId(value) {
+    return typeof value === "string" && PAGE_PATTERN.test(value);
+}
+
+// Everything outside the two control blocks and DEL. One rule, so a line and a block of text agree on what
+// a character is.
+function isPrintable(code) {
+    return code >= 0x20 && code !== 0x7f && !(code >= 0x80 && code <= 0x9f);
+}
+
+/**
+ * One clean line of text, cut to a length: control characters (escape sequences included) dropped, runs of
+ * whitespace collapsed, ends trimmed. Everything a page or a caller supplies passes through here before it
+ * is shown, logged, or typed into a terminal - a title is text, and a terminal reads some text as commands.
+ */
+export function cleanLine(raw, max = MAX_TITLE_LENGTH) {
+    if (typeof raw !== "string") return "";
+    let kept = "";
+    for (const ch of raw) {
+        const code = ch.codePointAt(0);
+        // Drop C0 controls, DEL and the C1 block; keep everything printable, other alphabets and emoji
+        // included. C1 goes because U+009B is CSI: no terminal this runs against decodes it from UTF-8 as an
+        // escape, but the whole point of this function is that nothing it returns is ever read as a command.
+        kept += isPrintable(code) ? ch : " ";
+    }
+    return kept.replace(/\s+/g, " ").trim().slice(0, max).trim();
+}
+
+/**
+ * The same cleaning for text that is allowed to be more than one line: a form's free-text box, a long answer
+ * typed into a textarea. Line breaks survive, every other control character does not, and the length is
+ * capped. Nothing typed into a terminal ever comes from here - this is what gets written to a file.
+ */
+export function cleanText(raw, max) {
+    if (typeof raw !== "string") return "";
+    let kept = "";
+    for (const ch of raw) {
+        const code = ch.codePointAt(0);
+        if (ch === "\n" || isPrintable(code)) kept += ch;
+        else if (ch === "\r" || ch === "\t") kept += ch === "\t" ? " " : "";
+        else kept += " ";
+    }
+    return kept.trim().slice(0, max);
+}
+
+/** The filename part a title turns into: lowercase, words joined by dashes, nothing else. */
+export function slugify(raw) {
+    const slug = cleanLine(raw, MAX_SLUG_LENGTH * 2)
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/g, "-")
+        .replace(/^-+|-+$/g, "")
+        .slice(0, MAX_SLUG_LENGTH)
+        .replace(/-+$/, "");
+    return slug || "page";
+}
+
+/**
+ * The first `limit` bytes of a file, as text, or "" when it cannot be read.
+ *
+ * Bounded at the read rather than after it: a page put into the directory by hand is not held to the publish
+ * cap, so a listing that read whole files would stall the whole interface on one huge one - and a listing
+ * happens on a timer, for every page, in every pane.
+ */
+function headOf(path, limit) {
+    let fd = null;
+    try {
+        fd = openSync(path, "r");
+        const buffer = Buffer.alloc(limit);
+        const read = readSync(fd, buffer, 0, limit, 0);
+        return buffer.subarray(0, read).toString("utf8");
+    } catch {
+        return "";
+    } finally {
+        if (fd !== null) closeSync(fd);
+    }
+}
+
+// The five entities a title can realistically carry. Titles are short human text, not markup, so this is not
+// a parser - it is the difference between showing "Ports &amp; keys" and "Ports & keys".
+const ENTITIES = { "&amp;": "&", "&lt;": "<", "&gt;": ">", "&quot;": '"', "&#39;": "'" };
+
+/**
+ * What a page calls itself: the text of its <title>, or the filename read back as words.
+ *
+ * The page's own title is used rather than anything recorded beside it, so a file dropped into the directory
+ * by hand is titled exactly like one the helper published, and there is no second copy of the truth to drift.
+ */
+function titleFor(path, id) {
+    const head = headOf(path, TITLE_SCAN_BYTES);
+    const match = /<title[^>]*>([\s\S]*?)<\/title>/i.exec(head);
+    if (match) {
+        const text = cleanLine(match[1].replace(/<[^>]*>/g, "").replace(/&(amp|lt|gt|quot|#39);/g, (entity) => ENTITIES[entity]));
+        if (text) return text;
+    }
+    return (
+        cleanLine(
+            id
+                .replace(/\.html$/i, "")
+                .replace(/^\d+-/, "")
+                .replace(/[-_]+/g, " "),
+        ) || id
+    );
+}
+
+/**
+ * The pane store over one artifacts root.
+ *
+ * `maxPageBytes` caps a single page, `maxTotalBytes` and `maxTotalFiles` the whole root, and
+ * `maxFeedbackPerPage` how many answers one page may file. All of them are enforced here rather than in the
+ * helper, because the helper is one way in and a file copied by hand is another.
+ */
+export function createPaneStore({ root, maxPageBytes, maxTotalBytes, maxTotalFiles, maxFeedbackPerPage }) {
+    // What the last scan of each key saw: id -> { mtime, unread }. This is where "new" lives. A key scanned
+    // for the first time has no new pages - they are history, and history does not badge; anything that
+    // turns up in a later scan is an arrival and does.
+    //
+    // Bounded, because a container that has been up for weeks has had every conversation it ever resumed
+    // through here. The oldest entry goes first, and all that is lost with it is which pages were unread in
+    // a conversation nobody has looked at for that long.
+    const scanned = new Map();
+    const MAX_SCANNED_KEYS = 256;
+
+    function remember(key, seen) {
+        scanned.set(key, seen);
+        while (scanned.size > MAX_SCANNED_KEYS) {
+            const oldest = scanned.keys().next();
+            if (oldest.done || oldest.value === key) break;
+            scanned.delete(oldest.value);
+        }
+    }
+
+    function keyDir(key) {
+        return join(root, key);
+    }
+
+    /**
+     * The directory for a key, created if needed, or null when the key is not one or the path is not ours.
+     * The realpath check is the containment rule at the directory level: a key directory that is a symlink,
+     * or that resolves anywhere but inside the root, is refused rather than followed.
+     */
+    function ensureDir(key) {
+        if (!isPaneKey(key)) return null;
+        const dir = keyDir(key);
+        try {
+            mkdirSync(dir, { recursive: true });
+            const real = realpathSync(dir);
+            const realRoot = realpathSync(root);
+            if (real !== join(realRoot, key)) return null;
+            return dir;
+        } catch {
+            return null;
+        }
+    }
+
+    /** The page files in a key's directory, oldest first, with their size and the number they carry. */
+    function pageFiles(key) {
+        if (!isPaneKey(key)) return [];
+        let names;
+        try {
+            names = readdirSync(keyDir(key));
+        } catch {
+            return []; // No directory yet, or nothing readable in it: an empty pane, not an error.
+        }
+        const found = [];
+        for (const id of names) {
+            if (!isPageId(id)) continue;
+            const path = join(keyDir(key), id);
+            let stat;
+            try {
+                stat = lstatSync(path);
+            } catch {
+                continue;
+            }
+            // A symlink is refused here as well as on read, so a link into the workspace is not even listed.
+            if (!stat.isFile()) continue;
+            const numberMatch = /^(\d+)-/.exec(id);
+            found.push({ id, path, bytes: stat.size, mtime: stat.mtimeMs, number: numberMatch ? Number(numberMatch[1]) : 0 });
+        }
+        return found.sort((a, b) => a.number - b.number || a.mtime - b.mtime);
+    }
+
+    /**
+     * What the whole root holds, pages and feedback together: bytes and file count. Walked per write, which
+     * is affordable only because the file count is capped too - both numbers come out of this one walk, and
+     * the count is what keeps the walk itself small.
+     */
+    function totals() {
+        let total = 0;
+        let files = 0;
+        let dirs;
+        try {
+            dirs = readdirSync(root, { withFileTypes: true });
+        } catch {
+            return { bytes: 0, files: 0 };
+        }
+        for (const entry of dirs) {
+            if (!entry.isDirectory() || !isPaneKey(entry.name)) continue;
+            for (const dir of [keyDir(entry.name), join(keyDir(entry.name), FEEDBACK_DIR)]) {
+                let names;
+                try {
+                    names = readdirSync(dir);
+                } catch {
+                    continue;
+                }
+                for (const name of names) {
+                    try {
+                        const stat = lstatSync(join(dir, name));
+                        if (!stat.isFile()) continue;
+                        total += stat.size;
+                        files += 1;
+                    } catch {
+                        // Vanished between the listing and the stat.
+                    }
+                }
+            }
+        }
+        return { bytes: total, files };
+    }
+
+    /** Bytes held by the whole root. The count that goes with it is a cap, not something a caller asks for. */
+    function totalBytes() {
+        return totals().bytes;
+    }
+
+    /** Room for one more file of this size, or the refusal that says which cap it met. */
+    function roomFor(bytes) {
+        const held = totals();
+        if (held.bytes + bytes > maxTotalBytes) return "full";
+        if (held.files + 1 > maxTotalFiles) return "full";
+        return null;
+    }
+
+    /**
+     * The pane's contents for a key, as the browser sees it: the id to ask for, what the page calls itself,
+     * when it arrived, and whether it is new since the window was last looking.
+     */
+    function list(key) {
+        const seen = scanned.get(key);
+        return pageFiles(key).map((file) => ({
+            id: file.id,
+            title: titleFor(file.path, file.id),
+            mtime: Math.round(file.mtime),
+            bytes: file.bytes,
+            unread: seen?.get(file.id)?.unread === true,
+        }));
+    }
+
+    /**
+     * Look at a key's directory again and report whether anything moved. Pages that turned up since the last
+     * look are new (unread), unless this is the first look at that key, when everything there is history.
+     * This is what makes a file copied in by hand appear in the pane with no helper involved.
+     */
+    function refresh(key) {
+        const files = pageFiles(key);
+        const before = scanned.get(key);
+        const after = new Map();
+        let changed = before === undefined && files.length > 0;
+        for (const file of files) {
+            const previous = before?.get(file.id);
+            if (previous === undefined) {
+                after.set(file.id, { mtime: file.mtime, unread: before !== undefined });
+                if (before !== undefined) changed = true;
+            } else {
+                after.set(file.id, { mtime: file.mtime, unread: previous.unread });
+                if (previous.mtime !== file.mtime) changed = true;
+            }
+        }
+        if (before !== undefined && before.size !== after.size) changed = true;
+        remember(key, after);
+        return changed;
+    }
+
+    /** The window opened this page, so it is no longer new. True when that actually changed something. */
+    function markRead(key, id) {
+        const entry = scanned.get(key)?.get(id);
+        if (!entry?.unread) return false;
+        entry.unread = false;
+        return true;
+    }
+
+    /**
+     * One page's content, or why it cannot be served. Every refusal is a containment rule:
+     * - "name": the key or the id is not a shape we file things under, so it never becomes a path.
+     * - "missing": there is no such page, or it is not a regular file - a directory, a device, a symlink.
+     *   Symlinks are refused rather than followed, which is what stops a link inside the pane from serving
+     *   the key file or anything else in the container.
+     * - "toobig": bigger than a page may be. A file put there by hand is not bound by the publish cap, and
+     *   reading an arbitrarily large file into memory to answer a request is not something to allow.
+     */
+    function read(key, id) {
+        if (!isPaneKey(key) || !isPageId(id)) return { ok: false, error: "name" };
+        const path = join(keyDir(key), id);
+        let stat;
+        try {
+            stat = lstatSync(path);
+        } catch {
+            return { ok: false, error: "missing" };
+        }
+        if (!stat.isFile()) return { ok: false, error: "missing" };
+        if (stat.size > maxPageBytes) return { ok: false, error: "toobig" };
+        // Belt and braces over the name check: whatever the path turned out to be, it has to still be the
+        // file inside this key's directory.
+        try {
+            if (realpathSync(path) !== join(realpathSync(root), key, id)) return { ok: false, error: "missing" };
+        } catch {
+            return { ok: false, error: "missing" };
+        }
+        try {
+            return { ok: true, id, title: titleFor(path, id), html: readFileSync(path, "utf8") };
+        } catch {
+            return { ok: false, error: "missing" };
+        }
+    }
+
+    /**
+     * Write a page into a key's directory and return what it became. `title` names it (and, slugified, names
+     * the file); the number in front of the name counts up per conversation, so the pane reads in the order
+     * the pages were published however they are sorted elsewhere.
+     *
+     * Refusals: "name" for a key that is not one, "toobig" for a page over the per-page cap, "full" when the
+     * root has reached its total - nothing is deleted to make room, because a page the user has not read yet
+     * is not the server's to throw away - and "write" when the filesystem said no.
+     */
+    function publish(key, { title, html }) {
+        if (!isPaneKey(key)) return { ok: false, error: "name" };
+        if (typeof html !== "string" || html === "") return { ok: false, error: "empty" };
+        const bytes = Buffer.byteLength(html);
+        if (bytes > maxPageBytes) return { ok: false, error: "toobig" };
+        const noRoom = roomFor(bytes);
+        if (noRoom !== null) return { ok: false, error: noRoom };
+        const dir = ensureDir(key);
+        if (dir === null) return { ok: false, error: "name" };
+        const files = pageFiles(key);
+        const next = files.reduce((highest, file) => Math.max(highest, file.number), 0) + 1;
+        const clean = cleanLine(title);
+        const id = `${String(next).padStart(2, "0")}-${slugify(clean)}.html`;
+        try {
+            writeFileSync(join(dir, id), html);
+        } catch {
+            return { ok: false, error: "write" };
+        }
+        // Straight into the scan as an arrival: a page nobody has opened is new, whoever wrote it.
+        const seen = scanned.get(key) ?? new Map();
+        seen.set(id, { mtime: Date.now(), unread: true });
+        remember(key, seen);
+        return { ok: true, id, title: clean || titleFor(join(dir, id), id), path: join(dir, id) };
+    }
+
+    /** The answers already filed for one page, by the name they all start with. */
+    function feedbackFiles(key, stem) {
+        let names;
+        try {
+            names = readdirSync(join(keyDir(key), FEEDBACK_DIR));
+        } catch {
+            return []; // No answers yet: an empty list, not an error.
+        }
+        return names.filter((name) => name.startsWith(`${stem}-`) && name.endsWith(".json"));
+    }
+
+    /**
+     * File one submission from a page's form. The page it belongs to is passed in already resolved against
+     * the listing, and the filename is built here from that page and the clock - nothing a submission carries
+     * decides where it lands.
+     */
+    function writeFeedback(key, id, { fields, text }) {
+        if (!isPaneKey(key) || !isPageId(id)) return { ok: false, error: "name" };
+        const dir = ensureDir(key);
+        if (dir === null) return { ok: false, error: "name" };
+        const feedbackDir = join(dir, FEEDBACK_DIR);
+        const stem = id.replace(/\.html$/i, "");
+        const body = {
+            page: stem,
+            submittedAt: new Date().toISOString(),
+            fields: fields ?? {},
+            text: text ?? "",
+        };
+        const contents = `${JSON.stringify(body, null, 2)}\n`;
+        // A form is submitted by whatever is inside the page - a person clicking, or a script in it - so an
+        // endless stream of them is a thing that can happen. Two caps stand in the way, and the count is the
+        // one that matters: answers are small, so bytes alone would allow a very large number of them, and
+        // every one of them also types a line into the agent's terminal.
+        if (feedbackFiles(key, stem).length + 1 > maxFeedbackPerPage) return { ok: false, error: "toomany" };
+        const noRoom = roomFor(Buffer.byteLength(contents));
+        if (noRoom !== null) return { ok: false, error: noRoom };
+        // The clock names the file, and a second submission in the same millisecond takes the next free name
+        // beside it. Nothing here ever overwrites: two answers are two answers, whatever the timing, and a
+        // millisecond with a hundred of them in it is refused rather than written over.
+        const at = Date.now();
+        let file = `${stem}-${at}.json`;
+        let attempt = 0;
+        while (existsSync(join(feedbackDir, file))) {
+            if (++attempt >= 100) return { ok: false, error: "write" };
+            file = `${stem}-${at}-${attempt}.json`;
+        }
+        try {
+            mkdirSync(feedbackDir, { recursive: true });
+            writeFileSync(join(feedbackDir, file), contents);
+        } catch {
+            return { ok: false, error: "write" };
+        }
+        return { ok: true, file, path: join(feedbackDir, file) };
+    }
+
+    /**
+     * Move everything published under one key to another - the moment a provisional key turns out to have a
+     * conversation id after all. A rename when the other side has no directory yet, which is the first time a
+     * conversation is seen; when it does exist - a conversation resumed after publishing something - the pages
+     * are carried over one at a time with fresh numbers, because the conversation's own numbering is the one
+     * that has to stay in order.
+     *
+     * What is new stays new: a page the user has not opened is still unread under its new name.
+     */
+    function adopt(fromKey, toKey) {
+        if (!isPaneKey(fromKey) || !isPaneKey(toKey)) return false;
+        if (fromKey === toKey) return true;
+        const fromDir = keyDir(fromKey);
+        const unread = scanned.get(fromKey) ?? new Map();
+        scanned.delete(fromKey);
+        if (!existsSync(fromDir)) {
+            refresh(toKey);
+            return true;
+        }
+        try {
+            if (!existsSync(keyDir(toKey))) {
+                renameSync(fromDir, keyDir(toKey));
+                const carried = new Map();
+                for (const file of pageFiles(toKey))
+                    carried.set(file.id, { mtime: file.mtime, unread: unread.get(file.id)?.unread === true });
+                remember(toKey, carried);
+                return true;
+            }
+        } catch {
+            return false;
+        }
+        return merge(fromKey, toKey, unread);
+    }
+
+    // The rarer half of adopt(): both directories exist, so each page is moved across under the next free
+    // number and its feedback follows it. The source directory is removed only when it is empty, which is
+    // the same "nothing is deleted" rule as everywhere else.
+    function merge(fromKey, toKey, unread) {
+        const target = ensureDir(toKey);
+        if (target === null) return false;
+        // A key nobody has scanned yet has no unread pages, so the carry starts from what is already there.
+        // Without that seed, remembering only the moved pages would make the target's own history arrive
+        // badged at the next refresh - the very rule the scan map exists to hold.
+        const carried = scanned.get(toKey) ?? new Map(pageFiles(toKey).map((file) => [file.id, { mtime: file.mtime, unread: false }]));
+        let next = pageFiles(toKey).reduce((highest, file) => Math.max(highest, file.number), 0);
+        for (const file of pageFiles(fromKey)) {
+            next += 1;
+            const id = `${String(next).padStart(2, "0")}-${file.id.replace(/^\d+-/, "")}`;
+            try {
+                renameSync(file.path, join(target, id));
+            } catch {
+                continue; // Leave it where it is rather than lose it.
+            }
+            carried.set(id, { mtime: file.mtime, unread: unread.get(file.id)?.unread === true });
+            moveFeedback(fromKey, toKey, file.id, id);
+        }
+        remember(toKey, carried);
+        cleanupEmpty(fromKey);
+        return true;
+    }
+
+    // A page's feedback files travel with it, renamed to match the page's new number.
+    function moveFeedback(fromKey, toKey, fromId, toId) {
+        const fromDir = join(keyDir(fromKey), FEEDBACK_DIR);
+        const toDir = join(keyDir(toKey), FEEDBACK_DIR);
+        const fromStem = fromId.replace(/\.html$/i, "");
+        const toStem = toId.replace(/\.html$/i, "");
+        let names;
+        try {
+            names = readdirSync(fromDir);
+        } catch {
+            return;
+        }
+        for (const name of names) {
+            if (!name.startsWith(`${fromStem}-`) || !name.endsWith(".json")) continue;
+            try {
+                mkdirSync(toDir, { recursive: true });
+                renameSync(join(fromDir, name), join(toDir, `${toStem}-${name.slice(fromStem.length + 1)}`));
+            } catch {
+                // Leave it behind rather than lose it.
+            }
+        }
+    }
+
+    // Remove the provisional directory once everything in it has moved. Only ever an empty directory, and
+    // only one this store created.
+    function cleanupEmpty(key) {
+        for (const dir of [join(keyDir(key), FEEDBACK_DIR), keyDir(key)]) {
+            try {
+                if (statSync(dir).isDirectory() && readdirSync(dir).length === 0) rmdirSync(dir);
+            } catch {
+                // Not empty, or already gone: either way there is nothing to do.
+            }
+        }
+    }
+
+    return { adopt, ensureDir, keyDir, list, markRead, publish, read, refresh, totalBytes, writeFeedback };
+}
+
+/**
+ * What a submission is allowed to be. Free text from a page's form, so it is bounded on every axis before it
+ * is written: how many fields, how long a name, how long a value, how long the free-text box. Anything else
+ * (numbers, nested objects, a field called `__proto__`) is turned into text or dropped rather than argued
+ * with - this becomes a JSON file an agent reads, not a structure anything acts on.
+ */
+export function cleanSubmission(body, { maxFields, maxFieldLength, maxTextLength }) {
+    const fields = {};
+    const raw = body && typeof body === "object" && typeof body.fields === "object" && body.fields !== null ? body.fields : {};
+    for (const [name, value] of Object.entries(raw)) {
+        if (Object.keys(fields).length >= maxFields) break;
+        const key = cleanLine(name, 64);
+        if (!key || key === "__proto__") continue;
+        if (value === null || typeof value === "object") continue;
+        fields[key] = cleanText(String(value), maxFieldLength);
+    }
+    const text = cleanText(body?.text, maxTextLength);
+    return { fields, text };
+}