@join-pantheon/cli 0.33.2 → 0.33.4-rc.1

package/dist/relay.js

@@ -1,8 +1,9 @@
 import { createServer } from "node:http";
 import { createHash, randomBytes, timingSafeEqual as timingSafeEqualBuf } from "node:crypto";
-import { rmSync, readdirSync, readFileSync, statSync, existsSync, writeFileSync, openSync, readSync, closeSync, fstatSync } from "node:fs";
+import { copyFileSync, readdirSync, readFileSync, statSync, existsSync, writeFileSync, openSync, readSync, closeSync, fstatSync } from "node:fs";
 import { spawn } from "node:child_process";
-import { join } from "node:path";
+import { monitorEventLoopDelay } from "node:perf_hooks";
+import { basename, join } from "node:path";
 import { WebSocketServer } from "ws";
 import { badRemote, Inbound, isContinuationFor, LIMITS, newId, PROTOCOL_VERSION, RELAY_FEATURES, SENDABLE, shouldDeferToHuman, } from "./protocol.js";
 import { VERSION } from "./version.js";
@@ -14,9 +15,11 @@ import { Accounts, normalizeUsername, validEmail, validUsername } from "./accoun
 import { emailSenderFromEnv } from "./email.js";
 import { Allowlist, RosterUnreadable } from "./roster-file.js";
 import { StateStore, ensurePrivateDir, setWaitForDisk, takeSlowestDiskWait, waitsForDisk } from "./store.js";
+import { busyWriter, DiskBusy, flushWrites, onWriteError, pendingView, readLatest, recordWrites, whenLanded, writerFor } from "./disk-writer.js";
 import { activity, mergeActivity, feed, noteRefusal, refusalsSince, countActs, spaceHealth, fleetReport, ESCALATIONS, MESSAGE_ACTS, ACT_TONE, DAY } from "./admin-stats.js";
 import { fmtLimit, inviteLimit, parseLimits, planLimits, PLAN_NAME } from "./plans.js";
 import { FEEDBACK_ADDRESS } from "./feedback.js";
+import { relayShellArg, teamNameProblem } from "./team-name.js";
 import { routeDecision } from "./profile.js";
 const presentSessions = (m) => [...m.sessions.values()].filter((s) => !s.quiet);
 function workState(m, now = Date.now(), busyTtlMs = LIMITS.activityBusyTtlMs, pruneMs = LIMITS.activityPruneMs) {
@@ -178,6 +181,19 @@ const fmtAgo = (t) => {
     return m < 1 ? "just now" : m < 60 ? `${m}m ago` : m < 1440 ? `${Math.round(m / 60)}h ago` : `${Math.round(m / 1440)}d ago`;
 };
 const safeName = (name) => name.replace(/[^a-z0-9._-]/gi, "_");
+const quoteTeam = (name) => relayShellArg(name);
+function parseLines(text) {
+    const records = [];
+    for (const line of text.split("\n")) {
+        if (!line.trim())
+            continue;
+        try {
+            records.push(JSON.parse(line));
+        }
+        catch { }
+    }
+    return records;
+}
 const spaceKey = (name) => safeName(name).toLowerCase();
 function firstLedgerName(file) {
     let fd = null;
@@ -276,6 +292,26 @@ function versionBelow(a, b) {
 export const RECOVERY_PROOF_MS = 15 * 60_000;
 const RECOVERY_KEEP_MS = 7 * 86_400_000;
 const LOCKED_MSG = "too many wrong codes from this device or your network in the last hour — wait an hour, then request a new code";
+const LOOP_RESOLUTION_MS = 10;
+const LOOP_WINDOW_MS = 60_000;
+function loopMs(ns) {
+    return Math.max(0, Math.round((ns / 1e6 - LOOP_RESOLUTION_MS) * 100) / 100);
+}
+const STORAGE_HELD = "the relay received this but has not saved it yet — its disk is refusing writes. It is kept, and reaches the team when the disk recovers; do not send it again.";
+export function backupExcludes(dir, keep) {
+    const always = ["--exclude=*.tmp"];
+    if (!keep)
+        return always;
+    let names = [];
+    try {
+        names = readdirSync(dir);
+    }
+    catch {
+        return always;
+    }
+    const kept = basename(keep);
+    return [...always, ...names.filter((f) => f !== kept && /^accounts-.*\.db$/.test(f)).map((f) => `--exclude=./backups/${f}`)];
+}
 const STORAGE_UNHEALTHY_MS = 5 * 60_000;
 function isStorageError(e) {
     const code = e?.code;
@@ -428,6 +464,10 @@ export class Relay {
             catch { }
         };
         try {
+            if (this.closing && req.method !== "GET" && req.method !== "HEAD") {
+                res.writeHead(503, { "content-type": "application/json", "retry-after": "10" });
+                return void res.end(JSON.stringify({ code: "SHUTTING_DOWN", error: "the relay is restarting — try again in a few seconds" }));
+            }
             if (!URL.canParse(req.url ?? "/", "http://x"))
                 return fail(new TypeError("unparseable request target"));
             const r = this.onHttp(req, res);
@@ -442,6 +482,12 @@ export class Relay {
     heartbeat;
     lagTimer;
     lagMax = 0;
+    loopDelay = null;
+    loopPrev = null;
+    loopTimer;
+    stopWriteErrors;
+    spaceOf = new WeakMap();
+    closing = false;
     fsyncOff = false;
     compactedAt = new Map();
     sessionSeq = 0;
@@ -483,6 +529,7 @@ export class Relay {
             this.log("* PANTHEON_RELAY_FSYNC=off: writes go to the operating system at once but the relay does not wait for the disk to confirm them (a relay crash loses nothing; a host power loss can lose the last seconds)");
         }
         ensurePrivateDir(this.dataDir);
+        this.stopWriteErrors = onWriteError(this.dataDir, (where, e, stem) => { this.noteStorageFailure(where, e); this.failHeld(stem); });
         this.maxLedgerBytes = opts.spaceMaxBytes
             ?? (Number(process.env.PANTHEON_SPACE_MAX_BYTES) > 0 ? Number(process.env.PANTHEON_SPACE_MAX_BYTES) : SPACE_MAX_BYTES_DEFAULT);
         this.scanUnbound();
@@ -544,6 +591,10 @@ export class Relay {
                 let last = Date.now();
                 this.lagTimer = setInterval(() => { const now = Date.now(); this.lagMax = Math.max(this.lagMax, now - last - 500); last = now; }, 500);
                 this.lagTimer.unref?.();
+                this.loopDelay = monitorEventLoopDelay({ resolution: LOOP_RESOLUTION_MS });
+                this.loopDelay.enable();
+                this.loopTimer = setInterval(() => this.rotateLoopWindow(), LOOP_WINDOW_MS);
+                this.loopTimer.unref?.();
                 this.startBackups();
                 const addr = this.http.address();
                 const port = typeof addr === "object" && addr ? addr.port : want;
@@ -552,6 +603,11 @@ export class Relay {
         });
     }
     async close() {
+        this.closing = true;
+        for (const ws of this.wss.clients)
+            ws.terminate();
+        await new Promise((r) => this.wss.close(() => r()));
+        await new Promise((r) => setImmediate(r));
         for (const sp of this.spaces.values()) {
             try {
                 sp.store.close(sp.goals, sp.obligations, sp.profiles);
@@ -560,6 +616,9 @@ export class Relay {
                 this.noteStorageFailure(`[${sp.name}] final flush`, e);
             }
         }
+        if (!(await flushWrites(this.dataDir, 30_000)))
+            this.log("! storage: writes were still queued when the relay closed (the disk did not take them in 30 s) — they are lost if the process exits now");
+        this.stopWriteErrors();
         if (this.backupTimer)
             clearInterval(this.backupTimer);
         this.installerGate.close();
@@ -569,15 +628,24 @@ export class Relay {
             clearInterval(this.heartbeat);
         if (this.lagTimer)
             clearInterval(this.lagTimer);
+        if (this.loopTimer)
+            clearInterval(this.loopTimer);
+        this.loopDelay?.disable();
         if (this.fsyncOff) {
             setWaitForDisk(true);
             this.fsyncOff = false;
         }
-        for (const ws of this.wss.clients)
-            ws.terminate();
-        await new Promise((r) => this.wss.close(() => r()));
         await new Promise((r) => this.http.close(() => r()));
     }
+    replyLanded(res, writes, reply) {
+        void whenLanded(writes()).then((ok) => {
+            if (res.destroyed)
+                return;
+            if (ok)
+                return reply();
+            this.json(res, 503, { code: "STORAGE", error: "the relay made that change but has not saved it yet — its disk is refusing writes. It is kept, and is saved when the disk recovers; do not send it again." });
+        });
+    }
     noteStorageFailure(where, e) {
         const message = e?.message ?? String(e);
         this.log(`! storage: ${where} failed: ${message}`);
@@ -608,12 +676,17 @@ export class Relay {
         }
     }
     unboundOnDisk(safe) {
-        const size = (suf) => { try {
-            return statSync(join(this.dataDir, `${safe}${suf}`)).size;
-        }
-        catch {
-            return -1;
-        } };
+        const size = (suf) => {
+            const v = pendingView(this.dataDir, safe, join(this.dataDir, `${safe}${suf}`));
+            if (v !== undefined)
+                return v === null ? -1 : v.length;
+            try {
+                return statSync(join(this.dataDir, `${safe}${suf}`)).size;
+            }
+            catch {
+                return -1;
+            }
+        };
         const ledger = size(".jsonl");
         if (size(".members.json") < 0)
             return ledger > 0;
@@ -652,6 +725,7 @@ export class Relay {
                 ...(adminView && this.unbound.size ? { unbound_spaces: [...this.unbound].sort() } : {}),
                 space_count: this.spaces.size,
                 space_capacity: this.limits.maxSpaces,
+                ...(this.loopDelay ? { loop_ms: this.loopStats() } : {}),
                 ...(adminView ? { spaces: [...this.spaces.values()].map(detail) }
                     : asked ? { spaces: one ? [detail(one)] : [] }
                         : {}),
@@ -765,7 +839,7 @@ export class Relay {
             const seen = Number(url.searchParams.get("seen"));
             if (Number.isFinite(seen) && seen > (member.seen_seq ?? 0))
                 member.seen_seq = seen;
-            this.broadcast(space, { t: "roster", roster: this.roster(space) });
+            this.broadcast(space, () => ({ t: "roster", roster: this.roster(space) }));
             return this.json(res, 200, { ok: true });
         }
         this.json(res, 404, { error: "not found" });
@@ -779,19 +853,19 @@ export class Relay {
         const dir = join(this.dataDir, "backups");
         const sessionDays = Number(process.env.PANTHEON_SESSION_RETENTION_DAYS ?? "30");
         const auditDays = Number(process.env.PANTHEON_AUDIT_RETENTION_DAYS ?? "365");
-        const run = () => {
+        const run = async () => {
             try {
                 const pruned = this.accounts.prune({ sessionDays, auditDays });
                 if (pruned.sessions || pruned.audit)
                     this.log(`accounts pruned: ${pruned.sessions} stale session(s), ${pruned.audit} old audit row(s)`);
-                const f = this.accounts.backup(dir);
+                const f = await this.accounts.backupAsync(dir);
                 this.log(`accounts backup written: ${f}`);
             }
             catch (e) {
                 this.log(`! accounts backup/prune failed: ${e.message}`);
             }
         };
-        setTimeout(run, 60_000);
+        setTimeout(run, 60_000).unref?.();
         this.backupTimer = setInterval(run, hours * 3_600_000);
         this.backupTimer.unref?.();
     }
@@ -849,7 +923,14 @@ export class Relay {
                 if (!authed)
                     return this.json(res, 403, { error: "sign in at /admin, or append ?token=…" });
                 A0.audit({ event: "admin_backup", ip });
-                return this.adminBackup(res, A0);
+                void this.adminBackup(res, A0).catch((e) => {
+                    this.log(`! backup archive failed: ${e.message}`);
+                    if (!res.headersSent)
+                        this.json(res, 500, { error: "the backup could not be made" });
+                    else
+                        res.end();
+                });
+                return;
             }
             if (url.pathname === "/admin/action" && req.method === "POST") {
                 if (!authed)
@@ -965,6 +1046,7 @@ export class Relay {
             });
         }
         if (url.pathname === "/account/revoke" && req.method === "POST") {
+            const writes = recordWrites();
             const who = this.verifySignedAccount(url, "POST", "/account/revoke", ip);
             if (!who)
                 return this.json(res, 403, { error: "this request must be signed by a device on the account" });
@@ -977,9 +1059,10 @@ export class Relay {
                 this.dropDeviceSockets(who.account.id, hit.deviceId);
                 this.noteAccountRemoval(who.account.username);
             }
-            return this.json(res, 200, { ok: true, revoked: hit.sessionIds.length, device_removed: !!hit.deviceId });
+            return this.replyLanded(res, writes, () => this.json(res, 200, { ok: true, revoked: hit.sessionIds.length, device_removed: !!hit.deviceId }));
         }
         if (url.pathname === "/account/delete" && req.method === "POST") {
+            const writes = recordWrites();
             const who = this.verifySignedAccount(url, "POST", "/account/delete", ip);
             if (!who)
                 return this.json(res, 403, { error: "this request must be signed by a device on the account" });
@@ -988,7 +1071,7 @@ export class Relay {
             for (const s of sessions)
                 this.dropSessionEverywhere(s.id);
             this.noteAccountRemoval(who.account.username);
-            return this.json(res, 200, { ok: true });
+            return this.replyLanded(res, writes, () => this.json(res, 200, { ok: true }));
         }
         if (url.pathname === "/account/recover" || url.pathname === "/account/recover/withdraw") {
             if (!this.authRateOk(ip))
@@ -1002,10 +1085,11 @@ export class Relay {
             if (method === "GET")
                 return this.json(res, 200, { seats: this.recoverableSeats(who.account, who.device) });
             const name = url.searchParams.get("space") ?? "";
+            const writes = recordWrites();
             const r = url.pathname === "/account/recover/withdraw"
                 ? this.withdrawRecovery(who.account, who.device, name, ip)
                 : this.recoverSeat(who.account, who.device, name, ip);
-            return this.json(res, r.status, r.body);
+            return this.replyLanded(res, writes, () => this.json(res, r.status, r.body));
         }
         if (url.pathname === "/audit" && req.method === "GET") {
             const who = this.verifySignedAccount(url, "GET", "/audit", ip);
@@ -1446,6 +1530,9 @@ button:hover{background:#1d4ed8}.err{color:var(--bad);margin-top:12px}
     sleepingCache = new Map();
     sleepingRecords(safe) {
         const file = join(this.dataDir, `${safe}.jsonl`);
+        const pending = pendingView(this.dataDir, safe, file);
+        if (pending !== undefined)
+            return parseLines(pending?.toString("utf8") ?? "");
         let st;
         try {
             st = statSync(file);
@@ -1457,15 +1544,7 @@ button:hover{background:#1d4ed8}.err{color:var(--bad);margin-top:12px}
         const hit = this.sleepingCache.get(safe);
         if (hit && hit.key === key)
             return hit.records;
-        const records = [];
-        for (const line of readFileSync(file, "utf8").split("\n")) {
-            if (!line.trim())
-                continue;
-            try {
-                records.push(JSON.parse(line));
-            }
-            catch { }
-        }
+        const records = parseLines(readFileSync(file, "utf8"));
         this.sleepingCache.set(safe, { key, records });
         if (this.sleepingCache.size > 200)
             this.sleepingCache.delete(this.sleepingCache.keys().next().value);
@@ -1488,7 +1567,7 @@ button:hover{background:#1d4ed8}.err{color:var(--bad);margin-top:12px}
         setTimeout(() => this.deleteSpaceFiles(name), 600).unref?.();
         const safe = name.replace(/[^a-z0-9._-]/gi, "_");
         try {
-            const hash = sp?.allow ? JSON.parse(readFileSync(join(this.dataDir, `${safe}.members.json`), "utf8")).tokenHash ?? null : null;
+            const hash = sp?.allow ? JSON.parse(readLatest(this.dataDir, safe, join(this.dataDir, `${safe}.members.json`))).tokenHash ?? null : null;
             const earlier = sp?.allow ? sp.allow.tokenHashes().filter((h) => h !== hash) : [];
             writeFileSync(join(this.dataDir, `${safe}.deleted.json`), JSON.stringify({ name, at: Date.now(), tokenHash: hash, ...(earlier.length ? { retiredTokenHashes: earlier } : {}) }));
             this.tombstoneFile(name);
@@ -1508,7 +1587,8 @@ button:hover{background:#1d4ed8}.err{color:var(--bad);margin-top:12px}
         if (body.csrf !== this.adminCsrf())
             return this.json(res, 403, { error: "stale form — reload the dashboard and try again" });
         const act = body.action ?? "", id = (body.id ?? "").trim();
-        const done = (msg) => { res.writeHead(303, { location: `/admin?done=${encodeURIComponent(msg)}` }); res.end(); };
+        const writes = recordWrites();
+        const done = (msg) => this.replyLanded(res, writes, () => { res.writeHead(303, { location: `/admin?done=${encodeURIComponent(msg)}` }); res.end(); });
         switch (act) {
             case "suspend":
             case "reinstate": {
@@ -1531,7 +1611,7 @@ button:hover{background:#1d4ed8}.err{color:var(--bad);margin-top:12px}
                 const allow = sp ? sp.allow : new Allowlist(this.dataDir, id.replace(/[^a-z0-9._-]/gi, "_"));
                 const token = (body.token ?? "").trim();
                 const inv = allow.findInvite(token);
-                const back = (msg) => { res.writeHead(303, { location: `/admin/space?name=${encodeURIComponent(id)}&done=${encodeURIComponent(msg)}` }); res.end(); };
+                const back = (msg) => this.replyLanded(res, writes, () => { res.writeHead(303, { location: `/admin/space?name=${encodeURIComponent(id)}&done=${encodeURIComponent(msg)}` }); res.end(); });
                 if (!inv)
                     return back(`no open invite …${token.slice(-6)} in ${id}`);
                 allow.revokeInvite(inv.token);
@@ -1778,9 +1858,9 @@ button:hover{background:#1d4ed8}.err{color:var(--bad);margin-top:12px}
             if (seen.has(stem))
                 continue;
             try {
-                const m = statSync(join(this.dataDir, f)).mtimeMs;
+                const m = busyWriter(this.dataDir, stem) ? -1 : statSync(join(this.dataDir, f)).mtimeMs;
                 let c = this.rosterCache.get(stem);
-                if (!c || c.mtimeMs !== m) {
+                if (!c || c.mtimeMs !== m || m < 0) {
                     c = { mtimeMs: m, allow: new Allowlist(this.dataDir, stem) };
                     this.rosterCache.set(stem, c);
                 }
@@ -1795,7 +1875,7 @@ button:hover{background:#1d4ed8}.err{color:var(--bad);margin-top:12px}
         const A = this.accounts;
         const u = acct.username;
         const out = [];
-        for (const { name, allow } of this.allRosters()) {
+        for (const { name, allow, live } of this.allRosters()) {
             const e = allow.get(u);
             if (!e?.admitted)
                 continue;
@@ -1804,15 +1884,18 @@ button:hover{background:#1d4ed8}.err{color:var(--bad);margin-top:12px}
             const fp = fingerprint(e.publicKey);
             if (e.publicKey === device.pubkey) {
                 const inv = this.openRecoveryInvite(allow, u);
-                out.push({ space: name, state: inv ? "ready" : "here", founder, alone, fp, ...(inv ? { invite: inv.token } : {}) });
+                out.push({ space: name, state: inv ? "ready" : "here", founder, alone, fp, in_use: false, ...(inv ? { invite: inv.token } : {}) });
                 continue;
             }
+            const in_use = (live?.members.get(u)?.sessions.size ?? 0) > 0;
+            const dev = in_use ? A.deviceByPubkey(e.publicKey) : null;
+            const use = { in_use, ...(dev && dev.account_id === acct.id && dev.label ? { in_use_label: dev.label } : {}) };
             const pend = allow.recoveryOf(u);
             if (pend && pend.pubkey === device.pubkey) {
-                out.push({ space: name, state: "waiting", founder, alone, fp, due_at: pend.due_at });
+                out.push({ space: name, state: "waiting", founder, alone, fp, due_at: pend.due_at, ...use });
                 continue;
             }
-            out.push({ space: name, state: A.wasDeviceOf(acct.id, fp, e.moved?.at ?? e.first_seen) ? "movable" : "unproven", founder, alone, fp });
+            out.push({ space: name, state: A.wasDeviceOf(acct.id, fp, e.moved?.at ?? e.first_seen) ? "movable" : "unproven", founder, alone, fp, ...use });
         }
         return out.sort((a, b) => a.space.localeCompare(b.space));
     }
@@ -1853,7 +1936,7 @@ button:hover{background:#1d4ed8}.err{color:var(--bad);margin-top:12px}
             if (now >= pend.due_at) {
                 if (now - pend.due_at > RECOVERY_KEEP_MS) {
                     sp.allow.cancelRecovery(u);
-                    return { status: 410, body: { reason: "lapsed", error: `that request lapsed (nobody finished it within a week of its time) — ask again: pantheon recover ${sp.name}` } };
+                    return { status: 410, body: { reason: "lapsed", error: `that request lapsed (nobody finished it within a week of its time) — ask again: pantheon recover ${quoteTeam(sp.name)}` } };
                 }
                 const inv = this.completeRecovery(sp, u, device.pubkey, "waited", null, null, ip, acct.id);
                 return { status: 200, body: { state: "moved", space: sp.name, invite: inv.token, how: "waited" } };
@@ -1886,8 +1969,8 @@ button:hover{background:#1d4ed8}.err{color:var(--bad);margin-top:12px}
             from: RELAY_MEMBER, act: "NOTE", to: null, goal: null, ref: null,
             body: `${u} asked to move their seat in this team to a new device ("${r.label}", key ${r.fp}), after signing in with the code emailed to their account. ` +
                 `It moves ${hours ? `in ${hours} hour${hours === 1 ? "" : "s"}` : "when they ask again"} unless someone cancels, or sooner when a teammate confirms it. ` +
-                `If you have checked with ${u} (a call, or a channel you trust) that the new key is theirs:  pantheon recover confirm ${u} --space ${sp.name}. ` +
-                `If it is not them:  pantheon recover cancel ${u} --space ${sp.name}.`,
+                `If you have checked with ${u} (a call, or a channel you trust) that the new key is theirs:  pantheon recover confirm ${u} --space ${quoteTeam(sp.name)}. ` +
+                `If it is not them:  pantheon recover cancel ${u} --space ${quoteTeam(sp.name)}.`,
             meta: { notice: "seat_recovery", recovery: { member: u, fp: r.fp, from_fp: oldFp, state: "waiting", due_at: r.due_at } },
         });
         this.broadcast(sp, { t: "members", members: this.members(sp) });
@@ -1896,7 +1979,7 @@ button:hover{background:#1d4ed8}.err{color:var(--bad);margin-top:12px}
     recoveryProofRefusal(A, acct, device, name, seat, founder) {
         const proof = A.lastEmailProof(acct.id, device.fingerprint);
         if (!proof || Date.now() - proof > RECOVERY_PROOF_MS) {
-            return { status: 403, body: { reason: "sign_in_again", error: `first prove it is you: run  pantheon recover ${name}  in a terminal (it emails a code to ${acct.email} and asks for it), or sign in with  pantheon signin  and within ${RECOVERY_PROOF_MS / 60_000} minutes run  pantheon recover ${name}  again` } };
+            return { status: 403, body: { reason: "sign_in_again", error: `first prove it is you: run  pantheon recover ${quoteTeam(name)}  in a terminal (it emails a code to ${acct.email} and asks for it), or sign in with  pantheon signin  and within ${RECOVERY_PROOF_MS / 60_000} minutes run  pantheon recover ${quoteTeam(name)}  again` } };
         }
         const oldFp = fingerprint(seat.publicKey);
         if (!A.wasDeviceOf(acct.id, oldFp, seat.moved?.at ?? seat.first_seen)) {
@@ -1954,7 +2037,7 @@ button:hover{background:#1d4ed8}.err{color:var(--bad);margin-top:12px}
             meta: { notice: "seat_recovery", recovery: { member, fp: toFp, from_fp: fromFp, state: "moved", how, ...(by ? { by } : {}) } },
         });
         this.broadcast(sp, { t: "members", members: this.members(sp) });
-        this.broadcast(sp, { t: "roster", roster: this.roster(sp) });
+        this.broadcast(sp, () => ({ t: "roster", roster: this.roster(sp) }));
         return inv;
     }
     mintRecoveryInvite(sp, member) {
@@ -2600,15 +2683,19 @@ ${url.searchParams.get("done") ? `<div class=done role=status>${esc(url.searchPa
         res.writeHead(200, { "content-type": "text/csv", "content-disposition": `attachment; filename="${table}.csv"` });
         res.end(csv);
     }
-    adminBackup(res, A) {
+    async adminBackup(res, A) {
+        const dir = join(this.dataDir, "backups");
+        let fresh = null;
         try {
-            A.backup(join(this.dataDir, "backups"));
+            fresh = await A.backupAsync(dir);
         }
         catch (e) {
             this.log(`! backup snapshot before archive failed: ${e.message}`);
         }
+        if (res.destroyed)
+            return;
         const stamp = new Date().toISOString().replace(/[:.]/g, "-");
-        const tar = spawn("tar", ["czf", "-", "-C", this.dataDir, "."], { stdio: ["ignore", "pipe", "pipe"] });
+        const tar = spawn("tar", ["czf", "-", "-C", this.dataDir, ...backupExcludes(dir, fresh), "."], { stdio: ["ignore", "pipe", "pipe"] });
         let err = "";
         tar.stderr.on("data", (c) => { err += String(c); });
         tar.on("error", (e) => {
@@ -2864,6 +2951,11 @@ pantheon start</code></pre>
                             return reject("NAME_TAKEN", `a team with a name too close to "${m.space}" already exists here — use its exact name, or pick another`);
                         }
                         const isNew = !this.live(m.space) && !this.spaceOnDisk(m.space);
+                        const badName = isNew ? teamNameProblem(m.space, { relay: true }) : null;
+                        if (badName) {
+                            this.log(`! REFUSED ${m.member}: cannot create a team called ${JSON.stringify(m.space.slice(0, 80))} (${badName.split(":")[0]})`);
+                            return reject("BAD_NAME", badName);
+                        }
                         if (isNew && this.nameCollides(m.space)) {
                             return reject("NAME_TAKEN", `a team with a name too close to "${m.space}" already exists here — pick another name`);
                         }
@@ -2896,8 +2988,14 @@ pantheon start</code></pre>
                             }
                         }
                         sp = this.getOrCreateSpace(m.space, m.token, { claim: { member: m.member, pubkey: m.pubkey }, retired: true });
-                        if (!sp)
+                        if (!sp) {
+                            const own = pre?.ok ? this.rosterOf(m.space)?.get(pre.account.username) : undefined;
+                            if (own?.admitted && pre?.ok) {
+                                const name = this.live(m.space)?.name ?? m.space;
+                                return reject("BAD_TOKEN", `You're already a member of ${name}. On this machine run: pantheon recover ${quoteTeam(name)}`);
+                            }
                             return reject("BAD_TOKEN", "wrong token for this space");
+                        }
                         staleToken = this.tokenState(sp, m.token) === "retired";
                         if (isNew) {
                             this.noteSpaceCreate(ip);
@@ -2964,10 +3062,10 @@ pantheon start</code></pre>
                         if (moved && moved.from_fp === fingerprint(m.pubkey)) {
                             const how = moved.how === "confirmed" ? `, confirmed by ${moved.by}` : moved.how === "alone" ? "" : ", after nobody cancelled it";
                             return reject("KEY_MISMATCH", `"${m.member}"'s seat in "${sp.name}" moved to another device on ${new Date(moved.at).toISOString().slice(0, 16).replace("T", " ")} UTC — signed in with the code emailed to ${m.member}'s account${how}. This machine's key no longer opens it. ` +
-                                `If that was you, use the new machine (or run  pantheon recover ${sp.name}  here to move it back). If it was not you, your email account may be compromised: secure it, then tell your team.`);
+                                `If that was you, use the new machine (or run  pantheon recover ${quoteTeam(sp.name)}  here to move it back). If it was not you, your email account may be compromised: secure it, then tell your team.`);
                         }
                         const recover = this.requireAccounts && this.accounts
-                            ? ` If this is really you on a new machine and the old one is gone: run  pantheon recover ${sp.name}  here, in a terminal — it signs you in with your email and moves your seat to this machine.`
+                            ? ` If this is really you on a new machine and the old one is gone: run  pantheon recover ${quoteTeam(sp.name)}  here, in a terminal — it signs you in with your email and moves your seat to this machine.`
                             : "";
                         if (sp.allow.founder() === m.member) {
                             return reject("KEY_MISMATCH", `"${m.member}" founded "${sp.name}" and is registered here with a different key.` + (recover ||
@@ -3013,6 +3111,7 @@ pantheon start</code></pre>
                         delegated = { pub: m.delegate.pub, run: m.delegate.run, expires: m.delegate.expires };
                     }
                     space = sp;
+                    this.spaceOf.set(ws, sp);
                     myPubkey = m.pubkey;
                     clearHelloTimer();
                     sessionId = acctSession ?? this.mintSession();
@@ -3082,7 +3181,7 @@ pantheon start</code></pre>
                     if (firstSession && m.role !== "cli") {
                         this.commit(sp, { from: me.id, act: "JOIN", body: `${me.display} online (${me.agent})`, to: null, goal: null, ref: null });
                     }
-                    this.broadcast(sp, { t: "roster", roster: this.roster(sp) });
+                    this.broadcast(sp, () => ({ t: "roster", roster: this.roster(sp) }));
                     this.log(`[${sp.name}] + ${me.id}/${sessionId} (${m.role}, ${me.agent})${verdict === "founder" ? " [founder, auto-admitted]" : ""}`);
                     return;
                 }
@@ -3170,7 +3269,7 @@ pantheon start</code></pre>
                         this.log(`[${space.name}] * ADMITTED ${m.member} by ${me.id}${m.may_wake ? " (may wake workers)" : ""}`);
                     }
                     this.broadcast(space, { t: "members", members: this.members(space) });
-                    this.broadcast(space, { t: "roster", roster: this.roster(space) });
+                    this.broadcast(space, () => ({ t: "roster", roster: this.roster(space) }));
                     this.send(ws, { t: "ack", ...(m.cid ? { cid: m.cid } : {}), id: m.member, seq: space.ledger.head });
                     return;
                 }
@@ -3178,7 +3277,7 @@ pantheon start</code></pre>
                     if (role !== "worker")
                         return;
                     me.worker = { state: m.state, ...(m.detail ? { detail: m.detail } : {}), ...(m.until ? { until: m.until } : {}), at: Date.now() };
-                    this.broadcast(space, { t: "roster", roster: this.roster(space) });
+                    this.broadcast(space, () => ({ t: "roster", roster: this.roster(space) }));
                     return;
                 }
                 if (m.t === "delete_space") {
@@ -3384,6 +3483,7 @@ pantheon start</code></pre>
                     if (m.deadline !== undefined && m.deadline <= Date.now()) {
                         return fail("DEADLINE_PAST", `deadline ${new Date(m.deadline).toISOString()} is already in the past`, m.cid);
                     }
+                    space.ledger.checkBacklog();
                     const goal = {
                         id: m.id && !space.goals.has(m.id) ? m.id : newId("g"),
                         title: m.title,
@@ -3489,6 +3589,7 @@ pantheon start</code></pre>
                 const wasFull = this.spaceFull(space);
                 if (wasFull && !recoveryStep && !this.freesRoom(space, me.id, m.act, m.ref ?? null))
                     return fail("SPACE_FULL", this.spaceFullMessage(space), m.cid);
+                space.ledger.checkBacklog();
                 const rec = this.commit(space, {
                     ...(m.sig && m.id ? { id: m.id } : {}),
                     from: me.id, session: sessionId, role, act: m.act, body: m.body,
@@ -3540,6 +3641,11 @@ pantheon start</code></pre>
                     send("SPACE_FULL", `${e.message}. Nothing new can be recorded until finished goals are closed (the relay then compacts their old acts), or the relay operator raises PANTHEON_SPACE_MAX_BYTES.`);
                     return;
                 }
+                if (e instanceof DiskBusy) {
+                    this.log(`! [${space?.name ?? "?"}] ${e.message} — refusing new acts until it catches up`);
+                    send("STORAGE_BUSY", `${e.message}, so this was not recorded. Wait a few seconds, then send it again.`);
+                    return;
+                }
                 if (e instanceof RosterUnreadable) {
                     this.log(`!! ${e.message} — every member of this space is refused until it is repaired (it was left in place)`);
                     if (!space)
@@ -3548,7 +3654,7 @@ pantheon start</code></pre>
                     return;
                 }
                 this.noteStorageFailure(`[${space?.name ?? "?"}] ${lastMsg || "message"}`, e);
-                send("STORAGE", "the relay could not save that — try again in a moment");
+                send("STORAGE", "the relay's disk is refusing writes, so this was not recorded — try again in a moment");
             }
         });
         const closeWork = (code) => {
@@ -3587,7 +3693,7 @@ pantheon start</code></pre>
                 this.commit(space, { from: me.id, act: "LEAVE", body: `${me.display} offline`, to: null, goal: null, ref: null });
             }
             this.pushObligations(space);
-            this.broadcast(space, { t: "roster", roster: this.roster(space) });
+            this.broadcast(space, () => ({ t: "roster", roster: this.roster(space) }));
             this.log(`[${space.name}] - ${me.id}/${sessionId}`);
         };
         ws.on("close", (code) => { try {
@@ -3598,6 +3704,20 @@ pantheon start</code></pre>
         } });
         ws.on("error", () => { });
     }
+    rotateLoopWindow() {
+        const h = this.loopDelay;
+        if (!h)
+            return;
+        if (h.count > 0)
+            this.loopPrev = { p50: loopMs(h.percentile(50)), p99: loopMs(h.percentile(99)), max: loopMs(h.max) };
+        h.reset();
+    }
+    loopStats() {
+        const h = this.loopDelay;
+        const cur = h.count > 0 ? { p50: loopMs(h.percentile(50)), p99: loopMs(h.percentile(99)), max: loopMs(h.max) } : null;
+        const at = cur ?? this.loopPrev ?? { p50: 0, p99: 0, max: 0 };
+        return { p50: at.p50, p99: at.p99, max_1m: Math.max(cur?.max ?? 0, this.loopPrev?.max ?? 0) };
+    }
     noteStalls() {
         const worst = this.lagMax;
         this.lagMax = 0;
@@ -3896,7 +4016,7 @@ pantheon start</code></pre>
             sp.store.rebuilt = null;
         this.pushObligations(sp);
         this.broadcast(sp, { t: "act", act: rec });
-        this.broadcast(sp, { t: "roster", roster: this.roster(sp) });
+        this.broadcast(sp, () => ({ t: "roster", roster: this.roster(sp) }));
         return rec;
     }
     effects(sp, rec, ctx) {
@@ -4158,6 +4278,8 @@ pantheon start</code></pre>
                 continue;
             if (this.liveSockets(sp))
                 continue;
+            if (busyWriter(this.dataDir, safeName(sp.name)))
+                continue;
             const idle = now - sp.lastTouch;
             const junk = this.isJunkSpace(sp);
             try {
@@ -4182,12 +4304,8 @@ pantheon start</code></pre>
     }
     deleteSpaceFiles(name) {
         const safe = name.replace(/[^a-z0-9._-]/gi, "_");
-        for (const suffix of [".jsonl", ".state.json", ".members.json"]) {
-            try {
-                rmSync(join(this.dataDir, `${safe}${suffix}`), { force: true });
-            }
-            catch { }
-        }
+        for (const suffix of [".jsonl", ".state.json", ".members.json"])
+            writerFor(this.dataDir, safe).remove(join(this.dataDir, `${safe}${suffix}`));
     }
     getOrCreateSpace(name, token, opts = {}) {
         const claim = opts.claim;
@@ -4229,6 +4347,17 @@ pantheon start</code></pre>
         store.headOf = () => ledger.head;
         store.onError = (e) => this.noteStorageFailure(`[${name}] snapshot save`, e);
         const restored = store.load({ quarantine: true, expectHistory: ledger.size > 0 });
+        if (!restored.unclean && restored.ledgerHead !== undefined && restored.ledgerHead > ledger.head) {
+            if (ledger.size > 0) {
+                try {
+                    copyFileSync(store.file, `${store.file}.ahead-${Date.now()}`);
+                }
+                catch { }
+                restored.unclean = `the state snapshot counts acts up to seq ${restored.ledgerHead}, but the ledger ends at seq ${ledger.head}`;
+            }
+            else
+                this.log(`!! [${name}] the state snapshot counts acts up to seq ${restored.ledgerHead}, but the ledger is empty — kept the snapshot; the ledger file may have been lost`);
+        }
         const sp = {
             name, token: raw,
             ledger,
@@ -4483,23 +4612,78 @@ pantheon start</code></pre>
         }));
     }
     send(ws, msg) {
-        if (ws.readyState === 1)
-            ws.send(JSON.stringify(msg));
+        const sp = this.spaceOf.get(ws);
+        const payload = JSON.stringify(msg);
+        const run = () => { if (ws.readyState === 1)
+            ws.send(payload); };
+        if (!sp)
+            return run();
+        const instead = msg.t === "ack" && msg.cid ? { cid: msg.cid } : msg.t === "welcome" ? {} : null;
+        const message = msg.t === "welcome" ? "the relay's disk is refusing writes, so it cannot let you in right now — reconnect in a moment" : STORAGE_HELD;
+        this.out(sp, run, instead ? { ws, payload: JSON.stringify({ t: "error", code: "STORAGE", message, ...instead }), hangUp: msg.t === "welcome" } : undefined);
     }
     broadcast(sp, msg) {
-        const payload = JSON.stringify(msg);
-        const seq = msg.t === "act" ? msg.act.seq : null;
-        for (const m of sp.members.values()) {
-            let reached = false;
-            for (const s of m.sessions.values())
-                if (s.ws.readyState === 1) {
-                    s.ws.send(payload);
-                    if (!s.quiet)
-                        reached = true;
-                }
-            if (reached && seq !== null && seq > m.delivered_seq)
-                m.delivered_seq = seq;
+        this.out(sp, () => {
+            const m = typeof msg === "function" ? msg() : msg;
+            const payload = JSON.stringify(m);
+            const seq = m.t === "act" ? m.act.seq : null;
+            for (const mem of sp.members.values()) {
+                let reached = false;
+                for (const s of mem.sessions.values())
+                    if (s.ws.readyState === 1) {
+                        s.ws.send(payload);
+                        if (!s.quiet)
+                            reached = true;
+                    }
+                if (reached && seq !== null && seq > mem.delivered_seq)
+                    mem.delivered_seq = seq;
+            }
+        });
+    }
+    out(sp, run, failed) {
+        const w = busyWriter(this.dataDir, safeName(sp.name));
+        if (!sp.outbox?.length && !w)
+            return run();
+        if (failed && w?.failing)
+            return this.sendFailed(failed);
+        (sp.outbox ??= []).push({ run, w, ticket: w ? w.ticket : 0, ...(failed ? { failed } : {}) });
+        if (w)
+            void w.whenWritten(w.ticket).then(() => this.release(sp));
+    }
+    release(sp) {
+        const q = sp.outbox;
+        while (q?.length) {
+            const h = q[0];
+            if (h.w && h.w.doneThrough < h.ticket)
+                break;
+            q.shift();
+            try {
+                h.run();
+            }
+            catch { }
+        }
+    }
+    failHeld(stem) {
+        const sp = [...this.spaces.values()].find((s) => safeName(s.name) === stem);
+        if (!sp?.outbox?.length)
+            return;
+        const failing = sp.outbox.filter((h) => h.failed && h.w && h.w.doneThrough < h.ticket);
+        if (!failing.length)
+            return;
+        sp.outbox = sp.outbox.filter((h) => !failing.includes(h));
+        for (const h of failing)
+            this.sendFailed(h.failed);
+        this.release(sp);
+    }
+    sendFailed(failed) {
+        if (failed.ws.readyState === 1) {
+            try {
+                failed.ws.send(failed.payload);
+            }
+            catch { }
         }
+        if (failed.hangUp)
+            this.hangUp(failed.ws, "STORAGE");
     }
 }
 function first(s, n = 120) {