@homie-rocks/studio 0.46.5 → 0.46.6

package/worker/index.mjs

@@ -90,7 +90,7 @@ import { appRole, openPath } from './app-format.mjs';
 import { playerEnabled, EMBED_GAME_JS } from './embed.mjs';
 import { NetRoom, WATCH_POLICIES, departure, errorLine, versionOf } from './room.mjs';
 import { hostedGame, hostedBuild, compiledFor, startHost } from './hosted.mjs';
-import { roomStore, restoreDelay } from './room-store.mjs';
+import { roomStore, restoreDelay, LOOSE_SAVE } from './room-store.mjs';
 import { appCors, isAppOrigin } from './standalone.mjs';
 import { ROOM_ID, badRoomPage, embedAncestors, embedPreview, frameAncestors, noWatchPage, playPage, shellKey, shellStandIn, watchPage } from './pages.mjs';
 import {
@@ -105,7 +105,7 @@ import { forgetLines, historyOf, keepLines, keptLine } from './lounge-store.mjs'
 import { isLoungePath, loungeRoutes } from './lounge.mjs';
 import { doorPage, serverPage, serversPage } from './site.mjs';
 import { isLocalOrigin, qrSvg } from './qr.mjs';
-import { acceptMultiplex, diagnosticsOf, gateFor, nextGate } from './gate.mjs';
+import { acceptMultiplex, diagnosticsOf, gateFor, gatePlayers, nextGate } from './gate.mjs';
 import { worldSource } from './world-wire.mjs';
 export { Gate, Concentrator } from './gate.mjs';
 import { seatCount, paramsFrom, perAddress, seatsOf } from './seats.mjs';
@@ -1361,6 +1361,9 @@ export class Table {
       if (this.saved?.retired === true) this.cleanupPending = true;
       this.recorded = (await ctx.storage.get('recorded')) ?? 0;
       this.officeSaved = (await ctx.storage.get('office')) ?? null;
+      // A big match room's last unconfirmed checkpoint (worker/room-store.mjs), read once as the object wakes.
+      try { this.looseSave = (await ctx.storage.list?.({ prefix: LOOSE_SAVE })) ?? null; }
+      catch (error) { this.looseSave = null; this.storageProblem(error); }
     });
   }
 
@@ -1434,7 +1437,7 @@ export class Table {
     try {
       const build = hostedBuild(this.game);
       const persistent = this.isPersistent();
-      try { this.durable ??= roomStore(this.ctx.storage, { persistent: this.isPersistent() }); } catch (error) { if (persistent) throw error; this.storageProblem(error); }
+      try { this.durable ??= this.openStore(); } catch (error) { if (persistent) throw error; this.storageProblem(error); }
       let saved = this.cleanupPending ? null : this.readRoom();
       restoring = Boolean(saved);
       if (this.persistentStored && build.settings.lifetime !== 'persistent') throw new Error('This room holds a lasting world. Keep room.lifetime persistent; its save is retained.');
@@ -1480,7 +1483,9 @@ export class Table {
       this.hostRt = startHost(this.game, {
         ...(saved ? { restore: new TextEncoder().encode((saved.hostText ?? JSON.stringify(saved.host))), restoreEpoch: epoch, startDelayMs: boot.count === 1 ? restoreDelay(`${this.game}/${this.code}`) : 0 } : {}),
         record: (owner,key,value) => this.queueRoundRecord(owner,key,value),
-        store: { save: (bytes, commitment) => this.saveRoom(bytes, commitment) },
+        store: { save: (bytes, commitment) => this.saveRoom(bytes, commitment, true) },
+        // A room behind Gates needs no snapshot as text: nothing is encoded that nobody is sent (room.mjs, hostSnap).
+        snapshotText: () => !this.relayed,
         ...(build.settings.characters ? { characterFor: (peer) => {
           const account = room.clients.get(peer.id)?.player;
           return account ? this.characterGrants.get(account) ?? null : null;
@@ -1520,12 +1525,19 @@ export class Table {
     }
   }
 
-  saveRoom(bytes, commitment) {
+  /** This object's room store, with the unconfirmed checkpoint it woke to (if any). */
+  openStore(persistent = this.isPersistent()) {
+    const loose = this.looseSave; this.looseSave = null;
+    return roomStore(this.ctx.storage, { persistent, loose, onProblem: (error) => this.storageProblem(error) });
+  }
+
+  /** `periodic`: the rules host's own checkpoint of a tick, as against a save an admission or a pause asked for. */
+  saveRoom(bytes, commitment, periodic = false) {
     if (this.persistentFailed) throw new Error('Lasting world stopped; restart from its committed save');
     const now = Date.now();
     if (now < (this.saveRetryAt ?? 0)) throw this.saveError ?? new Error('room save retry is waiting');
     try {
-      this.durable ??= roomStore(this.ctx.storage, { persistent: this.isPersistent() });
+      this.durable ??= this.openStore();
       const build = hostedBuild(this.game);
       const host = new TextDecoder().decode(bytes);
       const net = this.room.saved();
@@ -1544,6 +1556,10 @@ export class Table {
         commitment ??= { force: false, seconds: c.settings.durability.movementSeconds, tickHz: c.settings.tickHz,
           fields: { kinds: c.kinds.map(k => k.fields.flatMap(([,f],i) => f.durable ? [i] : [])), shared: c.shared.flatMap(([,f],i) => f.durable ? [i] : []) } };
         this.durable.commit(saved, commitment);
+      } else if (this.relayed && periodic && !clearing) {
+        // A room behind Gates: its once-a-second checkpoint holds none of its frames (worker/room-store.mjs). A save
+        // an admission asks for is still confirmed before the welcome that names the seat can leave.
+        this.durable.write(saved, { unconfirmed: true });
       } else this.durable.write(saved);
       if (clearing) { this.cleanupGeneration = (this.cleanupGeneration ?? 0) + 1; this.queueNetCleanup(false); }
       this.roomSave = saved;
@@ -1742,7 +1758,7 @@ export class Table {
     const room = this.room;
     if (!room || !this.hostRt) {
       if (this.ctx.storage.sql?.exec("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'save'").toArray().length) {
-        this.durable ??= roomStore(this.ctx.storage);
+        this.durable ??= this.openStore(false);
         const saved = this.cleanupPending ? null : this.readRoom(true);
         if (saved) {
           this.game = saved.game; this.code = saved.room;
@@ -2118,7 +2134,7 @@ export class Table {
       // players sent to a Gate in the last ten seconds who have not reached it yet.
       const gates = Number(new URL(request.url).searchParams.get('gates')) || 1, now = Date.now(), promised = new Map();
       for (const [index, due] of this.gatePromised) { const live = due.filter((at) => at > now); if (live.length) { this.gatePromised.set(index, live); promised.set(index, live.length); } else this.gatePromised.delete(index); }
-      const gate = nextGate(gates, this.gateHeld, promised);
+      const gate = nextGate(gates, this.gateHeld, promised, gatePlayers(this.env));
       if (!this.gatePromised.has(gate)) this.gatePromised.set(gate, []);
       this.gatePromised.get(gate).push(now + 10_000);
       return json({ gate });
@@ -2398,6 +2414,8 @@ export class Table {
       // A Gate's stamp on a player's message to this object handling it, ms.
       inputTransitMs: { n: transit.length, p50: at(0.5), p95: at(0.95), max: transit.at(-1) ?? null },
       durability: this.storageHealth ?? { ok: true },
+      // Checkpoints written without holding the room's frames for them (worker/room-store.mjs), so far.
+      save: { unconfirmed: this.durable?.unconfirmedWrites ?? 0 },
     });
     const work = diagnosticsOf(this.env, this.game, this.code).fetch('https://diagnostics/__diagnostics', { method: 'POST', body })
       .then((r) => r.body?.cancel?.())

package/worker/office.mjs

@@ -62,7 +62,7 @@ import { toolIdentity } from './tool-identity.mjs';
  * from an office key it is ALWAYS an ask (the AI may only propose one), and so is marking a referrer paid.
  */
 import { seatsOf } from './seats.mjs';
-import { diagnosticsOf, roomLayout } from './gate.mjs';
+import { diagnosticsOf, gatePlayers, roomLayout } from './gate.mjs';
 import { PUBLIC_SERVER, SERVER_LIMITS, checkServer, levelName, narrows, policyOf, roomServer, rowFor, serverView, serversOf, serverPassCookie, writeServer } from './servers.mjs';
 import { BRAIN_BUDGET, aiName, brainFor, fillSpot, passById, passCreate, passList, passRevoke, think } from './agents.mjs';
 import { DEFAULT_MODEL, NEURONS_PER_M, OWNER_MODEL, vocabularyOf } from './brain.mjs';
@@ -432,7 +432,7 @@ async function roomFacts(env, game, room, max) {
  * small room answers for itself, as it always has.
  */
 export async function roomDiagnostics(env, game, room, max) {
-  if (!env.GATE || roomLayout(max).gates === 1) return roomFacts(env, game, room, max);
+  if (!env.GATE || roomLayout(max, gatePlayers(env)).gates === 1) return roomFacts(env, game, room, max);
   let kept = null;
   try {
     const res = await diagnosticsOf(env, game, room).fetch(`https://diagnostics/__diagnostics?room=${enc(room)}`);

package/worker/room-store.mjs

@@ -12,8 +12,42 @@ import { normalizeWorld, denormalizeWorld, durableWorld, ownDurable, checkpointW
  * Only integer primary keys: no secondary-index write amplification.
  * The boot row outlives a save being replayed, so two restores of that save can never reuse an epoch.
  */
-export function roomStore(storage, { persistent = false } = {}) {
+export function roomStore(storage, { persistent = false, loose = null, onProblem = () => {} } = {}) {
   const sql = storage.sql;
+  /*
+   * THE CHECKPOINT THAT DOES NOT HOLD THE ROOM'S FRAMES. Cloudflare holds everything an object sends after a write
+   * until that write is confirmed. A match room behind Gates writes its whole self once a second, and every world
+   * and every receipt of the ticks that followed waited on it: one or two worlds a second reached every Gate a
+   * tick or more late, at every size. Such a checkpoint is a second old by design, and the worlds between two of
+   * them were never durable, so nothing a player sees is being promised. `write(value, { unconfirmed: true })`
+   * puts it in the object's key-value storage with `allowUnconfirmed`, which is Cloudflare's own way to say so.
+   *
+   * Every confirmed write removes it, so one found on disk was written after the last confirmed save, unless the
+   * object stopped between a confirmed write and that removal: each save says when it was made (`at`), and a read
+   * takes the later of the two. `loose` is what the object read of it when it woke (a Map from
+   * `storage.list({ prefix: LOOSE_SAVE })`); reads after that are answered from memory.
+   */
+  let looseText = null, looseOnDisk = false, looseWrites = 0;
+  if (loose && typeof loose.get === 'function' && loose.get(LOOSE_SAVE)) {
+    looseOnDisk = true;
+    try {
+      const meta = loose.get(LOOSE_SAVE), pieces = [];
+      for (let i = 0; i < meta.pieces; i += 1) { const piece = loose.get(`${LOOSE_SAVE}/${i}`); if (!(piece instanceof Uint8Array)) throw new Error('unconfirmed save pieces missing'); pieces.push(piece); }
+      const bytes = new Uint8Array(pieces.reduce((n, p) => n + p.length, 0));
+      let at = 0;
+      for (const piece of pieces) { bytes.set(piece, at); at += piece.length; }
+      if (!Number.isInteger(meta.pieces) || meta.pieces < 1 || bytes.length !== meta.bytes || bytes.length > 16_000_000) throw new Error('unconfirmed save does not fit its record');
+      looseText = new TextDecoder('utf-8', { fatal: true }).decode(bytes);
+      JSON.parse(looseText);
+    } catch (error) { looseText = null; onProblem(error); }
+  }
+  /** Removed with whatever confirmed write is being made: no await between them, so they are one transaction. */
+  function dropLoose() {
+    looseText = null;
+    if (!looseOnDisk || typeof storage.delete !== 'function') return;
+    looseOnDisk = false;
+    Promise.resolve(storage.delete([LOOSE_SAVE, ...Array.from({ length: LOOSE_PIECES }, (_, i) => `${LOOSE_SAVE}/${i}`)])).catch(onProblem);
+  }
   sql.exec('CREATE TABLE IF NOT EXISTS save (id INTEGER PRIMARY KEY, data BLOB NOT NULL)');
   sql.exec('CREATE TABLE IF NOT EXISTS boots (id INTEGER PRIMARY KEY, data TEXT NOT NULL)');
   let journalReady = false;
@@ -47,13 +81,22 @@ export function roomStore(storage, { persistent = false } = {}) {
       const total = sql.exec('SELECT SUM(length(data)) AS bytes FROM save').toArray()[0]?.bytes ?? 0;
       if (total > maxBytes) throw new Error('save exceeds 16 MB');
       const rows = sql.exec('SELECT id, data FROM save ORDER BY id').toArray();
-      if (!rows.length) return null;
-      if (rows.some((r, i) => r.id !== i) || rows.reduce((n, r) => n + r.data.byteLength, 0) > maxBytes) throw new Error('save pieces missing or save exceeds 16 MB');
-      const pieces = rows.map((r) => new Uint8Array(r.data));
-      const bytes = new Uint8Array(pieces.reduce((n, p) => n + p.length, 0));
-      let at = 0;
-      for (const p of pieces) { bytes.set(p, at); at += p.length; }
-      let value = JSON.parse(decoder.decode(bytes));
+      if (!rows.length && looseText === null) return null;
+      let value = null;
+      if (rows.length) {
+        if (rows.some((r, i) => r.id !== i) || rows.reduce((n, r) => n + r.data.byteLength, 0) > maxBytes) throw new Error('save pieces missing or save exceeds 16 MB');
+        const pieces = rows.map((r) => new Uint8Array(r.data));
+        const bytes = new Uint8Array(pieces.reduce((n, p) => n + p.length, 0));
+        let at = 0;
+        for (const p of pieces) { bytes.set(p, at); at += p.length; }
+        value = JSON.parse(decoder.decode(bytes));
+      }
+      // An unconfirmed checkpoint is the room when it is the later of the two (above).
+      if (looseText !== null && !boot?.persistent) {
+        const unconfirmed = JSON.parse(looseText);
+        if (!value || !(Number(value.at) >= Number(unconfirmed.at))) value = unconfirmed;
+      }
+      if (!value) return null;
       if (lasting) journal();
       sequence = floor;
       const journals = lasting ? sql.exec('SELECT id, seq, data FROM world_journal WHERE id > 0 AND seq > ? ORDER BY seq', floor).toArray() : [];
@@ -127,10 +170,20 @@ export function roomStore(storage, { persistent = false } = {}) {
       // Brought up to this tick by what changed in it (each value copied), never by copying the room.
       durable = applyChanges(durable, moved);
     },
-    write(value) {
+    /** `unconfirmed`: a match room's periodic checkpoint, which no frame waits on (above). Never a lasting world's. */
+    write(value, { unconfirmed = false } = {}) {
       if (lasting) journal();
-      const bytes = encoder.encode(JSON.stringify(value));
+      const text = JSON.stringify(value);
+      const bytes = encoder.encode(text);
       if (bytes.length > 16_000_000 && !lasting) throw new Error('save exceeds 16 MB');
+      if (unconfirmed && !lasting && typeof storage.put === 'function' && bytes.length <= LOOSE_PIECES * pieceBytes) {
+        const entries = { [LOOSE_SAVE]: { pieces: Math.max(1, Math.ceil(bytes.length / pieceBytes)), bytes: bytes.length, at: value.at ?? null } };
+        for (let i = 0; i < entries[LOOSE_SAVE].pieces; i += 1) entries[`${LOOSE_SAVE}/${i}`] = bytes.slice(i * pieceBytes, (i + 1) * pieceBytes);
+        // One put is one transaction: a reader never finds the record of one checkpoint with the pieces of another.
+        Promise.resolve(storage.put(entries, { allowUnconfirmed: true })).catch(onProblem);
+        looseText = text; looseOnDisk = true; looseWrites += 1;
+        return;
+      }
       storage.transactionSync(() => {
         let id = 0;
         for (let at = 0; at < bytes.length; at += pieceBytes) sql.exec('INSERT INTO save (id, data) VALUES (?, ?) ON CONFLICT(id) DO UPDATE SET data = excluded.data', id++, bytes.slice(at, at + pieceBytes));
@@ -138,16 +191,23 @@ export function roomStore(storage, { persistent = false } = {}) {
         // The checkpoint holds every record so far: raising the floor retires them without touching their rows.
         if (lasting && sequence !== floor) sql.exec("INSERT INTO world_journal (id, seq, data) VALUES (0, ?, x'') ON CONFLICT(id) DO UPDATE SET seq = excluded.seq", sequence);
       });
+      dropLoose();
       floor = sequence;
       if (lasting) settle(value);
     },
+    /** Checkpoints written without holding the room's frames, so far. */
+    get unconfirmedWrites() { return looseWrites; },
     boots() { const value = JSON.parse(sql.exec('SELECT data FROM boots WHERE id = 1').toArray()[0]?.data ?? 'null'); lasting ||= value?.persistent === true; return value; },
     boot(value) { sql.exec('INSERT INTO boots (id, data) VALUES (1, ?) ON CONFLICT(id) DO UPDATE SET data = excluded.data', JSON.stringify({ ...value, ...(lasting ? { persistent: true } : {}) })); },
     retire() { this.boot({ retired: true }); },
-    clear() { if (lasting) journal(); storage.transactionSync(() => { sql.exec('DELETE FROM save'); sql.exec('DELETE FROM boots'); if (lasting) sql.exec('DELETE FROM world_journal'); }); journalReady = false; base = null; durable = null; checkpoint = null; committedInputs = null; sequence = 0; floor = 0; },
+    clear() { dropLoose(); if (lasting) journal(); storage.transactionSync(() => { sql.exec('DELETE FROM save'); sql.exec('DELETE FROM boots'); if (lasting) sql.exec('DELETE FROM world_journal'); }); journalReady = false; base = null; durable = null; checkpoint = null; committedInputs = null; sequence = 0; floor = 0; },
   };
 }
 
+/** Where a match room's unconfirmed checkpoint is kept in the object's key-value storage: a record, and its pieces under it. */
+export const LOOSE_SAVE = 'room-save';
+const LOOSE_PIECES = 16;
+
 /** A repeatable delay spreads rooms that shared a failing isolate over thirty seconds. */
 export function restoreDelay(id) {
   let h = 2166136261;

package/worker/room.mjs

@@ -148,7 +148,10 @@ export function medianVote(list) {
   return v.length ? v[Math.floor((v.length - 1) / 2)] : null;
 }
 const utf8 = new TextEncoder();
-const byteLength = text => utf8.encode(text).length;
+// A message's size in bytes. Text that is all ASCII (nearly every frame a game sends) is as long as it is wide: it is
+// not encoded to be measured, which at ten inputs a second from a thousand players was a twentieth of a room's second.
+const ASCII = /^[\x00-\x7f]*$/;
+const byteLength = text => ASCII.test(text) ? text.length : utf8.encode(text).length;
 const PALETTE_SIZE = 12;
 const DEVICE_RANK = { desk: 0, tv: 1, phone: 2 };
 
@@ -473,6 +476,12 @@ export class NetRoom {
     const w = (c.rttWindow ??= []);
     w.push([now, ms]);
     while (w.length > 256 || (w.length > 1 && w[0][0] < now - 10000)) w.shift();
+    // A low percentile of ten seconds does not move in one. For a player behind a Gate (a big room) a full window
+    // is read once a second, not once an input: at ten inputs a second from a thousand players, sorting it each time
+    // was a tenth of the room's second when its clock moved between frames (0.46.3 against the build before it,
+    // test/crowd-room-cost.mjs). A room its players reach directly reads it as before.
+    if (c.conn.snapshotGroup && w.length >= 32 && now - (c.rttReadAt ?? 0) < 1000) return;
+    c.rttReadAt = now;
     const sorted = w.map((x) => x[1]).sort((a, b) => a - b);
     const low = sorted[Math.floor((sorted.length - 1) * 0.1)];
     if (c.measuredRtt !== undefined && Math.abs(low - c.measuredRtt) < 2 && now - (c.rttToldAt ?? 0) < 1000) return;
@@ -495,7 +504,10 @@ export class NetRoom {
     while (this.snapTimes.length && this.snapTimes[0] < now - 1000) this.snapTimes.shift();
     // A heartbeat (revision 9): the host's last state again while its frames hitch. It counts as a snapshot here
     // (the host is alive), and replicas are told which it is.
-    const out = text ?? JSON.stringify({ t: 'snap', from: c ? c.seat : null, ...snap, ...(m.hb === 1 ? { hb: 1 } : {}) });
+    // The whole snapshot as text: what a player with no view of their own is sent. The server's host hands it over
+    // unless this room sends it to nobody (a room behind Gates); then it is made only if somebody turns out to need it.
+    let out = text ?? (c ? JSON.stringify({ t: 'snap', from: c.seat, ...snap, ...(m.hb === 1 ? { hb: 1 } : {}) }) : null);
+    const whole = () => (out ??= JSON.stringify({ t: 'snap', from: null, ...snap, ...(m.hb === 1 ? { hb: 1 } : {}) }));
     if (m.hb === 1) this.stats.heartbeats = (this.stats.heartbeats ?? 0) + 1;
     this.lastSnapText = out;
     // A server-hosted room behind Gates hands each Gate ONE world a tick; the Gate selects, encodes and paces
@@ -513,9 +525,10 @@ export class NetRoom {
       }
       // A congested socket skips a snapshot rather than queueing a stale one.
       if (o.conn.buffered && o.conn.buffered() > 256 * 1024) { this.stats.drops += 1; continue; }
-      this.sendText(o.conn, this.hasViewSchedule() ? JSON.stringify({ t: 'snap', from: null, ...this.playerSnapshot(o, snap) }) : out);
+      this.sendText(o.conn, this.hasViewSchedule() ? JSON.stringify({ t: 'snap', from: null, ...this.playerSnapshot(o, snap) }) : whole());
     }
     if (groups) for (const group of groups) group.snapshot(snap, this.server.viewSettings ?? {}, this.tickHz);
+    if (out !== null) this.lastSnapText = out;
   }
 
   hasViewSchedule() {

package/worker/version.mjs

@@ -1,2 +1,2 @@
 /** The @homie-rocks/studio version this Worker code is (package.json's; a test keeps them equal). Cache-busts /_homie/site.js. */
-export const STUDIO_VERSION_TAG = '0.46.5';
+export const STUDIO_VERSION_TAG = '0.46.6';