@homie-rocks/studio 0.46.5 → 0.46.6

package/lib/crowd-probe/worker.mjs

@@ -11,6 +11,7 @@
  * It proves the room and its Gates at that many players. It says nothing of the last hop to a real player.
  */
 import { Tally, crowdClient } from './client.mjs';
+import { referenceWork } from './reference-work.mjs';
 
 const sleep = (ms) => new Promise((done) => setTimeout(done, ms));
 
@@ -18,20 +19,41 @@ export default {
   async fetch(request, env, ctx) {
     const url = new URL(request.url);
     if (!env.PROBE_KEY || request.headers.get('x-probe-key') !== env.PROBE_KEY) return new Response('not found', { status: 404 });
-    if (url.pathname === '/hello') return Response.json({ ok: true, probe: 1 });
+    if (url.pathname === '/hello') return Response.json({ ok: true, probe: 2 });
+    if (url.pathname === '/calibrate') {
+      // The reference work, `rounds` times, in one event. This Worker cannot time it: its clock stands still while
+      // its code runs and reads a timer's due time when the timer fires, so whoever asked holds the stopwatch (the
+      // time this request took, less the time one with no rounds took). `own` is what this Worker's clock said the
+      // rounds took, across an awaited timer, for the record of what such a figure is worth.
+      const rounds = Math.min(20, Math.max(0, Math.floor(Number(url.searchParams.get('rounds') ?? 0)) || 0));
+      await sleep(2);
+      const at = Date.now(); let check = 0;
+      for (let i = 0; i < rounds; i += 1) check = referenceWork();
+      await sleep(1);
+      return Response.json({ ok: true, colo: request.cf?.colo ?? null, rounds, check, own: Date.now() - at });
+    }
     const origin = url.searchParams.get('origin'), game = url.searchParams.get('game') ?? 'crowd-circuit', room = url.searchParams.get('room');
     const from = Number(url.searchParams.get('from') ?? 0), count = Math.min(5, Math.max(1, Number(url.searchParams.get('count') ?? 5)));
     const joinAt = Number(url.searchParams.get('joinAt')), rampMs = Number(url.searchParams.get('rampMs') ?? 50);
     const measureAt = Number(url.searchParams.get('measureAt')), endAt = Number(url.searchParams.get('endAt'));
+    // `compress=1`: offer permessage-deflate as a browser does, so the studio's Gates compress for these players too.
+    const compress = url.searchParams.get('compress') === '1';
+    // `slowEvery=K`: every Kth player is a slow one (so every Gate holds some, beside healthy players). Each takes what it is sent at six frames a second, in
+    // order, as a connection that carries no more does: it falls behind, says so (the tick it holds), and the
+    // question is what that costs the players beside it. (A Worker cannot slow a socket's reads; a real slow
+    // connection is played from a computer: `crowd-load.mjs --slow-every`.)
+    const slowEvery = Math.max(0, Math.floor(Number(url.searchParams.get('slowEvery') ?? 0)) || 0);
     if (!/^https:\/\/[a-z0-9.-]+$/.test(origin ?? '') || !/^[A-Za-z0-9_-]{1,40}$/.test(room ?? '') || ![joinAt, measureAt, endAt].every(Number.isFinite) || endAt - joinAt > 900_000) return new Response('bad request', { status: 400 });
     const { readable, writable } = new TransformStream(), writer = writable.getWriter(), encoder = new TextEncoder();
     const line = (o) => writer.write(encoder.encode(JSON.stringify(o) + '\n')).catch(() => {});
     const tallies = { ack: new Tally(), gaps: new Tally(), loops: new Tally() }, players = [];
+    // A slow player's own figures, apart: its acknowledgements are late by what it is.
+    const slowTallies = { ack: new Tally(), gaps: new Tally(), loops: new Tally() };
     const run = async () => {
       const join = async (i) => {
         const key = crypto.randomUUID().replaceAll('-', '');
-        const player = { socket: null, client: null };
-        player.client = crowdClient({ index: i, game, tallies, send: (text) => { try { player.socket?.send(text); } catch { /* closing */ } } });
+        const player = { socket: null, client: null, compressed: false, slow: slowEvery > 0 && i % slowEvery === 0, held: [], nextAt: 0 };
+        player.client = crowdClient({ index: i, game, tallies: player.slow ? slowTallies : tallies, send: (text) => { try { player.socket?.send(text); } catch { /* closing */ } } });
         players.push(player);
         try {
           await sleep(Math.max(0, joinAt + i * rampMs - Date.now()));
@@ -40,31 +62,36 @@ export default {
           const match = (await page.text()).match(/window\.HOMIE_NET=(\{[^<]+\})<\/script>/);
           if (!page.ok || !match) throw new Error(`bootstrap HTTP ${page.status}`);
           const cfg = JSON.parse(match[1]);
-          const answer = await fetch(cfg.url.replace(/^ws/, 'http'), { headers: { Upgrade: 'websocket', Origin: 'null' } });
+          const answer = await fetch(cfg.url.replace(/^ws/, 'http'), { headers: { Upgrade: 'websocket', Origin: 'null', ...(compress ? { 'Sec-WebSocket-Extensions': 'permessage-deflate; client_max_window_bits' } : {}) } });
+          player.compressed = /permessage-deflate/.test(answer.headers?.get?.('sec-websocket-extensions') ?? '');
           const socket = answer.webSocket;
           if (!socket) throw new Error(`upgrade HTTP ${answer.status}`);
           socket.accept(); player.socket = socket;
-          socket.addEventListener('message', (event) => { if (typeof event.data === 'string') player.client.message(event.data, Date.now()); });
+          socket.addEventListener('message', (event) => { if (typeof event.data !== 'string') return; if (player.slow) player.held.push(event.data); else player.client.message(event.data, Date.now()); });
           socket.addEventListener('close', (event) => player.client.close(event.code, event.reason));
           socket.addEventListener('error', () => player.client.close(1006, 'error'));
           player.client.open(Date.now());
         } catch (error) { player.client.c.errors.push(String(error?.message ?? error).slice(0, 120)); player.client.close(0, 'never connected'); }
       };
       const joins = Promise.all(Array.from({ length: count }, (_, n) => join(from + n)));
-      let measuring = false, lastLine = 0, lastTick = Date.now(), worstLag = 0;
+      let measuring = false, lastLine = 0, lastWhole = 0, lastTick = Date.now(), worstLag = 0;
+      // Everything measured so far: the last line of a step, and every thirty seconds on the way, so a request that
+      // is cut off before its end has still said most of what it saw.
+      const whole = (t, now) => ({ t, from, count, colo: request.cf?.colo ?? null, measuredMs: Math.max(0, Math.min(now, endAt) - measureAt), worstTimerLagMs: worstLag, players: players.map((p) => ({ ...p.client.facts(), compressed: p.compressed, slow: p.slow, ...(p.slow ? { waiting: p.held.length } : {}) })), tallies: { ack: tallies.ack.bins, gaps: tallies.gaps.bins, loops: tallies.loops.bins }, slowAck: slowTallies.ack.bins });
       while (Date.now() < endAt) {
         const now = Date.now();
         // This Worker's own lateness: how long past ten milliseconds its timer came back.
         worstLag = Math.max(worstLag, now - lastTick - 10); lastTick = now;
         if (!measuring && now >= measureAt) { measuring = true; for (const p of players) p.client.measure(); }
-        for (const p of players) p.client.tick(now);
+        for (const p of players) { if (p.slow && p.held.length && now >= p.nextAt) { p.nextAt = now + 160; p.client.message(p.held.shift(), now); } p.client.tick(now); }
+        if (measuring && now - lastWhole >= 30000) { lastWhole = now; await line(whole('partial', now)); }
         if (now - lastLine >= 5000) { lastLine = now; await line({ t: 'sample', at: now, from, joined: players.filter((p) => p.client.c.first !== null).length, open: players.filter((p) => p.socket && !p.client.c.closed).length, measuring }); }
         await sleep(10);
       }
-      await joins;
-      const facts = players.map((p) => p.client.facts());
+      await Promise.race([joins, sleep(5000)]);
+      const last = whole('final', endAt);
       for (const p of players) { try { p.socket?.close(1000, 'probe step complete'); } catch { /* gone */ } }
-      await line({ t: 'final', from, count, colo: request.cf?.colo ?? null, measuredMs: Math.max(0, endAt - measureAt), worstTimerLagMs: worstLag, players: facts, tallies: { ack: tallies.ack.bins, gaps: tallies.gaps.bins, loops: tallies.loops.bins } });
+      await line(last);
     };
     ctx.waitUntil(run().catch((error) => line({ t: 'failed', from, error: String(error?.stack ?? error).slice(0, 400) })).finally(() => writer.close().catch(() => {})));
     return new Response(readable, { headers: { 'content-type': 'application/x-ndjson', 'cache-control': 'no-store' } });

package/lib/crowd-summary.mjs

@@ -32,10 +32,17 @@ export function summarize(report, rows = [], link = null) {
     from: report.generator === 'cloudflare' ? 'cloudflare' : 'home',
     n: report.n, total: report.total, room: report.room, toolkit: report.toolkit ?? null, stopReason: report.stopReason ?? null,
     joined: report.joined, connectedAtEnd: report.connectedAtEnd, disconnects: report.disconnects,
+    // In-Cloudflare steps: probe requests whose report never came (their players were in the room all the same).
+    lostReports: report.lostReports ?? null,
+    // What the step varied: compression offered or not.
+    variant: (report.variant ?? (report.compression?.offered === false ? 'plain' : 'deflate')) + (report.slowPlayers ? ` +${report.slowPlayers} slow` : ''),
+    // With deliberately slow players in the step: what everybody else got.
+    healthy: report.slowPlayers ? report.healthy : null,
     client: {
       inputAckMs: { p50: report.inputAckMs?.p50 ?? null, p95: report.inputAckMs?.p95 ?? null, p99: report.inputAckMs?.p99 ?? null },
       // A player's tick progression and the frames they received, a second: the median player, and the worst.
-      tickHz: { median: round(report.tickHz?.median), min: round(report.tickHz?.min) },
+      // (From the in-Cloudflare probe this is by a Worker's clock, which runs slow when the Worker is late: not shown.)
+      tickHz: report.tickHzClock === 'probe' ? { median: null, min: null } : { median: round(report.tickHz?.median), min: round(report.tickHz?.min) },
       deliveredHz: { median: round(report.deliveredHz?.median), min: round(report.deliveredHz?.min) },
       receiveGapMs: { p95: report.receiveGapMs?.p95 ?? null, p99: report.receiveGapMs?.p99 ?? null },
       ackExpired: report.ackExpired ?? null,
@@ -78,9 +85,28 @@ export function summarize(report, rows = [], link = null) {
       roomTickHzAtGates: { median: round(pick(gates.map((g) => g.roomTickHz), 0.5)), p05: round(pick(gates.map((g) => g.roomTickHz), 0.05)) }, worldGapMsMax: pick(gates.map((g) => g.worldGapMs), 1),
       // The room's clock less a Gate's (median Gate), and how long the room says it held a Gate's frame.
       roomClockOffsetMs: pick(gates.map((g) => g.roomClock?.offsetMs), 0.5), channelDelayMs: pick(gates.map((g) => g.roomClock?.delayMs), 0.5),
+      // A Gate's own lateness: the worst its 20 ms timer came late (the worst Gate-second), and the share of Gate-seconds with any timer over 20 ms late.
+      gateLoopLateMs: { median: pick(gates.map((g) => g.loopLateMs), 0.5), p95: pick(gates.map((g) => g.loopLateMs), 0.95) }, gateBusyShare: gates.length ? round(gates.filter((g) => (g.loopLateTimers ?? 0) > 0).length / gates.length, 3) : null,
       gateRttMs: { median: round(pick(gates.map((g) => g.rttMs), 0.5), 0), p95: round(pick(gates.map((g) => g.rttMs), 0.95), 0) },
       inputOffsetMs: pick(gates.map((g) => g.inputOffsetMs), 0.5),
       clientLagTicks: { p95: pick(gates.map((g) => g.clientLagTicks?.p95), 0.95), max: pick(gates.map((g) => g.clientLagTicks?.max), 1) },
+      // Players more than a second behind at one moment, the worst second (counted, whether or not they were paced).
+      behindPlayers: pick(during.map((r) => (r.gates ?? []).reduce((a, g) => a + (g.behind ?? 0), 0)), 1),
+      // Of every world every Gate wrote: the share over 100, 200 and 400 ms old, and worlds a Gate-second that came
+      // more than 75, 150 and 300 ms after the one before.
+      worldsOld: (() => { const n = gates.reduce((a, g) => a + (g.worldsOld?.n ?? 0), 0), of = (key) => (n ? round(gates.reduce((a, g) => a + (g.worldsOld?.[key] ?? 0), 0) / n, 4) : null); return n ? { worlds: n, over100: of('over100'), over200: of('over200'), over400: of('over400') } : null; })(),
+      worldGaps: (() => { const with_ = gates.filter((g) => g.worldGaps), of = (key) => (with_.length ? round(with_.reduce((a, g) => a + g.worldGaps[key], 0) / with_.length, 3) : null); return with_.length ? { over75: of('over75'), over150: of('over150'), over300: of('over300') } : null; })(),
+      // What crossed the room's channels a second: rows (a row is a player's message, mostly) and frames in, frames
+      // and megabytes out, and worlds replaced before they were sent because eight were unacknowledged.
+      roomTraffic: (() => {
+        const first = during[0], last = during.at(-1), seconds = first && last ? (last.at - first.at) / 1000 : 0;
+        if (!(seconds > 0)) return null;
+        const total = (row, f) => (row.transport ?? []).reduce((a, t) => a + (f(t) ?? 0), 0), rate = (f) => round((total(last, f) - total(first, f)) / seconds, 1);
+        const heard = (last.transport ?? []).some((t) => t.in);
+        return { inRows: heard ? rate((t) => t.in?.rows) : null, inFrames: heard ? rate((t) => t.in?.frames) : null, inKB: heard ? rate((t) => (t.in?.bytes ?? 0) / 1000) : null, outFrames: rate((t) => t.frames), outMB: rate((t) => t.bytes / 1e6), replaced: rate((t) => t.coalesced) };
+      })(),
+      // Checkpoints the room wrote without holding its frames for them.
+      unconfirmedSaves: during.at(-1)?.save?.unconfirmed ?? null,
     },
   };
 }
@@ -105,11 +131,11 @@ export function table(steps) {
   const head = ['N', 'from', 'joined/end/drops', 'ack p50/p95/p99 ms', 'loop p50/p95 ms', 'lead (late)', 'client tick Hz', 'client delivered Hz', 'wire MB/s (link)', 'KB/s a player', 'compressed (ratio)', 'driver lag ms', 'server late', 'Gates', 'room Hz at Gates med/p05', 'Gate delivered Hz med/p05', 'Gate rtt med/p95 ms', 'world age p50/p95 ms', 'paced', 'diag ok/failed'];
   const lines = [`| ${head.join(' | ')} |`, `| ${head.map(() => '---').join(' | ')} |`];
   for (const s of steps) lines.push(`| ${[
-    s.n, s.from, `${cell(s.joined)}/${cell(s.connectedAtEnd)}/${cell(s.disconnects)}`,
+    s.n, `${s.from} ${s.variant}${s.lostReports ? ` (${s.lostReports} reports lost)` : ''}`, `${cell(s.joined)}/${cell(s.connectedAtEnd)}/${cell(s.disconnects)}`,
     `${cell(s.client.inputAckMs.p50)}/${cell(s.client.inputAckMs.p95)}/${cell(s.client.inputAckMs.p99)}`,
     `${cell(s.client.inputLoopMs?.p50)}/${cell(s.client.inputLoopMs?.p95)}`,
     `${cell(s.client.inputLead?.ticksMedian)} (${cell(s.client.inputLead?.lateShare)})`,
-    cell(s.client.tickHz.median), cell(s.client.deliveredHz.median),
+    cell(s.client.tickHz.median), `${cell(s.client.deliveredHz.median)}${s.healthy ? ` (healthy: ${cell(round(s.healthy.deliveredHz?.median))}, worst ${cell(round(s.healthy.deliveredHz?.min))})` : ''}`,
     `${cell(s.client.wireMBps)} (${cell(s.link.downMBps)}${s.link.limited ? ', LINK-LIMITED' : ''})`,
     cell(s.client.wireKBsPerPlayer),
     s.client.compression ? `${s.client.compression.sockets}/${s.client.compression.of} (${cell(s.client.compression.ratio)})` : cell(s.client.compressedSockets),
@@ -123,18 +149,59 @@ export function table(steps) {
   return lines.join('\n');
 }
 
+/**
+ * The second table: what the room carried, a line a step. Read with the first.
+ *   - "rows in": players' messages reaching the room a second. "worlds out": frames to Gates a second, megabytes,
+ *     and worlds replaced before sending because eight were unacknowledged on a channel (a room that is behind).
+ *   - "worlds old": of every world every Gate wrote, the share over 100 / 200 / 400 ms old.
+ *   - "behind": players more than a second behind at one moment, the worst second. "paced": how many were being
+ *     sent fewer frames for it, the worst second.
+ * There is no column for the room's processor time: Cloudflare's count of it could not be read from a tail (the
+ * requests that are a Gate's channel reported under 1% for a room that was out of time), and nothing an object
+ * measures about itself can stand in, because its clock stands still while it runs.
+ */
+export function threadTable(steps) {
+  const cell = (v) => (v === null || v === undefined ? '—' : String(v));
+  const pct = (v) => (Number.isFinite(v) ? `${Math.round(v * 1000) / 10}%` : '—');
+  const head = ['N', 'from', 'rows in/s', 'worlds out/s (MB/s, replaced/s)', 'unconfirmed checkpoints', 'worlds old >100/>200/>400', 'gaps >75/>150/>300 a Gate-s', 'behind', 'paced'];
+  const lines = [`| ${head.join(' | ')} |`, `| ${head.map(() => '---').join(' | ')} |`];
+  for (const s of steps) {
+    const t = s.server.roomTraffic, old = s.server.worldsOld, gaps = s.server.worldGaps;
+    lines.push(`| ${[
+      s.n, `${s.from} ${s.variant}`,
+      cell(t?.inRows), t ? `${cell(t.outFrames)} (${cell(t.outMB)}, ${cell(t.replaced)})` : '—',
+      cell(s.server.unconfirmedSaves),
+      old ? `${pct(old.over100)}/${pct(old.over200)}/${pct(old.over400)}` : '—', gaps ? `${cell(gaps.over75)}/${cell(gaps.over150)}/${cell(gaps.over300)}` : '—',
+      cell(s.server.behindPlayers), cell(s.server.pacedPlayers),
+    ].join(' | ')} |`);
+  }
+  return lines.join('\n');
+}
+
 if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
   const dir = resolve(process.argv[2] ?? '.');
   const read = (file) => (existsSync(file) ? JSON.parse(readFileSync(file, 'utf8')) : null);
   const steps = [];
-  for (const name of readdirSync(dir).filter((n) => /^n\d+(-cloud)?$/.test(n)).sort((a, b) => parseInt(a.slice(1), 10) - parseInt(b.slice(1), 10) || a.length - b.length)) {
+  for (const name of readdirSync(dir).filter((n) => /^n\d+(-[a-z0-9-]+)?$/.test(n)).sort((a, b) => parseInt(a.slice(1), 10) - parseInt(b.slice(1), 10) || a.localeCompare(b))) {
     const report = read(join(dir, name, 'report.json'));
     if (!report) continue;
     const file = join(dir, name, 'diagnostics.jsonl');
     const rows = existsSync(file) ? readFileSync(file, 'utf8').split('\n').filter(Boolean).flatMap((line) => { try { return [JSON.parse(line)]; } catch { return []; } }) : [];
-    steps.push(summarize(report, rows, read(join(dir, name, 'link.json'))));
+    steps.push({ ...summarize(report, rows, read(join(dir, name, 'link.json'))), folder: name });
   }
+  // The same step played more than once (n1000-cloud, n1000-cloud-r2, ...): the spread of what matters.
+  const groups = new Map();
+  for (const step of steps) { const key = `${step.n} ${step.from} ${step.variant}`; if (!groups.has(key)) groups.set(key, []); groups.get(key).push(step); }
+  const spread = [...groups].filter(([, runs]) => runs.length > 1).map(([key, runs]) => {
+    const of = (f) => { const a = runs.map(f).filter(Number.isFinite).sort((x, y) => x - y); return a.length ? `${a[0]} to ${a.at(-1)}` : '—'; };
+    return `| ${key} | ${runs.length} | ${of((r) => r.client.deliveredHz.median)} | ${of((r) => r.client.inputAckMs.p95)} | ${of((r) => r.server.gateRttMs.p95)} | ${of((r) => r.server.worldAgeMs.p95)} | ${of((r) => r.server.worldsOld?.over200)} | ${of((r) => r.server.lateInputs)} | ${of((r) => r.server.gateDeliveredHz.median)} |`;
+  });
+  const spreadTable = spread.length ? `\n\nRepeated steps, lowest to highest:\n\n| step | runs | client delivered Hz | ack p95 ms | Gate rtt p95 ms | world age p95 ms | share of worlds over 200 ms old | server late | Gate delivered Hz |\n| --- | --- | --- | --- | --- | --- | --- | --- | --- |\n${spread.join('\n')}` : '';
   writeFileSync(join(dir, 'summary.json'), JSON.stringify(steps, null, 2));
-  writeFileSync(join(dir, 'summary.md'), `${table(steps)}\n`);
-  console.log(table(steps));
+  // How fast Cloudflare ran the reference work against this computer (crowd-cloud.mjs --calibrate), when the run measured it.
+  const calibrations = readdirSync(dir).filter((n) => /^calibration.*\.json$/.test(n)).map((n) => read(join(dir, n))).filter((c) => c?.ratio);
+  const calibration = calibrations.length ? `\n\nThe reference work (lib/crowd-probe/reference-work.mjs), a round: this computer ${calibrations.map((c) => Math.min(...c.localMs)).join(', ')} ms (its quickest); inside Cloudflare, by this computer's stopwatch, ${calibrations.map((c) => `${c.cloudMsARound.min} to ${c.cloudMsARound.max} (median ${c.cloudMsARound.median})`).join('; ')} ms. Cloudflare over this computer: ${calibrations.map((c) => `${c.ratio.min} to ${c.ratio.max} (median ${c.ratio.median})`).join('; ')}. The Worker's own clock said the same rounds took ${calibrations.map((c) => (c.workerOwnClockMs ?? []).join(', ')).join('; ')} ms in all.` : '';
+  const text = `${table(steps)}\n\nWhat the room carried:\n\n${threadTable(steps)}${spreadTable}${calibration}`;
+  writeFileSync(join(dir, 'summary.md'), `${text}\n`);
+  console.log(text);
 }

package/lib/crowd-worker.mjs

@@ -46,6 +46,7 @@ export function createLoad({
   workers = 1,
   lead = 3,
   compress = true,
+  slowEvery = 0,
   onProblem = () => {}
 }) {
   const clients = [],
@@ -53,6 +54,8 @@ export function createLoad({
     first = new Histogram(),
     welcome = new Histogram(),
     ack = new Histogram(),
+    // A deliberately slow player's own figures, apart: its acknowledgements are seconds late by what it is.
+    slowAck = new Histogram(),
     gaps = new Histogram(),
     // From the tick a player held when it stamped an input to the tick it
     // reached the rules, ms: what the system took, whatever the lead.
@@ -144,7 +147,7 @@ export function createLoad({
       c.lastTick = snap.k;
       c.lastAt = now;
       c.frames++;
-      gaps.add(gap);
+      if (!c.slow) gaps.add(gap);
     }
     const control = snap.c?.find((row) => row[0] === c.seat);
     // The room says how early this player's inputs arrive, in sixteenths of a
@@ -155,7 +158,7 @@ export function createLoad({
     // early, one fewer when it was over a tick and a half.
     if (control && control[3] !== -128 && control[3] !== undefined) {
       const early = control[3] / 16;
-      if (measuring) { if (early <= 0) lateAcks++; onTime++; loops.add((c.lead - early) * 50); }
+      if (measuring && !c.slow) { if (early <= 0) lateAcks++; onTime++; loops.add((c.lead - early) * 50); }
       (c.early ??= []).push(early); if (c.early.length > 40) c.early.shift();
       if (c.early.length >= 20 && now - (c.leadAt ?? 0) > 2000) {
         const low = c.early.slice().sort((a, b) => a - b)[Math.floor(c.early.length / 20)];
@@ -169,7 +172,7 @@ export function createLoad({
         if (tick > control[2]) break;
         c.pending.delete(tick);
         if (measuring) {
-          if (tick === control[2]) ack.add(now - at);
+          if (tick === control[2]) (c.slow ? slowAck : ack).add(now - at);
           else skipped++;
         }
       }
@@ -230,6 +233,9 @@ export function createLoad({
       });
       c.ws.on('open', () => {
         c.helloAt = performance.now();
+        // A deliberately slow player: this socket stops reading for four seconds in every five, as a connection
+        // that cannot keep up does. What it costs the players beside it is the question.
+        if (slowEvery > 0 && i % slowEvery === 0) { c.slow = true; c.slowTimer = setInterval(() => { c.ws._socket?.pause(); setTimeout(() => c.ws._socket?.resume(), 4000); }, 5000); }
         c.compressed = /permessage-deflate/.test(c.ws.extensions ?? '');
         send(c, {
           t: 'hello',
@@ -434,6 +440,7 @@ export function createLoad({
       measuring = false;
       stopping = true;
       clearInterval(timer);
+      for (const c of clients) clearInterval(c.slowTimer);
       loop.disable();
       for (const c of clients) c.ws?.close(1000, 'load step complete');
       await new Promise((r) => setTimeout(r, 250));
@@ -441,7 +448,7 @@ export function createLoad({
       return {
         ...end,
         errors: [...errors].map(([text, count]) => ({ text, count })),
-        histograms: { first, welcome, ack, gaps, loops },
+        histograms: { first, welcome, ack, slowAck, gaps, loops },
         inputReports: onTime,
         inputReportsLate: lateAcks,
         leadTicks: clients.filter((c) => !c.retired).map((c) => c.lead),
@@ -456,6 +463,7 @@ export function createLoad({
           priorSeat: c.priorSeat,
           sameSeat: c.priorSeat === null ? null : c.seat === c.priorSeat,
           retired: Boolean(c.retired),
+          slow: Boolean(c.slow),
           firstStateMs: c.first === null ? null : c.first - c.began,
           helloAfterMs: c.helloAt ? c.helloAt - c.began : null,
           closeCode: c.closeCode,

package/lib/template-history.json

@@ -2859,6 +2859,48 @@
      }
     }
    }
+  },
+  "0.46.6": {
+   "files": {
+    "CLAUDE.md": "ddf2b2a0376202f6",
+    "README.md": "684b3cca8001673c",
+    "HANDOFF.md": "29337e2ace95b2e7",
+    "games/README.md": "5ad32fe80c61f117",
+    "music/README.md": "a3dc042dadd39df8",
+    "videos/README.md": "16babc3437d5545b",
+    "posts/README.md": "37841aae476bd540",
+    "site/README.md": "732894273a32dc3f",
+    "site/src/worker.mjs": "6cc5c6131eaf6fd0"
+   },
+   "sections": {
+    "AGENTS.md": {
+     "(intro)": "394cbbe934e80722",
+     "## Version-matched authoring guides": "f6261a0e5801b96b",
+     "## Layout": "f3a280b8c96cd068",
+     "## Apps: one screen, roles and parts": "8596f2b25edda3e2",
+     "## Commands (all through the pinned CLI in node_modules)": "5222e08ca55e326e",
+     "## The site": "0cb8dd9a6f8a0096",
+     "## Making games": "bb715272fd1a615b",
+     "## The Game Codex and progress": "0738ef719cef3a4c",
+     "## Art direction and models": "a48fff7bf0a5f2fc",
+     "## Running live games (the back office)": "51ebf2ab756f6e4d",
+     "## Servers and AI seats": "1ccfd549d2369a60",
+     "## Selling things (the shop)": "9e020d69dd6a954e",
+     "## Continuing a build from the Claude app": "0d68081e924ca419",
+     "## Rules": "27881d11d3ff3803",
+     "## Telling Homie": "80b46fdd9e578db8",
+     "## Beta": "9d02638207a26e73"
+    }
+   },
+   "named": {
+    "Homie Arcade": {
+     "sections": {
+      "AGENTS.md": {
+       "## Commands (all through the pinned CLI in node_modules)": "611b437f5b3303f0"
+      }
+     }
+    }
+   }
   }
  }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@homie-rocks/studio",
-  "version": "0.46.5",
+  "version": "0.46.6",
   "description": "A studio for games, music and video on its own Cloudflare: scaffold, multiplayer starters on the netplay contract, a port toolkit that makes existing single-player web games multiplayer, the site Worker with public rooms (Table/Lobby Durable Objects and D1, all on the free Workers plan, no payment method), a site with the hub's sections (Home, Games, Music, Videos, Rooms, Posts with feeds) in the studio's own look, an epic landing page for every game, big songs and videos in the studio's own R2 once it has storage (uploaded, checked by SHA-256, served at the same addresses), deploy, and the homie.rocks directory.",
   "type": "module",
   "license": "Apache-2.0",

package/rules/host.ts

@@ -80,6 +80,8 @@ export interface HostOptions {
   clock: HostClock;
   /** Synchronous save: the caller's storage output gate holds this tick's outgoing frames until confirmed. */
   store?: { save(bytes: Uint8Array, commitment?: { force: boolean; seconds: number; tickHz: number; fields: { kinds: number[][]; shared: number[] } }): void };
+  /** Whether a snapshot is also handed over as text (the default). A room that sends the whole world to nobody answers false. */
+  snapshotText?: () => boolean;
   /** Table recovery reserves this epoch in storage before starting this host. Ordinary deterministic restores omit it. */
   restoreEpoch?: number;
   startDelayMs?: number;
@@ -764,7 +766,8 @@ export function createHost(o: HostOptions): Host {
     // apart to whoever interpolates between them. The stamp always rises: a replica drops one that does not.
     lastSt = Math.max(lastSt + 1, Math.round(Math.min(now, dueOf(core.tick))));
     const snap = { t: 'snap', from: null, e: epoch, k: core.tick, st: lastSt, d: data, c: rows };
-    const encoded = JSON.stringify(snap);
+    // The whole world as text, for whoever is sent it whole. A room that sends it to nobody says so and is spared it.
+    const encoded = o.snapshotText?.() === false ? undefined : JSON.stringify(snap);
     timing.record('encode', encodeAt);
     const sendAt = timing.now();
     o.send(snap, encoded);

package/worker/build-id.mjs

@@ -1,2 +1,2 @@
 // Generated by lib/build-identity.mjs; identifies the bundled runtime sources.
-export const STUDIO_BUILD = Object.freeze({"version":"0.46.5","sha256":"6e01cf0c45cbe8649134cb3332feed0ba34c9ac39f3431d4c8374e0237345c0b"});
+export const STUDIO_BUILD = Object.freeze({"version":"0.46.6","sha256":"efd3ddbf72033f5d24b8107a41db2bc1a518f1d3f92963f5afd77c5c0e2f5059"});

package/worker/gate.mjs

@@ -36,9 +36,11 @@ function parseFrame(event) {
  * as full as it can serve, and a room opens only the Gates its players fill.
  * A studio chooses its room's size and nothing else. */
 export const GATE_PLAYERS = 50;
-export function roomLayout(seats) {
+/** `per`: players a Gate, for a trial that is finding the right number (`HOMIE_GATE_PLAYERS` on the Worker). A studio never sets it. */
+export const gatePlayers = (env) => { const n = Number(env?.HOMIE_GATE_PLAYERS); return Number.isInteger(n) && n >= 4 && n <= 200 ? n : GATE_PLAYERS; };
+export function roomLayout(seats, per = GATE_PLAYERS) {
   // A room of up to 64 is its Table alone, as it has been since 0.45.0.
-  const gates = seats <= 64 ? 1 : Math.ceil(seats / GATE_PLAYERS);
+  const gates = seats <= 64 ? 1 : Math.ceil(seats / per);
   return { gates, concentrators: 0 };
 }
 /** The Gate a key lands on when the room cannot be asked: spread evenly, as 0.45.0 did. */
@@ -52,17 +54,17 @@ function hashedGate(key, gates) {
  * Gate, not twenty with two or three players each, every one of them costing the room a channel. `held` is each
  * Gate's players now and `promised` those sent to it who have not arrived yet.
  */
-export function nextGate(gates, held, promised = new Map()) {
+export function nextGate(gates, held, promised = new Map(), per = GATE_PLAYERS) {
   let least = 0, fewest = Infinity;
   for (let i = 0; i < gates; i++) {
     const n = (held.get(i) ?? 0) + (promised.get(i) ?? 0);
-    if (n < GATE_PLAYERS) return i;
+    if (n < per) return i;
     if (n < fewest) { fewest = n; least = i; }
   }
   return least;
 }
 export function gateFor(env, game, room, seats, key) {
-  const layout = roomLayout(seats);
+  const layout = roomLayout(seats, gatePlayers(env));
   // One logical Gate is embedded in the Table for a small room. Same relay,
   // admission and delivery protocol; no extra Durable Object or seat clamp.
   const table = () => env.TABLE.get(env.TABLE.idFromName(`${game}/${room}`));
@@ -211,7 +213,9 @@ export function multiplexSession(socket, connect, { onMetrics = () => {}, delay
   const scheduleAdmission = () => {
     if (!closed && admissions.length && attaching < 8 && out.buffered() < 512 * 1024 && admissionTimer === null) admissionTimer = setTimeout(pump, 1);
   };
-  const transportFacts = () => ({ ...out.facts(), admissions: admissions.length, attaching });
+  // What this Gate has sent the room: frames, the rows in them (a row is a player's message, mostly) and their size.
+  const heard = { frames: 0, rows: 0, bytes: 0 };
+  const transportFacts = () => ({ ...out.facts(), in: { ...heard }, admissions: admissions.length, attaching });
   const shutdown = () => {
     if (closed) return;
     closed = true; out.close();
@@ -226,6 +230,7 @@ export function multiplexSession(socket, connect, { onMetrics = () => {}, delay
       if (closed) return;
       const rows = parseFrame(event);
       if (!Array.isArray(rows)) throw new Error('invalid multiplex batch');
+      heard.frames += 1; heard.rows += rows.length; heard.bytes += event.data.length;
       for (const [op, id, value, ip, agent] of expandBatch(rows)) {
         if (op === 'metrics') { onMetrics(id, value); continue; }
         if (op === 'open') {
@@ -291,26 +296,38 @@ export function acceptMultiplex(request, connect, options) {
  * A socket to a slow player has no back-pressure a Worker can read, so a Gate
  * that writes every tick to a link that carries fewer leaves that player
  * seconds behind for good: every frame arrives, each one later than the last.
- * A client names the newest tick it holds (`g`, on its inputs and pings). While
- * that trails the room by more than a second the Gate halves this player's
- * rate, down to one a second; when it is back within half a second the rate
- * doubles again. Each frame is a delta from the last one SENT, so a skipped
- * tick costs nothing later, and the player's own body is in every frame.
- * A client that names nothing is sent every tick, as before.
+ * A client names the newest tick it holds (`g`, on its inputs and pings).
+ *
+ * Being behind is not the reason to send less: a connection that stalls for a
+ * second and then delivers what it held catches up by itself, and sending it
+ * less only makes it worse (0.46.2 paced all 600 players of a healthy room
+ * when their one home link hiccuped). The reason is being behind and NOT
+ * catching up: more than a second behind on two reports half a second apart,
+ * the second not a quarter of a second nearer than the first. Then this
+ * player's rate halves, down to one a second, and halves again while that
+ * stays so. Within half a second of the room, the rate doubles back, a step a
+ * second. Each frame is a delta
+ * from the last one SENT, so a skipped tick costs nothing later, and the
+ * player's own body is in every frame. A client that names nothing is sent
+ * every tick, as before.
  */
 export function deliveryPace(hz = 20) {
-  let interval = 1, sent = -Infinity, got = null, changedAt = -Infinity, lag = null;
+  let interval = 1, sent = -Infinity, changedAt = -Infinity, lag = null, behind = null;
   const most = Math.max(1, Math.round(hz));
   return {
     /** The client holds tick `tick`; the room is at `world`. `now` in ms. */
     report(tick, world, now) {
       if (!Number.isFinite(tick) || !Number.isFinite(world)) return;
       // A new room epoch restarts its ticks: a tick from the old one says nothing.
-      got = tick > world ? null : tick;
-      lag = got === null ? null : world - got;
-      if (lag === null) return;
-      if (lag > hz && now - changedAt >= 500 && interval < most) { interval = Math.min(most, interval * 2); changedAt = now; }
-      else if (lag <= hz / 2 && now - changedAt >= 1000 && interval > 1) { interval = Math.max(1, interval / 2); changedAt = now; }
+      lag = tick > world ? null : world - tick;
+      if (lag === null) { behind = null; return; }
+      if (lag <= hz / 2) { behind = null; if (interval > 1 && now - changedAt >= 1000) { interval = Math.max(1, interval / 2); changedAt = now; } return; }
+      if (lag <= hz) { behind = null; return; }
+      if (behind === null) { behind = { lag, at: now }; return; }
+      if (now - behind.at < 500) return;
+      // Catching up means a quarter of a second nearer in half a second; anything less is a link that cannot keep this rate.
+      if (lag > behind.lag - hz / 4 && now - changedAt >= 500 && interval < most) { interval = Math.min(most, interval * 2); changedAt = now; }
+      behind = { lag, at: now };
     },
     due(tick) { return tick < sent || tick - sent >= interval; },
     sent(tick) { sent = tick; },
@@ -324,8 +341,19 @@ const COASTS = /^\{"t":"hello".{0,2000}?"coast":1[,}]/;
 const percentile = (sorted, p) => sorted.length ? sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * p))] : null;
 
 export class Gate {
-  constructor(ctx, env) { this.ctx = ctx; this.env = env; this.clients = new Map(); this.admissions = new Map(); this.sequence = 0; this.link = null; this.opening = null; this.timing = stageTimings(); this.metricsAt = 0; this.window = this.freshWindow(); this.worldTick = null; }
-  freshWindow() { return { worlds: 0, frames: 0, bytes: 0, skipped: 0, inputs: 0, worldAge: [], tickFrom: null, tickFromAt: 0, tickAt: 0, gapMax: 0 }; }
+  constructor(ctx, env) { this.ctx = ctx; this.env = env; this.clients = new Map(); this.admissions = new Map(); this.sequence = 0; this.link = null; this.opening = null; this.timing = stageTimings(); this.metricsAt = 0; this.window = this.freshWindow(); this.worldTick = null; this.behind = new Set(); this.idle = null; }
+  freshWindow() { return { worlds: 0, frames: 0, bytes: 0, skipped: 0, inputs: 0, worldAge: [], tickFrom: null, tickFromAt: 0, tickAt: 0, gapMax: 0, gaps: [0, 0, 0], late: 0, busy: 0 }; }
+  /** This Gate's own lateness: a timer every 20 ms, and how long past it each one came. A Gate that is out of time is late to its own timer. */
+  watchLoop() {
+    if (this.loopTimer) return;
+    let due = Date.now() + 20;
+    this.loopTimer = setInterval(() => {
+      const now = Date.now(), late = Math.max(0, now - due); due = now + 20;
+      this.window.late = Math.max(this.window.late, late); if (late > 20) this.window.busy += 1;
+      if (!this.link || !this.clients.size) { clearInterval(this.loopTimer); this.loopTimer = null; }
+    }, 20);
+    this.loopTimer?.unref?.();
+  }
   /** `[sent, received, answered]`: this Gate's clock when its frame left, the room's when it read it and when it answered. */
   roomClock([sent, received, answered]) {
     const now = Date.now(), offset = ((received - sent) + (answered - now)) / 2, delay = (now - sent) - (answered - received);
@@ -380,6 +408,13 @@ export class Gate {
         } catch (error) { lost({ error, reason: 'invalid room frame' }); try { socket.close(1011, 'room link failed'); } catch {} }
       });
       socket.addEventListener('close', lost); socket.addEventListener('error', lost);
+      // A Gate nobody is on gives its channel back after ten seconds: an open channel keeps the room and this Gate in
+      // memory (and billed) with nothing to carry. The next player to arrive opens another, as the first did.
+      this.rest = () => {
+        if (this.link !== link || this.clients.size) return;
+        this.link = null; this.worlds = null; link.close();
+        try { socket.close(1000, 'no players'); } catch { /* gone */ }
+      };
       this.link = link; return link;
     })();
     try { return await this.opening; } finally { this.opening = null; }
@@ -388,7 +423,7 @@ export class Gate {
   deliver(snap, settings, hz, drop) {
     const w = this.window, now = Date.now(); w.worlds++;
     if (w.tickFrom === null) { w.tickFrom = this.worldTick ?? snap.k; w.tickFromAt = this.worldAt ?? now; }
-    if (this.worldAt) w.gapMax = Math.max(w.gapMax, now - this.worldAt);
+    if (this.worldAt) { const gap = now - this.worldAt; w.gapMax = Math.max(w.gapMax, gap); if (gap > 75) w.gaps[0] += 1; if (gap > 150) w.gaps[1] += 1; if (gap > 300) w.gaps[2] += 1; }
     this.worldTick = snap.k; this.worldAt = now; w.tickAt = now;
     // By this Gate's clock, corrected for what the room's clock is measured to differ by.
     if (Number.isFinite(snap.st)) w.worldAge.push(Math.max(0, now - snap.st + (this.clock?.offsetMs ?? 0)));
@@ -431,7 +466,7 @@ export class Gate {
   async connect(request, server) {
     const link = await this.upstream(new URL(request.url));
     const id = String(++this.sequence);
-    this.clients.set(id, server);
+    this.clients.set(id, server); clearTimeout(this.idle); this.idle = null; this.watchLoop();
     let opened = false;
     const deadline = setTimeout(() => {
       this.admissions.delete(id);
@@ -458,12 +493,18 @@ export class Gate {
       }
       const now = Date.now();
       const said = REPORTED_TICK.exec(event.data);
-      if (said && server.pace && this.worldTick !== null) server.pace.report(Number(said[1]), this.worldTick, now);
+      if (said && server.pace && this.worldTick !== null) {
+        server.pace.report(Number(said[1]), this.worldTick, now);
+        // Counted, never acted on: players more than a second behind at this moment. Many at once on one Gate is a
+        // connection they share or something upstream of them all; each is still paced by their own reports alone
+        // (a full shared link is cured by sending each of them less, and nothing else cures it).
+        if ((server.pace.facts().lag ?? 0) > 20) this.behind.add(id); else this.behind.delete(id);
+      }
       this.window.inputs++;
       // Only the public Gate stamps ingress; a player cannot supply it inside their JSON.
       link.send(['data', id, event.data, Number.isFinite(event.receivedAt) ? event.receivedAt : now]);
     });
-    const left = () => { clearTimeout(deadline); this.admissions.delete(id); if (this.clients.delete(id) && opened) link.send(['close', id]); };
+    const left = () => { this.behind.delete(id); clearTimeout(deadline); this.admissions.delete(id); if (this.clients.delete(id) && opened) link.send(['close', id]); if (!this.clients.size && !this.idle) { this.idle = setTimeout(() => { this.idle = null; this.rest?.(); }, 10_000); this.idle?.unref?.(); } };
     server.addEventListener('close', left); server.addEventListener('error', left);
   }
   /** What this Gate did in the last second, for whoever is measuring the room. */
@@ -480,15 +521,21 @@ export class Gate {
       bytesPerSecond: Math.round(w.bytes / seconds), inputsPerSecond: Math.round(w.inputs / seconds), skipped: w.skipped,
       // Room stamp to this Gate writing it to players, ms (both are Cloudflare's clock at the start of an event).
       worldAgeMs: { p50: percentile(ages, 0.5), p95: percentile(ages, 0.95), max: ages.at(-1) ?? null },
+      // Of this second's worlds: how many were over 100, 200 and 400 ms old when this Gate wrote them, and how many came
+      // more than 75, 150 and 300 ms after the one before (a world is due every 50).
+      worldsOld: { n: ages.length, over100: ages.filter(a => a > 100).length, over200: ages.filter(a => a > 200).length, over400: ages.filter(a => a > 400).length },
+      worldGaps: { over75: w.gaps[0], over150: w.gaps[1], over300: w.gaps[2] },
       // How far behind the room players say they are, in ticks, and how many are being sent fewer frames for it.
       clientLagTicks: { reporting: lags.length, p50: percentile(lags, 0.5), p95: percentile(lags, 0.95), max: lags.at(-1) ?? null },
-      paced: paces.filter(p => p.interval > 1).length,
+      paced: paces.filter(p => p.interval > 1).length, behind: this.behind.size,
       coasting: subscribed.filter(c => c.coasts).length,
       // Ticks the room's worlds advanced a second, by THIS Gate's clock: the room's own clock cannot see itself run late.
       roomTickHz: w.tickFrom !== null && w.tickAt > w.tickFromAt ? Math.round((this.worldTick - w.tickFrom) / (w.tickAt - w.tickFromAt) * 100000) / 100 : null,
       worldGapMs: w.gapMax,
       // The room's clock less this Gate's, and the channel's delay, from stamps exchanged on the channel (as NTP does).
       roomClock: this.clock ?? null,
+      // The worst this Gate's own 20 ms timer came late in the second, and how many of the fifty came over 20 ms late.
+      loopLateMs: w.late, loopLateTimers: w.busy,
     };
   }
   publishMetrics(url, link) {