@homie-rocks/studio 0.46.3 → 0.46.6

package/lib/crowd-probe/reference-work.mjs

@@ -0,0 +1,45 @@
+/**
+ * A fixed piece of work, the same everywhere it runs: what a room's thread spends its time on, in miniature (parsing
+ * players' small messages, building and reading plain objects, arithmetic on floats, writing a block of text).
+ *
+ * A Worker cannot time a piece of its own work (its clock stands still while its code runs, and reads a timer's due
+ * time when the timer fires), so nothing measured on a laptop says by itself what a room costs on Cloudflare. This
+ * runs on both: the probe does it inside Cloudflare while `crowd-cloud.mjs --calibrate` holds the stopwatch on the
+ * request, and the same call is timed on the computer that asked. The ratio turns `test/crowd-room-cost.mjs`'s
+ * milliseconds into Cloudflare's.
+ *
+ * It returns a number that depends on every step, so no engine may skip the work, and the same number everywhere.
+ */
+export function referenceWork(rounds = 40) {
+  let check = 0, seed = 20261011;
+  const random = () => { seed = (Math.imul(seed, 1664525) + 1013904223) >>> 0; return seed / 4294967296; };
+  for (let round = 0; round < rounds; round += 1) {
+    // A tick's inputs: five hundred small messages parsed, held to numbers, queued by seat.
+    const queues = new Map();
+    for (let i = 0; i < 500; i += 1) {
+      const text = JSON.stringify({ t: 'in', e: 3914181188, k: 3000 + round, s: [[0, i % 2 ? 127 : -127, Math.round(random() * 70 - 35)]], r: i & 7, g: 2997 + round });
+      const m = JSON.parse(text), seat = i * 2;
+      const entry = { at: m.k >>> 0, values: { ax: Math.max(-127, Math.min(127, m.s[0][1] | 0)), ay: Math.max(-127, Math.min(127, m.s[0][2] | 0)) } };
+      let q = queues.get(seat); if (!q) { q = []; queues.set(seat, q); }
+      q.push(entry); check = (check + entry.values.ax + entry.values.ay + text.length) | 0;
+    }
+    // A tick of a thousand bodies: frozen vectors made and read, a little float arithmetic each.
+    const bodies = [];
+    for (let i = 0; i < 1000; i += 1) {
+      const q = queues.get(i), held = q ? q[q.length - 1].values : { ax: 0, ay: 0 };
+      const vel = Object.freeze({ x: Math.fround(held.ax / 127 * 6), y: Math.fround(held.ay / 127 * 6), z: 0 });
+      const pos = Object.freeze({ x: Math.fround(random() * 16 - 8 + vel.x * 0.05), y: Math.fround(random() * 120 - 60 + vel.y * 0.05), z: 0 });
+      const speed = Math.hypot(vel.x, vel.y);
+      bodies.push([`r${i}`, 0, round & 0xffff, [pos.x, pos.y], [vel.x, vel.y], [speed > 0 ? vel.x / speed : 1, speed > 0 ? vel.y / speed : 0], 0, [round, pos.x > 0], [], i, 0]);
+      check = (check + Math.round(pos.x * 100) + Math.round(pos.y * 100)) | 0;
+    }
+    // The world as it leaves: a block of numbers, and the room's checkpoint as text.
+    const block = new Float32Array(bodies.length * 6);
+    for (const [i, b] of bodies.entries()) block.set([b[3][0], b[3][1], b[4][0], b[4][1], b[5][0], b[5][1]], i * 6);
+    const bytes = new Uint8Array(block.buffer); let sum = 0;
+    for (let i = 0; i < bytes.length; i += 97) sum += bytes[i];
+    const text = JSON.stringify(bodies);
+    check = (check + sum + text.length + JSON.parse(text).length) | 0;
+  }
+  return check;
+}

package/lib/crowd-probe/run-trial.sh

@@ -0,0 +1,207 @@
+#!/usr/bin/env bash
+# A crowd trial of one game of a deployed studio, on your own Cloudflare account: each step from your computer and
+# from inside Cloudflare, the in-Cloudflare steps three times each (one run does not tell a limit from luck), a
+# step with slow players among healthy ones, the room's own figures a second beside every step, how fast Cloudflare
+# ran a fixed piece of work against this computer, and one summary.md at the end.
+#
+# It deploys nothing and changes nothing in the studio. It writes under reports/<game>/trial/<stamp>/ in the
+# studio's folder, creates temporary office keys for the diagnostics and revokes them.
+#
+#   cd <your studio>                       # deployed, with this toolkit installed
+#   (cd node_modules/@homie-rocks/studio/lib/crowd-probe && KEY=$(openssl rand -hex 24) && npx wrangler deploy --var PROBE_KEY:$KEY)
+#   URL=https://<studio> WORKER=<its worker's name> PROBE_URL=https://homie-crowd-probe.<account>.workers.dev PROBE_KEY=$KEY \
+#     bash node_modules/@homie-rocks/studio/lib/crowd-probe/run-trial.sh
+#   npx wrangler delete --name homie-crowd-probe      # when the trial is over
+#
+# Without PROBE_URL only the steps from this computer run. The players steer as the crowd-circuit fixture does:
+# for another game, change the input lib/crowd-probe/client.mjs and lib/crowd-worker.mjs send.
+#
+#   GAME=crowd-circuit                     the game's id
+#   PLAN=$'300 home\n600 cloud 3'          a different plan: "players mode [runs]" a line (modes below)
+#   LABEL=evening                          a word added to the report folder's name
+#   SECONDS_PER_STEP=150                   steady measurement per step
+#   EXPECTED_RUNTIME_SHA=<sha256>          stop unless the installed and the deployed toolkit are this build
+#   CHROME_PATH=/path/to/chrome            for the two real Chrome players in a step from this computer
+#
+# Read docs/crowd-transport-0462.md first for what each column can and cannot say. In short: nothing an object
+# measures about itself by its own clock is to be trusted on Cloudflare; a Gate's round trip to the room, the share
+# of worlds over 200 ms old and the players' own loop are the figures that show a room out of time.
+set -euo pipefail
+[ -d node_modules/@homie-rocks/studio ] || { echo 'Run this in a studio folder with @homie-rocks/studio installed.'; exit 1; }
+: "${URL:?URL=https://<your studio>}"
+: "${WORKER:?WORKER=<the name of the studio Worker, as in wrangler.jsonc>}"
+GAME="${GAME:-crowd-circuit}"
+export WRANGLER_SEND_METRICS=false
+# The probe inflates every frame, as a browser does; give zlib's pool room.
+export UV_THREADPOOL_SIZE=16
+TOOLKIT="$PWD/node_modules/@homie-rocks/studio"
+SECONDS_PER_STEP="${SECONDS_PER_STEP:-150}"
+PROBE_URL="${PROBE_URL:-}"
+PROBE_KEY="${PROBE_KEY:-}"
+STAMP=$(date -u +%Y%m%dT%H%M%SZ)
+OUT="$PWD/reports/$GAME/trial/$STAMP${LABEL:+-$LABEL}"
+mkdir -p "$OUT"
+
+# 1. The deployed Worker is this toolkit, byte for byte.
+node --input-type=module - "$URL" "$TOOLKIT" "$OUT" <<'JS'
+import { pathToFileURL } from 'node:url';
+import { writeFile } from 'node:fs/promises';
+const [url, toolkit, out] = process.argv.slice(2);
+const { STUDIO_BUILD } = await import(pathToFileURL(`${toolkit}/worker/build-id.mjs`));
+const expected = process.env.EXPECTED_RUNTIME_SHA;
+if (expected && STUDIO_BUILD.sha256 !== expected) throw Error(`Installed toolkit's runtime is ${STUDIO_BUILD.sha256}, not this build's ${expected}: install that build first`);
+const response = await fetch(`${url}/.well-known/homie-studio.json?verify=${Date.now()}`);
+if (!response.ok) throw Error(`manifest HTTP ${response.status}`);
+const manifest = await response.json();
+await writeFile(`${out}/deployment.json`, JSON.stringify(manifest, null, 2));
+if (manifest.toolkit?.sha256 !== STUDIO_BUILD.sha256 || manifest.toolkit?.version !== STUDIO_BUILD.version)
+  throw Error(`Wrong deployed toolkit: expected ${JSON.stringify(STUDIO_BUILD)}, got ${JSON.stringify(manifest.toolkit)}`);
+console.log('Verified deployed runtime', STUDIO_BUILD);
+JS
+
+# What this connection carries: twelve parallel downloads from Cloudflare for eight seconds, to nowhere.
+probe_link() {
+  local file="$1" tmp start end bytes pids=""
+  tmp=$(mktemp -d)
+  start=$(node -e 'console.log(Date.now())')
+  for i in 1 2 3 4 5 6 7 8 9 10 11 12; do
+    curl -s -o /dev/null -H 'Referer: https://speed.cloudflare.com/' -A 'Mozilla/5.0' --max-time 8 \
+      -w '%{size_download}\n' 'https://speed.cloudflare.com/__down?bytes=100000000' > "$tmp/$i" 2>/dev/null &
+    pids="$pids $!"
+  done
+  wait $pids
+  end=$(node -e 'console.log(Date.now())')
+  bytes=$(cat "$tmp"/* 2>/dev/null | awk '{s+=$1} END {print s+0}')
+  rm -rf "$tmp"
+  node -e '
+    const [file, bytes, start, end] = process.argv.slice(1);
+    const seconds = (end - start) / 1000, down = bytes > 0 && seconds > 0 ? bytes / seconds / 1e6 : null;
+    require("fs").writeFileSync(file, JSON.stringify({ at: new Date().toISOString(), streams: 12, seconds, bytes: Number(bytes), downMBps: down }, null, 2));
+    console.log("Link before this step:", down === null ? "not measured" : down.toFixed(2) + " MB/s (" + (down * 8).toFixed(0) + " Mbit/s)");
+  ' "$file" "$bytes" "$start" "$end"
+}
+
+TAIL_PID=
+DIAG_PID=
+cleanup() {
+  if [ -n "$DIAG_PID" ]; then kill -TERM "$DIAG_PID" 2>/dev/null || true; wait "$DIAG_PID" 2>/dev/null || true; fi
+  if [ -n "$TAIL_PID" ]; then kill -TERM "$TAIL_PID" 2>/dev/null || true; wait "$TAIL_PID" 2>/dev/null || true; fi
+  # Whatever finished is summarised, even when a step failed.
+  node "$TOOLKIT/lib/crowd-summary.mjs" "$OUT" || true
+  printf 'All reports: %s\nSummary: %s\n' "$OUT" "$OUT/summary.md"
+}
+trap cleanup EXIT
+trap 'exit 130' INT TERM
+
+# The tail is sampled by Cloudflare under load and is not the record; diagnostics.jsonl is.
+# Only failures are kept, so the sample is not spent on healthy requests.
+./node_modules/.bin/wrangler tail "$WORKER" --format json --status error > "$OUT/worker-tail.jsonl" 2> "$OUT/worker-tail.stderr" &
+TAIL_PID=$!
+sleep 3
+kill -0 "$TAIL_PID" 2>/dev/null || { printf 'Worker tail did not start (the run continues without it): %s\n' "$OUT/worker-tail.stderr"; TAIL_PID=; }
+# One step: the diagnostics collector beside a driver. $1 the step's folder, $2 the room, the rest the driver's command.
+run_step() {
+  local STEP="$1" ROOM="$2"; shift 2
+  mkdir -p "$STEP"
+  # The collector keeps a failed poll as a row and carries on; it is stopped when the step ends.
+  node "$TOOLKIT/lib/crowd-diagnostics.mjs" --root "$PWD" --url "$URL" \
+    --game "$GAME" --room "$ROOM" --seconds $((SECONDS_PER_STEP + 600)) --out "$STEP/diagnostics.jsonl" \
+    > "$STEP/diagnostics.log" 2>&1 &
+  DIAG_PID=$!
+  local READY=0 WAIT
+  for WAIT in $(seq 1 60); do
+    if [ -s "$STEP/diagnostics.jsonl" ]; then READY=1; break; fi
+    kill -0 "$DIAG_PID" 2>/dev/null || break
+    sleep 1
+  done
+  if [ "$READY" -ne 1 ]; then printf 'Diagnostics did not start for %s; the step runs without them:\n' "$ROOM"; cat "$STEP/diagnostics.log" || true; fi
+  local STATUS=0
+  "$@" > "$STEP.log" 2>&1 || STATUS=$?
+  printf '%s\n' "$STATUS" > "$STEP/driver-exit-code.txt"
+  kill -TERM "$DIAG_PID" 2>/dev/null || true
+  local DIAG_STATUS=0
+  wait "$DIAG_PID" 2>/dev/null || DIAG_STATUS=$?
+  printf '%s\n' "$DIAG_STATUS" > "$STEP/diagnostics-exit-code.txt"
+  DIAG_PID=
+  local ROWS FAILED
+  ROWS=$(wc -l < "$STEP/diagnostics.jsonl" 2>/dev/null | tr -d ' ' || echo 0)
+  FAILED=$(grep -c '"ok":false' "$STEP/diagnostics.jsonl" 2>/dev/null || true)
+  printf '%s finished (driver exit %s; diagnostics %s rows, %s failed polls): %s\n' "$ROOM" "$STATUS" "$ROWS" "${FAILED:-0}" "$STEP/report.json"
+  if [ -n "$TAIL_PID" ] && ! kill -0 "$TAIL_PID" 2>/dev/null; then
+    printf 'Worker tail stopped; inspect %s\n' "$OUT/worker-tail.stderr"; TAIL_PID=
+  fi
+  # A Gate gives its channel to the room back ten seconds after its last player leaves: the next step starts clean.
+  sleep 15
+}
+
+if [ -n "$PROBE_URL" ]; then
+  HELLO=$(curl -fsS -H "x-probe-key: $PROBE_KEY" "$PROBE_URL/hello" || true)
+  case "$HELLO" in
+    *'"probe":2'*) ;;
+    '') printf 'The probe at %s did not answer with that key; running from this computer only.\n' "$PROBE_URL"; PROBE_URL= ;;
+    *) printf 'The probe at %s is an older one (%s): deploy it again from the installed toolkit (lib/crowd-probe): it has the slow players and the calibration.\n' "$PROBE_URL" "$HELLO"; exit 1 ;;
+  esac
+fi
+
+# How fast Cloudflare runs a fixed piece of work against this computer: $1 the file. Eight probe requests.
+calibrate() {
+  [ -n "$PROBE_URL" ] || return 0
+  node "$TOOLKIT/lib/crowd-cloud.mjs" --probe "$PROBE_URL" --key "$PROBE_KEY" --calibrate "$1" || true
+}
+uptime > "$OUT/load-average-start.txt" || true
+calibrate "$OUT/calibration-start.json"
+
+# The plan: one step a line, "players mode [repeat]". Modes:
+#   home           this computer, compression on (what one home connection carries)
+#   home-slow      the same with every tenth player's socket not read for four seconds in five (a real slow socket)
+#   home-plain     this computer, compression off
+#   cloud          inside Cloudflare, compression off
+#   cloud-deflate  inside Cloudflare, compression on
+#   cloud-slow, cloud-deflate-slow   the same with every tenth player slow (it takes six frames a second)
+# The cloud steps run three times each: one run of each was not enough to tell a limit from luck.
+# From home: sizes one connection carries with room to spare. A thousand players on one connection measure the connection.
+PLAN="${PLAN:-300 home
+300 home-slow
+600 home
+800 cloud 3
+800 cloud-deflate 3
+900 cloud 3
+900 cloud-deflate 3
+1000 cloud 3
+1000 cloud-deflate 3
+1000 cloud-slow
+1000 cloud-deflate-slow}"
+printf '%s\n' "$PLAN" | while read -r N MODE REPEAT; do
+  [ -n "$N" ] || continue
+  for RUN in $(seq 1 "${REPEAT:-1}"); do
+    SUFFIX=""; [ "$RUN" -gt 1 ] && SUFFIX="-r$RUN"
+    case "$MODE" in
+      home|home-plain|home-slow)
+        STEP="$OUT/n$N"; COMPRESS=on; EXTRA=()
+        if [ "$MODE" = home-plain ]; then STEP="$OUT/n$N-home-plain"; COMPRESS=off; fi
+        if [ "$MODE" = home-slow ]; then STEP="$OUT/n$N-home-slow"; EXTRA=(--slow-every 10); fi
+        STEP="$STEP$SUFFIX"
+        mkdir -p "$STEP"
+        uptime > "$STEP/load-average.txt" || true
+        probe_link "$STEP/link.json" || true
+        # A full room is 998 simulated and two Chrome players.
+        SIZE=(--n "$N")
+        if [ "$N" -ge 999 ]; then SIZE=(--total 1000); fi
+        ROOM="h$(printf '%s' "$MODE" | cut -c6-7)$N-$RUN-$STAMP"
+        run_step "$STEP" "$ROOM" node "$TOOLKIT/lib/crowd-load.mjs" --url "$URL" "${SIZE[@]}" \
+          --workers 4 --seconds "$SECONDS_PER_STEP" --ramp-ms 50 --rss-mb 3072 --compress "$COMPRESS" ${EXTRA[@]+"${EXTRA[@]}"} --room "$ROOM" --out "$STEP" < /dev/null
+        ;;
+      cloud|cloud-deflate|cloud-slow|cloud-deflate-slow)
+        if [ -z "$PROBE_URL" ]; then continue; fi
+        STEP="$OUT/n$N-$MODE$SUFFIX"; COMPRESS=off; EXTRA=(); TAG=c
+        case "$MODE" in cloud-deflate*) COMPRESS=on; TAG=cd ;; esac
+        case "$MODE" in *-slow) EXTRA=(--slow-every 10); TAG="${TAG}s" ;; esac
+        ROOM="$TAG$N-$RUN-$STAMP"
+        run_step "$STEP" "$ROOM" node "$TOOLKIT/lib/crowd-cloud.mjs" --probe "$PROBE_URL" --key "$PROBE_KEY" \
+          --url "$URL" --n "$N" --seconds "$SECONDS_PER_STEP" --ramp-ms 50 --compress "$COMPRESS" --game "$GAME" ${EXTRA[@]+"${EXTRA[@]}"} --room "$ROOM" --out "$STEP" < /dev/null
+        ;;
+    esac
+  done
+done
+calibrate "$OUT/calibration-end.json"
+uptime > "$OUT/load-average-end.txt" || true

package/lib/crowd-probe/worker.mjs

@@ -11,6 +11,7 @@
  * It proves the room and its Gates at that many players. It says nothing of the last hop to a real player.
  */
 import { Tally, crowdClient } from './client.mjs';
+import { referenceWork } from './reference-work.mjs';
 
 const sleep = (ms) => new Promise((done) => setTimeout(done, ms));
 
@@ -18,20 +19,41 @@ export default {
   async fetch(request, env, ctx) {
     const url = new URL(request.url);
     if (!env.PROBE_KEY || request.headers.get('x-probe-key') !== env.PROBE_KEY) return new Response('not found', { status: 404 });
-    if (url.pathname === '/hello') return Response.json({ ok: true, probe: 1 });
+    if (url.pathname === '/hello') return Response.json({ ok: true, probe: 2 });
+    if (url.pathname === '/calibrate') {
+      // The reference work, `rounds` times, in one event. This Worker cannot time it: its clock stands still while
+      // its code runs and reads a timer's due time when the timer fires, so whoever asked holds the stopwatch (the
+      // time this request took, less the time one with no rounds took). `own` is what this Worker's clock said the
+      // rounds took, across an awaited timer, for the record of what such a figure is worth.
+      const rounds = Math.min(20, Math.max(0, Math.floor(Number(url.searchParams.get('rounds') ?? 0)) || 0));
+      await sleep(2);
+      const at = Date.now(); let check = 0;
+      for (let i = 0; i < rounds; i += 1) check = referenceWork();
+      await sleep(1);
+      return Response.json({ ok: true, colo: request.cf?.colo ?? null, rounds, check, own: Date.now() - at });
+    }
     const origin = url.searchParams.get('origin'), game = url.searchParams.get('game') ?? 'crowd-circuit', room = url.searchParams.get('room');
     const from = Number(url.searchParams.get('from') ?? 0), count = Math.min(5, Math.max(1, Number(url.searchParams.get('count') ?? 5)));
     const joinAt = Number(url.searchParams.get('joinAt')), rampMs = Number(url.searchParams.get('rampMs') ?? 50);
     const measureAt = Number(url.searchParams.get('measureAt')), endAt = Number(url.searchParams.get('endAt'));
+    // `compress=1`: offer permessage-deflate as a browser does, so the studio's Gates compress for these players too.
+    const compress = url.searchParams.get('compress') === '1';
+    // `slowEvery=K`: every Kth player is a slow one (so every Gate holds some, beside healthy players). Each takes what it is sent at six frames a second, in
+    // order, as a connection that carries no more does: it falls behind, says so (the tick it holds), and the
+    // question is what that costs the players beside it. (A Worker cannot slow a socket's reads; a real slow
+    // connection is played from a computer: `crowd-load.mjs --slow-every`.)
+    const slowEvery = Math.max(0, Math.floor(Number(url.searchParams.get('slowEvery') ?? 0)) || 0);
     if (!/^https:\/\/[a-z0-9.-]+$/.test(origin ?? '') || !/^[A-Za-z0-9_-]{1,40}$/.test(room ?? '') || ![joinAt, measureAt, endAt].every(Number.isFinite) || endAt - joinAt > 900_000) return new Response('bad request', { status: 400 });
     const { readable, writable } = new TransformStream(), writer = writable.getWriter(), encoder = new TextEncoder();
     const line = (o) => writer.write(encoder.encode(JSON.stringify(o) + '\n')).catch(() => {});
     const tallies = { ack: new Tally(), gaps: new Tally(), loops: new Tally() }, players = [];
+    // A slow player's own figures, apart: its acknowledgements are late by what it is.
+    const slowTallies = { ack: new Tally(), gaps: new Tally(), loops: new Tally() };
     const run = async () => {
       const join = async (i) => {
         const key = crypto.randomUUID().replaceAll('-', '');
-        const player = { socket: null, client: null };
-        player.client = crowdClient({ index: i, game, tallies, send: (text) => { try { player.socket?.send(text); } catch { /* closing */ } } });
+        const player = { socket: null, client: null, compressed: false, slow: slowEvery > 0 && i % slowEvery === 0, held: [], nextAt: 0 };
+        player.client = crowdClient({ index: i, game, tallies: player.slow ? slowTallies : tallies, send: (text) => { try { player.socket?.send(text); } catch { /* closing */ } } });
         players.push(player);
         try {
           await sleep(Math.max(0, joinAt + i * rampMs - Date.now()));
@@ -40,31 +62,36 @@ export default {
           const match = (await page.text()).match(/window\.HOMIE_NET=(\{[^<]+\})<\/script>/);
           if (!page.ok || !match) throw new Error(`bootstrap HTTP ${page.status}`);
           const cfg = JSON.parse(match[1]);
-          const answer = await fetch(cfg.url.replace(/^ws/, 'http'), { headers: { Upgrade: 'websocket', Origin: 'null' } });
+          const answer = await fetch(cfg.url.replace(/^ws/, 'http'), { headers: { Upgrade: 'websocket', Origin: 'null', ...(compress ? { 'Sec-WebSocket-Extensions': 'permessage-deflate; client_max_window_bits' } : {}) } });
+          player.compressed = /permessage-deflate/.test(answer.headers?.get?.('sec-websocket-extensions') ?? '');
           const socket = answer.webSocket;
           if (!socket) throw new Error(`upgrade HTTP ${answer.status}`);
           socket.accept(); player.socket = socket;
-          socket.addEventListener('message', (event) => { if (typeof event.data === 'string') player.client.message(event.data, Date.now()); });
+          socket.addEventListener('message', (event) => { if (typeof event.data !== 'string') return; if (player.slow) player.held.push(event.data); else player.client.message(event.data, Date.now()); });
           socket.addEventListener('close', (event) => player.client.close(event.code, event.reason));
           socket.addEventListener('error', () => player.client.close(1006, 'error'));
           player.client.open(Date.now());
         } catch (error) { player.client.c.errors.push(String(error?.message ?? error).slice(0, 120)); player.client.close(0, 'never connected'); }
       };
       const joins = Promise.all(Array.from({ length: count }, (_, n) => join(from + n)));
-      let measuring = false, lastLine = 0, lastTick = Date.now(), worstLag = 0;
+      let measuring = false, lastLine = 0, lastWhole = 0, lastTick = Date.now(), worstLag = 0;
+      // Everything measured so far: the last line of a step, and every thirty seconds on the way, so a request that
+      // is cut off before its end has still said most of what it saw.
+      const whole = (t, now) => ({ t, from, count, colo: request.cf?.colo ?? null, measuredMs: Math.max(0, Math.min(now, endAt) - measureAt), worstTimerLagMs: worstLag, players: players.map((p) => ({ ...p.client.facts(), compressed: p.compressed, slow: p.slow, ...(p.slow ? { waiting: p.held.length } : {}) })), tallies: { ack: tallies.ack.bins, gaps: tallies.gaps.bins, loops: tallies.loops.bins }, slowAck: slowTallies.ack.bins });
       while (Date.now() < endAt) {
         const now = Date.now();
         // This Worker's own lateness: how long past ten milliseconds its timer came back.
         worstLag = Math.max(worstLag, now - lastTick - 10); lastTick = now;
         if (!measuring && now >= measureAt) { measuring = true; for (const p of players) p.client.measure(); }
-        for (const p of players) p.client.tick(now);
+        for (const p of players) { if (p.slow && p.held.length && now >= p.nextAt) { p.nextAt = now + 160; p.client.message(p.held.shift(), now); } p.client.tick(now); }
+        if (measuring && now - lastWhole >= 30000) { lastWhole = now; await line(whole('partial', now)); }
         if (now - lastLine >= 5000) { lastLine = now; await line({ t: 'sample', at: now, from, joined: players.filter((p) => p.client.c.first !== null).length, open: players.filter((p) => p.socket && !p.client.c.closed).length, measuring }); }
         await sleep(10);
       }
-      await joins;
-      const facts = players.map((p) => p.client.facts());
+      await Promise.race([joins, sleep(5000)]);
+      const last = whole('final', endAt);
       for (const p of players) { try { p.socket?.close(1000, 'probe step complete'); } catch { /* gone */ } }
-      await line({ t: 'final', from, count, colo: request.cf?.colo ?? null, measuredMs: Math.max(0, endAt - measureAt), worstTimerLagMs: worstLag, players: facts, tallies: { ack: tallies.ack.bins, gaps: tallies.gaps.bins, loops: tallies.loops.bins } });
+      await line(last);
     };
     ctx.waitUntil(run().catch((error) => line({ t: 'failed', from, error: String(error?.stack ?? error).slice(0, 400) })).finally(() => writer.close().catch(() => {})));
     return new Response(readable, { headers: { 'content-type': 'application/x-ndjson', 'cache-control': 'no-store' } });

package/lib/crowd-summary.mjs

@@ -32,10 +32,17 @@ export function summarize(report, rows = [], link = null) {
     from: report.generator === 'cloudflare' ? 'cloudflare' : 'home',
     n: report.n, total: report.total, room: report.room, toolkit: report.toolkit ?? null, stopReason: report.stopReason ?? null,
     joined: report.joined, connectedAtEnd: report.connectedAtEnd, disconnects: report.disconnects,
+    // In-Cloudflare steps: probe requests whose report never came (their players were in the room all the same).
+    lostReports: report.lostReports ?? null,
+    // What the step varied: compression offered or not.
+    variant: (report.variant ?? (report.compression?.offered === false ? 'plain' : 'deflate')) + (report.slowPlayers ? ` +${report.slowPlayers} slow` : ''),
+    // With deliberately slow players in the step: what everybody else got.
+    healthy: report.slowPlayers ? report.healthy : null,
     client: {
       inputAckMs: { p50: report.inputAckMs?.p50 ?? null, p95: report.inputAckMs?.p95 ?? null, p99: report.inputAckMs?.p99 ?? null },
       // A player's tick progression and the frames they received, a second: the median player, and the worst.
-      tickHz: { median: round(report.tickHz?.median), min: round(report.tickHz?.min) },
+      // (From the in-Cloudflare probe this is by a Worker's clock, which runs slow when the Worker is late: not shown.)
+      tickHz: report.tickHzClock === 'probe' ? { median: null, min: null } : { median: round(report.tickHz?.median), min: round(report.tickHz?.min) },
       deliveredHz: { median: round(report.deliveredHz?.median), min: round(report.deliveredHz?.min) },
       receiveGapMs: { p95: report.receiveGapMs?.p95 ?? null, p99: report.receiveGapMs?.p99 ?? null },
       ackExpired: report.ackExpired ?? null,
@@ -78,9 +85,28 @@ export function summarize(report, rows = [], link = null) {
       roomTickHzAtGates: { median: round(pick(gates.map((g) => g.roomTickHz), 0.5)), p05: round(pick(gates.map((g) => g.roomTickHz), 0.05)) }, worldGapMsMax: pick(gates.map((g) => g.worldGapMs), 1),
       // The room's clock less a Gate's (median Gate), and how long the room says it held a Gate's frame.
       roomClockOffsetMs: pick(gates.map((g) => g.roomClock?.offsetMs), 0.5), channelDelayMs: pick(gates.map((g) => g.roomClock?.delayMs), 0.5),
+      // A Gate's own lateness: the worst its 20 ms timer came late (the worst Gate-second), and the share of Gate-seconds with any timer over 20 ms late.
+      gateLoopLateMs: { median: pick(gates.map((g) => g.loopLateMs), 0.5), p95: pick(gates.map((g) => g.loopLateMs), 0.95) }, gateBusyShare: gates.length ? round(gates.filter((g) => (g.loopLateTimers ?? 0) > 0).length / gates.length, 3) : null,
       gateRttMs: { median: round(pick(gates.map((g) => g.rttMs), 0.5), 0), p95: round(pick(gates.map((g) => g.rttMs), 0.95), 0) },
       inputOffsetMs: pick(gates.map((g) => g.inputOffsetMs), 0.5),
       clientLagTicks: { p95: pick(gates.map((g) => g.clientLagTicks?.p95), 0.95), max: pick(gates.map((g) => g.clientLagTicks?.max), 1) },
+      // Players more than a second behind at one moment, the worst second (counted, whether or not they were paced).
+      behindPlayers: pick(during.map((r) => (r.gates ?? []).reduce((a, g) => a + (g.behind ?? 0), 0)), 1),
+      // Of every world every Gate wrote: the share over 100, 200 and 400 ms old, and worlds a Gate-second that came
+      // more than 75, 150 and 300 ms after the one before.
+      worldsOld: (() => { const n = gates.reduce((a, g) => a + (g.worldsOld?.n ?? 0), 0), of = (key) => (n ? round(gates.reduce((a, g) => a + (g.worldsOld?.[key] ?? 0), 0) / n, 4) : null); return n ? { worlds: n, over100: of('over100'), over200: of('over200'), over400: of('over400') } : null; })(),
+      worldGaps: (() => { const with_ = gates.filter((g) => g.worldGaps), of = (key) => (with_.length ? round(with_.reduce((a, g) => a + g.worldGaps[key], 0) / with_.length, 3) : null); return with_.length ? { over75: of('over75'), over150: of('over150'), over300: of('over300') } : null; })(),
+      // What crossed the room's channels a second: rows (a row is a player's message, mostly) and frames in, frames
+      // and megabytes out, and worlds replaced before they were sent because eight were unacknowledged.
+      roomTraffic: (() => {
+        const first = during[0], last = during.at(-1), seconds = first && last ? (last.at - first.at) / 1000 : 0;
+        if (!(seconds > 0)) return null;
+        const total = (row, f) => (row.transport ?? []).reduce((a, t) => a + (f(t) ?? 0), 0), rate = (f) => round((total(last, f) - total(first, f)) / seconds, 1);
+        const heard = (last.transport ?? []).some((t) => t.in);
+        return { inRows: heard ? rate((t) => t.in?.rows) : null, inFrames: heard ? rate((t) => t.in?.frames) : null, inKB: heard ? rate((t) => (t.in?.bytes ?? 0) / 1000) : null, outFrames: rate((t) => t.frames), outMB: rate((t) => t.bytes / 1e6), replaced: rate((t) => t.coalesced) };
+      })(),
+      // Checkpoints the room wrote without holding its frames for them.
+      unconfirmedSaves: during.at(-1)?.save?.unconfirmed ?? null,
     },
   };
 }
@@ -105,11 +131,11 @@ export function table(steps) {
   const head = ['N', 'from', 'joined/end/drops', 'ack p50/p95/p99 ms', 'loop p50/p95 ms', 'lead (late)', 'client tick Hz', 'client delivered Hz', 'wire MB/s (link)', 'KB/s a player', 'compressed (ratio)', 'driver lag ms', 'server late', 'Gates', 'room Hz at Gates med/p05', 'Gate delivered Hz med/p05', 'Gate rtt med/p95 ms', 'world age p50/p95 ms', 'paced', 'diag ok/failed'];
   const lines = [`| ${head.join(' | ')} |`, `| ${head.map(() => '---').join(' | ')} |`];
   for (const s of steps) lines.push(`| ${[
-    s.n, s.from, `${cell(s.joined)}/${cell(s.connectedAtEnd)}/${cell(s.disconnects)}`,
+    s.n, `${s.from} ${s.variant}${s.lostReports ? ` (${s.lostReports} reports lost)` : ''}`, `${cell(s.joined)}/${cell(s.connectedAtEnd)}/${cell(s.disconnects)}`,
     `${cell(s.client.inputAckMs.p50)}/${cell(s.client.inputAckMs.p95)}/${cell(s.client.inputAckMs.p99)}`,
     `${cell(s.client.inputLoopMs?.p50)}/${cell(s.client.inputLoopMs?.p95)}`,
     `${cell(s.client.inputLead?.ticksMedian)} (${cell(s.client.inputLead?.lateShare)})`,
-    cell(s.client.tickHz.median), cell(s.client.deliveredHz.median),
+    cell(s.client.tickHz.median), `${cell(s.client.deliveredHz.median)}${s.healthy ? ` (healthy: ${cell(round(s.healthy.deliveredHz?.median))}, worst ${cell(round(s.healthy.deliveredHz?.min))})` : ''}`,
     `${cell(s.client.wireMBps)} (${cell(s.link.downMBps)}${s.link.limited ? ', LINK-LIMITED' : ''})`,
     cell(s.client.wireKBsPerPlayer),
     s.client.compression ? `${s.client.compression.sockets}/${s.client.compression.of} (${cell(s.client.compression.ratio)})` : cell(s.client.compressedSockets),
@@ -123,18 +149,59 @@ export function table(steps) {
   return lines.join('\n');
 }
 
+/**
+ * The second table: what the room carried, a line a step. Read with the first.
+ *   - "rows in": players' messages reaching the room a second. "worlds out": frames to Gates a second, megabytes,
+ *     and worlds replaced before sending because eight were unacknowledged on a channel (a room that is behind).
+ *   - "worlds old": of every world every Gate wrote, the share over 100 / 200 / 400 ms old.
+ *   - "behind": players more than a second behind at one moment, the worst second. "paced": how many were being
+ *     sent fewer frames for it, the worst second.
+ * There is no column for the room's processor time: Cloudflare's count of it could not be read from a tail (the
+ * requests that are a Gate's channel reported under 1% for a room that was out of time), and nothing an object
+ * measures about itself can stand in, because its clock stands still while it runs.
+ */
+export function threadTable(steps) {
+  const cell = (v) => (v === null || v === undefined ? '—' : String(v));
+  const pct = (v) => (Number.isFinite(v) ? `${Math.round(v * 1000) / 10}%` : '—');
+  const head = ['N', 'from', 'rows in/s', 'worlds out/s (MB/s, replaced/s)', 'unconfirmed checkpoints', 'worlds old >100/>200/>400', 'gaps >75/>150/>300 a Gate-s', 'behind', 'paced'];
+  const lines = [`| ${head.join(' | ')} |`, `| ${head.map(() => '---').join(' | ')} |`];
+  for (const s of steps) {
+    const t = s.server.roomTraffic, old = s.server.worldsOld, gaps = s.server.worldGaps;
+    lines.push(`| ${[
+      s.n, `${s.from} ${s.variant}`,
+      cell(t?.inRows), t ? `${cell(t.outFrames)} (${cell(t.outMB)}, ${cell(t.replaced)})` : '—',
+      cell(s.server.unconfirmedSaves),
+      old ? `${pct(old.over100)}/${pct(old.over200)}/${pct(old.over400)}` : '—', gaps ? `${cell(gaps.over75)}/${cell(gaps.over150)}/${cell(gaps.over300)}` : '—',
+      cell(s.server.behindPlayers), cell(s.server.pacedPlayers),
+    ].join(' | ')} |`);
+  }
+  return lines.join('\n');
+}
+
 if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
   const dir = resolve(process.argv[2] ?? '.');
   const read = (file) => (existsSync(file) ? JSON.parse(readFileSync(file, 'utf8')) : null);
   const steps = [];
-  for (const name of readdirSync(dir).filter((n) => /^n\d+(-cloud)?$/.test(n)).sort((a, b) => parseInt(a.slice(1), 10) - parseInt(b.slice(1), 10) || a.length - b.length)) {
+  for (const name of readdirSync(dir).filter((n) => /^n\d+(-[a-z0-9-]+)?$/.test(n)).sort((a, b) => parseInt(a.slice(1), 10) - parseInt(b.slice(1), 10) || a.localeCompare(b))) {
     const report = read(join(dir, name, 'report.json'));
     if (!report) continue;
     const file = join(dir, name, 'diagnostics.jsonl');
     const rows = existsSync(file) ? readFileSync(file, 'utf8').split('\n').filter(Boolean).flatMap((line) => { try { return [JSON.parse(line)]; } catch { return []; } }) : [];
-    steps.push(summarize(report, rows, read(join(dir, name, 'link.json'))));
+    steps.push({ ...summarize(report, rows, read(join(dir, name, 'link.json'))), folder: name });
   }
+  // The same step played more than once (n1000-cloud, n1000-cloud-r2, ...): the spread of what matters.
+  const groups = new Map();
+  for (const step of steps) { const key = `${step.n} ${step.from} ${step.variant}`; if (!groups.has(key)) groups.set(key, []); groups.get(key).push(step); }
+  const spread = [...groups].filter(([, runs]) => runs.length > 1).map(([key, runs]) => {
+    const of = (f) => { const a = runs.map(f).filter(Number.isFinite).sort((x, y) => x - y); return a.length ? `${a[0]} to ${a.at(-1)}` : '—'; };
+    return `| ${key} | ${runs.length} | ${of((r) => r.client.deliveredHz.median)} | ${of((r) => r.client.inputAckMs.p95)} | ${of((r) => r.server.gateRttMs.p95)} | ${of((r) => r.server.worldAgeMs.p95)} | ${of((r) => r.server.worldsOld?.over200)} | ${of((r) => r.server.lateInputs)} | ${of((r) => r.server.gateDeliveredHz.median)} |`;
+  });
+  const spreadTable = spread.length ? `\n\nRepeated steps, lowest to highest:\n\n| step | runs | client delivered Hz | ack p95 ms | Gate rtt p95 ms | world age p95 ms | share of worlds over 200 ms old | server late | Gate delivered Hz |\n| --- | --- | --- | --- | --- | --- | --- | --- | --- |\n${spread.join('\n')}` : '';
   writeFileSync(join(dir, 'summary.json'), JSON.stringify(steps, null, 2));
-  writeFileSync(join(dir, 'summary.md'), `${table(steps)}\n`);
-  console.log(table(steps));
+  // How fast Cloudflare ran the reference work against this computer (crowd-cloud.mjs --calibrate), when the run measured it.
+  const calibrations = readdirSync(dir).filter((n) => /^calibration.*\.json$/.test(n)).map((n) => read(join(dir, n))).filter((c) => c?.ratio);
+  const calibration = calibrations.length ? `\n\nThe reference work (lib/crowd-probe/reference-work.mjs), a round: this computer ${calibrations.map((c) => Math.min(...c.localMs)).join(', ')} ms (its quickest); inside Cloudflare, by this computer's stopwatch, ${calibrations.map((c) => `${c.cloudMsARound.min} to ${c.cloudMsARound.max} (median ${c.cloudMsARound.median})`).join('; ')} ms. Cloudflare over this computer: ${calibrations.map((c) => `${c.ratio.min} to ${c.ratio.max} (median ${c.ratio.median})`).join('; ')}. The Worker's own clock said the same rounds took ${calibrations.map((c) => (c.workerOwnClockMs ?? []).join(', ')).join('; ')} ms in all.` : '';
+  const text = `${table(steps)}\n\nWhat the room carried:\n\n${threadTable(steps)}${spreadTable}${calibration}`;
+  writeFileSync(join(dir, 'summary.md'), `${text}\n`);
+  console.log(text);
 }

package/lib/crowd-worker.mjs

@@ -46,6 +46,7 @@ export function createLoad({
   workers = 1,
   lead = 3,
   compress = true,
+  slowEvery = 0,
   onProblem = () => {}
 }) {
   const clients = [],
@@ -53,6 +54,8 @@ export function createLoad({
     first = new Histogram(),
     welcome = new Histogram(),
     ack = new Histogram(),
+    // A deliberately slow player's own figures, apart: its acknowledgements are seconds late by what it is.
+    slowAck = new Histogram(),
     gaps = new Histogram(),
     // From the tick a player held when it stamped an input to the tick it
     // reached the rules, ms: what the system took, whatever the lead.
@@ -144,7 +147,7 @@ export function createLoad({
       c.lastTick = snap.k;
       c.lastAt = now;
       c.frames++;
-      gaps.add(gap);
+      if (!c.slow) gaps.add(gap);
     }
     const control = snap.c?.find((row) => row[0] === c.seat);
     // The room says how early this player's inputs arrive, in sixteenths of a
@@ -155,7 +158,7 @@ export function createLoad({
     // early, one fewer when it was over a tick and a half.
     if (control && control[3] !== -128 && control[3] !== undefined) {
       const early = control[3] / 16;
-      if (measuring) { if (early <= 0) lateAcks++; onTime++; loops.add((c.lead - early) * 50); }
+      if (measuring && !c.slow) { if (early <= 0) lateAcks++; onTime++; loops.add((c.lead - early) * 50); }
       (c.early ??= []).push(early); if (c.early.length > 40) c.early.shift();
       if (c.early.length >= 20 && now - (c.leadAt ?? 0) > 2000) {
         const low = c.early.slice().sort((a, b) => a - b)[Math.floor(c.early.length / 20)];
@@ -169,7 +172,7 @@ export function createLoad({
         if (tick > control[2]) break;
         c.pending.delete(tick);
         if (measuring) {
-          if (tick === control[2]) ack.add(now - at);
+          if (tick === control[2]) (c.slow ? slowAck : ack).add(now - at);
           else skipped++;
         }
       }
@@ -230,6 +233,9 @@ export function createLoad({
       });
       c.ws.on('open', () => {
         c.helloAt = performance.now();
+        // A deliberately slow player: this socket stops reading for four seconds in every five, as a connection
+        // that cannot keep up does. What it costs the players beside it is the question.
+        if (slowEvery > 0 && i % slowEvery === 0) { c.slow = true; c.slowTimer = setInterval(() => { c.ws._socket?.pause(); setTimeout(() => c.ws._socket?.resume(), 4000); }, 5000); }
         c.compressed = /permessage-deflate/.test(c.ws.extensions ?? '');
         send(c, {
           t: 'hello',
@@ -434,6 +440,7 @@ export function createLoad({
       measuring = false;
       stopping = true;
       clearInterval(timer);
+      for (const c of clients) clearInterval(c.slowTimer);
       loop.disable();
       for (const c of clients) c.ws?.close(1000, 'load step complete');
       await new Promise((r) => setTimeout(r, 250));
@@ -441,7 +448,7 @@ export function createLoad({
       return {
         ...end,
         errors: [...errors].map(([text, count]) => ({ text, count })),
-        histograms: { first, welcome, ack, gaps, loops },
+        histograms: { first, welcome, ack, slowAck, gaps, loops },
         inputReports: onTime,
         inputReportsLate: lateAcks,
         leadTicks: clients.filter((c) => !c.retired).map((c) => c.lead),
@@ -456,6 +463,7 @@ export function createLoad({
           priorSeat: c.priorSeat,
           sameSeat: c.priorSeat === null ? null : c.seat === c.priorSeat,
           retired: Boolean(c.retired),
+          slow: Boolean(c.slow),
           firstStateMs: c.first === null ? null : c.first - c.began,
           helloAfterMs: c.helloAt ? c.helloAt - c.began : null,
           closeCode: c.closeCode,

package/lib/rules-check.mjs

@@ -28,10 +28,10 @@ export const LONG_ALLOWANCE = Object.freeze({ ...ALLOWANCE, ticks: 8 * ALLOWANCE
 
 /** The ordered declarations a save depends on. Functions contribute their presence, never their implementation. */
 export function stateHash(c) {
-  const { kinds, events, commands, effects, shared, rounds, bots, roomOn, asks, map: geometry, dims, contract, seats, save, historySeconds, rewindSeconds } = c;
+  const { kinds, events, commands, effects, shared, rounds, bots, roomOn, asks, map: geometry, dims, contract, seats, save, historySeconds, rewindSeconds, botNames } = c;
   // Keep existing 2D save compatibility hashes when the 3D map vocabulary grows.
   const map = dims === 2 ? { name: geometry.name, bounds: geometry.bounds, boxes: geometry.boxes, circles: geometry.circles, spots: geometry.spots } : geometry;
-  return createHash('sha256').update(JSON.stringify({ kinds, events, commands, effects, shared, rounds, bots, roomOn, asks, map, dims, contract, seats, ...(historySeconds?{historySeconds}:{}), ...(rewindSeconds!==undefined?{rewindSeconds}:{}), tickHz: c.settings.tickHz, save }, (k, v) => typeof v === 'function' ? true : v)).digest('hex').slice(0, 32);
+  return createHash('sha256').update(JSON.stringify({ kinds, events, commands, effects, shared, rounds, bots, roomOn, asks, map, dims, contract, seats, ...(historySeconds?{historySeconds}:{}), ...(rewindSeconds!==undefined?{rewindSeconds}:{}), ...(botNames?{botNames}:{}), tickHz: c.settings.tickHz, save }, (k, v) => typeof v === 'function' ? true : v)).digest('hex').slice(0, 32);
 }
 
 /**

package/lib/rules-types.mjs

@@ -78,7 +78,7 @@ type W = World<${parameters}>;
 type RW = World<${parameters}, true>;
 type Definition = Omit<RulesDef, 'entities' | 'room' | 'move' | 'asks'> & {
   entities: ${entities}; move?: Moves;
-  room?: { records?: {key:string;identity:string;eligible?:string;fields:Readonly<Record<string,string>>}; rounds?: { seconds: number; breakSeconds: number }; bots?: { keep: number }; historySeconds?: number; rewindSeconds?: number;
+  room?: { records?: {key:string;identity:string;eligible?:string;fields:Readonly<Record<string,string>>}; rounds?: { seconds: number; breakSeconds: number }; bots?: { keep: number; names?: readonly string[] }; historySeconds?: number; rewindSeconds?: number;
     start?: (world: RW) => void;
     join?: (ctx: Pick<W, 'map' | 'shared' | 'tune' | 'math' | 'round'>, player: { seat: number; driver: 'person' | 'bot' | 'ai'; owner: string; character?: (${joins}) | null }) => ${joins};
     on?: { [K in keyof (Omit<Events, keyof GameRoomEvents> & GameRoomEvents)]?: (world: RW, event: EventData<(Omit<Events, keyof GameRoomEvents> & GameRoomEvents)[K]>) => void };