@irtio/bots 0.5.1 → 0.6.0

package/dist/index.js

@@ -13,14 +13,16 @@ var INVARIANT_NAMES = [
   "correction-storm",
   "misprediction",
   "snaps",
-  "disconnects"
+  "disconnects",
+  "tick-health"
 ];
 var DEFAULT_THRESHOLDS = {
   budgetBytesPerSec: 128e3,
   correctionsPerSecMax: 5,
   handlerErrorsMax: 0,
   mispredictionMagnitudeMax: Number.POSITIVE_INFINITY,
-  snapsMax: Number.POSITIVE_INFINITY
+  snapsMax: Number.POSITIVE_INFINITY,
+  overrunsMax: 0
 };
 var widenedSchemas = /* @__PURE__ */ new WeakMap();
 function widenGrids(ext, slack) {
@@ -108,29 +110,382 @@ function frameVisibilityLeaks(ext, role, type, payload, spatial) {
   return deltaVisibilityLeaks(ext, role, decodeDelta(ext, payload), spatial);
 }
 
+// src/conditions.ts
+import { FrameType as FrameType2 } from "@irtio/protocol";
+var STATE_FRAMES = /* @__PURE__ */ new Set([
+  FrameType2.WRITE,
+  FrameType2.DELTA,
+  FrameType2.CORRECT,
+  FrameType2.MSG
+]);
+var DUPLICATE_GAP_MS = 1;
+var DEFAULT_REORDER_MS = 50;
+function newConditionCounters() {
+  return {
+    droppedOut: 0,
+    droppedIn: 0,
+    duplicatedOut: 0,
+    duplicatedIn: 0,
+    reorderedOut: 0,
+    reorderedIn: 0,
+    delayed: 0
+  };
+}
+function hasConditions(c) {
+  if (!c) return false;
+  return (c.rttMs ?? 0) > 0 || (c.jitterMs ?? 0) > 0 || (c.loss ?? 0) > 0 || (c.duplicate ?? 0) > 0 || (c.reorder ?? 0) > 0;
+}
+function describeConditions(c) {
+  const parts = [];
+  if ((c.rttMs ?? 0) > 0) parts.push(`rtt ${c.rttMs} ms`);
+  if ((c.jitterMs ?? 0) > 0) parts.push(`jitter ${c.jitterMs} ms`);
+  if ((c.loss ?? 0) > 0) parts.push(`loss ${pct(c.loss ?? 0)}`);
+  if ((c.duplicate ?? 0) > 0) parts.push(`duplicate ${pct(c.duplicate ?? 0)}`);
+  if ((c.reorder ?? 0) > 0) {
+    parts.push(`reorder ${pct(c.reorder ?? 0)} within ${c.reorderMs ?? DEFAULT_REORDER_MS} ms`);
+  }
+  return parts.length === 0 ? "none" : parts.join(", ");
+}
+function pct(v) {
+  return `${Math.round(v * 1e3) / 10}%`;
+}
+var realTimers = {
+  now: () => Date.now(),
+  setTimeout(fn, ms) {
+    const timer = setTimeout(fn, ms);
+    timer.unref?.();
+    return () => clearTimeout(timer);
+  }
+};
+function conditionedTransport(base, conditions, rng, options = {}) {
+  const counters = options.counters ?? newConditionCounters();
+  const timers = options.timers ?? realTimers;
+  const rttMs = Math.max(0, conditions.rttMs ?? 0);
+  const jitterMs = Math.max(0, conditions.jitterMs ?? 0);
+  const loss = clamp01(conditions.loss ?? 0);
+  const duplicate = clamp01(conditions.duplicate ?? 0);
+  const reorder = clamp01(conditions.reorder ?? 0);
+  const reorderMs = Math.max(0, conditions.reorderMs ?? DEFAULT_REORDER_MS);
+  return {
+    connect(url) {
+      const socket = base.connect(url);
+      const cancels = /* @__PURE__ */ new Set();
+      let closed = false;
+      let nextIn = 0;
+      let nextOut = 0;
+      function schedule(delayMs, run) {
+        if (closed) return;
+        counters.delayed++;
+        let cancel = () => void 0;
+        const fire = () => {
+          cancels.delete(cancel);
+          if (!closed) run();
+        };
+        if (delayMs <= 0) {
+          cancel = timers.setTimeout(fire, 0);
+        } else {
+          cancel = timers.setTimeout(fire, delayMs);
+        }
+        cancels.add(cancel);
+      }
+      function deliver(dir, bytes, run) {
+        const type = bytes.length > 0 ? bytes[0] ?? -1 : -1;
+        const stateful = STATE_FRAMES.has(type);
+        if (stateful && loss > 0 && rng.next() < loss) {
+          if (dir === "in") counters.droppedIn++;
+          else counters.droppedOut++;
+          return;
+        }
+        const now = timers.now();
+        let at = now + rttMs / 2 + (jitterMs > 0 ? rng.next() * jitterMs : 0);
+        const reordered = stateful && reorder > 0 && rng.next() < reorder;
+        if (reordered) {
+          at += reorderMs > 0 ? rng.next() * reorderMs : 0;
+          if (dir === "in") counters.reorderedIn++;
+          else counters.reorderedOut++;
+        } else {
+          const floor = dir === "in" ? nextIn : nextOut;
+          at = Math.max(at, floor);
+          if (dir === "in") nextIn = at;
+          else nextOut = at;
+        }
+        schedule(at - now, run);
+        if (stateful && duplicate > 0 && rng.next() < duplicate) {
+          if (dir === "in") counters.duplicatedIn++;
+          else counters.duplicatedOut++;
+          schedule(at - now + DUPLICATE_GAP_MS, run);
+        }
+      }
+      const wrapper = {
+        onopen: null,
+        onmessage: null,
+        onclose: null,
+        onerror: null,
+        send(bytes) {
+          const copy = bytes.slice();
+          deliver("out", copy, () => socket.send(copy));
+        },
+        close() {
+          closed = true;
+          for (const cancel of cancels) cancel();
+          cancels.clear();
+          socket.close();
+        }
+      };
+      socket.onopen = () => wrapper.onopen?.();
+      socket.onmessage = (bytes) => {
+        const copy = bytes.slice();
+        deliver("in", copy, () => wrapper.onmessage?.(copy));
+      };
+      socket.onclose = (info) => {
+        closed = true;
+        for (const cancel of cancels) cancel();
+        cancels.clear();
+        wrapper.onclose?.(info);
+      };
+      socket.onerror = (error) => wrapper.onerror?.(error);
+      return wrapper;
+    }
+  };
+}
+function clamp01(v) {
+  return Math.min(1, Math.max(0, v));
+}
+
+// src/hits.ts
+function entityValue(frameState, collection, id) {
+  const table = frameState[collection];
+  if (typeof table !== "object" || table === null) return void 0;
+  const record = table[id];
+  if (typeof record !== "object" || record === null) return void 0;
+  const value = record.value;
+  return typeof value === "object" && value !== null ? value : void 0;
+}
+function correlateShots(shots, dump) {
+  const frames = dump.frames;
+  return shots.map((shot) => {
+    const arrivedAt = shot.sentAt + shot.uplinkMs;
+    const frame = frames.find((f) => f.at >= arrivedAt);
+    if (frame === void 0) {
+      return {
+        ...shot,
+        serverTick: void 0,
+        estimated: shot.uplinkMs > 0,
+        authoritative: void 0,
+        missDistance: void 0,
+        unresolved: frames.length === 0 ? "nothing was recorded, so there is no tick to judge this shot against" : `the shot lands after the last recorded tick (${frames[frames.length - 1]?.tick})`
+      };
+    }
+    const value = entityValue(frame.state, shot.collection, shot.target);
+    if (value === void 0) {
+      return {
+        ...shot,
+        serverTick: frame.tick,
+        estimated: shot.uplinkMs > 0,
+        authoritative: void 0,
+        missDistance: void 0,
+        unresolved: `${shot.collection}.${shot.target} is not in the recording at tick ${frame.tick}`
+      };
+    }
+    const authoritative = {};
+    let sum = 0;
+    let missing;
+    for (const [field, aimed] of Object.entries(shot.aim)) {
+      const actual = value[field];
+      if (typeof actual !== "number") {
+        missing ??= `${shot.collection}.${shot.target}.${field} is not a number on the timeline`;
+        continue;
+      }
+      authoritative[field] = actual;
+      sum += (actual - aimed) ** 2;
+    }
+    if (missing !== void 0) {
+      return {
+        ...shot,
+        serverTick: frame.tick,
+        estimated: shot.uplinkMs > 0,
+        authoritative,
+        missDistance: void 0,
+        unresolved: missing
+      };
+    }
+    return {
+      ...shot,
+      serverTick: frame.tick,
+      estimated: shot.uplinkMs > 0,
+      authoritative,
+      missDistance: Math.sqrt(sum)
+    };
+  });
+}
+
+// src/truth.ts
+function captureClientState(schema, state) {
+  const view = state ?? {};
+  const out = {};
+  for (const desc of schema.collections) {
+    const raw = view[desc.name];
+    if (raw === void 0) continue;
+    const collection = asCollection(raw);
+    if (collection) {
+      const records = {};
+      for (const id of collection.ids()) {
+        const value = collection.get(id);
+        if (value === void 0) continue;
+        records[id] = { owner: collection.ownerOf(id), value: plain(value) };
+      }
+      out[desc.name] = records;
+    } else {
+      out[desc.name] = plain(raw);
+    }
+  }
+  return out;
+}
+function asCollection(value) {
+  return value !== null && typeof value === "object" && typeof value.ownerOf === "function" ? value : void 0;
+}
+function plain(value) {
+  if (value === null || typeof value !== "object") return value;
+  const out = {};
+  for (const [k, v] of Object.entries(value)) out[k] = plain(v);
+  return out;
+}
+function diffAgainstSave(clients, saveState, meta) {
+  const bots = [];
+  const all = [];
+  let compared = 0;
+  for (const client of clients) {
+    const differences = [];
+    let seen = 0;
+    for (const [collection, held] of Object.entries(client.state)) {
+      const authoritative = saveState[collection];
+      if (isTable(held)) {
+        const table = isTable(authoritative) ? authoritative : {};
+        for (const [id, record] of Object.entries(held)) {
+          seen++;
+          const mine = record;
+          const theirs = table[id];
+          if (theirs === void 0) {
+            differences.push({
+              bot: client.bot,
+              collection,
+              id,
+              client: mine.value,
+              save: void 0,
+              detail: `bot ${client.bot} holds ${collection}.${id}, which the save does not have. A client holding an entity authority does not is a desync, not a visibility gap.`
+            });
+            continue;
+          }
+          if (mine.owner !== theirs.owner) {
+            differences.push({
+              bot: client.bot,
+              collection,
+              id,
+              field: "owner",
+              client: mine.owner,
+              save: theirs.owner,
+              detail: `bot ${client.bot} has ${collection}.${id} owned by ${String(mine.owner)}, the save says ${String(theirs.owner)}`
+            });
+          }
+          compareValues(client.bot, collection, id, mine.value, theirs.value, differences);
+        }
+      } else {
+        seen++;
+        compareValues(client.bot, collection, void 0, held, authoritative, differences);
+      }
+    }
+    compared += seen;
+    bots.push({ bot: client.bot, compared: seen, differences, ok: differences.length === 0 });
+    all.push(...differences);
+  }
+  return {
+    ok: all.length === 0,
+    bots,
+    compared,
+    differences: all,
+    saveTick: meta.saveTick,
+    saveVersion: meta.saveVersion
+  };
+}
+function isTable(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function compareValues(bot, collection, id, client, save, out) {
+  const where = id === void 0 ? collection : `${collection}.${id}`;
+  if (!isTable(client)) {
+    if (!same(client, save)) {
+      out.push({
+        bot,
+        collection,
+        ...id !== void 0 ? { id } : {},
+        client,
+        save,
+        detail: `bot ${bot} has ${where} = ${show(client)}, the save has ${show(save)}`
+      });
+    }
+    return;
+  }
+  const theirs = isTable(save) ? save : void 0;
+  for (const [field, value] of Object.entries(client)) {
+    const actual = theirs?.[field];
+    if (same(value, actual)) continue;
+    out.push({
+      bot,
+      collection,
+      ...id !== void 0 ? { id } : {},
+      field,
+      client: value,
+      save: actual,
+      detail: `bot ${bot} has ${where}.${field} = ${show(value)}, the save has ${show(actual)}`
+    });
+  }
+}
+function same(a, b) {
+  if (a === b) return true;
+  if (typeof a === "number" && typeof b === "number") {
+    return Number.isNaN(a) && Number.isNaN(b);
+  }
+  if (isTable(a) && isTable(b)) {
+    const keys = /* @__PURE__ */ new Set([...Object.keys(a), ...Object.keys(b)]);
+    for (const k of keys) if (!same(a[k], b[k])) return false;
+    return true;
+  }
+  return false;
+}
+function show(value) {
+  if (value === void 0) return "nothing";
+  if (typeof value === "number") return String(Math.round(value * 1e3) / 1e3);
+  if (typeof value === "string") return JSON.stringify(value);
+  if (isTable(value)) return JSON.stringify(value);
+  return String(value);
+}
+
 // src/observer.ts
 import {
-  FrameType as FrameType3,
+  FrameType as FrameType4,
   decodeErrorPayload,
   decodeFrame,
   decodeReply,
   decodeWelcome,
   errorByCode,
-  readCorrectClientTick
+  readCorrectClientTick,
+  readSchemaPayload,
+  withBuiltins
 } from "@irtio/protocol";
 import {
   ByteReader,
   applyDelta,
   decodeDelta as decodeDelta2,
   decodeDeltaFrom,
-  decodeSnapshot
+  decodeSnapshot,
+  schemaFromCanonical
 } from "@irtio/schema";
 
 // src/trace.ts
 import { writeFile } from "fs/promises";
-import { FrameType as FrameType2 } from "@irtio/protocol";
+import { FrameType as FrameType3 } from "@irtio/protocol";
 var FRAME_NAMES = new Map(
-  Object.entries(FrameType2).map(([name, value]) => [value, name])
+  Object.entries(FrameType3).map(([name, value]) => [value, name])
 );
 function frameName(type) {
   return FRAME_NAMES.get(type) ?? `UNKNOWN(${type})`;
@@ -202,9 +557,15 @@ function simulationOnly(ext, delta, predicts) {
       const predicted = predicts(dc.name, op.id);
       for (const index of op.mask.fields) {
         const field = desc.fields[index];
-        if (!field || !physics.bodyFields.has(field.name)) return "mixed";
-        if (predicted) sawPredicted = true;
-        else sawSync = true;
+        if (!field) return "mixed";
+        if (physics.bodyFields.has(field.name)) {
+          if (predicted) sawPredicted = true;
+          else sawSync = true;
+        } else if (!predicted && !physics.intents.includes(field.name)) {
+          sawSync = true;
+        } else {
+          return "mixed";
+        }
       }
     }
   }
@@ -260,16 +621,33 @@ var BotObserver = class {
   constructor(options) {
     this.options = options;
     this.ring = new TraceRing(options.traceLimit);
+    this.ext = options.ext;
   }
   options;
   ring;
   violations = /* @__PURE__ */ new Map();
   counts = /* @__PURE__ */ new Map();
+  /**
+   * Bug #28: the decode extension, *mutable*. It starts as `options.ext` (the schema the run was
+   * spawned with) and is rebuilt in place when an inbound `SCHEMA` frame (13) lands — a D50
+   * additive `migrate` swaps the wire mid-session, the real client rebuilds its own decoders
+   * (`session.swapSchema`), and an observer still holding the v1 descriptor underruns on the v2
+   * resync WELCOME and fails `schema-validity` on a swap that was perfect.
+   */
+  ext;
+  /** How many `SCHEMA` frames rebuilt {@link ext}. For tests and the trace, like the client's. */
+  schemaSwaps = 0;
   id = "";
   role = "";
   roomId = "";
   /** Set before `leave()`, so a deliberate teardown is not reported as a disconnect. */
   stopping = false;
+  /**
+   * D36: the last connection status this bot's client reported, recorded whatever it was. This
+   * is the *fact* of where the connection is — the room-gone watcher and the join-timeout
+   * message read it — while `disconnects` below stays the judgement, with its old rules.
+   */
+  lastStatus = "connecting";
   framesIn = 0;
   framesOut = 0;
   bytesIn = 0;
@@ -353,6 +731,7 @@ var BotObserver = class {
   }
   /** `room.on('status')`: transitions away from a healthy connection, ignored during teardown. */
   onStatus(status) {
+    this.lastStatus = status;
     if (this.stopping) return;
     if (status !== "reconnecting" && status !== "closed") return;
     this.disconnects++;
@@ -394,30 +773,32 @@ var BotObserver = class {
       ...note !== void 0 ? { note } : {}
     };
     this.ring.push(entry);
-    if (dir === "in" && type === FrameType3.CORRECT) this.lastCorrectEntry = entry;
+    if (dir === "in" && type === FrameType4.CORRECT) this.lastCorrectEntry = entry;
   }
   /** Decodes the payload independently. Throws on a bad frame; the caller counts that. */
   inspect(dir, type, bytes, now) {
     const payload = decodeFrame(bytes).payload;
     if (dir === "out") {
-      if (type === FrameType3.CALL) this.calls++;
-      if (type === FrameType3.WRITE) this.recordWrites(decodeDelta2(this.options.ext, payload), now);
+      if (type === FrameType4.CALL) this.calls++;
+      if (type === FrameType4.WRITE) this.recordWrites(decodeDelta2(this.ext, payload), now);
       return void 0;
     }
     switch (type) {
-      case FrameType3.WELCOME:
+      case FrameType4.WELCOME:
         return this.onWelcome(decodeWelcome(payload));
-      case FrameType3.DELTA:
-        return this.onDelta(decodeDelta2(this.options.ext, payload), now);
-      case FrameType3.CORRECT: {
+      case FrameType4.DELTA:
+        return this.onDelta(decodeDelta2(this.ext, payload), now);
+      case FrameType4.CORRECT: {
         const r = new ByteReader(payload);
-        const delta = decodeDeltaFrom(this.options.ext, r);
+        const delta = decodeDeltaFrom(this.ext, r);
         return this.onCorrect(delta, readCorrectClientTick(r), now);
       }
-      case FrameType3.ERROR:
+      case FrameType4.ERROR:
         return this.onError(payload);
-      case FrameType3.REPLY:
+      case FrameType4.REPLY:
         return this.onReply(payload);
+      case FrameType4.SCHEMA:
+        return this.onSchema(payload);
       default:
         return void 0;
     }
@@ -426,12 +807,12 @@ var BotObserver = class {
     this.id = welcome.clientId;
     this.role = welcome.role;
     this.roomId = welcome.roomId;
-    const snapshot = decodeSnapshot(this.options.ext, welcome.snapshot);
+    const snapshot = decodeSnapshot(this.ext, welcome.snapshot);
     this.view = snapshot.state;
     this.rememberSnapshot(snapshot.state);
     this.countSpatialSnapshot(snapshot.state);
     for (const leak of snapshotVisibilityLeaks(
-      this.options.ext,
+      this.ext,
       this.role,
       snapshot.state,
       this.spatialContext()
@@ -440,6 +821,22 @@ var BotObserver = class {
     }
     return `joined ${welcome.roomId} as ${welcome.clientId}/${welcome.role || "default"}`;
   }
+  /**
+   * Bug #28: a `SCHEMA` frame (D50 additive migrate). Rebuild the decode extension from the
+   * descriptor the frame carries, exactly as the real client's `swapSchema` does, so the resync
+   * WELCOME about to arrive decodes under the schema it was encoded with. The old view and
+   * seen-ids are dropped — they were laid out by descriptors that no longer exist, and the resync
+   * WELCOME re-seeds both (`onWelcome`).
+   */
+  onSchema(payload) {
+    const canonical = readSchemaPayload(payload);
+    const next = schemaFromCanonical(canonical);
+    this.ext = withBuiltins(next);
+    this.schemaSwaps++;
+    this.view = void 0;
+    this.seen.clear();
+    return `schema swapped (hash ${next.hash}) \u2014 decode extension rebuilt`;
+  }
   onDelta(delta, now) {
     this.checkVisibility(delta);
     this.matchWrites(delta, now);
@@ -449,7 +846,7 @@ var BotObserver = class {
     this.checkVisibility(delta);
     const names = delta.collections.map((c) => c.name).join(",");
     const kind = simulationOnly(
-      this.options.ext,
+      this.ext,
       delta,
       (c, id) => this.predictsBody ? this.predictsBody(c, id) : false
     );
@@ -470,7 +867,20 @@ var BotObserver = class {
         true
       );
     }
-    return `corrected ${names}${clientTick !== void 0 ? ` (clientTick ${clientTick})` : ""}`;
+    const fields = /* @__PURE__ */ new Set();
+    for (const dc of delta.collections) {
+      const desc = this.ext.collections.find(
+        (c) => c.name === dc.name
+      );
+      for (const op of dc.ops) {
+        if (op.op !== "update") {
+          fields.add(`${op.op} ${op.id}`);
+          continue;
+        }
+        for (const index of op.mask.fields) fields.add(desc?.fields[index]?.name ?? `#${index}`);
+      }
+    }
+    return `corrected ${names} [${[...fields].join(" ")}]${clientTick !== void 0 ? ` (clientTick ${clientTick})` : ""}`;
   }
   /**
    * The room's `correct` event, one per correction op — the only source of `previous` (the local
@@ -483,7 +893,7 @@ var BotObserver = class {
       this.suppressedCorrections++;
       return;
     }
-    const desc = this.options.ext.collections.find(
+    const desc = this.ext.collections.find(
       (c) => c.name === correction.collection
     );
     const predictedBody = desc?.physics !== void 0 && correction.fields.length > 0 && correction.fields.every((f) => desc.physics?.bodyFields.has(f));
@@ -551,14 +961,9 @@ var BotObserver = class {
    * frame — a departing neighbour, not a leak.
    */
   checkVisibility(delta) {
-    if (this.view) applyDelta(this.options.ext, this.view, delta);
+    if (this.view) applyDelta(this.ext, this.view, delta);
     this.countSpatialDelta(delta);
-    for (const leak of deltaVisibilityLeaks(
-      this.options.ext,
-      this.role,
-      delta,
-      this.spatialContext()
-    )) {
+    for (const leak of deltaVisibilityLeaks(this.ext, this.role, delta, this.spatialContext())) {
       this.violate("visibility-leak", leak);
     }
     this.remember(delta);
@@ -566,12 +971,12 @@ var BotObserver = class {
   countSpatialDelta(delta) {
     if (!this.view || this.id === "") return;
     for (const collection of delta.collections) {
-      const desc = this.options.ext.collection(collection.name);
+      const desc = this.ext.collection(collection.name);
       if (desc.visibility === "spatial-grid") this.spatialOpsJudged += collection.ops.length;
     }
   }
   countSpatialSnapshot(state) {
-    for (const desc of this.options.ext.collections) {
+    for (const desc of this.ext.collections) {
       if (desc.visibility !== "spatial-grid") continue;
       this.spatialOpsJudged += state[desc.name]?.size ?? 0;
     }
@@ -585,7 +990,7 @@ var BotObserver = class {
     return ids;
   }
   rememberSnapshot(state) {
-    for (const desc of this.options.ext.collections) {
+    for (const desc of this.ext.collections) {
       if (desc.kind !== "entity") continue;
       const collection = state[desc.name];
       if (!collection) continue;
@@ -610,6 +1015,7 @@ var BotObserver = class {
     } catch {
       name = `code ${error.code}`;
     }
+    if (name === "E_STARTING") return `${name} (room starting \u2014 not counted)`;
     if (!error.fatal) {
       this.errors++;
       this.violate("handler-error", `${name}: ${error.message}`);
@@ -738,6 +1144,7 @@ function freshValue(rng, desc, ctx) {
 }
 
 // src/report.ts
+import { mergeProfiles } from "@irtio/protocol";
 function percentile(sorted, p) {
   if (sorted.length === 0) return 0;
   const index = Math.min(sorted.length - 1, Math.max(0, Math.ceil(p / 100 * sorted.length) - 1));
@@ -809,7 +1216,39 @@ function detailFor(name, observers, violations, thresholds) {
       const count = observers.reduce((sum, o) => sum + o.disconnects, 0);
       return count === 0 ? "every bot stayed connected" : `${count} disconnect(s)${examples(observers, name)}`;
     }
+    case "tick-health":
+      return "";
+  }
+}
+function tickHealthResult(reading, thresholds) {
+  const where = reading?.source !== void 0 ? ` (from ${reading.source})` : "";
+  if (reading === void 0 || reading.unavailable !== void 0) {
+    return {
+      name: "tick-health",
+      ok: true,
+      state: "unavailable",
+      violations: 0,
+      detail: reading?.unavailable ?? "the server was not asked for its tick counters, so this run says nothing about them"
+    };
+  }
+  if (reading.overruns === void 0) {
+    return {
+      name: "tick-health",
+      ok: true,
+      state: "unavailable",
+      violations: 0,
+      detail: `no overrun count came back${where}`
+    };
   }
+  const worst = reading.maxTickMs !== void 0 ? `, worst tick ${round(reading.maxTickMs)} ms` : "";
+  const violated = reading.overruns > thresholds.overrunsMax;
+  return {
+    name: "tick-health",
+    ok: !violated,
+    state: violated ? "violation" : "ok",
+    violations: violated ? reading.overruns : 0,
+    detail: `${reading.overruns} tick overrun(s) in the run window, tolerated ${thresholds.overrunsMax}${worst}${where}` + (violated ? " \u2014 the room fell behind and dropped the backlog" : "")
+  };
 }
 function buildReport(options) {
   const { observers, roomId, lags } = options;
@@ -817,9 +1256,17 @@ function buildReport(options) {
   const durationMs = Math.max(1, options.durationMs);
   const seconds = durationMs / 1e3;
   const invariants = INVARIANT_NAMES.map((name) => {
+    if (name === "tick-health") return tickHealthResult(options.tickHealth, thresholds);
     const violations = total(observers, name);
     const ok = name === "handler-error" ? violations <= thresholds.handlerErrorsMax : violations === 0;
-    return { name, ok, violations, detail: detailFor(name, observers, violations, thresholds) };
+    const state = ok ? "ok" : "violation";
+    return {
+      name,
+      ok,
+      state,
+      violations,
+      detail: detailFor(name, observers, violations, thresholds)
+    };
   });
   const perBot = observers.map((o) => o.stats);
   const totals = {
@@ -852,10 +1299,20 @@ function buildReport(options) {
     bytesOutPerSecondPerBot: round(totals.bytesOut / seconds / bots),
     convergence,
     convergenceLagMs: convergence?.p50Ms,
-    tracePath: options.tracePath
+    tracePath: options.tracePath,
+    ...options.profiles !== void 0 && options.profiles.length > 0 ? { profile: options.profiles.reduce(mergeProfiles) } : {}
   };
 }
 
+// src/scenario.ts
+function defineScenario(scenario) {
+  return scenario;
+}
+function looksLikeScenario(value) {
+  const s = value;
+  return typeof s === "object" && s !== null && Number.isInteger(s.bots) && s.bots > 0 && typeof s.script === "function" && typeof s.assert === "function";
+}
+
 // src/script.ts
 import { rpcTable } from "@irtio/protocol";
 function ownableCollections(schema) {
@@ -871,11 +1328,16 @@ function writableFields(desc, cheat) {
 function voidServerRpcs(schema) {
   return rpcTable(schema).filter((r) => r.direction === "server" && r.returns === void 0);
 }
-function asCollection(value) {
+function asCollection2(value) {
   return value && typeof value.ownerOf === "function" ? value : void 0;
 }
+function cheatPredicate(cheat) {
+  if (cheat === void 0) return () => false;
+  if (typeof cheat === "function") return (index) => cheat(index) === true;
+  return () => cheat;
+}
 function randomScript(schema, options = {}) {
-  const cheat = options.cheat ?? false;
+  const cheats = cheatPredicate(options.cheat);
   const stepMs = options.stepMs ?? 50;
   const step = options.step ?? 8;
   const rpcChance = options.rpcChance ?? 0.05;
@@ -883,24 +1345,25 @@ function randomScript(schema, options = {}) {
   const collections = ownableCollections(schema);
   const rpcs = voidServerRpcs(schema);
   return async (bot) => {
+    const cheat = cheats(bot.index);
     const room = bot.room;
     const origins = /* @__PURE__ */ new Map();
     while (!bot.stopped) {
       for (const desc of collections) {
-        const collection = asCollection(room.state[desc.name]);
+        const collection = asCollection2(room.state[desc.name]);
         if (!collection) continue;
         for (const id of [...collection.ids()]) {
           if (collection.ownerOf(id) !== room.me) continue;
           const instance = collection.get(id);
           if (!instance) continue;
-          writeFields(bot, desc, id, instance, origins);
+          writeFields(bot, desc, id, instance, origins, cheat);
         }
       }
       if (rpcs.length > 0 && bot.rng.chance(rpcChance)) await callRandomRpc(bot, room.call, rpcs);
       await bot.wait(stepMs);
     }
   };
-  function writeFields(bot, desc, id, instance, origins) {
+  function writeFields(bot, desc, id, instance, origins, cheat) {
     const fields = writableFields(desc, cheat);
     if (fields.length === 0) return;
     const count = Math.min(fieldsPerStep, fields.length);
@@ -947,30 +1410,48 @@ function relayEchoScript(options = {}) {
 // src/spawn.ts
 import {
   joinRelay,
-  joinRoom
+  joinRoom,
+  webSocketTransport
 } from "@irtio/client";
-import { relaySchema, withBuiltins } from "@irtio/protocol";
+import { relaySchema, withBuiltins as withBuiltins2 } from "@irtio/protocol";
 var DEFAULT_SEED = 96016;
 var DEFAULT_TRACE_LIMIT = 4096;
+var DEFAULT_JOIN_TIMEOUT_MS = 2e4;
+var DEFAULT_ROOM_GONE_GRACE_MS = 5e3;
+var ROOM_GONE_POLL_MS = 250;
 function per(value, index) {
   if (value === void 0) return void 0;
   return typeof value === "function" ? value(index) : value;
 }
 var BotImpl = class {
-  constructor(index, room, observer, seed, startedAt) {
+  constructor(index, room, observer, seed, startedAt, conditions, shots) {
     this.index = index;
     this.room = room;
     this.observer = observer;
     this.startedAt = startedAt;
+    this.conditions = conditions;
+    this.shots = shots;
     this.rng = makeRng(seed);
   }
   index;
   room;
   observer;
   startedAt;
+  conditions;
+  shots;
   stopped = false;
   rng;
   stopListeners = /* @__PURE__ */ new Set();
+  shot(request) {
+    const tick = this.room.tick;
+    this.shots.push({
+      ...request,
+      bot: this.index,
+      sentAt: Date.now(),
+      clientTick: typeof tick === "number" ? tick : void 0,
+      uplinkMs: (this.conditions?.rttMs ?? 0) / 2
+    });
+  }
   get id() {
     return this.observer.id;
   }
@@ -1025,6 +1506,43 @@ var BotImpl = class {
     }
   }
 };
+var JoinTimeoutError = class extends Error {
+  constructor(bot, timeoutMs, lastStatus) {
+    super(
+      `irtio bots: bot ${bot} did not finish joining within ${timeoutMs} ms (last status: ${lastStatus})` + (lastStatus === "starting" ? " \u2014 the room kept answering E_STARTING, so it never came up" : "")
+    );
+    this.bot = bot;
+    this.timeoutMs = timeoutMs;
+    this.lastStatus = lastStatus;
+  }
+  bot;
+  timeoutMs;
+  lastStatus;
+  name = "JoinTimeoutError";
+};
+async function withJoinTimeout(joining, index, timeoutMs, observer, sockets) {
+  if (timeoutMs <= 0) return joining;
+  let timer;
+  try {
+    return await Promise.race([
+      joining,
+      new Promise((_resolve, reject) => {
+        timer = setTimeout(() => {
+          for (const socket of sockets) {
+            try {
+              socket.close();
+            } catch {
+            }
+          }
+          reject(new JoinTimeoutError(index, timeoutMs, observer.lastStatus));
+        }, timeoutMs);
+      })
+    ]);
+  } finally {
+    if (timer !== void 0) clearTimeout(timer);
+  }
+}
+var CONDITIONS_SEED_OFFSET = 377022;
 async function spawnBots(n, options = {}) {
   if (!Number.isInteger(n) || n < 1) {
     throw new Error(`irtio bots: spawnBots needs at least one bot, got ${String(n)}`);
@@ -1035,15 +1553,21 @@ async function spawnBots(n, options = {}) {
     correctionsPerSecMax: options.correctionsPerSecMax ?? DEFAULT_THRESHOLDS.correctionsPerSecMax,
     handlerErrorsMax: options.handlerErrorsMax ?? DEFAULT_THRESHOLDS.handlerErrorsMax,
     mispredictionMagnitudeMax: options.mispredictionMagnitudeMax ?? DEFAULT_THRESHOLDS.mispredictionMagnitudeMax,
-    snapsMax: options.snapsMax ?? DEFAULT_THRESHOLDS.snapsMax
+    snapsMax: options.snapsMax ?? DEFAULT_THRESHOLDS.snapsMax,
+    overrunsMax: options.overrunsMax ?? DEFAULT_THRESHOLDS.overrunsMax
   };
-  const ext = options.schema ? withBuiltins(options.schema) : relaySchema;
+  const ext = options.schema ? withBuiltins2(options.schema) : relaySchema;
   const seed = options.seed ?? DEFAULT_SEED;
   const traceLimit = options.traceLimit ?? DEFAULT_TRACE_LIMIT;
   const writeLog = new WriteLog();
   const lags = [];
   const observers = [];
   const bots = [];
+  const shots = [];
+  const conditions = [];
+  const joinTimeoutMs = options.joinTimeoutMs ?? DEFAULT_JOIN_TIMEOUT_MS;
+  const roomGoneGraceMs = options.roomGoneGraceMs ?? DEFAULT_ROOM_GONE_GRACE_MS;
+  const baseTransport = options.transport ?? webSocketTransport;
   async function joinOne(index, roomId2) {
     const observer = new BotObserver({
       index,
@@ -1056,34 +1580,76 @@ async function spawnBots(n, options = {}) {
     });
     const role = per(options.role, index);
     const name = per(options.name, index);
+    const sockets = [];
+    const asked = per(options.conditions, index);
+    const injected = hasConditions(asked) ? asked : void 0;
+    const counters = newConditionCounters();
+    conditions[index] = { bot: index, conditions: injected, counters };
+    const recording = {
+      connect(url) {
+        const socket = baseTransport.connect(url);
+        sockets.push(socket);
+        return socket;
+      }
+    };
+    const transport = injected === void 0 ? recording : conditionedTransport(
+      recording,
+      injected,
+      makeRng(seed + index + CONDITIONS_SEED_OFFSET),
+      {
+        counters
+      }
+    );
     const common = {
       room: roomId2,
       ...options.url !== void 0 ? { url: options.url } : {},
       ...options.key !== void 0 ? { key: options.key } : {},
       ...role !== void 0 ? { role } : {},
       ...name !== void 0 ? { name } : {},
-      ...options.transport !== void 0 ? { transport: options.transport } : {},
+      transport,
       onFrame: (dir, type, bytes) => observer.onFrame(dir, type, bytes),
       onStatus: (status) => observer.onStatus(status)
     };
-    const room = options.schema ? await joinRoom(options.schema, {
+    const joining = options.schema ? joinRoom(options.schema, {
       ...common,
       ...options.flushMs !== void 0 ? { writeIntervalMs: options.flushMs } : {},
       ...options.rpc !== void 0 ? { rpc: options.rpc } : {},
-      ...options.physics !== void 0 ? { physics: options.physics } : {}
-    }) : await joinRelay(common);
+      ...options.physics !== void 0 ? { physics: options.physics } : {},
+      ...options.profile === true ? { profile: true } : {}
+    }) : joinRelay(common);
+    const room = await withJoinTimeout(joining, index, joinTimeoutMs, observer, sockets);
     room.on("correct", (correction) => observer.onCorrection(correction));
     const prediction = room.prediction;
     if (prediction) observer.predictsBody = (c, id) => prediction.predicts(c, id);
     observers[index] = observer;
-    return new BotImpl(index, room, observer, seed + index, startedAt);
+    return new BotImpl(
+      index,
+      room,
+      observer,
+      seed + index,
+      startedAt,
+      injected,
+      shots
+    );
   }
   const first = await joinOne(0, options.room ?? "");
   bots.push(first);
   const roomId = first.room.id;
   if (n > 1) {
-    const rest = await Promise.all(Array.from({ length: n - 1 }, (_, i) => joinOne(i + 1, roomId)));
-    bots.push(...rest);
+    const rest = await Promise.allSettled(
+      Array.from({ length: n - 1 }, (_, i) => joinOne(i + 1, roomId))
+    );
+    const failure = rest.find((r) => r.status === "rejected");
+    if (failure !== void 0) {
+      for (const settled of rest) {
+        if (settled.status === "fulfilled") settled.value.room.leave();
+      }
+      first.room.leave();
+      throw failure.reason;
+    }
+    for (const settled of rest) {
+      if (settled.status === "fulfilled") bots.push(settled.value);
+    }
   }
   const scriptErrors = [];
   let firstError;
@@ -1101,36 +1667,77 @@ async function spawnBots(n, options = {}) {
     }
   });
   const finished = Promise.all(scripts).then(() => void 0);
+  let endedBy;
   let deadline;
   if (options.durationMs !== void 0) {
     deadline = setTimeout(() => {
+      endedBy ??= "duration";
       for (const bot of bots) bot.stop();
     }, options.durationMs);
-    deadline.unref?.();
   }
+  let goneSince = 0;
+  let goneTimer;
+  if (roomGoneGraceMs > 0) {
+    goneTimer = setInterval(() => {
+      const down = observers.every(
+        (o) => o.lastStatus === "reconnecting" || o.lastStatus === "closed"
+      );
+      if (!down) {
+        goneSince = 0;
+        return;
+      }
+      goneSince ||= Date.now();
+      if (Date.now() - goneSince < roomGoneGraceMs) return;
+      endedBy ??= "room-gone";
+      for (const bot of bots) bot.stop();
+    }, ROOM_GONE_POLL_MS);
+    goneTimer.unref?.();
+  }
+  void finished.then(() => {
+    endedBy ??= "scripts";
+    if (deadline) clearTimeout(deadline);
+    if (goneTimer) clearInterval(goneTimer);
+  });
   let stoppedAt;
+  let tickHealth;
   const rings = () => observers.map((o) => o.ring);
+  let profiles;
   const runner = {
     bots,
     roomId,
     trace: makeTrace(startedAt, rings),
     scriptErrors,
+    shots,
+    conditions,
+    get endedBy() {
+      return endedBy;
+    },
     [Symbol.iterator]: () => bots[Symbol.iterator](),
     async done() {
       await finished;
       if (firstError !== void 0) throw firstError;
     },
+    recordTickHealth(reading) {
+      tickHealth = reading;
+    },
     report() {
+      if (options.profile === true) {
+        const live = bots.map((bot) => bot.room.profile?.total()).filter((p) => p !== void 0);
+        if (live.length > 0) profiles = live;
+      }
       return buildReport({
         observers,
         roomId,
         durationMs: (stoppedAt ?? Date.now()) - startedAt,
         lags,
-        thresholds
+        thresholds,
+        tickHealth,
+        ...profiles !== void 0 ? { profiles } : {}
       });
     },
     async stop() {
       if (deadline) clearTimeout(deadline);
+      if (goneTimer) clearInterval(goneTimer);
       for (const bot of bots) bot.stop();
       await finished;
       stoppedAt ??= Date.now();
@@ -1141,22 +1748,170 @@ async function spawnBots(n, options = {}) {
   };
   return runner;
 }
+
+// src/timeline.ts
+var Collection = class {
+  constructor(raw, entity) {
+    this.raw = raw;
+    this.entity = entity;
+  }
+  raw;
+  entity;
+  entry(id) {
+    if (!this.entity) return void 0;
+    const row = this.raw[id];
+    return typeof row === "object" && row !== null ? row : void 0;
+  }
+  get(id) {
+    const value = this.entry(id)?.value;
+    return typeof value === "object" && value !== null ? value : void 0;
+  }
+  has(id) {
+    return this.entity && Object.hasOwn(this.raw, id);
+  }
+  ids() {
+    return this.entity ? Object.keys(this.raw) : [];
+  }
+  get size() {
+    return this.entity ? Object.keys(this.raw).length : 0;
+  }
+  owner(id) {
+    const owner = this.entry(id)?.owner;
+    return typeof owner === "string" && owner !== "" ? owner : void 0;
+  }
+  get value() {
+    return this.entity ? void 0 : this.raw;
+  }
+};
+var UnknownCollectionError = class extends Error {
+  name = "UnknownCollectionError";
+  constructor(collection, known) {
+    super(
+      `irtio scenario: this room has no collection named "${collection}". It has: ${known.length > 0 ? known.join(", ") : "none"}.`
+    );
+  }
+};
+var TickNotRecordedError = class extends Error {
+  constructor(tick, first, last, dropped) {
+    super(
+      first === void 0 ? `irtio scenario: nothing was recorded, so tick ${tick} cannot be read.` : `irtio scenario: tick ${tick} is not in the recording, which holds ticks ${first} to ${last}` + (dropped > 0 ? `. ${dropped} earlier tick(s) were dropped by the recorder's cap, so raise it or shorten the run.` : ".")
+    );
+    this.tick = tick;
+  }
+  tick;
+  name = "TickNotRecordedError";
+};
+function momentOf(dump, index, read) {
+  const frame = dump.frames[index];
+  read(frame.tick);
+  const known = Object.keys(dump.collections);
+  const cache = /* @__PURE__ */ new Map();
+  return new Proxy({}, {
+    get(_target, property) {
+      if (typeof property !== "string") return void 0;
+      const cached = cache.get(property);
+      if (cached) return cached;
+      const kind = dump.collections[property];
+      if (kind === void 0) throw new UnknownCollectionError(property, known);
+      const raw = frame.state[property];
+      const view = new Collection(
+        typeof raw === "object" && raw !== null ? raw : {},
+        kind === "entity"
+      );
+      cache.set(property, view);
+      return view;
+    },
+    has: (_target, property) => typeof property === "string" && property in dump.collections,
+    ownKeys: () => [...known],
+    getOwnPropertyDescriptor: () => ({ enumerable: true, configurable: true })
+  });
+}
+function makeTimeline(dump) {
+  const index = /* @__PURE__ */ new Map();
+  dump.frames.forEach((f, i) => index.set(f.tick, i));
+  const ticks = dump.frames.map((f) => f.tick);
+  const results = [];
+  let readTicks = [];
+  const read = (tick) => {
+    readTicks.push(tick);
+  };
+  const at = (tick) => {
+    const i = index.get(tick);
+    if (i === void 0) {
+      throw new TickNotRecordedError(tick, ticks[0], ticks[ticks.length - 1], dump.dropped);
+    }
+    return momentOf(dump, i, read);
+  };
+  return {
+    roomId: dump.roomId,
+    ticks,
+    dropped: dump.dropped,
+    at,
+    find(match) {
+      for (let i = 0; i < dump.frames.length; i++) {
+        const tick = dump.frames[i].tick;
+        const state = momentOf(dump, i, read);
+        if (match(state, tick)) return { tick, state };
+      }
+      return void 0;
+    },
+    check(name, assertion) {
+      readTicks = [];
+      try {
+        assertion();
+        results.push({
+          name,
+          ok: true,
+          ...readTicks.length > 0 ? { tick: readTicks[readTicks.length - 1] } : {},
+          detail: readTicks.length === 0 ? "held (read no tick)" : `held, reading tick ${readTicks[readTicks.length - 1]}` + (readTicks.length > 1 ? ` (${readTicks.length} ticks read)` : "")
+        });
+      } catch (err) {
+        results.push({
+          name,
+          ok: false,
+          ...readTicks.length > 0 ? { tick: readTicks[readTicks.length - 1] } : {},
+          detail: err instanceof Error ? err.message : String(err)
+        });
+      }
+    },
+    get results() {
+      return results;
+    }
+  };
+}
 export {
   BotObserver,
+  DEFAULT_JOIN_TIMEOUT_MS,
+  DEFAULT_REORDER_MS,
+  DEFAULT_ROOM_GONE_GRACE_MS,
   DEFAULT_SEED,
   DEFAULT_THRESHOLDS,
   DEFAULT_TRACE_LIMIT,
   INVARIANT_NAMES,
+  JoinTimeoutError,
+  TickNotRecordedError,
   TraceRing,
+  UnknownCollectionError,
   WriteLog,
   buildReport,
+  captureClientState,
+  cheatPredicate,
+  conditionedTransport,
   convergenceStats,
+  correlateShots,
+  defineScenario,
   deltaVisibilityLeaks,
+  describeConditions,
+  diffAgainstSave,
   frameName,
   frameVisibilityLeaks,
   freshValue,
+  hasConditions,
+  looksLikeScenario,
   makeRng,
+  makeTimeline,
   makeTrace,
+  newConditionCounters,
   nextValue,
   randomScript,
   relayEchoScript,