@pylonsync/realtime 0.19.0 → 0.20.0

package/dist/replication.d.ts

@@ -87,6 +87,8 @@ export interface ReplicationSummary {
     updated: number[];
     despawned: number[];
 }
+/** True when a replication frame is a full frame (it replaces the table). */
+export declare function isFullFrame(frame: Uint8Array): boolean;
 /**
  * A datagram's frame number, tick, and input ack, without applying it.
  * Throws on bytes that are not a datagram.

package/dist/wire.d.ts

@@ -131,6 +131,11 @@ export declare function decodeWebTransportInfo(body: unknown): WebTransportInfo;
  * stream.
  */
 export declare function encodeDatagramAcks(acks: ReadonlyArray<readonly [number, number]>): Uint8Array;
+/**
+ * `encodeDatagramAcks` split into messages of at most `maxBytes` bytes and
+ * `MAX_ACKS_PER_MESSAGE` acks each, so every one fits in a datagram.
+ */
+export declare function encodeDatagramAckBatches(acks: ReadonlyArray<readonly [number, number]>, maxBytes: number): Uint8Array[];
 /** A message for a WebTransport stream: a 4-byte big-endian length, then `bytes`. */
 export declare function lengthPrefixed(bytes: Uint8Array): Uint8Array;
 /** The first message on a WebTransport stream: who connects to which shard. */

package/package.json

@@ -3,7 +3,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "0.19.0",
+  "version": "0.20.0",
   "description": "Pylon realtime shard client pieces with no framework: the wire protocol, codecs, and the entity replication decoder and table.",
   "type": "module",
   "main": "./src/index.ts",

package/src/connection-webtransport.test.ts

@@ -541,3 +541,84 @@ test("a closing frame refuses fixed credentials, though the browser gets no code
   expect(FakeWebTransport.sessions.length).toBe(1);
   client.close();
 });
+
+test("a stream frame waits for its tick's datagrams, and the handlers get both with the tick's ack", async () => {
+  const { client, wt } = await openWithBaseline();
+  const seen: Array<{ spawned: number[]; updated: number[]; ack: number }> = [];
+  client.onReplication((_, summary, _tick, ack) => {
+    if (!summary.full) seen.push({ spawned: summary.spawned, updated: summary.updated, ack });
+  });
+  // Tick 2's stream frame: entity 4 spawns at x = 50 (q 5000).
+  const precision = [...new Uint8Array(new Float32Array([0.01]).buffer)];
+  const spawn = Uint8Array.from([1, 0, ...precision, 0, 1, 4, ...varint(10000), 0, 0, 0, 0]);
+  wt.sendFrames([frame(3, 4, 2, 3, spawn)]);
+  await settleSoon();
+  expect(client.entities.get(4)).toBeUndefined();
+  expect(seen).toEqual([]);
+
+  wt.sendDatagram(dg({ frame: 1, tick: 2, ack: 3, streamTick: 2, parts: 1 }, [[1, 150]]));
+  await until("tick 2", () => seen.length === 1);
+  expect(client.entities.get(4)?.qx).toBe(5000);
+  expect(seen[0]).toEqual({ spawned: [4], updated: [1], ack: 3 });
+  // Acked with the stream tick the table had then.
+  expect([...wt.datagramsIn[0]]).toEqual([1, 1, 1, 2]);
+  client.close();
+});
+
+test("a session whose datagrams stop arriving closes", async () => {
+  const errors: string[] = [];
+  const { client, wt } = await openWithBaseline();
+  client.onError((e) => errors.push(e.message));
+  // Every tick misses a part, so none is ever whole.
+  for (let tick = 2; tick <= 103; tick++) {
+    wt.sendDatagram(dg({ frame: tick, tick, ack: 0, parts: 2 }, [[1, tick]]));
+  }
+  await until("the close", () => wt.clientClose !== null);
+  expect(errors).toContain("WebTransport to shard field: datagrams stopped arriving");
+  client.close();
+});
+
+test("a session with no whole tick for 5 s closes, even when nothing arrives", async () => {
+  let clock = 1000;
+  const errors: string[] = [];
+  const client = connectShard("field", {
+    subscriberId: "p1",
+    baseUrl: "h",
+    ticket: "t1",
+    transport: "webtransport",
+    now: () => clock,
+  });
+  client.onError((e) => errors.push(e.message));
+  await until("the session", () => client.connected);
+  const wt = FakeWebTransport.sessions[FakeWebTransport.sessions.length - 1];
+  wt.sendFrames([frame(3, 4, 1, 0, fullFrame)]);
+  await until("the full frame", () => client.entities.size === 3);
+  // Silence: no stream frame, no datagram. The watchdog runs each second.
+  clock += 4000;
+  await new Promise((r) => setTimeout(r, 1100));
+  expect(wt.clientClose).toBeNull();
+  clock += 2000;
+  await until("the close", () => wt.clientClose !== null);
+  expect(errors).toContain("WebTransport to shard field: datagrams stopped arriving");
+  client.close();
+});
+
+test("a snapshot shard over WebTransport is not taken for a stall", async () => {
+  let clock = 1000;
+  const client = connectShard("field", {
+    subscriberId: "p1",
+    baseUrl: "h",
+    ticket: "t1",
+    transport: "webtransport",
+    now: () => clock,
+  });
+  await until("the session", () => client.connected);
+  const wt = FakeWebTransport.sessions[FakeWebTransport.sessions.length - 1];
+  wt.sendFrames([frame(1, 0, 1, 0, json({ n: 1 }))]);
+  await until("the snapshot", () => client.tick === 1);
+  clock += 10_000;
+  await new Promise((r) => setTimeout(r, 1100));
+  expect(wt.clientClose).toBeNull();
+  expect(client.connected).toBe(true);
+  client.close();
+});

package/src/connection.ts

@@ -6,7 +6,7 @@
  */
 
 import { ShardClock } from "./clock";
-import { EntityTable, readDatagramHeader, type ReplicationSummary } from "./replication";
+import { EntityTable, isFullFrame, readDatagramHeader, type ReplicationSummary } from "./replication";
 import {
   SHARD_PROTOCOL_VERSION,
   ShardFrameKind,
@@ -14,8 +14,7 @@ import {
   StreamFrames,
   decodeWebTransportInfo,
   type WebTransportInfo,
-  MAX_ACKS_PER_MESSAGE,
-  encodeDatagramAcks,
+  encodeDatagramAckBatches,
   encodeWebTransportHello,
   encodeWebTransportInput,
   decodeShardPayload,
@@ -187,8 +186,8 @@ interface Link {
   readonly open: boolean;
   /** Send an input, as `encodeShardInput` made it. */
   send(input: string | Uint8Array): void;
-  /** Send datagram acks (`encodeDatagramAcks`). WebTransport only. */
-  ack(acks: Uint8Array): void;
+  /** Send datagram acks: (datagram number, stream tick). WebTransport only. */
+  ack(acks: Array<[number, number]>): void;
   /**
    * What the server said before it closed the session (a closing frame):
    * a WebTransport close from the server does not reach the browser's
@@ -207,8 +206,21 @@ interface PendingTick {
   datagrams: Map<number, Uint8Array>;
 }
 
-/** Ticks of datagrams kept waiting at most; older ones count as lost. */
-const MAX_PENDING_TICKS = 32;
+/** A stream frame waiting until its tick is whole. */
+interface PendingFrame {
+  tick: number;
+  payload: Uint8Array;
+}
+
+/**
+ * How long a WebTransport session may go without a whole tick (ms). Past
+ * that, its datagrams have stopped arriving: the client closes it. The
+ * server sends at least one datagram every tick.
+ */
+const STALL_MS = 5000;
+
+/** Ticks kept waiting at most; more means the datagrams have stopped. */
+const MAX_PENDING_TICKS = 100;
 
 /** The parts of the browser's `WebTransport` the client uses. */
 interface WebTransportSession {
@@ -216,6 +228,8 @@ interface WebTransportSession {
   readonly closed: Promise<{ closeCode?: number; reason?: string } | undefined>;
   readonly datagrams: {
     readonly readable: ReadableStream<Uint8Array>;
+    /** The largest datagram the session can send now. */
+    readonly maxDatagramSize?: number;
     readonly writable?: WritableStream<Uint8Array>;
     createWritable?: () => WritableStream<Uint8Array>;
   };
@@ -286,6 +300,15 @@ export function connectShard<TSnapshot = unknown, TInput = unknown>(
   // newer ticks that are not whole yet (see `pylon_replication::datagram`).
   let wholeTick = -1;
   const pending = new Map<number, PendingTick>();
+  // Stream frames (not full) that wait with their tick's datagrams, in
+  // order, and the tick of the newest one.
+  const pendingStream: PendingFrame[] = [];
+  let receivedStreamTick = -1;
+  // When the table last held a whole tick (or the session opened).
+  let lastWholeAt = 0;
+  // Whether this link replicates entities. Only then does the server send
+  // datagrams every tick; a snapshot shard sends none.
+  let replicating = false;
   let lastKind: Link["kind"] | null = null;
   let clientSeq = 0;
   let closed = false;
@@ -422,6 +445,10 @@ export function connectShard<TSnapshot = unknown, TInput = unknown>(
     reportedTick = -1;
     wholeTick = -1;
     pending.clear();
+    pendingStream.length = 0;
+    receivedStreamTick = -1;
+    lastWholeAt = now();
+    replicating = false;
     // Inputs sent on the old connection are never acknowledged on this
     // one (acks restart with the connection).
     sentAt.clear();
@@ -452,25 +479,54 @@ export function connectShard<TSnapshot = unknown, TInput = unknown>(
   };
 
   /**
-   * Apply the buffered datagrams of the newest whole tick and of every
-   * tick before it, oldest first, then take that tick's ack and ack the
-   * datagrams. A tick is whole when all its datagrams are here and the
-   * table has the stream frames sent by then.
+   * Apply everything buffered up to the newest whole tick, oldest tick
+   * first (each tick's stream frames, then its datagrams), then take that
+   * tick's ack, tell the handlers once, and ack the datagrams. A tick is
+   * whole when all its datagrams are here and so are the stream frames
+   * sent by then. Until then its changes wait, so the handlers never see a
+   * table newer than the ack they get with it (prediction relies on that).
    */
   const applyWhole = (from: Link, at: number) => {
+    try {
+      applyWholeTick(from, at);
+    } catch (e) {
+      // Out of sync with the server. Reconnecting gets a full baseline.
+      dispatchError(e instanceof Error ? e : new Error(String(e)));
+      entities.clear();
+      from.close();
+    }
+  };
+
+  const applyWholeTick = (from: Link, at: number) => {
     let whole = -1;
     for (const [tick, p] of pending) {
-      if (tick > whole && p.datagrams.size === p.parts && entities.streamTick >= p.streamTick) {
+      if (tick > whole && p.datagrams.size === p.parts && receivedStreamTick >= p.streamTick) {
         whole = tick;
       }
     }
     if (whole < 0) return;
     const ack = (pending.get(whole) as PendingTick).ack;
-    const ticks = [...pending.keys()].filter((t) => t <= whole).sort((a, b) => a - b);
+    const ticks = new Set<number>();
+    for (const t of pending.keys()) if (t <= whole) ticks.add(t);
+    for (const f of pendingStream) if (f.tick <= whole) ticks.add(f.tick);
+    const spawned = new Set<number>();
+    const despawned = new Set<number>();
     const updated = new Set<number>();
     const acks: Array<[number, number]> = [];
-    for (const t of ticks) {
-      const p = pending.get(t) as PendingTick;
+    for (const t of [...ticks].sort((a, b) => a - b)) {
+      while (pendingStream.length > 0 && pendingStream[0].tick === t) {
+        const f = pendingStream.shift() as PendingFrame;
+        const s = entities.apply(f.payload, f.tick);
+        for (const id of s.despawned) {
+          despawned.add(id);
+          spawned.delete(id);
+          updated.delete(id);
+        }
+        for (const id of s.spawned) spawned.add(id);
+        for (const id of s.updated) updated.add(id);
+      }
+      const p = pending.get(t);
+      if (!p) continue;
       pending.delete(t);
       for (const number of [...p.datagrams.keys()].sort((a, b) => a - b)) {
         const s = entities.applyDatagram(p.datagrams.get(number) as Uint8Array);
@@ -478,14 +534,39 @@ export function connectShard<TSnapshot = unknown, TInput = unknown>(
         acks.push([s.frame, entities.streamTick]);
       }
     }
+    for (const id of spawned) updated.delete(id);
     wholeTick = whole;
+    lastWholeAt = at;
     takeAck(ack, at);
     backoff = options.reconnectBackoffMs ?? 500;
-    report({ full: false, spawned: [], updated: [...updated], despawned: [] }, whole);
-    for (let i = 0; i < acks.length; i += MAX_ACKS_PER_MESSAGE) {
-      from.ack(encodeDatagramAcks(acks.slice(i, i + MAX_ACKS_PER_MESSAGE)));
+    report(
+      { full: false, spawned: [...spawned], updated: [...updated], despawned: [...despawned] },
+      whole,
+    );
+    if (acks.length > 0) from.ack(acks);
+  };
+
+  /**
+   * Too many ticks without one whole: the datagrams stopped arriving (UDP
+   * blocked partway, say). Close the session; `"auto"` uses WebSockets
+   * from then on.
+   */
+  const stalled = (from: Link) => {
+    if (
+      from !== link ||
+      !replicating ||
+      (now() - lastWholeAt <= STALL_MS &&
+        pending.size <= MAX_PENDING_TICKS &&
+        pendingStream.length <= MAX_PENDING_TICKS)
+    ) {
+      return false;
     }
+    dispatchError(new Error(`WebTransport to shard ${currentShard}: datagrams stopped arriving`));
+    if ((options.transport ?? "websocket") === "auto") webSocketOnly = true;
+    from.close();
+    return true;
   };
+
   /** A frame from the server: a WebSocket message, or one from the WebTransport stream. */
   const onFrame = (from: Link, data: ArrayBuffer, at: number) => {
     if (from !== link) return;
@@ -531,6 +612,14 @@ export function connectShard<TSnapshot = unknown, TInput = unknown>(
         // tick's frame is built, and would make the clock run early.
         observeTick(frame.tick, at);
       }
+      if (frame.kind === ShardFrameKind.Replication) replicating = true;
+      if (frame.kind === ShardFrameKind.Replication && from.kind === "webtransport" && !isFullFrame(frame.payload)) {
+        // It waits with its tick's datagrams (see `applyWhole`).
+        pendingStream.push({ tick: frame.tick, payload: frame.payload });
+        receivedStreamTick = Math.max(receivedStreamTick, frame.tick);
+        if (!stalled(from)) applyWhole(from, at);
+        return;
+      }
       if (frame.kind === ShardFrameKind.Replication) {
         let summary: ReplicationSummary;
         try {
@@ -542,14 +631,15 @@ export function connectShard<TSnapshot = unknown, TInput = unknown>(
           from.close();
           return;
         }
-        // Over WebTransport a stream frame holds only the tick's spawns
-        // and despawns; its ack describes the table once the tick's
-        // datagrams are in too. A full frame holds everything.
-        if (from.kind === "websocket" || summary.full) takeAck(frame.ack, at);
-        if (from.kind === "webtransport" && summary.full) {
-          // Datagrams built before it describe a table that is gone.
+        // A full frame holds everything, so its ack describes the table.
+        takeAck(frame.ack, at);
+        if (from.kind === "webtransport") {
+          // What was built before it describes a table that is gone.
           wholeTick = Math.max(wholeTick, frame.tick);
+          lastWholeAt = at;
+          receivedStreamTick = Math.max(receivedStreamTick, frame.tick);
           for (const t of [...pending.keys()]) if (t <= frame.tick) pending.delete(t);
+          while (pendingStream.length > 0 && pendingStream[0].tick <= frame.tick) pendingStream.shift();
         }
         // A frame applied: the connection works, so the next reconnect
         // starts from the short delay again. (Resetting on open would
@@ -587,16 +677,16 @@ export function connectShard<TSnapshot = unknown, TInput = unknown>(
     if (from !== link) return;
     try {
       const h = readDatagramHeader(datagram);
+      replicating = true;
       if (h.tick <= wholeTick || h.parts < 1) return;
       observeTick(h.tick, at);
       let p = pending.get(h.tick);
       if (!p) {
         p = { parts: h.parts, streamTick: h.streamTick, ack: h.ack, datagrams: new Map() };
         pending.set(h.tick, p);
-        if (pending.size > MAX_PENDING_TICKS) pending.delete(Math.min(...pending.keys()));
       }
       p.datagrams.set(h.frame, datagram);
-      applyWhole(from, at);
+      if (!stalled(from)) applyWhole(from, at);
     } catch (e) {
       // Not a datagram this client can read: start over with a baseline.
       dispatchError(e instanceof Error ? e : new Error(String(e)));
@@ -781,7 +871,11 @@ export function connectShard<TSnapshot = unknown, TInput = unknown>(
         writer.write(encodeWebTransportInput(input)).catch(ignore);
       },
       ack(acks) {
-        datagrams.write(acks).catch(ignore);
+        // A datagram larger than the session allows never arrives.
+        const max = session.datagrams.maxDatagramSize ?? 1200;
+        for (const message of encodeDatagramAckBatches(acks, max)) {
+          datagrams.write(message).catch(ignore);
+        }
       },
       notice: null,
       close() {
@@ -794,6 +888,10 @@ export function connectShard<TSnapshot = unknown, TInput = unknown>(
       },
     };
     began(l);
+    // Checks for a stall also while nothing arrives at all.
+    const watchdog = setInterval(() => {
+      if (link === l) stalled(l);
+    }, 1000);
 
     void (async () => {
       const reader = stream.readable.getReader();
@@ -833,6 +931,7 @@ export function connectShard<TSnapshot = unknown, TInput = unknown>(
     // `closed` does not get them from a server close.
     const closedWith = (info: { closeCode?: number; reason?: string } | null) => {
       isOpen = false;
+      clearInterval(watchdog);
       const code = l.notice?.code ?? info?.closeCode;
       const reason = l.notice?.reason ?? info?.reason ?? "";
       const refused = code === WEBTRANSPORT_CLOSE.Policy && reason.startsWith("unauthorized");

package/src/replication.ts

@@ -105,6 +105,11 @@ export interface ReplicationSummary {
   despawned: number[];
 }
 
+/** True when a replication frame is a full frame (it replaces the table). */
+export function isFullFrame(frame: Uint8Array): boolean {
+  return frame.length >= 2 && (frame[1] & FLAG_FULL) !== 0;
+}
+
 /**
  * A datagram's frame number, tick, and input ack, without applying it.
  * Throws on bytes that are not a datagram.

package/src/wire.test.ts

@@ -2,11 +2,13 @@ import { describe, expect, test } from "bun:test";
 import { encode as msgpackEncode, decode as msgpackDecode } from "@msgpack/msgpack";
 
 import {
+  MAX_ACKS_PER_MESSAGE,
   SHARD_HEADER_LEN,
   ShardCodec,
   ShardFrameKind,
   decodeShardPayload,
   decodeShardRejection,
+  encodeDatagramAckBatches,
   encodeShardInput,
   parseShardFrame,
 } from "./wire";
@@ -70,3 +72,18 @@ test("inputs are MessagePack for MessagePack shards and JSON otherwise", () => {
   expect(encodeShardInput(null, 5, 1)).toBe('{"input":5,"client_seq":1}');
   expect(encodeShardInput(ShardCodec.Json, 5, 2)).toBe('{"input":5,"client_seq":2}');
 });
+
+test("ack batches stay within the datagram size", () => {
+  const acks = Array.from({ length: 1300 }, (_, i) => [2_000_000 + i, 1_000_000] as [number, number]);
+  const batches = encodeDatagramAckBatches(acks, 200);
+  let total = 0;
+  for (const b of batches) {
+    expect(b.length).toBeLessThanOrEqual(200);
+    expect(b[0]).toBe(1);
+    // The count is the second byte (under 128 here).
+    total += b[1];
+  }
+  expect(total).toBe(1300);
+  const big = encodeDatagramAckBatches(Array.from({ length: 1300 }, (_, i) => [i % 100, 1] as [number, number]), 1_000_000);
+  expect(big.length).toBe(Math.ceil(1300 / MAX_ACKS_PER_MESSAGE));
+});

package/src/wire.ts

@@ -238,6 +238,42 @@ export function encodeDatagramAcks(acks: ReadonlyArray<readonly [number, number]
   return Uint8Array.from(out);
 }
 
+/** Bytes `pushVarint` writes for `v`. */
+function varintLen(v: number): number {
+  let n = 1;
+  while (v >= 0x80) {
+    v = Math.floor(v / 0x80);
+    n += 1;
+  }
+  return n;
+}
+
+/**
+ * `encodeDatagramAcks` split into messages of at most `maxBytes` bytes and
+ * `MAX_ACKS_PER_MESSAGE` acks each, so every one fits in a datagram.
+ */
+export function encodeDatagramAckBatches(
+  acks: ReadonlyArray<readonly [number, number]>,
+  maxBytes: number,
+): Uint8Array[] {
+  const out: Uint8Array[] = [];
+  let start = 0;
+  // Type byte plus the count, which is below 2^14 (two varint bytes).
+  let size = 3;
+  for (let i = 0; i < acks.length; i++) {
+    const [frame, applied] = acks[i];
+    const n = varintLen(frame) + varintLen(applied);
+    if (i > start && (size + n > maxBytes || i - start === MAX_ACKS_PER_MESSAGE)) {
+      out.push(encodeDatagramAcks(acks.slice(start, i)));
+      start = i;
+      size = 3;
+    }
+    size += n;
+  }
+  if (start < acks.length) out.push(encodeDatagramAcks(acks.slice(start)));
+  return out;
+}
+
 /** A message for a WebTransport stream: a 4-byte big-endian length, then `bytes`. */
 export function lengthPrefixed(bytes: Uint8Array): Uint8Array {
   const out = new Uint8Array(4 + bytes.length);