@pylonsync/realtime 0.18.1 → 0.19.1

package/src/replication.test.ts

@@ -35,6 +35,52 @@ describe("the Rust encoder's frames (replication.fixtures.json)", () => {
   });
 });
 
+describe("the Rust encoder's datagrams (replication.fixtures.json)", () => {
+  test("apply, and skip, exactly as the Rust table does", () => {
+    const table = new EntityTable();
+    const tableJson = () =>
+      [...table.entities.values()]
+        .sort((a, b) => a.id - b.id)
+        .map((e) => ({
+          id: e.id,
+          q: [e.qx, e.qy, e.qz],
+          components: Object.fromEntries(
+            [...e.components.entries()].map(([k, v]) => [String(k), hex(v)]),
+          ),
+        }));
+    for (const [i, event] of fixtures.datagrams.entries()) {
+      if ("stream" in event && event.stream) {
+        table.apply(bytes(event.stream), event.tick as number);
+      } else if ("datagram" in event && event.datagram) {
+        const s = table.applyDatagram(bytes(event.datagram));
+        expect(s.frame, `event ${i}`).toBe(event.frame);
+        expect(s.tick).toBe(event.tick);
+        expect(s.ack).toBe(event.ack);
+        expect(s.streamTick).toBe(event.sentStreamTick);
+        expect(s.parts).toBe(event.parts);
+        expect(s.updated, `event ${i}`).toEqual(event.updated);
+        expect(s.skipped, `event ${i}`).toBe(event.skipped);
+        expect(table.streamTick).toBe(event.streamTick);
+      }
+      expect(tableJson(), `after event ${i}`).toEqual(event.table as unknown as ReturnType<typeof tableJson>);
+    }
+  });
+
+  test("hostile datagrams are refused, not half-applied", () => {
+    const t = new EntityTable();
+    expect(() => t.applyDatagram(new Uint8Array())).toThrow(ReplicationError);
+    expect(() => t.applyDatagram(new Uint8Array([1]))).toThrow("datagram version 1");
+    // version, frame 1, tick 1, ack 0, stream tick 1, precision 1.0,
+    // parts 1, count 1, id 0, then nothing.
+    expect(() =>
+      t.applyDatagram(new Uint8Array([2, 1, 1, 0, 1, 0, 0, 0x80, 0x3f, 1, 1, 0])),
+    ).toThrow("ends early");
+    expect(() =>
+      t.applyDatagram(new Uint8Array([2, 1, 1, 0, 1, 0, 0, 0x80, 0x3f, 1, 0, 7])),
+    ).toThrow("trailing bytes");
+  });
+});
+
 describe("hostile frames", () => {
   const head = (...rest: number[]) => {
     const f = new Uint8Array(6 + rest.length);

package/src/replication.ts

@@ -23,9 +23,31 @@
  * JavaScript numbers hold integers exactly up to 2^53. Entity ids and
  * quantized positions must stay below that; a frame that exceeds it is
  * refused rather than rounded.
+ *
+ * **Datagrams** (`pylon_replication::datagram`): on WebTransport, spawns,
+ * despawns, and full frames come as frames on the stream, and updates as
+ * datagrams that may be lost, duplicated, or reordered. A datagram update
+ * carries the entity's absolute position and the tick of the stream frame
+ * that spawned it (low 16 bits). {@link EntityTable.applyDatagram} applies
+ * an update only to the entity spawned at that tick and only when it is
+ * newer than the entity's last datagram, and skips it otherwise. Stream
+ * frames must go through `apply(frame, tick)` so the table knows the
+ * ticks; a full frame starts them over.
+ *
+ * ```text
+ * u8      version (2)
+ * varint  frame number, tick, input ack, stream tick (of the last stream
+ *         frame sent by this tick)
+ * f32 LE  precision
+ * varint  parts (datagrams sent for this tick)
+ * varint  update count, per entity (ids ascending, delta-coded): id,
+ *         u16 LE spawn tick, u8 mask, the masked axes (zigzag, absolute),
+ *         components if bit 8
+ * ```
  */
 
 export const REPLICATION_VERSION = 1;
+export const DATAGRAM_VERSION = 2;
 
 const FLAG_FULL = 1;
 const MASK_X = 1;
@@ -57,6 +79,24 @@ export interface ReplicatedEntity {
   components: Map<number, Uint8Array>;
 }
 
+/** What one datagram did. */
+export interface DatagramSummary {
+  frame: number;
+  tick: number;
+  ack: number;
+  /** The tick of the last stream frame the server had sent by `tick`. */
+  streamTick: number;
+  /** Datagrams the server sent for `tick`. */
+  parts: number;
+  /** Entities it updated. */
+  updated: number[];
+  /**
+   * Updates it skipped: an unknown entity, another spawn, an older
+   * datagram than the entity's last, or another precision.
+   */
+  skipped: number;
+}
+
 /** What one frame did. */
 export interface ReplicationSummary {
   full: boolean;
@@ -65,6 +105,33 @@ export interface ReplicationSummary {
   despawned: number[];
 }
 
+/** True when a replication frame is a full frame (it replaces the table). */
+export function isFullFrame(frame: Uint8Array): boolean {
+  return frame.length >= 2 && (frame[1] & FLAG_FULL) !== 0;
+}
+
+/**
+ * A datagram's frame number, tick, and input ack, without applying it.
+ * Throws on bytes that are not a datagram.
+ */
+export function readDatagramHeader(datagram: Uint8Array): Omit<DatagramSummary, "updated" | "skipped"> {
+  const r = new Reader(datagram);
+  const version = r.u8();
+  if (version !== DATAGRAM_VERSION) throw new ReplicationError(`datagram version ${version}`);
+  const frame = toSafe(r.varint(), "frame number");
+  const tick = toSafe(r.varint(), "tick");
+  const ack = toSafe(r.varint(), "ack");
+  const streamTick = toSafe(r.varint(), "stream tick");
+  r.f32();
+  const parts = toSafe(r.varint(), "parts");
+  return { frame, tick, ack, streamTick, parts };
+}
+
+/** The low 16 bits of a spawn tick, as a datagram update carries them. */
+function spawnTag(tick: number | undefined): number | undefined {
+  return tick === undefined ? undefined : tick % 0x10000;
+}
+
 class Reader {
   private offset = 0;
   private readonly view: DataView;
@@ -161,6 +228,18 @@ export class EntityTable {
   readonly entities = new Map<number, ReplicatedEntity>();
   /** World units per quantization step, from the last frame. */
   precision = 0.01;
+  /**
+   * The tick of the stream frame that spawned each entity, when frames are
+   * applied with their tick. Datagram updates name it.
+   */
+  readonly spawnTicks = new Map<number, number>();
+  /** The last datagram applied to each entity. */
+  readonly datagramFrames = new Map<number, number>();
+  /**
+   * The tick of the last frame applied with its tick. A client acks each
+   * datagram with it, so the server knows which spawns it had.
+   */
+  streamTick = 0;
 
   get size(): number {
     return this.entities.size;
@@ -170,18 +249,32 @@ export class EntityTable {
     return this.entities.get(id);
   }
 
+  /** Forget everything. */
   clear(): void {
     this.entities.clear();
+    this.spawnTicks.clear();
+    this.datagramFrames.clear();
+    this.streamTick = 0;
   }
 
-  apply(frame: Uint8Array): ReplicationSummary {
+  /**
+   * Apply a replication frame. On a connection that also gets datagrams,
+   * pass the tick from the frame's header: the table records the tick each
+   * entity spawned at and the tick of the last frame, which datagrams and
+   * their acks name. A full frame starts that record over.
+   */
+  apply(frame: Uint8Array, tick?: number): ReplicationSummary {
     const r = new Reader(frame);
     const version = r.u8();
     if (version !== REPLICATION_VERSION) throw new ReplicationError(`version ${version}`);
     const full = (r.u8() & FLAG_FULL) !== 0;
     const precision = r.f32();
     if (!Number.isFinite(precision) || precision <= 0) throw new ReplicationError("bad precision");
-    if (full) this.entities.clear();
+    if (full) {
+      this.entities.clear();
+      this.datagramFrames.clear();
+      this.spawnTicks.clear();
+    }
     this.precision = precision;
     const summary: ReplicationSummary = { full, spawned: [], updated: [], despawned: [] };
 
@@ -190,6 +283,8 @@ export class EntityTable {
     for (let i = 0; i < n; i++) {
       last = nextId(r, last);
       this.entities.delete(last);
+      this.datagramFrames.delete(last);
+      this.spawnTicks.delete(last);
       summary.despawned.push(last);
     }
 
@@ -212,6 +307,9 @@ export class EntityTable {
         z: qz * precision,
         components,
       });
+      this.datagramFrames.delete(last);
+      if (tick === undefined) this.spawnTicks.delete(last);
+      else this.spawnTicks.set(last, tick);
       summary.spawned.push(last);
     }
 
@@ -236,6 +334,78 @@ export class EntityTable {
       e.y = e.qy * precision;
       e.z = e.qz * precision;
     }
+    if (tick !== undefined) this.streamTick = tick;
+    return summary;
+  }
+
+  /**
+   * Apply one datagram. It changes nothing it cannot apply exactly (see
+   * `DatagramSummary.skipped`). Throws only on bytes that are not a
+   * datagram.
+   */
+  applyDatagram(datagram: Uint8Array): DatagramSummary {
+    const r = new Reader(datagram);
+    const version = r.u8();
+    if (version !== DATAGRAM_VERSION) throw new ReplicationError(`datagram version ${version}`);
+    const frame = toSafe(r.varint(), "frame number");
+    const tick = toSafe(r.varint(), "tick");
+    const ack = toSafe(r.varint(), "ack");
+    const streamTick = toSafe(r.varint(), "stream tick");
+    const precision = r.f32();
+    const parts = toSafe(r.varint(), "parts");
+    const n = r.varint();
+    if (n > 1n << 16n) throw new ReplicationError("count too large");
+    const summary: DatagramSummary = { frame, tick, ack, streamTick, parts, updated: [], skipped: 0 };
+    // Positions in another precision mean nothing to this table: the full
+    // frame of the new precision is still on its way.
+    const usable = precision === Math.fround(this.precision);
+    let last: number | null = null;
+    for (let i = 0; i < Number(n); i++) {
+      last = nextId(r, last);
+      const tag = r.u8() | (r.u8() << 8);
+      const mask = r.u8();
+      const qx = mask & MASK_X ? toSafe(r.zigzag(), "position") : null;
+      const qy = mask & MASK_Y ? toSafe(r.zigzag(), "position") : null;
+      const qz = mask & MASK_Z ? toSafe(r.zigzag(), "position") : null;
+      const changes = new Map<number, Uint8Array>();
+      const removed: number[] = [];
+      if (mask & MASK_COMPONENTS) {
+        const count = r.varint();
+        if (count > 256n) throw new ReplicationError("more than 256 components");
+        for (let c = 0; c < Number(count); c++) {
+          const id = r.u8();
+          const len = r.varint();
+          if (len === 0n) {
+            removed.push(id);
+            continue;
+          }
+          if (len - 1n > BigInt(Number.MAX_SAFE_INTEGER)) throw new ReplicationError("component too large");
+          changes.set(id, r.take(Number(len - 1n)));
+        }
+      }
+      const e = this.entities.get(last);
+      const lastFrame = this.datagramFrames.get(last);
+      const current =
+        usable &&
+        e !== undefined &&
+        spawnTag(this.spawnTicks.get(last)) === tag &&
+        (lastFrame === undefined || frame > lastFrame);
+      if (!current || !e) {
+        summary.skipped += 1;
+        continue;
+      }
+      if (qx !== null) e.qx = qx;
+      if (qy !== null) e.qy = qy;
+      if (qz !== null) e.qz = qz;
+      e.x = e.qx * this.precision;
+      e.y = e.qy * this.precision;
+      e.z = e.qz * this.precision;
+      for (const id of removed) e.components.delete(id);
+      for (const [id, bytes] of changes) e.components.set(id, bytes);
+      this.datagramFrames.set(last, frame);
+      summary.updated.push(last);
+    }
+    if (!r.done) throw new ReplicationError("trailing bytes");
     return summary;
   }
 }

package/src/wire.test.ts

@@ -2,11 +2,13 @@ import { describe, expect, test } from "bun:test";
 import { encode as msgpackEncode, decode as msgpackDecode } from "@msgpack/msgpack";
 
 import {
+  MAX_ACKS_PER_MESSAGE,
   SHARD_HEADER_LEN,
   ShardCodec,
   ShardFrameKind,
   decodeShardPayload,
   decodeShardRejection,
+  encodeDatagramAckBatches,
   encodeShardInput,
   parseShardFrame,
 } from "./wire";
@@ -70,3 +72,18 @@ test("inputs are MessagePack for MessagePack shards and JSON otherwise", () => {
   expect(encodeShardInput(null, 5, 1)).toBe('{"input":5,"client_seq":1}');
   expect(encodeShardInput(ShardCodec.Json, 5, 2)).toBe('{"input":5,"client_seq":2}');
 });
+
+test("ack batches stay within the datagram size", () => {
+  const acks = Array.from({ length: 1300 }, (_, i) => [2_000_000 + i, 1_000_000] as [number, number]);
+  const batches = encodeDatagramAckBatches(acks, 200);
+  let total = 0;
+  for (const b of batches) {
+    expect(b.length).toBeLessThanOrEqual(200);
+    expect(b[0]).toBe(1);
+    // The count is the second byte (under 128 here).
+    total += b[1];
+  }
+  expect(total).toBe(1300);
+  const big = encodeDatagramAckBatches(Array.from({ length: 1300 }, (_, i) => [i % 100, 1] as [number, number]), 1_000_000);
+  expect(big.length).toBe(Math.ceil(1300 / MAX_ACKS_PER_MESSAGE));
+});

package/src/wire.ts

@@ -5,7 +5,8 @@
  * message is a binary frame with an 18-byte header:
  *
  *   0   1  frame kind: 1 snapshot, 2 input rejected, 3 entity replication,
- *            4 transfer (JSON `{ shard, ticket }`: the last frame; reconnect there)
+ *            4 transfer (JSON `{ shard, ticket }`: the last frame; reconnect there),
+ *            6 closing (WebTransport only, JSON `{ code, reason }`)
  *   1   1  codec: 0 JSON, 1 MessagePack, 2 bincode, 3 custom, 4 replication
  *   2   8  tick (u64 big-endian)
  *   10  8  ack: highest client_seq the shard processed for this subscriber (0 = none)
@@ -13,6 +14,12 @@
  *
  * Inputs go up as `{ input, client_seq }`: JSON in a text frame, or the
  * shard's codec in a binary frame.
+ *
+ * Over WebTransport (wire version 3 in Rust) the same frames travel on one
+ * bidirectional stream, each after a 4-byte big-endian length, and entity
+ * updates travel as QUIC datagrams (`EntityTable.applyDatagram`). The
+ * client's hello, its inputs, and its datagram acks are described at
+ * `encodeWebTransportHello`, `ShardClientMessage`, and `encodeDatagramAcks`.
  */
 
 import { decode as msgpackDecode, encode as msgpackEncode } from "@msgpack/msgpack";
@@ -30,6 +37,13 @@ export const ShardFrameKind = {
    * last frame on the connection.
    */
   Transfer: 4,
+  /** A datagram frame (wire version 3 WebSocket only). */
+  Datagram: 5,
+  /**
+   * WebTransport only: the server is about to close the session, with
+   * JSON `{ code, reason }` (`WEBTRANSPORT_CLOSE`). The client closes it.
+   */
+  Closing: 6,
 } as const;
 
 /** Where the subscriber went: connect to `shard` with `ticket`. */
@@ -142,3 +156,209 @@ export function encodeShardInput(
   if (codec === ShardCodec.MessagePack) return msgpackEncode(envelope);
   return JSON.stringify(envelope);
 }
+
+/** Acks one message may carry (the server refuses more). */
+export const MAX_ACKS_PER_MESSAGE = 512;
+
+/**
+ * Type bytes of the client's messages on a WebTransport stream (the first
+ * byte; the rest is the message).
+ */
+export const ShardClientMessage = {
+  /** An input envelope in the shard's codec (MessagePack shards). */
+  Input: 0,
+  /** Datagram acks (`encodeDatagramAcks`). */
+  Acks: 1,
+  /** An input envelope as JSON. */
+  JsonInput: 2,
+} as const;
+
+/** WebTransport session close codes the server uses. */
+export const WEBTRANSPORT_CLOSE = {
+  Normal: 0,
+  /** Refused: bad credentials, an unknown shard. */
+  Policy: 1,
+  /** The client broke the protocol. */
+  Protocol: 2,
+  /** Try again: the client was too slow, or the server was busy. */
+  Again: 3,
+} as const;
+
+/** What `GET /_pylon/shard/webtransport` returns. */
+export interface WebTransportInfo {
+  /** The `https://` URL of the WebTransport endpoint. */
+  url: string;
+  /**
+   * SHA-256 hashes of the server's self-signed certificates, for
+   * `serverCertificateHashes`. Empty when the certificate has a public CA.
+   */
+  certHashes: Uint8Array[];
+}
+
+/** Parse the body of `GET /_pylon/shard/webtransport`. */
+export function decodeWebTransportInfo(body: unknown): WebTransportInfo {
+  const raw = body as { url?: unknown; certHashes?: unknown };
+  if (typeof raw?.url !== "string") throw new Error("WebTransport info without a url");
+  const hashes = Array.isArray(raw.certHashes) ? raw.certHashes : [];
+  return {
+    url: raw.url,
+    certHashes: hashes.map((h) => {
+      if (typeof h !== "string") throw new Error("a WebTransport certificate hash is not a string");
+      const bin = atob(h);
+      const bytes = new Uint8Array(bin.length);
+      for (let i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i);
+      if (bytes.length !== 32) throw new Error(`a ${bytes.length}-byte certificate hash`);
+      return bytes;
+    }),
+  };
+}
+
+function pushVarint(out: number[], v: number): void {
+  if (!Number.isSafeInteger(v) || v < 0) throw new Error(`varint out of range: ${v}`);
+  while (v >= 0x80) {
+    out.push((v % 0x80) | 0x80);
+    v = Math.floor(v / 0x80);
+  }
+  out.push(v);
+}
+
+/**
+ * Datagram acks: the type byte `ShardClientMessage.Acks`, a varint count,
+ * then per ack the datagram's frame number and `EntityTable.streamTick`
+ * when the client applied it, both varints. Sent as a datagram, or on the
+ * stream.
+ */
+export function encodeDatagramAcks(acks: ReadonlyArray<readonly [number, number]>): Uint8Array {
+  const out: number[] = [ShardClientMessage.Acks];
+  pushVarint(out, acks.length);
+  for (const [frame, applied] of acks) {
+    pushVarint(out, frame);
+    pushVarint(out, applied);
+  }
+  return Uint8Array.from(out);
+}
+
+/** Bytes `pushVarint` writes for `v`. */
+function varintLen(v: number): number {
+  let n = 1;
+  while (v >= 0x80) {
+    v = Math.floor(v / 0x80);
+    n += 1;
+  }
+  return n;
+}
+
+/**
+ * `encodeDatagramAcks` split into messages of at most `maxBytes` bytes and
+ * `MAX_ACKS_PER_MESSAGE` acks each, so every one fits in a datagram.
+ */
+export function encodeDatagramAckBatches(
+  acks: ReadonlyArray<readonly [number, number]>,
+  maxBytes: number,
+): Uint8Array[] {
+  const out: Uint8Array[] = [];
+  let start = 0;
+  // Type byte plus the count, which is below 2^14 (two varint bytes).
+  let size = 3;
+  for (let i = 0; i < acks.length; i++) {
+    const [frame, applied] = acks[i];
+    const n = varintLen(frame) + varintLen(applied);
+    if (i > start && (size + n > maxBytes || i - start === MAX_ACKS_PER_MESSAGE)) {
+      out.push(encodeDatagramAcks(acks.slice(start, i)));
+      start = i;
+      size = 3;
+    }
+    size += n;
+  }
+  if (start < acks.length) out.push(encodeDatagramAcks(acks.slice(start)));
+  return out;
+}
+
+/** A message for a WebTransport stream: a 4-byte big-endian length, then `bytes`. */
+export function lengthPrefixed(bytes: Uint8Array): Uint8Array {
+  const out = new Uint8Array(4 + bytes.length);
+  new DataView(out.buffer).setUint32(0, bytes.length);
+  out.set(bytes, 4);
+  return out;
+}
+
+/** The first message on a WebTransport stream: who connects to which shard. */
+export function encodeWebTransportHello(hello: {
+  shard: string;
+  sid: string;
+  ticket?: string;
+  token?: string;
+}): Uint8Array {
+  return lengthPrefixed(new TextEncoder().encode(JSON.stringify(hello)));
+}
+
+/**
+ * An input as a WebTransport stream message: `encodeShardInput`'s output
+ * after its type byte.
+ */
+export function encodeWebTransportInput(encoded: string | Uint8Array): Uint8Array {
+  const body = typeof encoded === "string" ? new TextEncoder().encode(encoded) : encoded;
+  const msg = new Uint8Array(1 + body.length);
+  msg[0] = typeof encoded === "string" ? ShardClientMessage.JsonInput : ShardClientMessage.Input;
+  msg.set(body, 1);
+  return lengthPrefixed(msg);
+}
+
+/** A frame on a WebTransport stream larger than this ends the session. */
+const MAX_STREAM_FRAME = 64 * 1024 * 1024;
+
+/**
+ * Splits a WebTransport stream's bytes into its length-prefixed frames.
+ * Chunks can end anywhere: in a length, in a frame.
+ */
+export class StreamFrames {
+  private chunks: Uint8Array[] = [];
+  private buffered = 0;
+
+  /** Add a chunk and return the frames it completed. */
+  push(chunk: Uint8Array): ArrayBuffer[] {
+    this.chunks.push(chunk);
+    this.buffered += chunk.length;
+    const frames: ArrayBuffer[] = [];
+    while (this.buffered >= 4) {
+      const head = this.peek(4);
+      const len = new DataView(head.buffer, head.byteOffset, 4).getUint32(0);
+      if (len > MAX_STREAM_FRAME) throw new Error(`a ${len}-byte stream frame`);
+      if (this.buffered < 4 + len) break;
+      this.take(4);
+      const frame = this.take(len);
+      frames.push(frame.buffer.slice(frame.byteOffset, frame.byteOffset + frame.length) as ArrayBuffer);
+    }
+    return frames;
+  }
+
+  private peek(n: number): Uint8Array {
+    if (this.chunks[0].length >= n) return this.chunks[0].subarray(0, n);
+    const out = new Uint8Array(n);
+    let at = 0;
+    for (const c of this.chunks) {
+      const part = c.subarray(0, n - at);
+      out.set(part, at);
+      at += part.length;
+      if (at === n) break;
+    }
+    return out;
+  }
+
+  private take(n: number): Uint8Array {
+    const out = this.peek(n);
+    let left = n;
+    while (left > 0) {
+      const c = this.chunks[0];
+      if (c.length <= left) {
+        this.chunks.shift();
+        left -= c.length;
+      } else {
+        this.chunks[0] = c.subarray(left);
+        left = 0;
+      }
+    }
+    this.buffered -= n;
+    return out;
+  }
+}