@pylonsync/realtime 0.14.0

package/src/replication.test.ts

@@ -0,0 +1,64 @@
+import { describe, expect, test } from "bun:test";
+
+import { EntityTable, ReplicationError } from "./replication";
+import fixtures from "./replication.fixtures.json";
+
+function bytes(hex: string): Uint8Array {
+  const out = new Uint8Array(hex.length / 2);
+  for (let i = 0; i < out.length; i++) out[i] = parseInt(hex.slice(i * 2, i * 2 + 2), 16);
+  return out;
+}
+
+function hex(b: Uint8Array): string {
+  return Array.from(b, (x) => x.toString(16).padStart(2, "0")).join("");
+}
+
+describe("the Rust encoder's frames (replication.fixtures.json)", () => {
+  test("apply to the same tables the Rust decoder holds", () => {
+    const table = new EntityTable();
+    for (const [i, step] of fixtures.frames.entries()) {
+      table.apply(bytes(step.frame));
+      const got = [...table.entities.values()]
+        .sort((a, b) => a.id - b.id)
+        .map((e) => ({
+          id: e.id,
+          q: [e.qx, e.qy, e.qz],
+          components: Object.fromEntries(
+            [...e.components.entries()].map(([k, v]) => [String(k), hex(v)]),
+          ),
+        }));
+      expect(got, `after frame ${i}`).toEqual(step.table as unknown as typeof got);
+      for (const e of table.entities.values()) {
+        expect(e.x).toBeCloseTo(e.qx * step.precision, 9);
+      }
+    }
+  });
+});
+
+describe("hostile frames", () => {
+  const head = (...rest: number[]) => {
+    const f = new Uint8Array(6 + rest.length);
+    f[0] = 1;
+    new DataView(f.buffer).setFloat32(2, 1, true);
+    f.set(rest, 6);
+    return f;
+  };
+
+  test("are refused, not half-applied into garbage", () => {
+    const t = new EntityTable();
+    expect(() => t.apply(new Uint8Array())).toThrow(ReplicationError);
+    expect(() => t.apply(new Uint8Array([2, 0, 0, 0, 0x80, 0x3f]))).toThrow("version 2");
+    expect(() => t.apply(new Uint8Array([1, 0, 0, 0, 0, 0]))).toThrow("bad precision");
+    expect(() => t.apply(head(0xff, 0xff, 0xff, 0xff, 0x0f))).toThrow("count too large");
+    expect(() => t.apply(head(0, 0, 1, 7, 0))).toThrow("unknown entity 7");
+    expect(() => t.apply(head(2, 5, 0, 0, 0))).toThrow("do not ascend");
+    expect(() => t.apply(head(0, 0, 0, 9))).toThrow("trailing bytes");
+    expect(() => t.apply(head(0, 1, 1, 0, 0, 0, 1, 4, 50, 1))).toThrow("runs past the end");
+  });
+
+  test("an id beyond 2^53 is refused rather than rounded", () => {
+    // One spawn with id 2^53 (varint 80 80 80 80 80 80 80 10).
+    const f = head(0, 1, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x10, 0, 0, 0, 0, 0);
+    expect(() => new EntityTable().apply(f)).toThrow("beyond 2^53");
+  });
+});

package/src/replication.ts

@@ -0,0 +1,241 @@
+/**
+ * Entity replication frames (see `pylon_replication::frame` in Rust).
+ *
+ * A shard that replicates entities sends each subscriber, per tick, the
+ * entities that left its view (despawn), came into view (spawn, full
+ * state), or changed (update: only the axes that moved, as a quantized
+ * difference, and the components that changed or were removed).
+ * {@link EntityTable} applies them; after each frame it holds what the
+ * server says this subscriber sees.
+ *
+ * ```text
+ * u8      version (1)
+ * u8      flags: bit 0 FULL: clear the table first
+ * f32 LE  precision
+ * varint  despawn count, ids (first absolute, then differences; ascending)
+ * varint  spawn count, per entity: id, x, y, z (zigzag, quantized), components
+ * varint  update count, per entity: id, u8 mask (1 x, 2 y, 4 z, 8 components),
+ *         changed axes (zigzag differences), components if bit 8
+ * components: varint count, per component: u8 id, varint (length + 1),
+ *         bytes; length field 0 means removed
+ * ```
+ *
+ * JavaScript numbers hold integers exactly up to 2^53. Entity ids and
+ * quantized positions must stay below that; a frame that exceeds it is
+ * refused rather than rounded.
+ */
+
+export const REPLICATION_VERSION = 1;
+
+const FLAG_FULL = 1;
+const MASK_X = 1;
+const MASK_Y = 2;
+const MASK_Z = 4;
+const MASK_COMPONENTS = 8;
+
+/** Longest id list a frame may declare. */
+const MAX_COUNT = 1 << 24;
+
+export class ReplicationError extends Error {
+  constructor(message: string) {
+    super(`replication frame: ${message}`);
+    this.name = "ReplicationError";
+  }
+}
+
+/** One entity as the client sees it. */
+export interface ReplicatedEntity {
+  readonly id: number;
+  /** Quantized position; `x/y/z` are these times the frame precision. */
+  qx: number;
+  qy: number;
+  qz: number;
+  x: number;
+  y: number;
+  z: number;
+  /** Component id to bytes, in the game's own encoding. */
+  components: Map<number, Uint8Array>;
+}
+
+/** What one frame did. */
+export interface ReplicationSummary {
+  full: boolean;
+  spawned: number[];
+  updated: number[];
+  despawned: number[];
+}
+
+class Reader {
+  private offset = 0;
+  private readonly view: DataView;
+
+  constructor(private readonly bytes: Uint8Array) {
+    this.view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+  }
+
+  get done(): boolean {
+    return this.offset >= this.bytes.length;
+  }
+
+  u8(): number {
+    if (this.offset >= this.bytes.length) throw new ReplicationError("ends early");
+    return this.bytes[this.offset++];
+  }
+
+  f32(): number {
+    if (this.offset + 4 > this.bytes.length) throw new ReplicationError("ends early");
+    const v = this.view.getFloat32(this.offset, true);
+    this.offset += 4;
+    return v;
+  }
+
+  /** An unsigned LEB128 varint, as a bigint (exact to 64 bits). */
+  varint(): bigint {
+    let v = 0n;
+    for (let i = 0; i < 10; i++) {
+      const b = this.u8();
+      const part = BigInt(b & 0x7f);
+      if (i === 9 && part > 1n) throw new ReplicationError("varint past 64 bits");
+      v |= part << BigInt(7 * i);
+      if ((b & 0x80) === 0) return v;
+    }
+    throw new ReplicationError("varint too long");
+  }
+
+  /** A zigzag signed varint, as a bigint. */
+  zigzag(): bigint {
+    const z = this.varint();
+    return (z >> 1n) ^ -(z & 1n);
+  }
+
+  take(n: number): Uint8Array {
+    if (this.offset + n > this.bytes.length) throw new ReplicationError("component runs past the end");
+    const out = this.bytes.slice(this.offset, this.offset + n);
+    this.offset += n;
+    return out;
+  }
+}
+
+function toSafe(v: bigint, what: string): number {
+  if (v > BigInt(Number.MAX_SAFE_INTEGER) || v < BigInt(Number.MIN_SAFE_INTEGER)) {
+    throw new ReplicationError(`${what} ${v} is beyond 2^53`);
+  }
+  return Number(v);
+}
+
+function count(r: Reader): number {
+  const n = r.varint();
+  if (n > BigInt(MAX_COUNT)) throw new ReplicationError("count too large");
+  return Number(n);
+}
+
+function nextId(r: Reader, last: number | null): number {
+  const v = r.varint();
+  if (last === null) return toSafe(v, "entity id");
+  if (v === 0n) throw new ReplicationError("ids do not ascend");
+  return toSafe(BigInt(last) + v, "entity id");
+}
+
+function readComponents(r: Reader, into: Map<number, Uint8Array>): void {
+  const n = r.varint();
+  if (n > 256n) throw new ReplicationError("more than 256 components");
+  for (let i = 0; i < Number(n); i++) {
+    const id = r.u8();
+    const len = r.varint();
+    if (len === 0n) {
+      into.delete(id);
+      continue;
+    }
+    if (len - 1n > BigInt(Number.MAX_SAFE_INTEGER)) throw new ReplicationError("component too large");
+    into.set(id, r.take(Number(len - 1n)));
+  }
+}
+
+/**
+ * The entities one subscriber has been told about. Apply every
+ * replication frame in order; a frame that fails to apply means the
+ * connection is out of sync and should be reopened (the server then sends
+ * a full frame).
+ */
+export class EntityTable {
+  readonly entities = new Map<number, ReplicatedEntity>();
+  /** World units per quantization step, from the last frame. */
+  precision = 0.01;
+
+  get size(): number {
+    return this.entities.size;
+  }
+
+  get(id: number): ReplicatedEntity | undefined {
+    return this.entities.get(id);
+  }
+
+  clear(): void {
+    this.entities.clear();
+  }
+
+  apply(frame: Uint8Array): ReplicationSummary {
+    const r = new Reader(frame);
+    const version = r.u8();
+    if (version !== REPLICATION_VERSION) throw new ReplicationError(`version ${version}`);
+    const full = (r.u8() & FLAG_FULL) !== 0;
+    const precision = r.f32();
+    if (!Number.isFinite(precision) || precision <= 0) throw new ReplicationError("bad precision");
+    if (full) this.entities.clear();
+    this.precision = precision;
+    const summary: ReplicationSummary = { full, spawned: [], updated: [], despawned: [] };
+
+    let n = count(r);
+    let last: number | null = null;
+    for (let i = 0; i < n; i++) {
+      last = nextId(r, last);
+      this.entities.delete(last);
+      summary.despawned.push(last);
+    }
+
+    n = count(r);
+    last = null;
+    for (let i = 0; i < n; i++) {
+      last = nextId(r, last);
+      const qx = toSafe(r.zigzag(), "position");
+      const qy = toSafe(r.zigzag(), "position");
+      const qz = toSafe(r.zigzag(), "position");
+      const components = new Map<number, Uint8Array>();
+      readComponents(r, components);
+      this.entities.set(last, {
+        id: last,
+        qx,
+        qy,
+        qz,
+        x: qx * precision,
+        y: qy * precision,
+        z: qz * precision,
+        components,
+      });
+      summary.spawned.push(last);
+    }
+
+    n = count(r);
+    last = null;
+    for (let i = 0; i < n; i++) {
+      last = nextId(r, last);
+      const mask = r.u8();
+      const e = this.entities.get(last);
+      if (!e) throw new ReplicationError(`update for unknown entity ${last}`);
+      if (mask & MASK_X) e.qx = toSafe(BigInt(e.qx) + r.zigzag(), "position");
+      if (mask & MASK_Y) e.qy = toSafe(BigInt(e.qy) + r.zigzag(), "position");
+      if (mask & MASK_Z) e.qz = toSafe(BigInt(e.qz) + r.zigzag(), "position");
+      if (mask & MASK_COMPONENTS) readComponents(r, e.components);
+      summary.updated.push(last);
+    }
+    if (!r.done) throw new ReplicationError("trailing bytes");
+
+    // Positions follow the frame's precision (a full frame may change it).
+    for (const e of this.entities.values()) {
+      e.x = e.qx * precision;
+      e.y = e.qy * precision;
+      e.z = e.qz * precision;
+    }
+    return summary;
+  }
+}

package/src/wire.test.ts

@@ -0,0 +1,72 @@
+import { describe, expect, test } from "bun:test";
+import { encode as msgpackEncode, decode as msgpackDecode } from "@msgpack/msgpack";
+
+import {
+  SHARD_HEADER_LEN,
+  ShardCodec,
+  ShardFrameKind,
+  decodeShardPayload,
+  decodeShardRejection,
+  encodeShardInput,
+  parseShardFrame,
+} from "./wire";
+
+function frame(kind: number, codec: number, tick: number, ack: number, payload: Uint8Array) {
+  const buf = new ArrayBuffer(SHARD_HEADER_LEN + payload.length);
+  const view = new DataView(buf);
+  view.setUint8(0, kind);
+  view.setUint8(1, codec);
+  view.setUint32(2, Math.floor(tick / 0x1_0000_0000));
+  view.setUint32(6, tick >>> 0);
+  view.setUint32(10, Math.floor(ack / 0x1_0000_0000));
+  view.setUint32(14, ack >>> 0);
+  new Uint8Array(buf, SHARD_HEADER_LEN).set(payload);
+  return buf;
+}
+
+describe("parseShardFrame", () => {
+  test("reads kind, codec, tick, ack, and payload", () => {
+    const f = parseShardFrame(
+      frame(ShardFrameKind.Snapshot, ShardCodec.Json, 2 ** 33 + 5, 7, new TextEncoder().encode("42")),
+    );
+    expect(f.kind).toBe(1);
+    expect(f.codec).toBe(0);
+    expect(f.tick).toBe(2 ** 33 + 5);
+    expect(f.ack).toBe(7);
+    expect(decodeShardPayload(f.codec, f.payload)).toBe(42);
+  });
+
+  test("rejects a frame shorter than the header", () => {
+    expect(() => parseShardFrame(new ArrayBuffer(10))).toThrow("too short");
+  });
+});
+
+test("MessagePack payloads decode to objects", () => {
+  const payload = msgpackEncode({ players: [{ id: "a", x: 1.5 }], tick: 3 });
+  const f = parseShardFrame(frame(ShardFrameKind.Snapshot, ShardCodec.MessagePack, 3, 0, payload));
+  expect(decodeShardPayload(f.codec, f.payload)).toEqual({ players: [{ id: "a", x: 1.5 }], tick: 3 });
+});
+
+test("rejection frames decode with camelCase fields", () => {
+  const payload = msgpackEncode({ client_seq: 9, code: "rate_limited", message: "slow down" });
+  const f = parseShardFrame(frame(ShardFrameKind.InputRejected, ShardCodec.MessagePack, 1, 8, payload));
+  expect(decodeShardRejection(f.codec, f.payload)).toEqual({
+    clientSeq: 9,
+    code: "rate_limited",
+    message: "slow down",
+  });
+});
+
+test("an unknown codec needs a custom decoder", () => {
+  const bytes = new Uint8Array([1, 2, 3]);
+  expect(() => decodeShardPayload(ShardCodec.Custom, bytes)).toThrow("custom decoder");
+  expect(decodeShardPayload(ShardCodec.Custom, bytes, (p) => p.length)).toBe(3);
+});
+
+test("inputs are MessagePack for MessagePack shards and JSON otherwise", () => {
+  const bin = encodeShardInput(ShardCodec.MessagePack, { move: "n" }, 4);
+  expect(bin).toBeInstanceOf(Uint8Array);
+  expect(msgpackDecode(bin as Uint8Array)).toEqual({ input: { move: "n" }, client_seq: 4 });
+  expect(encodeShardInput(null, 5, 1)).toBe('{"input":5,"client_seq":1}');
+  expect(encodeShardInput(ShardCodec.Json, 5, 2)).toBe('{"input":5,"client_seq":2}');
+});

package/src/wire.ts

@@ -0,0 +1,129 @@
+/**
+ * The shard wire protocol, version 2 (see `pylon_realtime::wire` in Rust).
+ *
+ * A client asks for it with `?v=2` on the shard WebSocket URL. Each server
+ * message is a binary frame with an 18-byte header:
+ *
+ *   0   1  frame kind: 1 snapshot, 2 input rejected, 3 entity replication
+ *   1   1  codec: 0 JSON, 1 MessagePack, 2 bincode, 3 custom, 4 replication
+ *   2   8  tick (u64 big-endian)
+ *   10  8  ack: highest client_seq the shard processed for this subscriber (0 = none)
+ *   18  .. payload in the codec
+ *
+ * Inputs go up as `{ input, client_seq }`: JSON in a text frame, or the
+ * shard's codec in a binary frame.
+ */
+
+import { decode as msgpackDecode, encode as msgpackEncode } from "@msgpack/msgpack";
+
+export const SHARD_PROTOCOL_VERSION = 2;
+export const SHARD_HEADER_LEN = 18;
+
+export const ShardFrameKind = {
+  Snapshot: 1,
+  InputRejected: 2,
+  /** An entity replication frame: apply it to an `EntityTable`. */
+  Replication: 3,
+} as const;
+
+export const ShardCodec = {
+  Json: 0,
+  MessagePack: 1,
+  Bincode: 2,
+  Custom: 3,
+  /** The replication frame format (frame kind 3). */
+  Replication: 4,
+} as const;
+
+export interface ShardFrame {
+  kind: number;
+  codec: number;
+  tick: number;
+  ack: number;
+  payload: Uint8Array;
+}
+
+/** Why an input did not take effect. */
+export interface ShardInputRejection {
+  clientSeq: number | null;
+  /** `unauthorized`, `rate_limited`, `queue_full`, `invalid`, `stopped`, or `apply_failed`. */
+  code: string;
+  message: string;
+}
+
+/**
+ * Decode a payload in a codec the JS client does not know (bincode, or a
+ * game's own codec `3`).
+ */
+export type ShardPayloadDecoder = (payload: Uint8Array, codec: number) => unknown;
+
+function readU64(view: DataView, offset: number): number {
+  return view.getUint32(offset) * 0x1_0000_0000 + view.getUint32(offset + 4);
+}
+
+/** Split a version 2 frame into its header fields and payload. */
+export function parseShardFrame(data: ArrayBuffer): ShardFrame {
+  if (data.byteLength < SHARD_HEADER_LEN) {
+    throw new Error(`shard frame too short (${data.byteLength} bytes)`);
+  }
+  const view = new DataView(data);
+  return {
+    kind: view.getUint8(0),
+    codec: view.getUint8(1),
+    tick: readU64(view, 2),
+    ack: readU64(view, 10),
+    payload: new Uint8Array(data, SHARD_HEADER_LEN),
+  };
+}
+
+/** Decode a payload. JSON and MessagePack are built in. */
+export function decodeShardPayload(
+  codec: number,
+  payload: Uint8Array,
+  custom?: ShardPayloadDecoder,
+): unknown {
+  switch (codec) {
+    case ShardCodec.Json:
+      return JSON.parse(new TextDecoder().decode(payload));
+    case ShardCodec.MessagePack:
+      return msgpackDecode(payload);
+    default:
+      if (custom) return custom(payload, codec);
+      throw new Error(
+        `shard codec ${codec} needs a custom decoder (pass \`decode\` to connectShard)`,
+      );
+  }
+}
+
+/** The payload of an input-rejected frame, with camelCase fields. */
+export function decodeShardRejection(
+  codec: number,
+  payload: Uint8Array,
+  custom?: ShardPayloadDecoder,
+): ShardInputRejection {
+  const raw = decodeShardPayload(codec, payload, custom) as {
+    client_seq?: number | null;
+    code?: string;
+    message?: string;
+  };
+  return {
+    clientSeq: typeof raw.client_seq === "number" ? raw.client_seq : null,
+    code: String(raw.code ?? "invalid"),
+    message: String(raw.message ?? ""),
+  };
+}
+
+/**
+ * Encode an input envelope. MessagePack shards get a binary frame; every
+ * other codec (and a connection that has not seen a frame yet) gets JSON
+ * text, which the server always accepts.
+ */
+export function encodeShardInput(
+  codec: number | null,
+  input: unknown,
+  clientSeq: number,
+): string | Uint8Array {
+  const envelope = { input, client_seq: clientSeq };
+  if (codec === ShardCodec.MessagePack) return msgpackEncode(envelope);
+  return JSON.stringify(envelope);
+}