@colyseus/schema 4.0.19 → 5.0.0

package/src/encoder/streaming.ts

@@ -0,0 +1,232 @@
+/**
+ * Shared routing helpers for streamable collections (`StreamSchema`,
+ * `MapSchema.stream()`, etc.).
+ *
+ * Each streamable class carries exactly one lazy slot (`_stream`) that
+ * holds the 6 per-view / broadcast bookkeeping structures. Keeping the
+ * slot undefined until streaming actually activates means non-streaming
+ * `MapSchema` / `SetSchema` instances pay zero Map/Set allocations. One
+ * declared slot → hidden-class shape stays stable across streaming and
+ * non-streaming instances, so V8's ICs on `$items` / `journal` / etc.
+ * stay monomorphic.
+ *
+ * Lives alongside `changeTree/*.ts` — another directory of module-level
+ * free functions that operate on ChangeTree instances.
+ */
+import { OPERATION } from "../encoding/spec.js";
+import type { Root, Streamable } from "./Root.js";
+
+/**
+ * Thrown (from both the `FieldBuilder` chainable and the decorator's
+ * `addField` auto-flag) when a user attempts to stream an ArraySchema.
+ * Centralized so the two callsites emit the same diagnostic.
+ */
+export const ARRAY_STREAM_NOT_SUPPORTED =
+    "ArraySchema does not support streaming — positional ops " +
+    "(splice / unshift / reverse) shift subsequent indexes, so holding " +
+    "ADDs back for a later tick under `maxPerTick` would desync the " +
+    "decoder. Use `t.stream(X)` (stable monotonic positions) or " +
+    "`t.map(X).stream()` (stable keys) instead.";
+
+/**
+ * Per-instance bookkeeping for a streamable collection. Lazily allocated
+ * by `ensureStreamState` when the collection's ChangeTree picks up the
+ * `isStreamCollection` flag (or when the user touches `maxPerTick`).
+ */
+export interface StreamableState {
+    /** Per-view ADD backlog: wire-indexes not yet sent to that view. */
+    pendingByView: Map<number, Set<number>>;
+    /** Per-view SENT set — decides whether `remove()` emits a DELETE. */
+    sentByView: Map<number, Set<number>>;
+    /** Broadcast-mode ADD backlog (no active views). */
+    broadcastPending: Set<number>;
+    /** Broadcast-mode SENT set. */
+    sentBroadcast: Set<number>;
+    /** Broadcast-mode DELETE queue — flushes next shared tick. */
+    broadcastDeletes: Set<number>;
+    /** Max ADD ops emitted per tick per view (or per shared tick). */
+    maxPerTick: number;
+    /**
+     * Priority callback seeded from the schema declaration. Receives the
+     * client's StateView and the candidate element; higher return values
+     * emit first. Broadcast `encode()` ignores this and drains FIFO.
+     * Instance-level override: assign to `stream.priority`.
+     */
+    priority?: (view: any, element: any) => number;
+}
+
+export function createStreamableState(): StreamableState {
+    return {
+        pendingByView: new Map(),
+        sentByView: new Map(),
+        broadcastPending: new Set(),
+        sentBroadcast: new Set(),
+        broadcastDeletes: new Set(),
+        maxPerTick: 32,
+    };
+}
+
+/** Allocate `_stream` on first use (idempotent). Returns the state. */
+export function ensureStreamState(s: Streamable): StreamableState {
+    return (s._stream ??= createStreamableState());
+}
+
+/**
+ * Route an ADD into the pending backlogs.
+ * - No active views: push into broadcast pending (shared encode drains up
+ *   to `maxPerTick` per tick).
+ * - With views: push into per-view pending for every currently-bound view.
+ */
+export function streamRouteAdd(s: Streamable, root: Root, index: number): void {
+    // Broadcast mode (no views registered): seed broadcast pending so
+    // the shared `encode()` pass drains it up to `maxPerTick` per tick.
+    // View mode: do nothing — users must call `view.add(element)` per
+    // entity to subscribe it for that view. This matches the StateView
+    // design philosophy: per-client visibility is imperative, not
+    // declarative. An encode-time predicate would be O(views × entities)
+    // each tick — the whole reason StateView exists is to push that
+    // bookkeeping to game-loop cadence.
+    if (root.activeViews.size === 0) {
+        ensureStreamState(s).broadcastPending.add(index);
+    }
+}
+
+/**
+ * Route a REMOVE: silent-drop if never sent, force DELETE if already sent.
+ * Returns `true` iff no wire op reached any channel (caller can skip
+ * follow-on work like snapshotting the deleted value).
+ */
+export function streamRouteRemove(
+    s: Streamable,
+    root: Root,
+    refId: number,
+    index: number,
+): boolean {
+    // If `_stream` is still undefined, streaming never saw any add/remove —
+    // nothing to unwind, and nothing was ever emitted.
+    const st = s._stream;
+    if (st === undefined) return true;
+
+    let neverSent = false;
+
+    // Broadcast side.
+    if (st.broadcastPending.delete(index)) {
+        neverSent = true;
+    } else if (st.sentBroadcast.delete(index)) {
+        st.broadcastDeletes.add(index);
+    }
+
+    // Per-view side.
+    root.forEachActiveView((view) => {
+        const pending = st.pendingByView.get(view.id);
+        if (pending?.has(index)) {
+            pending.delete(index);
+            neverSent = true;
+            return;
+        }
+        const sent = st.sentByView.get(view.id);
+        if (sent?.has(index)) {
+            sent.delete(index);
+            let changes = view.changes.get(refId);
+            if (changes === undefined) {
+                changes = new Map();
+                view.changes.set(refId, changes);
+            }
+            changes.set(index, OPERATION.DELETE);
+        }
+    });
+
+    return neverSent;
+}
+
+/**
+ * Queue DELETE ops for every already-sent entry on all channels and
+ * reset pending. Caller is responsible for actually clearing its own
+ * storage and releasing any element refs it owns.
+ */
+export function streamRouteClear(s: Streamable, root: Root, refId: number): void {
+    const st = s._stream;
+    if (st === undefined) return;
+
+    // Broadcast: drop never-sent pending; force DELETE for sent entries.
+    st.broadcastPending.clear();
+    for (const index of st.sentBroadcast) st.broadcastDeletes.add(index);
+    st.sentBroadcast.clear();
+
+    // Per-view: clear pending; force DELETE for sent entries via
+    // `view.changes` (drained first in encodeView).
+    root.forEachActiveView((view) => {
+        st.pendingByView.get(view.id)?.clear();
+
+        const sent = st.sentByView.get(view.id);
+        if (sent !== undefined && sent.size > 0) {
+            let changes = view.changes.get(refId);
+            if (changes === undefined) {
+                changes = new Map();
+                view.changes.set(refId, changes);
+            }
+            for (const index of sent) changes.set(index, OPERATION.DELETE);
+            sent.clear();
+        }
+    });
+}
+
+/**
+ * Push a single position into `_pendingByView[viewId]` — the building
+ * block for `StateView.add(element)` when the element lives under a
+ * streamable collection. Idempotent for already-pending positions.
+ */
+export function streamEnqueueForView(s: Streamable, viewId: number, index: number): void {
+    const st = ensureStreamState(s);
+    let pending = st.pendingByView.get(viewId);
+    if (pending === undefined) {
+        pending = new Set();
+        st.pendingByView.set(viewId, pending);
+    }
+    pending.add(index);
+}
+
+/**
+ * Unsubscribe a single position from a view. Returns true iff the
+ * element had already been sent and a DELETE op was queued on
+ * `view.changes`; false if it was only pending (silent drop) or not
+ * present at all.
+ */
+export function streamDequeueForView(
+    s: Streamable,
+    viewId: number,
+    refId: number,
+    index: number,
+    viewChanges: Map<number, Map<number, number>>,
+): boolean {
+    const st = s._stream;
+    if (st === undefined) return false;
+    const pending = st.pendingByView.get(viewId);
+    if (pending?.has(index)) {
+        pending.delete(index);
+        return false;
+    }
+    const sent = st.sentByView.get(viewId);
+    if (sent?.has(index)) {
+        sent.delete(index);
+        let changes = viewChanges.get(refId);
+        if (changes === undefined) {
+            changes = new Map();
+            viewChanges.set(refId, changes);
+        }
+        changes.set(index, OPERATION.DELETE);
+        return true;
+    }
+    return false;
+}
+
+/**
+ * Drop all per-view state for a disposing/GC'd StateView. Keeps memory
+ * bounded in long-running rooms with client churn.
+ */
+export function streamDropView(s: Streamable, viewId: number): void {
+    const st = s._stream;
+    if (st === undefined) return;
+    st.pendingByView.delete(viewId);
+    st.sentByView.delete(viewId);
+}

package/src/encoder/subscriptions.ts

@@ -0,0 +1,71 @@
+/**
+ * Per-view collection subscriptions — `view.subscribe(collection)` opts
+ * a view into ALL future content changes of a collection, not just a
+ * one-shot snapshot. Covers every collection type:
+ *
+ * - `ArraySchema` / `MapSchema` / `SetSchema` / `CollectionSchema`: new
+ *   children are force-shipped immediately via `view._addImmediate(child)`.
+ *   Subsequent field mutations on those children emit via the normal
+ *   view pass (the children are now visible).
+ * - `StreamSchema` (or a `.stream()` map/set): new positions are
+ *   enqueued into `_pendingByView` so the encoder's priority pass
+ *   drains them respecting `maxPerTick`.
+ *
+ * The propagation hook is in `changeTree/treeAttachment.ts setParent`
+ * — every new child attachment to a collection checks the parent tree's
+ * `subscribedViews` bitmap and fans out to subscribed views.
+ */
+import type { ChangeTree, Ref } from "./ChangeTree.js";
+import type { Root, Streamable } from "./Root.js";
+import { streamEnqueueForView } from "./streaming.js";
+import { $changes } from "../types/symbols.js";
+
+/**
+ * Walk the `subscribedViews` bitmap of `parentTree` and propagate a new
+ * child attachment to every subscribed view. Streams route through the
+ * priority/pending queue; all other collections force-ship immediately.
+ */
+export function propagateNewChildToSubscribers(
+    parentTree: ChangeTree,
+    childIndex: number,
+    childRef: Ref,
+    root: Root,
+): void {
+    const subs = parentTree.subscribedViews;
+    if (subs === undefined) return;
+
+    const isStream = parentTree.isStreamCollection;
+    const streamable = isStream ? (parentTree.ref as unknown as Streamable) : undefined;
+    const childTree = isStream ? undefined : childRef[$changes];
+
+    // Walk set bits via clz32 — same pattern as the inline recorder
+    // iteration elsewhere in the encoder.
+    for (let slot = 0, n = subs.length; slot < n; slot++) {
+        let bits = subs[slot];
+        while (bits !== 0) {
+            const bit = bits & -bits;
+            bits ^= bit;
+            const viewId = slot * 32 + (31 - Math.clz32(bit));
+            const weakRef = root.activeViews.get(viewId);
+            const view = weakRef?.deref();
+            if (view === undefined) {
+                // View was disposed / GC'd; clear the stale subscription bit.
+                subs[slot] &= ~bit;
+                continue;
+            }
+            if (isStream) {
+                // Streams bypass the recorder — enqueue for the priority
+                // pass to drain under `maxPerTick`.
+                streamEnqueueForView(streamable!, viewId, childIndex);
+            } else if (childTree !== undefined) {
+                // Non-stream collections: just markVisible. The parent's
+                // recorder already carries the ADD op (triggered by the
+                // push/set/add that led to this setParent), and the
+                // child's tree carries its construction-time dirty state
+                // — the encoder's normal view pass picks both up on the
+                // next encode, no view.changes seeding needed.
+                view.markVisible(childTree);
+            }
+        }
+    }
+}

package/src/index.ts

@@ -14,6 +14,9 @@ export { CollectionSchema };
 import { SetSchema } from "./types/custom/SetSchema.js";
 export { SetSchema };
 
+import { StreamSchema } from "./types/custom/StreamSchema.js";
+export { StreamSchema };
+
 import { registerType, defineCustomTypes } from "./types/registry.js";
 export { registerType, defineCustomTypes };
 
@@ -21,6 +24,8 @@ registerType("map", { constructor: MapSchema });
 registerType("array", { constructor: ArraySchema });
 registerType("set", { constructor: SetSchema });
 registerType("collection", { constructor: CollectionSchema, });
+// "stream" is registered inside StreamSchema.ts (same pattern as others
+// that co-locate registerType with the class for side-effect safety).
 
 // Utils
 export { dumpChanges } from "./utils.js";
@@ -44,19 +49,25 @@ export { Metadata } from "./Metadata.js";
 export {
     type,
     deprecated,
-    defineTypes,
+    owned,
+    unreliable,
+    transient,
     view,
     schema,
     entity,
     type DefinitionType,
     type PrimitiveType,
     type Definition,
+    type FieldsAndMethods,
     // Raw schema() return types
     type SchemaWithExtendsConstructor,
     type SchemaWithExtends,
     type SchemaType,
 } from "./annotations.js";
 
+// zod-style chainable builders
+export { t, FieldBuilder, isBuilder, type BuilderDefinition, type ChildType } from "./types/builder.js";
+
 export { TypeContext } from "./types/TypeContext.js";
 
 // Helper types for type inference
@@ -67,8 +78,9 @@ export { Callbacks, StateCallbackStrategy } from "./decoder/strategy/Callbacks.j
 export { getRawChangesCallback } from "./decoder/strategy/RawChanges.js";
 
 export { Encoder } from "./encoder/Encoder.js";
-export { encodeSchemaOperation, encodeArray, encodeKeyValueOperation } from "./encoder/EncodeOperation.js";
-export { ChangeTree, type Ref, type IRef, type ChangeSetName, type ChangeSet} from "./encoder/ChangeTree.js";
+export { Root } from "./encoder/Root.js";
+export { encodeSchemaOperation, encodeArray, encodeKeyValueOperation, encodeMapEntry, encodeIndexedEntry } from "./encoder/EncodeOperation.js";
+export { ChangeTree, type Ref, type IRef } from "./encoder/ChangeTree.js";
 export { StateView } from "./encoder/StateView.js";
 
 export { Decoder } from "./decoder/Decoder.js";

package/src/input/InputDecoder.ts

@@ -0,0 +1,57 @@
+import { Decoder } from "../decoder/Decoder.js";
+import { decode, type Iterator } from "../encoding/decode.js";
+import type { Schema } from "../Schema.js";
+
+/**
+ * Bound single-struct decoder for input packets. Wraps the standard
+ * `Decoder` so bytes emitted by `InputEncoder` land on the bound instance.
+ *
+ * - `decode(bytes)`: single-input packet (reliable mode).
+ * - `decodeAll(bytes, cb)`: multi-input length-framed packet (unreliable
+ *   mode). Invokes `cb` with the mutated instance once per framed input,
+ *   oldest → newest. The instance is re-used across callbacks — consume
+ *   synchronously (apply to game state) rather than holding the reference.
+ */
+export class InputDecoder<T extends Schema = any> {
+    readonly instance: T;
+    private readonly _decoder: Decoder<T>;
+    private readonly _it: Iterator = { offset: 0 };
+
+    constructor(instance: T) {
+        this.instance = instance;
+        this._decoder = new Decoder(instance);
+    }
+
+    /**
+     * Decode a single-input (reliable) packet into the bound instance.
+     * Returns the instance for chaining.
+     */
+    decode(bytes: Uint8Array): T {
+        this._decoder.decode(bytes);
+        return this.instance;
+    }
+
+    /**
+     * Walk a multi-input (unreliable) packet, decoding each length-framed
+     * input into the bound instance in order and invoking `onInput` after
+     * each decode. `onInput` receives the bound instance itself — reads
+     * must be synchronous; downstream code should apply the input to game
+     * state, not retain the reference.
+     *
+     * Returns the number of inputs decoded.
+     */
+    decodeAll(bytes: Uint8Array, onInput: (instance: T, index: number) => void): number {
+        const it = this._it;
+        it.offset = 0;
+        let count = 0;
+        while (it.offset < bytes.length) {
+            const len = decode.number(bytes, it);
+            const end = it.offset + len;
+            this._decoder.decode(bytes.subarray(it.offset, end));
+            onInput(this.instance, count);
+            it.offset = end;
+            count++;
+        }
+        return count;
+    }
+}