@colyseus/schema 4.0.20 → 5.0.1

package/src/decoder/Decoder.ts

@@ -1,10 +1,9 @@
 import { TypeContext } from "../types/TypeContext.js";
-import { $changes, $childType, $decoder, $onDecodeEnd, $refId } from "../types/symbols.js";
+import { $childType, $decoder, $onDecodeEnd, $refId } from "../types/symbols.js";
 import { Schema } from "../Schema.js";
-
 import { decode } from "../encoding/decode.js";
 import { OPERATION, SWITCH_TO_STRUCTURE, TYPE_ID } from '../encoding/spec.js';
-import type { IRef, Ref } from "../encoder/ChangeTree.js";
+import { type IRef, type Ref } from "../encoder/ChangeTree.js";
 import type { Iterator } from "../encoding/decode.js";
 import { ReferenceTracker } from "./ReferenceTracker.js";
 import { DEFINITION_MISMATCH, type DataChange, type DecodeOperation } from "./DecodeOperation.js";
@@ -42,7 +41,13 @@ export class Decoder<T extends IRef = any> {
         it: Iterator = { offset: 0 },
         ref: IRef = this.state,
     ) {
-        const allChanges: DataChange[] = [];
+        // Only allocate a collection array when there's a subscriber. Every
+        // decode-op push site uses `allChanges?.push(...)` — optional
+        // chaining short-circuits the object literal too, so a listener-
+        // free decoder does zero per-field allocation.
+        const allChanges: DataChange[] | null = (this.triggerChanges !== undefined)
+            ? []
+            : null;
 
         const $root = this.root;
         const totalBytes = bytes.byteLength;
@@ -90,11 +95,16 @@ export class Decoder<T extends IRef = any> {
             }
         }
 
-        // FIXME: DRY with SWITCH_TO_STRUCTURE block.
+        // Close out the last ref's decode session — mirrors the
+        // SWITCH_TO_STRUCTURE block above, which fires it at every
+        // intra-loop structure transition. ArraySchema uses this to
+        // compact `items` after a tick's deletes. No other consumer
+        // currently hooks it; the dual call sites stay as-is rather
+        // than being extracted into a helper for a one-line body.
         (ref as any)[$onDecodeEnd]?.()
 
         // trigger changes
-        this.triggerChanges?.(allChanges);
+        if (allChanges !== null) this.triggerChanges?.(allChanges);
 
         // drop references of unused schemas
         $root.garbageCollectDeletedRefs();
@@ -131,16 +141,16 @@ export class Decoder<T extends IRef = any> {
         return type || defaultType;
     }
 
-    createInstanceOfType (type: typeof Schema): Schema {
-        return new (type as any)();
+    createInstanceOfType(type: typeof Schema): Schema {
+        return type.initializeForDecoder();
     }
 
-    removeChildRefs(ref: Collection, allChanges: DataChange[]) {
+    removeChildRefs(ref: Collection, allChanges: DataChange[] | null) {
         const needRemoveRef = typeof ((ref as any)[$childType]) !== "string";
         const refId = (ref as Ref)[$refId];
 
         ref.forEach((value: any, key: any) => {
-            allChanges.push({
+            allChanges?.push({
                 ref: ref as Ref,
                 refId,
                 op: OPERATION.DELETE,

package/src/decoder/ReferenceTracker.ts

@@ -41,6 +41,10 @@ export class ReferenceTracker {
     addRef(refId: number, ref: IRef, incrementCount: boolean = true) {
         this.refs.set(refId, ref);
 
+        // `enumerable: false` is load-bearing: tests use `deepStrictEqual`
+        // on decoded instances, which WOULD walk enumerable Symbol-keyed
+        // properties and include `$refId` in the comparison. Keep the
+        // descriptor dance for semantic compatibility.
         Object.defineProperty(ref, $refId, {
             value: refId,
             enumerable: false,

package/src/decoder/strategy/Callbacks.ts

@@ -129,7 +129,7 @@ export class StateCallbackStrategy<TState extends IRef> {
     /**
      * Listen to property changes on a nested instance.
      */
-    listen<TInstance, K extends PublicPropNames<TInstance>>(
+    listen<TInstance extends Schema, K extends PublicPropNames<TInstance>>(
         instance: TInstance,
         property: K,
         handler: PropertyChangeCallback<TInstance[K]>,
@@ -333,6 +333,13 @@ export class StateCallbackStrategy<TState extends IRef> {
     protected triggerChanges(allChanges: DataChange[]): void {
         this.uniqueRefIds.clear();
 
+        // Flag stays set for the whole dispatch pass — any `listen()` /
+        // `onAdd(...)` registered while a callback is firing needs to
+        // suppress its immediate-trigger. One toggle per pass is enough;
+        // every callback invocation below is wrapped in try/catch so
+        // nothing bubbles out to leave the flag stuck.
+        this.isTriggering = true;
+
         for (let i = 0, l = allChanges.length; i < l; i++) {
             const change = allChanges[i];
             const refId = change.refId;
@@ -354,7 +361,7 @@ export class StateCallbackStrategy<TState extends IRef> {
                 const deleteCallbacks = this.callbacks[childRefId]?.[OPERATION.DELETE];
                 if (deleteCallbacks) {
                     for (let j = deleteCallbacks.length - 1; j >= 0; j--) {
-                        deleteCallbacks[j]();
+                        try { deleteCallbacks[j](); } catch (e) { console.error(e); }
                     }
                 }
             }
@@ -369,11 +376,7 @@ export class StateCallbackStrategy<TState extends IRef> {
                     const replaceCallbacks = $callbacks[OPERATION.REPLACE];
                     if (replaceCallbacks) {
                         for (let j = replaceCallbacks.length - 1; j >= 0; j--) {
-                            try {
-                                replaceCallbacks[j]();
-                            } catch (e) {
-                                console.error(e);
-                            }
+                            try { replaceCallbacks[j](); } catch (e) { console.error(e); }
                         }
                     }
                 }
@@ -382,14 +385,7 @@ export class StateCallbackStrategy<TState extends IRef> {
                 const fieldCallbacks = $callbacks[change.field];
                 if (fieldCallbacks) {
                     for (let j = fieldCallbacks.length - 1; j >= 0; j--) {
-                        try {
-                            this.isTriggering = true;
-                            fieldCallbacks[j](change.value, change.previousValue);
-                        } catch (e) {
-                            console.error(e);
-                        } finally {
-                            this.isTriggering = false;
-                        }
+                        try { fieldCallbacks[j](change.value, change.previousValue); } catch (e) { console.error(e); }
                     }
                 }
 
@@ -400,15 +396,25 @@ export class StateCallbackStrategy<TState extends IRef> {
                 const dynamicIndex = change.dynamicIndex ?? change.field;
 
                 if ((change.op & OPERATION.DELETE) === OPERATION.DELETE) {
-                    //
-                    // FIXME: `previousValue` should always be available.
-                    //
+                    // DELETE can arrive with `previousValue === undefined` in two
+                    // legitimate cases — neither is fixable decoder-side:
+                    //   1. DELETE_AND_ADD (op byte 192) for an index/refId the
+                    //      decoder never held — e.g. a client whose @view
+                    //      subscription just began sees the replacement op
+                    //      first. `previousValue` is undefined; `value` is the
+                    //      newly-decoded instance. The ADD half still fires below.
+                    //   2. DELETE_BY_REFID (decodeArray, DecodeOperation.ts) for
+                    //      a filtered ArraySchema ref that was never ADDed on
+                    //      this decoder. That push site is unconditional because
+                    //      `removeRef` has already run; raw-change observers
+                    //      still want the event even with unknown previousValue.
+                    // The guard prevents `onRemove(undefined, key)` from firing.
                     if (change.previousValue !== undefined) {
                         // trigger onRemove (value, key)
                         const deleteCallbacks = $callbacks[OPERATION.DELETE];
                         if (deleteCallbacks) {
                             for (let j = deleteCallbacks.length - 1; j >= 0; j--) {
-                                deleteCallbacks[j](change.previousValue, dynamicIndex);
+                                try { deleteCallbacks[j](change.previousValue, dynamicIndex); } catch (e) { console.error(e); }
                             }
                         }
                     }
@@ -417,11 +423,9 @@ export class StateCallbackStrategy<TState extends IRef> {
                     if ((change.op & OPERATION.ADD) === OPERATION.ADD) {
                         const addCallbacks = $callbacks[OPERATION.ADD];
                         if (addCallbacks) {
-                            this.isTriggering = true;
                             for (let j = addCallbacks.length - 1; j >= 0; j--) {
-                                addCallbacks[j](change.value, dynamicIndex);
+                                try { addCallbacks[j](change.value, dynamicIndex); } catch (e) { console.error(e); }
                             }
-                            this.isTriggering = false;
                         }
                     }
 
@@ -432,11 +436,9 @@ export class StateCallbackStrategy<TState extends IRef> {
                     // trigger onAdd (value, key)
                     const addCallbacks = $callbacks[OPERATION.ADD];
                     if (addCallbacks) {
-                        this.isTriggering = true;
                         for (let j = addCallbacks.length - 1; j >= 0; j--) {
-                            addCallbacks[j](change.value, dynamicIndex);
+                            try { addCallbacks[j](change.value, dynamicIndex); } catch (e) { console.error(e); }
                         }
-                        this.isTriggering = false;
                     }
                 }
 
@@ -445,7 +447,7 @@ export class StateCallbackStrategy<TState extends IRef> {
                     const replaceCallbacks = $callbacks[OPERATION.REPLACE];
                     if (replaceCallbacks) {
                         for (let j = replaceCallbacks.length - 1; j >= 0; j--) {
-                            replaceCallbacks[j](dynamicIndex, change.value);
+                            try { replaceCallbacks[j](dynamicIndex, change.value); } catch (e) { console.error(e); }
                         }
                     }
                 }
@@ -453,6 +455,8 @@ export class StateCallbackStrategy<TState extends IRef> {
 
             this.uniqueRefIds.add(refId);
         }
+
+        this.isTriggering = false;
     }
 }
 

package/src/decoder/strategy/getDecoderStateCallbacks.ts

@@ -155,10 +155,6 @@ export function getDecoderStateCallbacks<T extends Schema>(decoder: Decoder<T>):
                     const replaceCallbacks = $callbacks?.[OPERATION.REPLACE];
                     for (let i = replaceCallbacks?.length - 1; i >= 0; i--) {
                         replaceCallbacks[i]();
-                        // try {
-                        // } catch (e) {
-                        //     console.error(e);
-                        // }
                     }
                 }
 
@@ -166,10 +162,6 @@ export function getDecoderStateCallbacks<T extends Schema>(decoder: Decoder<T>):
                     const fieldCallbacks = $callbacks[change.field];
                     for (let i = fieldCallbacks?.length - 1; i >= 0; i--) {
                         fieldCallbacks[i](change.value, change.previousValue);
-                        // try {
-                        // } catch (e) {
-                        //     console.error(e);
-                        // }
                     }
                 }
 
@@ -180,9 +172,16 @@ export function getDecoderStateCallbacks<T extends Schema>(decoder: Decoder<T>):
                 //
 
                 if ((change.op & OPERATION.DELETE) === OPERATION.DELETE) {
-                    //
-                    // FIXME: `previousValue` should always be available.
-                    //
+                    // DELETE can arrive with `previousValue === undefined` in
+                    // two legitimate cases — neither is fixable decoder-side:
+                    //   1. DELETE_AND_ADD (op byte 192) for an index/refId the
+                    //      decoder never held (e.g. a client whose @view
+                    //      subscription just began sees the replacement op
+                    //      first). The ADD half still fires below.
+                    //   2. DELETE_BY_REFID (decodeArray) for a filtered
+                    //      ArraySchema ref that was never ADDed on this
+                    //      decoder; that push site is unconditional.
+                    // The guard prevents `onRemove(undefined, key)` from firing.
                     if (change.previousValue !== undefined) {
                         // triger onRemove
                         const deleteCallbacks = $callbacks[OPERATION.DELETE];
@@ -211,10 +210,14 @@ export function getDecoderStateCallbacks<T extends Schema>(decoder: Decoder<T>):
                 }
 
                 // trigger onChange
+                // The `value !== undefined || previousValue !== undefined` half
+                // suppresses spurious onChange calls for DELETEs the decoder
+                // never had context for — same two cases documented above
+                // (DELETE_AND_ADD post-view-grant, DELETE_BY_REFID for unseen
+                // refIds), plus the historical "ADD+DELETE collapsed to DELETE
+                // on the wire" path now neutralized at the push site.
                 if (
                     change.value !== change.previousValue &&
-                    // FIXME: see "should not encode item if added and removed at the same patch" test case.
-                    // some "ADD" + "DELETE" operations on same patch are being encoded as "DELETE"
                     (change.value !== undefined || change.previousValue !== undefined)
                 ) {
                     const replaceCallbacks = $callbacks[OPERATION.REPLACE];

package/src/encoder/ChangeRecorder.ts

@@ -0,0 +1,276 @@
+import { OPERATION } from "../encoding/spec.js";
+
+/**
+ * ChangeRecorder — "what changed this tick" for a single ref.
+ *
+ * This file holds the two standalone recorder classes used for the
+ * unreliable channel (lazy, opt-in). The reliable channel is inlined on
+ * `ChangeTree` for perf; see `ChangeTree._isSchema` dispatch.
+ *
+ * Interface design (ISP):
+ *   - {@link ChangeRecorder}: common ops, implemented by both Schema and
+ *     Collection recorders.
+ *   - {@link ICollectionChangeRecorder}: extends with `recordPure` +
+ *     `shift` — collection-only. Schema recorders do NOT carry these.
+ *
+ * Per-field filter/visibility is decided at encode time, not record time.
+ * Full-sync output is derived structurally via `ChangeTree.forEachLive`.
+ */
+export interface ChangeRecorder {
+    /**
+     * Record a change at the given index. Handles op merge
+     * (DELETE followed by ADD becomes DELETE_AND_ADD).
+     */
+    record(index: number, op: OPERATION): void;
+
+    /** Record a DELETE at the given index. */
+    recordDelete(index: number, op: OPERATION): void;
+
+    /**
+     * Record an operation without op-merge semantics. Used by ArraySchema
+     * positional writes where DELETE→ADD merge is undesirable.
+     */
+    recordRaw(index: number, op: OPERATION): void;
+
+    /** Current operation at index, or undefined if none. */
+    operationAt(index: number): OPERATION | undefined;
+
+    /** Overwrite the operation at index. */
+    setOperationAt(index: number, op: OPERATION): void;
+
+    /**
+     * Iterate (index, op) pairs in record order.
+     * Pure operations emit with index = -op (wire convention).
+     */
+    forEach(cb: (index: number, op: OPERATION) => void): void;
+
+    /** Closure-free forEach variant for the hot encode path. */
+    forEachWithCtx<T>(ctx: T, cb: (ctx: T, index: number, op: OPERATION) => void): void;
+
+    size(): number;
+    has(): boolean;
+    reset(): void;
+}
+
+/**
+ * Extended recorder for collection types — adds `recordPure` (CLEAR /
+ * REVERSE) and `shift` (ArraySchema.unshift support).
+ */
+export interface ICollectionChangeRecorder extends ChangeRecorder {
+    /**
+     * Record a pure operation (CLEAR / REVERSE) with no index.
+     * Interleaves with indexed ops at record order.
+     */
+    recordPure(op: OPERATION): void;
+
+    /** Shift current-tick dirty indexes by `shiftIndex`. */
+    shift(shiftIndex: number): void;
+}
+
+// Module-scope adapter: lets `forEach(cb)` delegate to `forEachWithCtx`
+// by passing the user's callback as ctx. No per-call allocation.
+const _invokeNoCtx = (
+    cb: (index: number, op: OPERATION) => void,
+    index: number,
+    op: OPERATION,
+) => cb(index, op);
+
+// ──────────────────────────────────────────────────────────────────────────
+// SchemaChangeRecorder — bitmask + Uint8Array, for Schema types (≤64 fields)
+// ──────────────────────────────────────────────────────────────────────────
+
+/**
+ * Schema field operations are limited to ADD(128), DELETE(64), and
+ * DELETE_AND_ADD(192). REPLACE(0) is collection-only, so `ops[i] === 0`
+ * is a safe "no operation" sentinel.
+ */
+export class SchemaChangeRecorder implements ChangeRecorder {
+    // Bitmask storage for fields 0-31 (low) and 32-63 (high).
+    private dirtyLow = 0;
+    private dirtyHigh = 0;
+
+    // ops[fieldIndex] = OPERATION value. Pre-sized to numFields+1.
+    private readonly ops: Uint8Array;
+
+    constructor(numFields: number) {
+        this.ops = new Uint8Array(Math.max(numFields + 1, 1));
+    }
+
+    record(index: number, op: OPERATION): void {
+        const prev = this.ops[index];
+        if (prev === 0) this.ops[index] = op;
+        else if (prev === OPERATION.DELETE) this.ops[index] = OPERATION.DELETE_AND_ADD;
+        // Promote ADD → DELETE_AND_ADD when a ref is replaced in the same
+        // tick. See `ChangeTree.record` for rationale — same logic, this
+        // interface implementation is kept in sync.
+        else if (prev === OPERATION.ADD && op === OPERATION.DELETE_AND_ADD) {
+            this.ops[index] = OPERATION.DELETE_AND_ADD;
+        }
+        // else preserve existing ADD / DELETE_AND_ADD.
+
+        if (index < 32) this.dirtyLow |= (1 << index);
+        else this.dirtyHigh |= (1 << (index - 32));
+    }
+
+    recordDelete(index: number, op: OPERATION): void {
+        this.ops[index] = op;
+        if (index < 32) this.dirtyLow |= (1 << index);
+        else this.dirtyHigh |= (1 << (index - 32));
+    }
+
+    recordRaw(index: number, op: OPERATION): void {
+        this.record(index, op);
+    }
+
+    operationAt(index: number): OPERATION | undefined {
+        const op = this.ops[index];
+        return op === 0 ? undefined : op;
+    }
+
+    setOperationAt(index: number, op: OPERATION): void {
+        this.ops[index] = op;
+    }
+
+    forEach(cb: (index: number, op: OPERATION) => void): void {
+        this.forEachWithCtx(cb, _invokeNoCtx);
+    }
+
+    forEachWithCtx<T>(ctx: T, cb: (ctx: T, index: number, op: OPERATION) => void): void {
+        let low = this.dirtyLow;
+        let high = this.dirtyHigh;
+        const ops = this.ops;
+        // Iterate set bits via clz32 (CPU-level bit scan).
+        while (low !== 0) {
+            const bit = low & -low;
+            const fieldIndex = 31 - Math.clz32(bit);
+            low ^= bit;
+            cb(ctx, fieldIndex, ops[fieldIndex]);
+        }
+        while (high !== 0) {
+            const bit = high & -high;
+            const fieldIndex = 31 - Math.clz32(bit) + 32;
+            high ^= bit;
+            cb(ctx, fieldIndex, ops[fieldIndex]);
+        }
+    }
+
+    size(): number {
+        return popcount32(this.dirtyLow) + popcount32(this.dirtyHigh);
+    }
+
+    has(): boolean {
+        return (this.dirtyLow | this.dirtyHigh) !== 0;
+    }
+
+    reset(): void {
+        this.dirtyLow = 0;
+        this.dirtyHigh = 0;
+        this.ops.fill(0);
+    }
+}
+
+// ──────────────────────────────────────────────────────────────────────────
+// CollectionChangeRecorder — Map-based, for collections with sparse indexes
+// ──────────────────────────────────────────────────────────────────────────
+
+/**
+ * Collection items have sparse indexes (e.g. 0, 7, 1024) exceeding the
+ * 64-field cap Schema imposes. Map-based storage handles arbitrary
+ * indexes; the value at each entry is the OPERATION.
+ *
+ * Pure operations (CLEAR, REVERSE) live in `pureOps` as `[position, op]`
+ * entries where `position` is `dirty.size` at record time — preserves
+ * insertion-order interleaving with indexed ops (e.g. CLEAR must emit
+ * BEFORE subsequent ADDs).
+ */
+export class CollectionChangeRecorder implements ICollectionChangeRecorder {
+    private dirty: Map<number, OPERATION> = new Map();
+    private pureOps: Array<[number, OPERATION]> = [];
+
+    record(index: number, op: OPERATION): void {
+        const prev = this.dirty.get(index);
+        if (prev === undefined) this.dirty.set(index, op);
+        else if (prev === OPERATION.DELETE) this.dirty.set(index, OPERATION.DELETE_AND_ADD);
+        // Promote ADD → DELETE_AND_ADD for same-tick replacement of a ref
+        // (see `SchemaChangeRecorder.record` for rationale).
+        else if (prev === OPERATION.ADD && op === OPERATION.DELETE_AND_ADD) {
+            this.dirty.set(index, OPERATION.DELETE_AND_ADD);
+        }
+        // else preserve existing op.
+    }
+
+    recordDelete(index: number, op: OPERATION): void {
+        this.dirty.set(index, op);
+    }
+
+    recordRaw(index: number, op: OPERATION): void {
+        this.dirty.set(index, op);
+    }
+
+    recordPure(op: OPERATION): void {
+        this.pureOps.push([this.dirty.size, op]);
+    }
+
+    operationAt(index: number): OPERATION | undefined {
+        return this.dirty.get(index);
+    }
+
+    setOperationAt(index: number, op: OPERATION): void {
+        if (this.dirty.has(index)) this.dirty.set(index, op);
+    }
+
+    forEach(cb: (index: number, op: OPERATION) => void): void {
+        this.forEachWithCtx(cb, _invokeNoCtx);
+    }
+
+    forEachWithCtx<T>(ctx: T, cb: (ctx: T, index: number, op: OPERATION) => void): void {
+        const pure = this.pureOps;
+        if (pure.length > 0) {
+            let pureIdx = 0, i = 0;
+            for (const [index, op] of this.dirty) {
+                while (pureIdx < pure.length && pure[pureIdx][0] <= i) {
+                    const pureOp = pure[pureIdx++][1];
+                    cb(ctx, -pureOp, pureOp);
+                }
+                cb(ctx, index, op);
+                i++;
+            }
+            while (pureIdx < pure.length) {
+                const pureOp = pure[pureIdx++][1];
+                cb(ctx, -pureOp, pureOp);
+            }
+        } else {
+            for (const [index, op] of this.dirty) cb(ctx, index, op);
+        }
+    }
+
+    size(): number {
+        return this.dirty.size + this.pureOps.length;
+    }
+
+    has(): boolean {
+        return this.dirty.size > 0 || this.pureOps.length > 0;
+    }
+
+    reset(): void {
+        this.dirty.clear();
+        this.pureOps.length = 0;
+    }
+
+    shift(shiftIndex: number): void {
+        const dst = new Map<number, OPERATION>();
+        for (const [idx, val] of this.dirty) dst.set(idx + shiftIndex, val);
+        this.dirty = dst;
+    }
+}
+
+// ──────────────────────────────────────────────────────────────────────────
+// Helpers
+// ──────────────────────────────────────────────────────────────────────────
+
+/** 32-bit Hamming weight (popcount). */
+export function popcount32(n: number): number {
+    n = n - ((n >>> 1) & 0x55555555);
+    n = (n & 0x33333333) + ((n >>> 2) & 0x33333333);
+    return (((n + (n >>> 4)) & 0x0f0f0f0f) * 0x01010101) >>> 24;
+}