@colyseus/schema 3.0.0-alpha.4 → 3.0.0-alpha.41

package/src/encoder/Encoder.ts

@@ -1,14 +1,16 @@
 import type { Schema } from "../Schema";
-import { TypeContext } from "../annotations";
-import { $changes, $encoder, $filter } from "../types/symbols";
+import { TypeContext } from "../types/TypeContext";
+import { $changes, $encoder, $filter, $onEncodeEnd } from "../types/symbols";
 
-import * as encode from "../encoding/encode";
+import { encode } from "../encoding/encode";
 import type { Iterator } from "../encoding/decode";
 
-import { SWITCH_TO_STRUCTURE, TYPE_ID } from '../encoding/spec';
-import { Root } from "./ChangeTree";
+import { OPERATION, SWITCH_TO_STRUCTURE, TYPE_ID } from '../encoding/spec';
+import { Root } from "./Root";
 import { getNextPowerOf2 } from "../utils";
+
 import type { StateView } from "./StateView";
+import type { Metadata } from "../Metadata";
 
 export class Encoder<T extends Schema = any> {
     static BUFFER_SIZE = 8 * 1024;// 8KB
@@ -19,14 +21,16 @@ export class Encoder<T extends Schema = any> {
 
     root: Root;
 
-    constructor(root: T) {
-        this.setRoot(root);
+    constructor(state: T) {
 
         //
         // TODO: cache and restore "Context" based on root schema
         // (to avoid creating a new context for every new room)
         //
-        this.context = new TypeContext(root.constructor as typeof Schema);
+        this.context = new TypeContext(state.constructor as typeof Schema);
+        this.root = new Root(this.context);
+
+        this.setState(state);
 
         // console.log(">>>>>>>>>>>>>>>> Encoder types");
         // this.context.schemas.forEach((id, schema) => {
@@ -34,32 +38,45 @@ export class Encoder<T extends Schema = any> {
         // });
     }
 
-    protected setRoot(state: T) {
-        this.root = new Root();
+    protected setState(state: T) {
         this.state = state;
-        state[$changes].setRoot(this.root);
+        this.state[$changes].setRoot(this.root);
     }
 
     encode(
         it: Iterator = { offset: 0 },
         view?: StateView,
-        bytes = this.sharedBuffer,
-        changeTrees = this.root.changes
+        buffer = this.sharedBuffer,
+        changeSetName: "changes" | "allChanges" | "filteredChanges" | "allFilteredChanges" = "changes",
+        isEncodeAll = changeSetName === "allChanges",
+        initialOffset = it.offset // cache current offset in case we need to resize the buffer
     ): Buffer {
-        const initialOffset = it.offset; // cache current offset in case we need to resize the buffer
-
-        const isEncodeAll = this.root.allChanges === changeTrees;
         const hasView = (view !== undefined);
         const rootChangeTree = this.state[$changes];
 
-        const changeTreesIterator = changeTrees.entries();
+        const shouldDiscardChanges = !isEncodeAll && !hasView;
+        const changeTrees = this.root[changeSetName];
 
-        for (const [changeTree, changes] of changeTreesIterator) {
+        for (let i = 0, numChangeTrees = changeTrees.length; i < numChangeTrees; i++) {
+            const changeTree = changeTrees[i];
+
+            // // Root#removeChangeFromChangeSet() is now removing instead of setting to "undefined"
+            // if (changeTree === undefined) { continue; }
+
+            const operations = changeTree[changeSetName];
             const ref = changeTree.ref;
 
-            const ctor = ref['constructor'];
+            const ctor = ref.constructor;
             const encoder = ctor[$encoder];
             const filter = ctor[$filter];
+            const metadata = ctor[Symbol.metadata];
+
+            // try { throw new Error(); } catch (e) {
+            //     // only print if not coming from Reflection.ts
+            //     if (!e.stack.includes("src/Reflection.ts")) {
+            //         console.log("ChangeTree:", { refId: changeTree.refId, ref: ref.constructor.name });
+            //     }
+            // }
 
             if (hasView) {
                 if (!view.items.has(changeTree)) {
@@ -72,14 +89,21 @@ export class Encoder<T extends Schema = any> {
             }
 
             // skip root `refId` if it's the first change tree
-            if (it.offset !== initialOffset || changeTree !== rootChangeTree) {
-                bytes[it.offset++] = SWITCH_TO_STRUCTURE & 255;
-                encode.number(bytes, changeTree.refId, it);
+            // (unless it "hasView", which will need to revisit the root)
+            if (hasView || it.offset > initialOffset || changeTree !== rootChangeTree) {
+                buffer[it.offset++] = SWITCH_TO_STRUCTURE & 255;
+                encode.number(buffer, changeTree.refId, it);
             }
 
-            const changesIterator = changes.entries();
+            for (let j = 0, numChanges = operations.operations.length; j < numChanges; j++) {
+                const fieldIndex = operations.operations[j];
+
+                const operation = (fieldIndex < 0)
+                    ? Math.abs(fieldIndex) // "pure" operation without fieldIndex (e.g. CLEAR, REVERSE, etc.)
+                    : (isEncodeAll)
+                        ? OPERATION.ADD
+                        : changeTree.indexedOperations[fieldIndex];
 
-            for (const [fieldIndex, operation] of changesIterator) {
                 //
                 // first pass (encodeAll), identify "filtered" operations without encoding them
                 // they will be encoded per client, based on their view.
@@ -87,116 +111,158 @@ export class Encoder<T extends Schema = any> {
                 // TODO: how can we optimize filtering out "encode all" operations?
                 // TODO: avoid checking if no view tags were defined
                 //
-                if (filter && !filter(ref, fieldIndex, view)) {
-                    // console.log("SKIP FIELD:", { ref: changeTree.ref.constructor.name, fieldIndex, })
-
+                if (fieldIndex === undefined || operation === undefined || (filter && !filter(ref, fieldIndex, view))) {
                     // console.log("ADD AS INVISIBLE:", fieldIndex, changeTree.ref.constructor.name)
                     // view?.invisible.add(changeTree);
                     continue;
                 }
 
-                // console.log("WILL ENCODE", {
-                //     ref: changeTree.ref.constructor.name,
-                //     fieldIndex,
-                //     operation: OPERATION[operation],
-                // });
+                // try { throw new Error(); } catch (e) {
+                //     // only print if not coming from Reflection.ts
+                //     if (!e.stack.includes("src/Reflection.ts")) {
+                //         console.log("WILL ENCODE", {
+                //             ref: changeTree.ref.constructor.name,
+                //             fieldIndex,
+                //             operation: OPERATION[operation],
+                //         });
+                //     }
+                // }
+
+                // console.log("encode...", { ref: changeTree.ref.constructor.name, refId: changeTree.refId, fieldIndex, operation });
+
+                encoder(this, buffer, changeTree, fieldIndex, operation, it, isEncodeAll, hasView, metadata);
+            }
+
+            if (shouldDiscardChanges) {
+                changeTree.discard();
 
-                encoder(this, bytes, changeTree, fieldIndex, operation, it, isEncodeAll, hasView);
+                // Not a new instance anymore
+                changeTree.isNew = false;
             }
         }
 
-        if (it.offset > bytes.byteLength) {
-            const newSize = getNextPowerOf2(this.sharedBuffer.byteLength * 2);
-            console.warn("@colyseus/schema encode buffer overflow. Current buffer size: " + bytes.byteLength + ", encoding offset: " + it.offset + ", new size: " + newSize);
+        if (it.offset > buffer.byteLength) {
+            const newSize = getNextPowerOf2(buffer.byteLength * 2);
+            console.warn(`@colyseus/schema buffer overflow. Encoded state is higher than default BUFFER_SIZE. Use the following to increase default BUFFER_SIZE:
+
+    import { Encoder } from "@colyseus/schema";
+    Encoder.BUFFER_SIZE = ${Math.round(newSize / 1024)} * 1024; // ${Math.round(newSize / 1024)} KB
+`);
 
             //
             // resize buffer and re-encode (TODO: can we avoid re-encoding here?)
             //
-            this.sharedBuffer = Buffer.allocUnsafeSlow(newSize);
-            return this.encode({ offset: initialOffset }, view);
+            buffer = Buffer.allocUnsafeSlow(newSize);
 
-        } else {
-            //
-            // only clear changes after making sure buffer resize is not required.
-            //
-            if (!isEncodeAll && !hasView) {
-                //
-                // FIXME: avoid iterating over change trees twice.
-                //
-                this.onEndEncode(changeTrees);
+            // assign resized buffer to local sharedBuffer
+            if (buffer === this.sharedBuffer) {
+                this.sharedBuffer = buffer;
             }
 
-            // return bytes;
-            return bytes.slice(0, it.offset);
+            return this.encode({ offset: initialOffset }, view, buffer, changeSetName, isEncodeAll);
+
+        } else {
+            // //
+            // // only clear changes after making sure buffer resize is not required.
+            // //
+            // if (shouldClearChanges) {
+            //     //
+            //     // FIXME: avoid iterating over change trees twice.
+            //     //
+            //     this.onEndEncode(changeTrees);
+            // }
+
+            return buffer.subarray(0, it.offset);
         }
     }
 
-    encodeAll(it: Iterator = { offset: 0 }) {
-        // console.log(`encodeAll(), this.$root.allChanges (${this.$root.allChanges.size})`);
+    encodeAll(it: Iterator = { offset: 0 }, buffer: Buffer = this.sharedBuffer) {
+        // console.log(`\nencodeAll(), this.root.allChanges (${(Object.keys(this.root.allChanges).length)})`);
+        // this.debugChanges("allChanges");
 
-        // Array.from(this.$root.allChanges.entries()).map((item) => {
-        //     console.log("->", item[0].refId, item[0].ref.toJSON());
-        // });
-
-        return this.encode(it, undefined, this.sharedBuffer, this.root.allChanges);
+        return this.encode(it, undefined, buffer, "allChanges", true);
     }
 
     encodeAllView(view: StateView, sharedOffset: number, it: Iterator, bytes = this.sharedBuffer) {
         const viewOffset = it.offset;
 
-        // console.log(`encodeAllView(), this.$root.allFilteredChanges (${this.$root.allFilteredChanges.size})`);
-        // this.debugAllFilteredChanges();
+        // console.log(`\nencodeAllView(), this.root.allFilteredChanges (${(Object.keys(this.root.allFilteredChanges).length)})`);
+        // this.debugChanges("allFilteredChanges");
+
+        // console.log("\n\nENCODE ALL FOR VIEW...\n\n")
 
         // try to encode "filtered" changes
-        this.encode(it, view, bytes, this.root.allFilteredChanges);
+        this.encode(it, view, bytes, "allFilteredChanges", true, viewOffset);
 
         return Buffer.concat([
-            bytes.slice(0, sharedOffset),
-            bytes.slice(viewOffset, it.offset)
+            bytes.subarray(0, sharedOffset),
+            bytes.subarray(viewOffset, it.offset)
         ]);
     }
 
-
-    // debugAllFilteredChanges() {
-    //     Array.from(this.$root.allFilteredChanges.entries()).map((item) => {
-    //         console.log("->", { refId: item[0].refId }, item[0].ref.toJSON());
-    //         if (Array.isArray(item[0].ref.toJSON())) {
-    //             item[1].forEach((op, key) => {
-    //                 console.log("  ->", { key, op: OPERATION[op] });
-    //             })
-    //         }
-    //     });
-    // }
+    debugChanges(field: "changes" | "allFilteredChanges" | "allChanges" | "filteredChanges") {
+        const rootChangeSet = (typeof (field) === "string")
+            ? this.root[field]
+            : field;
+
+        rootChangeSet.forEach((changeTree) => {
+            const changeSet = changeTree[field];
+
+            const metadata: Metadata = changeTree.ref.constructor[Symbol.metadata];
+            console.log("->", { ref: changeTree.ref.constructor.name, refId: changeTree.refId, changes: Object.keys(changeSet).length });
+            for (const index in changeSet) {
+                const op = changeSet[index];
+                console.log("  ->", {
+                    index,
+                    field: metadata?.[index],
+                    op: OPERATION[op],
+                });
+            }
+        });
+    }
 
     encodeView(view: StateView, sharedOffset: number, it: Iterator, bytes = this.sharedBuffer) {
         const viewOffset = it.offset;
 
-        // try to encode "filtered" changes
-        this.encode(it, view, bytes, this.root.filteredChanges);
+        // console.log(`\nencodeView(), view.changes (${view.changes.size})`);
+        // this.debugChanges(view.changes);
+
+        // console.log(`\nencodeView(), this.root.filteredChanges (${this.root.filteredChanges.size})`);
+        // this.debugChanges("filteredChanges");
 
         // encode visibility changes (add/remove for this view)
-        const viewChangesIterator = view.changes.entries();
-        for (const [changeTree, changes] of viewChangesIterator) {
-            if (changes.size === 0) {
-                // FIXME: avoid having empty changes if no changes were made
-                // console.log("changes.size === 0", changeTree.ref.constructor.name);
+        const refIds = Object.keys(view.changes);
+        // console.log("ENCODE VIEW:", refIds);
+        for (let i = 0, numRefIds = refIds.length; i < numRefIds; i++) {
+            const refId = refIds[i];
+            const changes = view.changes[refId];
+            const changeTree = this.root.changeTrees[refId];
+
+            if (
+                changeTree === undefined ||
+                Object.keys(changes).length === 0 // FIXME: avoid having empty changes if no changes were made
+            ) {
+                // console.log("changes.size === 0, skip", changeTree.ref.constructor.name);
                 continue;
             }
 
             const ref = changeTree.ref;
 
-            const ctor = ref['constructor'];
+            const ctor = ref.constructor;
             const encoder = ctor[$encoder];
+            const metadata = ctor[Symbol.metadata];
 
             bytes[it.offset++] = SWITCH_TO_STRUCTURE & 255;
             encode.number(bytes, changeTree.refId, it);
 
-            const changesIterator = changes.entries();
+            const keys = Object.keys(changes);
+            for (let i = 0, numChanges = keys.length; i < numChanges; i++) {
+                const key = keys[i];
+                const operation = changes[key];
 
-            for (const [fieldIndex, operation] of changesIterator) {
                 // isEncodeAll = false
                 // hasView = true
-                encoder(this, bytes, changeTree, fieldIndex, operation, it, false, true);
+                encoder(this, bytes, changeTree, Number(key), operation, it, false, true, metadata);
             }
         }
 
@@ -205,31 +271,58 @@ export class Encoder<T extends Schema = any> {
         // (to allow re-using StateView's for multiple clients)
         //
         // clear "view" changes after encoding
-        view.changes.clear();
+        view.changes = {};
+
+        // console.log("FILTERED CHANGES:", this.root.filteredChanges);
+
+        // try to encode "filtered" changes
+        this.encode(it, view, bytes, "filteredChanges", false, viewOffset);
 
         return Buffer.concat([
-            bytes.slice(0, sharedOffset),
-            bytes.slice(viewOffset, it.offset)
+            bytes.subarray(0, sharedOffset),
+            bytes.subarray(viewOffset, it.offset)
         ]);
     }
 
     onEndEncode(changeTrees = this.root.changes) {
-        const changeTreesIterator = changeTrees.entries();
-        for (const [changeTree, _] of changeTreesIterator) {
-            changeTree.endEncode();
-        }
+        // changeTrees.forEach(function(changeTree) {
+        //     changeTree.endEncode();
+        // });
+
+
+        // for (const refId in changeTrees) {
+        //     const changeTree = this.root.changeTrees[refId];
+        //     changeTree.endEncode();
+
+        //     // changeTree.changes.clear();
+
+        //     // // ArraySchema and MapSchema have a custom "encode end" method
+        //     // changeTree.ref[$onEncodeEnd]?.();
+
+        //     // // Not a new instance anymore
+        //     // delete changeTree[$isNew];
+        // }
     }
 
     discardChanges() {
+        // console.log("DISCARD CHANGES!");
+
         // discard shared changes
-        if (this.root.changes.size > 0) {
-            this.onEndEncode(this.root.changes);
-            this.root.changes.clear();
+        let length = this.root.changes.length;
+        if (length > 0) {
+            while (length--) {
+                this.root.changes[length]?.endEncode();
+            }
+            this.root.changes.length = 0;
         }
+
         // discard filtered changes
-        if (this.root.filteredChanges.size > 0) {
-            this.onEndEncode(this.root.filteredChanges);
-            this.root.filteredChanges.clear();
+        length = this.root.filteredChanges.length;
+        if (length > 0) {
+            while (length--) {
+                this.root.filteredChanges[length]?.endEncode();
+            }
+            this.root.filteredChanges.length = 0;
         }
     }
 
@@ -237,9 +330,21 @@ export class Encoder<T extends Schema = any> {
         const baseTypeId = this.context.getTypeId(baseType);
         const targetTypeId = this.context.getTypeId(targetType);
 
+        if (targetTypeId === undefined) {
+            console.warn(`@colyseus/schema WARNING: Class "${targetType.name}" is not registered on TypeRegistry - Please either tag the class with @entity or define a @type() field.`);
+            return;
+        }
+
         if (baseTypeId !== targetTypeId) {
             bytes[it.offset++] = TYPE_ID & 255;
             encode.number(bytes, targetTypeId, it);
         }
     }
-}
+
+    get hasChanges() {
+        return (
+            this.root.changes.length > 0 ||
+            this.root.filteredChanges.length > 0
+        );
+    }
+}

package/src/encoder/Root.ts

@@ -0,0 +1,93 @@
+import { OPERATION } from "../encoding/spec";
+import { TypeContext } from "../types/TypeContext";
+import { spliceOne } from "../types/utils";
+import { ChangeTree, setOperationAtIndex } from "./ChangeTree";
+
+export class Root {
+    protected nextUniqueId: number = 0;
+
+    refCount: {[id: number]: number} = {};
+    changeTrees: {[refId: number]: ChangeTree} = {};
+
+    // all changes
+    allChanges: ChangeTree[] = [];
+    allFilteredChanges: ChangeTree[] = [];// TODO: do not initialize it if filters are not used
+
+    // pending changes to be encoded
+    changes: ChangeTree[] = [];
+    filteredChanges: ChangeTree[] = [];// TODO: do not initialize it if filters are not used
+
+    constructor(public types: TypeContext) { }
+
+    getNextUniqueId() {
+        return this.nextUniqueId++;
+    }
+
+    add(changeTree: ChangeTree) {
+        // FIXME: move implementation of `ensureRefId` to `Root` class
+        changeTree.ensureRefId();
+
+        const isNewChangeTree = (this.changeTrees[changeTree.refId] === undefined);
+        if (isNewChangeTree) { this.changeTrees[changeTree.refId] = changeTree; }
+
+        const previousRefCount = this.refCount[changeTree.refId];
+        if (previousRefCount === 0) {
+            //
+            // When a ChangeTree is re-added, it means that it was previously removed.
+            // We need to re-add all changes to the `changes` map.
+            //
+            const ops = changeTree.allChanges.operations;
+            let len = ops.length;
+            while (len--) {
+                changeTree.indexedOperations[ops[len]] = OPERATION.ADD;
+                setOperationAtIndex(changeTree.changes, len);
+            }
+        }
+
+        this.refCount[changeTree.refId] = (previousRefCount || 0) + 1;
+
+        return isNewChangeTree;
+    }
+
+    remove(changeTree: ChangeTree) {
+        const refCount = (this.refCount[changeTree.refId]) - 1;
+
+        if (refCount <= 0) {
+            //
+            // Only remove "root" reference if it's the last reference
+            //
+            changeTree.root = undefined;
+            delete this.changeTrees[changeTree.refId];
+
+            this.removeChangeFromChangeSet("allChanges", changeTree);
+            this.removeChangeFromChangeSet("changes", changeTree);
+
+            if (changeTree.isFiltered || changeTree.isPartiallyFiltered) {
+                this.removeChangeFromChangeSet("allFilteredChanges", changeTree);
+                this.removeChangeFromChangeSet("filteredChanges", changeTree);
+            }
+
+            this.refCount[changeTree.refId] = 0;
+
+        } else {
+            this.refCount[changeTree.refId] = refCount;
+        }
+
+        changeTree.forEachChild((child, _) => this.remove(child));
+
+        return refCount;
+    }
+
+    removeChangeFromChangeSet(changeSetName: "allChanges" | "changes" | "filteredChanges" | "allFilteredChanges", changeTree: ChangeTree) {
+        const changeSet = this[changeSetName];
+        const index = changeSet.indexOf(changeTree);
+        if (index !== -1) {
+            spliceOne(changeSet, index);
+            // changeSet[index] = undefined;
+        }
+    }
+
+    clear() {
+        this.changes.length = 0;
+    }
+}