@colyseus/schema 3.0.42 → 3.0.43

package/src/bench_encode.ts

@@ -0,0 +1,108 @@
+import { nanoid } from "nanoid";
+import { Schema, type, MapSchema, ArraySchema, Encoder } from ".";
+
+class Attribute extends Schema {
+    @type("string") name: string;
+    @type("number") value: number;
+}
+
+class Item extends Schema {
+    @type("number") price: number;
+    @type([ Attribute ]) attributes = new ArraySchema<Attribute>();
+}
+
+class Position extends Schema {
+    @type("number") x: number;
+    @type("number") y: number;
+}
+
+class Player extends Schema {
+    @type(Position) position = new Position();
+    @type({ map: Item }) items = new MapSchema<Item>();
+}
+
+class State extends Schema {
+    @type({ map: Player }) players = new MapSchema<Player>();
+    @type("string") currentTurn;
+}
+
+const state = new State();
+
+Encoder.BUFFER_SIZE = 4096 * 4096;
+const encoder = new Encoder(state);
+
+
+let now = Date.now();
+
+// for (let i = 0; i < 10000; i++) {
+//     const player = new Player();
+//     state.players.set(`p-${nanoid()}`, player);
+//
+//     player.position.x = (i + 1) * 100;
+//     player.position.y = (i + 1) * 100;
+//     for (let j = 0; j < 10; j++) {
+//         const item = new Item();
+//         player.items.set(`item-${j}`, item);
+//         item.price = (i + 1) * 50;
+//         for (let k = 0; k < 5; k++) {
+//             const attr = new Attribute();
+//             attr.name = `Attribute ${k}`;
+//             attr.value = k;
+//             item.attributes.push(attr);
+//         }
+//     }
+// }
+// console.log("time to make changes:", Date.now() - now);
+
+
+// measure time to .encodeAll()
+
+now = Date.now();
+// for (let i = 0; i < 1000; i++) {
+//     encoder.encodeAll();
+// }
+// console.log(Date.now() - now);
+
+const total = 100;
+const allEncodes = Date.now();
+
+let avgTimeToEncode = 0;
+let avgTimeToMakeChanges = 0;
+
+for (let i = 0; i < total; i++) {
+    now = Date.now();
+    for (let j = 0; j < 50; j++) {
+        const player = new Player();
+        state.players.set(`p-${nanoid()}`, player);
+
+        player.position.x = (j + 1) * 100;
+        player.position.y = (j + 1) * 100;
+        for (let k = 0; k < 10; k++) {
+            const item = new Item();
+            item.price = (j + 1) * 50;
+            for (let l = 0; l < 5; l++) {
+                const attr = new Attribute();
+                attr.name = `Attribute ${l}`;
+                attr.value = l;
+                item.attributes.push(attr);
+            }
+            player.items.set(`item-${k}`, item);
+        }
+    }
+    const timeToMakeChanges = Date.now() - now;
+    console.log("time to make changes:", timeToMakeChanges);
+    avgTimeToMakeChanges += timeToMakeChanges;
+
+    now = Date.now();
+    encoder.encode();
+    encoder.discardChanges();
+
+    const timeToEncode = Date.now() - now;
+    console.log("time to encode:", timeToEncode);
+    avgTimeToEncode += timeToEncode;
+}
+console.log("avg time to encode:", (avgTimeToEncode) / total);
+console.log("avg time to make changes:", (avgTimeToMakeChanges) / total);
+console.log("time for all encodes:", Date.now() - allEncodes);
+
+console.log(Array.from(encoder.encodeAll()).length, "bytes");

package/src/debug.ts

@@ -0,0 +1,55 @@
+import * as fs from "fs";
+import { Reflection, Decoder } from "./index";
+
+const contents = fs.readFileSync("/Users/endel/Projects/colyseus/clients/bubbits/project/@bubbits/backend/schema-debug.txt", { encoding: "utf8" }).toString();
+
+let isCommentBlock = false;
+let lastComment = "";
+
+let decoder: Decoder;
+
+function getBuffer(line: string) {
+    const start = line.lastIndexOf(":");
+    const buffer = Buffer.from(new Uint8Array(line.substring(start + 1).split(",").map(n => Number(n))));
+    console.log(`(${buffer.byteLength}) ${Array.from(buffer).join(",")}`)
+    // console.log("");
+    // console.log("");
+    // console.log("> ", line);
+    // console.log("> substring:", line.substring(start + 1))
+    return buffer;
+}
+
+function decode(buffer: Buffer) {
+    try {
+        decoder.decode(buffer);
+    } catch (e) {
+        console.error(e);
+        console.log("Last log:\n\n")
+        console.log(lastComment);
+    }
+}
+
+contents.split("\n").forEach((line) => {
+    if (line.startsWith("#")) {
+        // reset last comment.
+        if (isCommentBlock === false) { lastComment = ""; }
+
+        isCommentBlock = true;
+        lastComment += line.substring(line.indexOf(":") + 1) + "\n";
+        return;
+    }
+
+    isCommentBlock = false;
+
+    if (line.startsWith("handshake:") && !decoder) {
+        decoder = Reflection.decode(getBuffer(line));
+
+    } else if (line.startsWith("state:")) {
+        decode(getBuffer(line));
+
+    } else if (line.startsWith("patch:")) {
+        decode(getBuffer(line));
+    }
+});
+
+console.log(decoder.state.toJSON());

package/src/decoder/Decoder.ts

@@ -58,6 +58,8 @@ export class Decoder<T extends Schema = any> {
             if (bytes[it.offset] == SWITCH_TO_STRUCTURE) {
                 it.offset++;
 
+                ref[$onDecodeEnd]?.()
+
                 const nextRefId = decode.number(bytes, it);
                 const nextRef = $root.refs.get(nextRefId);
 
@@ -65,17 +67,17 @@ export class Decoder<T extends Schema = any> {
                 // Trying to access a reference that haven't been decoded yet.
                 //
                 if (!nextRef) {
+                    // throw new Error(`"refId" not found: ${nextRefId}`);
                     console.error(`"refId" not found: ${nextRefId}`, { previousRef: ref, previousRefId: this.currentRefId });
                     console.warn("Please report this to the developers. All refIds =>");
                     console.warn(Schema.debugRefIdsDecoder(this));
                     this.skipCurrentStructure(bytes, it, totalBytes);
-                }
-                ref[$onDecodeEnd]?.()
 
-                this.currentRefId = nextRefId;
-
-                ref = nextRef;
-                decoder = ref.constructor[$decoder];
+                } else {
+                    ref = nextRef;
+                    decoder = ref.constructor[$decoder];
+                    this.currentRefId = nextRefId;
+                }
 
                 continue;
             }
@@ -131,12 +133,6 @@ export class Decoder<T extends Schema = any> {
     }
 
     createInstanceOfType (type: typeof Schema): Schema {
-        // let instance: Schema = new (type as any)();
-
-        // // assign root on $changes
-        // instance[$changes].root = this.root[$changes].root;
-
-        // return instance;
         return new (type as any)();
     }
 

package/src/encoder/ChangeTree.ts

@@ -36,17 +36,53 @@ export interface IndexedOperations {
     [index: number]: OPERATION;
 }
 
+// Linked list node for change trees
+export interface ChangeTreeNode {
+    changeTree: ChangeTree;
+    next?: ChangeTreeNode;
+    prev?: ChangeTreeNode;
+}
+
+// Linked list for change trees
+export interface ChangeTreeList {
+    next?: ChangeTreeNode;
+    tail?: ChangeTreeNode;
+    length: number;
+}
+
 export interface ChangeSet {
     // field index -> operation index
     indexes: { [index: number]: number };
     operations: number[];
-    queueRootIndex?: number; // index of ChangeTree structure in `root.changes` or `root.filteredChanges`
+    queueRootNode?: ChangeTreeNode; // direct reference to ChangeTreeNode in the linked list
 }
 
 function createChangeSet(): ChangeSet {
     return { indexes: {}, operations: [] };
 }
 
+// Linked list helper functions
+export function createChangeTreeList(): ChangeTreeList {
+    return { next: undefined, tail: undefined, length: 0 };
+}
+
+export function addToChangeTreeList(list: ChangeTreeList, changeTree: ChangeTree): ChangeTreeNode {
+    const node: ChangeTreeNode = { changeTree, next: undefined, prev: undefined };
+
+    if (!list.next) {
+        list.next = node;
+        list.tail = node;
+    } else {
+        node.prev = list.tail;
+        list.tail!.next = node;
+        list.tail = node;
+    }
+
+    list.length++;
+
+    return node;
+}
+
 export function setOperationAtIndex(changeSet: ChangeSet, index: number) {
     const operationsIndex = changeSet.indexes[index];
     if (operationsIndex === undefined) {
@@ -96,25 +132,32 @@ export function debugChangeSet(label: string, changeSet: ChangeSet) {
 export function enqueueChangeTree(
     root: Root,
     changeTree: ChangeTree,
-    changeSet: 'changes' | 'filteredChanges' | 'allFilteredChanges',
-    queueRootIndex = changeTree[changeSet].queueRootIndex
+    changeSet: 'changes' | 'filteredChanges' | 'allFilteredChanges' | 'allChanges',
+    queueRootNode = changeTree[changeSet].queueRootNode
 ) {
-    if (!root) {
-        // skip
-        return;
+    // skip
+    if (!root) { return; }
 
-    } else if (root[changeSet][queueRootIndex] !== changeTree) {
-        changeTree[changeSet].queueRootIndex = root[changeSet].push(changeTree) - 1;
+    if (queueRootNode) {
+    } else {
+        // Add to linked list if not already present
+        changeTree[changeSet].queueRootNode = addToChangeTreeList(root[changeSet], changeTree);
     }
 }
 
+export interface ParentChain {
+    ref: Ref;
+    index: number;
+    next?: ParentChain;
+}
+
 export class ChangeTree<T extends Ref=any> {
     ref: T;
     refId: number;
+    metadata: Metadata;
 
     root?: Root;
-    parent?: Ref;
-    parentIndex?: number;
+    parentChain?: ParentChain; // Linked list for tracking parents
 
     /**
      * Whether this structure is parent of a filtered structure.
@@ -145,12 +188,12 @@ export class ChangeTree<T extends Ref=any> {
 
     constructor(ref: T) {
         this.ref = ref;
+        this.metadata = ref.constructor[Symbol.metadata];
 
         //
         // Does this structure have "filters" declared?
         //
-        const metadata = ref.constructor[Symbol.metadata];
-        if (metadata?.[$viewFieldIndexes]) {
+        if (this.metadata?.[$viewFieldIndexes]) {
             this.allFilteredChanges = { indexes: {}, operations: [] };
             this.filteredChanges = { indexes: {}, operations: [] };
         }
@@ -158,35 +201,18 @@ export class ChangeTree<T extends Ref=any> {
 
     setRoot(root: Root) {
         this.root = root;
-        this.checkIsFiltered(this.parent, this.parentIndex);
 
-        //
-        // TODO: refactor and possibly unify .setRoot() and .setParent()
-        //
+        const isNewChangeTree = this.root.add(this);
 
-        // Recursively set root on child structures
-        const metadata: Metadata = this.ref.constructor[Symbol.metadata];
-        if (metadata) {
-            metadata[$refTypeFieldIndexes]?.forEach((index) => {
-                const field = metadata[index as any as number];
-                const changeTree: ChangeTree = this.ref[field.name]?.[$changes];
-                if (changeTree) {
-                    if (changeTree.root !== root) {
-                        changeTree.setRoot(root);
-                    } else {
-                        root.add(changeTree); // increment refCount
-                    }
-                }
-            });
+        this.checkIsFiltered(this.parent, this.parentIndex, isNewChangeTree);
 
-        } else if (this.ref[$childType] && typeof(this.ref[$childType]) !== "string") {
-            // MapSchema / ArraySchema, etc.
-            (this.ref as MapSchema).forEach((value, key) => {
-                const changeTree: ChangeTree = value[$changes];
-                if (changeTree.root !== root) {
-                    changeTree.setRoot(root);
+        // Recursively set root on child structures
+        if (isNewChangeTree) {
+            this.forEachChild((child, _) => {
+                if (child.root !== root) {
+                    child.setRoot(root);
                 } else {
-                    root.add(changeTree); // increment refCount
+                    root.add(child); // increment refCount
                 }
             });
         }
@@ -197,63 +223,57 @@ export class ChangeTree<T extends Ref=any> {
         root?: Root,
         parentIndex?: number,
     ) {
-        this.parent = parent;
-        this.parentIndex = parentIndex;
+        this.addParent(parent, parentIndex);
 
         // avoid setting parents with empty `root`
         if (!root) { return; }
 
+        const isNewChangeTree = root.add(this);
+
         // skip if parent is already set
         if (root !== this.root) {
             this.root = root;
-            this.checkIsFiltered(parent, parentIndex);
-
-        } else {
-            root.add(this);
+            this.checkIsFiltered(parent, parentIndex, isNewChangeTree);
         }
 
         // assign same parent on child structures
-        const metadata: Metadata = this.ref.constructor[Symbol.metadata];
-        if (metadata) {
-            metadata[$refTypeFieldIndexes]?.forEach((index) => {
-                const field = metadata[index as any as number];
-                const changeTree: ChangeTree = this.ref[field.name]?.[$changes];
-                if (changeTree && changeTree.root !== root) {
-                    changeTree.setParent(this.ref, root, index);
-                }
-            });
-
-        } else if (this.ref[$childType] && typeof(this.ref[$childType]) !== "string") {
-            // MapSchema / ArraySchema, etc.
-            (this.ref as MapSchema).forEach((value, key) => {
-                const changeTree: ChangeTree = value[$changes];
-                if (changeTree.root !== root) {
-                    changeTree.setParent(this.ref, root, this.indexes[key] ?? key);
+        if (isNewChangeTree) {
+            //
+            // assign same parent on child structures
+            //
+            this.forEachChild((child, index) => {
+                if (child.root === root) {
+                    //
+                    // re-assigning a child of the same root, move it to the end
+                    // of the changes queue so encoding order is preserved
+                    //
+                    root.moveToEndOfChanges(child);
+                    return;
                 }
+                child.setParent(this.ref, root, index);
             });
         }
-
     }
 
-    forEachChild(callback: (change: ChangeTree, atIndex: number) => void) {
+    forEachChild(callback: (change: ChangeTree, at: any) => void) {
         //
         // assign same parent on child structures
         //
-        const metadata: Metadata = this.ref.constructor[Symbol.metadata];
-        if (metadata) {
-            metadata[$refTypeFieldIndexes]?.forEach((index) => {
-                const field = metadata[index as any as number];
-                const value = this.ref[field.name];
-                if (value) {
-                    callback(value[$changes], index);
-                }
-            });
+        if (this.ref[$childType]) {
+            if (typeof(this.ref[$childType]) !== "string") {
+                // MapSchema / ArraySchema, etc.
+                for (const [key, value] of (this.ref as MapSchema).entries()) {
+                    callback(value[$changes], key);
+                };
+            }
 
-        } else if (this.ref[$childType] && typeof(this.ref[$childType]) !== "string") {
-            // MapSchema / ArraySchema, etc.
-            (this.ref as MapSchema).forEach((value, key) => {
-                callback(value[$changes], this.indexes[key] ?? key);
-            });
+        } else {
+            for (const index of this.metadata?.[$refTypeFieldIndexes] ?? []) {
+                const field = this.metadata[index as any as number];
+                const value = this.ref[field.name];
+                if (!value) { continue; }
+                callback(value[$changes], index);
+            }
         }
     }
 
@@ -271,9 +291,7 @@ export class ChangeTree<T extends Ref=any> {
     }
 
     change(index: number, operation: OPERATION = OPERATION.ADD) {
-        const metadata = this.ref.constructor[Symbol.metadata] as Metadata;
-
-        const isFiltered = this.isFiltered || (metadata?.[index]?.tag !== undefined);
+        const isFiltered = this.isFiltered || (this.metadata?.[index]?.tag !== undefined);
         const changeSet = (isFiltered)
             ? this.filteredChanges
             : this.changes;
@@ -376,19 +394,16 @@ export class ChangeTree<T extends Ref=any> {
     }
 
     getType(index?: number) {
-        if (Metadata.isValidInstance(this.ref)) {
-            const metadata = this.ref.constructor[Symbol.metadata] as Metadata;
-            return metadata[index].type;
-
-        } else {
+        return (
             //
             // Get the child type from parent structure.
             // - ["string"] => "string"
             // - { map: "string" } => "string"
             // - { set: "string" } => "string"
             //
-            return this.ref[$childType];
-        }
+            this.ref[$childType] || // ArraySchema | MapSchema | SetSchema | CollectionSchema
+            this.metadata[index].type // Schema
+        );
     }
 
     getChange(index: number) {
@@ -458,9 +473,7 @@ export class ChangeTree<T extends Ref=any> {
         this.indexedOperations = {};
 
         // clear changeset
-        this[changeSetName].indexes = {};
-        this[changeSetName].operations.length = 0;
-        this[changeSetName].queueRootIndex = undefined;
+        this[changeSetName] = createChangeSet();
 
         // ArraySchema and MapSchema have a custom "encode end" method
         this.ref[$onEncodeEnd]?.();
@@ -478,24 +491,17 @@ export class ChangeTree<T extends Ref=any> {
         this.ref[$onEncodeEnd]?.();
 
         this.indexedOperations = {};
-
-        this.changes.indexes = {};
-        this.changes.operations.length = 0;
-        this.changes.queueRootIndex = undefined;
+        this.changes = createChangeSet();
 
         if (this.filteredChanges !== undefined) {
-            this.filteredChanges.indexes = {};
-            this.filteredChanges.operations.length = 0;
-            this.filteredChanges.queueRootIndex = undefined;
+            this.filteredChanges = createChangeSet();
         }
 
         if (discardAll) {
-            this.allChanges.indexes = {};
-            this.allChanges.operations.length = 0;
+            this.allChanges = createChangeSet();
 
             if (this.allFilteredChanges !== undefined) {
-                this.allFilteredChanges.indexes = {};
-                this.allFilteredChanges.operations.length = 0;
+                this.allFilteredChanges = createChangeSet();
             }
         }
     }
@@ -517,22 +523,11 @@ export class ChangeTree<T extends Ref=any> {
         this.discard();
     }
 
-    ensureRefId() {
-        // skip if refId is already set.
-        if (this.refId !== undefined) {
-            return;
-        }
-
-        this.refId = this.root.getNextUniqueId();
-    }
-
     get changed() {
         return (Object.entries(this.indexedOperations).length > 0);
     }
 
-    protected checkIsFiltered(parent: Ref, parentIndex: number) {
-        const isNewChangeTree = this.root.add(this);
-
+    protected checkIsFiltered(parent: Ref, parentIndex: number, isNewChangeTree: boolean) {
         if (this.root.types.hasFilters) {
             //
             // At Schema initialization, the "root" structure might not be available
@@ -545,7 +540,7 @@ export class ChangeTree<T extends Ref=any> {
             if (this.filteredChanges !== undefined) {
                 enqueueChangeTree(this.root, this, 'filteredChanges');
                 if (isNewChangeTree) {
-                    this.root.allFilteredChanges.push(this);
+                    enqueueChangeTree(this.root, this, 'allFilteredChanges');
                 }
             }
         }
@@ -553,7 +548,7 @@ export class ChangeTree<T extends Ref=any> {
         if (!this.isFiltered) {
             enqueueChangeTree(this.root, this, 'changes');
             if (isNewChangeTree) {
-                this.root.allChanges.push(this);
+                enqueueChangeTree(this.root, this, 'allChanges');
             }
         }
     }
@@ -568,7 +563,7 @@ export class ChangeTree<T extends Ref=any> {
         //
         const refType = Metadata.isValidInstance(this.ref)
             ? this.ref.constructor
-            :  this.ref[$childType];
+            : this.ref[$childType];
 
         let parentChangeTree: ChangeTree;
 
@@ -627,4 +622,95 @@ export class ChangeTree<T extends Ref=any> {
         }
     }
 
-}
+    /**
+     * Get the immediate parent
+     */
+    get parent(): Ref | undefined {
+        return this.parentChain?.ref;
+    }
+
+    /**
+     * Get the immediate parent index
+     */
+    get parentIndex(): number | undefined {
+        return this.parentChain?.index;
+    }
+
+    /**
+     * Add a parent to the chain
+     */
+    addParent(parent: Ref, index: number) {
+        // Check if this parent already exists in the chain
+        if (this.hasParent((p, i) => p === parent && i === index)) {
+            return;
+        }
+
+        this.parentChain = {
+            ref: parent,
+            index,
+            next: this.parentChain
+        };
+    }
+
+    /**
+     * Remove a parent from the chain
+     * @param parent - The parent to remove
+     * @returns true if parent was removed
+     */
+    removeParent(parent: Ref): boolean {
+        let current = this.parentChain;
+        let previous = null;
+        while (current) {
+            //
+            // FIXME: it is required to check against `$changes` here because
+            // ArraySchema is instance of Proxy
+            //
+            if (current.ref[$changes] === parent[$changes]) {
+                if (previous) {
+                    previous.next = current.next;
+                } else {
+                    this.parentChain = current.next;
+                }
+                return true;
+            }
+            previous = current;
+            current = current.next;
+        }
+        return this.parentChain === undefined;
+    }
+
+    /**
+     * Find a specific parent in the chain
+     */
+    findParent(predicate: (parent: Ref, index: number) => boolean): ParentChain | undefined {
+        let current = this.parentChain;
+        while (current) {
+            if (predicate(current.ref, current.index)) {
+                return current;
+            }
+            current = current.next;
+        }
+        return undefined;
+    }
+
+    /**
+     * Check if this ChangeTree has a specific parent
+     */
+    hasParent(predicate: (parent: Ref, index: number) => boolean): boolean {
+        return this.findParent(predicate) !== undefined;
+    }
+
+    /**
+     * Get all parents as an array (for debugging/testing)
+     */
+    getAllParents(): Array<{ref: Ref, index: number}> {
+        const parents: Array<{ref: Ref, index: number}> = [];
+        let current = this.parentChain;
+        while (current) {
+            parents.push({ref: current.ref, index: current.index});
+            current = current.next;
+        }
+        return parents;
+    }
+
+}