@colyseus/schema 3.0.42 → 3.0.44

package/lib/utils.js

@@ -15,10 +15,13 @@ function dumpChanges(schema) {
         refs: []
     };
     // for (const refId in $root.changes) {
-    $root.changes.forEach(changeTree => {
+    let current = $root.changes.next;
+    while (current) {
+        const changeTree = current.changeTree;
         // skip if ChangeTree is undefined
         if (changeTree === undefined) {
-            return;
+            current = current.next;
+            continue;
         }
         const changes = changeTree.indexedOperations;
         dump.refs.push(`refId#${changeTree.refId}`);
@@ -30,7 +33,8 @@ function dumpChanges(schema) {
             }
             dump.ops[spec_1.OPERATION[op]]++;
         }
-    });
+        current = current.next;
+    }
     return dump;
 }
 function getNextPowerOf2(number) {

package/lib/utils.js.map

@@ -1 +1 @@
-{"version":3,"file":"utils.js","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":";;AAeA,8BAIC;AAED,kCAyBC;AAED,0CAeC;AA9DD,0CAA4C;AAC5C,6CAA2C;AAa3C,SAAgB,SAAS,CAAC,KAAa;IACnC,OAAO,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAC3C,CAAC,CAAC,KAAK,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CACpC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;AACf,CAAC;AAED,SAAgB,WAAW,CAAC,MAAc;IACtC,MAAM,KAAK,GAAG,MAAM,CAAC,kBAAQ,CAAC,CAAC,IAAI,CAAC;IAEpC,MAAM,IAAI,GAAe;QACrB,GAAG,EAAE,EAAE;QACP,IAAI,EAAE,EAAE;KACX,CAAC;IAEF,uCAAuC;IACvC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE;QAC/B,kCAAkC;QAClC,IAAI,UAAU,KAAK,SAAS,EAAG,CAAC;YAAC,OAAO;QAAC,CAAC;QAE1C,MAAM,OAAO,GAAG,UAAU,CAAC,iBAAiB,CAAC;QAE7C,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,UAAU,CAAC,KAAK,EAAE,CAAC,CAAC;QAC5C,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC1B,MAAM,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;YAC1B,MAAM,MAAM,GAAG,gBAAS,CAAC,EAAE,CAAC,CAAC;YAC7B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;gBAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAAC,CAAC;YAChD,IAAI,CAAC,GAAG,CAAC,gBAAS,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;QAC9B,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,OAAO,IAAI,CAAC;AAChB,CAAC;AAED,SAAgB,eAAe,CAAC,MAAc;IAC1C,+CAA+C;IAC/C,IAAI,CAAC,MAAM,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;QAChC,OAAO,MAAM,CAAC;IAClB,CAAC;IAED,gDAAgD;IAChD,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,OAAO,MAAM,GAAG,CAAC,EAAE,CAAC;QAChB,MAAM,KAAK,CAAC,CAAC;QACb,WAAW,EAAE,CAAC;IAClB,CAAC;IAED,6BAA6B;IAC7B,OAAO,CAAC,IAAI,WAAW,CAAC;AAC5B,CAAC","sourcesContent":["import type { Schema } from \"./Schema\";\nimport { OPERATION } from \"./encoding/spec\";\nimport { $changes } from \"./types/symbols\";\n\ntype ChangeItem = [string, number | string, any?];\n\ninterface ChangeDump {\n    ops: {\n        ADD?: number;\n        REMOVE?: number;\n        REPLACE?: number;\n    },\n    refs: string[],\n}\n\nexport function getIndent(level: number) {\n    return (new Array(level).fill(0)).map((_, i) =>\n        (i === level - 1) ? `└─ ` : `   `\n    ).join(\"\");\n}\n\nexport function dumpChanges(schema: Schema) {\n    const $root = schema[$changes].root;\n\n    const dump: ChangeDump = {\n        ops: {},\n        refs: []\n    };\n\n    // for (const refId in $root.changes) {\n    $root.changes.forEach(changeTree => {\n        // skip if ChangeTree is undefined\n        if (changeTree === undefined)  { return; }\n\n        const changes = changeTree.indexedOperations;\n\n        dump.refs.push(`refId#${changeTree.refId}`);\n        for (const index in changes) {\n            const op = changes[index];\n            const opName = OPERATION[op];\n            if (!dump.ops[opName]) { dump.ops[opName] = 0; }\n            dump.ops[OPERATION[op]]++;\n        }\n    });\n\n    return dump;\n}\n\nexport function getNextPowerOf2(number: number) {\n    // If number is already a power of 2, return it\n    if ((number & (number - 1)) === 0) {\n        return number;\n    }\n\n    // Find the position of the most significant bit\n    let msbPosition = 0;\n    while (number > 0) {\n        number >>= 1;\n        msbPosition++;\n    }\n\n    // Return the next power of 2\n    return 1 << msbPosition;\n}"]}
+{"version":3,"file":"utils.js","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":";;AAeA,8BAIC;AAED,kCA+BC;AAED,0CAeC;AApED,0CAA4C;AAC5C,6CAA2C;AAa3C,SAAgB,SAAS,CAAC,KAAa;IACnC,OAAO,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAC3C,CAAC,CAAC,KAAK,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CACpC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;AACf,CAAC;AAED,SAAgB,WAAW,CAAC,MAAc;IACtC,MAAM,KAAK,GAAG,MAAM,CAAC,kBAAQ,CAAC,CAAC,IAAI,CAAC;IAEpC,MAAM,IAAI,GAAe;QACrB,GAAG,EAAE,EAAE;QACP,IAAI,EAAE,EAAE;KACX,CAAC;IAEF,uCAAuC;IACvC,IAAI,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;IACjC,OAAO,OAAO,EAAE,CAAC;QACb,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;QACtC,kCAAkC;QAClC,IAAI,UAAU,KAAK,SAAS,EAAG,CAAC;YAC5B,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;YACvB,SAAS;QACb,CAAC;QAED,MAAM,OAAO,GAAG,UAAU,CAAC,iBAAiB,CAAC;QAE7C,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,UAAU,CAAC,KAAK,EAAE,CAAC,CAAC;QAC5C,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC1B,MAAM,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;YAC1B,MAAM,MAAM,GAAG,gBAAS,CAAC,EAAE,CAAC,CAAC;YAC7B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;gBAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAAC,CAAC;YAChD,IAAI,CAAC,GAAG,CAAC,gBAAS,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;QAC9B,CAAC;QACD,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAC3B,CAAC;IAED,OAAO,IAAI,CAAC;AAChB,CAAC;AAED,SAAgB,eAAe,CAAC,MAAc;IAC1C,+CAA+C;IAC/C,IAAI,CAAC,MAAM,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;QAChC,OAAO,MAAM,CAAC;IAClB,CAAC;IAED,gDAAgD;IAChD,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,OAAO,MAAM,GAAG,CAAC,EAAE,CAAC;QAChB,MAAM,KAAK,CAAC,CAAC;QACb,WAAW,EAAE,CAAC;IAClB,CAAC;IAED,6BAA6B;IAC7B,OAAO,CAAC,IAAI,WAAW,CAAC;AAC5B,CAAC","sourcesContent":["import type { Schema } from \"./Schema\";\nimport { OPERATION } from \"./encoding/spec\";\nimport { $changes } from \"./types/symbols\";\n\ntype ChangeItem = [string, number | string, any?];\n\ninterface ChangeDump {\n    ops: {\n        ADD?: number;\n        REMOVE?: number;\n        REPLACE?: number;\n    },\n    refs: string[],\n}\n\nexport function getIndent(level: number) {\n    return (new Array(level).fill(0)).map((_, i) =>\n        (i === level - 1) ? `└─ ` : `   `\n    ).join(\"\");\n}\n\nexport function dumpChanges(schema: Schema) {\n    const $root = schema[$changes].root;\n\n    const dump: ChangeDump = {\n        ops: {},\n        refs: []\n    };\n\n    // for (const refId in $root.changes) {\n    let current = $root.changes.next;\n    while (current) {\n        const changeTree = current.changeTree;\n        // skip if ChangeTree is undefined\n        if (changeTree === undefined)  {\n            current = current.next;\n            continue;\n        }\n\n        const changes = changeTree.indexedOperations;\n\n        dump.refs.push(`refId#${changeTree.refId}`);\n        for (const index in changes) {\n            const op = changes[index];\n            const opName = OPERATION[op];\n            if (!dump.ops[opName]) { dump.ops[opName] = 0; }\n            dump.ops[OPERATION[op]]++;\n        }\n        current = current.next;\n    }\n\n    return dump;\n}\n\nexport function getNextPowerOf2(number: number) {\n    // If number is already a power of 2, return it\n    if ((number & (number - 1)) === 0) {\n        return number;\n    }\n\n    // Find the position of the most significant bit\n    let msbPosition = 0;\n    while (number > 0) {\n        number >>= 1;\n        msbPosition++;\n    }\n\n    // Return the next power of 2\n    return 1 << msbPosition;\n}"]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@colyseus/schema",
-  "version": "3.0.42",
+  "version": "3.0.44",
   "description": "Binary state serializer with delta encoding for games",
   "bin": {
     "schema-codegen": "bin/schema-codegen",
@@ -66,6 +66,7 @@
     "@types/rimraf": "^2.0.3",
     "@types/sinon": "^7.0.3",
     "benchmark": "^2.1.4",
+    "core-js": "^3.44.0",
     "flatbuffers": "^1.10.2",
     "fossil-delta": "^1.0.2",
     "glob": "^7.1.5",

package/src/Schema.ts

@@ -3,7 +3,7 @@ import { DEFAULT_VIEW_TAG, type DefinitionType } from "./annotations";
 
 import { NonFunctionPropNames, ToJSON } from './types/HelperTypes';
 
-import { ChangeSet, ChangeTree, Ref } from './encoder/ChangeTree';
+import { ChangeSet, ChangeSetName, ChangeTree, Ref } from './encoder/ChangeTree';
 import { $changes, $decoder, $deleteByIndex, $descriptors, $encoder, $filter, $getByIndex, $track } from './types/symbols';
 import { StateView } from './encoder/StateView';
 
@@ -185,7 +185,7 @@ export class Schema {
      * @param showContents display JSON contents of the instance
      * @returns
      */
-    static debugRefIds(ref: Ref, showContents: boolean = false, level: number = 0, decoder?: Decoder) {
+    static debugRefIds(ref: Ref, showContents: boolean = false, level: number = 0, decoder?: Decoder, keyPrefix: string = "") {
         const contents = (showContents) ? ` - ${JSON.stringify(ref.toJSON())}` : "";
         const changeTree: ChangeTree = ref[$changes];
 
@@ -197,15 +197,29 @@ export class Schema {
             ? ` [×${root.refCount[refId]}]`
             : '';
 
-        let output = `${getIndent(level)}${ref.constructor.name} (refId: ${refId})${refCount}${contents}\n`;
+        let output = `${getIndent(level)}${keyPrefix}${ref.constructor.name} (refId: ${refId})${refCount}${contents}\n`;
 
-        changeTree.forEachChild((childChangeTree) =>
-            output += this.debugRefIds(childChangeTree.ref, showContents, level + 1, decoder));
+        changeTree.forEachChild((childChangeTree, key) => {
+            const keyPrefix = (ref['forEach'] !== undefined && key !== undefined) ? `["${key}"]: ` : "";
+            output += this.debugRefIds(childChangeTree.ref, showContents, level + 1, decoder, keyPrefix);
+        });
 
         return output;
     }
 
-    static debugRefIdsDecoder(decoder: Decoder) {
+    static debugRefIdEncodingOrder(ref: Ref, changeSet: ChangeSetName = 'allChanges') {
+        let encodeOrder: number[] = [];
+        let current = ref[$changes].root[changeSet].next;
+        while (current) {
+            if (current.changeTree) {
+                encodeOrder.push(current.changeTree.refId);
+            }
+            current = current.next;
+        }
+        return encodeOrder;
+    }
+
+    static debugRefIdsFromDecoder(decoder: Decoder) {
         return this.debugRefIds(decoder.state, false, 0, decoder);
     }
 
@@ -263,15 +277,17 @@ export class Schema {
     static debugChangesDeep(ref: Ref, changeSetName: "changes" | "allChanges" | "allFilteredChanges" | "filteredChanges" = "changes") {
         let output = "";
 
-        const rootChangeTree = ref[$changes];
+        const rootChangeTree: ChangeTree = ref[$changes];
         const root = rootChangeTree.root;
         const changeTrees: Map<ChangeTree, ChangeTree[]> = new Map();
 
         const instanceRefIds = [];
         let totalOperations = 0;
 
+        // TODO: FIXME: this method is not working as expected
         for (const [refId, changes] of Object.entries(root[changeSetName])) {
             const changeTree = root.changeTrees[refId];
+            if (!changeTree) { continue; }
 
             let includeChangeTree = false;
             let parentChangeTrees: ChangeTree[] = [];

package/src/bench_encode.ts

@@ -0,0 +1,108 @@
+import { nanoid } from "nanoid";
+import { Schema, type, MapSchema, ArraySchema, Encoder } from ".";
+
+class Attribute extends Schema {
+    @type("string") name: string;
+    @type("number") value: number;
+}
+
+class Item extends Schema {
+    @type("number") price: number;
+    @type([ Attribute ]) attributes = new ArraySchema<Attribute>();
+}
+
+class Position extends Schema {
+    @type("number") x: number;
+    @type("number") y: number;
+}
+
+class Player extends Schema {
+    @type(Position) position = new Position();
+    @type({ map: Item }) items = new MapSchema<Item>();
+}
+
+class State extends Schema {
+    @type({ map: Player }) players = new MapSchema<Player>();
+    @type("string") currentTurn;
+}
+
+const state = new State();
+
+Encoder.BUFFER_SIZE = 4096 * 4096;
+const encoder = new Encoder(state);
+
+
+let now = Date.now();
+
+// for (let i = 0; i < 10000; i++) {
+//     const player = new Player();
+//     state.players.set(`p-${nanoid()}`, player);
+//
+//     player.position.x = (i + 1) * 100;
+//     player.position.y = (i + 1) * 100;
+//     for (let j = 0; j < 10; j++) {
+//         const item = new Item();
+//         player.items.set(`item-${j}`, item);
+//         item.price = (i + 1) * 50;
+//         for (let k = 0; k < 5; k++) {
+//             const attr = new Attribute();
+//             attr.name = `Attribute ${k}`;
+//             attr.value = k;
+//             item.attributes.push(attr);
+//         }
+//     }
+// }
+// console.log("time to make changes:", Date.now() - now);
+
+
+// measure time to .encodeAll()
+
+now = Date.now();
+// for (let i = 0; i < 1000; i++) {
+//     encoder.encodeAll();
+// }
+// console.log(Date.now() - now);
+
+const total = 100;
+const allEncodes = Date.now();
+
+let avgTimeToEncode = 0;
+let avgTimeToMakeChanges = 0;
+
+for (let i = 0; i < total; i++) {
+    now = Date.now();
+    for (let j = 0; j < 50; j++) {
+        const player = new Player();
+        state.players.set(`p-${nanoid()}`, player);
+
+        player.position.x = (j + 1) * 100;
+        player.position.y = (j + 1) * 100;
+        for (let k = 0; k < 10; k++) {
+            const item = new Item();
+            item.price = (j + 1) * 50;
+            for (let l = 0; l < 5; l++) {
+                const attr = new Attribute();
+                attr.name = `Attribute ${l}`;
+                attr.value = l;
+                item.attributes.push(attr);
+            }
+            player.items.set(`item-${k}`, item);
+        }
+    }
+    const timeToMakeChanges = Date.now() - now;
+    console.log("time to make changes:", timeToMakeChanges);
+    avgTimeToMakeChanges += timeToMakeChanges;
+
+    now = Date.now();
+    encoder.encode();
+    encoder.discardChanges();
+
+    const timeToEncode = Date.now() - now;
+    console.log("time to encode:", timeToEncode);
+    avgTimeToEncode += timeToEncode;
+}
+console.log("avg time to encode:", (avgTimeToEncode) / total);
+console.log("avg time to make changes:", (avgTimeToMakeChanges) / total);
+console.log("time for all encodes:", Date.now() - allEncodes);
+
+console.log(Array.from(encoder.encodeAll()).length, "bytes");

package/src/debug.ts

@@ -0,0 +1,55 @@
+import * as fs from "fs";
+import { Reflection, Decoder } from "./index";
+
+const contents = fs.readFileSync("/Users/endel/Projects/colyseus/clients/bubbits/project/@bubbits/backend/schema-debug.txt", { encoding: "utf8" }).toString();
+
+let isCommentBlock = false;
+let lastComment = "";
+
+let decoder: Decoder;
+
+function getBuffer(line: string) {
+    const start = line.lastIndexOf(":");
+    const buffer = Buffer.from(new Uint8Array(line.substring(start + 1).split(",").map(n => Number(n))));
+    console.log(`(${buffer.byteLength}) ${Array.from(buffer).join(",")}`)
+    // console.log("");
+    // console.log("");
+    // console.log("> ", line);
+    // console.log("> substring:", line.substring(start + 1))
+    return buffer;
+}
+
+function decode(buffer: Buffer) {
+    try {
+        decoder.decode(buffer);
+    } catch (e) {
+        console.error(e);
+        console.log("Last log:\n\n")
+        console.log(lastComment);
+    }
+}
+
+contents.split("\n").forEach((line) => {
+    if (line.startsWith("#")) {
+        // reset last comment.
+        if (isCommentBlock === false) { lastComment = ""; }
+
+        isCommentBlock = true;
+        lastComment += line.substring(line.indexOf(":") + 1) + "\n";
+        return;
+    }
+
+    isCommentBlock = false;
+
+    if (line.startsWith("handshake:") && !decoder) {
+        decoder = Reflection.decode(getBuffer(line));
+
+    } else if (line.startsWith("state:")) {
+        decode(getBuffer(line));
+
+    } else if (line.startsWith("patch:")) {
+        decode(getBuffer(line));
+    }
+});
+
+console.log(decoder.state.toJSON());

package/src/decoder/Decoder.ts

@@ -58,6 +58,8 @@ export class Decoder<T extends Schema = any> {
             if (bytes[it.offset] == SWITCH_TO_STRUCTURE) {
                 it.offset++;
 
+                ref[$onDecodeEnd]?.()
+
                 const nextRefId = decode.number(bytes, it);
                 const nextRef = $root.refs.get(nextRefId);
 
@@ -65,17 +67,17 @@ export class Decoder<T extends Schema = any> {
                 // Trying to access a reference that haven't been decoded yet.
                 //
                 if (!nextRef) {
+                    throw new Error(`"refId" not found: ${nextRefId}`);
                     console.error(`"refId" not found: ${nextRefId}`, { previousRef: ref, previousRefId: this.currentRefId });
                     console.warn("Please report this to the developers. All refIds =>");
-                    console.warn(Schema.debugRefIdsDecoder(this));
+                    console.warn(Schema.debugRefIdsFromDecoder(this));
                     this.skipCurrentStructure(bytes, it, totalBytes);
-                }
-                ref[$onDecodeEnd]?.()
 
-                this.currentRefId = nextRefId;
-
-                ref = nextRef;
-                decoder = ref.constructor[$decoder];
+                } else {
+                    ref = nextRef;
+                    decoder = ref.constructor[$decoder];
+                    this.currentRefId = nextRefId;
+                }
 
                 continue;
             }
@@ -131,12 +133,6 @@ export class Decoder<T extends Schema = any> {
     }
 
     createInstanceOfType (type: typeof Schema): Schema {
-        // let instance: Schema = new (type as any)();
-
-        // // assign root on $changes
-        // instance[$changes].root = this.root[$changes].root;
-
-        // return instance;
         return new (type as any)();
     }