@colyseus/schema 3.0.0-alpha.3 → 3.0.0-alpha.31

package/src/encoder/EncodeOperation.ts

@@ -3,7 +3,7 @@ import { $changes } from "../types/symbols";
 import { getType } from "../types/registry";
 
 import * as encode from "../encoding/encode";
-import { EncodeSchemaError, assertInstanceType, assertType } from "../encoding/assert";
+// import { EncodeSchemaError, assertInstanceType, assertType } from "../encoding/assert";
 
 import type { ChangeTree, Ref } from "./ChangeTree";
 import type { Encoder } from "./Encoder";
@@ -12,6 +12,7 @@ import type { PrimitiveType } from "../annotations";
 
 import type { Iterator } from "../encoding/decode";
 import type { ArraySchema } from "../types/custom/ArraySchema";
+import type { Metadata } from "../Metadata";
 
 export type EncodeOperation<T extends Ref = any> = (
     encoder: Encoder,
@@ -24,40 +25,27 @@ export type EncodeOperation<T extends Ref = any> = (
     hasView: boolean,
 ) => void;
 
-export function encodePrimitiveType(
-    type: PrimitiveType,
-    bytes: Buffer,
-    value: any,
-    klass: Schema,
-    field: string | number,
-    it: Iterator,
-) {
-    assertType(value, type as string, klass, field);
-
-    const encodeFunc = encode[type as string];
-
-    if (encodeFunc) {
-        encodeFunc(bytes, value, it);
-        // encodeFunc(bytes, value);
-
-    } else {
-        throw new EncodeSchemaError(`a '${type}' was expected, but ${value} was provided in ${klass.constructor.name}#${field}`);
-    }
-};
-
 export function encodeValue(
     encoder: Encoder,
     bytes: Buffer,
-    ref: Ref,
+    // ref: Ref,
     type: any,
     value: any,
-    field: string | number,
+    // field: string | number,
     operation: OPERATION,
     it: Iterator,
 ) {
-    if (type[Symbol.metadata] !== undefined) {
-        // TODO: move this to the `@type()` annotation
-        assertInstanceType(value, type as typeof Schema, ref as Schema, field);
+    if (typeof (type) === "string") {
+        //
+        // Primitive values
+        //
+        // assertType(value, type as string, ref as Schema, field);
+
+        encode[type]?.(bytes, value, it);
+
+    } else if (type[Symbol.metadata] !== undefined) {
+        // // TODO: move this to the `@type()` annotation
+        // assertInstanceType(value, type as typeof Schema, ref as Schema, field);
 
         //
         // Encode refId for this instance.
@@ -70,22 +58,16 @@ export function encodeValue(
             encoder.tryEncodeTypeId(bytes, type as typeof Schema, value.constructor as typeof Schema, it);
         }
 
-    } else if (typeof (type) === "string") {
-        //
-        // Primitive values
-        //
-        encodePrimitiveType(type as PrimitiveType, bytes, value, ref as Schema, field, it);
-
     } else {
-        //
-        // Custom type (MapSchema, ArraySchema, etc)
-        //
-        const definition = getType(Object.keys(type)[0]);
+        // //
+        // // Custom type (MapSchema, ArraySchema, etc)
+        // //
+        // const definition = getType(Object.keys(type)[0]);
 
-        //
-        // ensure a ArraySchema has been provided
-        //
-        assertInstanceType(ref[field], definition.constructor, ref as Schema, field);
+        // //
+        // // ensure a ArraySchema has been provided
+        // //
+        // assertInstanceType(ref[field], definition.constructor, ref as Schema, field);
 
         //
         // Encode refId for this instance.
@@ -107,13 +89,6 @@ export const encodeSchemaOperation: EncodeOperation = function (
     operation: OPERATION,
     it: Iterator,
 ) {
-    const ref = changeTree.ref;
-    const metadata = ref['constructor'][Symbol.metadata];
-
-    const field = metadata[index];
-    const type = metadata[field].type;
-    const value = ref[field];
-
     // "compress" field index + operation
     bytes[it.offset++] = (index | operation) & 255;
 
@@ -122,8 +97,21 @@ export const encodeSchemaOperation: EncodeOperation = function (
         return;
     }
 
+    const ref = changeTree.ref;
+    const metadata: Metadata = ref['constructor'][Symbol.metadata];
+    const field = metadata[index];
+
     // TODO: inline this function call small performance gain
-    encodeValue(encoder, bytes, ref, type, value, field, operation, it);
+    encodeValue(
+        encoder,
+        bytes,
+        // ref,
+        metadata[index].type,
+        ref[field.name],
+        // index,
+        operation,
+        it
+    );
 }
 
 /**
@@ -134,12 +122,10 @@ export const encodeKeyValueOperation: EncodeOperation = function (
     encoder: Encoder,
     bytes: Buffer,
     changeTree: ChangeTree,
-    field: number,
+    index: number,
     operation: OPERATION,
     it: Iterator,
 ) {
-    const ref = changeTree.ref;
-
     // encode operation
     bytes[it.offset++] = operation & 255;
 
@@ -149,13 +135,15 @@ export const encodeKeyValueOperation: EncodeOperation = function (
     }
 
     // encode index
-    encode.number(bytes, field, it);
+    encode.number(bytes, index, it);
 
     // Do not encode value for DELETE operations
     if (operation === OPERATION.DELETE) {
         return;
     }
 
+    const ref = changeTree.ref;
+
     //
     // encode "alias" for dynamic fields (maps)
     //
@@ -164,16 +152,38 @@ export const encodeKeyValueOperation: EncodeOperation = function (
             //
             // MapSchema dynamic key
             //
-            const dynamicIndex = changeTree.ref['$indexes'].get(field);
+            const dynamicIndex = changeTree.ref['$indexes'].get(index);
             encode.string(bytes, dynamicIndex, it);
         }
     }
 
-    const type = changeTree.getType(field);
-    const value = changeTree.getValue(field);
+    const type = changeTree.getType(index);
+    const value = changeTree.getValue(index);
+
+    // try { throw new Error(); } catch (e) {
+    //     // only print if not coming from Reflection.ts
+    //     if (!e.stack.includes("src/Reflection.ts")) {
+    //         console.log("encodeKeyValueOperation -> ", {
+    //             ref: changeTree.ref.constructor.name,
+    //             field,
+    //             operation: OPERATION[operation],
+    //             value: value?.toJSON(),
+    //             items: ref.toJSON(),
+    //         });
+    //     }
+    // }
 
     // TODO: inline this function call small performance gain
-    encodeValue(encoder, bytes, ref, type, value, field, operation, it);
+    encodeValue(
+        encoder,
+        bytes,
+        // ref,
+        type,
+        value,
+        // index,
+        operation,
+        it
+    );
 }
 
 /**
@@ -191,18 +201,22 @@ export const encodeArray: EncodeOperation = function (
     hasView: boolean,
 ) {
     const ref = changeTree.ref;
+    const useOperationByRefId = hasView && changeTree.isFiltered && (typeof (changeTree.getType(field)) !== "string");
 
-    if (
-        hasView &&
-        operation === OPERATION.DELETE &&
-        typeof (changeTree.getType(field)) !== "string"
-    ) {
-        // encode delete by refId (array of schemas)
-        bytes[it.offset++] = OPERATION.DELETE_BY_REFID;
-        const value =  ref['tmpItems'][field];
-        const refId = value[$changes].refId;
-        encode.number(bytes, refId, it);
-        return;
+    let refOrIndex: number;
+
+    if (useOperationByRefId) {
+        refOrIndex = ref['tmpItems'][field][$changes].refId;
+
+        if (operation === OPERATION.DELETE) {
+            operation = OPERATION.DELETE_BY_REFID;
+
+        } else if (operation === OPERATION.ADD) {
+            operation = OPERATION.ADD_BY_REFID;
+        }
+
+    } else {
+        refOrIndex = field;
     }
 
     // encode operation
@@ -214,7 +228,7 @@ export const encodeArray: EncodeOperation = function (
     }
 
     // encode index
-    encode.number(bytes, field, it);
+    encode.number(bytes, refOrIndex, it);
 
     // Do not encode value for DELETE operations
     if (operation === OPERATION.DELETE) {
@@ -233,5 +247,14 @@ export const encodeArray: EncodeOperation = function (
     // });
 
     // TODO: inline this function call small performance gain
-    encodeValue(encoder, bytes, ref, type, value, field, operation, it);
+    encodeValue(
+        encoder,
+        bytes,
+        // ref,
+        type,
+        value,
+        // field,
+        operation,
+        it
+    );
 }

package/src/encoder/Encoder.ts

@@ -1,14 +1,17 @@
 import type { Schema } from "../Schema";
-import { TypeContext } from "../annotations";
-import { $changes, $encoder, $filter } from "../types/symbols";
+import { TypeContext } from "../types/TypeContext";
+import { $changes, $encoder, $filter, $isNew, $onEncodeEnd } from "../types/symbols";
 
 import * as encode from "../encoding/encode";
 import type { Iterator } from "../encoding/decode";
 
-import { SWITCH_TO_STRUCTURE, TYPE_ID } from '../encoding/spec';
-import { Root } from "./ChangeTree";
+import { OPERATION, SWITCH_TO_STRUCTURE, TYPE_ID } from '../encoding/spec';
+import { Root } from "./Root";
 import { getNextPowerOf2 } from "../utils";
+
 import type { StateView } from "./StateView";
+import type { Metadata } from "../Metadata";
+import type { ChangeTree } from "./ChangeTree";
 
 export class Encoder<T extends Schema = any> {
     static BUFFER_SIZE = 8 * 1024;// 8KB
@@ -19,14 +22,16 @@ export class Encoder<T extends Schema = any> {
 
     root: Root;
 
-    constructor(root: T) {
-        this.setRoot(root);
+    constructor(state: T) {
 
         //
         // TODO: cache and restore "Context" based on root schema
         // (to avoid creating a new context for every new room)
         //
-        this.context = new TypeContext(root.constructor as typeof Schema);
+        this.context = new TypeContext(state.constructor as typeof Schema);
+        this.root = new Root(this.context);
+
+        this.setState(state);
 
         // console.log(">>>>>>>>>>>>>>>> Encoder types");
         // this.context.schemas.forEach((id, schema) => {
@@ -34,33 +39,38 @@ export class Encoder<T extends Schema = any> {
         // });
     }
 
-    protected setRoot(state: T) {
-        this.root = new Root();
+    protected setState(state: T) {
         this.state = state;
-        state[$changes].setRoot(this.root);
+        this.state[$changes].setRoot(this.root);
     }
 
     encode(
         it: Iterator = { offset: 0 },
         view?: StateView,
-        bytes = this.sharedBuffer,
-        changeTrees = this.root.changes
+        buffer = this.sharedBuffer,
+        changeTrees = this.root.changes,
+        isEncodeAll = this.root.allChanges === changeTrees,
+        initialOffset = it.offset // cache current offset in case we need to resize the buffer
     ): Buffer {
-        const initialOffset = it.offset; // cache current offset in case we need to resize the buffer
-
-        const isEncodeAll = this.root.allChanges === changeTrees;
         const hasView = (view !== undefined);
         const rootChangeTree = this.state[$changes];
 
-        const changeTreesIterator = changeTrees.entries();
+        const shouldClearChanges = !isEncodeAll && !hasView;
 
-        for (const [changeTree, changes] of changeTreesIterator) {
+        for (const [changeTree, changes] of changeTrees.entries()) {
             const ref = changeTree.ref;
 
             const ctor = ref['constructor'];
             const encoder = ctor[$encoder];
             const filter = ctor[$filter];
 
+            // try { throw new Error(); } catch (e) {
+            //     // only print if not coming from Reflection.ts
+            //     if (!e.stack.includes("src/Reflection.ts")) {
+            //         console.log("ChangeTree:", { ref: ref.constructor.name, });
+            //     }
+            // }
+
             if (hasView) {
                 if (!view.items.has(changeTree)) {
                     view.invisible.add(changeTree);
@@ -72,9 +82,10 @@ export class Encoder<T extends Schema = any> {
             }
 
             // skip root `refId` if it's the first change tree
-            if (it.offset !== initialOffset || changeTree !== rootChangeTree) {
-                bytes[it.offset++] = SWITCH_TO_STRUCTURE & 255;
-                encode.number(bytes, changeTree.refId, it);
+            // (unless it "hasView", which will need to revisit the root)
+            if (hasView || it.offset > initialOffset || changeTree !== rootChangeTree) {
+                buffer[it.offset++] = SWITCH_TO_STRUCTURE & 255;
+                encode.number(buffer, changeTree.refId, it);
             }
 
             const changesIterator = changes.entries();
@@ -88,91 +99,115 @@ export class Encoder<T extends Schema = any> {
                 // TODO: avoid checking if no view tags were defined
                 //
                 if (filter && !filter(ref, fieldIndex, view)) {
-                    // console.log("SKIP FIELD:", { ref: changeTree.ref.constructor.name, fieldIndex, })
-
                     // console.log("ADD AS INVISIBLE:", fieldIndex, changeTree.ref.constructor.name)
                     // view?.invisible.add(changeTree);
                     continue;
                 }
 
-                // console.log("WILL ENCODE", {
-                //     ref: changeTree.ref.constructor.name,
-                //     fieldIndex,
-                //     operation: OPERATION[operation],
-                // });
-
-                encoder(this, bytes, changeTree, fieldIndex, operation, it, isEncodeAll, hasView);
+                // try { throw new Error(); } catch (e) {
+                //     // only print if not coming from Reflection.ts
+                //     if (!e.stack.includes("src/Reflection.ts")) {
+                //         console.log("WILL ENCODE", {
+                //             ref: changeTree.ref.constructor.name,
+                //             fieldIndex,
+                //             operation: OPERATION[operation],
+                //         });
+                //     }
+                // }
+
+                encoder(this, buffer, changeTree, fieldIndex, operation, it, isEncodeAll, hasView);
             }
+
+            // if (shouldClearChanges) {
+            //     changeTree.endEncode();
+            // }
         }
 
-        if (it.offset > bytes.byteLength) {
-            const newSize = getNextPowerOf2(this.sharedBuffer.byteLength * 2);
-            console.warn("@colyseus/schema encode buffer overflow. Current buffer size: " + bytes.byteLength + ", encoding offset: " + it.offset + ", new size: " + newSize);
+        if (it.offset > buffer.byteLength) {
+            const newSize = getNextPowerOf2(buffer.byteLength * 2);
+            console.warn(`@colyseus/schema buffer overflow. Encoded state is higher than default BUFFER_SIZE. Use the following to increase default BUFFER_SIZE:
+
+    import { Encoder } from "@colyseus/schema";
+    Encoder.BUFFER_SIZE = ${Math.round(newSize / 1024)} * 1024; // ${Math.round(newSize / 1024)} KB
+`);
 
             //
             // resize buffer and re-encode (TODO: can we avoid re-encoding here?)
             //
-            this.sharedBuffer = Buffer.allocUnsafeSlow(newSize);
-            return this.encode({ offset: initialOffset }, view);
+            buffer = Buffer.allocUnsafeSlow(newSize);
+
+            // assign resized buffer to local sharedBuffer
+            if (buffer === this.sharedBuffer) {
+                this.sharedBuffer = buffer;
+            }
+
+            return this.encode({ offset: initialOffset }, view, buffer, changeTrees, isEncodeAll);
 
         } else {
             //
             // only clear changes after making sure buffer resize is not required.
             //
-            if (!isEncodeAll && !hasView) {
+            if (shouldClearChanges) {
                 //
                 // FIXME: avoid iterating over change trees twice.
                 //
                 this.onEndEncode(changeTrees);
             }
 
-            // return bytes;
-            return bytes.slice(0, it.offset);
+            return buffer.subarray(0, it.offset);
         }
     }
 
-    encodeAll(it: Iterator = { offset: 0 }) {
-        // console.log(`encodeAll(), this.$root.allChanges (${this.$root.allChanges.size})`);
+    encodeAll(it: Iterator = { offset: 0 }, buffer: Buffer = this.sharedBuffer) {
+        // console.log(`\nencodeAll(), this.root.allChanges (${this.root.allChanges.size})`);
+        // this.debugChanges("allChanges");
 
-        // Array.from(this.$root.allChanges.entries()).map((item) => {
-        //     console.log("->", item[0].refId, item[0].ref.toJSON());
-        // });
-
-        return this.encode(it, undefined, this.sharedBuffer, this.root.allChanges);
+        return this.encode(it, undefined, buffer, this.root.allChanges, true);
     }
 
     encodeAllView(view: StateView, sharedOffset: number, it: Iterator, bytes = this.sharedBuffer) {
         const viewOffset = it.offset;
 
-        // console.log(`encodeAllView(), this.$root.allFilteredChanges (${this.$root.allFilteredChanges.size})`);
-        // this.debugAllFilteredChanges();
+        // console.log(`\nencodeAllView(), this.root.allFilteredChanges (${this.root.allFilteredChanges.size})`);
+        // this.debugChanges("allFilteredChanges");
 
         // try to encode "filtered" changes
-        this.encode(it, view, bytes, this.root.allFilteredChanges);
+        this.encode(it, view, bytes, this.root.allFilteredChanges, true, viewOffset);
 
         return Buffer.concat([
-            bytes.slice(0, sharedOffset),
-            bytes.slice(viewOffset, it.offset)
+            bytes.subarray(0, sharedOffset),
+            bytes.subarray(viewOffset, it.offset)
         ]);
     }
 
-
-    // debugAllFilteredChanges() {
-    //     Array.from(this.$root.allFilteredChanges.entries()).map((item) => {
-    //         console.log("->", { refId: item[0].refId }, item[0].ref.toJSON());
-    //         if (Array.isArray(item[0].ref.toJSON())) {
-    //             item[1].forEach((op, key) => {
-    //                 console.log("  ->", { key, op: OPERATION[op] });
-    //             })
-    //         }
-    //     });
-    // }
+    debugChanges(
+        field:  "changes" | "allFilteredChanges" | "allChanges" | "filteredChanges" | Map<ChangeTree, Map<number, OPERATION>>
+    ) {
+        const changeSet = (typeof (field) === "string")
+            ? this.root[field]
+            : field;
+
+        Array.from(changeSet.entries()).map((item) => {
+            const metadata: Metadata = item[0].ref.constructor[Symbol.metadata];
+            console.log("->", { ref: item[0].ref.constructor.name, refId: item[0].refId, changes: item[1].size });
+            item[1].forEach((op, index) => {
+                console.log("  ->", {
+                    index,
+                    field: metadata?.[index],
+                    op: OPERATION[op],
+                });
+            });
+        });
+    }
 
     encodeView(view: StateView, sharedOffset: number, it: Iterator, bytes = this.sharedBuffer) {
         const viewOffset = it.offset;
 
-        // try to encode "filtered" changes
-        this.encode(it, view, bytes, this.root.filteredChanges);
+        // console.log(`\nencodeView(), view.changes (${view.changes.size})`);
+        // this.debugChanges(view.changes);
+
+        // console.log(`\nencodeView(), this.root.filteredChanges (${this.root.filteredChanges.size})`);
+        // this.debugChanges("filteredChanges");
 
         // encode visibility changes (add/remove for this view)
         const viewChangesIterator = view.changes.entries();
@@ -207,9 +242,12 @@ export class Encoder<T extends Schema = any> {
         // clear "view" changes after encoding
         view.changes.clear();
 
+        // try to encode "filtered" changes
+        this.encode(it, view, bytes, this.root.filteredChanges, false, viewOffset);
+
         return Buffer.concat([
-            bytes.slice(0, sharedOffset),
-            bytes.slice(viewOffset, it.offset)
+            bytes.subarray(0, sharedOffset),
+            bytes.subarray(viewOffset, it.offset)
         ]);
     }
 
@@ -217,6 +255,14 @@ export class Encoder<T extends Schema = any> {
         const changeTreesIterator = changeTrees.entries();
         for (const [changeTree, _] of changeTreesIterator) {
             changeTree.endEncode();
+            // changeTree.changes.clear();
+
+            // // ArraySchema and MapSchema have a custom "encode end" method
+            // changeTree.ref[$onEncodeEnd]?.();
+
+            // // Not a new instance anymore
+            // delete changeTree[$isNew];
+
         }
     }
 
@@ -242,4 +288,4 @@ export class Encoder<T extends Schema = any> {
             encode.number(bytes, targetTypeId, it);
         }
     }
-}
+}

package/src/encoder/Root.ts

@@ -0,0 +1,51 @@
+import { OPERATION } from "../encoding/spec";
+import { TypeContext } from "../types/TypeContext";
+import { ChangeTree } from "./ChangeTree";
+
+export class Root {
+    protected nextUniqueId: number = 0;
+    refCount = new WeakMap<ChangeTree, number>();
+
+    // all changes
+    allChanges = new Map<ChangeTree, Map<number, OPERATION>>();
+    allFilteredChanges = new Map<ChangeTree, Map<number, OPERATION>>();
+
+    // pending changes to be encoded
+    changes = new Map<ChangeTree, Map<number, OPERATION>>();
+    filteredChanges = new Map<ChangeTree, Map<number, OPERATION>>();
+
+    constructor(public types: TypeContext) { }
+
+    getNextUniqueId() {
+        return this.nextUniqueId++;
+    }
+
+    add(changeTree: ChangeTree) {
+        const refCount = this.refCount.get(changeTree) || 0;
+        this.refCount.set(changeTree, refCount + 1);
+    }
+
+    remove(changeTree: ChangeTree) {
+        const refCount = this.refCount.get(changeTree);
+        if (refCount <= 1) {
+            this.allChanges.delete(changeTree);
+            this.changes.delete(changeTree);
+
+            if (changeTree.isFiltered || changeTree.isPartiallyFiltered) {
+                this.allFilteredChanges.delete(changeTree);
+                this.filteredChanges.delete(changeTree);
+            }
+
+            this.refCount.delete(changeTree);
+
+        } else {
+            this.refCount.set(changeTree, refCount - 1);
+        }
+
+        changeTree.forEachChild((child, _) => this.remove(child));
+    }
+
+    clear() {
+        this.changes.clear();
+    }
+}