@colyseus/schema 3.0.0-alpha.8 → 3.0.0

package/src/encoder/EncodeOperation.ts

@@ -1,17 +1,15 @@
 import { OPERATION } from "../encoding/spec";
-import { $changes } from "../types/symbols";
-import { getType } from "../types/registry";
+import { $changes, $childType, $getByIndex } from "../types/symbols";
 
-import * as encode from "../encoding/encode";
-import { EncodeSchemaError, assertInstanceType, assertType } from "../encoding/assert";
+import { encode } from "../encoding/encode";
 
 import type { ChangeTree, Ref } from "./ChangeTree";
 import type { Encoder } from "./Encoder";
 import type { Schema } from "../Schema";
-import type { PrimitiveType } from "../annotations";
 
 import type { Iterator } from "../encoding/decode";
 import type { ArraySchema } from "../types/custom/ArraySchema";
+import type { Metadata } from "../Metadata";
 
 export type EncodeOperation<T extends Ref = any> = (
     encoder: Encoder,
@@ -22,43 +20,21 @@ export type EncodeOperation<T extends Ref = any> = (
     it: Iterator,
     isEncodeAll: boolean,
     hasView: boolean,
+    metadata?: Metadata,
 ) => void;
 
-export function encodePrimitiveType(
-    type: PrimitiveType,
-    bytes: Buffer,
-    value: any,
-    klass: Schema,
-    field: string | number,
-    it: Iterator,
-) {
-    assertType(value, type as string, klass, field);
-
-    const encodeFunc = encode[type as string];
-
-    if (encodeFunc) {
-        encodeFunc(bytes, value, it);
-        // encodeFunc(bytes, value);
-
-    } else {
-        throw new EncodeSchemaError(`a '${type}' was expected, but ${value} was provided in ${klass.constructor.name}#${field}`);
-    }
-};
-
 export function encodeValue(
     encoder: Encoder,
     bytes: Buffer,
-    ref: Ref,
     type: any,
     value: any,
-    field: string | number,
     operation: OPERATION,
     it: Iterator,
 ) {
-    if (type[Symbol.metadata] !== undefined) {
-        // TODO: move this to the `@type()` annotation
-        assertInstanceType(value, type as typeof Schema, ref as Schema, field);
+    if (typeof (type) === "string") {
+        encode[type]?.(bytes, value, it);
 
+    } else if (type[Symbol.metadata] !== undefined) {
         //
         // Encode refId for this instance.
         // The actual instance is going to be encoded on next `changeTree` iteration.
@@ -70,23 +46,7 @@ export function encodeValue(
             encoder.tryEncodeTypeId(bytes, type as typeof Schema, value.constructor as typeof Schema, it);
         }
 
-    } else if (typeof (type) === "string") {
-        //
-        // Primitive values
-        //
-        encodePrimitiveType(type as PrimitiveType, bytes, value, ref as Schema, field, it);
-
     } else {
-        //
-        // Custom type (MapSchema, ArraySchema, etc)
-        //
-        const definition = getType(Object.keys(type)[0]);
-
-        //
-        // ensure a ArraySchema has been provided
-        //
-        assertInstanceType(ref[field], definition.constructor, ref as Schema, field);
-
         //
         // Encode refId for this instance.
         // The actual instance is going to be encoded on next `changeTree` iteration.
@@ -106,14 +66,10 @@ export const encodeSchemaOperation: EncodeOperation = function (
     index: number,
     operation: OPERATION,
     it: Iterator,
+    _: any,
+    __: any,
+    metadata: Metadata,
 ) {
-    const ref = changeTree.ref;
-    const metadata = ref['constructor'][Symbol.metadata];
-
-    const field = metadata[index];
-    const type = metadata[field].type;
-    const value = ref[field];
-
     // "compress" field index + operation
     bytes[it.offset++] = (index | operation) & 255;
 
@@ -122,8 +78,18 @@ export const encodeSchemaOperation: EncodeOperation = function (
         return;
     }
 
+    const ref = changeTree.ref;
+    const field = metadata[index];
+
     // TODO: inline this function call small performance gain
-    encodeValue(encoder, bytes, ref, type, value, field, operation, it);
+    encodeValue(
+        encoder,
+        bytes,
+        metadata[index].type,
+        ref[field.name],
+        operation,
+        it
+    );
 }
 
 /**
@@ -134,12 +100,10 @@ export const encodeKeyValueOperation: EncodeOperation = function (
     encoder: Encoder,
     bytes: Buffer,
     changeTree: ChangeTree,
-    field: number,
+    index: number,
     operation: OPERATION,
     it: Iterator,
 ) {
-    const ref = changeTree.ref;
-
     // encode operation
     bytes[it.offset++] = operation & 255;
 
@@ -149,31 +113,53 @@ export const encodeKeyValueOperation: EncodeOperation = function (
     }
 
     // encode index
-    encode.number(bytes, field, it);
+    encode.number(bytes, index, it);
 
     // Do not encode value for DELETE operations
     if (operation === OPERATION.DELETE) {
         return;
     }
 
+    const ref = changeTree.ref;
+
     //
     // encode "alias" for dynamic fields (maps)
     //
-    if ((operation & OPERATION.ADD) == OPERATION.ADD) { // ADD or DELETE_AND_ADD
+    if ((operation & OPERATION.ADD) === OPERATION.ADD) { // ADD or DELETE_AND_ADD
         if (typeof(ref['set']) === "function") {
             //
             // MapSchema dynamic key
             //
-            const dynamicIndex = changeTree.ref['$indexes'].get(field);
+            const dynamicIndex = changeTree.ref['$indexes'].get(index);
             encode.string(bytes, dynamicIndex, it);
         }
     }
 
-    const type = changeTree.getType(field);
-    const value = changeTree.getValue(field);
+    const type = ref[$childType];
+    const value = ref[$getByIndex](index);
+
+    // try { throw new Error(); } catch (e) {
+    //     // only print if not coming from Reflection.ts
+    //     if (!e.stack.includes("src/Reflection.ts")) {
+    //         console.log("encodeKeyValueOperation -> ", {
+    //             ref: changeTree.ref.constructor.name,
+    //             field,
+    //             operation: OPERATION[operation],
+    //             value: value?.toJSON(),
+    //             items: ref.toJSON(),
+    //         });
+    //     }
+    // }
 
     // TODO: inline this function call small performance gain
-    encodeValue(encoder, bytes, ref, type, value, field, operation, it);
+    encodeValue(
+        encoder,
+        bytes,
+        type,
+        value,
+        operation,
+        it
+    );
 }
 
 /**
@@ -191,30 +177,37 @@ export const encodeArray: EncodeOperation = function (
     hasView: boolean,
 ) {
     const ref = changeTree.ref;
+    const useOperationByRefId = hasView && changeTree.isFiltered && (typeof (changeTree.getType(field)) !== "string");
 
-    if (
-        hasView &&
-        operation === OPERATION.DELETE &&
-        typeof (changeTree.getType(field)) !== "string"
-    ) {
-        // encode delete by refId (array of schemas)
-        bytes[it.offset++] = OPERATION.DELETE_BY_REFID;
-        const value =  ref['tmpItems'][field];
-        const refId = value[$changes].refId;
-        encode.number(bytes, refId, it);
-        return;
+    let refOrIndex: number;
+
+    if (useOperationByRefId) {
+        refOrIndex = ref['tmpItems'][field][$changes].refId;
+
+        if (operation === OPERATION.DELETE) {
+            operation = OPERATION.DELETE_BY_REFID;
+
+        } else if (operation === OPERATION.ADD) {
+            operation = OPERATION.ADD_BY_REFID;
+        }
+
+    } else {
+        refOrIndex = field;
     }
 
     // encode operation
     bytes[it.offset++] = operation & 255;
 
     // custom operations
-    if (operation === OPERATION.CLEAR) {
+    if (
+        operation === OPERATION.CLEAR ||
+        operation === OPERATION.REVERSE
+    ) {
         return;
     }
 
     // encode index
-    encode.number(bytes, field, it);
+    encode.number(bytes, refOrIndex, it);
 
     // Do not encode value for DELETE operations
     if (operation === OPERATION.DELETE) {
@@ -233,5 +226,12 @@ export const encodeArray: EncodeOperation = function (
     // });
 
     // TODO: inline this function call small performance gain
-    encodeValue(encoder, bytes, ref, type, value, field, operation, it);
+    encodeValue(
+        encoder,
+        bytes,
+        type,
+        value,
+        operation,
+        it
+    );
 }

package/src/encoder/Encoder.ts

@@ -1,32 +1,35 @@
 import type { Schema } from "../Schema";
-import { TypeContext } from "../annotations";
-import { $changes, $encoder, $filter } from "../types/symbols";
+import { TypeContext } from "../types/TypeContext";
+import { $changes, $encoder, $filter, $onEncodeEnd } from "../types/symbols";
 
-import * as encode from "../encoding/encode";
+import { encode } from "../encoding/encode";
 import type { Iterator } from "../encoding/decode";
 
-import { SWITCH_TO_STRUCTURE, TYPE_ID } from '../encoding/spec';
-import { Root } from "./ChangeTree";
+import { OPERATION, SWITCH_TO_STRUCTURE, TYPE_ID } from '../encoding/spec';
+import { Root } from "./Root";
 import { getNextPowerOf2 } from "../utils";
+
 import type { StateView } from "./StateView";
+import type { Metadata } from "../Metadata";
 
 export class Encoder<T extends Schema = any> {
-    static BUFFER_SIZE = 8 * 1024;// 8KB
-    sharedBuffer = Buffer.allocUnsafeSlow(Encoder.BUFFER_SIZE);
+    static BUFFER_SIZE = Buffer.poolSize ?? 8 * 1024; // 8KB
+    sharedBuffer = Buffer.allocUnsafe(Encoder.BUFFER_SIZE);
 
     context: TypeContext;
     state: T;
 
     root: Root;
 
-    constructor(root: T) {
-        this.setRoot(root);
-
+    constructor(state: T) {
         //
         // TODO: cache and restore "Context" based on root schema
         // (to avoid creating a new context for every new room)
         //
-        this.context = new TypeContext(root.constructor as typeof Schema);
+        this.context = new TypeContext(state.constructor as typeof Schema);
+        this.root = new Root(this.context);
+
+        this.setState(state);
 
         // console.log(">>>>>>>>>>>>>>>> Encoder types");
         // this.context.schemas.forEach((id, schema) => {
@@ -34,32 +37,35 @@ export class Encoder<T extends Schema = any> {
         // });
     }
 
-    protected setRoot(state: T) {
-        this.root = new Root();
+    protected setState(state: T) {
         this.state = state;
-        state[$changes].setRoot(this.root);
+        this.state[$changes].setRoot(this.root);
     }
 
     encode(
         it: Iterator = { offset: 0 },
         view?: StateView,
         buffer = this.sharedBuffer,
-        changeTrees = this.root.changes
+        changeSetName: "changes" | "allChanges" | "filteredChanges" | "allFilteredChanges" = "changes",
+        isEncodeAll = changeSetName === "allChanges",
+        initialOffset = it.offset // cache current offset in case we need to resize the buffer
     ): Buffer {
-        const initialOffset = it.offset; // cache current offset in case we need to resize the buffer
-
-        const isEncodeAll = this.root.allChanges === changeTrees;
         const hasView = (view !== undefined);
         const rootChangeTree = this.state[$changes];
 
-        const changeTreesIterator = changeTrees.entries();
+        const shouldDiscardChanges = !isEncodeAll && !hasView;
+        const changeTrees = this.root[changeSetName];
+
+        for (let i = 0, numChangeTrees = changeTrees.length; i < numChangeTrees; i++) {
+            const changeTree = changeTrees[i];
 
-        for (const [changeTree, changes] of changeTreesIterator) {
+            const operations = changeTree[changeSetName];
             const ref = changeTree.ref;
 
-            const ctor = ref['constructor'];
+            const ctor = ref.constructor;
             const encoder = ctor[$encoder];
             const filter = ctor[$filter];
+            const metadata = ctor[Symbol.metadata];
 
             if (hasView) {
                 if (!view.items.has(changeTree)) {
@@ -72,14 +78,21 @@ export class Encoder<T extends Schema = any> {
             }
 
             // skip root `refId` if it's the first change tree
-            if (it.offset !== initialOffset || changeTree !== rootChangeTree) {
+            // (unless it "hasView", which will need to revisit the root)
+            if (hasView || it.offset > initialOffset || changeTree !== rootChangeTree) {
                 buffer[it.offset++] = SWITCH_TO_STRUCTURE & 255;
                 encode.number(buffer, changeTree.refId, it);
             }
 
-            const changesIterator = changes.entries();
+            for (let j = 0, numChanges = operations.operations.length; j < numChanges; j++) {
+                const fieldIndex = operations.operations[j];
+
+                const operation = (fieldIndex < 0)
+                    ? Math.abs(fieldIndex) // "pure" operation without fieldIndex (e.g. CLEAR, REVERSE, etc.)
+                    : (isEncodeAll)
+                        ? OPERATION.ADD
+                        : changeTree.indexedOperations[fieldIndex];
 
-            for (const [fieldIndex, operation] of changesIterator) {
                 //
                 // first pass (encodeAll), identify "filtered" operations without encoding them
                 // they will be encoded per client, based on their view.
@@ -87,122 +100,138 @@ export class Encoder<T extends Schema = any> {
                 // TODO: how can we optimize filtering out "encode all" operations?
                 // TODO: avoid checking if no view tags were defined
                 //
-                if (filter && !filter(ref, fieldIndex, view)) {
-                    // console.log("SKIP FIELD:", { ref: changeTree.ref.constructor.name, fieldIndex, })
-
+                if (fieldIndex === undefined || operation === undefined || (filter && !filter(ref, fieldIndex, view))) {
                     // console.log("ADD AS INVISIBLE:", fieldIndex, changeTree.ref.constructor.name)
                     // view?.invisible.add(changeTree);
                     continue;
                 }
 
-                // console.log("WILL ENCODE", {
-                //     ref: changeTree.ref.constructor.name,
-                //     fieldIndex,
-                //     operation: OPERATION[operation],
-                // });
-
-                encoder(this, buffer, changeTree, fieldIndex, operation, it, isEncodeAll, hasView);
+                encoder(this, buffer, changeTree, fieldIndex, operation, it, isEncodeAll, hasView, metadata);
             }
+
+            // if (shouldDiscardChanges) {
+            //     changeTree.discard();
+            //     changeTree.isNew = false; // Not a new instance anymore
+            // }
         }
 
         if (it.offset > buffer.byteLength) {
-            const newSize = getNextPowerOf2(buffer.byteLength * 2);
-            console.warn("@colyseus/schema encode buffer overflow. Current buffer size: " + buffer.byteLength + ", encoding offset: " + it.offset + ", new size: " + newSize);
+            // we can assume that n + 1 poolSize will suffice given that we are likely done with encoding at this point
+            // multiples of poolSize are faster to allocate than arbitrary sizes
+            // if we are on an older platform that doesn't implement pooling use 8kb as poolSize (that's the default for node)
+            const newSize = Math.ceil(it.offset / (Buffer.poolSize ?? 8 * 1024)) * (Buffer.poolSize ?? 8 * 1024);            
+
+            console.warn(`@colyseus/schema buffer overflow. Encoded state is higher than default BUFFER_SIZE. Use the following to increase default BUFFER_SIZE:
+
+    import { Encoder } from "@colyseus/schema";
+    Encoder.BUFFER_SIZE = ${Math.round(newSize / 1024)} * 1024; // ${Math.round(newSize / 1024)} KB
+`);
 
             //
             // resize buffer and re-encode (TODO: can we avoid re-encoding here?)
+            // -> No we probably can't unless we catch the need for resize before encoding which is likely more computationally expensive than resizing on demand
             //
-            buffer = Buffer.allocUnsafeSlow(newSize);
+            buffer = Buffer.alloc(newSize, buffer); // fill with buffer here to memcpy previous encoding steps beyond the initialOffset
 
             // assign resized buffer to local sharedBuffer
             if (buffer === this.sharedBuffer) {
                 this.sharedBuffer = buffer;
             }
 
-            return this.encode({ offset: initialOffset }, view, buffer);
+            return this.encode({ offset: initialOffset }, view, buffer, changeSetName, isEncodeAll);
 
         } else {
             //
             // only clear changes after making sure buffer resize is not required.
             //
-            if (!isEncodeAll && !hasView) {
+            if (shouldDiscardChanges) {
                 //
-                // FIXME: avoid iterating over change trees twice.
+                // TODO: avoid iterating over change trees twice.
                 //
-                this.onEndEncode(changeTrees);
+                for (let i = 0, numChangeTrees = changeTrees.length; i < numChangeTrees; i++) {
+                    const changeTree = changeTrees[i];
+                    changeTree.discard();
+                    changeTree.isNew = false; // Not a new instance anymore
+                }
             }
 
-            // return bytes;
-            return buffer.slice(0, it.offset);
+            return buffer.subarray(0, it.offset);
         }
     }
 
     encodeAll(it: Iterator = { offset: 0 }, buffer: Buffer = this.sharedBuffer) {
-        // console.log(`encodeAll(), this.$root.allChanges (${this.$root.allChanges.size})`);
-
-        // Array.from(this.$root.allChanges.entries()).map((item) => {
-        //     console.log("->", item[0].refId, item[0].ref.toJSON());
-        // });
-
-        return this.encode(it, undefined, buffer, this.root.allChanges);
+        return this.encode(it, undefined, buffer, "allChanges", true);
     }
 
     encodeAllView(view: StateView, sharedOffset: number, it: Iterator, bytes = this.sharedBuffer) {
         const viewOffset = it.offset;
 
-        // console.log(`encodeAllView(), this.$root.allFilteredChanges (${this.$root.allFilteredChanges.size})`);
-        // this.debugAllFilteredChanges();
-
         // try to encode "filtered" changes
-        this.encode(it, view, bytes, this.root.allFilteredChanges);
+        this.encode(it, view, bytes, "allFilteredChanges", true, viewOffset);
 
         return Buffer.concat([
-            bytes.slice(0, sharedOffset),
-            bytes.slice(viewOffset, it.offset)
+            bytes.subarray(0, sharedOffset),
+            bytes.subarray(viewOffset, it.offset)
         ]);
     }
 
-
-    // debugAllFilteredChanges() {
-    //     Array.from(this.$root.allFilteredChanges.entries()).map((item) => {
-    //         console.log("->", { refId: item[0].refId }, item[0].ref.toJSON());
-    //         if (Array.isArray(item[0].ref.toJSON())) {
-    //             item[1].forEach((op, key) => {
-    //                 console.log("  ->", { key, op: OPERATION[op] });
-    //             })
-    //         }
-    //     });
-    // }
+    debugChanges(field: "changes" | "allFilteredChanges" | "allChanges" | "filteredChanges") {
+        const rootChangeSet = (typeof (field) === "string")
+            ? this.root[field]
+            : field;
+
+        rootChangeSet.forEach((changeTree) => {
+            const changeSet = changeTree[field];
+
+            const metadata: Metadata = changeTree.ref.constructor[Symbol.metadata];
+            console.log("->", { ref: changeTree.ref.constructor.name, refId: changeTree.refId, changes: Object.keys(changeSet).length });
+            for (const index in changeSet) {
+                const op = changeSet[index];
+                console.log("  ->", {
+                    index,
+                    field: metadata?.[index],
+                    op: OPERATION[op],
+                });
+            }
+        });
+    }
 
     encodeView(view: StateView, sharedOffset: number, it: Iterator, bytes = this.sharedBuffer) {
         const viewOffset = it.offset;
 
-        // try to encode "filtered" changes
-        this.encode(it, view, bytes, this.root.filteredChanges);
-
         // encode visibility changes (add/remove for this view)
-        const viewChangesIterator = view.changes.entries();
-        for (const [changeTree, changes] of viewChangesIterator) {
-            if (changes.size === 0) {
-                // FIXME: avoid having empty changes if no changes were made
-                // console.log("changes.size === 0", changeTree.ref.constructor.name);
+        const refIds = Object.keys(view.changes);
+
+        for (let i = 0, numRefIds = refIds.length; i < numRefIds; i++) {
+            const refId = refIds[i];
+            const changes = view.changes[refId];
+            const changeTree = this.root.changeTrees[refId];
+
+            if (
+                changeTree === undefined ||
+                Object.keys(changes).length === 0 // FIXME: avoid having empty changes if no changes were made
+            ) {
+                // console.log("changes.size === 0, skip", changeTree.ref.constructor.name);
                 continue;
             }
 
             const ref = changeTree.ref;
 
-            const ctor = ref['constructor'];
+            const ctor = ref.constructor;
             const encoder = ctor[$encoder];
+            const metadata = ctor[Symbol.metadata];
 
             bytes[it.offset++] = SWITCH_TO_STRUCTURE & 255;
             encode.number(bytes, changeTree.refId, it);
 
-            const changesIterator = changes.entries();
+            const keys = Object.keys(changes);
+            for (let i = 0, numChanges = keys.length; i < numChanges; i++) {
+                const key = keys[i];
+                const operation = changes[key];
 
-            for (const [fieldIndex, operation] of changesIterator) {
                 // isEncodeAll = false
                 // hasView = true
-                encoder(this, bytes, changeTree, fieldIndex, operation, it, false, true);
+                encoder(this, bytes, changeTree, Number(key), operation, it, false, true, metadata);
             }
         }
 
@@ -211,31 +240,54 @@ export class Encoder<T extends Schema = any> {
         // (to allow re-using StateView's for multiple clients)
         //
         // clear "view" changes after encoding
-        view.changes.clear();
+        view.changes = {};
+
+        // try to encode "filtered" changes
+        this.encode(it, view, bytes, "filteredChanges", false, viewOffset);
 
         return Buffer.concat([
-            bytes.slice(0, sharedOffset),
-            bytes.slice(viewOffset, it.offset)
+            bytes.subarray(0, sharedOffset),
+            bytes.subarray(viewOffset, it.offset)
         ]);
     }
 
     onEndEncode(changeTrees = this.root.changes) {
-        const changeTreesIterator = changeTrees.entries();
-        for (const [changeTree, _] of changeTreesIterator) {
-            changeTree.endEncode();
-        }
+        // changeTrees.forEach(function(changeTree) {
+        //     changeTree.endEncode();
+        // });
+
+
+        // for (const refId in changeTrees) {
+        //     const changeTree = this.root.changeTrees[refId];
+        //     changeTree.endEncode();
+
+        //     // changeTree.changes.clear();
+
+        //     // // ArraySchema and MapSchema have a custom "encode end" method
+        //     // changeTree.ref[$onEncodeEnd]?.();
+
+        //     // // Not a new instance anymore
+        //     // delete changeTree[$isNew];
+        // }
     }
 
     discardChanges() {
         // discard shared changes
-        if (this.root.changes.size > 0) {
-            this.onEndEncode(this.root.changes);
-            this.root.changes.clear();
+        let length = this.root.changes.length;
+        if (length > 0) {
+            while (length--) {
+                this.root.changes[length]?.endEncode();
+            }
+            this.root.changes.length = 0;
         }
+
         // discard filtered changes
-        if (this.root.filteredChanges.size > 0) {
-            this.onEndEncode(this.root.filteredChanges);
-            this.root.filteredChanges.clear();
+        length = this.root.filteredChanges.length;
+        if (length > 0) {
+            while (length--) {
+                this.root.filteredChanges[length]?.endEncode();
+            }
+            this.root.filteredChanges.length = 0;
         }
     }
 
@@ -243,9 +295,21 @@ export class Encoder<T extends Schema = any> {
         const baseTypeId = this.context.getTypeId(baseType);
         const targetTypeId = this.context.getTypeId(targetType);
 
+        if (targetTypeId === undefined) {
+            console.warn(`@colyseus/schema WARNING: Class "${targetType.name}" is not registered on TypeRegistry - Please either tag the class with @entity or define a @type() field.`);
+            return;
+        }
+
         if (baseTypeId !== targetTypeId) {
             bytes[it.offset++] = TYPE_ID & 255;
             encode.number(bytes, targetTypeId, it);
         }
     }
-}
+
+    get hasChanges() {
+        return (
+            this.root.changes.length > 0 ||
+            this.root.filteredChanges.length > 0
+        );
+    }
+}