@colyseus/schema 3.0.0-alpha.3 → 3.0.0-alpha.31

package/src/annotations.ts

@@ -3,9 +3,12 @@ import { Schema } from './Schema';
 import { ArraySchema } from './types/custom/ArraySchema';
 import { MapSchema } from './types/custom/MapSchema';
 import { Metadata } from "./Metadata";
-import { $changes, $childType, $track } from "./types/symbols";
+import { $changes, $childType, $descriptors, $track } from "./types/symbols";
 import { TypeDefinition, getType } from "./types/registry";
 import { OPERATION } from "./encoding/spec";
+import { TypeContext } from "./types/TypeContext";
+import { assertInstanceType, assertType } from "./encoding/assert";
+import type { Ref } from "./encoder/ChangeTree";
 
 /**
  * Data types
@@ -42,113 +45,6 @@ export interface TypeOptions {
 
 export const DEFAULT_VIEW_TAG = -1;
 
-export class TypeContext {
-    types: {[id: number]: typeof Schema} = {};
-    schemas = new Map<typeof Schema, number>();
-
-    hasFilters: boolean = false;
-
-    /**
-     * For inheritance support
-     * Keeps track of which classes extends which. (parent -> children)
-     */
-    static inheritedTypes = new Map<typeof Schema, Set<typeof Schema>>();
-
-    static register(target: typeof Schema) {
-        const parent = Object.getPrototypeOf(target);
-        if (parent !== Schema) {
-            let inherits = TypeContext.inheritedTypes.get(parent);
-            if (!inherits) {
-                inherits = new Set<typeof Schema>();
-                TypeContext.inheritedTypes.set(parent, inherits);
-            }
-            inherits.add(target);
-        }
-    }
-
-    constructor(rootClass?: typeof Schema) {
-        if (rootClass) {
-            this.discoverTypes(rootClass);
-        }
-    }
-
-    has(schema: typeof Schema) {
-        return this.schemas.has(schema);
-    }
-
-    get(typeid: number) {
-        return this.types[typeid];
-    }
-
-    add(schema: typeof Schema, typeid: number = this.schemas.size) {
-        // skip if already registered
-        if (this.schemas.has(schema)) {
-            return false;
-        }
-
-        this.types[typeid] = schema;
-        this.schemas.set(schema, typeid);
-        return true;
-    }
-
-    getTypeId(klass: typeof Schema) {
-        return this.schemas.get(klass);
-    }
-
-    private discoverTypes(klass: typeof Schema) {
-        if (!this.add(klass)) {
-            return;
-        }
-
-        // add classes inherited from this base class
-        TypeContext.inheritedTypes.get(klass)?.forEach((child) => {
-            this.discoverTypes(child);
-        });
-
-        // skip if no fields are defined for this class.
-        if (klass[Symbol.metadata] === undefined) {
-            klass[Symbol.metadata] = {};
-        }
-
-        // const metadata = Metadata.getFor(klass);
-        const metadata = klass[Symbol.metadata];
-
-        // if any schema/field has filters, mark "context" as having filters.
-        if (metadata[-2]) {
-            this.hasFilters = true;
-        }
-
-        for (const field in metadata) {
-            const fieldType = metadata[field].type;
-
-            if (typeof(fieldType) === "string") {
-                continue;
-            }
-
-            if (Array.isArray(fieldType)) {
-                const type = fieldType[0];
-                if (type === "string") {
-                    continue;
-                }
-                this.discoverTypes(type as typeof Schema);
-
-            } else if (typeof(fieldType) === "function") {
-                this.discoverTypes(fieldType);
-
-            } else {
-                const type = Object.values(fieldType)[0];
-
-                // skip primitive types
-                if (typeof(type) === "string") {
-                    continue;
-                }
-
-                this.discoverTypes(type as typeof Schema);
-            }
-        }
-    }
-}
-
 export function entity(constructor, context: ClassDecoratorContext) {
     if (!constructor._definition) {
         // for inheritance support
@@ -330,19 +226,22 @@ export function view<T> (tag: number = DEFAULT_VIEW_TAG) {
 
         const parentClass = Object.getPrototypeOf(constructor);
         const parentMetadata = parentClass[Symbol.metadata];
-        const metadata: Metadata = (constructor[Symbol.metadata] ??= Object.assign({}, constructor[Symbol.metadata], parentMetadata ?? Object.create(null)));
 
-        if (!metadata[fieldName]) {
-            //
-            // detect index for this field, considering inheritance
-            //
-            metadata[fieldName] = {
-                type: undefined,
-                index: (metadata[-1] // current structure already has fields defined
-                    ?? (parentMetadata && parentMetadata[-1]) // parent structure has fields defined
-                    ?? -1) + 1 // no fields defined
-            }
-        }
+        // TODO: use Metadata.initialize()
+        const metadata: Metadata = (constructor[Symbol.metadata] ??= Object.assign({}, constructor[Symbol.metadata], parentMetadata ?? Object.create(null)));
+        // const fieldIndex = metadata[fieldName];
+
+        // if (!metadata[fieldIndex]) {
+        //     //
+        //     // detect index for this field, considering inheritance
+        //     //
+        //     metadata[fieldIndex] = {
+        //         type: undefined,
+        //         index: (metadata[-1] // current structure already has fields defined
+        //             ?? (parentMetadata && parentMetadata[-1]) // parent structure has fields defined
+        //             ?? -1) + 1 // no fields defined
+        //     }
+        // }
 
         Metadata.setTag(metadata, fieldName, tag);
     }
@@ -356,22 +255,24 @@ export function unreliable<T> (target: T, field: string) {
 
     const parentClass = Object.getPrototypeOf(constructor);
     const parentMetadata = parentClass[Symbol.metadata];
+
+    // TODO: use Metadata.initialize()
     const metadata: Metadata = (constructor[Symbol.metadata] ??= Object.assign({}, constructor[Symbol.metadata], parentMetadata ?? Object.create(null)));
 
-    if (!metadata[field]) {
-        //
-        // detect index for this field, considering inheritance
-        //
-        metadata[field] = {
-            type: undefined,
-            index: (metadata[-1] // current structure already has fields defined
-                ?? (parentMetadata && parentMetadata[-1]) // parent structure has fields defined
-                ?? -1) + 1 // no fields defined
-        }
-    }
+    // if (!metadata[field]) {
+    //     //
+    //     // detect index for this field, considering inheritance
+    //     //
+    //     metadata[field] = {
+    //         type: undefined,
+    //         index: (metadata[-1] // current structure already has fields defined
+    //             ?? (parentMetadata && parentMetadata[-1]) // parent structure has fields defined
+    //             ?? -1) + 1 // no fields defined
+    //     }
+    // }
 
     // add owned flag to the field
-    metadata[field].unreliable = true;
+    metadata[metadata[field]].unreliable = true;
 }
 
 export function type (
@@ -389,20 +290,20 @@ export function type (
         TypeContext.register(constructor);
 
         const parentClass = Object.getPrototypeOf(constructor);
-        const parentMetadata = parentClass[Symbol.metadata];
-        const metadata: Metadata = (constructor[Symbol.metadata] ??= Object.assign({}, constructor[Symbol.metadata], parentMetadata ?? Object.create(null)));
+        const parentMetadata = parentClass && parentClass[Symbol.metadata];
+        const metadata = Metadata.initialize(constructor, parentMetadata);
 
-        let fieldIndex: number;
+        let fieldIndex: number = metadata[field];
 
         /**
          * skip if descriptor already exists for this field (`@deprecated()`)
          */
-        if (metadata[field]) {
-            if (metadata[field].deprecated) {
+        if (metadata[fieldIndex]) {
+            if (metadata[fieldIndex].deprecated) {
                 // do not create accessors for deprecated properties.
                 return;
 
-            } else if (metadata[field].descriptor !== undefined) {
+            } else if (metadata[fieldIndex].type !== undefined) {
                 // trying to define same property multiple times across inheritance.
                 // https://github.com/colyseus/colyseus-unity3d/issues/131#issuecomment-814308572
                 try {
@@ -412,9 +313,6 @@ export function type (
                     const definitionAtLine = e.stack.split("\n")[4].trim();
                     throw new Error(`${e.message} ${definitionAtLine}`);
                 }
-
-            } else {
-                fieldIndex = metadata[field].index;
             }
 
         } else {
@@ -428,12 +326,18 @@ export function type (
         }
 
         if (options && options.manual) {
-            Metadata.addField(metadata, fieldIndex, field, type, {
-                // do not declare getter/setter descriptor
-                enumerable: true,
-                configurable: true,
-                writable: true,
-            });
+            Metadata.addField(
+                metadata,
+                fieldIndex,
+                field,
+                type,
+                {
+                    // do not declare getter/setter descriptor
+                    enumerable: true,
+                    configurable: true,
+                    writable: true,
+                }
+            );
 
         } else {
             const complexTypeKlass = (Array.isArray(type))
@@ -449,7 +353,7 @@ export function type (
                 fieldIndex,
                 field,
                 type,
-                getPropertyDescriptor(`_${field}`, fieldIndex, childType, complexTypeKlass, metadata, field)
+                getPropertyDescriptor(`_${field}`, fieldIndex, childType, complexTypeKlass)
             );
         }
     }
@@ -460,8 +364,6 @@ export function getPropertyDescriptor(
     fieldIndex: number,
     type: DefinitionType,
     complexTypeKlass: TypeDefinition,
-    metadata: Metadata,
-    field: string,
 ) {
     return {
         get: function () { return this[fieldCached]; },
@@ -487,30 +389,32 @@ export function getPropertyDescriptor(
                     }
 
                     value[$childType] = type;
+
+                } else if (typeof (type) !== "string") {
+                    assertInstanceType(value, type as typeof Schema, this, fieldCached.substring(1));
+
+                } else {
+                    assertType(value, type, this, fieldCached.substring(1));
                 }
 
+                const changeTree = this[$changes];
+
                 //
                 // Replacing existing "ref", remove it from root.
                 // TODO: if there are other references to this instance, we should not remove it from root.
                 //
                 if (previousValue !== undefined && previousValue[$changes]) {
-                    this[$changes].root?.remove(previousValue[$changes]);
+                    changeTree.root?.remove(previousValue[$changes]);
                 }
 
                 // flag the change for encoding.
-                this.constructor[$track](this[$changes], fieldIndex, OPERATION.ADD);
+                this.constructor[$track](changeTree, fieldIndex, OPERATION.ADD);
 
                 //
                 // call setParent() recursively for this and its child
                 // structures.
                 //
-                if (value[$changes]) {
-                    value[$changes].setParent(
-                        this,
-                        this[$changes].root,
-                        metadata[field].index,
-                    );
-                }
+                (value as Ref)[$changes]?.setParent(this, changeTree.root, fieldIndex);
 
             } else if (previousValue !== undefined) {
                 //
@@ -542,23 +446,25 @@ export function deprecated(throws: boolean = true): PropertyDecorator {
         const parentClass = Object.getPrototypeOf(constructor);
         const parentMetadata = parentClass[Symbol.metadata];
         const metadata: Metadata = (constructor[Symbol.metadata] ??= Object.assign({}, constructor[Symbol.metadata], parentMetadata ?? Object.create(null)));
-
-        if (!metadata[field]) {
-            //
-            // detect index for this field, considering inheritance
-            //
-            metadata[field] = {
-                type: undefined,
-                index: (metadata[-1] // current structure already has fields defined
-                    ?? (parentMetadata && parentMetadata[-1]) // parent structure has fields defined
-                    ?? -1) + 1 // no fields defined
-            }
-        }
-
-        metadata[field].deprecated = true;
+        const fieldIndex = metadata[field];
+
+        // if (!metadata[field]) {
+        //     //
+        //     // detect index for this field, considering inheritance
+        //     //
+        //     metadata[field] = {
+        //         type: undefined,
+        //         index: (metadata[-1] // current structure already has fields defined
+        //             ?? (parentMetadata && parentMetadata[-1]) // parent structure has fields defined
+        //             ?? -1) + 1 // no fields defined
+        //     }
+        // }
+
+        metadata[fieldIndex].deprecated = true;
 
         if (throws) {
-            metadata[field].descriptor = {
+            metadata[$descriptors] ??= {};
+            metadata[$descriptors][field] = {
                 get: function () { throw new Error(`${field} is deprecated.`); },
                 set: function (this: Schema, value: any) { /* throw new Error(`${field} is deprecated.`); */ },
                 enumerable: false,
@@ -567,8 +473,8 @@ export function deprecated(throws: boolean = true): PropertyDecorator {
         }
 
         // flag metadata[field] as non-enumerable
-        Object.defineProperty(metadata, field, {
-            value: metadata[field],
+        Object.defineProperty(metadata, fieldIndex, {
+            value: metadata[fieldIndex],
             enumerable: false,
             configurable: true
         });

package/src/bench_encode.ts

@@ -0,0 +1,121 @@
+import { nanoid } from "nanoid";
+import { Schema, type, MapSchema, ArraySchema, Encoder } from ".";
+import * as benchmark from "benchmark";
+
+const suite = new benchmark.Suite();
+
+class Attribute extends Schema {
+    @type("string") name: string;
+    @type("number") value: number;
+}
+
+class Item extends Schema {
+    @type("number") price: number;
+    @type([ Attribute ]) attributes = new ArraySchema<Attribute>();
+}
+
+class Position extends Schema {
+    @type("number") x: number;
+    @type("number") y: number;
+}
+
+class Player extends Schema {
+    @type(Position) position = new Position();
+    @type({ map: Item }) items = new MapSchema<Item>();
+}
+
+class State extends Schema {
+    @type({ map: Player }) players = new MapSchema<Player>();
+    @type("string") currentTurn;
+}
+
+const state = new State();
+
+Encoder.BUFFER_SIZE = 4096 * 4096;
+const encoder = new Encoder(state);
+
+// // SINGLE TESTING
+// console.log("> Will create player...");
+// const player = new Player();
+// console.log("... players.set()")
+// state.players.set(`p-${nanoid()}`, player);
+// player.position.x = 100;
+// player.position.y = 100;
+// console.log("> Will create Item...");
+// const item = new Item();
+// item.price = 50;
+
+// console.log("> Will create Attribute...");
+// const attr = new Attribute();
+// attr.name = `Attribute 1`;
+// attr.value = 1;
+// console.log("... item.attributes.push()")
+// item.attributes.push(attr);
+
+// console.log("... player.items.set()")
+// player.items.set(`item-1`, item);
+
+let now = Date.now();
+for (let i = 0; i < 10000; i++) {
+// for (let i = 0; i < 10; i++) {
+    const player = new Player();
+    state.players.set(`p-${nanoid()}`, player);
+
+    player.position.x = (i + 1) * 100;
+    player.position.y = (i + 1) * 100;
+    for (let j = 0; j < 10; j++) {
+        const item = new Item();
+        player.items.set(`item-${j}`, item);
+        item.price = (i + 1) * 50;
+        for (let k = 0; k < 5; k++) {
+            const attr = new Attribute();
+            attr.name = `Attribute ${k}`;
+            attr.value = k;
+            item.attributes.push(attr);
+        }
+    }
+}
+console.log("time to make changes:", Date.now() - now);
+
+// process.exit();
+
+
+// measure time to .encodeAll()
+
+now = Date.now();
+// for (let i = 0; i < 1000; i++) {
+//     encoder.encodeAll();
+// }
+// console.log(Date.now() - now);
+
+const allEncodes = Date.now();
+for (let i = 0; i < 100; i++) {
+    now = Date.now();
+    for (let j = 0; j < 50; j++) {
+        const player = new Player();
+        state.players.set(`p-${nanoid()}`, player);
+
+        player.position.x = (j + 1) * 100;
+        player.position.y = (j + 1) * 100;
+        for (let k = 0; k < 10; k++) {
+            const item = new Item();
+            item.price = (j + 1) * 50;
+            for (let l = 0; l < 5; l++) {
+                const attr = new Attribute();
+                attr.name = `Attribute ${l}`;
+                attr.value = l;
+                item.attributes.push(attr);
+            }
+            player.items.set(`item-${k}`, item);
+        }
+    }
+    console.log("time to make changes:", Date.now() - now);
+
+    now = Date.now();
+    // encoder.encode();
+    encoder.discardChanges();
+    console.log("time to encode:", Date.now() - now);
+}
+console.log("time for all encodes:", Date.now() - allEncodes);
+
+// console.log(Array.from(encoder.encodeAll()).length, "bytes");

package/src/debug.ts

@@ -0,0 +1,56 @@
+import * as fs from "fs";
+import { Reflection, Decoder } from "./index";
+
+const contents = fs.readFileSync("/Users/endel/Projects/colyseus/clients/bubbits/project/@bubbits/backend/schema-debug.txt", { encoding: "utf8" }).toString();
+
+let isCommentBlock = false;
+let lastComment = "";
+
+let decoder: Decoder;
+
+function getBuffer(line: string) {
+    const start = line.lastIndexOf(":");
+    const buffer = Buffer.from(new Uint8Array(line.substring(start + 1).split(",").map(n => Number(n))));
+    console.log(`(${buffer.byteLength}) ${Array.from(buffer).join(",")}`)
+    // console.log("");
+    // console.log("");
+    // console.log("> ", line);
+    // console.log("> substring:", line.substring(start + 1))
+    return buffer;
+}
+
+function decode(buffer: Buffer) {
+    try {
+        decoder.decode(buffer);
+    } catch (e) {
+        console.error(e);
+        console.log("Last log:\n\n")
+        console.log(lastComment);
+    }
+}
+
+contents.split("\n").forEach((line) => {
+    if (line.startsWith("#")) {
+        // reset last comment.
+        if (isCommentBlock === false) { lastComment = ""; }
+
+        isCommentBlock = true;
+        lastComment += line.substring(line.indexOf(":") + 1) + "\n";
+        return;
+    }
+
+    isCommentBlock = false;
+
+    if (line.startsWith("handshake:") && !decoder) {
+        const state = Reflection.decode(getBuffer(line));
+        decoder = new Decoder(state);
+
+    } else if (line.startsWith("state:")) {
+        decode(getBuffer(line));
+
+    } else if (line.startsWith("patch:")) {
+        decode(getBuffer(line));
+    }
+});
+
+console.log(decoder.state.toJSON());

package/src/decoder/DecodeOperation.ts

@@ -4,7 +4,7 @@ import { Schema } from "../Schema";
 import type { Ref } from "../encoder/ChangeTree";
 import type { Decoder } from "./Decoder";
 import * as decode from "../encoding/decode";
-import { $changes, $childType, $deleteByIndex, $getByIndex } from "../types/symbols";
+import { $childType, $deleteByIndex, $getByIndex } from "../types/symbols";
 
 import type { MapSchema } from "../types/custom/MapSchema";
 import type { ArraySchema } from "../types/custom/ArraySchema";
@@ -43,7 +43,7 @@ export function decodeValue(
     it: decode.Iterator,
     allChanges: DataChange[],
 ) {
-    const $root = decoder.$root;
+    const $root = decoder.root;
     const previousValue = ref[$getByIndex](index);
 
     let value: any;
@@ -174,7 +174,7 @@ export const decodeSchemaOperation: DecodeOperation = function (
     allChanges: DataChange[],
 ) {
     const first_byte = bytes[it.offset++];
-    const metadata: Metadata = ref['constructor'][Symbol.metadata];
+    const metadata: Metadata = ref.constructor[Symbol.metadata];
 
     // "compressed" index + operation
     const operation = (first_byte >> 6) << 6
@@ -182,21 +182,24 @@ export const decodeSchemaOperation: DecodeOperation = function (
 
     // skip early if field is not defined
     const field = metadata[index];
-    if (field === undefined) { return DEFINITION_MISMATCH; }
+    if (field === undefined) {
+        console.warn("@colyseus/schema: field not defined at", { index, ref: ref.constructor.name, metadata });
+        return DEFINITION_MISMATCH;
+    }
 
     const { value, previousValue } = decodeValue(
         decoder,
         operation,
         ref,
         index,
-        metadata[field].type,
+        field.type,
         bytes,
         it,
         allChanges,
     );
 
     if (value !== null && value !== undefined) {
-        ref[field] = value;
+        ref[field.name] = value;
     }
 
     // add change
@@ -205,7 +208,7 @@ export const decodeSchemaOperation: DecodeOperation = function (
             ref,
             refId: decoder.currentRefId,
             op: operation,
-            field: field,
+            field: field.name,
             value,
             previousValue,
         });
@@ -251,6 +254,7 @@ export const decodeKeyValueOperation: DecodeOperation = function (
         dynamicIndex = ref['getIndex'](index);
     }
 
+
     const { value, previousValue } = decodeValue(
         decoder,
         operation,
@@ -303,7 +307,8 @@ export const decodeArray: DecodeOperation = function (
     allChanges: DataChange[]
 ) {
     // "uncompressed" index + operation (array/map items)
-    const operation = bytes[it.offset++];
+    let operation = bytes[it.offset++];
+    let index: number;
 
     if (operation === OPERATION.CLEAR) {
         //
@@ -318,8 +323,8 @@ export const decodeArray: DecodeOperation = function (
     } else if (operation === OPERATION.DELETE_BY_REFID) {
         // TODO: refactor here, try to follow same flow as below
         const refId = decode.number(bytes, it);
-        const previousValue = decoder.$root.refs.get(refId);
-        const index = ref.findIndex((value) => value === previousValue);
+        const previousValue = decoder.root.refs.get(refId);
+        index = ref.findIndex((value) => value === previousValue);
         ref[$deleteByIndex](index);
         allChanges.push({
             ref,
@@ -330,10 +335,22 @@ export const decodeArray: DecodeOperation = function (
             value: undefined,
             previousValue,
         });
+
         return;
+
+    } else if (operation === OPERATION.ADD_BY_REFID) {
+        const refId = decode.number(bytes, it);
+        const itemByRefId = decoder.root.refs.get(refId);
+
+        // use existing index, or push new value
+        index = (itemByRefId)
+            ? ref.findIndex((value) => value === itemByRefId)
+            : ref.length;
+
+    } else {
+        index = decode.number(bytes, it);
     }
 
-    const index = decode.number(bytes, it);
     const type = ref[$childType];
 
     let dynamicIndex: number | string = index;