@colyseus/schema 3.0.0-alpha.3 → 3.0.0-alpha.30

package/src/Metadata.ts

@@ -117,6 +117,53 @@ export const Metadata = {
         return metadata[field].deprecated === true;
     },
 
+    init(klass: any) {
+        //
+        // Used only to initialize an empty Schema (Encoder#constructor)
+        // TODO: remove/refactor this...
+        //
+        const metadata = {};
+        klass[Symbol.metadata] = metadata;
+        Object.defineProperty(metadata, -1, {
+            value: 0,
+            enumerable: false,
+            configurable: true,
+        });
+    },
+
+    initialize(constructor: any, parentMetadata?: any) {
+        let metadata: Metadata = constructor[Symbol.metadata] ?? Object.create(null);
+
+        // make sure inherited classes have their own metadata object.
+        if (constructor[Symbol.metadata] === parentMetadata) {
+            metadata = Object.create(null);
+
+            if (parentMetadata) {
+                // assign parent metadata to current
+                Object.assign(metadata, parentMetadata);
+
+                for (let i = 0; i <= parentMetadata[-1]; i++) {
+                    Object.defineProperty(metadata, i, {
+                        value: parentMetadata[i],
+                        enumerable: false,
+                        configurable: true,
+                    });
+                }
+
+                Object.defineProperty(metadata, -1, {
+                    value: parentMetadata[-1],
+                    enumerable: false,
+                    configurable: true,
+                    writable: true,
+                });
+            }
+        }
+
+        constructor[Symbol.metadata] = metadata;
+
+        return metadata;
+    },
+
     isValidInstance(klass: any) {
         return (
             klass.constructor[Symbol.metadata] &&

package/src/Reflection.ts

@@ -1,4 +1,5 @@
-import { type, PrimitiveType, DefinitionType, TypeContext } from "./annotations";
+import { type, PrimitiveType, DefinitionType } from "./annotations";
+import { TypeContext } from "./types/TypeContext";
 import { Metadata } from "./Metadata";
 import { ArraySchema } from "./types/custom/ArraySchema";
 import { Iterator } from "./encoding/decode";
@@ -24,10 +25,8 @@ export class ReflectionType extends Schema {
 export class Reflection extends Schema {
     @type([ ReflectionType ]) types: ArraySchema<ReflectionType> = new ArraySchema<ReflectionType>();
 
-    static encode (instance: Schema, context?: TypeContext) {
-        if (!context) {
-            context = new TypeContext(instance.constructor as typeof Schema);
-        }
+    static encode(instance: Schema, context?: TypeContext, it: Iterator = { offset: 0 }) {
+        context ??= new TypeContext(instance.constructor as typeof Schema);
 
         const reflection = new Reflection();
         const encoder = new Encoder(reflection);
@@ -96,7 +95,6 @@ export class Reflection extends Schema {
             buildType(type, klass[Symbol.metadata]);
         }
 
-        const it = { offset: 0 };
         const buf = encoder.encodeAll(it);
         return Buffer.from(buf, 0, it.offset);
     }
@@ -107,62 +105,72 @@ export class Reflection extends Schema {
         const reflectionDecoder = new Decoder(reflection);
         reflectionDecoder.decode(bytes, it);
 
-        const context = new TypeContext();
+        const typeContext = new TypeContext();
 
-        const schemaTypes = reflection.types.reduce((types, reflectionType) => {
-            const parentKlass: typeof Schema = types[reflectionType.extendsId] || Schema;
-            const schema: typeof Schema = class _ extends parentKlass {};
+        // 1st pass, initialize metadata + inheritance
+        reflection.types.forEach((reflectionType) => {
+            const parentClass: typeof Schema = typeContext.get(reflectionType.extendsId) ?? Schema;
+            const schema: typeof Schema = class _ extends parentClass {};
 
-            // const _metadata = Object.create(_classSuper[Symbol.metadata] ?? null);
-            const _metadata = parentKlass && parentKlass[Symbol.metadata] || Object.create(null);
-            Object.defineProperty(schema, Symbol.metadata, { enumerable: true, configurable: true, writable: true, value: _metadata })
+            const parentMetadata = parentClass[Symbol.metadata];
 
             // register for inheritance support
             TypeContext.register(schema);
 
-            const typeid = reflectionType.id;
-            types[typeid] = schema
-            context.add(schema, typeid);
-            return types;
+            // for inheritance support
+            Metadata.initialize(schema, parentMetadata);
+
+            typeContext.add(schema, reflectionType.id);
         }, {});
 
+        // 2nd pass, set fields
         reflection.types.forEach((reflectionType) => {
-            const schemaType = schemaTypes[reflectionType.id];
+            const schemaType = typeContext.get(reflectionType.id);
             const metadata = schemaType[Symbol.metadata];
 
-            const parentKlass = reflection.types[reflectionType.extendsId];
-            const parentFieldIndex = parentKlass && parentKlass.fields.length || 0;
+            // FIXME: use metadata[-1] to get field count
+            const parentFieldIndex = 0;
+
+            // console.log("--------------------");
+            // // console.log("reflectionType", reflectionType.toJSON());
+            // console.log("reflectionType.fields", reflectionType.fields.toJSON());
+            // console.log("parentFieldIndex", parentFieldIndex);
+
+            //
+            // FIXME: set fields using parentKlass as well
+            // currently the fields are duplicated on inherited classes
+            //
+            // // const parentKlass = reflection.types[reflectionType.extendsId];
+            // // parentKlass.fields
 
             reflectionType.fields.forEach((field, i) => {
                 const fieldIndex = parentFieldIndex + i;
 
                 if (field.referencedType !== undefined) {
                     let fieldType = field.type;
-                    let refType = schemaTypes[field.referencedType];
+                    let refType: PrimitiveType = typeContext.get(field.referencedType);
 
                     // map or array of primitive type (-1)
                     if (!refType) {
                         const typeInfo = field.type.split(":");
                         fieldType = typeInfo[0];
-                        refType = typeInfo[1];
+                        refType = typeInfo[1] as PrimitiveType; // string
                     }
 
                     if (fieldType === "ref") {
-                        // type(refType)(schemaType.prototype, field.name);
                         Metadata.addField(metadata, fieldIndex, field.name, refType);
 
                     } else {
-                        // type({ [fieldType]: refType } as DefinitionType)(schemaType.prototype, field.name);
                         Metadata.addField(metadata, fieldIndex, field.name, { [fieldType]: refType } as DefinitionType);
                     }
 
                 } else {
-                    // type(field.type as PrimitiveType)(schemaType.prototype, field.name);
                     Metadata.addField(metadata, fieldIndex, field.name, field.type as PrimitiveType);
                 }
             });
         });
 
-        return new (schemaTypes[0])();
+        // @ts-ignore
+        return new (typeContext.get(0))();
     }
 }

package/src/Schema.ts

@@ -248,7 +248,7 @@ export abstract class Schema {
         return output;
     }
 
-    static debugChangesDeep(ref: Ref) {
+    static debugChangesDeep(ref: Ref, changeSetName: "changes" | "allChanges" | "allFilteredChanges" | "filteredChanges" = "changes") {
         let output = "";
 
         const rootChangeTree = ref[$changes];
@@ -257,7 +257,7 @@ export abstract class Schema {
         let totalInstances = 0;
         let totalOperations = 0;
 
-        for (const [changeTree, changes] of (rootChangeTree.root.changes.entries())) {
+        for (const [changeTree, changes] of (rootChangeTree.root[changeSetName].entries())) {
             let includeChangeTree = false;
             let parentChangeTrees: ChangeTree[] = [];
             let parentChangeTree = changeTree.parent?.[$changes];

package/src/annotations.ts

@@ -6,6 +6,7 @@ import { Metadata } from "./Metadata";
 import { $changes, $childType, $track } from "./types/symbols";
 import { TypeDefinition, getType } from "./types/registry";
 import { OPERATION } from "./encoding/spec";
+import { TypeContext } from "./types/TypeContext";
 
 /**
  * Data types
@@ -42,113 +43,6 @@ export interface TypeOptions {
 
 export const DEFAULT_VIEW_TAG = -1;
 
-export class TypeContext {
-    types: {[id: number]: typeof Schema} = {};
-    schemas = new Map<typeof Schema, number>();
-
-    hasFilters: boolean = false;
-
-    /**
-     * For inheritance support
-     * Keeps track of which classes extends which. (parent -> children)
-     */
-    static inheritedTypes = new Map<typeof Schema, Set<typeof Schema>>();
-
-    static register(target: typeof Schema) {
-        const parent = Object.getPrototypeOf(target);
-        if (parent !== Schema) {
-            let inherits = TypeContext.inheritedTypes.get(parent);
-            if (!inherits) {
-                inherits = new Set<typeof Schema>();
-                TypeContext.inheritedTypes.set(parent, inherits);
-            }
-            inherits.add(target);
-        }
-    }
-
-    constructor(rootClass?: typeof Schema) {
-        if (rootClass) {
-            this.discoverTypes(rootClass);
-        }
-    }
-
-    has(schema: typeof Schema) {
-        return this.schemas.has(schema);
-    }
-
-    get(typeid: number) {
-        return this.types[typeid];
-    }
-
-    add(schema: typeof Schema, typeid: number = this.schemas.size) {
-        // skip if already registered
-        if (this.schemas.has(schema)) {
-            return false;
-        }
-
-        this.types[typeid] = schema;
-        this.schemas.set(schema, typeid);
-        return true;
-    }
-
-    getTypeId(klass: typeof Schema) {
-        return this.schemas.get(klass);
-    }
-
-    private discoverTypes(klass: typeof Schema) {
-        if (!this.add(klass)) {
-            return;
-        }
-
-        // add classes inherited from this base class
-        TypeContext.inheritedTypes.get(klass)?.forEach((child) => {
-            this.discoverTypes(child);
-        });
-
-        // skip if no fields are defined for this class.
-        if (klass[Symbol.metadata] === undefined) {
-            klass[Symbol.metadata] = {};
-        }
-
-        // const metadata = Metadata.getFor(klass);
-        const metadata = klass[Symbol.metadata];
-
-        // if any schema/field has filters, mark "context" as having filters.
-        if (metadata[-2]) {
-            this.hasFilters = true;
-        }
-
-        for (const field in metadata) {
-            const fieldType = metadata[field].type;
-
-            if (typeof(fieldType) === "string") {
-                continue;
-            }
-
-            if (Array.isArray(fieldType)) {
-                const type = fieldType[0];
-                if (type === "string") {
-                    continue;
-                }
-                this.discoverTypes(type as typeof Schema);
-
-            } else if (typeof(fieldType) === "function") {
-                this.discoverTypes(fieldType);
-
-            } else {
-                const type = Object.values(fieldType)[0];
-
-                // skip primitive types
-                if (typeof(type) === "string") {
-                    continue;
-                }
-
-                this.discoverTypes(type as typeof Schema);
-            }
-        }
-    }
-}
-
 export function entity(constructor, context: ClassDecoratorContext) {
     if (!constructor._definition) {
         // for inheritance support
@@ -330,6 +224,8 @@ export function view<T> (tag: number = DEFAULT_VIEW_TAG) {
 
         const parentClass = Object.getPrototypeOf(constructor);
         const parentMetadata = parentClass[Symbol.metadata];
+
+        // TODO: use Metadata.initialize()
         const metadata: Metadata = (constructor[Symbol.metadata] ??= Object.assign({}, constructor[Symbol.metadata], parentMetadata ?? Object.create(null)));
 
         if (!metadata[fieldName]) {
@@ -356,6 +252,8 @@ export function unreliable<T> (target: T, field: string) {
 
     const parentClass = Object.getPrototypeOf(constructor);
     const parentMetadata = parentClass[Symbol.metadata];
+
+    // TODO: use Metadata.initialize()
     const metadata: Metadata = (constructor[Symbol.metadata] ??= Object.assign({}, constructor[Symbol.metadata], parentMetadata ?? Object.create(null)));
 
     if (!metadata[field]) {
@@ -389,8 +287,8 @@ export function type (
         TypeContext.register(constructor);
 
         const parentClass = Object.getPrototypeOf(constructor);
-        const parentMetadata = parentClass[Symbol.metadata];
-        const metadata: Metadata = (constructor[Symbol.metadata] ??= Object.assign({}, constructor[Symbol.metadata], parentMetadata ?? Object.create(null)));
+        const parentMetadata = parentClass && parentClass[Symbol.metadata];
+        const metadata = Metadata.initialize(constructor, parentMetadata);
 
         let fieldIndex: number;
 

package/src/bench_encode.ts

@@ -0,0 +1,97 @@
+import { nanoid } from "nanoid";
+import { Schema, type, MapSchema, ArraySchema, Encoder } from ".";
+import * as benchmark from "benchmark";
+
+const suite = new benchmark.Suite();
+
+class Attribute extends Schema {
+    @type("string") name: string;
+    @type("number") value: number;
+}
+
+class Item extends Schema {
+    @type("number") price: number;
+    @type([ Attribute ]) attributes = new ArraySchema<Attribute>();
+}
+
+class Position extends Schema {
+    @type("number") x: number;
+    @type("number") y: number;
+}
+
+class Player extends Schema {
+    @type(Position) position = new Position();
+    @type({ map: Item }) items = new MapSchema<Item>();
+}
+
+class State extends Schema {
+    @type({ map: Player }) players = new MapSchema<Player>();
+    @type("string") currentTurn;
+}
+
+const state = new State();
+
+for (let i = 0; i < 50; i++) {
+    const player = new Player();
+    state.players.set(`p-${nanoid()}`, player);
+
+    player.position.x = (i + 1) * 100;
+    player.position.y = (i + 1) * 100;
+    for (let j = 0; j < 10; j++) {
+        const item = new Item();
+        item.price = (i + 1) * 50;
+        for (let k = 0; k < 5; k++) {
+            const attr = new Attribute();
+            attr.name = `Attribute ${k}`;
+            attr.value = k;
+            item.attributes.push(attr);
+
+        }
+        player.items.set(`item-${j}`, item);
+    }
+}
+
+
+Encoder.BUFFER_SIZE = 4096 * 4096;
+const encoder = new Encoder(state);
+
+// measure time to .encodeAll()
+
+let now = Date.now();
+for (let i = 0; i < 1000; i++) {
+    encoder.encodeAll();
+}
+console.log(Date.now() - now);
+
+const allEncodes = Date.now();
+for (let i = 0; i < 100; i++) {
+    now = Date.now();
+    for (let j = 0; j < 50; j++) {
+        const player = new Player();
+        state.players.set(`p-${nanoid()}`, player);
+
+        player.position.x = (j + 1) * 100;
+        player.position.y = (j + 1) * 100;
+        for (let k = 0; k < 10; k++) {
+            const item = new Item();
+            item.price = (j + 1) * 50;
+            for (let l = 0; l < 5; l++) {
+                const attr = new Attribute();
+                attr.name = `Attribute ${l}`;
+                attr.value = l;
+                item.attributes.push(attr);
+
+            }
+            player.items.set(`item-${k}`, item);
+        }
+    }
+    console.log("time to make changes:", Date.now() - now);
+
+    now = Date.now();
+    encoder.encode();
+    encoder.discardChanges();
+    console.log("time to encode:", Date.now() - now);
+}
+console.log("time for all encodes:", Date.now() - allEncodes);
+
+console.log(Array.from(encoder.encodeAll()).length, "bytes");

package/src/debug.ts

@@ -0,0 +1,56 @@
+import * as fs from "fs";
+import { Reflection, Decoder } from "./index";
+
+const contents = fs.readFileSync("/Users/endel/Projects/colyseus/clients/bubbits/project/@bubbits/backend/schema-debug.txt", { encoding: "utf8" }).toString();
+
+let isCommentBlock = false;
+let lastComment = "";
+
+let decoder: Decoder;
+
+function getBuffer(line: string) {
+    const start = line.lastIndexOf(":");
+    const buffer = Buffer.from(new Uint8Array(line.substring(start + 1).split(",").map(n => Number(n))));
+    console.log(`(${buffer.byteLength}) ${Array.from(buffer).join(",")}`)
+    // console.log("");
+    // console.log("");
+    // console.log("> ", line);
+    // console.log("> substring:", line.substring(start + 1))
+    return buffer;
+}
+
+function decode(buffer: Buffer) {
+    try {
+        decoder.decode(buffer);
+    } catch (e) {
+        console.error(e);
+        console.log("Last log:\n\n")
+        console.log(lastComment);
+    }
+}
+
+contents.split("\n").forEach((line) => {
+    if (line.startsWith("#")) {
+        // reset last comment.
+        if (isCommentBlock === false) { lastComment = ""; }
+
+        isCommentBlock = true;
+        lastComment += line.substring(line.indexOf(":") + 1) + "\n";
+        return;
+    }
+
+    isCommentBlock = false;
+
+    if (line.startsWith("handshake:") && !decoder) {
+        const state = Reflection.decode(getBuffer(line));
+        decoder = new Decoder(state);
+
+    } else if (line.startsWith("state:")) {
+        decode(getBuffer(line));
+
+    } else if (line.startsWith("patch:")) {
+        decode(getBuffer(line));
+    }
+});
+
+console.log(decoder.state.toJSON());

package/src/decoder/DecodeOperation.ts

@@ -4,7 +4,7 @@ import { Schema } from "../Schema";
 import type { Ref } from "../encoder/ChangeTree";
 import type { Decoder } from "./Decoder";
 import * as decode from "../encoding/decode";
-import { $changes, $childType, $deleteByIndex, $getByIndex } from "../types/symbols";
+import { $childType, $deleteByIndex, $getByIndex } from "../types/symbols";
 
 import type { MapSchema } from "../types/custom/MapSchema";
 import type { ArraySchema } from "../types/custom/ArraySchema";
@@ -43,7 +43,7 @@ export function decodeValue(
     it: decode.Iterator,
     allChanges: DataChange[],
 ) {
-    const $root = decoder.$root;
+    const $root = decoder.root;
     const previousValue = ref[$getByIndex](index);
 
     let value: any;
@@ -182,7 +182,10 @@ export const decodeSchemaOperation: DecodeOperation = function (
 
     // skip early if field is not defined
     const field = metadata[index];
-    if (field === undefined) { return DEFINITION_MISMATCH; }
+    if (field === undefined) {
+        console.warn("@colyseus/schema: field not defined at", { index, ref: ref.constructor.name, metadata });
+        return DEFINITION_MISMATCH;
+    }
 
     const { value, previousValue } = decodeValue(
         decoder,
@@ -251,6 +254,7 @@ export const decodeKeyValueOperation: DecodeOperation = function (
         dynamicIndex = ref['getIndex'](index);
     }
 
+
     const { value, previousValue } = decodeValue(
         decoder,
         operation,
@@ -303,7 +307,10 @@ export const decodeArray: DecodeOperation = function (
     allChanges: DataChange[]
 ) {
     // "uncompressed" index + operation (array/map items)
-    const operation = bytes[it.offset++];
+    let operation = bytes[it.offset++];
+
+    let isSchemaChild: boolean;
+    let index: number;
 
     if (operation === OPERATION.CLEAR) {
         //
@@ -318,8 +325,8 @@ export const decodeArray: DecodeOperation = function (
     } else if (operation === OPERATION.DELETE_BY_REFID) {
         // TODO: refactor here, try to follow same flow as below
         const refId = decode.number(bytes, it);
-        const previousValue = decoder.$root.refs.get(refId);
-        const index = ref.findIndex((value) => value === previousValue);
+        const previousValue = decoder.root.refs.get(refId);
+        index = ref.findIndex((value) => value === previousValue);
         ref[$deleteByIndex](index);
         allChanges.push({
             ref,
@@ -330,10 +337,26 @@ export const decodeArray: DecodeOperation = function (
             value: undefined,
             previousValue,
         });
+
         return;
+
+    } else if (operation === OPERATION.ADD_BY_REFID) {
+        isSchemaChild = true;
+        // operation = OPERATION.ADD;
+
+        const refId = decode.number(bytes, it);
+        const itemByRefId = decoder.root.refs.get(refId);
+
+        // use existing index, or push new value
+        index = (itemByRefId)
+            ? ref.findIndex((value) => value === itemByRefId)
+            : ref.length;
+
+    } else {
+        isSchemaChild = false;
+        index = decode.number(bytes, it);
     }
 
-    const index = decode.number(bytes, it);
     const type = ref[$childType];
 
     let dynamicIndex: number | string = index;

package/src/decoder/Decoder.ts

@@ -1,27 +1,28 @@
-import { TypeContext } from "../annotations";
+import { TypeContext } from "../types/TypeContext";
 import { $changes, $childType, $decoder, $onDecodeEnd } from "../types/symbols";
 import { Schema } from "../Schema";
 
 import * as decode from "../encoding/decode";
 import { OPERATION, SWITCH_TO_STRUCTURE, TYPE_ID } from '../encoding/spec';
-import { Ref } from "../encoder/ChangeTree";
-import { Iterator } from "../encoding/decode";
+import type { Ref } from "../encoder/ChangeTree";
+import type { Iterator } from "../encoding/decode";
 import { ReferenceTracker } from "./ReferenceTracker";
-import { DEFINITION_MISMATCH, DataChange, DecodeOperation } from "./DecodeOperation";
+import { DEFINITION_MISMATCH, type DataChange, type DecodeOperation } from "./DecodeOperation";
 import { Collection } from "../types/HelperTypes";
 
 export class Decoder<T extends Schema = any> {
     context: TypeContext;
 
     state: T;
-    $root: ReferenceTracker;
+    root: ReferenceTracker;
 
     currentRefId: number = 0;
 
     triggerChanges?: (allChanges: DataChange[]) => void;
 
     constructor(root: T, context?: TypeContext) {
-        this.setRoot(root);
+        this.setState(root);
+
         this.context = context || new TypeContext(root.constructor as typeof Schema);
 
         // console.log(">>>>>>>>>>>>>>>> Decoder types");
@@ -30,10 +31,10 @@ export class Decoder<T extends Schema = any> {
         // });
     }
 
-    protected setRoot(root: T) {
+    protected setState(root: T) {
         this.state = root;
-        this.$root = new ReferenceTracker();
-        this.$root.addRef(0, root);
+        this.root = new ReferenceTracker();
+        this.root.addRef(0, root);
     }
 
     decode(
@@ -43,7 +44,7 @@ export class Decoder<T extends Schema = any> {
     ) {
         const allChanges: DataChange[] = [];
 
-        const $root = this.$root;
+        const $root = this.root;
         const totalBytes = bytes.byteLength;
 
         let decoder: DecodeOperation = ref['constructor'][$decoder];
@@ -66,6 +67,7 @@ export class Decoder<T extends Schema = any> {
                 if (!nextRef) { throw new Error(`"refId" not found: ${this.currentRefId}`); }
                 ref[$onDecodeEnd]?.()
                 ref = nextRef;
+
                 decoder = ref['constructor'][$decoder];
 
                 continue;
@@ -146,7 +148,7 @@ export class Decoder<T extends Schema = any> {
             });
 
             if (needRemoveRef) {
-                this.$root.removeRef(this.$root.refIds.get(value));
+                this.root.removeRef(this.root.refIds.get(value));
             }
         });
     }