@colyseus/schema 3.0.0-alpha.27 → 3.0.0-alpha.29

package/src/annotations.ts

@@ -6,6 +6,7 @@ import { Metadata } from "./Metadata";
 import { $changes, $childType, $track } from "./types/symbols";
 import { TypeDefinition, getType } from "./types/registry";
 import { OPERATION } from "./encoding/spec";
+import { TypeContext } from "./types/TypeContext";
 
 /**
  * Data types
@@ -42,132 +43,6 @@ export interface TypeOptions {
 
 export const DEFAULT_VIEW_TAG = -1;
 
-export class TypeContext {
-    types: {[id: number]: typeof Schema} = {};
-    schemas = new Map<typeof Schema, number>();
-
-    hasFilters: boolean = false;
-
-    /**
-     * For inheritance support
-     * Keeps track of which classes extends which. (parent -> children)
-     */
-    static inheritedTypes = new Map<typeof Schema, Set<typeof Schema>>();
-
-    static register(target: typeof Schema) {
-        const parent = Object.getPrototypeOf(target);
-        if (parent !== Schema) {
-            let inherits = TypeContext.inheritedTypes.get(parent);
-            if (!inherits) {
-                inherits = new Set<typeof Schema>();
-                TypeContext.inheritedTypes.set(parent, inherits);
-            }
-            inherits.add(target);
-        }
-    }
-
-    constructor(rootClass?: typeof Schema) {
-        if (rootClass) {
-            this.discoverTypes(rootClass);
-        }
-    }
-
-    has(schema: typeof Schema) {
-        return this.schemas.has(schema);
-    }
-
-    get(typeid: number) {
-        return this.types[typeid];
-    }
-
-    add(schema: typeof Schema, typeid: number = this.schemas.size) {
-        // skip if already registered
-        if (this.schemas.has(schema)) {
-            return false;
-        }
-
-        this.types[typeid] = schema;
-
-        //
-        // Workaround to allow using an empty Schema (with no `@type()` fields)
-        //
-        if (schema[Symbol.metadata] === undefined) {
-            Metadata.init(schema);
-        }
-
-        this.schemas.set(schema, typeid);
-        return true;
-    }
-
-    getTypeId(klass: typeof Schema) {
-        return this.schemas.get(klass);
-    }
-
-    private discoverTypes(klass: typeof Schema, parentFieldViewTag?: number) {
-        if (!this.add(klass)) {
-            return;
-        }
-
-        // add classes inherited from this base class
-        TypeContext.inheritedTypes.get(klass)?.forEach((child) => {
-            this.discoverTypes(child, parentFieldViewTag);
-        });
-
-        // skip if no fields are defined for this class.
-        if (klass[Symbol.metadata] === undefined) {
-            klass[Symbol.metadata] = {};
-        }
-
-        // const metadata = Metadata.getFor(klass);
-        const metadata = klass[Symbol.metadata];
-
-        // if any schema/field has filters, mark "context" as having filters.
-        if (metadata[-2]) {
-            this.hasFilters = true;
-        }
-
-        for (const field in metadata) {
-            //
-            // Modify the field's metadata to include the parent field's view tag
-            //
-            if (
-                parentFieldViewTag !== undefined &&
-                metadata[field].tag === undefined
-            ) {
-                metadata[field].tag = parentFieldViewTag;
-            }
-
-            const fieldType = metadata[field].type;
-            const viewTag = metadata[field].tag;
-
-            if (typeof(fieldType) === "string") {
-                continue;
-            }
-
-            if (Array.isArray(fieldType)) {
-                const type = fieldType[0];
-                if (type === "string") {
-                    continue;
-                }
-                this.discoverTypes(type as typeof Schema, viewTag);
-
-            } else if (typeof(fieldType) === "function") {
-                this.discoverTypes(fieldType, viewTag);
-
-            } else {
-                const type = Object.values(fieldType)[0];
-
-                // skip primitive types
-                if (typeof(type) === "string") {
-                    continue;
-                }
-
-                this.discoverTypes(type as typeof Schema, viewTag);
-            }
-        }
-    }
-}
-
 export function entity(constructor, context: ClassDecoratorContext) {
     if (!constructor._definition) {
         // for inheritance support

package/src/bench_encode.ts

@@ -4,44 +4,44 @@ import * as benchmark from "benchmark";
 
 const suite = new benchmark.Suite();
 
-class AttributeNew extends Schema {
+class Attribute extends Schema {
     @type("string") name: string;
     @type("number") value: number;
 }
 
-class ItemNew extends Schema {
+class Item extends Schema {
     @type("number") price: number;
-    @type([ AttributeNew ]) attributes = new ArraySchema<AttributeNew>();
+    @type([ Attribute ]) attributes = new ArraySchema<Attribute>();
 }
 
-class PositionNew extends Schema {
+class Position extends Schema {
     @type("number") x: number;
     @type("number") y: number;
 }
 
-class PlayerNew extends Schema {
-    @type(PositionNew) position = new PositionNew();
-    @type({ map: ItemNew }) items = new MapSchema<ItemNew>();
+class Player extends Schema {
+    @type(Position) position = new Position();
+    @type({ map: Item }) items = new MapSchema<Item>();
 }
 
-class StateNew extends Schema {
-    @type({ map: PlayerNew }) players = new MapSchema<PlayerNew>();
+class State extends Schema {
+    @type({ map: Player }) players = new MapSchema<Player>();
     @type("string") currentTurn;
 }
 
-const state = new StateNew();
+const state = new State();
 
 for (let i = 0; i < 50; i++) {
-    const player = new PlayerNew();
+    const player = new Player();
     state.players.set(`p-${nanoid()}`, player);
 
     player.position.x = (i + 1) * 100;
     player.position.y = (i + 1) * 100;
     for (let j = 0; j < 10; j++) {
-        const item = new ItemNew();
+        const item = new Item();
         item.price = (i + 1) * 50;
         for (let k = 0; k < 5; k++) {
-            const attr = new AttributeNew();
+            const attr = new Attribute();
             attr.name = `Attribute ${k}`;
             attr.value = k;
             item.attributes.push(attr);
@@ -52,15 +52,46 @@ for (let i = 0; i < 50; i++) {
 }
 
 
-Encoder.BUFFER_SIZE = 1024 * 128;
+Encoder.BUFFER_SIZE = 4096 * 4096;
 const encoder = new Encoder(state);
 
 // measure time to .encodeAll()
 
-const now = Date.now();
+let now = Date.now();
 for (let i = 0; i < 1000; i++) {
     encoder.encodeAll();
 }
 console.log(Date.now() - now);
 
-console.log(Array.from(encoder.encodeAll()).join(","));
+const allEncodes = Date.now();
+for (let i = 0; i < 100; i++) {
+    now = Date.now();
+    for (let j = 0; j < 50; j++) {
+        const player = new Player();
+        state.players.set(`p-${nanoid()}`, player);
+
+        player.position.x = (j + 1) * 100;
+        player.position.y = (j + 1) * 100;
+        for (let k = 0; k < 10; k++) {
+            const item = new Item();
+            item.price = (j + 1) * 50;
+            for (let l = 0; l < 5; l++) {
+                const attr = new Attribute();
+                attr.name = `Attribute ${l}`;
+                attr.value = l;
+                item.attributes.push(attr);
+
+            }
+            player.items.set(`item-${k}`, item);
+        }
+    }
+    console.log("time to make changes:", Date.now() - now);
+
+    now = Date.now();
+    encoder.encode();
+    encoder.discardChanges();
+    console.log("time to encode:", Date.now() - now);
+}
+console.log("time for all encodes:", Date.now() - allEncodes);
+
+console.log(Array.from(encoder.encodeAll()).length, "bytes");

package/src/decoder/Decoder.ts

@@ -1,4 +1,4 @@
-import { TypeContext } from "../annotations";
+import { TypeContext } from "../types/TypeContext";
 import { $changes, $childType, $decoder, $onDecodeEnd } from "../types/symbols";
 import { Schema } from "../Schema";
 

package/src/decoder/strategy/StateCallbacks.ts

@@ -214,7 +214,7 @@ export function getDecoderStateCallbacks<T extends Schema>(decoder: Decoder<T>):
 
         if (metadata && !isCollection) {
 
-            const onAdd = function (
+            const onAddListen = function (
                 ref: Ref,
                 prop: string,
                 callback: (value: any, previousValue: any) => void, immediate: boolean
@@ -223,7 +223,7 @@ export function getDecoderStateCallbacks<T extends Schema>(decoder: Decoder<T>):
                 if (
                     immediate &&
                     context.instance[prop] !== undefined &&
-                    !onAddCalls.has(callback) // Workaround for https://github.com/colyseus/schema/issues/147
+                    !onAddCalls.has(currentOnAddCallback) // Workaround for https://github.com/colyseus/schema/issues/147
                 ) {
                     callback(context.instance[prop], undefined);
                 }
@@ -236,14 +236,14 @@ export function getDecoderStateCallbacks<T extends Schema>(decoder: Decoder<T>):
             return new Proxy({
                 listen: function listen(prop: string, callback: (value: any, previousValue: any) => void, immediate: boolean = true) {
                     if (context.instance) {
-                        return onAdd(context.instance, prop, callback, immediate);
+                        return onAddListen(context.instance, prop, callback, immediate);
 
                     } else {
                         // collection instance not received yet
                         let detachCallback = () => {};
 
                         context.onInstanceAvailable((ref: Ref, existing: boolean) => {
-                            detachCallback = onAdd(ref, prop, callback, immediate && existing)
+                            detachCallback = onAddListen(ref, prop, callback, immediate && existing && !onAddCalls.has(currentOnAddCallback))
                         });
 
                         return () => detachCallback();
@@ -330,6 +330,7 @@ export function getDecoderStateCallbacks<T extends Schema>(decoder: Decoder<T>):
                     currentOnAddCallback = callback;
                     callback(value, key);
                     onAddCalls.delete(callback)
+                    currentOnAddCallback = undefined;
                 });
             };
 

package/src/encoder/ChangeTree.ts

@@ -7,6 +7,7 @@ import type { ArraySchema } from "../types/custom/ArraySchema";
 import type { CollectionSchema } from "../types/custom/CollectionSchema";
 import type { SetSchema } from "../types/custom/SetSchema";
 
+import { Root } from "./Root";
 import { Metadata } from "../Metadata";
 import type { EncodeOperation } from "./EncodeOperation";
 import type { DecodeOperation } from "../decoder/DecodeOperation";
@@ -26,53 +27,6 @@ export type Ref = Schema
     | CollectionSchema
     | SetSchema;
 
-export class Root {
-    protected nextUniqueId: number = 0;
-    refCount = new WeakMap<ChangeTree, number>();
-
-    // all changes
-    allChanges = new Map<ChangeTree, Map<number, OPERATION>>();
-    allFilteredChanges = new Map<ChangeTree, Map<number, OPERATION>>();
-
-    // pending changes to be encoded
-    changes = new Map<ChangeTree, Map<number, OPERATION>>();
-    filteredChanges = new Map<ChangeTree, Map<number, OPERATION>>();
-
-    getNextUniqueId() {
-        return this.nextUniqueId++;
-    }
-
-    add (changeTree: ChangeTree) {
-        const refCount = this.refCount.get(changeTree) || 0;
-        this.refCount.set(changeTree, refCount + 1);
-    }
-
-    remove(changeTree: ChangeTree) {
-        const refCount = this.refCount.get(changeTree);
-        if (refCount <= 1) {
-            this.allChanges.delete(changeTree);
-            this.changes.delete(changeTree);
-
-            if (changeTree.isFiltered || changeTree.isPartiallyFiltered) {
-                this.allFilteredChanges.delete(changeTree);
-                this.filteredChanges.delete(changeTree);
-            }
-
-            this.refCount.delete(changeTree);
-
-        } else {
-            this.refCount.set(changeTree, refCount - 1);
-        }
-
-        changeTree.forEachChild((child, _) =>
-            this.remove(child));
-    }
-
-    clear() {
-        this.changes.clear();
-    }
-}
-
 export class ChangeTree<T extends Ref=any> {
     ref: T;
     refId: number;
@@ -122,9 +76,6 @@ export class ChangeTree<T extends Ref=any> {
         if (this.isFiltered || this.isPartiallyFiltered) {
             this.root.allFilteredChanges.set(this, this.allFilteredChanges);
             this.root.filteredChanges.set(this, this.filteredChanges);
-
-        // } else {
-        //     this.root.allChanges.set(this, this.allChanges);
         }
 
         if (!this.isFiltered) {
@@ -453,19 +404,39 @@ export class ChangeTree<T extends Ref=any> {
         // Detect if current structure has "filters" declared
         this.isPartiallyFiltered = (this.ref['constructor']?.[Symbol.metadata]?.[-2] !== undefined);
 
-        // TODO: support "partially filtered", where the instance is visible, but only a field is not.
+        if (parent && !Metadata.isValidInstance(parent)) {
+            const parentChangeTree = parent[$changes];
+            parent = parentChangeTree.parent;
+            parentIndex = parentChangeTree.parentIndex;
+        }
+
+        const parentMetadata = parent?.['constructor']?.[Symbol.metadata];
 
-        // Detect if parent has "filters" declared
-        while (parent && !this.isFiltered) {
-            const metadata: Metadata = parent['constructor'][Symbol.metadata];
+        this.isFiltered = (
+            parent &&
+            parentMetadata?.[-2]?.includes(parentIndex)
+        );
 
-            const fieldName = metadata?.[parentIndex];
-            const isParentOwned = metadata?.[fieldName]?.tag !== undefined;
+        // this.isFiltered = this.ref['constructor']?.[Symbol.metadata]?.[-4];
 
-            this.isFiltered = isParentOwned || parent[$changes].isFiltered; // metadata?.[-2]
+        // // Detect if parent has "filters" declared
+        // while (parent && !this.isFiltered) {
+        //     const metadata: Metadata = parent['constructor'][Symbol.metadata];
+        //     // this.isFiltered = metadata?.[-4];
 
-            parent = parent[$changes].parent;
-        };
+        //     const fieldName = metadata?.[parentIndex];
+        //     const isParentOwned = metadata?.[fieldName]?.tag !== undefined;
+        //     this.isFiltered = isParentOwned || parent[$changes].isFiltered; // metadata?.[-2]
+
+        //     parent = parent[$changes].parent;
+        // };
+
+        // console.log("ChangeTree.checkIsFiltered", {
+        //     parent: parent?.constructor.name,
+        //     ref: this.ref.constructor.name,
+        //     isFiltered: this.isFiltered,
+        //     isPartiallyFiltered: this.isPartiallyFiltered,
+        // });
 
         //
         // TODO: refactor this!

package/src/encoder/Encoder.ts

@@ -1,15 +1,17 @@
 import type { Schema } from "../Schema";
-import { TypeContext } from "../annotations";
+import { TypeContext } from "../types/TypeContext";
 import { $changes, $encoder, $filter } from "../types/symbols";
 
 import * as encode from "../encoding/encode";
 import type { Iterator } from "../encoding/decode";
 
 import { OPERATION, SWITCH_TO_STRUCTURE, TYPE_ID } from '../encoding/spec';
-import { Root } from "./ChangeTree";
+import { Root } from "./Root";
 import { getNextPowerOf2 } from "../utils";
+
 import type { StateView } from "./StateView";
-import { Metadata } from "../Metadata";
+import type { Metadata } from "../Metadata";
+import type { ChangeTree } from "./ChangeTree";
 
 export class Encoder<T extends Schema = any> {
     static BUFFER_SIZE = 8 * 1024;// 8KB
@@ -21,13 +23,13 @@ export class Encoder<T extends Schema = any> {
     root: Root;
 
     constructor(state: T) {
-        this.root = new Root();
 
         //
         // TODO: cache and restore "Context" based on root schema
         // (to avoid creating a new context for every new room)
         //
         this.context = new TypeContext(state.constructor as typeof Schema);
+        this.root = new Root(this.context);
 
         this.setState(state);
 
@@ -80,7 +82,8 @@ export class Encoder<T extends Schema = any> {
             }
 
             // skip root `refId` if it's the first change tree
-            if (it.offset !== initialOffset || changeTree !== rootChangeTree) {
+            // (unless it "hasView", which will need to revisit the root)
+            if (hasView || changeTree !== rootChangeTree) {
                 buffer[it.offset++] = SWITCH_TO_STRUCTURE & 255;
                 encode.number(buffer, changeTree.refId, it);
             }
@@ -124,7 +127,6 @@ export class Encoder<T extends Schema = any> {
     Encoder.BUFFER_SIZE = ${Math.round(newSize / 1024)} * 1024; // ${Math.round(newSize / 1024)} KB
 `);
 
-
             //
             // resize buffer and re-encode (TODO: can we avoid re-encoding here?)
             //
@@ -153,11 +155,8 @@ export class Encoder<T extends Schema = any> {
     }
 
     encodeAll(it: Iterator = { offset: 0 }, buffer: Buffer = this.sharedBuffer) {
-        // console.log(`encodeAll(), this.root.allChanges (${this.root.allChanges.size})`);
-
-        // Array.from(this.root.allChanges.entries()).map((item) => {
-        //     console.log("->", { ref: item[0].ref.constructor.name, refId: item[0].refId, changes: item[1].size });
-        // });
+        // console.log(`\nencodeAll(), this.root.allChanges (${this.root.allChanges.size})`);
+        // this.debugChanges("allChanges");
 
         return this.encode(it, undefined, buffer, this.root.allChanges, true);
     }
@@ -165,11 +164,11 @@ export class Encoder<T extends Schema = any> {
     encodeAllView(view: StateView, sharedOffset: number, it: Iterator, bytes = this.sharedBuffer) {
         const viewOffset = it.offset;
 
-        // console.log(`encodeAllView(), this.root.allFilteredChanges (${this.root.allFilteredChanges.size})`);
-        // this.debugAllFilteredChanges();
+        // console.log(`\nencodeAllView(), this.root.allFilteredChanges (${this.root.allFilteredChanges.size})`);
+        // this.debugChanges("allFilteredChanges");
 
         // try to encode "filtered" changes
-        this.encode(it, view, bytes, this.root.allFilteredChanges, true);
+        this.encode(it, view, bytes, this.root.allFilteredChanges, true, viewOffset);
 
         return Buffer.concat([
             bytes.subarray(0, sharedOffset),
@@ -177,21 +176,35 @@ export class Encoder<T extends Schema = any> {
         ]);
     }
 
-
-    debugAllFilteredChanges() {
-        Array.from(this.root.allFilteredChanges.entries()).map((item) => {
-            console.log("->", { refId: item[0].refId, changes: item[1].size }, item[0].ref.toJSON());
-            if (Array.isArray(item[0].ref.toJSON())) {
-                item[1].forEach((op, key) => {
-                    console.log("  ->", { key, op: OPERATION[op] });
-                })
-            }
+    debugChanges(
+        field:  "changes" | "allFilteredChanges" | "allChanges" | "filteredChanges" | Map<ChangeTree, Map<number, OPERATION>>
+    ) {
+        const changeSet = (typeof (field) === "string")
+            ? this.root[field]
+            : field;
+
+        Array.from(changeSet.entries()).map((item) => {
+            const metadata: Metadata = item[0].ref.constructor[Symbol.metadata];
+            console.log("->", { ref: item[0].ref.constructor.name, refId: item[0].refId, changes: item[1].size });
+            item[1].forEach((op, index) => {
+                console.log("  ->", {
+                    index,
+                    field: metadata?.[index],
+                    op: OPERATION[op],
+                });
+            });
         });
     }
 
     encodeView(view: StateView, sharedOffset: number, it: Iterator, bytes = this.sharedBuffer) {
         const viewOffset = it.offset;
 
+        // console.log(`\nencodeView(), view.changes (${view.changes.size})`);
+        // this.debugChanges(view.changes);
+
+        // console.log(`\nencodeView(), this.root.filteredChanges (${this.root.filteredChanges.size})`);
+        // this.debugChanges("filteredChanges");
+
         // encode visibility changes (add/remove for this view)
         const viewChangesIterator = view.changes.entries();
         for (const [changeTree, changes] of viewChangesIterator) {

package/src/encoder/Root.ts

@@ -0,0 +1,51 @@
+import { OPERATION } from "../encoding/spec";
+import { TypeContext } from "../types/TypeContext";
+import { ChangeTree } from "./ChangeTree";
+
+export class Root {
+    protected nextUniqueId: number = 0;
+    refCount = new WeakMap<ChangeTree, number>();
+
+    // all changes
+    allChanges = new Map<ChangeTree, Map<number, OPERATION>>();
+    allFilteredChanges = new Map<ChangeTree, Map<number, OPERATION>>();
+
+    // pending changes to be encoded
+    changes = new Map<ChangeTree, Map<number, OPERATION>>();
+    filteredChanges = new Map<ChangeTree, Map<number, OPERATION>>();
+
+    constructor(public types: TypeContext) { }
+
+    getNextUniqueId() {
+        return this.nextUniqueId++;
+    }
+
+    add(changeTree: ChangeTree) {
+        const refCount = this.refCount.get(changeTree) || 0;
+        this.refCount.set(changeTree, refCount + 1);
+    }
+
+    remove(changeTree: ChangeTree) {
+        const refCount = this.refCount.get(changeTree);
+        if (refCount <= 1) {
+            this.allChanges.delete(changeTree);
+            this.changes.delete(changeTree);
+
+            if (changeTree.isFiltered || changeTree.isPartiallyFiltered) {
+                this.allFilteredChanges.delete(changeTree);
+                this.filteredChanges.delete(changeTree);
+            }
+
+            this.refCount.delete(changeTree);
+
+        } else {
+            this.refCount.set(changeTree, refCount - 1);
+        }
+
+        changeTree.forEachChild((child, _) => this.remove(child));
+    }
+
+    clear() {
+        this.changes.clear();
+    }
+}

package/src/index.ts

@@ -42,18 +42,10 @@ export {
     ReflectionField,
 } from "./Reflection";
 
+// Annotations, Metadata and TypeContext
 export { Metadata } from "./Metadata";
-
-export {
-    // Annotations
-    type,
-    deprecated,
-    defineTypes,
-    view,
-
-    // Internals
-    TypeContext,
-} from "./annotations";
+export { type, deprecated, defineTypes, view, } from "./annotations";
+export { TypeContext } from "./types/TypeContext";
 
 // Annotation types
 export type { DefinitionType, PrimitiveType, Definition, } from "./annotations";