@colyseus/schema 3.0.0-alpha.3 → 3.0.0-alpha.30

package/src/encoder/Encoder.ts

@@ -1,14 +1,17 @@
 import type { Schema } from "../Schema";
-import { TypeContext } from "../annotations";
+import { TypeContext } from "../types/TypeContext";
 import { $changes, $encoder, $filter } from "../types/symbols";
 
 import * as encode from "../encoding/encode";
 import type { Iterator } from "../encoding/decode";
 
-import { SWITCH_TO_STRUCTURE, TYPE_ID } from '../encoding/spec';
-import { Root } from "./ChangeTree";
+import { OPERATION, SWITCH_TO_STRUCTURE, TYPE_ID } from '../encoding/spec';
+import { Root } from "./Root";
 import { getNextPowerOf2 } from "../utils";
+
 import type { StateView } from "./StateView";
+import type { Metadata } from "../Metadata";
+import type { ChangeTree } from "./ChangeTree";
 
 export class Encoder<T extends Schema = any> {
     static BUFFER_SIZE = 8 * 1024;// 8KB
@@ -19,14 +22,16 @@ export class Encoder<T extends Schema = any> {
 
     root: Root;
 
-    constructor(root: T) {
-        this.setRoot(root);
+    constructor(state: T) {
 
         //
         // TODO: cache and restore "Context" based on root schema
         // (to avoid creating a new context for every new room)
         //
-        this.context = new TypeContext(root.constructor as typeof Schema);
+        this.context = new TypeContext(state.constructor as typeof Schema);
+        this.root = new Root(this.context);
+
+        this.setState(state);
 
         // console.log(">>>>>>>>>>>>>>>> Encoder types");
         // this.context.schemas.forEach((id, schema) => {
@@ -34,21 +39,19 @@ export class Encoder<T extends Schema = any> {
         // });
     }
 
-    protected setRoot(state: T) {
-        this.root = new Root();
+    protected setState(state: T) {
         this.state = state;
-        state[$changes].setRoot(this.root);
+        this.state[$changes].setRoot(this.root);
     }
 
     encode(
         it: Iterator = { offset: 0 },
         view?: StateView,
-        bytes = this.sharedBuffer,
-        changeTrees = this.root.changes
+        buffer = this.sharedBuffer,
+        changeTrees = this.root.changes,
+        isEncodeAll = this.root.allChanges === changeTrees,
+        initialOffset = it.offset // cache current offset in case we need to resize the buffer
     ): Buffer {
-        const initialOffset = it.offset; // cache current offset in case we need to resize the buffer
-
-        const isEncodeAll = this.root.allChanges === changeTrees;
         const hasView = (view !== undefined);
         const rootChangeTree = this.state[$changes];
 
@@ -61,6 +64,13 @@ export class Encoder<T extends Schema = any> {
             const encoder = ctor[$encoder];
             const filter = ctor[$filter];
 
+            // try { throw new Error(); } catch (e) {
+            //     // only print if not coming from Reflection.ts
+            //     if (!e.stack.includes("src/Reflection.ts")) {
+            //         console.log("ChangeTree:", { ref: ref.constructor.name, });
+            //     }
+            // }
+
             if (hasView) {
                 if (!view.items.has(changeTree)) {
                     view.invisible.add(changeTree);
@@ -72,9 +82,10 @@ export class Encoder<T extends Schema = any> {
             }
 
             // skip root `refId` if it's the first change tree
-            if (it.offset !== initialOffset || changeTree !== rootChangeTree) {
-                bytes[it.offset++] = SWITCH_TO_STRUCTURE & 255;
-                encode.number(bytes, changeTree.refId, it);
+            // (unless it "hasView", which will need to revisit the root)
+            if (hasView || it.offset > initialOffset || changeTree !== rootChangeTree) {
+                buffer[it.offset++] = SWITCH_TO_STRUCTURE & 255;
+                encode.number(buffer, changeTree.refId, it);
             }
 
             const changesIterator = changes.entries();
@@ -88,32 +99,45 @@ export class Encoder<T extends Schema = any> {
                 // TODO: avoid checking if no view tags were defined
                 //
                 if (filter && !filter(ref, fieldIndex, view)) {
-                    // console.log("SKIP FIELD:", { ref: changeTree.ref.constructor.name, fieldIndex, })
-
                     // console.log("ADD AS INVISIBLE:", fieldIndex, changeTree.ref.constructor.name)
                     // view?.invisible.add(changeTree);
                     continue;
                 }
 
-                // console.log("WILL ENCODE", {
-                //     ref: changeTree.ref.constructor.name,
-                //     fieldIndex,
-                //     operation: OPERATION[operation],
-                // });
-
-                encoder(this, bytes, changeTree, fieldIndex, operation, it, isEncodeAll, hasView);
+                // try { throw new Error(); } catch (e) {
+                //     // only print if not coming from Reflection.ts
+                //     if (!e.stack.includes("src/Reflection.ts")) {
+                //         console.log("WILL ENCODE", {
+                //             ref: changeTree.ref.constructor.name,
+                //             fieldIndex,
+                //             operation: OPERATION[operation],
+                //         });
+                //     }
+                // }
+
+                encoder(this, buffer, changeTree, fieldIndex, operation, it, isEncodeAll, hasView);
             }
         }
 
-        if (it.offset > bytes.byteLength) {
-            const newSize = getNextPowerOf2(this.sharedBuffer.byteLength * 2);
-            console.warn("@colyseus/schema encode buffer overflow. Current buffer size: " + bytes.byteLength + ", encoding offset: " + it.offset + ", new size: " + newSize);
+        if (it.offset > buffer.byteLength) {
+            const newSize = getNextPowerOf2(buffer.byteLength * 2);
+            console.warn(`@colyseus/schema buffer overflow. Encoded state is higher than default BUFFER_SIZE. Use the following to increase default BUFFER_SIZE:
+
+    import { Encoder } from "@colyseus/schema";
+    Encoder.BUFFER_SIZE = ${Math.round(newSize / 1024)} * 1024; // ${Math.round(newSize / 1024)} KB
+`);
 
             //
             // resize buffer and re-encode (TODO: can we avoid re-encoding here?)
             //
-            this.sharedBuffer = Buffer.allocUnsafeSlow(newSize);
-            return this.encode({ offset: initialOffset }, view);
+            buffer = Buffer.allocUnsafeSlow(newSize);
+
+            // assign resized buffer to local sharedBuffer
+            if (buffer === this.sharedBuffer) {
+                this.sharedBuffer = buffer;
+            }
+
+            return this.encode({ offset: initialOffset }, view, buffer, changeTrees, isEncodeAll);
 
         } else {
             //
@@ -126,53 +150,60 @@ export class Encoder<T extends Schema = any> {
                 this.onEndEncode(changeTrees);
             }
 
-            // return bytes;
-            return bytes.slice(0, it.offset);
+            return buffer.subarray(0, it.offset);
         }
     }
 
-    encodeAll(it: Iterator = { offset: 0 }) {
-        // console.log(`encodeAll(), this.$root.allChanges (${this.$root.allChanges.size})`);
-
-        // Array.from(this.$root.allChanges.entries()).map((item) => {
-        //     console.log("->", item[0].refId, item[0].ref.toJSON());
-        // });
+    encodeAll(it: Iterator = { offset: 0 }, buffer: Buffer = this.sharedBuffer) {
+        // console.log(`\nencodeAll(), this.root.allChanges (${this.root.allChanges.size})`);
+        // this.debugChanges("allChanges");
 
-        return this.encode(it, undefined, this.sharedBuffer, this.root.allChanges);
+        return this.encode(it, undefined, buffer, this.root.allChanges, true);
     }
 
     encodeAllView(view: StateView, sharedOffset: number, it: Iterator, bytes = this.sharedBuffer) {
         const viewOffset = it.offset;
 
-        // console.log(`encodeAllView(), this.$root.allFilteredChanges (${this.$root.allFilteredChanges.size})`);
-        // this.debugAllFilteredChanges();
+        // console.log(`\nencodeAllView(), this.root.allFilteredChanges (${this.root.allFilteredChanges.size})`);
+        // this.debugChanges("allFilteredChanges");
 
         // try to encode "filtered" changes
-        this.encode(it, view, bytes, this.root.allFilteredChanges);
+        this.encode(it, view, bytes, this.root.allFilteredChanges, true, viewOffset);
 
         return Buffer.concat([
-            bytes.slice(0, sharedOffset),
-            bytes.slice(viewOffset, it.offset)
+            bytes.subarray(0, sharedOffset),
+            bytes.subarray(viewOffset, it.offset)
         ]);
     }
 
-
-    // debugAllFilteredChanges() {
-    //     Array.from(this.$root.allFilteredChanges.entries()).map((item) => {
-    //         console.log("->", { refId: item[0].refId }, item[0].ref.toJSON());
-    //         if (Array.isArray(item[0].ref.toJSON())) {
-    //             item[1].forEach((op, key) => {
-    //                 console.log("  ->", { key, op: OPERATION[op] });
-    //             })
-    //         }
-    //     });
-    // }
+    debugChanges(
+        field:  "changes" | "allFilteredChanges" | "allChanges" | "filteredChanges" | Map<ChangeTree, Map<number, OPERATION>>
+    ) {
+        const changeSet = (typeof (field) === "string")
+            ? this.root[field]
+            : field;
+
+        Array.from(changeSet.entries()).map((item) => {
+            const metadata: Metadata = item[0].ref.constructor[Symbol.metadata];
+            console.log("->", { ref: item[0].ref.constructor.name, refId: item[0].refId, changes: item[1].size });
+            item[1].forEach((op, index) => {
+                console.log("  ->", {
+                    index,
+                    field: metadata?.[index],
+                    op: OPERATION[op],
+                });
+            });
+        });
+    }
 
     encodeView(view: StateView, sharedOffset: number, it: Iterator, bytes = this.sharedBuffer) {
         const viewOffset = it.offset;
 
-        // try to encode "filtered" changes
-        this.encode(it, view, bytes, this.root.filteredChanges);
+        // console.log(`\nencodeView(), view.changes (${view.changes.size})`);
+        // this.debugChanges(view.changes);
+
+        // console.log(`\nencodeView(), this.root.filteredChanges (${this.root.filteredChanges.size})`);
+        // this.debugChanges("filteredChanges");
 
         // encode visibility changes (add/remove for this view)
         const viewChangesIterator = view.changes.entries();
@@ -207,9 +238,12 @@ export class Encoder<T extends Schema = any> {
         // clear "view" changes after encoding
         view.changes.clear();
 
+        // try to encode "filtered" changes
+        this.encode(it, view, bytes, this.root.filteredChanges, false, viewOffset);
+
         return Buffer.concat([
-            bytes.slice(0, sharedOffset),
-            bytes.slice(viewOffset, it.offset)
+            bytes.subarray(0, sharedOffset),
+            bytes.subarray(viewOffset, it.offset)
         ]);
     }
 
@@ -242,4 +276,4 @@ export class Encoder<T extends Schema = any> {
             encode.number(bytes, targetTypeId, it);
         }
     }
-}
+}

package/src/encoder/Root.ts

@@ -0,0 +1,51 @@
+import { OPERATION } from "../encoding/spec";
+import { TypeContext } from "../types/TypeContext";
+import { ChangeTree } from "./ChangeTree";
+
+export class Root {
+    protected nextUniqueId: number = 0;
+    refCount = new WeakMap<ChangeTree, number>();
+
+    // all changes
+    allChanges = new Map<ChangeTree, Map<number, OPERATION>>();
+    allFilteredChanges = new Map<ChangeTree, Map<number, OPERATION>>();
+
+    // pending changes to be encoded
+    changes = new Map<ChangeTree, Map<number, OPERATION>>();
+    filteredChanges = new Map<ChangeTree, Map<number, OPERATION>>();
+
+    constructor(public types: TypeContext) { }
+
+    getNextUniqueId() {
+        return this.nextUniqueId++;
+    }
+
+    add(changeTree: ChangeTree) {
+        const refCount = this.refCount.get(changeTree) || 0;
+        this.refCount.set(changeTree, refCount + 1);
+    }
+
+    remove(changeTree: ChangeTree) {
+        const refCount = this.refCount.get(changeTree);
+        if (refCount <= 1) {
+            this.allChanges.delete(changeTree);
+            this.changes.delete(changeTree);
+
+            if (changeTree.isFiltered || changeTree.isPartiallyFiltered) {
+                this.allFilteredChanges.delete(changeTree);
+                this.filteredChanges.delete(changeTree);
+            }
+
+            this.refCount.delete(changeTree);
+
+        } else {
+            this.refCount.set(changeTree, refCount - 1);
+        }
+
+        changeTree.forEachChild((child, _) => this.remove(child));
+    }
+
+    clear() {
+        this.changes.clear();
+    }
+}

package/src/encoder/StateView.ts

@@ -28,25 +28,23 @@ export class StateView {
     changes = new Map<ChangeTree, Map<number, OPERATION>>();
 
     // TODO: allow to set multiple tags at once
-    add(obj: Ref, tag: number = DEFAULT_VIEW_TAG) {
+    add(obj: Ref, tag: number = DEFAULT_VIEW_TAG, checkIncludeParent: boolean = true) {
         if (!obj[$changes]) {
             console.warn("StateView#add(), invalid object:", obj);
             return this;
         }
 
-        let changeTree: ChangeTree = obj[$changes];
-        this.items.add(changeTree);
-
-        // Add children of this ChangeTree to this view
-        changeTree.forEachChild((change, _) =>
-            this.add(change.ref, tag));
-
         // FIXME: ArraySchema/MapSchema does not have metadata
         const metadata: Metadata = obj.constructor[Symbol.metadata];
+        const changeTree: ChangeTree = obj[$changes];
+        this.items.add(changeTree);
 
-        // add parent ChangeTree's, if they are invisible to this view
-        // TODO: REFACTOR addParent()
-        this.addParent(changeTree, tag);
+        // add parent ChangeTree's
+        // - if it was invisible to this view
+        // - if it were previously filtered out
+        if (checkIncludeParent && changeTree.parent) {
+            this.addParent(changeTree.parent[$changes], changeTree.parentIndex, tag);
+        }
 
         //
         // TODO: when adding an item of a MapSchema, the changes may not
@@ -72,8 +70,6 @@ export class StateView {
             }
             tags.add(tag);
 
-            // console.log("BY TAG:", tag);
-
             // Ref: add tagged properties
             metadata?.[-3]?.[tag]?.forEach((index) => {
                 if (changeTree.getChange(index) !== OPERATION.DELETE) {
@@ -82,89 +78,77 @@ export class StateView {
             });
 
         } else {
-
-            // console.log("DEFAULT TAG", changeTree.allChanges);
-
-            // // add default tag properties
-            // metadata?.[-3]?.[DEFAULT_VIEW_TAG]?.forEach((index) => {
-            //     if (changeTree.getChange(index) !== OPERATION.DELETE) {
-            //         changes.set(index, OPERATION.ADD);
-            //     }
-            // });
-
-            const allChangesSet = (changeTree.isFiltered || changeTree.isPartiallyFiltered)
+            const isInvisible = this.invisible.has(changeTree);
+            const changeSet = (changeTree.isFiltered || changeTree.isPartiallyFiltered)
                 ? changeTree.allFilteredChanges
                 : changeTree.allChanges;
-            const it = allChangesSet.keys();
-            const isInvisible = this.invisible.has(changeTree);
 
-            for (const index of it) {
+            changeSet.forEach((op, index) => {
+                const tagAtIndex = metadata?.[metadata?.[index]].tag;
                 if (
-                    (isInvisible || metadata?.[metadata?.[index]].tag === tag) &&
-                    changeTree.getChange(index) !== OPERATION.DELETE
+                    (
+                        isInvisible || // if "invisible", include all
+                        tagAtIndex === undefined || // "all change" with no tag
+                        tagAtIndex === tag // tagged property
+                    ) &&
+                    op !== OPERATION.DELETE
                 ) {
-                    changes.set(index, OPERATION.ADD);
+                    changes.set(index, op);
                 }
-            }
+            });
         }
 
-        // TODO: avoid unnecessary iteration here
-        while (
-            changeTree.parent &&
-            (changeTree = changeTree.parent[$changes]) &&
-            (changeTree.isFiltered || changeTree.isPartiallyFiltered)
-        ) {
-            this.items.add(changeTree);
-        }
+        // Add children of this ChangeTree to this view
+        changeTree.forEachChild((change, index) => {
+            // Do not ADD children that don't have the same tag
+            if (metadata && metadata[metadata[index]].tag !== tag) {
+                return;
+            }
+            this.add(change.ref, tag, false);
+        });
 
         return this;
     }
 
-    protected addParent(changeTree: ChangeTree, tag: number) {
-        const parentRef = changeTree.parent;
-        if (!parentRef) { return; }
+    protected addParent(changeTree: ChangeTree, parentIndex: number, tag: number) {
+        // view must have all "changeTree" parent tree
+        this.items.add(changeTree);
 
-        const parentChangeTree = parentRef[$changes];
-        const parentIndex = changeTree.parentIndex;
+        // add parent's parent
+        const parentChangeTree = changeTree.parent?.[$changes];
+        if (parentChangeTree && (parentChangeTree.isFiltered || parentChangeTree.isPartiallyFiltered)) {
+            this.addParent(parentChangeTree, changeTree.parentIndex, tag);
+        }
 
-        if (!this.invisible.has(parentChangeTree)) {
-            // parent is already available, no need to add it!
+        // parent is already available, no need to add it!
+        if (!this.invisible.has(changeTree)) {
             return;
         }
 
-        this.addParent(parentChangeTree, tag);
-
         // add parent's tag properties
-        if (parentChangeTree.getChange(parentIndex) !== OPERATION.DELETE) {
+        if (changeTree.getChange(parentIndex) !== OPERATION.DELETE) {
 
-            let parentChanges = this.changes.get(parentChangeTree);
-            if (parentChanges === undefined) {
-                parentChanges = new Map<number, OPERATION>();
-                this.changes.set(parentChangeTree, parentChanges);
+            let changes = this.changes.get(changeTree);
+            if (changes === undefined) {
+                changes = new Map<number, OPERATION>();
+                this.changes.set(changeTree, changes);
             }
 
-            // console.log("add parent change", {
-            //     parentIndex,
-            //     parentChanges,
-            //     parentChange: (
-            //         parentChangeTree.getChange(parentIndex) &&
-            //         OPERATION[parentChangeTree.getChange(parentIndex)]
-            //     ),
-            // })
+            if (!this.tags) {
+                this.tags = new WeakMap<ChangeTree, Set<number>>();
+            }
 
-            if (!this.tags) { this.tags = new WeakMap<ChangeTree, Set<number>>(); }
             let tags: Set<number>;
-            if (!this.tags.has(parentChangeTree)) {
+            if (!this.tags.has(changeTree)) {
                 tags = new Set<number>();
-                this.tags.set(parentChangeTree, tags);
+                this.tags.set(changeTree, tags);
             } else {
-                tags = this.tags.get(parentChangeTree);
+                tags = this.tags.get(changeTree);
             }
             tags.add(tag);
 
-            parentChanges.set(parentIndex, OPERATION.ADD);
+            changes.set(parentIndex, OPERATION.ADD);
         }
-
     }
 
     remove(obj: Ref, tag: number = DEFAULT_VIEW_TAG) {

package/src/encoding/decode.ts

@@ -32,8 +32,6 @@ import type { BufferLike } from "./encode";
 export interface Iterator { offset: number; }
 
 export function utf8Read(bytes: BufferLike, it: Iterator, length: number) {
-  it.offset += length;
-
   var string = '', chr = 0;
   for (var i = it.offset, end = it.offset + length; i < end; i++) {
     var byte = bytes[i];
@@ -74,6 +72,7 @@ export function utf8Read(bytes: BufferLike, it: Iterator, length: number) {
     // (do not throw error to avoid server/client from crashing due to hack attemps)
     // throw new Error('Invalid byte ' + byte.toString(16));
   }
+  it.offset += length;
   return string;
 }
 

package/src/encoding/encode.ts

@@ -35,28 +35,32 @@ let textEncoder: TextEncoder;
 // @ts-ignore
 try { textEncoder = new TextEncoder(); } catch (e) { }
 
-export function utf8Length(str) {
-  var c = 0, length = 0;
-  for (var i = 0, l = str.length; i < l; i++) {
-    c = str.charCodeAt(i);
-    if (c < 0x80) {
-      length += 1;
-    }
-    else if (c < 0x800) {
-      length += 2;
-    }
-    else if (c < 0xd800 || c >= 0xe000) {
-      length += 3;
-    }
-    else {
-      i++;
-      length += 4;
+const hasBufferByteLength = (typeof Buffer !== 'undefined' && Buffer.byteLength);
+
+export const utf8Length = (hasBufferByteLength)
+    ? Buffer.byteLength // node
+    : function (str: string, _?: any) {
+        var c = 0, length = 0;
+        for (var i = 0, l = str.length; i < l; i++) {
+            c = str.charCodeAt(i);
+            if (c < 0x80) {
+                length += 1;
+            }
+            else if (c < 0x800) {
+                length += 2;
+            }
+            else if (c < 0xd800 || c >= 0xe000) {
+                length += 3;
+            }
+            else {
+                i++;
+                length += 4;
+            }
+        }
+        return length;
     }
-  }
-  return length;
-}
 
-export function utf8Write(view, str, it) {
+export function utf8Write(view: BufferLike, str: string, it: Iterator) {
   var c = 0;
   for (var i = 0, l = str.length; i < l; i++) {
     c = str.charCodeAt(i);
@@ -166,8 +170,7 @@ export function string(bytes: BufferLike, value: string, it: Iterator) {
   // encode `null` strings as empty.
   if (!value) { value = ""; }
 
-  // let length = utf8Length(value);
-  let length = Buffer.byteLength(value, "utf8");
+  let length = utf8Length(value, "utf8");
   let size = 0;
 
   // fixstr

package/src/encoding/spec.ts

@@ -25,5 +25,6 @@ export enum OPERATION {
     REVERSE = 15,
     MOVE = 32,
     DELETE_BY_REFID = 33, // This operation is only used at ENCODING time. During DECODING, DELETE_BY_REFID is converted to DELETE
+    ADD_BY_REFID = 129,
 
 }

package/src/index.ts

@@ -4,6 +4,8 @@ export type { DataChange } from "./decoder/DecodeOperation";
 import { $track, $encoder, $decoder, $filter, $getByIndex, $deleteByIndex, $changes, $childType } from "./types/symbols";
 export { $track, $encoder, $decoder, $filter, $getByIndex, $deleteByIndex, $changes, $childType };
 
+export type { ToJSON } from "./types/HelperTypes";
+
 import { MapSchema } from "./types/custom/MapSchema"
 export { MapSchema };
 
@@ -40,22 +42,17 @@ export {
     ReflectionField,
 } from "./Reflection";
 
+// Annotations, Metadata and TypeContext
 export { Metadata } from "./Metadata";
-
-export {
-    // Annotations
-    type,
-    deprecated,
-    defineTypes,
-    view,
-
-    // Internals
-    TypeContext,
-} from "./annotations";
+export { type, deprecated, defineTypes, view, } from "./annotations";
+export { TypeContext } from "./types/TypeContext";
 
 // Annotation types
 export type { DefinitionType, PrimitiveType, Definition, } from "./annotations";
 
+export { getDecoderStateCallbacks, CallbackProxy, GetCallbackProxy } from "./decoder/strategy/StateCallbacks";
+export { getRawChangesCallback } from "./decoder/strategy/RawChanges";
+
 export { Encoder } from "./encoder/Encoder";
 export { encodeSchemaOperation, encodeArray as encodeKeyValueOperation } from "./encoder/EncodeOperation";
 export { ChangeTree, Ref } from "./encoder/ChangeTree";