@colyseus/schema 3.0.0-alpha.2 → 3.0.0-alpha.23

package/src/encoder/Encoder.ts

@@ -5,10 +5,11 @@ import { $changes, $encoder, $filter } from "../types/symbols";
 import * as encode from "../encoding/encode";
 import type { Iterator } from "../encoding/decode";
 
-import { SWITCH_TO_STRUCTURE, TYPE_ID } from '../encoding/spec';
+import { OPERATION, SWITCH_TO_STRUCTURE, TYPE_ID } from '../encoding/spec';
 import { Root } from "./ChangeTree";
 import { getNextPowerOf2 } from "../utils";
 import type { StateView } from "./StateView";
+import { Metadata } from "../Metadata";
 
 export class Encoder<T extends Schema = any> {
     static BUFFER_SIZE = 8 * 1024;// 8KB
@@ -37,18 +38,24 @@ export class Encoder<T extends Schema = any> {
     protected setRoot(state: T) {
         this.root = new Root();
         this.state = state;
+
+        // Workaround to allow using an empty Schema.
+        if (state.constructor[Symbol.metadata] === undefined) {
+            Metadata.init(state);
+        }
+
         state[$changes].setRoot(this.root);
     }
 
     encode(
         it: Iterator = { offset: 0 },
         view?: StateView,
-        bytes = this.sharedBuffer,
-        changeTrees = this.root.changes
+        buffer = this.sharedBuffer,
+        changeTrees = this.root.changes,
+        isEncodeAll = this.root.allChanges === changeTrees,
     ): Buffer {
         const initialOffset = it.offset; // cache current offset in case we need to resize the buffer
 
-        const isEncodeAll = this.root.allChanges === changeTrees;
         const hasView = (view !== undefined);
         const rootChangeTree = this.state[$changes];
 
@@ -61,6 +68,13 @@ export class Encoder<T extends Schema = any> {
             const encoder = ctor[$encoder];
             const filter = ctor[$filter];
 
+            // try { throw new Error(); } catch (e) {
+            //     // only print if not coming from Reflection.ts
+            //     if (!e.stack.includes("src/Reflection.ts")) {
+            //         console.log("ChangeTree:", { ref: ref.constructor.name, });
+            //     }
+            // }
+
             if (hasView) {
                 if (!view.items.has(changeTree)) {
                     view.invisible.add(changeTree);
@@ -73,8 +87,8 @@ export class Encoder<T extends Schema = any> {
 
             // skip root `refId` if it's the first change tree
             if (it.offset !== initialOffset || changeTree !== rootChangeTree) {
-                bytes[it.offset++] = SWITCH_TO_STRUCTURE & 255;
-                encode.number(bytes, changeTree.refId, it);
+                buffer[it.offset++] = SWITCH_TO_STRUCTURE & 255;
+                encode.number(buffer, changeTree.refId, it);
             }
 
             const changesIterator = changes.entries();
@@ -95,25 +109,41 @@ export class Encoder<T extends Schema = any> {
                     continue;
                 }
 
-                // console.log("WILL ENCODE", {
-                //     ref: changeTree.ref.constructor.name,
-                //     fieldIndex,
-                //     operation: OPERATION[operation],
-                // });
-
-                encoder(this, bytes, changeTree, fieldIndex, operation, it, isEncodeAll, hasView);
+                // try { throw new Error(); } catch (e) {
+                //     // only print if not coming from Reflection.ts
+                //     if (!e.stack.includes("src/Reflection.ts")) {
+                //         console.log("WILL ENCODE", {
+                //             ref: changeTree.ref.constructor.name,
+                //             fieldIndex,
+                //             operation: OPERATION[operation],
+                //         });
+                //     }
+                // }
+
+                encoder(this, buffer, changeTree, fieldIndex, operation, it, isEncodeAll, hasView);
             }
         }
 
-        if (it.offset > bytes.byteLength) {
-            const newSize = getNextPowerOf2(this.sharedBuffer.byteLength * 2);
-            console.warn("@colyseus/schema encode buffer overflow. Current buffer size: " + bytes.byteLength + ", encoding offset: " + it.offset + ", new size: " + newSize);
+        if (it.offset > buffer.byteLength) {
+            const newSize = getNextPowerOf2(buffer.byteLength * 2);
+            console.warn(`@colyseus/schema buffer overflow. Encoded state is higher than default BUFFER_SIZE. Use the following to increase default BUFFER_SIZE:
+
+    import { Encoder } from "@colyseus/schema";
+    Encoder.BUFFER_SIZE = ${Math.round(newSize / 1024)} * 1024; // ${Math.round(newSize / 1024)} KB
+`);
+
 
             //
             // resize buffer and re-encode (TODO: can we avoid re-encoding here?)
             //
-            this.sharedBuffer = Buffer.allocUnsafeSlow(newSize);
-            return this.encode({ offset: initialOffset }, view);
+            buffer = Buffer.allocUnsafeSlow(newSize);
+
+            // assign resized buffer to local sharedBuffer
+            if (buffer === this.sharedBuffer) {
+                this.sharedBuffer = buffer;
+            }
+
+            return this.encode({ offset: initialOffset }, view, buffer, changeTrees, isEncodeAll);
 
         } else {
             //
@@ -126,47 +156,46 @@ export class Encoder<T extends Schema = any> {
                 this.onEndEncode(changeTrees);
             }
 
-            // return bytes;
-            return bytes.slice(0, it.offset);
+            return buffer.subarray(0, it.offset);
         }
     }
 
-    encodeAll(it: Iterator = { offset: 0 }) {
-        // console.log(`encodeAll(), this.$root.allChanges (${this.$root.allChanges.size})`);
+    encodeAll(it: Iterator = { offset: 0 }, buffer: Buffer = this.sharedBuffer) {
+        // console.log(`encodeAll(), this.root.allChanges (${this.root.allChanges.size})`);
 
-        // Array.from(this.$root.allChanges.entries()).map((item) => {
-        //     console.log("->", item[0].refId, item[0].ref.toJSON());
+        // Array.from(this.root.allChanges.entries()).map((item) => {
+        //     console.log("->", { ref: item[0].ref.constructor.name, refId: item[0].refId, changes: item[1].size });
         // });
 
-        return this.encode(it, undefined, this.sharedBuffer, this.root.allChanges);
+        return this.encode(it, undefined, buffer, this.root.allChanges, true);
     }
 
     encodeAllView(view: StateView, sharedOffset: number, it: Iterator, bytes = this.sharedBuffer) {
         const viewOffset = it.offset;
 
-        // console.log(`encodeAllView(), this.$root.allFilteredChanges (${this.$root.allFilteredChanges.size})`);
+        // console.log(`encodeAllView(), this.root.allFilteredChanges (${this.root.allFilteredChanges.size})`);
         // this.debugAllFilteredChanges();
 
         // try to encode "filtered" changes
-        this.encode(it, view, bytes, this.root.allFilteredChanges);
+        this.encode(it, view, bytes, this.root.allFilteredChanges, true);
 
         return Buffer.concat([
-            bytes.slice(0, sharedOffset),
-            bytes.slice(viewOffset, it.offset)
+            bytes.subarray(0, sharedOffset),
+            bytes.subarray(viewOffset, it.offset)
         ]);
     }
 
 
-    // debugAllFilteredChanges() {
-    //     Array.from(this.$root.allFilteredChanges.entries()).map((item) => {
-    //         console.log("->", { refId: item[0].refId }, item[0].ref.toJSON());
-    //         if (Array.isArray(item[0].ref.toJSON())) {
-    //             item[1].forEach((op, key) => {
-    //                 console.log("  ->", { key, op: OPERATION[op] });
-    //             })
-    //         }
-    //     });
-    // }
+    debugAllFilteredChanges() {
+        Array.from(this.root.allFilteredChanges.entries()).map((item) => {
+            console.log("->", { refId: item[0].refId, changes: item[1].size }, item[0].ref.toJSON());
+            if (Array.isArray(item[0].ref.toJSON())) {
+                item[1].forEach((op, key) => {
+                    console.log("  ->", { key, op: OPERATION[op] });
+                })
+            }
+        });
+    }
 
     encodeView(view: StateView, sharedOffset: number, it: Iterator, bytes = this.sharedBuffer) {
         const viewOffset = it.offset;
@@ -208,8 +237,8 @@ export class Encoder<T extends Schema = any> {
         view.changes.clear();
 
         return Buffer.concat([
-            bytes.slice(0, sharedOffset),
-            bytes.slice(viewOffset, it.offset)
+            bytes.subarray(0, sharedOffset),
+            bytes.subarray(viewOffset, it.offset)
         ]);
     }
 
@@ -242,4 +271,4 @@ export class Encoder<T extends Schema = any> {
             encode.number(bytes, targetTypeId, it);
         }
     }
-}
+}

package/src/encoder/StateView.ts

@@ -34,15 +34,20 @@ export class StateView {
             return this;
         }
 
+        // FIXME: ArraySchema/MapSchema does not have metadata
+        const metadata: Metadata = obj.constructor[Symbol.metadata];
+
         let changeTree: ChangeTree = obj[$changes];
         this.items.add(changeTree);
 
         // Add children of this ChangeTree to this view
-        changeTree.forEachChild((change, _) =>
-            this.add(change.ref, tag));
-
-        // FIXME: ArraySchema/MapSchema does not have metadata
-        const metadata: Metadata = obj.constructor[Symbol.metadata];
+        changeTree.forEachChild((change, index) => {
+            // Do not ADD children that don't have the same tag
+            if (metadata && metadata[metadata[index]].tag !== tag) {
+                return;
+            }
+            this.add(change.ref, tag);
+        });
 
         // add parent ChangeTree's, if they are invisible to this view
         // TODO: REFACTOR addParent()
@@ -72,8 +77,6 @@ export class StateView {
             }
             tags.add(tag);
 
-            // console.log("BY TAG:", tag);
-
             // Ref: add tagged properties
             metadata?.[-3]?.[tag]?.forEach((index) => {
                 if (changeTree.getChange(index) !== OPERATION.DELETE) {

package/src/encoding/decode.ts

@@ -22,6 +22,7 @@
  */
 
 import { SWITCH_TO_STRUCTURE } from "./spec";
+import type { BufferLike } from "./encode";
 
 /**
  * msgpack implementation highly based on notepack.io
@@ -30,9 +31,9 @@ import { SWITCH_TO_STRUCTURE } from "./spec";
 
 export interface Iterator { offset: number; }
 
-function utf8Read(bytes, offset, length) {
+export function utf8Read(bytes: BufferLike, it: Iterator, length: number) {
   var string = '', chr = 0;
-  for (var i = offset, end = offset + length; i < end; i++) {
+  for (var i = it.offset, end = it.offset + length; i < end; i++) {
     var byte = bytes[i];
     if ((byte & 0x80) === 0x00) {
       string += String.fromCharCode(byte);
@@ -71,48 +72,49 @@ function utf8Read(bytes, offset, length) {
     // (do not throw error to avoid server/client from crashing due to hack attemps)
     // throw new Error('Invalid byte ' + byte.toString(16));
   }
+  it.offset += length;
   return string;
 }
 
-export function int8 (bytes: number[], it: Iterator) {
+export function int8 (bytes: BufferLike, it: Iterator) {
     return uint8(bytes, it) << 24 >> 24;
 };
 
-export function uint8 (bytes: number[], it: Iterator) {
+export function uint8 (bytes: BufferLike, it: Iterator) {
     return bytes[it.offset++];
 };
 
-export function int16 (bytes: number[], it: Iterator) {
+export function int16 (bytes: BufferLike, it: Iterator) {
     return uint16(bytes, it) << 16 >> 16;
 };
 
-export function uint16 (bytes: number[], it: Iterator) {
+export function uint16 (bytes: BufferLike, it: Iterator) {
     return bytes[it.offset++] | bytes[it.offset++] << 8;
 };
 
-export function int32 (bytes: number[], it: Iterator) {
+export function int32 (bytes: BufferLike, it: Iterator) {
     return bytes[it.offset++] | bytes[it.offset++] << 8 | bytes[it.offset++] << 16 | bytes[it.offset++] << 24;
 };
 
-export function uint32 (bytes: number[], it: Iterator) {
+export function uint32 (bytes: BufferLike, it: Iterator) {
     return int32(bytes, it) >>> 0;
 };
 
-export function float32(bytes: number[], it: Iterator) {
+export function float32(bytes: BufferLike, it: Iterator) {
   return readFloat32(bytes, it);
 }
 
-export function float64(bytes: number[], it: Iterator) {
+export function float64(bytes: BufferLike, it: Iterator) {
   return readFloat64(bytes, it);
 }
 
-export function int64(bytes: number[], it: Iterator) {
+export function int64(bytes: BufferLike, it: Iterator) {
   const low = uint32(bytes, it);
   const high = int32(bytes, it) * Math.pow(2, 32);
   return high + low;
 };
 
-export function uint64(bytes: number[], it: Iterator) {
+export function uint64(bytes: BufferLike, it: Iterator) {
   const low = uint32(bytes, it);
   const high = uint32(bytes, it) * Math.pow(2, 32);
   return high + low;
@@ -124,22 +126,22 @@ const _int32 = new Int32Array(2);
 const _float32 = new Float32Array(_int32.buffer);
 const _float64 = new Float64Array(_int32.buffer);
 
-export function readFloat32 (bytes: number[], it: Iterator) {
+export function readFloat32 (bytes: BufferLike, it: Iterator) {
     _int32[0] = int32(bytes, it);
     return _float32[0];
 };
 
-export function readFloat64 (bytes: number[], it: Iterator) {
+export function readFloat64 (bytes: BufferLike, it: Iterator) {
     _int32[_isLittleEndian ? 0 : 1] = int32(bytes, it);
     _int32[_isLittleEndian ? 1 : 0] = int32(bytes, it);
     return _float64[0];
 };
 
-export function boolean (bytes: number[], it: Iterator) {
+export function boolean (bytes: BufferLike, it: Iterator) {
     return uint8(bytes, it) > 0;
 };
 
-export function string (bytes, it: Iterator) {
+export function string (bytes: BufferLike, it: Iterator) {
   const prefix = bytes[it.offset++];
   let length: number;
 
@@ -157,13 +159,10 @@ export function string (bytes, it: Iterator) {
     length = uint32(bytes, it);
   }
 
-  const value = utf8Read(bytes, it.offset, length);
-  it.offset += length;
-
-  return value;
+  return utf8Read(bytes, it, length);
 }
 
-export function stringCheck(bytes, it: Iterator) {
+export function stringCheck(bytes: BufferLike, it: Iterator) {
   const prefix = bytes[it.offset];
   return (
     // fixstr
@@ -177,7 +176,7 @@ export function stringCheck(bytes, it: Iterator) {
   );
 }
 
-export function number (bytes, it: Iterator) {
+export function number (bytes: BufferLike, it: Iterator) {
   const prefix = bytes[it.offset++];
 
   if (prefix < 0x80) {
@@ -230,7 +229,7 @@ export function number (bytes, it: Iterator) {
   }
 };
 
-export function numberCheck (bytes, it: Iterator) {
+export function numberCheck (bytes: BufferLike, it: Iterator) {
   const prefix = bytes[it.offset];
   // positive fixint - 0x00 - 0x7f
   // float 32        - 0xca
@@ -249,7 +248,7 @@ export function numberCheck (bytes, it: Iterator) {
   );
 }
 
-export function arrayCheck (bytes, it: Iterator) {
+export function arrayCheck (bytes: BufferLike, it: Iterator) {
   return bytes[it.offset] < 0xa0;
 
   // const prefix = bytes[it.offset] ;
@@ -268,7 +267,7 @@ export function arrayCheck (bytes, it: Iterator) {
   // return prefix;
 }
 
-export function switchStructureCheck(bytes, it: Iterator) {
+export function switchStructureCheck(bytes: BufferLike, it: Iterator) {
   return (
       // previous byte should be `SWITCH_TO_STRUCTURE`
       bytes[it.offset - 1] === SWITCH_TO_STRUCTURE &&

package/src/encoding/encode.ts

@@ -35,28 +35,32 @@ let textEncoder: TextEncoder;
 // @ts-ignore
 try { textEncoder = new TextEncoder(); } catch (e) { }
 
-export function utf8Length(str) {
-  var c = 0, length = 0;
-  for (var i = 0, l = str.length; i < l; i++) {
-    c = str.charCodeAt(i);
-    if (c < 0x80) {
-      length += 1;
-    }
-    else if (c < 0x800) {
-      length += 2;
-    }
-    else if (c < 0xd800 || c >= 0xe000) {
-      length += 3;
-    }
-    else {
-      i++;
-      length += 4;
+const hasBufferByteLength = (typeof Buffer !== 'undefined' && Buffer.byteLength);
+
+export const utf8Length = (hasBufferByteLength)
+    ? Buffer.byteLength // node
+    : function (str: string, _?: any) {
+        var c = 0, length = 0;
+        for (var i = 0, l = str.length; i < l; i++) {
+            c = str.charCodeAt(i);
+            if (c < 0x80) {
+                length += 1;
+            }
+            else if (c < 0x800) {
+                length += 2;
+            }
+            else if (c < 0xd800 || c >= 0xe000) {
+                length += 3;
+            }
+            else {
+                i++;
+                length += 4;
+            }
+        }
+        return length;
     }
-  }
-  return length;
-}
 
-export function utf8Write(view, str, it) {
+export function utf8Write(view: BufferLike, str: string, it: Iterator) {
   var c = 0;
   for (var i = 0, l = str.length; i < l; i++) {
     c = str.charCodeAt(i);
@@ -166,8 +170,7 @@ export function string(bytes: BufferLike, value: string, it: Iterator) {
   // encode `null` strings as empty.
   if (!value) { value = ""; }
 
-  // let length = utf8Length(value);
-  let length = Buffer.byteLength(value, "utf8");
+  let length = utf8Length(value, "utf8");
   let size = 0;
 
   // fixstr

package/src/encoding/spec.ts

@@ -25,5 +25,6 @@ export enum OPERATION {
     REVERSE = 15,
     MOVE = 32,
     DELETE_BY_REFID = 33, // This operation is only used at ENCODING time. During DECODING, DELETE_BY_REFID is converted to DELETE
+    ADD_BY_REFID = 129,
 
 }

package/src/index.ts

@@ -4,6 +4,8 @@ export type { DataChange } from "./decoder/DecodeOperation";
 import { $track, $encoder, $decoder, $filter, $getByIndex, $deleteByIndex, $changes, $childType } from "./types/symbols";
 export { $track, $encoder, $decoder, $filter, $getByIndex, $deleteByIndex, $changes, $childType };
 
+export type { ToJSON } from "./types/HelperTypes";
+
 import { MapSchema } from "./types/custom/MapSchema"
 export { MapSchema };
 
@@ -56,6 +58,9 @@ export {
 // Annotation types
 export type { DefinitionType, PrimitiveType, Definition, } from "./annotations";
 
+export { getDecoderStateCallbacks, CallbackProxy, GetCallbackProxy } from "./decoder/strategy/StateCallbacks";
+export { getRawChangesCallback } from "./decoder/strategy/RawChanges";
+
 export { Encoder } from "./encoder/Encoder";
 export { encodeSchemaOperation, encodeArray as encodeKeyValueOperation } from "./encoder/EncodeOperation";
 export { ChangeTree, Ref } from "./encoder/ChangeTree";

package/src/types/custom/ArraySchema.ts

@@ -471,6 +471,7 @@ export class ArraySchema<V = any> implements Array<V>, Collection<number, V> {
      * @param thisArg An object to which the this keyword can refer in the callbackfn function.
      * If thisArg is omitted, undefined is used as the this value.
      */
+    every<S extends V>(predicate: (value: V, index: number, array: V[]) => value is S, thisArg?: any): this is S[];
     every(callbackfn: (value: V, index: number, array: V[]) => unknown, thisArg?: any): boolean {
         return this.items.every(callbackfn, thisArg);
     }
@@ -690,7 +691,7 @@ export class ArraySchema<V = any> implements Array<V>, Collection<number, V> {
     //
     // ES2023
     //
-    with(index: number, value: V): V[] {
+    with(index: number, value: V): ArraySchema<V> {
         const copy = this.items.slice();
         copy[index] = value;
         return new ArraySchema(...copy);