@colyseus/schema 3.0.0-alpha.9 → 3.0.0

package/src/encoding/encode.ts

@@ -35,9 +35,17 @@ let textEncoder: TextEncoder;
 // @ts-ignore
 try { textEncoder = new TextEncoder(); } catch (e) { }
 
+// force little endian to facilitate decoding on multiple implementations
+const _isLittleEndian = true;  // new Uint16Array(new Uint8Array([1, 0]).buffer)[0] === 1;
+const _convoBuffer = new ArrayBuffer(8);
+const _int32 = new Int32Array(_convoBuffer);
+const _float32 = new Float32Array(_convoBuffer);
+const _float64 = new Float64Array(_convoBuffer);
+const _int64 = new BigInt64Array(_convoBuffer);
+
 const hasBufferByteLength = (typeof Buffer !== 'undefined' && Buffer.byteLength);
 
-export const utf8Length = (hasBufferByteLength)
+const utf8Length: (str: string, _?: any) => number = (hasBufferByteLength)
     ? Buffer.byteLength // node
     : function (str: string, _?: any) {
         var c = 0, length = 0;
@@ -60,7 +68,7 @@ export const utf8Length = (hasBufferByteLength)
         return length;
     }
 
-export function utf8Write(view: BufferLike, str: string, it: Iterator) {
+function utf8Write(view: BufferLike, str: string, it: Iterator) {
   var c = 0;
   for (var i = 0, l = str.length; i < l; i++) {
     c = str.charCodeAt(i);
@@ -68,51 +76,54 @@ export function utf8Write(view: BufferLike, str: string, it: Iterator) {
       view[it.offset++] = c;
     }
     else if (c < 0x800) {
-      view[it.offset++] = 0xc0 | (c >> 6);
-      view[it.offset++] = 0x80 | (c & 0x3f);
+      view[it.offset] = 0xc0 | (c >> 6);
+      view[it.offset + 1] = 0x80 | (c & 0x3f);
+      it.offset += 2;
     }
     else if (c < 0xd800 || c >= 0xe000) {
-      view[it.offset++] = 0xe0 | (c >> 12);
-      view[it.offset++] = 0x80 | (c >> 6 & 0x3f);
-      view[it.offset++] = 0x80 | (c & 0x3f);
+      view[it.offset] = 0xe0 | (c >> 12);
+      view[it.offset+1] = 0x80 | (c >> 6 & 0x3f);
+      view[it.offset+2] = 0x80 | (c & 0x3f);
+      it.offset += 3;
     }
     else {
       i++;
       c = 0x10000 + (((c & 0x3ff) << 10) | (str.charCodeAt(i) & 0x3ff));
-      view[it.offset++] = 0xf0 | (c >> 18);
-      view[it.offset++] = 0x80 | (c >> 12 & 0x3f);
-      view[it.offset++] = 0x80 | (c >> 6 & 0x3f);
-      view[it.offset++] = 0x80 | (c & 0x3f);
+      view[it.offset] = 0xf0 | (c >> 18);
+      view[it.offset+1] = 0x80 | (c >> 12 & 0x3f);
+      view[it.offset+2] = 0x80 | (c >> 6 & 0x3f);
+      view[it.offset+3] = 0x80 | (c & 0x3f);
+      it.offset += 4;
     }
   }
 }
 
-export function int8(bytes: BufferLike, value: number, it: Iterator) {
+function int8(bytes: BufferLike, value: number, it: Iterator) {
     bytes[it.offset++] = value & 255;
 };
 
-export function uint8(bytes: BufferLike, value: number, it: Iterator) {
+function uint8(bytes: BufferLike, value: number, it: Iterator) {
     bytes[it.offset++] = value & 255;
 };
 
-export function int16(bytes: BufferLike, value: number, it: Iterator) {
+function int16(bytes: BufferLike, value: number, it: Iterator) {
     bytes[it.offset++] = value & 255;
     bytes[it.offset++] = (value >> 8) & 255;
 };
 
-export function uint16(bytes: BufferLike, value: number, it: Iterator) {
+function uint16(bytes: BufferLike, value: number, it: Iterator) {
     bytes[it.offset++] = value & 255;
     bytes[it.offset++] = (value >> 8) & 255;
 };
 
-export function int32(bytes: BufferLike, value: number, it: Iterator) {
+function int32(bytes: BufferLike, value: number, it: Iterator) {
   bytes[it.offset++] = value & 255;
   bytes[it.offset++] = (value >> 8) & 255;
   bytes[it.offset++] = (value >> 16) & 255;
   bytes[it.offset++] = (value >> 24) & 255;
 };
 
-export function uint32(bytes: BufferLike, value: number, it: Iterator) {
+function uint32(bytes: BufferLike, value: number, it: Iterator) {
   const b4 = value >> 24;
   const b3 = value >> 16;
   const b2 = value >> 8;
@@ -123,50 +134,48 @@ export function uint32(bytes: BufferLike, value: number, it: Iterator) {
   bytes[it.offset++] = b4 & 255;
 };
 
-export function int64(bytes: BufferLike, value: number, it: Iterator) {
+function int64(bytes: BufferLike, value: number, it: Iterator) {
   const high = Math.floor(value / Math.pow(2, 32));
   const low = value >>> 0;
   uint32(bytes, low, it);
   uint32(bytes, high, it);
 };
 
-export function uint64(bytes: BufferLike, value: number, it: Iterator) {
+function uint64(bytes: BufferLike, value: number, it: Iterator) {
   const high = (value / Math.pow(2, 32)) >> 0;
   const low = value >>> 0;
   uint32(bytes, low, it);
   uint32(bytes, high, it);
 };
 
-export function float32(bytes: BufferLike, value: number, it: Iterator) {
-  writeFloat32(bytes, value, it);
+function bigint64(bytes: BufferLike, value: bigint, it: Iterator) {
+    _int64[0] = BigInt.asIntN(64, value);
+    int32(bytes, _int32[0], it);
+    int32(bytes, _int32[1], it);
 }
 
-export function float64(bytes: BufferLike, value: number, it: Iterator) {
-  writeFloat64(bytes, value, it);
+function biguint64(bytes: BufferLike, value: bigint, it: Iterator) {
+    _int64[0] = BigInt.asIntN(64, value);
+    int32(bytes, _int32[0], it);
+    int32(bytes, _int32[1], it);
 }
 
-// force little endian to facilitate decoding on multiple implementations
-const _isLittleEndian = true;  // new Uint16Array(new Uint8Array([1, 0]).buffer)[0] === 1;
-const _int32 = new Int32Array(2);
-const _float32 = new Float32Array(_int32.buffer);
-const _float64 = new Float64Array(_int32.buffer);
-
-export function writeFloat32(bytes: BufferLike, value: number, it: Iterator) {
+function float32(bytes: BufferLike, value: number, it: Iterator) {
   _float32[0] = value;
   int32(bytes, _int32[0], it);
-};
+}
 
-export function writeFloat64(bytes: BufferLike, value: number, it: Iterator) {
+function float64(bytes: BufferLike, value: number, it: Iterator) {
   _float64[0] = value;
   int32(bytes, _int32[_isLittleEndian ? 0 : 1], it);
   int32(bytes, _int32[_isLittleEndian ? 1 : 0], it);
-};
+}
 
-export function boolean(bytes: BufferLike, value: number, it: Iterator) {
+function boolean(bytes: BufferLike, value: number, it: Iterator) {
   bytes[it.offset++] = value ? 1 : 0; // uint8
 };
 
-export function string(bytes: BufferLike, value: string, it: Iterator) {
+function string(bytes: BufferLike, value: string, it: Iterator) {
   // encode `null` strings as empty.
   if (!value) { value = ""; }
 
@@ -204,7 +213,7 @@ export function string(bytes: BufferLike, value: string, it: Iterator) {
   return size + length;
 }
 
-export function number(bytes: BufferLike, value: number, it: Iterator) {
+function number(bytes: BufferLike, value: number, it: Iterator) {
   if (isNaN(value)) {
     return number(bytes, 0, it);
 
@@ -212,17 +221,19 @@ export function number(bytes: BufferLike, value: number, it: Iterator) {
     return number(bytes, (value > 0) ? Number.MAX_SAFE_INTEGER : -Number.MAX_SAFE_INTEGER, it);
 
   } else if (value !== (value|0)) {
+    if (Math.abs(value) <= 3.4028235e+38) { // range check
+        _float32[0] = value;
+        if (Math.abs(Math.abs(_float32[0]) - Math.abs(value)) < 1e-4) { // precision check; adjust 1e-n (n = precision) to in-/decrease acceptable precision loss
+            // now we know value is in range for f32 and has acceptable precision for f32
+            bytes[it.offset++] = 0xca;
+            float32(bytes, value, it);
+            return 5;
+        }
+    }
+
     bytes[it.offset++] = 0xcb;
-    writeFloat64(bytes, value, it);
+    float64(bytes, value, it);
     return 9;
-
-    // TODO: encode float 32?
-    // is it possible to differentiate between float32 / float64 here?
-
-    // // float 32
-    // bytes.push(0xca);
-    // writeFloat32(bytes, value);
-    // return 5;
   }
 
   if (value >= 0) {
@@ -293,3 +304,23 @@ export function number(bytes: BufferLike, value: number, it: Iterator) {
     return 9;
   }
 }
+
+export const encode = {
+    int8,
+    uint8,
+    int16,
+    uint16,
+    int32,
+    uint32,
+    int64,
+    uint64,
+    bigint64,
+    biguint64,
+    float32,
+    float64,
+    boolean,
+    string,
+    number,
+    utf8Write,
+    utf8Length,
+}

package/src/encoding/spec.ts

@@ -8,8 +8,8 @@ export enum OPERATION {
     ADD = 128,            // (10000000) add new structure/primitive
     REPLACE = 0,          // (00000001) replace structure/primitive
     DELETE = 64,          // (01000000) delete field
-    DELETE_AND_MOVE = 96, // () add new structure/primitive
-    MOVE_AND_ADD = 160,   // () add new structure/primitive
+    DELETE_AND_MOVE = 96, // () ArraySchema only
+    MOVE_AND_ADD = 160,   // () ArraySchema only
     DELETE_AND_ADD = 192, // (11000000) DELETE field, followed by an ADD
 
     /**
@@ -20,10 +20,8 @@ export enum OPERATION {
     /**
      * ArraySchema operations
      */
-    PUSH = 11,
-    UNSHIFT = 12,
     REVERSE = 15,
     MOVE = 32,
     DELETE_BY_REFID = 33, // This operation is only used at ENCODING time. During DECODING, DELETE_BY_REFID is converted to DELETE
-
+    ADD_BY_REFID = 129,
 }

package/src/index.ts

@@ -1,8 +1,6 @@
 export { Schema } from "./Schema";
 export type { DataChange } from "./decoder/DecodeOperation";
-
-import { $track, $encoder, $decoder, $filter, $getByIndex, $deleteByIndex, $changes, $childType } from "./types/symbols";
-export { $track, $encoder, $decoder, $filter, $getByIndex, $deleteByIndex, $changes, $childType };
+export type { ToJSON } from "./types/HelperTypes";
 
 import { MapSchema } from "./types/custom/MapSchema"
 export { MapSchema };
@@ -16,8 +14,8 @@ export { CollectionSchema };
 import { SetSchema } from "./types/custom/SetSchema";
 export { SetSchema };
 
-import { registerType } from "./types/registry";
-export { registerType };
+import { registerType, defineCustomTypes } from "./types/registry";
+export { registerType, defineCustomTypes };
 
 registerType("map", { constructor: MapSchema });
 registerType("array", { constructor: ArraySchema });
@@ -28,10 +26,9 @@ registerType("collection", { constructor: CollectionSchema, });
 export { dumpChanges } from "./utils";
 
 // Encoder / Decoder
-export type { Iterator } from "./encoding/decode";
-import * as encode from "./encoding/encode";
-import * as decode from "./encoding/decode";
-export { encode, decode };
+export { $track, $encoder, $decoder, $filter, $getByIndex, $deleteByIndex, $changes, $childType } from "./types/symbols";
+export { encode } from "./encoding/encode";
+export { decode, type Iterator } from "./encoding/decode";
 
 // Reflection
 export {
@@ -40,22 +37,17 @@ export {
     ReflectionField,
 } from "./Reflection";
 
+// Annotations, Metadata and TypeContext
 export { Metadata } from "./Metadata";
-
-export {
-    // Annotations
-    type,
-    deprecated,
-    defineTypes,
-    view,
-
-    // Internals
-    TypeContext,
-} from "./annotations";
+export { type, deprecated, defineTypes, view, schema, type SchemaWithExtends, } from "./annotations";
+export { TypeContext } from "./types/TypeContext";
 
 // Annotation types
 export type { DefinitionType, PrimitiveType, Definition, } from "./annotations";
 
+export { getDecoderStateCallbacks, CallbackProxy, GetCallbackProxy } from "./decoder/strategy/StateCallbacks";
+export { getRawChangesCallback } from "./decoder/strategy/RawChanges";
+
 export { Encoder } from "./encoder/Encoder";
 export { encodeSchemaOperation, encodeArray as encodeKeyValueOperation } from "./encoder/EncodeOperation";
 export { ChangeTree, Ref } from "./encoder/ChangeTree";

package/src/types/HelperTypes.ts

@@ -1,5 +1,11 @@
-import { ArraySchema } from "./custom/ArraySchema";
-import { MapSchema } from "./custom/MapSchema";
+import type { Definition, DefinitionType, PrimitiveType, RawPrimitiveType } from "../annotations";
+import type { Schema } from "../Schema";
+import type { ArraySchema } from "./custom/ArraySchema";
+import type { CollectionSchema } from "./custom/CollectionSchema";
+import type { MapSchema } from "./custom/MapSchema";
+import type { SetSchema } from "./custom/SetSchema";
+
+export type Constructor<T = {}> = new (...args: any[]) => T;
 
 export interface Collection<K = any, V = any, IT = V> {
     [Symbol.iterator](): IterableIterator<IT>;
@@ -7,6 +13,52 @@ export interface Collection<K = any, V = any, IT = V> {
     entries(): IterableIterator<[K, V]>;
 }
 
+export type InferValueType<T extends DefinitionType> =
+    T extends "string" ? string
+    : T extends "number" ? number
+    : T extends "int8" ? number
+    : T extends "uint8" ? number
+    : T extends "int16" ? number
+    : T extends "uint16" ? number
+    : T extends "int32" ? number
+    : T extends "uint32" ? number
+    : T extends "int64" ? number
+    : T extends "uint64" ? number
+    : T extends "float32" ? number
+    : T extends "float64" ? number
+    : T extends "boolean" ? boolean
+
+    : T extends { type: infer ChildType extends Constructor } ? InstanceType<ChildType>
+    : T extends { type: infer ChildType extends PrimitiveType } ? ChildType
+
+    : T extends Array<infer ChildType extends Constructor> ? InstanceType<ChildType>[]
+    : T extends Array<infer ChildType extends RawPrimitiveType> ? ChildType[]
+
+    : T extends { array: infer ChildType extends Constructor } ? InstanceType<ChildType>[]
+    : T extends { array: infer ChildType extends PrimitiveType } ? ChildType[]
+
+    : T extends { map: infer ChildType extends Constructor } ? MapSchema<InstanceType<ChildType>>
+    : T extends { map: infer ChildType extends PrimitiveType } ? MapSchema<ChildType>
+
+    : T extends { set: infer ChildType extends Constructor } ? SetSchema<InstanceType<ChildType>>
+    : T extends { set: infer ChildType extends PrimitiveType } ? SetSchema<ChildType>
+
+    : T extends { collection: infer ChildType extends Constructor } ? CollectionSchema<InstanceType<ChildType>>
+    : T extends { collection: infer ChildType extends PrimitiveType } ? CollectionSchema<ChildType>
+
+    : T extends Constructor ? InstanceType<T>
+    : T extends PrimitiveType ? T
+
+    : never;
+
+export type InferSchemaInstanceType<T extends Definition> = {
+    [K in keyof T]: InferValueType<T[K]>
+} & Schema;
+
+export type DefinedSchemaType<T extends Definition, P extends typeof Schema> = {
+    new (): InferSchemaInstanceType<T> & InstanceType<P>;
+} & typeof Schema;
+
 export type NonFunctionProps<T> = Omit<T, {
     [K in keyof T]: T[K] extends Function ? K : never;
 }[keyof T]>;

package/src/types/TypeContext.ts

@@ -0,0 +1,175 @@
+import { Metadata } from "../Metadata";
+import { Schema } from "../Schema";
+import { $viewFieldIndexes } from "./symbols";
+
+export class TypeContext {
+    types: { [id: number]: typeof Schema; } = {};
+    schemas = new Map<typeof Schema, number>();
+
+    hasFilters: boolean = false;
+    parentFiltered: {[typeIdAndParentIndex: string]: boolean} = {};
+
+    /**
+     * For inheritance support
+     * Keeps track of which classes extends which. (parent -> children)
+     */
+    static inheritedTypes = new Map<typeof Schema, Set<typeof Schema>>();
+
+    static register(target: typeof Schema) {
+        const parent = Object.getPrototypeOf(target);
+        if (parent !== Schema) {
+            let inherits = TypeContext.inheritedTypes.get(parent);
+            if (!inherits) {
+                inherits = new Set<typeof Schema>();
+                TypeContext.inheritedTypes.set(parent, inherits);
+            }
+            inherits.add(target);
+        }
+    }
+
+    constructor(rootClass?: typeof Schema) {
+        if (rootClass) {
+            //
+            // TODO:
+            //      cache "discoverTypes" results for each rootClass
+            //      to avoid re-discovering types for each new context/room
+            //
+            this.discoverTypes(rootClass);
+        }
+    }
+
+    has(schema: typeof Schema) {
+        return this.schemas.has(schema);
+    }
+
+    get(typeid: number) {
+        return this.types[typeid];
+    }
+
+    add(schema: typeof Schema, typeid = this.schemas.size) {
+        // skip if already registered
+        if (this.schemas.has(schema)) {
+            return false;
+        }
+
+        this.types[typeid] = schema;
+
+        //
+        // Workaround to allow using an empty Schema (with no `@type()` fields)
+        //
+        if (schema[Symbol.metadata] === undefined) {
+            Metadata.initialize(schema);
+        }
+
+        this.schemas.set(schema, typeid);
+        return true;
+    }
+
+    getTypeId(klass: typeof Schema) {
+        return this.schemas.get(klass);
+    }
+
+    private discoverTypes(klass: typeof Schema, parentType?: typeof Schema, parentIndex?: number, parentHasViewTag?: boolean) {
+        if (parentHasViewTag) {
+            this.registerFilteredByParent(klass, parentType, parentIndex);
+        }
+
+        // skip if already registered
+        if (!this.add(klass)) { return; }
+
+        // add classes inherited from this base class
+        TypeContext.inheritedTypes.get(klass)?.forEach((child) => {
+            this.discoverTypes(child, parentType, parentIndex, parentHasViewTag);
+        });
+
+        // add parent classes
+        let parent: any = klass;
+        while (
+            (parent = Object.getPrototypeOf(parent)) &&
+            parent !== Schema && // stop at root (Schema)
+            parent !== Function.prototype // stop at root (non-Schema)
+        ) {
+            this.discoverTypes(parent);
+        }
+
+        const metadata: Metadata = (klass[Symbol.metadata] ??= {});
+
+        // if any schema/field has filters, mark "context" as having filters.
+        if (metadata[$viewFieldIndexes]) {
+            this.hasFilters = true;
+        }
+
+        for (const fieldIndex in metadata) {
+            const index = fieldIndex as any as number;
+
+            const fieldType = metadata[index].type;
+            const fieldHasViewTag = (metadata[index].tag !== undefined);
+
+            if (typeof (fieldType) === "string") {
+                continue;
+            }
+
+            if (Array.isArray(fieldType)) {
+                const type = fieldType[0];
+
+                // skip primitive types
+                if (type === "string") {
+                    continue;
+                }
+
+                this.discoverTypes(type as typeof Schema, klass, index, parentHasViewTag || fieldHasViewTag);
+
+            } else if (typeof (fieldType) === "function") {
+                this.discoverTypes(fieldType as typeof Schema, klass, index, parentHasViewTag || fieldHasViewTag);
+
+            } else {
+                const type = Object.values(fieldType)[0];
+
+                // skip primitive types
+                if (typeof (type) === "string") {
+                    continue;
+                }
+
+                this.discoverTypes(type as typeof Schema, klass, index, parentHasViewTag || fieldHasViewTag);
+            }
+        }
+    }
+
+    /**
+     * Keep track of which classes have filters applied.
+     * Format: `${typeid}-${parentTypeid}-${parentIndex}`
+     */
+    private registerFilteredByParent(schema: typeof Schema, parentType?: typeof Schema, parentIndex?: number) {
+        const typeid = this.schemas.get(schema) ?? this.schemas.size;
+
+        let key = `${typeid}`;
+        if (parentType) { key += `-${this.schemas.get(parentType)}`; }
+
+        key += `-${parentIndex}`;
+        this.parentFiltered[key] = true;
+    }
+
+    debug() {
+        let parentFiltered = "";
+
+        for (const key in this.parentFiltered) {
+            const keys: number[] = key.split("-").map(Number);
+            const fieldIndex = keys.pop();
+
+            parentFiltered += `\n\t\t`;
+            parentFiltered += `${key}: ${keys.reverse().map((id, i) => {
+                const klass = this.types[id];
+                const metadata: Metadata = klass[Symbol.metadata];
+                let txt = klass.name;
+                if (i === 0) { txt += `[${metadata[fieldIndex].name}]`; }
+                return `${txt}`;
+            }).join(" -> ")}`;
+        }
+
+        return `TypeContext ->\n` +
+            `\tSchema types: ${this.schemas.size}\n` +
+            `\thasFilters: ${this.hasFilters}\n` +
+            `\tparentFiltered:${parentFiltered}`;
+    }
+
+}