@colyseus/schema 3.0.0-alpha.9 → 3.0.0

package/src/codegen/languages/csharp.ts

@@ -59,16 +59,16 @@ function generateClass(klass: Class, namespace: string) {
     return `${getCommentHeader()}
 
 using Colyseus.Schema;
-using Action = System.Action;
+#if UNITY_5_3_OR_NEWER
+using UnityEngine.Scripting;
+#endif
 ${namespace ? `\nnamespace ${namespace} {` : ""}
 ${indent}public partial class ${klass.name} : ${klass.extends} {
+#if UNITY_5_3_OR_NEWER
+[Preserve]
+#endif
+public ${klass.name}() { }
 ${klass.properties.map((prop) => generateProperty(prop, indent)).join("\n\n")}
-
-${indent}\t/*
-${indent}\t * Support for individual property change callbacks below...
-${indent}\t */
-
-${generateAllFieldCallbacks(klass, indent)}
 ${indent}}
 ${namespace ? "}" : ""}
 `;
@@ -119,7 +119,7 @@ function generateProperty(prop: Property, indent: string = "") {
             typeArgs += `, "${prop.childType}"`;
         }
 
-        initializer = `new ${langType}()`;
+        initializer = `null`;
 
     } else {
         langType = getType(prop);
@@ -147,45 +147,6 @@ ${namespace ? "}" : ""}
 `;
 }
 
-function generateAllFieldCallbacks(klass: Class, indent: string) {
-    //
-    // TODO: improve me. It would be great to generate less boilerplate in favor
-    // of a single implementation on C# Schema class itself.
-    //
-    const eventNames: string[] = [];
-    return `${klass.properties
-        .filter(prop => !prop.deprecated) // generate only for properties that haven't been deprecated.
-        .map(prop => {
-        const eventName = `__${prop.name}Change`;
-        eventNames.push(eventName);
-
-        const defaultNull = (prop.childType)
-            ? "null"
-            : `default(${getType(prop)})`;
-
-        return `\t${indent}protected event PropertyChangeHandler<${getType(prop)}> ${eventName};
-\t${indent}public Action On${capitalize(prop.name)}Change(PropertyChangeHandler<${getType(prop)}> __handler, bool __immediate = true) {
-\t${indent}\tif (__callbacks == null) { __callbacks = new SchemaCallbacks(); }
-\t${indent}\t__callbacks.AddPropertyCallback(nameof(this.${prop.name}));
-\t${indent}\t${eventName} += __handler;
-\t${indent}\tif (__immediate && this.${prop.name} != ${defaultNull}) { __handler(this.${prop.name}, ${defaultNull}); }
-\t${indent}\treturn () => {
-\t${indent}\t\t__callbacks.RemovePropertyCallback(nameof(${prop.name}));
-\t${indent}\t\t${eventName} -= __handler;
-\t${indent}\t};
-\t${indent}}`;
-    }).join("\n\n")}
-
-\t${indent}protected override void TriggerFieldChange(DataChange change) {
-\t${indent}\tswitch (change.Field) {
-${klass.properties.filter(prop => !prop.deprecated).map((prop, i) => {
-    return `\t${indent}\t\tcase nameof(${prop.name}): ${eventNames[i]}?.Invoke((${getType(prop)}) change.Value, (${getType(prop)}) change.PreviousValue); break;`;
-}).join("\n")}
-\t${indent}\t\tdefault: break;
-\t\t${indent}}
-\t${indent}}`;
-}
-
 function getChildType(prop: Property) {
     return typeMaps[prop.childType];
 }

package/src/codegen/languages/haxe.ts

@@ -106,5 +106,9 @@ function generateProperty(prop: Property) {
         initializer = typeInitializer[prop.type];
     }
 
+    // TODO: remove initializer. The callbacks at the Haxe decoder side have a
+    // "FIXME" comment about this on Decoder.hx
+
     return `\t@:type(${typeArgs})\n\tpublic var ${prop.name}: ${langType} = ${initializer};\n`
+    // return `\t@:type(${typeArgs})\n\tpublic var ${prop.name}: ${langType};\n`
 }

package/src/codegen/languages/lua.ts

@@ -43,11 +43,14 @@ function generateClass(klass: Class, namespace: string, allClasses: Class[]) {
         }
     });
 
-// TOOD: inheritance
+    // Inheritance support
+    const inherits = (klass.extends !== "Schema")
+        ? `, ${klass.extends}`
+        : "";
 
     return `${getCommentHeader().replace(/\/\//mg, "--")}
 
-local schema = require 'colyseus.serialization.schema.schema'
+local schema = require 'colyseus.serializer.schema.schema'
 ${allRefs.
     filter(ref => ref.childType && typeMaps[ref.childType] === undefined).
     map(ref => ref.childType).
@@ -56,25 +59,15 @@ ${allRefs.
     map(childType => `local ${childType} = require '${(namespace ? `${namespace}.` : '')}${childType}'`).
     join("\n")}
 
+---@class ${klass.name}: ${klass.extends}
+${klass.properties.map(prop => `---@field ${prop.name} ${getLUATypeAnnotation(prop)}`).join("\n")}
 local ${klass.name} = schema.define({
 ${klass.properties.map(prop => generatePropertyDeclaration(prop)).join(",\n")},
     ["_fields_by_index"] = { ${klass.properties.map(prop => `"${prop.name}"`).join(", ")} },
-})
+}${inherits})
 
 return ${klass.name}
 `;
-
-    // ["on_change"] = function(changes)
-    //     -- on change logic here
-    // end,
-
-    // ["on_add"] = function()
-    //     -- on add logic here
-    //  end,
-
-    // ["on_remove"] = function()
-    //     -- on remove logic here
-    // end,
 }
 
 function generatePropertyDeclaration(prop: Property) {
@@ -108,18 +101,17 @@ function generatePropertyDeclaration(prop: Property) {
     return `    ["${prop.name}"] = ${typeArgs}`;
 }
 
-// function generatePropertyInitializer(prop: Property) {
-//     let initializer = "";
+function getLUATypeAnnotation(prop: Property) {
+    if (prop.type === "ref") {
+        return prop.childType;
 
-//     if(prop.type === "ref") {
-//         initializer = `new ${prop.childType}()`;
+    } else if (prop.type === "array") {
+        return "ArraySchema";
 
-//     } else if(prop.type === "array") {
-//         initializer = `new schema.ArraySchema()`;
+    } else if (prop.type === "map") {
+        return "MapSchema";
 
-//     } else if(prop.type === "map") {
-//         initializer = `new schema.MapSchema()`;
-//     }
-
-//     return `this.${prop.name} = ${initializer}`;
-// }
+    } else {
+        return typeMaps[prop.type];
+    }
+}

package/src/codegen/parser.ts

@@ -148,6 +148,49 @@ function inspectNode(node: ts.Node, context: Context, decoratorName: string) {
                     defineProperty(property, typeArgument);
                 }
 
+            } else if (
+                node.getText() === "setFields" &&
+                (
+                    node.parent.kind === ts.SyntaxKind.CallExpression ||
+                    node.parent.kind === ts.SyntaxKind.PropertyAccessExpression
+                )
+            ) {
+                /**
+                 * Metadata.setFields(klassName, { ... })
+                 */
+                const callExpression = (node.parent.kind === ts.SyntaxKind.PropertyAccessExpression)
+                    ? node.parent.parent as ts.CallExpression
+                    : node.parent as ts.CallExpression;
+
+                if (callExpression.kind !== ts.SyntaxKind.CallExpression) {
+                    break;
+                }
+
+                const classNameNode = callExpression.arguments[0];
+                const className = ts.isClassExpression(classNameNode)
+                    ? classNameNode.name?.escapedText.toString()
+                    : classNameNode.getText();
+
+                // skip if no className is provided
+                if (!className) { break; }
+
+                if (currentStructure.name !== className) {
+                    currentStructure = new Class();
+                }
+                context.addStructure(currentStructure);
+                (currentStructure as Class).extends = "Schema"; // force extends to Schema
+                currentStructure.name = className;
+
+                const types = callExpression.arguments[1] as any;
+                for (let i = 0; i < types.properties.length; i++) {
+                    const prop = types.properties[i];
+
+                    const property = currentProperty || new Property();
+                    property.name = prop.name.escapedText;
+
+                    currentStructure.addProperty(property);
+                    defineProperty(property, prop.initializer);
+                }
 
             } else if (
                 node.getText() === "defineTypes" &&
@@ -192,6 +235,70 @@ function inspectNode(node: ts.Node, context: Context, decoratorName: string) {
 
             break;
 
+        case ts.SyntaxKind.CallExpression:
+            /**
+             * Defining schema via `schema.schema({ ... })`
+             * - schema.schema({})
+             * - schema({})
+             * - ClassName.extends({})
+             */
+            if (
+                (
+                    (
+                        (node as ts.CallExpression).expression?.getText() === "schema.schema" ||
+                        (node as ts.CallExpression).expression?.getText() === "schema"
+                    ) ||
+                    (
+                        (node as ts.CallExpression).expression?.getText().indexOf(".extends") !== -1
+                    )
+                ) &&
+                (node as ts.CallExpression).arguments[0].kind === ts.SyntaxKind.ObjectLiteralExpression
+            ) {
+                const callExpression = node as ts.CallExpression;
+
+                let className = callExpression.arguments[1]?.getText();
+
+                if (!className && callExpression.parent.kind === ts.SyntaxKind.VariableDeclaration) {
+                    className = (callExpression.parent as ts.VariableDeclaration).name?.getText();
+                }
+
+                // skip if no className is provided
+                if (!className) { break; }
+
+                if (currentStructure.name !== className) {
+                    currentStructure = new Class();
+                    context.addStructure(currentStructure);
+                }
+
+                if ((node as ts.CallExpression).expression?.getText().indexOf(".extends") !== -1) {
+                    // if it's using `.extends({})`
+                    const extendsClass = (node as any).expression?.expression?.escapedText;
+
+                    // skip if no extendsClass is provided
+                    if (!extendsClass) { break; }
+                    (currentStructure as Class).extends = extendsClass;
+
+                } else {
+                    // if it's using `schema({})`
+                    (currentStructure as Class).extends = "Schema"; // force extends to Schema
+                }
+
+                currentStructure.name = className;
+
+                const types = callExpression.arguments[0] as any;
+                for (let i = 0; i < types.properties.length; i++) {
+                    const prop = types.properties[i];
+
+                    const property = currentProperty || new Property();
+                    property.name = prop.name.escapedText;
+
+                    currentStructure.addProperty(property);
+                    defineProperty(property, prop.initializer);
+                }
+            }
+
+            break;
+
         case ts.SyntaxKind.EnumMember:
             if (currentStructure instanceof Enum) {
                 const initializer = (node as any).initializer?.text;

package/src/codegen/types.ts

@@ -39,6 +39,7 @@ export class Context {
     }
 
     addStructure(structure: IStructure) {
+        if (structure.context === this) { return; } // skip if already added.
         structure.context = this;
 
         if (structure instanceof Class) {

package/src/debug.ts

@@ -0,0 +1,55 @@
+import * as fs from "fs";
+import { Reflection, Decoder } from "./index";
+
+const contents = fs.readFileSync("/Users/endel/Projects/colyseus/clients/bubbits/project/@bubbits/backend/schema-debug.txt", { encoding: "utf8" }).toString();
+
+let isCommentBlock = false;
+let lastComment = "";
+
+let decoder: Decoder;
+
+function getBuffer(line: string) {
+    const start = line.lastIndexOf(":");
+    const buffer = Buffer.from(new Uint8Array(line.substring(start + 1).split(",").map(n => Number(n))));
+    console.log(`(${buffer.byteLength}) ${Array.from(buffer).join(",")}`)
+    // console.log("");
+    // console.log("");
+    // console.log("> ", line);
+    // console.log("> substring:", line.substring(start + 1))
+    return buffer;
+}
+
+function decode(buffer: Buffer) {
+    try {
+        decoder.decode(buffer);
+    } catch (e) {
+        console.error(e);
+        console.log("Last log:\n\n")
+        console.log(lastComment);
+    }
+}
+
+contents.split("\n").forEach((line) => {
+    if (line.startsWith("#")) {
+        // reset last comment.
+        if (isCommentBlock === false) { lastComment = ""; }
+
+        isCommentBlock = true;
+        lastComment += line.substring(line.indexOf(":") + 1) + "\n";
+        return;
+    }
+
+    isCommentBlock = false;
+
+    if (line.startsWith("handshake:") && !decoder) {
+        decoder = Reflection.decode(getBuffer(line));
+
+    } else if (line.startsWith("state:")) {
+        decode(getBuffer(line));
+
+    } else if (line.startsWith("patch:")) {
+        decode(getBuffer(line));
+    }
+});
+
+console.log(decoder.state.toJSON());

package/src/decoder/DecodeOperation.ts

@@ -3,7 +3,7 @@ import { Metadata } from "../Metadata";
 import { Schema } from "../Schema";
 import type { Ref } from "../encoder/ChangeTree";
 import type { Decoder } from "./Decoder";
-import * as decode from "../encoding/decode";
+import { Iterator, decode } from "../encoding/decode";
 import { $childType, $deleteByIndex, $getByIndex } from "../types/symbols";
 
 import type { MapSchema } from "../types/custom/MapSchema";
@@ -28,7 +28,7 @@ export const DEFINITION_MISMATCH = -1;
 export type DecodeOperation<T extends Schema = any> = (
     decoder: Decoder<T>,
     bytes: Buffer,
-    it: decode.Iterator,
+    it: Iterator,
     ref: Ref,
     allChanges: DataChange[],
 ) => number | void;
@@ -40,7 +40,7 @@ export function decodeValue(
     index: number,
     type: any,
     bytes: Buffer,
-    it: decode.Iterator,
+    it: Iterator,
     allChanges: DataChange[],
 ) {
     const $root = decoder.root;
@@ -105,7 +105,14 @@ export function decodeValue(
                 value = decoder.createInstanceOfType(childType);
             }
 
-            $root.addRef(refId, value, (value !== previousValue));
+            $root.addRef(
+                refId,
+                value,
+                (
+                    value !== previousValue || // increment ref count if value has changed
+                    (operation === OPERATION.DELETE_AND_ADD && value === previousValue) // increment ref count if the same instance is being added again
+                )
+            );
         }
 
 
@@ -169,12 +176,12 @@ export function decodeValue(
 export const decodeSchemaOperation: DecodeOperation = function (
     decoder: Decoder<any>,
     bytes: Buffer,
-    it: decode.Iterator,
+    it: Iterator,
     ref: Ref,
     allChanges: DataChange[],
 ) {
     const first_byte = bytes[it.offset++];
-    const metadata: Metadata = ref['constructor'][Symbol.metadata];
+    const metadata: Metadata = ref.constructor[Symbol.metadata];
 
     // "compressed" index + operation
     const operation = (first_byte >> 6) << 6
@@ -182,21 +189,24 @@ export const decodeSchemaOperation: DecodeOperation = function (
 
     // skip early if field is not defined
     const field = metadata[index];
-    if (field === undefined) { return DEFINITION_MISMATCH; }
+    if (field === undefined) {
+        console.warn("@colyseus/schema: field not defined at", { index, ref: ref.constructor.name, metadata });
+        return DEFINITION_MISMATCH;
+    }
 
     const { value, previousValue } = decodeValue(
         decoder,
         operation,
         ref,
         index,
-        metadata[field].type,
+        field.type,
         bytes,
         it,
         allChanges,
     );
 
     if (value !== null && value !== undefined) {
-        ref[field] = value;
+        ref[field.name] = value;
     }
 
     // add change
@@ -205,7 +215,7 @@ export const decodeSchemaOperation: DecodeOperation = function (
             ref,
             refId: decoder.currentRefId,
             op: operation,
-            field: field,
+            field: field.name,
             value,
             previousValue,
         });
@@ -215,7 +225,7 @@ export const decodeSchemaOperation: DecodeOperation = function (
 export const decodeKeyValueOperation: DecodeOperation = function (
     decoder: Decoder<any>,
     bytes: Buffer,
-    it: decode.Iterator,
+    it: Iterator,
     ref: Ref,
     allChanges: DataChange[]
 ) {
@@ -251,6 +261,7 @@ export const decodeKeyValueOperation: DecodeOperation = function (
         dynamicIndex = ref['getIndex'](index);
     }
 
+
     const { value, previousValue } = decodeValue(
         decoder,
         operation,
@@ -298,12 +309,13 @@ export const decodeKeyValueOperation: DecodeOperation = function (
 export const decodeArray: DecodeOperation = function (
     decoder: Decoder<any>,
     bytes: Buffer,
-    it: decode.Iterator,
+    it: Iterator,
     ref: ArraySchema,
     allChanges: DataChange[]
 ) {
     // "uncompressed" index + operation (array/map items)
-    const operation = bytes[it.offset++];
+    let operation = bytes[it.offset++];
+    let index: number;
 
     if (operation === OPERATION.CLEAR) {
         //
@@ -315,11 +327,15 @@ export const decodeArray: DecodeOperation = function (
         (ref as ArraySchema).clear();
         return;
 
+    } else if (operation === OPERATION.REVERSE) {
+        (ref as ArraySchema).reverse();
+        return;
+
     } else if (operation === OPERATION.DELETE_BY_REFID) {
         // TODO: refactor here, try to follow same flow as below
         const refId = decode.number(bytes, it);
         const previousValue = decoder.root.refs.get(refId);
-        const index = ref.findIndex((value) => value === previousValue);
+        index = ref.findIndex((value) => value === previousValue);
         ref[$deleteByIndex](index);
         allChanges.push({
             ref,
@@ -330,10 +346,22 @@ export const decodeArray: DecodeOperation = function (
             value: undefined,
             previousValue,
         });
+
         return;
+
+    } else if (operation === OPERATION.ADD_BY_REFID) {
+        const refId = decode.number(bytes, it);
+        const itemByRefId = decoder.root.refs.get(refId);
+
+        // use existing index, or push new value
+        index = (itemByRefId)
+            ? ref.findIndex((value) => value === itemByRefId)
+            : ref.length;
+
+    } else {
+        index = decode.number(bytes, it);
     }
 
-    const index = decode.number(bytes, it);
     const type = ref[$childType];
 
     let dynamicIndex: number | string = index;

package/src/decoder/Decoder.ts

@@ -1,13 +1,13 @@
-import { TypeContext } from "../annotations";
+import { TypeContext } from "../types/TypeContext";
 import { $changes, $childType, $decoder, $onDecodeEnd } from "../types/symbols";
 import { Schema } from "../Schema";
 
-import * as decode from "../encoding/decode";
+import { decode } from "../encoding/decode";
 import { OPERATION, SWITCH_TO_STRUCTURE, TYPE_ID } from '../encoding/spec';
-import { Ref } from "../encoder/ChangeTree";
-import { Iterator } from "../encoding/decode";
+import type { Ref } from "../encoder/ChangeTree";
+import type { Iterator } from "../encoding/decode";
 import { ReferenceTracker } from "./ReferenceTracker";
-import { DEFINITION_MISMATCH, DataChange, DecodeOperation } from "./DecodeOperation";
+import { DEFINITION_MISMATCH, type DataChange, type DecodeOperation } from "./DecodeOperation";
 import { Collection } from "../types/HelperTypes";
 
 export class Decoder<T extends Schema = any> {
@@ -21,7 +21,8 @@ export class Decoder<T extends Schema = any> {
     triggerChanges?: (allChanges: DataChange[]) => void;
 
     constructor(root: T, context?: TypeContext) {
-        this.setRoot(root);
+        this.setState(root);
+
         this.context = context || new TypeContext(root.constructor as typeof Schema);
 
         // console.log(">>>>>>>>>>>>>>>> Decoder types");
@@ -30,7 +31,7 @@ export class Decoder<T extends Schema = any> {
         // });
     }
 
-    protected setRoot(root: T) {
+    protected setState(root: T) {
         this.state = root;
         this.root = new ReferenceTracker();
         this.root.addRef(0, root);
@@ -66,7 +67,8 @@ export class Decoder<T extends Schema = any> {
                 if (!nextRef) { throw new Error(`"refId" not found: ${this.currentRefId}`); }
                 ref[$onDecodeEnd]?.()
                 ref = nextRef;
-                decoder = ref['constructor'][$decoder];
+
+                decoder = ref.constructor[$decoder];
 
                 continue;
             }
@@ -80,9 +82,9 @@ export class Decoder<T extends Schema = any> {
                 // keep skipping next bytes until reaches a known structure
                 // by local decoder.
                 //
-                const nextIterator: decode.Iterator = { offset: it.offset };
+                const nextIterator: Iterator = { offset: it.offset };
                 while (it.offset < totalBytes) {
-                    if (decode.switchStructureCheck(bytes, it)) {
+                    if (bytes[it.offset] === SWITCH_TO_STRUCTURE) {
                         nextIterator.offset = it.offset + 1;
                         if ($root.refs.has(decode.number(bytes, nextIterator))) {
                             break;

package/src/decoder/ReferenceTracker.ts

@@ -81,6 +81,7 @@ export class ReferenceTracker {
     clearRefs() {
         this.refs.clear();
         this.deletedRefs.clear();
+        this.callbacks = {};
         this.refCounts = {};
     }
 
@@ -98,8 +99,9 @@ export class ReferenceTracker {
             // Ensure child schema instances have their references removed as well.
             //
             if (Metadata.isValidInstance(ref)) {
-                const metadata: Metadata = ref['constructor'][Symbol.metadata];
-                for (const field in metadata) {
+                const metadata: Metadata = ref.constructor[Symbol.metadata];
+                for (const index in metadata) {
+                    const field = metadata[index as any as number].name;
                     const childRefId = typeof(ref[field]) === "object" && this.refIds.get(ref[field]);
                     if (childRefId) {
                         this.removeRef(childRefId);
@@ -148,4 +150,4 @@ export class ReferenceTracker {
         }
     }
 
-}
+}