@openhps/protobuf 1.0.0

package/package.json

@@ -0,0 +1,96 @@
+{
+  "name": "@openhps/protobuf",
+  "version": "1.0.0",
+  "description": "OpenHPS Protocol Buffer Generator",
+  "main": "dist/cjs/bin/generator.js",
+  "bin": {
+    "openhps-protobuf": "./dist/cjs/bin/generator.js"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "benchmark:1": "ts-node test/bench/serialization.bench.ts",
+    "benchmark:2": "ts-node test/bench/socket.bench.ts",
+    "benchmark:3": "ts-node test/bench/mqtt.bench.ts",
+    "start": "node dist/cjs/bin/generator.js",
+    "start:debug": "ts-node src/bin/generator.ts",
+    "test": "mocha --config test/.mocharc.json",
+    "cover": "nyc mocha --config test/.mocharc.json && nyc report --reporter=cobertura",
+    "cover:ci": "nyc mocha --config test/.mocharc.json --reporter mocha-junit-reporter && nyc report --reporter=cobertura",
+    "lint": "eslint src/**",
+    "clean": "shx rm -rf ./dist && shx rm -rf ./docs/out",
+    "build": "npm run clean && npm-run-all --parallel build:ts:* && npm run build:webpack",
+    "build:quick": "npm-run-all --parallel build:ts:* && npm run build:webpack",
+    "build:typedoc": "typedoc --options docs/typedoc.json",
+    "build:ts:cjs": "tsc --build ./tsconfig/tsconfig.bundle.cjs.json",
+    "build:ts:esm": "tsc --build ./tsconfig/tsconfig.bundle.esm.json",
+    "build:ts:esm5": "tsc --build ./tsconfig/tsconfig.bundle.esm5.json",
+    "build:ts:types": "tsc --build ./tsconfig/tsconfig.bundle.types.json",
+    "build:webpack": "npm-run-all --parallel build:webpack:*",
+    "build:webpack:prod": "webpack --env prod --",
+    "build:webpack:dev": "webpack --env dev --",
+    "publish:development": "npm run bump:beta && npm publish --tag dev",
+    "publish:beta": "npm run bump:beta && npm publish --tag beta",
+    "publish:release": "npm run bump:beta && npm publish --tag latest",
+    "bump:development": "standard-version --prerelease alpha -m 'chore(alpha): %s' --author='ci <ci@openhps.org>'",
+    "bump:beta": "standard-version --prerelease beta -m 'chore(beta): %s' --author='ci <ci@openhps.org>'",
+    "bump:release": "standard-version -m 'chore(release): %s' --author='ci <ci@openhps.org>'"
+  },
+  "files": [
+    "dist/**/*",
+    "src/**/*",
+    "LICENSE"
+  ],
+  "author": "Maxim Van de Wynckel",
+  "license": "Apache-2.0",
+  "devDependencies": {
+    "@commitlint/cli": "^19.3.0",
+    "@commitlint/config-conventional": "^19.2.2",
+    "@openhps/core": "^0.7.2",
+    "@openhps/mqtt": "^0.1.9",
+    "@openhps/rdf": "^0.4.64",
+    "@openhps/rf": "^0.1.62",
+    "@openhps/socket": "^0.1.8",
+    "@types/benchmark": "^2.1.5",
+    "@types/chai": "^4.3.16",
+    "@types/fs-extra": "^11.0.4",
+    "@types/long": "^5.0.0",
+    "@types/mocha": "^10.0.6",
+    "@types/node": "^20.14.2",
+    "@typescript-eslint/eslint-plugin": ">=7.13.0",
+    "@typescript-eslint/parser": ">=7.13.0",
+    "benchmark": "^2.1.4",
+    "buffer": "^6.0.3",
+    "eslint": "^8.56.0",
+    "eslint-config-prettier": "^9.1.0",
+    "eslint-plugin-deprecation": "^2.0.0",
+    "eslint-plugin-import": "^2.29.1",
+    "eslint-plugin-jsdoc": "^48.0.3",
+    "eslint-plugin-prettier": "^5.1.3",
+    "husky": "^9.0.11",
+    "mocha": "^10.4.0",
+    "mocha-junit-reporter": "^2.2.1",
+    "npm-run-all": "^4.1.5",
+    "nyc": "^17.0.0",
+    "prettier": "^3.3.2",
+    "reflect-metadata": ">=0.2.2",
+    "shx": "^0.3.4",
+    "standard-version": "^9.5.0",
+    "ts-node": "^10.9.2",
+    "typedoc": ">=0.25.13",
+    "typescript": "^5.4.5",
+    "webpack": "^5.92.0",
+    "webpack-cli": "^5.1.4",
+    "wkt": "^0.1.1"
+  },
+  "dependencies": {
+    "@inquirer/prompts": "^5.0.5",
+    "chai": "^4",
+    "chalk": "4.x",
+    "fs-extra": "^11.2.0",
+    "long": "^5.2.3",
+    "protobufjs": "^7.3.2",
+    "yargs": "^17.7.2"
+  }
+}

package/src/InternalProtobufDeserializer.ts

@@ -0,0 +1,245 @@
+import {
+    ConcreteTypeDescriptor,
+    Deserializer,
+    IndexedObject,
+    MapTypeDescriptor,
+    ObjectMemberMetadata,
+    ObjectMetadata,
+    Serializable,
+    TypeDescriptor,
+    TypedJSON,
+    UUID,
+} from '@openhps/core';
+import * as protobuf from 'protobufjs';
+import { AnyT, JsonObjectMetadata } from 'typedjson';
+
+export class InternalProtobufDeserializer extends Deserializer {
+    protected static primitiveWrapper: protobuf.Type;
+
+    constructor() {
+        super();
+
+        InternalProtobufDeserializer.primitiveWrapper = new protobuf.Type('PrimitiveWrapperMessage');
+        InternalProtobufDeserializer.primitiveWrapper.add(new protobuf.Field('value', 1, 'string'));
+
+        this.setDeserializationStrategy(Number, (object: any) => {
+            if (typeof object === 'number') {
+                return object;
+            } else if (typeof object === 'string') {
+                return Number(object);
+            } else if (object.toNumber) {
+                return object.toNumber();
+            }
+            return object;
+        });
+        this.setDeserializationStrategy(
+            Map,
+            (
+                object: any,
+                typeDescriptor: MapTypeDescriptor,
+                knownTypes: Map<string, Serializable<any>>,
+                memberName: string,
+                deserializer: Deserializer,
+                memberOptions?: ObjectMemberMetadata,
+                serializerOptions?: any,
+            ) => {
+                const result = new Map();
+                Object.keys(object).forEach((key) => {
+                    result.set(
+                        key,
+                        deserializer.convertSingleValue(
+                            object[key],
+                            typeDescriptor.valueType,
+                            knownTypes,
+                            memberName,
+                            memberOptions,
+                            serializerOptions,
+                        ),
+                    );
+                });
+                return result;
+            },
+        );
+    }
+
+    convertSingleValue(
+        sourceObject: any,
+        typeDescriptor: TypeDescriptor,
+        knownTypes: Map<string, Serializable<any>>,
+        memberName?: string,
+        memberOptions?: ObjectMemberMetadata,
+        serializerOptions?: any,
+    ): any {
+        if (this.retrievePreserveNull(memberOptions) && sourceObject === null) {
+            return null;
+        } else if (!TypedJSON.utils.isValueDefined(sourceObject)) {
+            return;
+        }
+
+        const deserializer = this.deserializationStrategy.get(typeDescriptor.ctor);
+        if (deserializer !== undefined) {
+            return deserializer(
+                sourceObject,
+                typeDescriptor,
+                knownTypes,
+                memberName,
+                this,
+                memberOptions,
+                serializerOptions,
+            );
+        }
+
+        if (typeof sourceObject === 'object') {
+            return this.convertAsObject(
+                sourceObject,
+                typeDescriptor,
+                knownTypes,
+                memberName,
+                this,
+                memberOptions,
+                serializerOptions,
+            );
+        }
+
+        let error = `Could not deserialize '${memberName}'; don't know how to deserialize type`;
+
+        if (typeDescriptor.hasFriendlyName()) {
+            error += ` '${typeDescriptor.ctor.name}'`;
+        }
+
+        this.errorHandler(new TypeError(`${error}.`));
+    }
+
+    convertAsObject<T>(
+        sourceObject: IndexedObject,
+        typeDescriptor: ConcreteTypeDescriptor,
+        knownTypes: Map<string, Serializable<any>>,
+        memberName: string,
+        deserializer: InternalProtobufDeserializer,
+        memberOptions?: ObjectMemberMetadata,
+        serializerOptions?: any,
+    ): IndexedObject | T | undefined {
+        let expectedSelfType = typeDescriptor.ctor;
+        let sourceObjectMetadata: ObjectMetadata = JsonObjectMetadata.getFromConstructor(expectedSelfType);
+        if (sourceObject.hasOwnProperty('_type') && sourceObject._type !== 0) {
+            const enumTypeClassname = sourceObjectMetadata.protobuf.enumMapping.get(sourceObject._type);
+            if (enumTypeClassname) {
+                const enumType = knownTypes.get(enumTypeClassname);
+                if (enumType !== expectedSelfType) {
+                    expectedSelfType = enumType;
+                    sourceObjectMetadata = JsonObjectMetadata.getFromConstructor(expectedSelfType);
+                }
+            }
+        } else if (sourceObject.type_url && sourceObject.value) {
+            const MessageType = serializerOptions.types.get(sourceObject.type_url) as protobuf.Type;
+            const message = MessageType.decode(sourceObject.value);
+            const data = this.convertAsObject(
+                message,
+                new ConcreteTypeDescriptor(knownTypes.get(sourceObject.type_url)),
+                knownTypes,
+                memberName,
+                deserializer,
+                memberOptions,
+                serializerOptions,
+            );
+            return data;
+        }
+
+        const sourceMetadata = sourceObjectMetadata;
+        // Strong-typed deserialization available, get to it.
+        // First deserialize properties into a temporary object.
+        const sourceObjectWithDeserializedProperties = {} as IndexedObject;
+
+        // Deserialize by expected properties.
+        sourceMetadata.dataMembers.forEach((objMemberMetadata, propKey) => {
+            const objMemberDebugName = `${TypedJSON.utils.nameof(sourceMetadata.classType)}.${propKey}`;
+            const objMemberOptions = TypedJSON.options.mergeOptions(sourceMetadata.options, objMemberMetadata.options);
+            const memberName =
+                (objMemberOptions && objMemberOptions.protobuf
+                    ? objMemberOptions.protobuf.name
+                    : objMemberMetadata.name) ?? objMemberMetadata.name;
+            const objMemberValue = sourceObject[memberName];
+
+            let revivedValue;
+            if (objMemberMetadata.name === 'uid') {
+                if (sourceObject.hasOwnProperty('uid_bytes') && sourceObject['uid_bytes'].byteLength > 0) {
+                    revivedValue = UUID.fromBuffer(sourceObject['uid_bytes']).toString();
+                } else if (sourceObject.hasOwnProperty('uid_string')) {
+                    revivedValue = sourceObject['uid_string'];
+                } else {
+                    return;
+                }
+            } else if (!sourceObject.hasOwnProperty(memberName) || !objMemberValue) {
+                return;
+            } else if (objMemberMetadata.type() === AnyT && objMemberValue.type_url) {
+                const MessageType = serializerOptions.types.get(sourceObject[memberName].type_url) as protobuf.Type;
+                if (MessageType) {
+                    const message = MessageType.decode(objMemberValue.value);
+                    revivedValue = deserializer.convertSingleValue(
+                        message,
+                        new ConcreteTypeDescriptor(knownTypes.get(objMemberValue.type_url)),
+                        knownTypes,
+                        objMemberDebugName,
+                        objMemberOptions,
+                        serializerOptions,
+                    );
+                } else if (objMemberValue.value) {
+                    const message = InternalProtobufDeserializer.primitiveWrapper.decode(objMemberValue.value) as any;
+                    switch (objMemberValue.type_url) {
+                        case 'Boolean':
+                            revivedValue = Boolean(message.value);
+                            break;
+                        case 'Number':
+                            revivedValue = Number(message.value);
+                            break;
+                        case 'String':
+                        default:
+                            revivedValue = message.value;
+                    }
+                }
+            } else {
+                revivedValue = deserializer.convertSingleValue(
+                    objMemberValue,
+                    objMemberMetadata.type(),
+                    knownTypes,
+                    objMemberDebugName,
+                    objMemberOptions,
+                    serializerOptions,
+                );
+            }
+
+            if (TypedJSON.utils.isValueDefined(revivedValue)) {
+                sourceObjectWithDeserializedProperties[objMemberMetadata.key] = revivedValue;
+            }
+        });
+
+        // Next, instantiate target object.
+        const targetObject: IndexedObject = deserializer.instantiateType(expectedSelfType);
+
+        // Finally, assign deserialized properties to target object.
+        Object.assign(targetObject, sourceObjectWithDeserializedProperties);
+
+        // Call onDeserialized method (if any).
+        const methodName = sourceObjectMetadata.onDeserializedMethodName;
+        if (methodName != null) {
+            if (typeof (targetObject as any)[methodName] === 'function') {
+                // check for member first
+                (targetObject as any)[methodName]();
+            } else if (typeof (targetObject.constructor as any)[methodName] === 'function') {
+                // check for static
+                (targetObject.constructor as any)[methodName]();
+            } else {
+                deserializer.getErrorHandler()(
+                    new TypeError(
+                        `onDeserialized callback` +
+                            `'${TypedJSON.utils.nameof(
+                                sourceObjectMetadata.classType,
+                            )}.${methodName}' is not a method.`,
+                    ),
+                );
+            }
+        }
+
+        return targetObject;
+    }
+}

package/src/InternalProtobufSerializer.ts

@@ -0,0 +1,212 @@
+import {
+    ConcreteTypeDescriptor,
+    IndexedObject,
+    MapTypeDescriptor,
+    ObjectMemberMetadata,
+    ObjectMetadata,
+    Serializer,
+    TypeDescriptor,
+    TypedJSON,
+    UUID,
+} from '@openhps/core';
+import { Type } from 'protobufjs';
+import { AnyT, JsonObjectMetadata } from 'typedjson';
+import * as protobuf from 'protobufjs';
+
+export class InternalProtobufSerializer extends Serializer {
+    protected static primitiveWrapper: Type;
+
+    constructor() {
+        super();
+        this.setSerializationStrategy(Map, this.convertAsMap.bind(this));
+        InternalProtobufSerializer.primitiveWrapper = new protobuf.Type('PrimitiveWrapperMessage');
+        InternalProtobufSerializer.primitiveWrapper.add(new protobuf.Field('value', 1, 'string'));
+    }
+
+    convertSingleValue(
+        sourceObject: any,
+        typeDescriptor: TypeDescriptor,
+        memberName?: string,
+        memberOptions?: ObjectMemberMetadata,
+        serializerOptions?: any,
+    ): any {
+        if (!TypedJSON.utils.isValueDefined(sourceObject)) {
+            return;
+        }
+
+        if (!TypedJSON.utils.isInstanceOf(sourceObject, typeDescriptor.ctor)) {
+            const expectedName = TypedJSON.utils.nameof(typeDescriptor.ctor);
+            const actualName = TypedJSON.utils.nameof(sourceObject.constructor);
+
+            this.errorHandler(
+                new TypeError(
+                    `Could not serialize '${memberName}': expected '${expectedName}',` + ` got '${actualName}'.`,
+                ),
+            );
+            return;
+        }
+
+        const serializer = this.serializationStrategy.get(typeDescriptor.ctor);
+        if (serializer !== undefined) {
+            return serializer(sourceObject, typeDescriptor, memberName, this, memberOptions, serializerOptions);
+        }
+        // if not present in the strategy do property by property serialization
+        if (typeof sourceObject === 'object') {
+            return this.convertAsObject(
+                sourceObject,
+                typeDescriptor,
+                memberName,
+                this,
+                memberOptions,
+                serializerOptions,
+            );
+        }
+
+        let error = `Could not serialize '${memberName}'; don't know how to serialize type`;
+
+        if (typeDescriptor.hasFriendlyName()) {
+            error += ` '${typeDescriptor.ctor.name}'`;
+        }
+
+        this.errorHandler(new TypeError(`${error}.`));
+    }
+
+    convertAsObject(
+        sourceObject: IndexedObject,
+        typeDescriptor: ConcreteTypeDescriptor,
+        memberName: string,
+        serializer: InternalProtobufSerializer,
+        memberOptions?: ObjectMemberMetadata,
+        serializerOptions?: any,
+    ) {
+        const typeMetadata: ObjectMetadata | undefined = JsonObjectMetadata.getFromConstructor(typeDescriptor.ctor);
+        let sourceTypeMetadata: ObjectMetadata | undefined = JsonObjectMetadata.getFromConstructor(
+            sourceObject.constructor,
+        );
+
+        if (!sourceTypeMetadata) {
+            sourceTypeMetadata = typeMetadata;
+        }
+
+        let targetObject: IndexedObject;
+
+        const sourceMeta = sourceTypeMetadata;
+        targetObject = {};
+
+        const classOptions = TypedJSON.options.mergeOptions(serializer.options, sourceMeta.options);
+        sourceMeta.dataMembers.forEach((objMemberMetadata) => {
+            const objMemberOptions = TypedJSON.options.mergeOptions(classOptions, objMemberMetadata.options);
+            let serialized;
+            let memberName =
+                (objMemberOptions && objMemberOptions.protobuf
+                    ? objMemberOptions.protobuf.name
+                    : objMemberMetadata.name) ?? objMemberMetadata.name;
+
+            if (sourceObject[objMemberMetadata.key] === undefined) {
+                return;
+            }
+
+            if (objMemberMetadata.type == null) {
+                throw new TypeError(
+                    `Could not serialize ${objMemberMetadata.name}, there is` +
+                        ` no constructor nor serialization function to use.`,
+                );
+            } else if (objMemberMetadata.name === 'uid') {
+                const uuid = UUID.fromString(sourceObject[objMemberMetadata.key]);
+                if (uuid) {
+                    serialized = uuid.toBuffer();
+                    memberName += '_bytes';
+                } else {
+                    serialized = sourceObject[objMemberMetadata.key];
+                    memberName += '_string';
+                }
+            } else {
+                const MessageType = sourceTypeMetadata.protobuf.messageType;
+                const field = MessageType.get(objMemberMetadata.name);
+                serialized = this.convertSingleValue(
+                    sourceObject[objMemberMetadata.key],
+                    objMemberMetadata.type(),
+                    `${TypedJSON.utils.nameof(sourceMeta.classType)}.${objMemberMetadata.key}`,
+                    objMemberOptions,
+                    {
+                        ...serializerOptions,
+                        field,
+                    },
+                );
+            }
+
+            if (TypedJSON.utils.isValueDefined(serialized)) {
+                if (objMemberMetadata.type() === AnyT) {
+                    const MessageType = serializerOptions.types.get(
+                        sourceObject[objMemberMetadata.key].constructor.name,
+                    ) as protobuf.Type;
+                    if (MessageType) {
+                        const message = MessageType.create(serialized);
+                        serialized = {
+                            type_url: sourceObject[objMemberMetadata.key].constructor.name,
+                            value: MessageType.encode(message).finish(),
+                        };
+                    } else {
+                        const message = InternalProtobufSerializer.primitiveWrapper.create({
+                            value: String(sourceObject[objMemberMetadata.key]),
+                        });
+                        serialized = {
+                            type_url: sourceObject[objMemberMetadata.key].constructor.name,
+                            value: InternalProtobufSerializer.primitiveWrapper.encode(message).finish(),
+                        };
+                    }
+                }
+                targetObject[memberName] = serialized;
+            }
+        });
+
+        const isAnyField = serializerOptions.field && serializerOptions.field.type === 'google.protobuf.Any';
+        if (!isAnyField && typeMetadata.knownTypes.size > 1) {
+            const MessageType = sourceTypeMetadata.protobuf.messageType;
+            const message = MessageType.create({
+                ...targetObject,
+                _type: sourceTypeMetadata.protobuf.messageTypeEnum,
+            });
+            targetObject = message;
+        } else if (isAnyField) {
+            const MessageType = sourceTypeMetadata.protobuf.messageType;
+            const message = MessageType.create(targetObject);
+            targetObject = {
+                type_url: sourceObject.constructor.name,
+                value: MessageType.encode(message).finish(),
+            };
+        }
+        return targetObject;
+    }
+
+    /**
+     * Performs the conversion of a map of typed objects (or primitive values) into an array
+     * of simple javascript objects with `key` and `value` properties.
+     * @param sourceObject
+     * @param typeDescriptor
+     * @param memberName
+     * @param serializer
+     * @param memberOptions
+     * @param serializerOptions
+     */
+    convertAsMap(
+        sourceObject: Map<any, any>,
+        typeDescriptor: MapTypeDescriptor,
+        memberName: string,
+        serializer: Serializer,
+        memberOptions?: ObjectMemberMetadata,
+        serializerOptions?: any,
+    ): IndexedObject | Array<{ key: any; value: any }> {
+        const result = {};
+        sourceObject.forEach((value, key) => {
+            result[key] = serializer.convertSingleValue(
+                value,
+                typeDescriptor.valueType,
+                memberName,
+                memberOptions,
+                serializerOptions,
+            );
+        });
+        return result;
+    }
+}