@tdxvolt/volt-client-grpc 0.18.2 → 0.18.4

package/package.json

@@ -3,7 +3,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "0.18.2",
+  "version": "0.18.4",
   "description": "tdx Volt library for nodejs clients",
   "type": "module",
   "exports": {
@@ -16,9 +16,10 @@
   "files": [
     "lib",
     "src",
-    "protobuf"
+    "scripts"
   ],
   "scripts": {
+    "prebuild": "node ./scripts/generate-proto-assets.js",
     "build": "rollup -c",
     "dev": "rollup -c -w",
     "prepare": "npm run build"
@@ -36,7 +37,7 @@
   "author": "toby.ealden@gmail.com",
   "license": "ISC",
   "dependencies": {
-    "@grpc/proto-loader": "^0.6.4",
+    "@grpc/proto-loader": "^0.7.13",
     "@tdxvolt/volt-client-web": "^0.18.1",
     "bent": "^7.3.12",
     "builtin-modules": "^3.3.0",
@@ -44,6 +45,8 @@
     "ip": "^1.1.5",
     "jsonwebtoken": "^8.5.1",
     "lodash": "^4.17.15",
+    "lodash.camelcase": "^4.3.0",
+    "protobufjs": "^7.4.0",
     "uuid": "^8.1.0"
   }
 }

package/scripts/generate-proto-assets.js

@@ -0,0 +1,71 @@
+// Recursively load all `.proto` files in the `protobuf` directory.
+//
+
+import { readdirSync, statSync, readFileSync, writeFileSync } from "fs";
+import { dirname, join } from "path";
+import { fileURLToPath } from "url";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+const protoFolder = "../../../../tdxvolt-core/src/volt_api";
+const folder = join(__dirname, protoFolder);
+
+// Verify that the proto folder exists.
+try {
+  statSync(folder);
+} catch (e) {
+  console.error(`\nThe proto folder '${folder}' does not exist.`);
+  console.error(
+    `\nTo build this package, you must clone the https://github.com/tdxvolt/tdxvolt-core repository such that the folder structure resembles the following:`
+  );
+  console.error(`\ntdxvolt`);
+  console.error(`  └── tdxvolt-core`);
+  console.error(`  └── tdxvolt-js`);
+  console.error(`\n`);
+  process.exit(1);
+}
+
+function getProtoFiles(dir) {
+  let protoFiles = [];
+
+  const files = readdirSync(dir);
+  files.forEach((file) => {
+    const filePath = join(dir, file);
+    const stats = statSync(filePath);
+    if (stats.isDirectory()) {
+      protoFiles = protoFiles.concat(getProtoFiles(filePath));
+    } else if (file.endsWith(".proto")) {
+      protoFiles.push(filePath);
+    }
+  });
+
+  return protoFiles;
+}
+
+const protoFiles = getProtoFiles(folder);
+
+// Load the contents of each proto file, and write as string literals
+// to the 'volt-proto-literals.js' file.
+let protoLiterals =
+  "/**\n * This file is generated by the 'generate-proto-assets.js' script.\n * Do not modify this file directly.\n */\n\n";
+protoFiles.forEach((filePath) => {
+  // Escape backticks in the proto file.
+  const contents = readFileSync(filePath, "utf8").replace(/`/g, "\\`");
+
+  // Get the base file name without the extension.
+  filePath = filePath.split("/").pop();
+
+  protoLiterals += `export const ${filePath.replace(
+    ".proto",
+    ""
+  )} = \`${contents}\`;\n`;
+});
+
+// Add an array that contains all the literals.
+protoLiterals += `export const voltProtos = [${protoFiles
+  .map((filePath) => filePath.split("/").pop().replace(".proto", ""))
+  .join(", ")}];`;
+
+writeFileSync(join(__dirname, "../src/volt-proto-literals.js"), protoLiterals);
+
+console.log("Proto files loaded and written to 'volt-proto-literals.js'");

package/src/constants.js

@@ -7,7 +7,7 @@ export const constants = {
   privateEncryptedKeyPrefix: "-----BEGIN ENCRYPTED PRIVATE KEY-----",
   /**
    * These top-level service types should map to a protobuf definition
-   * in @tdxvolt/volt-proto.
+   * in the volt-proto-literals.js file.
    */
   serviceType: {
     fileAPI: "tdx.volt_api.volt.v1.FileAPI",
@@ -15,6 +15,7 @@ export const constants = {
     sqliteDatabaseAPI: "tdx.volt_api.data.v1.SqliteDatabaseAPI",
     ssiAPI: "tdx.volt_api.volt.v1.SsiAPI",
     relayAPI: "tdx.volt_api.relay.v1.RelayAPI",
+    terminalAPI: "tdx.volt_api.volt.v1.TerminalAPI",
     voltAPI: "tdx.volt_api.volt.v1.VoltAPI",
     wireAPI: "tdx.volt_api.volt.v1.WireAPI"
   }

package/src/grpc-call.js

@@ -330,7 +330,7 @@ export default class GRPCCall extends EventEmitter {
     this._encryptionKey = null;
   }
 
-  start(grpcClient, request) {
+  start(grpcClient, request, unary = false) {
     this._grpcClient = grpcClient;
 
     if (typeof this._grpcClient[this._methodName] !== "function") {
@@ -441,8 +441,12 @@ export default class GRPCCall extends EventEmitter {
     });
 
     if (this._initialRequest.request) {
-      // Allow callee to attach event handlers before we actually send the initial payload.
-      process.nextTick(() => this._call.write(this._initialRequest.request));
+      if (unary) {
+        this._call.end(this._initialRequest.request);
+      } else {
+        // Allow callee to attach event handlers before we actually send the initial payload.
+        process.nextTick(() => this._call.write(this._initialRequest.request));
+      }
     }
 
     return this;

package/src/proto-package-definition.js

@@ -0,0 +1,183 @@
+/**
+ *
+ * This is copied from the proto-loader package file @grpc/proto-loader/build/src/index.js,
+ * simply in order to export the createPackageDefinition function, which we need to be
+ * able to dynamically load protobuf from strings stored in the Volt database rather than
+ * from files.
+ *
+ * There doesn't seem to be any plans to export this function from the package itself,
+ * see https://github.com/grpc/grpc-node/issues/550.
+ *
+ * This is deemed to be low-risk as the protobuf code is not expected to change often.
+ *
+ * Used by `getServiceDescriptors` in `proto-utils.js`.
+ *
+ */
+
+import camelCase from "lodash.camelcase";
+import Protobuf from "protobufjs";
+import descriptor from "protobufjs/ext/descriptor/index.js";
+import IdempotencyLevel from "@grpc/proto-loader";
+
+const descriptorOptions = {
+  longs: String,
+  enums: String,
+  bytes: String,
+  defaults: true,
+  oneofs: true,
+  json: true
+};
+function joinName(baseName, name) {
+  if (baseName === "") {
+    return name;
+  } else {
+    return baseName + "." + name;
+  }
+}
+function isHandledReflectionObject(obj) {
+  return (
+    obj instanceof Protobuf.Service ||
+    obj instanceof Protobuf.Type ||
+    obj instanceof Protobuf.Enum
+  );
+}
+function isNamespaceBase(obj) {
+  return obj instanceof Protobuf.Namespace || obj instanceof Protobuf.Root;
+}
+function getAllHandledReflectionObjects(obj, parentName) {
+  const objName = joinName(parentName, obj.name);
+  if (isHandledReflectionObject(obj)) {
+    return [[objName, obj]];
+  } else {
+    if (isNamespaceBase(obj) && typeof obj.nested !== "undefined") {
+      return Object.keys(obj.nested)
+        .map((name) => {
+          return getAllHandledReflectionObjects(obj.nested[name], objName);
+        })
+        .reduce(
+          (accumulator, currentValue) => accumulator.concat(currentValue),
+          []
+        );
+    }
+  }
+  return [];
+}
+function createDeserializer(cls, options) {
+  return function deserialize(argBuf) {
+    return cls.toObject(cls.decode(argBuf), options);
+  };
+}
+function createSerializer(cls) {
+  return function serialize(arg) {
+    if (Array.isArray(arg)) {
+      throw new Error(
+        `Failed to serialize message: expected object with ${cls.name} structure, got array instead`
+      );
+    }
+    const message = cls.fromObject(arg);
+    return cls.encode(message).finish();
+  };
+}
+function mapMethodOptions(options) {
+  return (options || []).reduce(
+    (obj, item) => {
+      for (const [key, value] of Object.entries(item)) {
+        switch (key) {
+          case "uninterpreted_option":
+            obj.uninterpreted_option.push(item.uninterpreted_option);
+            break;
+          default:
+            obj[key] = value;
+        }
+      }
+      return obj;
+    },
+    {
+      deprecated: false,
+      idempotency_level: IdempotencyLevel.IDEMPOTENCY_UNKNOWN,
+      uninterpreted_option: []
+    }
+  );
+}
+function createMethodDefinition(method, serviceName, options, fileDescriptors) {
+  /* This is only ever called after the corresponding root.resolveAll(), so we
+   * can assume that the resolved request and response types are non-null */
+  const requestType = method.resolvedRequestType;
+  const responseType = method.resolvedResponseType;
+  return {
+    path: "/" + serviceName + "/" + method.name,
+    requestStream: !!method.requestStream,
+    responseStream: !!method.responseStream,
+    requestSerialize: createSerializer(requestType),
+    requestDeserialize: createDeserializer(requestType, options),
+    responseSerialize: createSerializer(responseType),
+    responseDeserialize: createDeserializer(responseType, options),
+    // TODO(murgatroid99): Find a better way to handle this
+    originalName: camelCase(method.name),
+    requestType: createMessageDefinition(requestType, fileDescriptors),
+    responseType: createMessageDefinition(responseType, fileDescriptors),
+    options: mapMethodOptions(method.parsedOptions)
+  };
+}
+function createServiceDefinition(service, name, options, fileDescriptors) {
+  const def = {};
+  for (const method of service.methodsArray) {
+    def[method.name] = createMethodDefinition(
+      method,
+      name,
+      options,
+      fileDescriptors
+    );
+  }
+  return def;
+}
+function createMessageDefinition(message, fileDescriptors) {
+  const messageDescriptor = message.toDescriptor("proto3");
+  return {
+    format: "Protocol Buffer 3 DescriptorProto",
+    type: messageDescriptor.$type.toObject(
+      messageDescriptor,
+      descriptorOptions
+    ),
+    fileDescriptorProtos: fileDescriptors
+  };
+}
+function createEnumDefinition(enumType, fileDescriptors) {
+  const enumDescriptor = enumType.toDescriptor("proto3");
+  return {
+    format: "Protocol Buffer 3 EnumDescriptorProto",
+    type: enumDescriptor.$type.toObject(enumDescriptor, descriptorOptions),
+    fileDescriptorProtos: fileDescriptors
+  };
+}
+/**
+ * function createDefinition(obj: Protobuf.Service, name: string, options:
+ * Options): ServiceDefinition; function createDefinition(obj: Protobuf.Type,
+ * name: string, options: Options): MessageTypeDefinition; function
+ * createDefinition(obj: Protobuf.Enum, name: string, options: Options):
+ * EnumTypeDefinition;
+ */
+function createDefinition(obj, name, options, fileDescriptors) {
+  if (obj instanceof Protobuf.Service) {
+    return createServiceDefinition(obj, name, options, fileDescriptors);
+  } else if (obj instanceof Protobuf.Type) {
+    return createMessageDefinition(obj, fileDescriptors);
+  } else if (obj instanceof Protobuf.Enum) {
+    return createEnumDefinition(obj, fileDescriptors);
+  } else {
+    throw new Error("Type mismatch in reflection object handling");
+  }
+}
+
+export function createPackageDefinition(root, options) {
+  const def = {};
+  root.resolveAll();
+  const descriptorList = root.toDescriptor("proto3").file;
+  const bufferList = descriptorList.map((value) =>
+    Buffer.from(descriptor.FileDescriptorProto.encode(value).finish())
+  );
+  for (const [name, obj] of getAllHandledReflectionObjects(root, "")) {
+    def[name] = createDefinition(obj, name, options, bufferList);
+  }
+  return def;
+}

package/src/proto-utils.js

@@ -1,130 +1,60 @@
-import protoLoader from "@grpc/proto-loader";
-import lodash from "lodash";
-import { dirname, join } from "path";
-import { fileURLToPath } from "url";
-import { writeFileSync, mkdirSync } from "fs";
-
-const { get: getProp, snakeCase } = lodash;
-const __dirname = dirname(fileURLToPath(import.meta.url));
-
-const defaultProtoPath = join(__dirname, "../protobuf");
+import debug from "debug";
+import protobuf from "protobufjs";
+import { createPackageDefinition } from "./proto-package-definition.js";
+import { voltProtos } from "./volt-proto-literals.js";
+import { constants } from "./constants.js";
+
+const log = debug("volt-client-grpc:proto-utils");
+
+const voltServices = [
+  constants.serviceType.voltAPI,
+  constants.serviceType.fileAPI,
+  constants.serviceType.sqliteDatabaseAPI,
+  constants.serviceType.sqliteServerAPI,
+  constants.serviceType.ssiAPI,
+  constants.serviceType.relayAPI,
+  constants.serviceType.terminalAPI,
+  constants.serviceType.wireAPI
+];
 
 const defaultLoaderOptions = {
   keepCase: true,
   longs: String,
   enums: String,
   defaults: true,
-  oneofs: true,
-  includeDirs: [defaultProtoPath],
+  oneofs: true
 };
 
-function getProtoDescriptors(grpc, protoPath, opts) {
-  const definition = protoLoader.loadSync(
-    protoPath,
-    opts || defaultLoaderOptions,
-  );
-  const descriptors = grpc.loadPackageDefinition(definition);
-  return descriptors;
-}
-
-function getProtoService(descriptors, servicePath) {
-  const serviceDef = getProp(descriptors, servicePath);
-  if (serviceDef?.service) {
-    return serviceDef.service;
-  } else {
-    return null;
-  }
-}
-
-function getServiceDescriptorsFromPath(
-  grpc,
-  protoPath,
-  servicePackageName,
-  opts,
-) {
-  if (!opts) {
-    opts = { ...defaultLoaderOptions };
-    opts.includeDirs = [protoPath];
-  }
-
-  // Extract the package components of the service proto package. The package name should be of the
-  // fully qualified proto path, e.g. tdx.volt_api.webcam.v1.WebcamControlAPI.
-  const protoServiceComponents = servicePackageName.split(".");
-
-  // The service name is the last component of the path.
-  const protoServiceName = protoServiceComponents.pop();
-
-  // The actual file name should match the snake case of the service name.
-  const protoFileName = `${snakeCase(protoServiceName).toLowerCase()}.proto`;
-
-  // The path to the proto file should match each component of the package name.
-  const serviceProtoPath = join(
-    protoPath,
-    ...protoServiceComponents,
-    protoFileName,
-  );
-
-  let serviceDescriptors;
-  const descriptors = getProtoDescriptors(grpc, serviceProtoPath, opts);
-  if (descriptors) {
-    serviceDescriptors = getProtoService(descriptors, servicePackageName);
+function getServiceDescriptors(service) {
+  let root = new protobuf.Root();
+  for (let protoFile of service.service_description.proto_file) {
+    protobuf.parse(protoFile.protobuf, root, defaultLoaderOptions);
+    log("parsed proto file %s", protoFile.file_path);
   }
 
-  return serviceDescriptors;
-}
-
-function getServiceDescriptors(grpc, servicePackageName, opts) {
-  // Extract the package components of the service proto package. The package name should be of the
-  // fully qualified proto path, e.g. tdx.volt_api.webcam.v1.WebcamControlAPI.
-  const protoServiceComponents = servicePackageName.split(".");
-
-  // The service name is the last component of the path.
-  const protoServiceName = protoServiceComponents.pop();
+  // Create a package definition from the root object.
+  const packageDefinition = createPackageDefinition(root, defaultLoaderOptions);
 
-  // The actual file name should match the snake case of the service name.
-  const protoFileName = `${snakeCase(protoServiceName).toLowerCase()}.proto`;
-
-  // The path to the proto file should match each component of the package name.
-  const protoPath = join(
-    defaultProtoPath,
-    ...protoServiceComponents,
-    protoFileName,
-  );
-
-  let serviceDescriptors;
-  const descriptors = getProtoDescriptors(grpc, protoPath, opts);
-  if (descriptors) {
-    serviceDescriptors = getProtoService(descriptors, servicePackageName);
+  // Merge the service API definitions into a single object.
+  let serviceDescriptors = {};
+  for (let api of service.service_description.service_api) {
+    const packageDescriptors = packageDefinition[api];
+    serviceDescriptors = { ...serviceDescriptors, ...packageDescriptors };
   }
 
   return serviceDescriptors;
 }
 
-function getProtoDescriptorMethods(protoDescriptor) {
-  const methods = [];
-  Object.keys(protoDescriptor).forEach((methodName) => {
-    const method = protoDescriptor[methodName];
-    methods.push({
-      path: method.path,
-      client_streaming: method.requestStream,
-      server_streaming: method.responseStream,
-    });
-  });
-  return methods;
-}
+function getBuiltInServiceDescriptors() {
+  // Create a placeholder service object for the Volt API service.
+  const voltServicePlaceholder = {
+    service_description: {
+      proto_file: voltProtos.map((proto) => ({ protobuf: proto })),
+      service_api: voltServices
+    }
+  };
 
-function createServiceProtobufFiles(service, protoPath) {
-  for (let protoFile of service.service_description.proto_file) {
-    const filePath = join(protoPath, protoFile.file_path);
-    mkdirSync(dirname(filePath), { recursive: true });
-    writeFileSync(filePath, protoFile.protobuf);
-  }
+  return getServiceDescriptors(voltServicePlaceholder);
 }
 
-export {
-  createServiceProtobufFiles,
-  getServiceDescriptors,
-  getProtoDescriptors,
-  getProtoDescriptorMethods,
-  getServiceDescriptorsFromPath,
-};
+export { getServiceDescriptors, getBuiltInServiceDescriptors };

package/src/volt-client-internal.js

@@ -4,11 +4,9 @@ import ip from "ip";
 import bent from "bent";
 import lodash from "lodash";
 import VoltConnection from "./volt-connection.js";
-import { join } from "path";
 import {
-  createServiceProtobufFiles,
   getServiceDescriptors,
-  getServiceDescriptorsFromPath
+  getBuiltInServiceDescriptors
 } from "./proto-utils.js";
 import { createClient as createGrpcClient } from "./grpc-utils.js";
 import * as voltUtils from "./utils.js";
@@ -23,15 +21,6 @@ const { publicKeyToPem, removeCarriageReturn, signBase64 } = voltClientUtils;
 const { filter, pick } = lodash;
 
 const log = debug("volt-client-grpc:volt-client-internal");
-const voltServices = [
-  constants.serviceType.voltAPI,
-  constants.serviceType.fileAPI,
-  constants.serviceType.sqliteDatabaseAPI,
-  constants.serviceType.sqliteServerAPI,
-  constants.serviceType.ssiAPI,
-  constants.serviceType.relayAPI,
-  constants.serviceType.wireAPI
-];
 
 function issueAuthenticate(authenticateRequest, ttl) {
   // eslint-disable-next-line no-use-before-define
@@ -371,11 +360,8 @@ export function getVoltAPIClientInternal() {
       ? this._voltConfig.relay.address
       : this._voltConfig.address;
     log("creating cached VoltAPI service client on %s", serviceAddress);
-    let serviceDescriptors = {};
-    voltServices.forEach((pkg) => {
-      const packageDescriptors = getServiceDescriptors(this._grpc, pkg);
-      serviceDescriptors = { ...serviceDescriptors, ...packageDescriptors };
-    });
+
+    const serviceDescriptors = getBuiltInServiceDescriptors();
 
     this._cachedClient = createGrpcClient(
       this._grpc,
@@ -396,20 +382,9 @@ export function getAPIClientInternal(service) {
     const serviceAddress = this.isRelayed
       ? this._voltConfig.relay.address
       : service.service_description.host_address;
-
-    createServiceProtobufFiles(service, "./service-proto");
-
     log("creating API service client on %s", serviceAddress);
-    let serviceDescriptors = {};
-    const fullProtoPath = join(process.cwd(), "./service-proto");
-    for (let api of service.service_description.service_api) {
-      const packageDescriptors = getServiceDescriptorsFromPath(
-        this._grpc,
-        fullProtoPath,
-        api
-      );
-      serviceDescriptors = { ...serviceDescriptors, ...packageDescriptors };
-    }
+
+    const serviceDescriptors = getServiceDescriptors(service);
 
     this._cachedService[service.id] = createGrpcClient(
       this._grpc,

package/src/volt-client.js

@@ -248,63 +248,6 @@ export class VoltClient extends EventEmitter {
       });
   }
 
-  /**
-   * Creates a GRPC client for the given Volt service description.
-   *
-   * Automatically handles remote connections.
-   *
-   * n.b. for this javascript client Protobuf definitions should conform to the recommended
-   * structure, which is that a package proto file resides in a folder path that matches the
-   * package name, and services contained in the package are named in in Pascal Case and using
-   * the suffix 'API'. For example the WebcamControlAPI in package tdx.volt_api.webcam.v1 should reside
-   * in a protobuf file at tdx/api/webcam/v1/webcam_control_api.proto
-   *
-   * @param {object} service a service description.
-   * @param {object} descriptors the service descriptors, must be specified unless the service
-   * packages are well-known Volt APIs.
-   */
-  createServiceClient(service, descriptors) {
-    const pkg = service.service_description.service_api;
-
-    let serviceDescriptors;
-    if (descriptors) {
-      serviceDescriptors = descriptors;
-    } else {
-      serviceDescriptors = {};
-      pkg.forEach((svc) => {
-        log("adding service %s to service client", svc);
-        const packageDescriptors = getServiceDescriptors(this._grpc, svc);
-        serviceDescriptors = { ...serviceDescriptors, ...packageDescriptors };
-      });
-    }
-
-    if (this.isRelayed) {
-      if (!this._voltConfig?.relay?.address) {
-        throw new Error(
-          "Remote connection enabled but no Relay address - have you called start()?"
-        );
-      }
-
-      // In remote mode => create a client using the discovered Relay address **and** passing the service reourceId
-      return createGrpcClient(
-        this._grpc,
-        serviceDescriptors,
-        this._voltConfig.relay.address,
-        this._credential,
-        false,
-        service.volt_id,
-        service.id
-      );
-    } else {
-      return createGrpcClient(
-        this._grpc,
-        serviceDescriptors,
-        service.service_description.address,
-        this._credential
-      );
-    }
-  }
-
   get isRelayed() {
     return !!(this._voltConfig?.relay && this._voltConfig.id);
   }