@coherentglobal/spark-execute-sdk 0.3.0

package/src/node.js

@@ -0,0 +1,254 @@
+const utils = require("./helpers/utils.js");
+const WasmRunnerErrors = require("./error.js");
+const { WasmRunner } = require("@coherentglobal/wasm-runner");
+const logger = require("./logger");
+const validate = require("./validate.js");
+const processModels = require("./models.js");
+const got = require("got");
+
+// const resolverDir = path.join(__dirname, "/tenant_resolver");
+let registry = [];
+let INVOCATION_CACHE = [];
+class Spark {
+  /**
+   * @param {object} config
+   */
+  constructor(config) {
+    /**
+     * @private
+     */
+    this.config = this.validateConfig(config);
+    const {
+      sparkEndpoint: { syntheticKey, bearerToken, authType, tenant, url },
+      nodeGenModels,
+    } = this.config;
+
+    /**
+     * @private
+     */
+    this.tenant = tenant;
+    /**
+     * @private
+     */
+    this.authType = authType;
+    /**
+     * @private
+     */
+    this.token = syntheticKey || bearerToken;
+    /**
+     * @private
+     */
+    this.url = url;
+    /**
+     * @private
+     */
+    this.isCompatible = utils.isWasmSupported();
+    /**
+     * @private
+     */
+    this.model = this._models(nodeGenModels);
+
+    /**
+     * @private
+     */
+    this.fallbackEnabled = false;
+
+    // (async () => {
+    //   if (!(await utils.isPathExist(resolverDir))) {
+    //     await fs.promises.mkdir(resolverDir, { mode: 0o750 });
+    //   }
+    // })();
+  }
+
+  /**
+   *
+   * @param {object} nodegen
+   * @returns {object}
+   */
+  _models(nodegen) {
+    const models = nodegen.map(processModels);
+    return models;
+  }
+
+  /**
+   *
+   * @param {object} config
+   * @returns {object}
+   */
+  validateConfig(config) {
+    const { value, error } = validate(config);
+    if (!error) {
+      return value;
+    } else {
+      const message = JSON.stringify(error.details);
+      logger.error({
+        EventType: "ValidationError",
+        TextMessage: message,
+      });
+      throw new Error(`ValidationError: ${message}`);
+    }
+  }
+
+  /**
+   *
+   * @param {string} id
+   * @returns {boolean}
+   */
+  _getModel(id) {
+    const model = this.model.find((m) => m.versionId === id);
+    if (!model) throw new WasmRunnerErrors.MissingModelError(id);
+
+    return model;
+  }
+
+  /**
+   *
+   * @param {string} versionId
+   */
+  async load(versionId) {
+    const { versionId: versionUuid, binary: modelPath } =
+      this._getModel(versionId);
+
+    logger.info({
+      EventType: "ModelInfo",
+      TextMessage: `Append model tenant: ${this.tenant} uuid: ${versionUuid}`,
+    });
+
+    // if (!registry[this.tenant]) {
+    if (!utils.isEmpty(registry)) {
+      // const resolverPath = path.join(
+      //   resolverDir,
+      //   `externalResolver_${this.tenant}.js`
+      // );
+
+      // const template = await fs.promises.readFile(
+      //   require.resolve("./resolver/externalResolver"),
+      //   {
+      //     encoding: "utf8",
+      //     flag: "r",
+      //   }
+      // );
+
+      // await fs.promises.writeFile(
+      //   resolverPath,
+      //   template.replace("%%tenant_to_replace%%", this.tenant)
+      // );
+
+      registry = new WasmRunner();
+    }
+
+    console.log("APPENDING");
+    await registry.append({
+      id: versionUuid,
+      url: modelPath,
+      // size,
+    });
+
+    INVOCATION_CACHE = [
+      ...INVOCATION_CACHE,
+      { id: versionUuid },
+      // { id: versionUuid, tenant: this.tenant },
+    ];
+  }
+
+  /**
+   * Check for exiting Model
+   *
+   * @param {string} model
+   * @returns {boolean}
+   */
+  isExist = (model) => {
+    return registry.isExist(model);
+  };
+
+  /**
+   * Execute model
+   *
+   * @param {object} input
+   * @param {string} version_id
+   * @returns {object}
+   */
+  async execute(input, version_id) {
+    if (!input.request_meta.version_id && !input.request_meta.version_uuid) {
+      input.request_meta.version_id = version_id;
+    }
+    const versionID =
+      input.request_meta.version_id || input.request_meta.version_uuid;
+
+    const model = this._getModel(versionID);
+
+    await this.load(model.versionId);
+
+    logger.info({
+      EventType: "ExecuteModel",
+      TextMessage: `Execute uuid: ${model.versionId}`,
+    });
+
+    if (this.isExist(model.versionId)) {
+      let modelUrl = "";
+
+      if (
+        !this.isCompatible &&
+        !utils.isEmpty(this.url) &&
+        this.fallbackEnabled
+      ) {
+        let token = "";
+        if (this.token.constructor.name === "AsyncFunction") {
+          token = await this.token();
+        } else {
+          token = this.token;
+        }
+        console.log("TOKEN", token);
+
+        if (utils.isEmpty(this.token)) {
+          throw new WasmRunnerErrors.UnauthorizedError();
+        }
+
+        const options = {
+          headers: utils.getAuthHeaders(token, this.authType, this.tenant),
+          json: {
+            request_data: {},
+            request_meta: input.request_meta,
+          },
+        };
+
+        console.log("OPTIONS", options);
+        modelUrl = new URL(
+          `/${this.tenant}/api/v3/version/${input.request_meta.version_id}`,
+          this.url
+        );
+
+        try {
+          const response = await got.post(modelUrl, options).json();
+
+          return response;
+        } catch (err) {
+          throw new WasmRunnerErrors.UnauthorizedError();
+        }
+      } else if (this.isCompatible) {
+        const modelIndex = INVOCATION_CACHE.findIndex(
+          (item) => item.id === model.versionId
+        );
+
+        INVOCATION_CACHE[modelIndex].lastUsed = Date.now();
+
+        const result = await registry.execute(input, model.versionId, true);
+
+        return result;
+      } else {
+        throw new WasmRunnerErrors.NotSupportedError(
+          "WebAssembly is not supported"
+        );
+      }
+    } else {
+      throw new WasmRunnerErrors.BadRequestError("Missing request input.");
+    }
+  }
+
+  /**
+   * Remove Model
+   */
+  // async remove(versionId) {}
+}
+
+module.exports = Spark;

package/src/resolver/externalResolver.js

@@ -0,0 +1,118 @@
+const { StatusCodes } = require("http-status-codes");
+const { logger } = require("@coherentglobal/node-sdk");
+// const config = require("../config"); env vars
+const {
+  handleDirectRequest,
+  modelServiceValidation,
+  getLatestVersionFromServiceMapTable,
+} = require("../logic/runnerLogic");
+const { getModelViaFolder } = require("../services/entityStore");
+const tenant = "%%tenant_to_replace%%";
+
+const CALL_TYPE = {
+  0: "SparkService",
+  1: "ExternalApi",
+};
+
+const ERROR_CODES = {
+  A1: {
+    code: "a1",
+    message: "ERROR:[Main item] missing in Xcall [NAME] table.",
+  },
+};
+
+const XcallResBuilder = {
+  Json: function (callType, name, status, errorCode, responseTime, data) {
+    return JSON.stringify({
+      metadata: {
+        calltype: callType,
+        name: name,
+        status: status,
+        error_code: errorCode,
+        response_time: responseTime,
+      },
+      data: data || {},
+    });
+  },
+  XML: function (callType, name, status, errorCode, responseTime, data) {
+    const trimmedData = data.replace(/\<\?xml.+\?\>/gm, "");
+
+    const response = `
+    <data calltype="${callType}" name="${name}" status="${status}" error_code="${errorCode}" response_time = "${responseTime}">
+      ${trimmedData}
+    </data>
+    `;
+
+    return response;
+  },
+};
+
+module.exports = {
+  sparkService: async (requestData, context) => {
+    const modelLocation = config().modelLocation;
+    const headers = context.headers;
+    const folderName = requestData.folder_name;
+    const serviceName = requestData.service_name;
+
+    logger.info({
+      eventType: "SparkService",
+      msg: `WASMSERVER:DISPATCH_TYPE.SPARK_SERVICE ${tenant}`,
+    });
+    logger.info({ eventType: "RequestData", msg: requestData });
+    const requestBodyFromEngine = JSON.parse(requestData.request_body);
+    const requestMetaFromEngine = requestBodyFromEngine.request_meta;
+    const findAvailableLatestModelFromLocal =
+      getLatestVersionFromServiceMapTable(
+        context.servicemap,
+        headers.tenant,
+        decodeURI(folderName),
+        decodeURI(serviceName)
+      );
+    requestMetaFromEngine.version_id = findAvailableLatestModelFromLocal;
+    const serviceMap = modelServiceValidation(
+      context.servicemap,
+      headers.tenant,
+      requestMetaFromEngine
+    );
+    // const modelVersion = modelLocation
+    // ? // ? await resolveOffline(requestMetaFromEngine, headers.tenant, headers, serviceMap)
+    // findAvailableLatestModelFromLocal
+    // : await getModelViaFolder(folderName, serviceName, headers);
+
+    const modelVersion = await getModelViaFolder(
+      folderName,
+      serviceName,
+      headers
+    );
+    const params = {
+      tenant: headers.tenant,
+    };
+    const body = {
+      ...requestBodyFromEngine,
+      request_meta: {
+        ...requestBodyFromEngine.request_meta,
+        version_id: modelVersion,
+        call_purpose: "Spark - API Tester",
+        source_system: "SPARK",
+        correlation_id: "",
+        requested_output: null,
+        service_category: "",
+      },
+    };
+    const payload = {
+      body,
+      headers,
+      params,
+    };
+    const results = await handleDirectRequest(payload);
+    const response = XcallResBuilder.Json(
+      CALL_TYPE[0],
+      serviceName,
+      StatusCodes.ACCEPTED,
+      "",
+      0,
+      results
+    );
+    return response;
+  },
+};

package/src/services/entityStore.js

@@ -0,0 +1,260 @@
+const tmp = require("tmp");
+const cuid = require("cuid");
+const fs = require("fs");
+const got = require("got");
+const path = require("path");
+const { logger } = require("@coherentglobal/node-sdk");
+const { EntityStoreError } = require("../error");
+const config = require("../config");
+
+const request = got.extend({
+  throwHttpErrors: false, // allows us to throw our own error based off got response
+  handlers: [
+    (options, next) =>
+      (async () => {
+        const response = await next(options);
+        // return an http error of error response but with local stacktrace and not from got api response
+        if (response && response.statusCode >= 400 && response.body) {
+          let extendError;
+          try {
+            // eslint-disable-next-line prefer-destructuring
+            extendError = JSON.parse(response.body).errors[0];
+          } catch (err) {
+            // eslint-disable-next-line prefer-destructuring
+            extendError = response.body.errors[0];
+          }
+
+          if (response.statusCode === 404) {
+            extendError = "MODEL_NOT_FOUND";
+          }
+
+          logger.info({
+            EventType: "EntityStoreError",
+            TextMessage: `EntityStoreError: ${JSON.stringify(response.body)}`,
+          });
+          throw new EntityStoreError(extendError, response.statusCode);
+        }
+        return response;
+      })(),
+  ],
+});
+
+/**
+ * Fetch Temporary file.
+ *
+ * @param {String} url
+ * Temporary blob download url.
+ *
+ * @param {String} token
+ * Keycloak Token.
+ *
+ * @returns {Buffer}
+ */
+const getFromTempUrl = async (url, headers) => {
+  let options = {};
+
+  if (headers) {
+    options = {
+      headers,
+    };
+  }
+  const output = await request.get(url, options);
+  return output.rawBody;
+};
+
+/**
+ * Fetch documnet using Document Version Id.
+ *
+ * @param {String} versionUuid
+ * Document Version Id.
+ *
+ * @param {String} token
+ * Keycloak token.
+ *
+ * @returns
+ */
+const getDocumentByVersionId = async (versionUuid, token) => {
+  const { entityStoreUrl } = config();
+  const url = new URL(
+    `/docstore/api/v1/documents/versions/${versionUuid}`,
+    entityStoreUrl
+  ).toString();
+  logger.info({
+    EventType: "GetVersionID",
+    TextMessage: `getDocumentByVersionId , ${url}`,
+  });
+  const options = {
+    headers: {
+      Authorization: token,
+    },
+    responseType: "json",
+  };
+
+  const output = await request.get(url, options);
+
+  return output.body;
+};
+
+const getParameterSetById = async (versionUuid, token, tenant = "coherent") => {
+  if (!parameterSetCache[`${tenant}::${versionUuid}`]) {
+    const { sparkUrl } = config();
+    const url = new URL(
+      `/${tenant}/api/v3/parameter/versions/${versionUuid}`,
+      sparkUrl
+    ).toString();
+    logger.info({
+      EventType: "ParameterSet",
+      TextMessage: `getParameterSetById , ${url}`,
+    });
+    const options = {
+      headers: {
+        Authorization: token,
+      },
+      responseType: "json",
+    };
+    const output = await request.get(url, options);
+    parameterSetCache[`${tenant}::${versionUuid}`] = output.body.response_data;
+  }
+
+  return parameterSetCache[`${tenant}::${versionUuid}`];
+};
+
+const TMP_DIR = tmp.dirSync().name;
+
+/**
+ * Save Temporary file.
+ *
+ * @param {String} versionUuid
+ * Document version ID.
+ *
+ * @param {Buffer} content
+ * Temporary file content.
+ *
+ * @param {String} token
+ * Keycloak token.
+ *
+ * @returns
+ */
+const downloadModelFile = async (url, versionUuid, headers) => {
+  const getWasmDocument = await getFromTempUrl(url, headers);
+
+  logger.info({
+    EventType: "DownloadModelFile",
+    TextMessage: `downloadModelFile url:${url}, versionUuid:${versionUuid}`,
+  });
+  const tmpWasm = tmp.fileSync({
+    dir: TMP_DIR,
+    name: `${versionUuid}-${Date.now()}.zip`,
+  });
+
+  const wasmDestination = fs.createWriteStream(tmpWasm.name.toString());
+  wasmDestination.write(getWasmDocument);
+  logger.info({
+    EventType: "DownloadComplete",
+    TextMessage: `downloadModelFile complete url:${url}, versionUuid:${versionUuid}`,
+  });
+  return tmpWasm.name;
+};
+
+const downloadJSON = async (url, versionUuid, headers) => {
+  try {
+    const getDocument = await getFromTempUrl(url, headers);
+
+    const filePath = path.join(TMP_DIR, `${versionUuid}.json`);
+    if (!fs.existsSync(filePath)) {
+      logger.info({
+        EventType: "DownloadJson",
+        TextMessage: `downloadJSON url:${url}, versionUuid:${versionUuid}`,
+      });
+      const tmpWasm = tmp.fileSync({
+        dir: TMP_DIR,
+        name: `${versionUuid}.json`,
+      });
+      const wasmDestination = fs.createWriteStream(tmpWasm.name.toString());
+      wasmDestination.write(getDocument);
+      logger.info({
+        EventType: "JsonDownloaded",
+        TextMessage: `downloadJSON complete url:${url}, versionUuid:${versionUuid}`,
+      });
+    }
+
+    const jsonString = await fs.promises.readFile(filePath, "utf8");
+
+    return JSON.parse(jsonString);
+  } catch (err) {
+    throw new Error("File is not a JSON document");
+  }
+};
+
+/* istanbul ignore next  */
+const downloadModel = async (versionUuid, token) => {
+  const { entityStoreUrl } = config();
+  const { url: documentURL } = await getDocumentByVersionId(versionUuid, token);
+  const downloadUrl = new URL(documentURL, entityStoreUrl).toString();
+  const modelFile = await downloadModelFile(downloadUrl, versionUuid);
+  logger.info({
+    EventType: "ModelDownloaded",
+    TextMessage: `downloadModel complete  versionUuid:${versionUuid}`,
+  });
+  return modelFile;
+};
+
+/* istanbul ignore next  */
+const parameterSetCache = {};
+
+/* istanbul ignore next  */
+const getParameterSet = async (versionUuid, tenant, token) => {
+  const { entityStoreUrl } = config();
+  const url = `https://excel.dev.coherent.global/coherent/api/v3/parameter/versions/${versionUuid}`;
+  if (!parameterSetCache[`${tenant}::${versionUuid}`]) {
+    const { url: documentURL } = await getDocumentByVersionId(
+      versionUuid,
+      token
+    );
+    const downloadUrl = new URL(documentURL, entityStoreUrl).toString();
+    const parameterSet = await downloadJSON(downloadUrl, versionUuid);
+    parameterSetCache[`${tenant}::${versionUuid}`] = parameterSet;
+    return parameterSet;
+  }
+
+  return parameterSetCache[`${tenant}::${versionUuid}`];
+};
+
+/**
+ * 
+ * @param {*} folder 
+ * @param {*} model 
+ * 
+ * https://excel.dev.coherent.global/api/filemanager/GetFolderDocs?folderPath=/products/FJ%20Test/productfactory/engines/DownstreamService_example
+ */
+const getModelViaFolder = async (folder, model, headers) => {
+  const { sparkUrl } = config();
+  let options = {}
+  const folderPath = `/products/${folder}/productfactory/engines/${model}`
+  const folderUrl = `/api/filemanager/GetFolderDocs?folderPath=${folderPath}`
+  const folderSparkUrl = new URL(folderUrl, sparkUrl).toString();
+  if (headers) {
+    options = {
+      headers,
+    };
+  }
+  const output = await request.get(folderSparkUrl, options);
+  const body = output.body ? JSON.parse(output.body) : {}
+  let response = ""
+  if(body) {
+    const documents = body.data.documents
+    const findModelFromDocuments = documents.find( name => name.name === `${model}.c1engine`)
+    if(findModelFromDocuments) response = findModelFromDocuments.latestVersionId
+  }
+  return response
+}
+
+module.exports = {
+  getDocumentByVersionId,
+  getFromTempUrl,
+  downloadModelFile,
+  downloadModel,
+  getParameterSet,
+  getParameterSetById,
+  getModelViaFolder
+};