@coherentglobal/spark-execute-sdk 0.3.0

package/src/services/resolver.js

@@ -0,0 +1,134 @@
+const got = require("got");
+const crypto = require("crypto");
+const config = require("../config");
+const { ResolverError } = require("../error");
+const path = require("path");
+
+const getCacheKey = (obj) =>
+  crypto.createHash("sha256").update(JSON.stringify(obj), "utf8").digest("hex");
+
+const resolverCache = new Map();
+
+const DEFAULT_CACHE = 600; // 600 seconds
+
+const resolve = async (requestMeta, tenant, headers) => {
+  const { sparkUrl } = config();
+  try {
+    const key = getCacheKey(requestMeta);
+    if (resolverCache.has(key)) {
+      const cache = resolverCache.get(key);
+
+      if (cache && cache.expireAt > Date.now()) {
+        return {
+          versionUuid: cache.versionUuid,
+          modelUrl: cache.modelUrl,
+        };
+      }
+
+      // If cache already expired
+      /* istanbul ignore next  */
+      resolverCache.delete(key);
+    }
+
+    const options = {
+      headers,
+      json: {
+        request_data: {},
+        request_meta: requestMeta,
+      },
+    };
+
+    const url = new URL(
+      `/${tenant}/api/v3/nodegen/resolver`,
+      sparkUrl
+    ).toString();
+
+    const response = await got.post(url, options).json();
+
+    const { response_data: responseData } = response;
+
+    const {
+      cache_duration: cacheDuration,
+      version_id: versionUuid,
+      wasm_url: wasmUrl,
+      instance_no: instanceNo,
+      paramset_version_id: paramSetVersionId,
+    } = responseData;
+
+    const date = new Date();
+
+    const cache = {
+      versionUuid: versionUuid,
+      modelUrl: wasmUrl,
+      instanceNo,
+      paramSetVersionId,
+      expireAt: date.setSeconds(
+        date.getSeconds() + (Number(cacheDuration) || DEFAULT_CACHE)
+      ),
+    };
+
+    resolverCache.set(key, cache);
+    return cache;
+  } catch (err) {
+    /* istanbul ignore next  */
+    throw new ResolverError(err.message);
+  }
+};
+
+const resolveOffline = async (requestMeta, tenant, headers, servicemap) => {
+  const { modelLocation } = config();
+  try {
+    const key = getCacheKey(requestMeta);
+    if (resolverCache.has(key)) {
+      const cache = resolverCache.get(key);
+
+      if (cache && cache.expireAt > Date.now()) {
+        return {
+          versionUuid: cache.versionUuid,
+          modelUrl: cache.modelUrl,
+        };
+      }
+
+      // If cache already expired
+      /* istanbul ignore next  */
+      resolverCache.delete(key);
+    }
+    // console.log("OFFLINE: ", servicemap)
+    const productName = servicemap.dataJson.EngineInformation.ProductName
+    const serviceName = servicemap.dataJson.EngineInformation.ServiceName
+    const revision = servicemap.dataJson.EngineInformation.Revision
+    const latestVersion = servicemap.latestVersionId
+    let wasmUrl = path.join(
+      `${modelLocation}/${tenant}/${productName}/${serviceName}/${revision}`,
+      `${latestVersion}.zip`
+    );
+    // console.log('wasmUrl: ', wasmUrl)
+
+    const responseData = {
+      cache_duration: 100,
+      version_id: latestVersion,
+      wasm_url: wasmUrl,
+      instance_no: 0,
+    };
+
+    const date = new Date();
+
+    const cache = {
+      versionUuid: responseData.version_id,
+      modelUrl: responseData.wasm_url,
+      instanceNo: responseData.instance_no,
+      expireAt: date.setSeconds(date.getSeconds() + DEFAULT_CACHE),
+    };
+
+    resolverCache.set(key, cache);
+    return cache;
+  } catch (err) {
+    /* istanbul ignore next  */
+    throw new ResolverError(err.message);
+  }
+};
+
+module.exports = {
+  resolve,
+  resolveOffline,
+};

package/src/services/spark.js

@@ -0,0 +1,80 @@
+const got = require("got");
+const fs = require("fs");
+const path = require("path");
+// const { isPathExist } = require("../utils");
+const config = require("../config");
+
+const DEFAULT_CACHE = 600; // 5 minutes
+const tenantConfigCache = new Map();
+
+const configDir = path.join(__dirname, "/tenant_config");
+
+async function fetchTenantConfig(tenant, authHeaders) {
+  /* istanbul ignore next  */
+  // if (!(await isPathExist(configDir))) {
+  //   await fs.promises.mkdir(configDir, { mode: 0o750 });
+  // }
+
+  const { sparkUrl } = config();
+
+  try {
+    const key = tenant;
+    if (tenantConfigCache.has(key)) {
+      const cache = tenantConfigCache.get(key);
+
+      if (cache.expire_at > Date.now()) {
+        return;
+      } else {
+        /* istanbul ignore next  */
+        tenantConfigCache.delete(key);
+      }
+    }
+
+    const options = {
+      headers: authHeaders,
+    };
+
+    const url = new URL(
+      "/api/v1/config/GetAllSparkConfiguration",
+      sparkUrl
+    ).toString();
+
+    const response = await got.get(url, options).json();
+
+    const { data } = response;
+
+    const { ApiDetails: apidetails } = data;
+
+    const cache = {
+      expire_at: Date.now() + DEFAULT_CACHE,
+    };
+
+    tenantConfigCache.set(key, cache);
+    await fs.promises.writeFile(
+      path.join(configDir, `${tenant}_config.json`),
+      JSON.stringify({
+        config: {
+          apidetails,
+        },
+      })
+    );
+    return cache;
+  } catch (err) {
+    throw new Error(err.message);
+  }
+}
+
+async function getConfig(tenant) {
+  const configPath = path.join(configDir, `${tenant}_config.json`);
+  if (fs.existsSync(configPath)) {
+    return JSON.parse(await fs.promises.readFile(configPath));
+  }
+
+  throw new Error(`Tenant: ${tenant} config is empty`);
+}
+
+module.exports = {
+  fetchTenantConfig,
+  getConfig,
+  configDir,
+};

package/src/tenant_resolver/externalResolver_coherent.js

@@ -0,0 +1,118 @@
+const { StatusCodes } = require("http-status-codes");
+const { logger } = require("@coherentglobal/node-sdk");
+// const config = require("../config"); env vars
+const {
+  handleDirectRequest,
+  modelServiceValidation,
+  getLatestVersionFromServiceMapTable,
+} = require("../logic/runnerLogic");
+const { getModelViaFolder } = require("../services/entityStore");
+const tenant = "coherent";
+
+const CALL_TYPE = {
+  0: "SparkService",
+  1: "ExternalApi",
+};
+
+const ERROR_CODES = {
+  A1: {
+    code: "a1",
+    message: "ERROR:[Main item] missing in Xcall [NAME] table.",
+  },
+};
+
+const XcallResBuilder = {
+  Json: function (callType, name, status, errorCode, responseTime, data) {
+    return JSON.stringify({
+      metadata: {
+        calltype: callType,
+        name: name,
+        status: status,
+        error_code: errorCode,
+        response_time: responseTime,
+      },
+      data: data || {},
+    });
+  },
+  XML: function (callType, name, status, errorCode, responseTime, data) {
+    const trimmedData = data.replace(/\<\?xml.+\?\>/gm, "");
+
+    const response = `
+    <data calltype="${callType}" name="${name}" status="${status}" error_code="${errorCode}" response_time = "${responseTime}">
+      ${trimmedData}
+    </data>
+    `;
+
+    return response;
+  },
+};
+
+module.exports = {
+  sparkService: async (requestData, context) => {
+    const modelLocation = config().modelLocation;
+    const headers = context.headers;
+    const folderName = requestData.folder_name;
+    const serviceName = requestData.service_name;
+
+    logger.info({
+      eventType: "SparkService",
+      msg: `WASMSERVER:DISPATCH_TYPE.SPARK_SERVICE ${tenant}`,
+    });
+    logger.info({ eventType: "RequestData", msg: requestData });
+    const requestBodyFromEngine = JSON.parse(requestData.request_body);
+    const requestMetaFromEngine = requestBodyFromEngine.request_meta;
+    const findAvailableLatestModelFromLocal =
+      getLatestVersionFromServiceMapTable(
+        context.servicemap,
+        headers.tenant,
+        decodeURI(folderName),
+        decodeURI(serviceName)
+      );
+    requestMetaFromEngine.version_id = findAvailableLatestModelFromLocal;
+    const serviceMap = modelServiceValidation(
+      context.servicemap,
+      headers.tenant,
+      requestMetaFromEngine
+    );
+    // const modelVersion = modelLocation
+    // ? // ? await resolveOffline(requestMetaFromEngine, headers.tenant, headers, serviceMap)
+    // findAvailableLatestModelFromLocal
+    // : await getModelViaFolder(folderName, serviceName, headers);
+
+    const modelVersion = await getModelViaFolder(
+      folderName,
+      serviceName,
+      headers
+    );
+    const params = {
+      tenant: headers.tenant,
+    };
+    const body = {
+      ...requestBodyFromEngine,
+      request_meta: {
+        ...requestBodyFromEngine.request_meta,
+        version_id: modelVersion,
+        call_purpose: "Spark - API Tester",
+        source_system: "SPARK",
+        correlation_id: "",
+        requested_output: null,
+        service_category: "",
+      },
+    };
+    const payload = {
+      body,
+      headers,
+      params,
+    };
+    const results = await handleDirectRequest(payload);
+    const response = XcallResBuilder.Json(
+      CALL_TYPE[0],
+      serviceName,
+      StatusCodes.ACCEPTED,
+      "",
+      0,
+      results
+    );
+    return response;
+  },
+};

package/src/tenant_resolver/externalResolver_test.js

@@ -0,0 +1,118 @@
+const { StatusCodes } = require("http-status-codes");
+const { logger } = require("@coherentglobal/node-sdk");
+// const config = require("../config"); env vars
+const {
+  handleDirectRequest,
+  modelServiceValidation,
+  getLatestVersionFromServiceMapTable,
+} = require("../logic/runnerLogic");
+const { getModelViaFolder } = require("../services/entityStore");
+const tenant = "test";
+
+const CALL_TYPE = {
+  0: "SparkService",
+  1: "ExternalApi",
+};
+
+const ERROR_CODES = {
+  A1: {
+    code: "a1",
+    message: "ERROR:[Main item] missing in Xcall [NAME] table.",
+  },
+};
+
+const XcallResBuilder = {
+  Json: function (callType, name, status, errorCode, responseTime, data) {
+    return JSON.stringify({
+      metadata: {
+        calltype: callType,
+        name: name,
+        status: status,
+        error_code: errorCode,
+        response_time: responseTime,
+      },
+      data: data || {},
+    });
+  },
+  XML: function (callType, name, status, errorCode, responseTime, data) {
+    const trimmedData = data.replace(/\<\?xml.+\?\>/gm, "");
+
+    const response = `
+    <data calltype="${callType}" name="${name}" status="${status}" error_code="${errorCode}" response_time = "${responseTime}">
+      ${trimmedData}
+    </data>
+    `;
+
+    return response;
+  },
+};
+
+module.exports = {
+  sparkService: async (requestData, context) => {
+    const modelLocation = config().modelLocation;
+    const headers = context.headers;
+    const folderName = requestData.folder_name;
+    const serviceName = requestData.service_name;
+
+    logger.info({
+      eventType: "SparkService",
+      msg: `WASMSERVER:DISPATCH_TYPE.SPARK_SERVICE ${tenant}`,
+    });
+    logger.info({ eventType: "RequestData", msg: requestData });
+    const requestBodyFromEngine = JSON.parse(requestData.request_body);
+    const requestMetaFromEngine = requestBodyFromEngine.request_meta;
+    const findAvailableLatestModelFromLocal =
+      getLatestVersionFromServiceMapTable(
+        context.servicemap,
+        headers.tenant,
+        decodeURI(folderName),
+        decodeURI(serviceName)
+      );
+    requestMetaFromEngine.version_id = findAvailableLatestModelFromLocal;
+    const serviceMap = modelServiceValidation(
+      context.servicemap,
+      headers.tenant,
+      requestMetaFromEngine
+    );
+    // const modelVersion = modelLocation
+    // ? // ? await resolveOffline(requestMetaFromEngine, headers.tenant, headers, serviceMap)
+    // findAvailableLatestModelFromLocal
+    // : await getModelViaFolder(folderName, serviceName, headers);
+
+    const modelVersion = await getModelViaFolder(
+      folderName,
+      serviceName,
+      headers
+    );
+    const params = {
+      tenant: headers.tenant,
+    };
+    const body = {
+      ...requestBodyFromEngine,
+      request_meta: {
+        ...requestBodyFromEngine.request_meta,
+        version_id: modelVersion,
+        call_purpose: "Spark - API Tester",
+        source_system: "SPARK",
+        correlation_id: "",
+        requested_output: null,
+        service_category: "",
+      },
+    };
+    const payload = {
+      body,
+      headers,
+      params,
+    };
+    const results = await handleDirectRequest(payload);
+    const response = XcallResBuilder.Json(
+      CALL_TYPE[0],
+      serviceName,
+      StatusCodes.ACCEPTED,
+      "",
+      0,
+      results
+    );
+    return response;
+  },
+};

package/src/validate.js

@@ -0,0 +1,38 @@
+const Joi = require("joi");
+
+const rules = Joi.object({
+  sparkEndpoint: Joi.object({
+    url: Joi.string()
+      .uri({
+        scheme: ["http", "https"],
+      })
+      .required(),
+    tenant: Joi.string().alphanum().required(),
+    authType: Joi.string()
+      .valid("public", "syntheticKey", "bearerToken")
+      .required(),
+    syntheticKey: [Joi.string().allow(""), Joi.function().allow("")], // allow fns
+    bearerToken: [Joi.string().allow(""), Joi.function().allow("")], // allow strings/fns
+  })
+    .and("url", "tenant", "authType")
+    .required(),
+  nodeGenModels: Joi.array()
+    .items(
+      Joi.object({
+        versionId: Joi.string(),
+        type: Joi.string(),
+        binary: Joi.string(),
+        metaData: Joi.object(),
+      })
+    )
+    .required(),
+  // xParameters: Joi.array().items(),
+  // xCallCallback: Joi.func()
+}).required();
+
+const validate = (config) => {
+  const isValidConfig = rules.validate(config);
+  return isValidConfig;
+};
+
+module.exports = validate;

package/test/config.test.js

@@ -0,0 +1,124 @@
+const { expect } = require("chai");
+const validate = require("../src/validate.js");
+
+describe("config", () => {
+  it("invalidate empty config", () => {
+    const fixture = {};
+    const checkIfValid = validate(fixture);
+    const errorName = checkIfValid.error.name;
+    const errorType = checkIfValid.error.details[0].type;
+    expect(errorName).to.equal("ValidationError");
+    expect(errorType).to.equal("any.required");
+  });
+  it('empty url not allowed', () => {
+    const fixture = {
+        sparkEndpoint: {
+            url: "",
+            tenant: "",
+            authType: "",
+            syntheticKey: "",
+            bearerToken: ""
+        },
+        nodeGenModels: []
+    }
+    const checkIfValid = validate(fixture)
+    const errorName = checkIfValid.error.name;
+    const errorType = checkIfValid.error.details[0].type;
+    const errorLabel = checkIfValid.error.details[0].context.label;
+    // console.log('ASD: ', JSON.stringify(checkIfValid))
+    expect(errorName).to.equal("ValidationError");
+    expect(errorType).to.equal("string.empty");
+    expect(errorLabel).to.equal("sparkEndpoint.url");
+  })
+  it('invalid url', () => {
+    const fixture = {
+        sparkEndpoint: {
+            url: "http:",
+            tenant: "",
+            authType: "",
+            syntheticKey: "",
+            bearerToken: ""
+        },
+        nodeGenModels: []
+    }
+    const checkIfValid = validate(fixture)
+    const errorName = checkIfValid.error.name;
+    const errorType = checkIfValid.error.details[0].type;
+    // console.log('ASD: ', JSON.stringify(checkIfValid))
+    expect(errorName).to.equal("ValidationError");
+    expect(errorType).to.equal("string.uriCustomScheme");
+  })
+  it('tenant not allowed to be emtpy', () => {
+    const fixture = {
+        sparkEndpoint: {
+            url: "https://spark.dev.coherent.global",
+            tenant: "",
+            authType: "",
+            syntheticKey: "",
+            bearerToken: ""
+        },
+        nodeGenModels: []
+    }
+    const checkIfValid = validate(fixture)
+    const errorName = checkIfValid.error.name;
+    const errorType = checkIfValid.error.details[0].type;
+    const errorLabel = checkIfValid.error.details[0].context.label;
+    expect(errorName).to.equal("ValidationError");
+    expect(errorType).to.equal("string.empty");
+    expect(errorLabel).to.equal("sparkEndpoint.tenant");
+  })
+  it('authType not allowed to be emtpy', () => {
+    const fixture = {
+        sparkEndpoint: {
+            url: "https://spark.dev.coherent.global",
+            tenant: "coherent",
+            authType: "",
+            syntheticKey: "",
+            bearerToken: ""
+        },
+        nodeGenModels: []
+    }
+    const checkIfValid = validate(fixture)
+    const errorName = checkIfValid.error.name;
+    const errorType = checkIfValid.error.details[0].type;
+    const errorLabel = checkIfValid.error.details[0].context.label;
+    expect(errorName).to.equal("ValidationError");
+    expect(errorType).to.equal("any.only");
+    expect(errorLabel).to.equal("sparkEndpoint.authType");
+  })
+  it('authType must use valid strings', () => {
+    const fixture = {
+        sparkEndpoint: {
+            url: "https://spark.dev.coherent.global",
+            tenant: "coherent",
+            authType: "publics",
+            syntheticKey: "",
+            bearerToken: ""
+        },
+        nodeGenModels: []
+    }
+    const checkIfValid = validate(fixture)
+    const errorName = checkIfValid.error.name;
+    const errorType = checkIfValid.error.details[0].type;
+    const errorLabel = checkIfValid.error.details[0].context.label;
+    expect(errorName).to.equal("ValidationError");
+    expect(errorType).to.equal("any.only");
+    expect(errorLabel).to.equal("sparkEndpoint.authType");
+  })
+  it('valid config', () => {
+    const fixture = {
+        sparkEndpoint: {
+            url: "https://spark.dev.coherent.global",
+            tenant: "coherent",
+            authType: "public",
+            syntheticKey: "",
+            bearerToken: ""
+        },
+        nodeGenModels: []
+    }
+    const checkIfValid = validate(fixture)
+    // console.log('ASD: ', JSON.stringify(checkIfValid))
+    const errorName = checkIfValid.error;
+    expect(errorName).to.equal(undefined);
+  })
+});