@accordproject/template-engine 2.4.1 → 2.5.4

package/dist/index.mjs

@@ -1,8 +1,10 @@
 import jp from 'jsonpath';
 import traverse from 'traverse';
+import os from 'os';
 import { Introspector, ModelManager, Factory, Serializer, ModelUtil } from '@accordproject/concerto-core';
 import dayjs from 'dayjs';
 import utc from 'dayjs/plugin/utc';
+import { ToWords } from 'to-words';
 import { TemplateMarkModel, ConcertoMetaModel, CommonMarkModel, CiceroMarkModel } from '@accordproject/markdown-common';
 import { createDefaultMapFromNodeModules, createDefaultMapFromCDN } from '@typescript/vfs';
 import { twoslasher } from '@typescript/twoslash';
@@ -11,6 +13,7 @@ import { InMemoryWriter } from '@accordproject/concerto-util';
 import { templatemarkutil } from '@accordproject/markdown-template';
 import { existsSync, mkdirSync, rmSync } from 'fs';
 import * as lzstring from 'lz-string';
+import child_process from 'child_process';
 
 function booleanDrafter(value) {
   if (value) {
@@ -34,21 +37,27 @@ function draftDoubleIEEE(value) {
   }
 }
 function draftDoubleFormat(value, format) {
-  return format.replace(/0(.)0((.)(0+))?/gi, function(_a, sep1, _b, sep2, digits) {
-    const len = digits ? digits.length : 0;
-    const vs = value.toFixed(len);
-    let res = "";
-    if (sep2) {
-      const d = vs.substring(vs.length - len);
-      res += sep2 + d;
-    }
-    let i = vs.substring(0, vs.length - (len === 0 ? 0 : len + 1));
-    while (i.length > 3) {
-      res = sep1 + i.substring(i.length - 3) + res;
-      i = i.substring(0, i.length - 3);
-    }
-    return i + res;
-  });
+  if (format === "text") {
+    const converter = new ToWords();
+    const res = converter.convert(value);
+    return res;
+  } else {
+    return format.replace(/0(.)0((.)(0+))?/gi, function(_a, sep1, _b, sep2, digits) {
+      const len = digits ? digits.length : 0;
+      const vs = value.toFixed(len);
+      let res = "";
+      if (sep2) {
+        const d = vs.substring(vs.length - len);
+        res += sep2 + d;
+      }
+      let i = vs.substring(0, vs.length - (len === 0 ? 0 : len + 1));
+      while (i.length > 3) {
+        res = sep1 + i.substring(i.length - 3) + res;
+        i = i.substring(0, i.length - 3);
+      }
+      return i + res;
+    });
+  }
 }
 
 function doubleDrafter(value, format) {
@@ -63,16 +72,22 @@ function draftInteger(value) {
   return "" + value;
 }
 function draftIntegerFormat(value, format) {
-  return format.replace(/0(.)0/gi, function(_a, sep1) {
-    const vs = value.toFixed(0);
-    let res = "";
-    let i = vs.substring(0, vs.length);
-    while (i.length > 3) {
-      res = sep1 + i.substring(i.length - 3) + res;
-      i = i.substring(0, i.length - 3);
-    }
-    return i + res;
-  });
+  if (format === "text") {
+    const converter = new ToWords();
+    const res = converter.convert(value);
+    return res;
+  } else {
+    return format.replace(/0(.)0/gi, function(_a, sep1) {
+      const vs = value.toFixed(0);
+      let res = "";
+      let i = vs.substring(0, vs.length);
+      while (i.length > 3) {
+        res = sep1 + i.substring(i.length - 3) + res;
+        i = i.substring(0, i.length - 3);
+      }
+      return i + res;
+    });
+  }
 }
 
 function integerDrafter(value, format) {
@@ -431,12 +446,138 @@ function joinList(data, joinDef, options) {
   }
 }
 
+async function sleep(msec) {
+  return new Promise((resolve) => setTimeout(resolve, msec));
+}
+class JavaScriptEvaluator {
+  options;
+  workers;
+  // child processes
+  queue;
+  // queue of work to do
+  constructor(options) {
+    this.options = options;
+    this.workers = [];
+    this.queue = [];
+  }
+  /**
+   * Evaluates a JS function in process.
+   * @param {EvalRequest} request - the eval request
+   * @returns {Promise} a promise to the result
+   */
+  async evalDangerously(request) {
+    return new Promise((resolve, reject) => {
+      try {
+        const start = (/* @__PURE__ */ new Date()).getTime();
+        const fun = new Function(...["dayjs", "jp", ...request.argumentNames], request.code);
+        const result = fun(...[dayjs, jp, ...request.arguments]);
+        const end = (/* @__PURE__ */ new Date()).getTime();
+        resolve({ result, elapsed: end - start });
+      } catch (err) {
+        reject({
+          message: err.message
+        });
+      }
+    });
+  }
+  /**
+   * Evaluates a JS function using a node child process
+   * @param {EvalRequest} request the eval request
+   * @param {EvalOptions} options the options for the request
+   * @returns {Promise<EvalResponse>} the async result
+   */
+  async evalChildProcess(request, options = { timeout: 1e4 }) {
+    return new Promise((resolve, reject) => {
+      const now = (/* @__PURE__ */ new Date()).getTime();
+      const workItem = {
+        startTime: now,
+        expireTime: now + options.timeout,
+        request,
+        resolve,
+        reject
+      };
+      if (this.queue.length >= this.options.maxQueueDepth) {
+        reject({ maxQueueDepthExceeded: true, elapsed: 0 });
+      }
+      this.queue.push(workItem);
+      this.processQueue(options);
+    });
+  }
+  processQueue(options) {
+    const now = (/* @__PURE__ */ new Date()).getTime();
+    const notExpired = this.queue.filter((w) => now < w.expireTime);
+    const expired = this.queue.filter((w) => now >= w.expireTime);
+    this.queue = notExpired;
+    expired.forEach((w) => {
+      w.reject({ timeout: true, starvation: true, elapsed: now - w.startTime });
+    });
+    if (this.workers.length < this.options.maxWorkers) {
+      const next = this.queue.shift();
+      if (next) {
+        this.doWork(next, options).then((result) => next.resolve(result)).catch((error) => next.reject(error));
+      }
+    } else {
+      sleep(this.options.waitInterval).then(() => {
+        this.processQueue(options);
+      });
+    }
+  }
+  doWork(work, options) {
+    return new Promise((resolve, reject) => {
+      const start = (/* @__PURE__ */ new Date()).getTime();
+      if (!child_process.fork) {
+        reject({ message: "Cannot use evalChildProcess because child_process.fork is not defined." });
+      }
+      const worker = child_process.fork("./dist/worker.js", { timeout: options.timeout, env: {} });
+      if (!worker.pid) {
+        throw new Error("Failed to fork child process");
+      }
+      this.workers.push(worker);
+      work.pid = worker.pid;
+      let result;
+      worker.on("error", (err) => {
+        this.workers = this.workers.filter((w) => w.pid !== worker.pid);
+        const end = (/* @__PURE__ */ new Date()).getTime();
+        reject({ message: err.message, elapsed: end - start });
+      });
+      worker.on("message", (msg) => {
+        result = msg;
+      });
+      worker.on("exit", (code) => {
+        if (code === null) {
+          this.workers = this.workers.filter((w) => w.pid !== worker.pid);
+          const end = (/* @__PURE__ */ new Date()).getTime();
+          reject({ timeout: true, elapsed: end - start });
+        } else if (code === 0 && result) {
+          this.workers = this.workers.filter((w) => w.pid !== worker.pid);
+          const end = (/* @__PURE__ */ new Date()).getTime();
+          resolve({ ...result, elapsed: end - start });
+        } else {
+          this.workers = this.workers.filter((w) => w.pid !== worker.pid);
+          const end = (/* @__PURE__ */ new Date()).getTime();
+          reject({ code, result, elapsed: end - start });
+        }
+      });
+      worker.send(work.request);
+    });
+  }
+}
+
 function checkCode(code) {
   if (code.type !== CodeType.ES_2020) {
     throw new Error(`Cannot run ${code.contents} as it is not ES_2020 JavaScript.`);
   }
 }
-function evaluateJavaScript(clauseLibrary, data, fn, options) {
+const availableProcessors = process.env.MAX_WORKERS ? Number.parseInt(process.env.MAX_WORKERS) : os.availableParallelism();
+const javaScriptEvaluator = new JavaScriptEvaluator({
+  maxWorkers: availableProcessors,
+  // how many child processes
+  waitInterval: process.env.WAIT_INTERVAL ? Number.parseInt(process.env.WAIT_INTERVAL) : 50,
+  // how long to wait before rescheduling work
+  maxQueueDepth: process.env.MAX_QUEUE_DEPTH ? Number.parseInt(process.env.MAX_QUEUE_DEPTH) : 1e3
+  // max requests to queue
+});
+async function evaluateJavaScript(clauseLibrary, data, fn, options) {
   if (options?.disableJavaScriptEvaluation) {
     throw new Error("JavaScript evaluation is disabled.");
   }
@@ -447,24 +588,26 @@ function evaluateJavaScript(clauseLibrary, data, fn, options) {
   functionArgNames.push("data");
   functionArgNames.push("library");
   functionArgNames.push("options");
-  functionArgNames.push("dayjs");
-  functionArgNames.push("jp");
   const functionArgValues = new Array();
   functionArgValues.push(data);
   functionArgValues.push(clauseLibrary);
   functionArgValues.push(options);
-  functionArgValues.push(dayjs);
-  functionArgValues.push(jp);
   const expression = fn.substring(fn.indexOf("{") + 1, fn.lastIndexOf("}"));
   if (expression.trim().length === 0) {
     throw new Error("Empty expression");
   }
   try {
-    const fun = new Function(...functionArgNames, expression);
-    const result = fun(...functionArgValues);
-    return result;
+    const request = { code: expression, argumentNames: functionArgNames, arguments: functionArgValues };
+    if (options?.childProcessJavaScriptEvaluation) {
+      const evalOptions = options?.timeout ? { timeout: options.timeout } : void 0;
+      const r = await javaScriptEvaluator.evalChildProcess(request, evalOptions);
+      return typeof r.result === "object" ? JSON.stringify(r.result) : r.result.toString();
+    } else {
+      const r = await javaScriptEvaluator.evalDangerously(request);
+      return typeof r.result === "object" ? JSON.stringify(r.result) : r.result.toString();
+    }
   } catch (err) {
-    throw new Error(`Caught error ${err} evaluating ${expression} with arguments ${functionArgValues}`);
+    throw new Error(`Caught error ${JSON.stringify(err)} evaluating ${expression} with arguments ${JSON.stringify(functionArgValues)}`);
   }
 }
 function getJsonPath(rootData, currentNode, paths) {
@@ -495,8 +638,70 @@ function getJsonPath(rootData, currentNode, paths) {
   }
   return withPath.length > 0 ? `$${withPath.join("")}` : "$";
 }
-function generateAgreement(modelManager, clauseLibrary, templateMark, data, options) {
+async function evaluateUserCode(clauseLibrary, templateMark, data, options) {
+  const result = {};
+  const paths = traverse(templateMark).paths();
+  for (let n = 0; n < paths.length; n++) {
+    const path = paths[n];
+    const context = traverse(templateMark).get(path);
+    if (typeof context === "object" && context.$class && typeof context.$class === "string") {
+      const nodeClass = context.$class;
+      if (FORMULA_DEFINITION_RE.test(nodeClass)) {
+        if (context.code) {
+          checkCode(context.code);
+          result[path.join("/")] = await evaluateJavaScript(clauseLibrary, data, context.code.contents, options);
+        } else {
+          throw new Error("Formula node is missing code.");
+        }
+      } else if (CONDITIONAL_DEFINITION_RE.test(nodeClass) || CLAUSE_DEFINITION_RE.test(nodeClass)) {
+        if (context.condition) {
+          checkCode(context.condition);
+          result[path.join("/")] = await evaluateJavaScript(clauseLibrary, data, context.condition.contents, options);
+        }
+      }
+    }
+  }
+  return result;
+}
+async function generateListBlocks(modelManager, clauseLibrary, templateMark, data, options) {
+  const result = {};
+  const paths = traverse(templateMark).paths();
+  for (let n = 0; n < paths.length; n++) {
+    const thisPath = paths[n];
+    const context = traverse(templateMark).get(thisPath);
+    if (typeof context === "object" && context.$class && typeof context.$class === "string") {
+      const nodeClass = context.$class;
+      if (LISTBLOCK_DEFINITION_RE.test(nodeClass)) {
+        const path = getJsonPath(templateMark, context, thisPath);
+        const variableValues = jp.query(data, path, 1);
+        if (variableValues.length === 0) {
+          throw new Error(`No values found for path '${path}' in data ${data}.`);
+        } else {
+          const arrayData = variableValues[0];
+          if (!Array.isArray(arrayData)) {
+            throw new Error(`Values found for path '${path}' in data ${data} is not an array: ${arrayData}.`);
+          } else {
+            const nodes = [];
+            for (let n2 = 0; n2 < arrayData.length; n2++) {
+              const arrayItem = arrayData[n2];
+              const subResult = await generateAgreement(modelManager, clauseLibrary, context.nodes[0].nodes[0], arrayItem, options);
+              nodes.push({
+                $class: `${CommonMarkModel.NAMESPACE}.Item`,
+                nodes: subResult.nodes ? subResult.nodes : []
+              });
+            }
+            result[thisPath.join("/")] = nodes;
+          }
+        }
+      }
+    }
+  }
+  return result;
+}
+async function generateAgreement(modelManager, clauseLibrary, templateMark, data, options) {
   const introspector = new Introspector(modelManager);
+  const userCodeResults = await evaluateUserCode(clauseLibrary, templateMark, data, options);
+  const listBlockResults = await generateListBlocks(modelManager, clauseLibrary, templateMark, data, options);
   return traverse(templateMark).map(function(context) {
     let stopHere = false;
     if (typeof context === "object" && context.$class && typeof context.$class === "string") {
@@ -516,8 +721,7 @@ function generateAgreement(modelManager, clauseLibrary, templateMark, data, opti
         delete context.elementType;
       } else if (FORMULA_DEFINITION_RE.test(nodeClass)) {
         if (context.code) {
-          checkCode(context.code);
-          const result = evaluateJavaScript(clauseLibrary, data, context.code.contents, options);
+          const result = userCodeResults[this.path.join("/")];
           if (result === null) {
             context.value = "<null>";
           } else if (typeof result === "string") {
@@ -530,28 +734,11 @@ function generateAgreement(modelManager, clauseLibrary, templateMark, data, opti
           throw new Error("Formula node is missing code.");
         }
       } else if (LISTBLOCK_DEFINITION_RE.test(nodeClass)) {
-        const path = getJsonPath(templateMark, context, this.path);
-        const variableValues = jp.query(data, path, 1);
-        if (variableValues.length === 0) {
-          throw new Error(`No values found for path '${path}' in data ${data}.`);
-        } else {
-          const arrayData = variableValues[0];
-          if (!Array.isArray(arrayData)) {
-            throw new Error(`Values found for path '${path}' in data ${data} is not an array: ${arrayData}.`);
-          } else {
-            context.$class = `${CommonMarkModel.NAMESPACE}.List`;
-            delete context.elementType;
-            delete context.name;
-            context.nodes = arrayData.map((arrayItem) => {
-              const subResult = generateAgreement(modelManager, clauseLibrary, context.nodes[0].nodes[0], arrayItem);
-              return {
-                $class: `${CommonMarkModel.NAMESPACE}.Item`,
-                nodes: subResult.nodes ? subResult.nodes : []
-              };
-            });
-            stopHere = true;
-          }
-        }
+        context.$class = `${CommonMarkModel.NAMESPACE}.List`;
+        delete context.elementType;
+        delete context.name;
+        context.nodes = listBlockResults[this.path.join("/")];
+        stopHere = true;
       } else if (JOIN_DEFINITION_RE.test(nodeClass)) {
         const path = getJsonPath(templateMark, context, this.path);
         const variableValues = jp.query(data, path, 1);
@@ -597,8 +784,8 @@ function generateAgreement(modelManager, clauseLibrary, templateMark, data, opti
         }
       } else if (CONDITIONAL_DEFINITION_RE.test(nodeClass)) {
         if (context.condition) {
-          checkCode(context.condition);
-          context.isTrue = !!evaluateJavaScript(clauseLibrary, data, context.condition.contents, options);
+          const result = userCodeResults[this.path.join("/")];
+          context.isTrue = !!result;
         } else {
           const path = getJsonPath(templateMark, context, this.path);
           const variableValues = jp.query(data, path, 1);
@@ -619,7 +806,7 @@ function generateAgreement(modelManager, clauseLibrary, templateMark, data, opti
       } else if (CLAUSE_DEFINITION_RE.test(nodeClass)) {
         if (context.condition) {
           checkCode(context.condition);
-          const result = !!evaluateJavaScript(clauseLibrary, data, context.condition.contents, options);
+          const result = !!userCodeResults[this.path.join("/")];
           if (!result) {
             delete context.nodes;
             stopHere = true;
@@ -718,7 +905,7 @@ class TemplateMarkInterpreter {
     }
     const typedTemplateMark = this.checkTypes(templateMark);
     const jsTemplateMark = await this.compileTypeScriptToJavaScript(typedTemplateMark);
-    const ciceroMark = generateAgreement(this.modelManager, this.clauseLibrary, jsTemplateMark, data, options);
+    const ciceroMark = await generateAgreement(this.modelManager, this.clauseLibrary, jsTemplateMark, data, options);
     return this.validateCiceroMark(ciceroMark);
   }
 }