@sdeverywhere/parse 0.1.0 → 0.1.2

package/dist/index.d.cts

@@ -1,3 +1,35 @@
+/**
+ * Format a model variable or subscript/dimension name into a valid C identifier (with
+ * special characters converted to underscore).
+ *
+ * Note that this should only be called with an individual variable base name (e.g.,
+ * 'Variable name') or a subscript/dimension name (e.g., 'DimA').  In the case where
+ * you have a full variable name that includes subscripts/dimensions (e.g.,
+ * 'Variable name[DimA,B2]'), use `canonicalVarId` to convert the base variable name
+ * and subscript/dimension parts to canonical form indepdendently.
+ *
+ * @param {string} name The name of the variable in the source model, e.g., "Variable name".
+ * @returns {string} The C identifier for the given name, e.g., "_variable_name".
+ */
+declare function canonicalId(name: string): string;
+/**
+ * Format a (subscripted or non-subscripted) model variable name into a canonical identifier,
+ * (with special characters converted to underscore, and subscript/dimension parts separated
+ * by commas).
+ *
+ * @param {string} name The name of the variable in the source model, e.g., "Variable name[DimA, B2]".
+ * @returns {string} The canonical identifier for the given name, e.g., "_variable_name[_dima,_b2]".
+ */
+declare function canonicalVarId(name: string): string;
+/**
+ * Format a model function name into a valid C identifier (with special characters
+ * converted to underscore, and the ID converted to uppercase).
+ *
+ * @param {string} name The name of the variable in the source model, e.g., "FUNCTION name".
+ * @returns {string} The C identifier for the given name, e.g., "_FUNCTION_NAME".
+ */
+declare function canonicalFunctionId(name: string): string;
+
 /** The original name of a dimension, as it appears in the model. */
 type DimName = string;
 /** The canonical identifier of a dimension, as it appears in generated code. */
@@ -442,4 +474,88 @@ declare function parseVensimEquation(input: string): Equation;
  */
 declare function parseVensimModel(input: string, context?: VensimParseContext, sort?: boolean): Model;
 
-export { BinaryOp, BinaryOpExpr, DimId, DimName, DimOrSubId, DimOrSubName, DimensionDef, Equation, EquationLhs, EquationRhs, EquationRhsConstList, EquationRhsData, EquationRhsExpr, EquationRhsLookup, Expr, FormatVariableRefFunc, FunctionCall, FunctionId, FunctionName, Keyword, LookupCall, LookupDef, LookupPoint, LookupRange, Model, NumberLiteral, ParensExpr, PrettyOpts, ReduceExprOptions, StringLiteral, SubId, SubName, SubscriptMapping, SubscriptRef, UnaryOp, UnaryOpExpr, VariableDef, VariableId, VariableName, VariableRef, VensimParseContext, debugPrintExpr, parseVensimEquation, parseVensimExpr, parseVensimModel, parseVensimSubscriptRange, prettyPrintExpr, printExprStats, reduceConditionals, reduceExpr, toPrettyString };
+/**
+ * A single Vensim definition (either a subscript range definition or an
+ * equation definition).  This contains the definition's text and metadata
+ * that was extracted during preprocessing.
+ */
+interface VensimDef {
+    /**
+     * A simplified key for the LHS of the definition, used for sorting
+     * and/or flattening.
+     */
+    key: string;
+    /**
+     * The preprocessed equation or subscript range definition (with
+     * units and comment replaced with `~~|`).
+     */
+    def: string;
+    /**
+     * The kind of definition; either 'eqn' for an equation containing an equals
+     * sign, 'dim' for a dimension (subscript range) definition, or 'decl' for
+     * all other declarations (e.g., a lookup or data variable definition).
+     */
+    kind: 'eqn' | 'dim' | 'decl';
+    /**
+     * The (1-based) line number where the definition begins.
+     */
+    line: number;
+    /**
+     * The units text.
+     */
+    units: string;
+    /**
+     * The comment text.
+     */
+    comment: string;
+    /**
+     * The optional group name, if the definition is contained within a group.
+     */
+    group?: string;
+}
+/**
+ * Result type for the `preprocessVensimModel` function.
+ */
+interface PreprocessedVensimModel {
+    /**
+     * The preprocessed definitions that were preserved.
+     */
+    defs: VensimDef[];
+    /**
+     * The macros that were removed by the preprocessor.
+     */
+    removedMacros: string[];
+    /**
+     * The text blocks that were removed by the preprocessor.  These include
+     * unsupported functions (such as `TABBED ARRAY`) and other definitions
+     * that were requested for removal.
+     */
+    removedBlocks: string[];
+}
+/**
+ * Process the given Vensim model content so that it can be parsed
+ * by `antlr4-vensim`.  This will:
+ *   - strip out group markers
+ *   - remove macro definitions, which are currently unsupported
+ *   - remove equations that reference certain unsupported functions
+ *     (e.g., `TABBED ARRAY`)
+ *   - remove everything in the private Vensim sketch section
+ *   - join lines that are separated by a continuation (backslash)
+ *   - split the input into distinct definitions (equations and
+ *     subscript ranges)
+ *
+ * The definitions are further processed to preserve the units and
+ * comment text in separate properties, but strips them from the
+ * equation string (replaced with `~~`) to make it easier for
+ * `antlr4-vensim` to process.
+ *
+ * @param input The original Vensim mdl file content.
+ * @param options The options that control preprocessing.
+ * @return A `PreprocessedVensimModel` instance containing the preprocessed
+ * Vensim definitions.
+ */
+declare function preprocessVensimModel(input: string, options?: {
+    removalKeys?: string[];
+}): PreprocessedVensimModel;
+
+export { type BinaryOp, type BinaryOpExpr, type DimId, type DimName, type DimOrSubId, type DimOrSubName, type DimensionDef, type Equation, type EquationLhs, type EquationRhs, type EquationRhsConstList, type EquationRhsData, type EquationRhsExpr, type EquationRhsLookup, type Expr, type FormatVariableRefFunc, type FunctionCall, type FunctionId, type FunctionName, type Keyword, type LookupCall, type LookupDef, type LookupPoint, type LookupRange, type Model, type NumberLiteral, type ParensExpr, type PreprocessedVensimModel, type PrettyOpts, type ReduceExprOptions, type StringLiteral, type SubId, type SubName, type SubscriptMapping, type SubscriptRef, type UnaryOp, type UnaryOpExpr, type VariableDef, type VariableId, type VariableName, type VariableRef, type VensimDef, type VensimParseContext, canonicalFunctionId, canonicalId, canonicalVarId, debugPrintExpr, parseVensimEquation, parseVensimExpr, parseVensimModel, parseVensimSubscriptRange, preprocessVensimModel, prettyPrintExpr, printExprStats, reduceConditionals, reduceExpr, toPrettyString };

package/dist/index.d.ts

@@ -1,3 +1,35 @@
+/**
+ * Format a model variable or subscript/dimension name into a valid C identifier (with
+ * special characters converted to underscore).
+ *
+ * Note that this should only be called with an individual variable base name (e.g.,
+ * 'Variable name') or a subscript/dimension name (e.g., 'DimA').  In the case where
+ * you have a full variable name that includes subscripts/dimensions (e.g.,
+ * 'Variable name[DimA,B2]'), use `canonicalVarId` to convert the base variable name
+ * and subscript/dimension parts to canonical form indepdendently.
+ *
+ * @param {string} name The name of the variable in the source model, e.g., "Variable name".
+ * @returns {string} The C identifier for the given name, e.g., "_variable_name".
+ */
+declare function canonicalId(name: string): string;
+/**
+ * Format a (subscripted or non-subscripted) model variable name into a canonical identifier,
+ * (with special characters converted to underscore, and subscript/dimension parts separated
+ * by commas).
+ *
+ * @param {string} name The name of the variable in the source model, e.g., "Variable name[DimA, B2]".
+ * @returns {string} The canonical identifier for the given name, e.g., "_variable_name[_dima,_b2]".
+ */
+declare function canonicalVarId(name: string): string;
+/**
+ * Format a model function name into a valid C identifier (with special characters
+ * converted to underscore, and the ID converted to uppercase).
+ *
+ * @param {string} name The name of the variable in the source model, e.g., "FUNCTION name".
+ * @returns {string} The C identifier for the given name, e.g., "_FUNCTION_NAME".
+ */
+declare function canonicalFunctionId(name: string): string;
+
 /** The original name of a dimension, as it appears in the model. */
 type DimName = string;
 /** The canonical identifier of a dimension, as it appears in generated code. */
@@ -442,4 +474,88 @@ declare function parseVensimEquation(input: string): Equation;
  */
 declare function parseVensimModel(input: string, context?: VensimParseContext, sort?: boolean): Model;
 
-export { BinaryOp, BinaryOpExpr, DimId, DimName, DimOrSubId, DimOrSubName, DimensionDef, Equation, EquationLhs, EquationRhs, EquationRhsConstList, EquationRhsData, EquationRhsExpr, EquationRhsLookup, Expr, FormatVariableRefFunc, FunctionCall, FunctionId, FunctionName, Keyword, LookupCall, LookupDef, LookupPoint, LookupRange, Model, NumberLiteral, ParensExpr, PrettyOpts, ReduceExprOptions, StringLiteral, SubId, SubName, SubscriptMapping, SubscriptRef, UnaryOp, UnaryOpExpr, VariableDef, VariableId, VariableName, VariableRef, VensimParseContext, debugPrintExpr, parseVensimEquation, parseVensimExpr, parseVensimModel, parseVensimSubscriptRange, prettyPrintExpr, printExprStats, reduceConditionals, reduceExpr, toPrettyString };
+/**
+ * A single Vensim definition (either a subscript range definition or an
+ * equation definition).  This contains the definition's text and metadata
+ * that was extracted during preprocessing.
+ */
+interface VensimDef {
+    /**
+     * A simplified key for the LHS of the definition, used for sorting
+     * and/or flattening.
+     */
+    key: string;
+    /**
+     * The preprocessed equation or subscript range definition (with
+     * units and comment replaced with `~~|`).
+     */
+    def: string;
+    /**
+     * The kind of definition; either 'eqn' for an equation containing an equals
+     * sign, 'dim' for a dimension (subscript range) definition, or 'decl' for
+     * all other declarations (e.g., a lookup or data variable definition).
+     */
+    kind: 'eqn' | 'dim' | 'decl';
+    /**
+     * The (1-based) line number where the definition begins.
+     */
+    line: number;
+    /**
+     * The units text.
+     */
+    units: string;
+    /**
+     * The comment text.
+     */
+    comment: string;
+    /**
+     * The optional group name, if the definition is contained within a group.
+     */
+    group?: string;
+}
+/**
+ * Result type for the `preprocessVensimModel` function.
+ */
+interface PreprocessedVensimModel {
+    /**
+     * The preprocessed definitions that were preserved.
+     */
+    defs: VensimDef[];
+    /**
+     * The macros that were removed by the preprocessor.
+     */
+    removedMacros: string[];
+    /**
+     * The text blocks that were removed by the preprocessor.  These include
+     * unsupported functions (such as `TABBED ARRAY`) and other definitions
+     * that were requested for removal.
+     */
+    removedBlocks: string[];
+}
+/**
+ * Process the given Vensim model content so that it can be parsed
+ * by `antlr4-vensim`.  This will:
+ *   - strip out group markers
+ *   - remove macro definitions, which are currently unsupported
+ *   - remove equations that reference certain unsupported functions
+ *     (e.g., `TABBED ARRAY`)
+ *   - remove everything in the private Vensim sketch section
+ *   - join lines that are separated by a continuation (backslash)
+ *   - split the input into distinct definitions (equations and
+ *     subscript ranges)
+ *
+ * The definitions are further processed to preserve the units and
+ * comment text in separate properties, but strips them from the
+ * equation string (replaced with `~~`) to make it easier for
+ * `antlr4-vensim` to process.
+ *
+ * @param input The original Vensim mdl file content.
+ * @param options The options that control preprocessing.
+ * @return A `PreprocessedVensimModel` instance containing the preprocessed
+ * Vensim definitions.
+ */
+declare function preprocessVensimModel(input: string, options?: {
+    removalKeys?: string[];
+}): PreprocessedVensimModel;
+
+export { type BinaryOp, type BinaryOpExpr, type DimId, type DimName, type DimOrSubId, type DimOrSubName, type DimensionDef, type Equation, type EquationLhs, type EquationRhs, type EquationRhsConstList, type EquationRhsData, type EquationRhsExpr, type EquationRhsLookup, type Expr, type FormatVariableRefFunc, type FunctionCall, type FunctionId, type FunctionName, type Keyword, type LookupCall, type LookupDef, type LookupPoint, type LookupRange, type Model, type NumberLiteral, type ParensExpr, type PreprocessedVensimModel, type PrettyOpts, type ReduceExprOptions, type StringLiteral, type SubId, type SubName, type SubscriptMapping, type SubscriptRef, type UnaryOp, type UnaryOpExpr, type VariableDef, type VariableId, type VariableName, type VariableRef, type VensimDef, type VensimParseContext, canonicalFunctionId, canonicalId, canonicalVarId, debugPrintExpr, parseVensimEquation, parseVensimExpr, parseVensimModel, parseVensimSubscriptRange, preprocessVensimModel, prettyPrintExpr, printExprStats, reduceConditionals, reduceExpr, toPrettyString };

package/dist/index.js

@@ -1,3 +1,26 @@
+// src/_shared/canonical-id.js
+var reTrailingMark = new RegExp("\\s+!$", "g");
+var reWhitespace = new RegExp("(\\s|_)+", "g");
+var reSpecialChars = new RegExp(`['"\\.,\\-\\$&%\\/\\|()]`, "g");
+function canonicalId(name) {
+  return "_" + name.trim().replace(reTrailingMark, "!").replace(reWhitespace, "_").replace(reSpecialChars, "_").toLowerCase();
+}
+function canonicalVarId(name) {
+  const m = name.match(/([^[]+)(?:\[([^\]]+)\])?/);
+  if (!m) {
+    throw new Error(`Invalid variable name: ${name}`);
+  }
+  let id = canonicalId(m[1]);
+  if (m[2]) {
+    const subscripts = m[2].split(",").map((x) => canonicalId(x));
+    id += `[${subscripts.join(",")}]`;
+  }
+  return id;
+}
+function canonicalFunctionId(name) {
+  return canonicalId(name).toUpperCase();
+}
+
 // src/ast/print-expr.ts
 import { assertNever } from "assert-never";
 function debugPrintExpr(expr, indent = 0) {
@@ -246,14 +269,6 @@ function fullIdForVarRef(varRef) {
 // src/ast/reduce-expr.ts
 import { assertNever as assertNever2 } from "assert-never";
 
-// src/_shared/names.js
-function canonicalName(name) {
-  return "_" + name.trim().replace(/"/g, "_").replace(/\s+!$/g, "!").replace(/\s/g, "_").replace(/,/g, "_").replace(/-/g, "_").replace(/\./g, "_").replace(/\$/g, "_").replace(/'/g, "_").replace(/&/g, "_").replace(/%/g, "_").replace(/\//g, "_").replace(/\|/g, "_").toLowerCase();
-}
-function cFunctionName(name) {
-  return canonicalName(name).toUpperCase();
-}
-
 // src/ast/ast-builders.ts
 function num(value, text) {
   return {
@@ -629,7 +644,7 @@ var SubscriptRangeReader = class extends ModelVisitor {
     const ids = ctx.Id();
     if (ids.length === 1) {
       const dimName = ids[0].getText();
-      const dimId = canonicalName(dimName);
+      const dimId = canonicalId(dimName);
       super.visitSubscriptRange(ctx);
       return {
         dimName,
@@ -639,7 +654,7 @@ var SubscriptRangeReader = class extends ModelVisitor {
         subscriptRefs: this.subscriptNames.map((subName) => {
           return {
             subName,
-            subId: canonicalName(subName)
+            subId: canonicalId(subName)
           };
         }),
         subscriptMappings: this.subscriptMappings,
@@ -647,9 +662,9 @@ var SubscriptRangeReader = class extends ModelVisitor {
       };
     } else if (ids.length === 2) {
       const dimName = ids[0].getText();
-      const dimId = canonicalName(dimName);
+      const dimId = canonicalId(dimName);
       const familyName = ids[1].getText();
-      const familyId = canonicalName(familyName);
+      const familyId = canonicalId(familyName);
       return {
         dimName,
         dimId,
@@ -689,11 +704,11 @@ var SubscriptRangeReader = class extends ModelVisitor {
     super.visitSubscriptMapping(ctx);
     this.subscriptMappings.push({
       toDimName,
-      toDimId: canonicalName(toDimName),
+      toDimId: canonicalId(toDimName),
       subscriptRefs: this.mappedSubscriptNames.map((subName) => {
         return {
           subName,
-          subId: canonicalName(subName)
+          subId: canonicalId(subName)
         };
       })
     });
@@ -703,7 +718,7 @@ var SubscriptRangeReader = class extends ModelVisitor {
   }
   visitCall(ctx) {
     const fnName = ctx.Id().getText();
-    const fnId = cFunctionName(fnName);
+    const fnId = canonicalFunctionId(fnName);
     if (fnId === "_GET_DIRECT_SUBSCRIPT") {
       super.visitCall(ctx);
     } else {
@@ -799,7 +814,7 @@ var ExprReader = class extends ModelVisitor2 {
   //
   visitCall(ctx) {
     const vensimFnName = ctx.Id().getText();
-    const fnId = cFunctionName(vensimFnName);
+    const fnId = canonicalFunctionId(vensimFnName);
     this.callStack.push({ fn: fnId, args: [] });
     super.visitCall(ctx);
     const callInfo = this.callStack.pop();
@@ -822,14 +837,14 @@ var ExprReader = class extends ModelVisitor2 {
   }
   visitVar(ctx) {
     const vensimVarName = ctx.Id().getText().trim();
-    const varId = canonicalName(vensimVarName);
+    const varId = canonicalId(vensimVarName);
     this.subscripts = void 0;
     super.visitVar(ctx);
     const subscriptNames = this.subscripts;
     const subscriptRefs = subscriptNames?.map((name) => {
       return {
         subName: name,
-        subId: canonicalName(name)
+        subId: canonicalId(name)
       };
     });
     this.subscripts = void 0;
@@ -879,7 +894,7 @@ var ExprReader = class extends ModelVisitor2 {
   }
   visitLookupCall(ctx) {
     const lookupVarName = ctx.Id().getText();
-    const lookupVarId = canonicalName(lookupVarName);
+    const lookupVarId = canonicalId(lookupVarName);
     if (ctx.subscriptList()) {
       ctx.subscriptList().accept(this);
     }
@@ -887,7 +902,7 @@ var ExprReader = class extends ModelVisitor2 {
     const subscriptRefs = subscriptNames?.map((name) => {
       return {
         subName: name,
-        subId: canonicalName(name)
+        subId: canonicalId(name)
       };
     });
     this.subscripts = void 0;
@@ -1084,13 +1099,13 @@ var EquationReader = class extends ModelVisitor3 {
   }
   visitLhs(ctx) {
     const lhsVarName = ctx.Id().getText();
-    const lhsVarId = canonicalName(lhsVarName);
+    const lhsVarId = canonicalId(lhsVarName);
     super.visitLhs(ctx);
     const subscriptNames = this.subscripts;
     const subscriptRefs = subscriptNames?.map((name) => {
       return {
         subName: name,
-        subId: canonicalName(name)
+        subId: canonicalId(name)
       };
     });
     const exceptSubscriptSets = this.exceptSubscriptSets;
@@ -1098,7 +1113,7 @@ var EquationReader = class extends ModelVisitor3 {
       return subscriptSet.map((name) => {
         return {
           subName: name,
-          subId: canonicalName(name)
+          subId: canonicalId(name)
         };
       });
     });
@@ -1178,16 +1193,41 @@ function parseVensimEquation(input) {
 
 // src/vensim/preprocess-vensim.ts
 import split from "split-string";
-function preprocessVensimModel(input) {
+function preprocessVensimModel(input, options) {
+  const removalKeys = options?.removalKeys;
+  function shouldRemove(text) {
+    if (text.includes("TABBED ARRAY")) {
+      return true;
+    }
+    if (removalKeys) {
+      for (const key of removalKeys) {
+        if (text.includes(key)) {
+          return true;
+        }
+      }
+    }
+    return false;
+  }
+  const macrosResult = removeMacros(input);
+  input = macrosResult.processed;
   const rawDefs = splitDefs(input);
   const vensimDefs = [];
+  const removedBlocks = [];
   for (const rawDef of rawDefs) {
+    if (shouldRemove(rawDef.text)) {
+      removedBlocks.push(rawDef.text.trim() + "|");
+      continue;
+    }
     const vensimDef = processDef(rawDef);
     if (vensimDef) {
       vensimDefs.push(vensimDef);
     }
   }
-  return vensimDefs;
+  return {
+    defs: vensimDefs,
+    removedMacros: macrosResult.removed,
+    removedBlocks
+  };
 }
 function splitDefs(input) {
   const defTexts = split(input, { separator: "|", quotes: ['"'], keep: () => true });
@@ -1229,6 +1269,9 @@ function splitDefs(input) {
 function splitLines(input) {
   return input.split(/\r\n|\n|\r/);
 }
+function splitExceptInQuoted(input, sep) {
+  return split(input, { separator: sep, quotes: ['"'] });
+}
 function processBackslashes(input) {
   const inputLines = splitLines(input);
   let output = "";
@@ -1274,22 +1317,27 @@ function replaceDelimitedStrings(str, open, close, newStr) {
 function reduceWhitespace(input) {
   return input.replace(/\s\s+/g, " ").trim();
 }
+var reWhitespace2 = new RegExp("(\\s|_)+", "g");
 function keyForDef(def) {
   let key = def;
   key = key.replace(/:INTERPOLATE:/g, "");
+  let kind;
   if (key.includes("=")) {
+    kind = "eqn";
     key = key.split("=")[0].trim();
   } else if (key.includes(":")) {
+    kind = "dim";
     key = key.split(":")[0].trim();
   } else {
+    kind = "decl";
   }
+  key = splitExceptInQuoted(key, "(")[0];
   key = key.replace(/"/g, "");
-  key = key.split("(")[0];
   key = key.trim();
-  key = key.replace(/\[\s*/g, "[");
-  key = key.replace(/\s*\]/g, "]");
+  key = key.replace(/(?<=\[).*?(?=\])/g, (match) => match.replace(/\s/g, ""));
+  key = key.replace(reWhitespace2, "_");
   key = key.toLowerCase();
-  return key;
+  return { key, kind };
 }
 function processDef(rawDef) {
   let input = rawDef.text;
@@ -1307,7 +1355,7 @@ function processDef(rawDef) {
 ${input}`);
   }
   const rawDefText = reduceWhitespace(parts[0]);
-  const key = keyForDef(rawDefText);
+  const { key, kind } = keyForDef(rawDefText);
   const def = `${rawDefText} ~~|`;
   const units = reduceWhitespace(parts[1]);
   const comment = reduceWhitespace(parts[2]);
@@ -1315,12 +1363,25 @@ ${input}`);
   return {
     key,
     def,
+    kind,
     line: rawDef.line,
     units,
     comment,
     ...group ? { group } : {}
   };
 }
+function removeMacros(input) {
+  const removed = [];
+  const processed = input.replace(/:MACRO:.*:END OF MACRO:/gms, (match) => {
+    removed.push(match);
+    const numBreaks = match.split(/\r\n|\n|\r/gms).length - 1;
+    return numBreaks > 0 ? "\n".repeat(numBreaks) : "";
+  });
+  return {
+    processed,
+    removed
+  };
+}
 
 // src/vensim/impl/model-reader.js
 import { ModelVisitor as ModelVisitor4 } from "antlr4-vensim";
@@ -1379,7 +1440,7 @@ var ModelReader = class extends ModelVisitor4 {
 function parseVensimModel(input, context, sort = false) {
   const dimensions = [];
   const equations = [];
-  const defs = preprocessVensimModel(input);
+  const { defs } = preprocessVensimModel(input);
   if (sort) {
     defs.sort((a, b) => {
       return a.key < b.key ? -1 : a.key > b.key ? 1 : 0;
@@ -1431,11 +1492,15 @@ Detail:
   };
 }
 export {
+  canonicalFunctionId,
+  canonicalId,
+  canonicalVarId,
   debugPrintExpr,
   parseVensimEquation,
   parseVensimExpr,
   parseVensimModel,
   parseVensimSubscriptRange,
+  preprocessVensimModel,
   prettyPrintExpr,
   printExprStats,
   reduceConditionals,