@sdeverywhere/compile 0.7.9 → 0.7.10

package/package.json

@@ -1,12 +1,8 @@
 {
   "name": "@sdeverywhere/compile",
-  "version": "0.7.9",
+  "version": "0.7.10",
   "description": "The core Vensim to C compiler for the SDEverywhere tool suite.",
   "type": "module",
-  "files": [
-    "src/**",
-    "!.DS_Store"
-  ],
   "main": "./src/index.js",
   "dependencies": {
     "antlr4": "4.12.0",
@@ -35,6 +31,10 @@
     "prettier:check": "prettier --check .",
     "prettier:fix": "prettier --write .",
     "precommit": "../../scripts/precommit",
-    "ci:build": "run-s lint prettier:check"
+    "type-check": "tsc --noEmit -p tsconfig-test.json",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "test:ci": "vitest run",
+    "ci:build": "run-s lint prettier:check type-check test:ci"
   }
 }

package/src/_shared/helpers.js

@@ -22,8 +22,17 @@ let nextLevelVarSeq = 1
 let nextAuxVarSeq = 1
 // parsed csv data cache
 let csvData = new Map()
-// string table for web apps
-export let strings = []
+
+// XXX: This is needed for tests due to sequence numbers being in module-level storage
+export function resetHelperState() {
+  nextTmpVarSeq = 1
+  nextLookupVarSeq = 1
+  nextFixedDelayVarSeq = 1
+  nextDepreciationVarSeq = 1
+  nextLevelVarSeq = 1
+  nextAuxVarSeq = 1
+  csvData.clear()
+}
 
 export let canonicalName = name => {
   // Format a model variable name into a valid C identifier.
@@ -200,15 +209,6 @@ export let isIterable = obj => {
   }
   return typeof obj[Symbol.iterator] === 'function'
 }
-export let stringToId = str => {
-  // Look up a string id. Create the id from the string if it is not found.
-  let stringIndex = R.indexOf(str, strings)
-  if (stringIndex < 0) {
-    stringIndex = strings.length
-    strings.push(str)
-  }
-  return `id${stringIndex}`
-}
 // Command helpers
 export let readXlsx = pathname => {
   return XLSX.readFile(pathname, { cellDates: true })

package/src/_shared/subscript.js

@@ -49,6 +49,11 @@ import { canonicalName, asort, vlog } from './helpers.js'
 // subscript name as the key and a subscript object as the value.
 let subscripts = new Map()
 
+// XXX: This is needed for tests due to subs/dims being in module-level storage
+export function resetSubscriptsAndDimensions() {
+  subscripts.clear()
+}
+
 export function Subscript(modelName, modelValue = null, modelFamily = null, modelMappings = null) {
   let name = canonicalName(modelName)
   if (modelValue === null) {

package/src/generate/equation-gen.js

@@ -17,6 +17,7 @@ import {
   newTmpVarName,
   permutationsOf,
   readCsv,
+  readXlsx,
   strToConst,
   vlog
 } from '../_shared/helpers.js'
@@ -208,6 +209,56 @@ export default class EquationGen extends ModelReader {
     }
     return value
   }
+  handleExcelWorkbook(fileOrTag, workbook, tab, dataKind, dataSource) {
+    // Return a `getCellValue` function for the given Excel workbook parsed from an XLS[X] file.
+    if (workbook) {
+      let sheet = workbook.Sheets[tab]
+      if (sheet) {
+        return (c, r) => {
+          let cell = sheet[XLSX.utils.encode_cell({ c, r })]
+          return cell != null ? cdbl(cell.v) : null
+        }
+      } else {
+        throw new Error(`Direct ${dataKind} worksheet ${tab} in ${dataSource} ${fileOrTag} not found`)
+      }
+    } else {
+      throw new Error(`Direct ${dataKind} workbook ${dataSource} ${fileOrTag} not found`)
+    }
+  }
+  handleCsvFile(file, dataPathname, tab, dataKind) {
+    // Return a `getCellValue` function for the given CSV file.
+    let data = readCsv(dataPathname, tab)
+    if (data) {
+      return (c, r) => {
+        let value = '0.0'
+        try {
+          value = data[r] != null && data[r][c] != null ? cdbl(data[r][c]) : null
+        } catch (error) {
+          console.error(`${error.message} in ${dataPathname}`)
+        }
+        return value
+      }
+    } else {
+      throw new Error(`Direct ${dataKind} file ${file} could not be read`)
+    }
+  }
+  handleExcelOrCsvFile(fileOrTag, tab, dataKind) {
+    // Return a `getCellValue` function that reads the CSV or XLS[X] content.
+    if (fileOrTag.startsWith('?')) {
+      // The file is a tag for an Excel file with data in the directData map.
+      let workbook = this.directData.get(fileOrTag)
+      return this.handleExcelWorkbook(fileOrTag, workbook, tab, dataKind, 'tagged')
+    } else {
+      // The file is a CSV or XLS[X] pathname. Read it now.
+      let dataPathname = path.resolve(this.modelDirname, fileOrTag)
+      if (dataPathname.toLowerCase().endsWith('csv')) {
+        return this.handleCsvFile(fileOrTag, dataPathname, tab, dataKind)
+      } else {
+        let workbook = readXlsx(dataPathname)
+        return this.handleExcelWorkbook(fileOrTag, workbook, tab, dataKind, 'file')
+      }
+    }
+  }
   lookupDataNameGen(subscripts) {
     // Construct a name for the static data array associated with a lookup variable.
     return R.map(subscript => {
@@ -360,40 +411,11 @@ export default class EquationGen extends ModelReader {
     // If direct data exists for this variable, copy it from the workbook into one or more lookups.
     let result = []
     if (this.mode === 'init-lookups') {
-      let getCellValue
       let { file, tab, timeRowOrCol, startCell } = this.var.directDataArgs
-      if (file.startsWith('?')) {
-        // The file is a tag for an Excel file with data in the directData map.
-        let workbook = this.directData.get(file)
-        if (workbook) {
-          let sheet = workbook.Sheets[tab]
-          if (sheet) {
-            getCellValue = (c, r) => {
-              let cell = sheet[XLSX.utils.encode_cell({ c, r })]
-              return cell != null ? cdbl(cell.v) : null
-            }
-          } else {
-            throw new Error(`ERROR: Direct data worksheet ${tab} tagged ${file} not found`)
-          }
-        } else {
-          throw new Error(`ERROR: Direct data workbook tagged ${file} not found`)
-        }
-      } else {
-        // The file is a CSV pathname. Read it now.
-        let csvPathname = path.resolve(this.modelDirname, file)
-        let data = readCsv(csvPathname, tab)
-        if (data) {
-          getCellValue = (c, r) => {
-            let value = '0.0'
-            try {
-              value = data[r] != null && data[r][c] != null ? cdbl(data[r][c]) : null
-            } catch (error) {
-              console.error(`${error.message} in ${csvPathname}`)
-            }
-            return value
-          }
-        }
-      }
+
+      // Create a function that reads the CSV or XLS[X] content
+      let getCellValue = this.handleExcelOrCsvFile(file, tab, 'data')
+
       // If the data was found, convert it to a lookup.
       if (getCellValue) {
         let indexNum = 0
@@ -427,12 +449,17 @@ export default class EquationGen extends ModelReader {
     let dataCol, dataRow, dataValue, timeCol, timeRow, timeValue, nextCell
     let lookupData = ''
     let lookupSize = 0
-    let dataAddress = XLSX.utils.decode_cell(startCell)
+    let dataAddress = XLSX.utils.decode_cell(startCell.toUpperCase())
     dataCol = dataAddress.c
     dataRow = dataAddress.r
+    if (dataCol < 0 || dataRow < 0) {
+      throw new Error(
+        `Failed to parse 'cell' argument for GET DIRECT {DATA,LOOKUPS} call for ${this.lhs}: ${startCell}`
+      )
+    }
     if (isNaN(parseInt(timeRowOrCol))) {
       // Time values are in a column.
-      timeCol = XLSX.utils.decode_col(timeRowOrCol)
+      timeCol = XLSX.utils.decode_col(timeRowOrCol.toUpperCase())
       timeRow = dataRow
       dataCol += indexNum
       nextCell = () => {
@@ -468,18 +495,10 @@ export default class EquationGen extends ModelReader {
     // The subscripts may be indices to pick out a subset of the data.
     let result = this.comments
     let { file, tab, startCell } = this.var.directConstArgs
-    let csvPathname = path.resolve(this.modelDirname, file)
-    let data = readCsv(csvPathname, tab)
-    if (data) {
-      let getCellValue = (c, r) => {
-        let value = '0.0'
-        try {
-          value = data[r] != null && data[r][c] != null ? cdbl(data[r][c]) : null
-        } catch (error) {
-          console.error(`${error.message} in ${csvPathname}`)
-        }
-        return value
-      }
+
+    // Create a function that reads the CSV or XLS[X] content
+    let getCellValue = this.handleExcelOrCsvFile(file, tab, 'constants')
+    if (getCellValue) {
       // Get C subscripts in text form for the LHS in normal order.
       let modelLHSReader = new ModelLHSReader()
       modelLHSReader.read(this.var.modelLHS)
@@ -526,12 +545,15 @@ export default class EquationGen extends ModelReader {
         }
         cellOffsets.push(entry)
       }
-      // Read CSV data into an indexed variable for each cell.
+      // Read tabular data into an indexed variable for each cell.
       let numericSubscripts = lhsIndexSubscripts.map(idx => idx.map(s => sub(s).value))
       let lhsSubscripts = numericSubscripts.map(s => s.reduce((a, v) => a.concat(`[${v}]`), ''))
-      let dataAddress = XLSX.utils.decode_cell(startCell)
+      let dataAddress = XLSX.utils.decode_cell(startCell.toUpperCase())
       let startCol = dataAddress.c
       let startRow = dataAddress.r
+      if (startCol < 0 || startRow < 0) {
+        throw new Error(`Failed to parse 'cell' argument for GET DIRECT CONSTANTS call for ${this.lhs}: ${startCell}`)
+      }
       for (let i = 0; i < cellOffsets.length; i++) {
         let rowOffset = cellOffsets[i][0] ? cellOffsets[i][0] : 0
         let colOffset = cellOffsets[i][1] ? cellOffsets[i][1] : 0
@@ -989,13 +1011,18 @@ export default class EquationGen extends ModelReader {
         // Emit the size of the dimension in place of the dimension name.
         this.emit(`${sub(varName).size}`)
       } else {
-        // A subscript masquerading as a variable takes the value of the loop index var plus one
-        // (since Vensim indices are one-based).
+        // A dimension masquerading as a variable (i.e., in expression position) takes the
+        // value of the loop index var plus one (since Vensim indices are one-based).
         let s = this.rhsSubscriptGen([varName])
         // Remove the brackets around the C subscript expression.
         s = s.slice(1, s.length - 1)
         this.emit(`(${s} + 1)`)
       }
+    } else if (isIndex(varName)) {
+      // A subscript masquerading as a variable (i.e., in expression position) takes the
+      // numeric index value plus one (since Vensim indices are one-based).
+      const index = sub(varName).value
+      this.emit(`${index + 1}`)
     } else {
       this.varNames.push(varName)
       if (functionName === '_VECTOR_SELECT') {

package/src/model/equation-reader.js

@@ -22,10 +22,11 @@ import {
 import {
   extractMarkedDims,
   indexNamesForSubscript,
+  isDimension,
+  isIndex,
   normalizeSubscripts,
   separatedVariableIndex,
-  sub,
-  isDimension
+  sub
 } from '../_shared/subscript.js'
 import ModelReader from '../parse/model-reader.js'
 import { createParser } from '../parse/parser.js'
@@ -238,8 +239,8 @@ export default class EquationReader extends ModelReader {
     // Get the var name of a variable in a call and save it as a reference.
     let id = ctx.Id().getText()
     let varName = canonicalName(id)
-    // Do not add a dimension name as a reference.
-    if (!isDimension(varName)) {
+    // Do not add a dimension or index name as a reference.
+    if (!isDimension(varName) && !isIndex(varName)) {
       let fn = this.currentFunctionName()
       this.refId = varName
       this.expandedRefIds = []

package/src/model/model.js

@@ -38,14 +38,45 @@ const PRINT_INIT_GRAPH = false
 const PRINT_AUX_GRAPH = false
 const PRINT_LEVEL_GRAPH = false
 
-function read(parseTree, spec, extData, directData, modelDirname) {
+// XXX: This is needed for tests due to variables being in module-level storage
+function resetModelState() {
+  variables.length = 0
+  inputVars.length = 0
+  variablesByName.clear()
+  constantExprs.clear()
+  nonAtoANames = Object.create(null)
+}
+
+/**
+ * Read the given model parse tree and resolve all subscript and variable/equation definitions.
+ *
+ * Note that this function currently does not return anything and instead stores the parsed subscript
+ * definitions in the `subscript` module and the parsed/analyzed variables in this module.
+ *
+ * @param {import('../parse/parser.js').VensimModelParseTree} parseTree The Vensim parse tree.
+ * @param {*} spec The parsed `spec.json` object.
+ * @param {Map<string, any>} extData The map of datasets from external `.dat` files.
+ * @param {Map<string, any>} directData The mapping of dataset name used in a `GET DIRECT DATA`
+ * call (e.g., `?data`) to the tabular data contained in the loaded data file.
+ * @param {string} modelDirname The path to the directory containing the model (used for resolving data
+ * files for `GET DIRECT SUBSCRIPT`).
+ * @param {*} opts An optional object used by tests to stop the read process after a specific phase.
+ */
+function read(parseTree, spec, extData, directData, modelDirname, opts) {
   // Some arrays need to be separated into variables with individual indices to
   // prevent eval cycles. They are manually added to the spec file.
   let specialSeparationDims = spec.specialSeparationDims
+
   // Subscript ranges must be defined before reading variables that use them.
-  readSubscriptRanges(parseTree, spec.dimensionFamilies, spec.indexFamilies, modelDirname)
+  readSubscriptRanges(parseTree, modelDirname)
+  if (opts?.stopAfterReadSubscripts) return
+  resolveSubscriptRanges(spec.dimensionFamilies)
+  if (opts?.stopAfterResolveSubscripts) return
+
   // Read variables from the model parse tree.
   readVariables(parseTree, specialSeparationDims, directData)
+  if (opts?.stopAfterReadVariables) return
+
   if (spec) {
     // If the spec file contains `input/outputVarNames` (with full Vensim variable names)
     // convert those to C names first.  Otherwise, use `input/outputNames` which are already
@@ -61,20 +92,50 @@ function read(parseTree, spec, extData, directData, modelDirname) {
       inputVars = spec.inputVars
     }
   }
+
   // Analyze model equations to fill in more details about variables.
   analyze()
+  if (opts?.stopAfterAnalyze) return
+
   // Check that all input and output vars in the spec actually exist in the model.
   checkSpecVars(spec, extData)
+
   // Remove variables that are not referenced by an input or output variable.
   removeUnusedVariables(spec)
+
   // Resolve duplicate declarations by converting to one variable type.
   resolveDuplicateDeclarations()
 }
-function readSubscriptRanges(tree, dimensionFamilies, indexFamilies, modelDirname) {
+
+/**
+ * Read subscript ranges from the given model.
+ *
+ * Note that this function currently does not return anything and instead stores the parsed subscript
+ * range definitions in the `subscript` module.
+ *
+ * @param {import('../parse/parser.js').VensimModelParseTree} parseTree The Vensim parse tree.
+ * @param {string} modelDirname The path to the directory containing the model (used for resolving data
+ * files for `GET DIRECT SUBSCRIPT`).
+ */
+function readSubscriptRanges(parseTree, modelDirname) {
   // Read subscript ranges from the model.
   let subscriptRangeReader = new SubscriptRangeReader(modelDirname)
-  subscriptRangeReader.visitModel(tree)
+  subscriptRangeReader.visitModel(parseTree)
+}
+
+/**
+ * Process the previously read subscript/dimension definitions (stored in the `subscript` module) to
+ * resolve aliases, families, and indices.
+ *
+ * Note that this function currently does not return anything and only updates the set of dimension
+ * and subscript definitions in the `subscript` module.
+ *
+ * @param {Object.<string, string>} dimensionFamilies The optional mapping of dimension name to family name
+ * as provided in a `spec.json` file.
+ */
+function resolveSubscriptRanges(dimensionFamilies) {
   let allDims = allDimensions()
+
   // Expand dimensions that appeared in subscript range definitions into indices.
   // Repeat until there are only indices in dimension values.
   let dimFoundInValue
@@ -204,6 +265,21 @@ function readSubscriptRanges(tree, dimensionFamilies, indexFamilies, modelDirnam
     }
   }
 }
+
+/**
+ * Read equations from the given model and generate `Variable` instances for all variables that
+ * are encountered while parsing.
+ *
+ * Note that this function currently does not return anything and instead stores the parsed
+ * variable definitions in the `model` module.
+ *
+ * @param {import('../parse/parser.js').VensimModelParseTree} tree The Vensim parse tree.
+ * @param {Object.<string, string>} specialSeparationDims The variable names that need to be
+ * separated because of circular references.  A mapping from "C" variable name to "C" dimension
+ * name to separate on.
+ * @param {Map<string, any>} directData The mapping of dataset name used in a `GET DIRECT DATA`
+ * call (e.g., `?data`) to the tabular data contained in the loaded data file.
+ */
 function readVariables(tree, specialSeparationDims, directData) {
   // Read all variables in the model parse tree.
   // This populates the variables table with basic information for each variable
@@ -216,6 +292,7 @@ function readVariables(tree, specialSeparationDims, directData) {
   v.varName = '_time'
   addVariable(v)
 }
+
 function analyze() {
   // Analyze the RHS of each equation in stages after all the variables are read.
   // Find non-apply-to-all vars that are defined with more than one equation.
@@ -1146,6 +1223,7 @@ export default {
   read,
   refIdForVar,
   refIdsWithName,
+  resetModelState,
   splitRefId,
   variables,
   varIndexInfo,

package/src/model/subscript-range-reader.js

@@ -115,9 +115,12 @@ export default class SubscriptRangeReader extends ModelReader {
     let lastCell = args[3]
     // let prefix = args[4]
     // If lastCell is a column letter, scan the column, else scan the row.
-    let dataAddress = XLSX.utils.decode_cell(firstCell)
+    let dataAddress = XLSX.utils.decode_cell(firstCell.toUpperCase())
     let col = dataAddress.c
     let row = dataAddress.r
+    if (col < 0 || row < 0) {
+      throw new Error(`Failed to parse 'firstcell' argument for GET DIRECT SUBSCRIPT call: ${firstCell}`)
+    }
     let nextCell
     if (isNaN(parseInt(lastCell))) {
       nextCell = () => row++

package/src/parse/parser.js

@@ -3,13 +3,17 @@
 import antlr4 from 'antlr4'
 import { ModelLexer, ModelParser } from 'antlr4-vensim'
 
+/**
+ * @typedef {object} VensimModelParseTree
+ */
+
 /**
  * Create a `ModelParser` for the given model text, which can be the
  * contents of an entire `mdl` file, or a portion of one (e.g., an
  * expression or definition).
  *
- * @param input The string containing the model text.
- * @return A `ModelParser` from which a parse tree can be obtained.
+ * @param {string} input The string containing the model text.
+ * @return {ModelParser} A `ModelParser` from which a parse tree can be obtained.
  */
 export function createParser(input) {
   let chars = new antlr4.InputStream(input)
@@ -23,8 +27,8 @@ export function createParser(input) {
 /**
  * Read the given model text and return a parse tree.
  *
- * @param input The string containing the model text.
- * @return A parse tree representation of the model.
+ * @param {string} input The string containing the model text.
+ * @return {VensimModelParseTree} A parse tree representation of the model.
  */
 export function parseModel(input) {
   let parser = createParser(input)