@sdeverywhere/compile 0.7.23 → 0.7.25

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sdeverywhere/compile",
-  "version": "0.7.23",
+  "version": "0.7.25",
   "description": "The core Vensim to C compiler for the SDEverywhere tool suite.",
   "type": "module",
   "main": "./src/index.js",

package/src/_shared/helpers.js

@@ -24,6 +24,8 @@ let nextLevelVarSeq = 1
 let nextAuxVarSeq = 1
 // parsed csv data cache
 let csvData = new Map()
+// parsed xlsx data cache
+let xlsxData = new Map()
 
 // Newer versions of the xlsx package require manually setting the `fs` instance
 // before using the `XLSX.readFile` function
@@ -38,6 +40,7 @@ export function resetHelperState() {
   nextLevelVarSeq = 1
   nextAuxVarSeq = 1
   csvData.clear()
+  xlsxData.clear()
 }
 
 export let canonicalName = name => {
@@ -200,7 +203,15 @@ export let isIterable = obj => {
 }
 // Command helpers
 export let readXlsx = pathname => {
-  return XLSX.readFile(pathname, { cellDates: true })
+  // Read the XLSX file at the pathname and parse it.
+  // Return a `XLSX.WorkBook` object that can be used to access the data.
+  // Cache parsed files to support multiple reads from different equations.
+  let xlsx = xlsxData.get(pathname)
+  if (!xlsx) {
+    xlsx = XLSX.readFile(pathname, { cellDates: true })
+    xlsxData.set(pathname, xlsx)
+  }
+  return xlsx
 }
 export let readCsv = (pathname, delimiter = ',') => {
   // Read the CSV file at the pathname and parse it with the given delimiter.
@@ -208,7 +219,7 @@ export let readCsv = (pathname, delimiter = ',') => {
   // If there is a header row, it is returned as the first row.
   // Cache parsed files to support multiple reads from different equations.
   let csv = csvData.get(pathname)
-  if (csv == null) {
+  if (!csv) {
     const CSV_PARSE_OPTS = {
       delimiter,
       columns: false,

package/src/_shared/subscript.js

@@ -107,7 +107,7 @@ export function sub(name) {
   let result
   try {
     result = subscripts.get(name)
-  } catch (e) {
+  } catch (_) {
     console.error(`sub name ${name} not found`)
   }
   return result
@@ -249,7 +249,7 @@ export function subscriptFamilies(subscripts) {
   // Return a list of the subscript families for each subscript.
   try {
     return R.map(subscriptName => sub(subscriptName).family, subscripts)
-  } catch (e) {
+  } catch (_) {
     console.error(`ERROR: subscript not found in "${subscripts.join(',')}" in subscriptFamilies`)
   }
 }

package/src/_shared/var-names.js

@@ -0,0 +1,31 @@
+import { canonicalName } from './helpers.js'
+import { isIndex, sub } from './subscript.js'
+
+/**
+ * Convert a Vensim variable name to a C name.
+ *
+ * WARNING: This function requires model analysis to be completed first when the variable
+ * has subscripts.
+ *
+ * @param {string} vensimVarName The full Vensim variable name (can contain subscripts).
+ * @returns {string} A canonical C representation of the variable name (e.g., '_variable_name').
+ */
+export function cName(vensimVarName) {
+  // Split the variable name from the subscripts
+  let matches = vensimVarName.match(/([^[]+)(?:\[([^\]]+)\])?/)
+  if (!matches) {
+    throw new Error(`Invalid variable name '${vensimVarName}' found when converting to C representation`)
+  }
+  let cVarName = canonicalName(matches[1])
+  if (matches[2]) {
+    // The variable name includes subscripts, so split them into individual IDs
+    let cSubIds = matches[2].split(',').map(x => canonicalName(x))
+    // If a subscript is an index, convert it to an index number to match Vensim data exports
+    let cSubIdParts = cSubIds.map(cSubId => {
+      return isIndex(cSubId) ? `[${sub(cSubId).value}]` : `[${cSubId}]`
+    })
+    // Append the subscript parts to the base variable name to create the full reference
+    cVarName += cSubIdParts.join('')
+  }
+  return cVarName
+}

package/src/generate/expand-var-names.js

@@ -1,16 +1,17 @@
 import * as R from 'ramda'
 
-import { cartesianProductOf, canonicalName } from '../_shared/helpers.js'
-import { sub, isDimension } from '../_shared/subscript.js'
+import { cName } from '../_shared/var-names.js'
 
+import { expandVar } from '../model/expand-var-instances.js'
 import Model from '../model/model.js'
 
 /**
- * Return an array of names for all variable in the model, sorted alphabetically and expanded to
+ * Return an array of names for all accessible variables, sorted alphabetically and expanded to
  * include the full set of subscripted variants for variables that include subscripts.
  *
- * @param canonical If true, convert names to canonical representation (variable identifiers), otherwise
- * return the original name of each variable as it appears in the model.
+ * @param {*} variables The `Variable` objects to process.
+ * @param {boolean} canonical If true, convert names to canonical representation (variable identifiers),
+ * otherwise return the original name of each variable as it appears in the model.
  * @returns {string[]} An array of variable names or identifiers.
  */
 export function expandVarNames(canonical) {
@@ -19,10 +20,12 @@ export function expandVarNames(canonical) {
     R.reduce(
       (a, v) => {
         if (v.varType !== 'lookup' && v.varType !== 'data' && v.includeInOutput) {
+          const varInstances = expandVar(v)
+          const varNames = varInstances.map(instance => instance.varName)
           if (canonical) {
-            return R.concat(a, R.map(Model.cName, namesForVar(v)))
+            return R.concat(a, R.map(cName, varNames))
           } else {
-            return R.concat(a, namesForVar(v))
+            return R.concat(a, varNames)
           }
         } else {
           return a
@@ -33,73 +36,3 @@ export function expandVarNames(canonical) {
     )
   )
 }
-
-/**
- * Return an array of names for the given variable including all subscript variants.
- *
- * @param {*} v A `Variable` instance.
- * @returns {string[]} An array of expanded names for the given variable.
- */
-function namesForVar(v) {
-  if (v.parsedEqn === undefined) {
-    // XXX: The special `Time` variable does not have a `parsedEqn`, so use the raw LHS
-    return [v.modelLHS]
-  }
-
-  // Expand each variable to get the names of all subscripted variants
-  const lhsVarDef = v.parsedEqn.lhs.varDef
-  const lhsSubRefs = lhsVarDef.subscriptRefs
-  if (lhsSubRefs?.length > 0) {
-    // At each position, expand any dimensions or use a subscript (index) directly
-    const subOrDimNames = lhsSubRefs.map(subRef => subRef.subName)
-    return expandDims(lhsVarDef.varName, subOrDimNames)
-  } else {
-    // No subscripts, so include a single variable name
-    return [lhsVarDef.varName]
-  }
-}
-
-/**
- * Return an array of all expanded subscript combinations.
- *
- * @param {string} baseVarName The base name of the variable.
- * @param {string[]} subOrDimNames The array of subscript or dimension names.
- * @returns {string[]} An array of string representations of subscripted references,
- * e.g., `'x[A1,B1]' ,'x[A1,B2]', ...]`.
- */
-function expandDims(baseVarName, subOrDimNames) {
-  // Expand the dimension for each position
-  const expanded = subOrDimNames.map(name => expandDim(name).flat(Infinity))
-
-  // Expand these into the set of all combinations of subscripts for the variable
-  const origCombos = cartesianProductOf(expanded)
-  return origCombos.map(combo => {
-    const subs = combo.join(',')
-    return `${baseVarName}[${subs}]`
-  })
-}
-
-/**
- * Return an array containing all subscript (index) names in the given dimension.  If
- * the given name is a subscript, it will return a single-element array with that
- * subscript name.
- *
- * @param {string} subOrDimName A subscript or dimension name.
- * @returns {string[]} A (possibly nested) array of subscript names.
- */
-function expandDim(subOrDimName) {
-  // Convert the name to an ID
-  const subOrDimId = canonicalName(subOrDimName)
-
-  if (isDimension(subOrDimId)) {
-    // Get the object for the dimension
-    const dimObj = sub(subOrDimId)
-
-    // The dimension may contain a mix of individual subscripts (indices) and/or subdimensions,
-    // so recursively expand them
-    return dimObj.modelValue.map(expandDim)
-  } else {
-    // This is an individual subscript (index), so return it directly
-    return [subOrDimName]
-  }
-}

package/src/model/expand-var-instances.js

@@ -0,0 +1,139 @@
+import { cartesianProductOf, canonicalName } from '../_shared/helpers.js'
+import { isDimension, sub } from '../_shared/subscript.js'
+
+/**
+ * A single instance of a variable.
+ * @typedef {Object} VarInstance
+ * @property {string} varName The full name of the variable instance, e.g., "Variable Name[SubA, SubB]".
+ * @property {number[]} [subscriptIndices] The array of subscript indices; only defined if this variable
+ * has subscripts.
+ */
+
+/**
+ * Return an array of names and subscript indices for the given variable, expanded to
+ * include all subscript variants.
+ *
+ * @param {*} v A `Variable` object.
+ * @returns {VarInstance[]} An array of `VarInstance` objects corresponding to the expanded variable.
+ */
+export function expandVar(v) {
+  if (v.parsedEqn === undefined) {
+    // XXX: The special `Time` variable does not have a `parsedEqn`, so use the raw LHS
+    return [
+      {
+        varName: v.modelLHS
+      }
+    ]
+  }
+
+  // Expand each subscript position to get the names of all subscripted variants
+  const lhsVarDef = v.parsedEqn.lhs.varDef
+  const lhsSubOrDimIds = v.subscripts
+  if (lhsSubOrDimIds === undefined || lhsSubOrDimIds.length === 0) {
+    // No subscripts, so include a single variable name
+    return [
+      {
+        varName: lhsVarDef.varName
+      }
+    ]
+  }
+
+  // At each position, expand any dimensions or use a subscript (index) directly
+  return expandDims(lhsVarDef.varName, lhsSubOrDimIds)
+}
+
+/**
+ * Return an array of all expanded subscript combinations.
+ *
+ * @param {string} baseVarName The base name of the variable.
+ * @param {string[]} subOrDimIds The array of subscript or dimension IDs.
+ * @returns {VarInstance[]} An array of `VarInstance` objects with string representations of
+ * subscripted references, e.g., `'x[A1,B1]', 'x[A1,B2]', ...`.
+ */
+function expandDims(baseVarName, subOrDimIds) {
+  // Expand the dimension for each position
+  const expanded = subOrDimIds.map(id => expandSubSpecs(id).flat(Infinity))
+
+  // Expand these into the set of all combinations of subscripts for the variable
+  const origCombos = cartesianProductOf(expanded)
+  return origCombos.map(combo => {
+    const subNames = combo.map(spec => spec.name).join(',')
+    const subIndices = combo.map(spec => spec.index)
+    return {
+      varName: `${baseVarName}[${subNames}]`,
+      subscriptIndices: subIndices
+    }
+  })
+}
+
+/**
+ * Pairs a subscript name with its index.
+ * @typedef {Object} SubSpec
+ * @property {string} name The name of the subscript, e.g., "A1".
+ * @property {number} index The subscript index relative to its parent dimension.
+ */
+
+/**
+ * Return an array containing all subscript specs in the given dimension.  If
+ * the given ID is a subscript, it will return a single-element array with that
+ * subscript name and index.
+ *
+ * @param {string} subOrDimId A subscript or dimension ID (e.g., '_a1', '_dima').
+ * @returns {SubSpec[]} A (possibly nested) array of `SubSpec` objects.
+ */
+function expandSubSpecs(subOrDimId) {
+  if (isDimension(subOrDimId)) {
+    // Get the object for the dimension
+    const dimObj = sub(subOrDimId)
+
+    // The dimension may contain a mix of individual subscripts (indices) and/or subdimensions,
+    // so recursively expand them
+    return dimObj.value.map(expandSubSpecs)
+  } else {
+    // This is an individual subscript (index), so return its name and index value
+    // XXX: Currently, the object returned by `sub` will not include the `modelName`
+    // for subscripts (it's only defined for dimensions?), so if we don't have the
+    // `modelName`, find it using the parent dimension object.  This could be avoided
+    // if subscript objects maintained their original model name.
+    const subObj = sub(subOrDimId)
+    const dimSubNames = expandSubNamesForDim(subObj.family).flat(Infinity)
+    const subName = dimSubNames[subObj.value]
+    if (subName === undefined) {
+      throw new Error(`Failed to resolve name of subscript ${subOrDimId} in dimension ${subObj.family}`)
+    }
+    return [
+      {
+        name: subName,
+        index: subObj.value
+      }
+    ]
+  }
+}
+
+/**
+ * Return an array containing all subscript (index) names in the given dimension,
+ * expanding subdimensions as needed.  If the given ID is a subscript, it will return
+ * single-element array with that subscript name.
+ *
+ * @param {string} dimId A dimension ID (e.g., '_dima').
+ * @returns {string[]} A (possibly nested) array of subscript names (e.g., 'A1').
+ */
+function expandSubNamesForDim(dimId) {
+  if (!isDimension(dimId)) {
+    throw new Error('expandSubNames should only be called with a dimension ID')
+  }
+
+  // Get the object for the dimension
+  const dimObj = sub(dimId)
+
+  // The dimension may contain a mix of individual subscripts (indices) and/or
+  // subdimensions, so recursively expand them
+  return dimObj.modelValue.map(subOrDimName => {
+    const subOrDimId = canonicalName(subOrDimName)
+    if (isDimension(subOrDimId)) {
+      return expandSubNamesForDim(subOrDimId)
+    } else {
+      return [subOrDimName]
+    }
+  })
+}

package/src/model/model.js

@@ -2,7 +2,7 @@ import B from 'bufx'
 import yaml from 'js-yaml'
 import * as R from 'ramda'
 
-import { canonicalName, decanonicalize, isIterable, /*listConcat,*/ strlist, vlog, vsort } from '../_shared/helpers.js'
+import { canonicalVensimName, decanonicalize, isIterable, strlist, vlog, vsort } from '../_shared/helpers.js'
 import {
   addIndex,
   allAliases,
@@ -13,7 +13,9 @@ import {
   sub,
   subscriptFamilies
 } from '../_shared/subscript.js'
+import { cName } from '../_shared/var-names.js'
 
+import { expandVar } from './expand-var-instances.js'
 import { readEquation } from './read-equations.js'
 import { readDimensionDefs } from './read-subscripts.js'
 import { readVariables } from './read-variables.js'
@@ -60,12 +62,12 @@ function resetModelState() {
  * TODO: FIX TYPE
  * @param {*} parsedModel The parsed model structure.
  * @param {*} spec The parsed `spec.json` object.
- * @param {Map<string, any>} extData The map of datasets from external `.dat` files.
- * @param {Map<string, any>} directData The mapping of dataset name used in a `GET DIRECT DATA`
+ * @param {Map<string, any>} [extData] The map of datasets from external `.dat` files.
+ * @param {Map<string, any>} [directData] The mapping of dataset name used in a `GET DIRECT DATA`
  * call (e.g., `?data`) to the tabular data contained in the loaded data file.
- * @param {string} modelDirname The path to the directory containing the model (used for resolving data
+ * @param {string} [modelDirname] The path to the directory containing the model (used for resolving data
  * files for `GET DIRECT SUBSCRIPT`).
- * @param {*} opts An optional object used by tests to stop the read process after a specific phase.
+ * @param {*} [opts] An optional object used by tests to stop the read process after a specific phase.
  */
 function read(parsedModel, spec, extData, directData, modelDirname, opts) {
   // Some arrays need to be separated into variables with individual indices to
@@ -425,7 +427,13 @@ function removeUnusedVariables(spec) {
   }
 
   // Filter out unneeded variables so we're left with the minimal set of variables to emit
-  variables = R.filter(v => referencedVarNames.has(v.varName), variables)
+  const filteredVariables = R.filter(v => referencedVarNames.has(v.varName), variables)
+  // TODO: Note that we reuse the same `variables` array instance here instead of reassigning
+  // to it because some code (like in `code-gen/expand-var-names.js` and in some tests) uses
+  // the `variables` array (from module-level storage) directly.  We need to fix those uses
+  // to use accessors to avoid these subtle issues.
+  variables.length = 0
+  variables.push(...filteredVariables)
 
   // Rebuild the variables-by-name map
   variablesByName.clear()
@@ -732,28 +740,6 @@ function vensimName(cVarName) {
   }
   return result
 }
-function cName(vensimVarName) {
-  // Convert a Vensim variable name to a C name.
-  // This function requires model analysis to be completed first when the variable has subscripts.
-
-  // Split the variable name from the subscripts
-  let matches = vensimVarName.match(/([^[]+)(?:\[([^\]]+)\])?/)
-  if (!matches) {
-    throw new Error(`Invalid variable name '${vensimVarName}' found when converting to C representation`)
-  }
-  let cVarName = canonicalName(matches[1])
-  if (matches[2]) {
-    // The variable name includes subscripts, so split them into individual IDs
-    let cSubIds = matches[2].split(',').map(x => canonicalName(x))
-    // If a subscript is an index, convert it to an index number to match Vensim data exports
-    let cSubIdParts = cSubIds.map(cSubId => {
-      return isIndex(cSubId) ? `[${sub(cSubId).value}]` : `[${cSubId}]`
-    })
-    // Append the subscript parts to the base variable name to create the full reference
-    cVarName += cSubIdParts.join('')
-  }
-  return cVarName
-}
 function isInputVar(varName) {
   // Return true if the given variable (in canonical form) is included in the list of
   // input variables in the spec file.
@@ -1105,12 +1091,13 @@ function allListedVars() {
   }
 
   // The order of execution/evaluation in the generated model is:
-  //   initConstants (vars of type `const` only)
-  //   initLookups (vars of type `lookup` only)
-  //   initData (vars of type `data` only)
-  //   initLevels (vars returned by `initVars`, a mix of initial, aux, and level vars)
-  //   evalAux (vars of type `aux` only)
-  //   evalLevels (vars of type `level` only)
+  //   initConstants (vars of type `const` only; called for t=0 only)
+  //   initLookups (vars of type `lookup` only; called for t=0 only)
+  //   initData (vars of type `data` only; called for t=0 only)
+  //   initLevels (vars returned by `initVars`, a mix of initial, aux, and level vars;
+  //     called for t=0 only)
+  //   evalAux (vars of type `aux` only; called for t>=0)
+  //   evalLevels (vars of type `level` only; called before `evalAux` for t>0)
   // So to make the ordering in the listing better match the order of evaluation,
   // we emit variables in the above order, but filter to avoid having duplicates.
   addUnique(constVars())
@@ -1151,7 +1138,7 @@ function varIndexInfoMap() {
 
   // Get the set of unique variable names, and assign a 1-based index to each.
   // This matches the index number used in `storeOutput` and `setLookup` in the
-  // generated C/JS code
+  // generated C/JS code.
   const infoMap = new Map()
   let varIndex = 1
   for (const v of sortedVars) {
@@ -1209,6 +1196,73 @@ function jsonList() {
     }
   }
 
+  //
+  // We include a `varInstances` object in the generated JSON listing that
+  // includes an array of expanded variable items (one item for every "instance"
+  // of a variable, including subscripted variables) in the same order that they
+  // are evaluated (assigned) in the generated model.
+  //
+  // Each object contains the following minimal set of fields that are needed
+  // for accessing the data for a single variable instance at runtime:
+  //   varId (e.g., '_variable_name')
+  //   varName (e.g., 'Variable Name')
+  //   varType ('const', 'data', 'lookup', 'initial', 'level', 'aux')
+  //   varIndex
+  //   subscriptIndices
+  //
+  // The order of execution/evaluation in the generated model is:
+  //   initConstants (vars of type `const` only, called for t=0 only)
+  //   initLookups (vars of type `lookup` only, called for t=0 only)
+  //   initData (vars of type `data` only, called for t=0 only)
+  //   initLevels (vars returned by `initVars`, a mix of initial, aux, and level vars,
+  //     called for t=0 only)
+  //   evalAux (vars of type `aux` only; called for t>=0)
+  //   evalLevels (vars of type `level` only; called before `evalAux` for t>0)
+  //
+  function expandedVarItems(vars) {
+    const expandedVars = []
+
+    for (const v of vars) {
+      // Filter out variables that are generated/used internally
+      if (v.includeInOutput === false) {
+        continue
+      }
+
+      const varInstances = expandVar(v)
+      for (const { varName, subscriptIndices } of varInstances) {
+        const varId = canonicalVensimName(varName)
+        const varItem = {
+          varId,
+          varName,
+          varType: v.varType
+        }
+        const varInfo = infoMap.get(v.varName)
+        if (varInfo) {
+          varItem.varIndex = varInfo.varIndex
+          if (subscriptIndices?.length > 0) {
+            varItem.subscriptIndices = subscriptIndices
+          }
+        }
+        expandedVars.push(varItem)
+      }
+    }
+
+    return expandedVars
+  }
+  const expandedConstants = expandedVarItems(constVars())
+  const expandedLookupVars = expandedVarItems(lookupVars())
+  const expandedDataVars = expandedVarItems(dataVars())
+  // The special exogenous `Time` variable may have already been removed by
+  // `removeUnusedVariables` if it is not referenced explicitly in the model,
+  // so we will only include it in the listing if it is defined here.  Note
+  // that `_time` is set to `_initial_time` as the first step in the
+  // `initLevels` function, which is why it is included in the "init" group.
+  const timeVar = varWithName('_time')
+  const specialInitVars = timeVar ? [timeVar] : []
+  const expandedInitVars = expandedVarItems([...specialInitVars, ...initVars()])
+  const expandedLevelVars = expandedVarItems(levelVars())
+  const expandedAuxVars = expandedVarItems(auxVars())
+
   // Derive minimal versions of the full arrays; these only contain the minimal
   // subset of fields that are needed by the `ModelListing` class from the
   // runtime package.  The property names in the minimal objects are slightly
@@ -1245,7 +1299,15 @@ function jsonList() {
   cachedJsonList = {
     full: {
       dimensions: sortedFullDims,
-      variables: sortedFullVars
+      variables: sortedFullVars,
+      varInstances: {
+        constants: expandedConstants,
+        lookupVars: expandedLookupVars,
+        dataVars: expandedDataVars,
+        initVars: expandedInitVars,
+        levelVars: expandedLevelVars,
+        auxVars: expandedAuxVars
+      }
     },
     minimal: {
       dimensions: sortedMinimalDims,
@@ -1261,7 +1323,6 @@ export default {
   addVariable,
   allVars,
   auxVars,
-  cName,
   constVars,
   dataVars,
   expansionFlags,

package/src/model/toposort.js

@@ -39,7 +39,7 @@ function toposort(nodes, edges) {
       var nodeRep
       try {
         nodeRep = '\n' + node + '\n'
-      } catch (e) {
+      } catch (_) {
         nodeRep = ''
       }
       throw new Error('Found cyclic dependency during toposort:\n' + [...predecessors].join(' →\n') + ' →' + nodeRep)

package/src/parse-and-generate.js

@@ -8,6 +8,7 @@ import { parseVensimModel } from '@sdeverywhere/parse'
 import { readXlsx } from './_shared/helpers.js'
 import { readDat } from './_shared/read-dat.js'
 import { printSubscripts, yamlSubsList } from './_shared/subscript.js'
+import { cName } from './_shared/var-names.js'
 import Model from './model/model.js'
 import { getDirectSubscripts } from './model/read-subscripts.js'
 import { generateCode } from './generate/gen-code.js'
@@ -120,7 +121,7 @@ export function printNames(namesPathname, operation) {
   for (let line of lines) {
     if (line.length > 0) {
       if (operation === 'to-c') {
-        B.emitLine(Model.cName(line))
+        B.emitLine(cName(line))
       } else {
         B.emitLine(Model.vensimName(line))
       }