@sdeverywhere/compile 0.7.10 → 0.7.11

package/package.json

@@ -1,10 +1,11 @@
 {
   "name": "@sdeverywhere/compile",
-  "version": "0.7.10",
+  "version": "0.7.11",
   "description": "The core Vensim to C compiler for the SDEverywhere tool suite.",
   "type": "module",
   "main": "./src/index.js",
   "dependencies": {
+    "@sdeverywhere/parse": "^0.1.0",
     "antlr4": "4.12.0",
     "antlr4-vensim": "0.6.2",
     "bufx": "^1.0.5",

package/src/generate/code-gen.js

@@ -4,15 +4,16 @@ import { asort, lines, strlist, abend, mapIndexed } from '../_shared/helpers.js'
 import { sub, allDimensions, allMappings, subscriptFamilies } from '../_shared/subscript.js'
 import Model from '../model/model.js'
 
+import { generateEquation } from './gen-equation.js'
 import EquationGen from './equation-gen.js'
-import ModelLHSReader from './model-lhs-reader.js'
+import { expandVarNames } from './expand-var-names.js'
 
-export function generateCode(parseTree, opts) {
-  return codeGenerator(parseTree, opts).generate()
+export function generateCode(parsedModel, opts) {
+  return codeGenerator(parsedModel, opts).generate()
 }
 
-let codeGenerator = (parseTree, opts) => {
-  const { spec, operation, extData, directData, modelDirname } = opts
+let codeGenerator = (parsedModel, opts) => {
+  const { spec, operations, extData, directData, modelDirname } = opts
   // Set to 'decl', 'init-lookups', 'eval', etc depending on the section being generated.
   let mode = ''
   // Set to true to output all variables when there is no model run spec.
@@ -25,21 +26,30 @@ let codeGenerator = (parseTree, opts) => {
     outputAllVars = true
   }
   // Function to generate a section of the code
-  let generateSection = R.map(v => new EquationGen(v, extData, directData, mode, modelDirname).generate())
+  let generateSection = R.map(v => {
+    if (parsedModel.kind === 'vensim-legacy') {
+      return new EquationGen(v, extData, directData, mode, modelDirname).generate()
+    } else {
+      return generateEquation(v, mode, extData, directData, modelDirname)
+    }
+  })
   let section = R.pipe(generateSection, R.flatten, lines)
   function generate() {
     // Read variables and subscript ranges from the model parse tree.
     // This is the main entry point for code generation and is called just once.
     try {
-      Model.read(parseTree, spec, extData, directData, modelDirname)
+      Model.read(parsedModel, spec, extData, directData, modelDirname)
       // In list mode, print variables to the console instead of generating code.
-      if (operation === 'printRefIdTest') {
+      if (operations.includes('printRefIdTest')) {
         Model.printRefIdTest()
-      } else if (operation === 'printRefGraph') {
+      }
+      if (operations.includes('printRefGraph')) {
         Model.printRefGraph(opts.varname)
-      } else if (operation === 'convertNames') {
+      }
+      if (operations.includes('convertNames')) {
         // Do not generate output, but leave the results of model analysis.
-      } else if (operation === 'generateC') {
+      }
+      if (operations.includes('generateC')) {
         // Generate code for each variable in the proper order.
         let code = emitDeclCode()
         code += emitInitLookupsCode()
@@ -198,10 +208,21 @@ void ${name}${idx}() {
     }
     let funcCalls = R.pipe(mapIndexed(funcCall), lines)
 
-    // Break the vars into chunks of 30; this number was empirically
-    // determined by looking at runtime performance and memory usage
-    // of the En-ROADS model on various devices
-    let chunks = R.splitEvery(30, vars)
+    // Break the vars into chunks.  The default value of 30 was empirically
+    // determined by looking at runtime performance and memory usage of the
+    // En-ROADS model on various devices.
+    let chunkSize
+    if (process.env.SDE_CODE_GEN_CHUNK_SIZE) {
+      chunkSize = parseInt(process.env.SDE_CODE_GEN_CHUNK_SIZE)
+    } else {
+      chunkSize = 30
+    }
+    let chunks
+    if (chunkSize > 0) {
+      chunks = R.splitEvery(chunkSize, vars)
+    } else {
+      chunks = [vars]
+    }
 
     if (!preStep) {
       preStep = ''
@@ -296,33 +317,8 @@ ${postStep}
     // Return a list of var names for all variables except lookups and data variables.
     // The names are in Vensim format if vensimNames is true, otherwise they are in C format.
     // Expand subscripted vars into separate var names with each index.
-    function sortedVars() {
-      // Return a list of all vars sorted by the model LHS var name (without subscripts), case insensitive.
-      return R.sortBy(v => {
-        let modelLHSReader = new ModelLHSReader()
-        modelLHSReader.read(v.modelLHS)
-        return modelLHSReader.varName.toUpperCase()
-      }, Model.variables)
-    }
-    return R.uniq(
-      R.reduce(
-        (a, v) => {
-          if (v.varType !== 'lookup' && v.varType !== 'data' && v.includeInOutput) {
-            let modelLHSReader = new ModelLHSReader()
-            modelLHSReader.read(v.modelLHS)
-            if (vensimNames) {
-              return R.concat(a, modelLHSReader.names())
-            } else {
-              return R.concat(a, R.map(Model.cName, modelLHSReader.names()))
-            }
-          } else {
-            return a
-          }
-        },
-        [],
-        sortedVars()
-      )
-    )
+    const canonicalNames = !vensimNames
+    return expandVarNames(canonicalNames)
   }
   //
   // Input/output section helpers

package/src/generate/direct-data-helpers.js

@@ -0,0 +1,86 @@
+import path from 'node:path'
+
+import XLSX from 'xlsx'
+
+import { cdbl, readCsv, readXlsx } from '../_shared/helpers.js'
+
+/**
+ * Return a `getCellValue` function that reads the CSV or XLS[X] content.
+ *
+ * @param {string} fileOrTag The filename (e.g., 'data.xlsx') or tag name (e.g., '?data').
+ * @param {string} tabOrDelimiter
+ * @param {'data' | 'constants'} dataKind The kind of `GET DIRECT ...` being used.
+ * @param {Map<string, any>} directData The mapping of dataset name used in a `GET DIRECT DATA` call (e.g.,
+ * `?data`) to the tabular data contained in the loaded data file.
+ * @param {string} modelDir The path to the directory containing the model (used for resolving data files).
+ * @returns A `getCellValue` function.
+ */
+export function handleExcelOrCsvFile(fileOrTag, tabOrDelimiter, dataKind, directData, modelDir) {
+  if (fileOrTag.startsWith('?')) {
+    // The file is a tag for an Excel file with data in the directData map.
+    const workbook = directData.get(fileOrTag)
+    return handleExcelWorkbook(fileOrTag, workbook, tabOrDelimiter, dataKind, 'tagged')
+  } else {
+    // The file is a CSV or XLS[X] pathname. Read it now.
+    const dataPathname = path.resolve(modelDir, fileOrTag)
+    if (dataPathname.toLowerCase().endsWith('csv')) {
+      return handleCsvFile(fileOrTag, dataPathname, tabOrDelimiter, dataKind)
+    } else {
+      const workbook = readXlsx(dataPathname)
+      return handleExcelWorkbook(fileOrTag, workbook, tabOrDelimiter, dataKind, 'file')
+    }
+  }
+}
+
+/**
+ * Return a `getCellValue` function for the given Excel workbook parsed from an XLS[X] file.
+ *
+ * @param {string} fileOrTag The filename (e.g., 'data.xlsx') or tag name (e.g., '?data').
+ * @param {*} workbook The workbook data loaded from the file.
+ * @param {string} tab The name of the tab within the workbook.
+ * @param {'data' | 'constants'} dataKind The kind of `GET DIRECT ...` being used.
+ * @param {'file' | 'tagged'} dataSource The reference kind, either 'file' or 'tagged'.
+ * @returns A `getCellValue` function.
+ */
+function handleExcelWorkbook(fileOrTag, workbook, tab, dataKind, dataSource) {
+  if (workbook) {
+    let sheet = workbook.Sheets[tab]
+    if (sheet) {
+      return (c, r) => {
+        let cell = sheet[XLSX.utils.encode_cell({ c, r })]
+        return cell != null ? cdbl(cell.v) : null
+      }
+    } else {
+      throw new Error(`Direct ${dataKind} worksheet ${tab} in ${dataSource} ${fileOrTag} not found`)
+    }
+  } else {
+    throw new Error(`Direct ${dataKind} workbook ${dataSource} ${fileOrTag} not found`)
+  }
+}
+
+/**
+ * Return a `getCellValue` function for the given CSV file.
+ *
+ * @param {string} file The filename of the data file.
+ * @param {string} dataFilename The full path to the data file.
+ * @param {string} delimiter The delimiter for the tabular data.
+ * @param {'data' | 'constants'} dataKind The kind of `GET DIRECT ...` being used.
+ * @returns A `getCellValue` function.
+ */
+function handleCsvFile(file, dataPathname, delimiter, dataKind) {
+  // Return a `getCellValue` function for the given CSV file.
+  let data = readCsv(dataPathname, delimiter)
+  if (data) {
+    return (c, r) => {
+      let value = '0.0'
+      try {
+        value = data[r] != null && data[r][c] != null ? cdbl(data[r][c]) : null
+      } catch (error) {
+        console.error(`${error.message} in ${dataPathname}`)
+      }
+      return value
+    }
+  } else {
+    throw new Error(`Direct ${dataKind} file ${file} could not be read`)
+  }
+}

package/src/generate/equation-gen.js

@@ -673,7 +673,7 @@ export default class EquationGen extends ModelReader {
           throw new Error(`ERROR: lookup size = ${this.var.points.length} in ${this.var.refId}`)
         }
         let lookupData = R.reduce((a, p) => listConcat(a, `${cdbl(p[0])}, ${cdbl(p[1])}`, true), '', this.var.points)
-        this.emit(`__new_lookup(${this.var.points.length}, /*copy=*/true, (double[]){ ${lookupData} });`)
+        this.emit(`__new_lookup(${this.var.points.length}, /*copy=*/true, (double[]){ ${lookupData} })`)
       }
     } else {
       super.visitEquation(ctx)

package/src/generate/expand-var-names.js

@@ -0,0 +1,115 @@
+import * as R from 'ramda'
+
+import { cartesianProductOf, canonicalName } from '../_shared/helpers.js'
+import { sub, isDimension } from '../_shared/subscript.js'
+
+import Model from '../model/model.js'
+
+import ModelLHSReader from './model-lhs-reader.js'
+
+/**
+ * Return an array of names for all variable in the model, sorted alphabetically and expanded to
+ * include the full set of subscripted variants for variables that include subscripts.
+ *
+ * @param canonical If true, convert names to canonical representation (variable identifiers), otherwise
+ * return the original name of each variable as it appears in the model.
+ * @returns {string[]} An array of variable names or identifiers.
+ */
+export function expandVarNames(canonical) {
+  const sortedVars = R.sortBy(v => v.varName, Model.variables)
+  return R.uniq(
+    R.reduce(
+      (a, v) => {
+        if (v.varType !== 'lookup' && v.varType !== 'data' && v.includeInOutput) {
+          if (canonical) {
+            return R.concat(a, R.map(Model.cName, namesForVar(v)))
+          } else {
+            return R.concat(a, namesForVar(v))
+          }
+        } else {
+          return a
+        }
+      },
+      [],
+      sortedVars
+    )
+  )
+}
+
+/**
+ * Return an array of names for the given variable including all subscript variants.
+ *
+ * @param {*} v A `Variable` instance.
+ * @returns {string[]} An array of expanded names for the given variable.
+ */
+function namesForVar(v) {
+  if (process.env.SDE_NONPUBLIC_USE_NEW_PARSE !== '1') {
+    // TODO: When the old parsing code is active, use the old ModelLHSReader.  This code path
+    // will be removed when the old parsing code is removed.
+    let modelLHSReader = new ModelLHSReader()
+    modelLHSReader.read(v.modelLHS)
+    return modelLHSReader.names()
+  }
+
+  if (v.parsedEqn === undefined) {
+    // XXX: The special `Time` variable does not have a `parsedEqn`, so use the raw LHS
+    return [v.modelLHS]
+  }
+
+  // Expand each variable to get the names of all subscripted variants
+  const lhsVarDef = v.parsedEqn.lhs.varDef
+  const lhsSubRefs = lhsVarDef.subscriptRefs
+  if (lhsSubRefs?.length > 0) {
+    // At each position, expand any dimensions or use a subscript (index) directly
+    const subOrDimNames = lhsSubRefs.map(subRef => subRef.subName)
+    return expandDims(lhsVarDef.varName, subOrDimNames)
+  } else {
+    // No subscripts, so include a single variable name
+    return [lhsVarDef.varName]
+  }
+}
+
+/**
+ * Return an array of all expanded subscript combinations.
+ *
+ * @param {string} baseVarName The base name of the variable.
+ * @param {string[]} subOrDimNames The array of subscript or dimension names.
+ * @returns {string[]} An array of string representations of subscripted references,
+ * e.g., `'x[A1,B1]' ,'x[A1,B2]', ...]`.
+ */
+function expandDims(baseVarName, subOrDimNames) {
+  // Expand the dimension for each position
+  const expanded = subOrDimNames.map(name => expandDim(name).flat(Infinity))
+
+  // Expand these into the set of all combinations of subscripts for the variable
+  const origCombos = cartesianProductOf(expanded)
+  return origCombos.map(combo => {
+    const subs = combo.join(',')
+    return `${baseVarName}[${subs}]`
+  })
+}
+
+/**
+ * Return an array containing all subscript (index) names in the given dimension.  If
+ * the given name is a subscript, it will return a single-element array with that
+ * subscript name.
+ *
+ * @param {string} subOrDimName A subscript or dimension name.
+ * @returns {string[]} A (possibly nested) array of subscript names.
+ */
+function expandDim(subOrDimName) {
+  // Convert the name to an ID
+  const subOrDimId = canonicalName(subOrDimName)
+
+  if (isDimension(subOrDimId)) {
+    // Get the object for the dimension
+    const dimObj = sub(subOrDimId)
+
+    // The dimension may contain a mix of individual subscripts (indices) and/or subdimensions,
+    // so recursively expand them
+    return dimObj.modelValue.map(expandDim)
+  } else {
+    // This is an individual subscript (index), so return it directly
+    return [subOrDimName]
+  }
+}

package/src/generate/gen-const-list.js

@@ -0,0 +1,89 @@
+import { cartesianProductOf, cdbl } from '../_shared/helpers.js'
+import { isDimension, normalizeSubscripts, sub } from '../_shared/subscript.js'
+
+/**
+ * Generate code for a single element in a const list definition.
+ *
+ * @param {*} variable The `Variable` instance to process.
+ * @param {*} parsedEqn The parsed equation.
+ * @return {string[]} An array of strings containing the generated C code for the variable,
+ * one string per line of code.
+ */
+export function generateConstListElement(variable, parsedEqn) {
+  // In the "read variables" phase, const lists are expanded into separated variable
+  // definitions, so `variable` here will have `subscripts` that represent specific
+  // subscript indices in normalized order (alphabetized by parent dimension/family
+  // name).  However, we need to consult the LHS subscripts/dimensions, which will
+  // be in the original order from the model equation.
+  //
+  // In the following example,
+  // we have a 2D variable whose original dimensions are not in normal order:
+  //   DimA: A1, A2 ~~|
+  //   DimB: B1, B2, B3 ~~|
+  //   x[DimB, DimA] = 1, 2; 3, 4; 5, 6; ~~|
+  //
+  // The variable `x` will have been separated into:
+  //   x[B1,A1]
+  //   x[B1,A2]
+  //   x[B2,A1]
+  //   ...
+  //
+  // Each one will refer to a single element from the original const list.  To determine
+  // which element in the const list goes with which variable instance, we build an array
+  // of all subscript combinations and then find the index of the one that matches the
+  // combination used for the separated variable instance.
+  const lhsSubRefs = variable.parsedEqn.lhs.varDef.subscriptRefs
+  const lhsSubIds = lhsSubRefs.map(subRef => subRef.subId)
+  const subIdArrays = lhsSubIds.map(subOrDimId => {
+    if (isDimension(subOrDimId)) {
+      // Use the full array of subscripts (indexes) for the dimension at this position
+      return sub(subOrDimId).value
+    } else {
+      // This is a single subscript (index), so use an array with a single element
+      return [subOrDimId]
+    }
+  })
+
+  // Continuing with the above example, at this point we will have a 2D array:
+  //   [
+  //     [_b1,_b2,_b3],
+  //     [_a1,_a2]
+  //   ]
+  // We expand these into the set of all combinations of subscripts in the original
+  // order of the dimensions from the equation LHS.
+  const origCombos = cartesianProductOf(subIdArrays)
+
+  // Now we have the combinations in original order:
+  //   [_b1,_a1]
+  //   [_b1,_a2]
+  //   [_b2,_a1]
+  //   ...
+  // But we need to put them into normalized order so that we can find the index of
+  // `variable.subscripts` (which is already in normalized order).
+  const normalizedCombos = origCombos.map(normalizeSubscripts)
+
+  // Convert to strings to make matching easier.  Now we have the strings in normalized order:
+  //   [_a1,_b1]
+  //   [_a2,_b1]
+  //   [_a1,_b2]
+  //   ...
+  const comboStrings = normalizedCombos.map(combo => combo.map(subId => `[${subId}]`).join(''))
+
+  // Convert `variable.subscripts` into the same format so that we can do an array lookup,
+  // for example if this separated variable instance is x[_a2,_b1], this will be:
+  //   [_a2,_b1]
+  const lhsComboString = variable.subscripts.map(subId => `[${subId}]`).join('')
+
+  // Find the index of the combination that matches `variable.subscripts`
+  const constIndex = comboStrings.indexOf(lhsComboString)
+  if (constIndex < 0) {
+    throw new Error(`Failed to determine index of const list element for ${variable.refId}`)
+  }
+
+  // Determine the LHS and RHS of the const assignment
+  const lhsVarId = variable.varName
+  const lhsIndicesString = variable.subscripts.map(subId => `[${sub(subId).value}]`).join('')
+  const lhsRef = `${lhsVarId}${lhsIndicesString}`
+  const rhsConstValue = cdbl(parsedEqn.rhs.constants[constIndex].value)
+  return `  ${lhsRef} = ${rhsConstValue};`
+}

package/src/generate/gen-direct-const.js

@@ -0,0 +1,91 @@
+import XLSX from 'xlsx'
+
+import { cartesianProductOf } from '../_shared/helpers.js'
+import { indexInSepDim, isDimension, sub } from '../_shared/subscript.js'
+
+import { handleExcelOrCsvFile } from './direct-data-helpers.js'
+
+/**
+ * Generate code for a variable that uses `GET DIRECT CONSTANTS` to source constant values from an external
+ * file (in CSV or Excel format).
+ *
+ * @param {*} variable The `Variable` instance to process.
+ * @param {Map<string, any>} directData The mapping of dataset name used in a `GET DIRECT CONSTANTS` call (e.g.,
+ * `?data`) to the tabular data contained in the loaded data file.
+ * @param {string} modelDir The path to the directory containing the model (used for resolving data files).
+ * @return {string[]} An array of strings containing the generated C code for the variable,
+ * one string per line of code.
+ */
+export function generateDirectConstInit(variable, directData, modelDir) {
+  // Create a function that reads the CSV or XLS[X] content
+  let { file, tab, startCell } = variable.directConstArgs
+  let getCellValue = handleExcelOrCsvFile(file, tab, 'constants', directData, modelDir)
+
+  // Get C subscripts in text form for the LHS in normal order.
+  let lhsSubIds = variable.parsedEqn.lhs.varDef.subscriptRefs?.map(s => s.subId) || []
+  let modelDimNames = lhsSubIds.filter(s => isDimension(s))
+
+  // Generate offsets from the start cell in the table corresponding to LHS indices.
+  let cellOffsets = []
+  let cSubscripts = variable.subscripts.map(s => (isDimension(s) ? sub(s).value : [s]))
+  let lhsIndexSubscripts = cartesianProductOf(cSubscripts)
+
+  // Find the table cell offset for each LHS index tuple.
+  let lines = []
+  for (let indexSubscripts of lhsIndexSubscripts) {
+    let entry = [null, null]
+    for (let i = 0; i < variable.subscripts.length; i++) {
+      // LHS dimensions or indices in a separated dimension map to table cells.
+      let lhsSubscript = variable.subscripts[i]
+      if (isDimension(lhsSubscript) || indexInSepDim(lhsSubscript, variable)) {
+        // Consider the LHS index subscript at this position.
+        let indexSubscript = indexSubscripts[i]
+        let ind = sub(indexSubscript)
+        // Find the model subscript position corresponding to the LHS index subscript.
+        for (let iModelDim = 0; iModelDim < modelDimNames.length; iModelDim++) {
+          // Only fill an entry position once.
+          if (entry[iModelDim] === null) {
+            let modelDim = sub(modelDimNames[iModelDim])
+            if (modelDim.family === ind.family) {
+              // Set the numeric index for the model dimension in the cell offset entry.
+              // Use the position within the dimension to map subdimensions onto cell offsets.
+              let pos = modelDim.value.indexOf(indexSubscript)
+              // Vectors use a 2D cell offset that maps to columns in the first row.
+              // Tables use a 2D cell offset with the row or column matching the model dimension.
+              let entryRowOrCol = modelDimNames.length > 1 ? iModelDim : 1
+              entry[entryRowOrCol] = pos
+              break
+            }
+          }
+        }
+      }
+    }
+    // Replace unfilled entry positions with zero.
+    entry = entry.map(x => (x === null ? 0 : x))
+    // Read values by column first when the start cell ends with an asterisk.
+    // Ref: https://www.vensim.com/documentation/fn_get_direct_constants.html
+    if (startCell.endsWith('*')) {
+      entry.reverse()
+    }
+    cellOffsets.push(entry)
+  }
+
+  // Read tabular data into an indexed variable for each cell.
+  let numericSubscripts = lhsIndexSubscripts.map(idx => idx.map(s => sub(s).value))
+  let lhsSubscripts = numericSubscripts.map(s => s.reduce((a, v) => a.concat(`[${v}]`), ''))
+  let dataAddress = XLSX.utils.decode_cell(startCell.toUpperCase())
+  let startCol = dataAddress.c
+  let startRow = dataAddress.r
+  if (startCol < 0 || startRow < 0) {
+    throw new Error(`Failed to parse 'cell' argument for GET DIRECT CONSTANTS call for ${variable.refId}: ${startCell}`)
+  }
+  for (let i = 0; i < cellOffsets.length; i++) {
+    let rowOffset = cellOffsets[i][0] ? cellOffsets[i][0] : 0
+    let colOffset = cellOffsets[i][1] ? cellOffsets[i][1] : 0
+    let dataValue = getCellValue(startCol + colOffset, startRow + rowOffset)
+    let lhs = `${variable.varName}${lhsSubscripts[i] || ''}`
+    lines.push(`  ${lhs} = ${dataValue};`)
+  }
+
+  return lines
+}