@sdeverywhere/compile 0.7.9 → 0.7.11

package/src/generate/gen-const-list.js

@@ -0,0 +1,89 @@
+import { cartesianProductOf, cdbl } from '../_shared/helpers.js'
+import { isDimension, normalizeSubscripts, sub } from '../_shared/subscript.js'
+
+/**
+ * Generate code for a single element in a const list definition.
+ *
+ * @param {*} variable The `Variable` instance to process.
+ * @param {*} parsedEqn The parsed equation.
+ * @return {string[]} An array of strings containing the generated C code for the variable,
+ * one string per line of code.
+ */
+export function generateConstListElement(variable, parsedEqn) {
+  // In the "read variables" phase, const lists are expanded into separated variable
+  // definitions, so `variable` here will have `subscripts` that represent specific
+  // subscript indices in normalized order (alphabetized by parent dimension/family
+  // name).  However, we need to consult the LHS subscripts/dimensions, which will
+  // be in the original order from the model equation.
+  //
+  // In the following example,
+  // we have a 2D variable whose original dimensions are not in normal order:
+  //   DimA: A1, A2 ~~|
+  //   DimB: B1, B2, B3 ~~|
+  //   x[DimB, DimA] = 1, 2; 3, 4; 5, 6; ~~|
+  //
+  // The variable `x` will have been separated into:
+  //   x[B1,A1]
+  //   x[B1,A2]
+  //   x[B2,A1]
+  //   ...
+  //
+  // Each one will refer to a single element from the original const list.  To determine
+  // which element in the const list goes with which variable instance, we build an array
+  // of all subscript combinations and then find the index of the one that matches the
+  // combination used for the separated variable instance.
+  const lhsSubRefs = variable.parsedEqn.lhs.varDef.subscriptRefs
+  const lhsSubIds = lhsSubRefs.map(subRef => subRef.subId)
+  const subIdArrays = lhsSubIds.map(subOrDimId => {
+    if (isDimension(subOrDimId)) {
+      // Use the full array of subscripts (indexes) for the dimension at this position
+      return sub(subOrDimId).value
+    } else {
+      // This is a single subscript (index), so use an array with a single element
+      return [subOrDimId]
+    }
+  })
+
+  // Continuing with the above example, at this point we will have a 2D array:
+  //   [
+  //     [_b1,_b2,_b3],
+  //     [_a1,_a2]
+  //   ]
+  // We expand these into the set of all combinations of subscripts in the original
+  // order of the dimensions from the equation LHS.
+  const origCombos = cartesianProductOf(subIdArrays)
+
+  // Now we have the combinations in original order:
+  //   [_b1,_a1]
+  //   [_b1,_a2]
+  //   [_b2,_a1]
+  //   ...
+  // But we need to put them into normalized order so that we can find the index of
+  // `variable.subscripts` (which is already in normalized order).
+  const normalizedCombos = origCombos.map(normalizeSubscripts)
+
+  // Convert to strings to make matching easier.  Now we have the strings in normalized order:
+  //   [_a1,_b1]
+  //   [_a2,_b1]
+  //   [_a1,_b2]
+  //   ...
+  const comboStrings = normalizedCombos.map(combo => combo.map(subId => `[${subId}]`).join(''))
+
+  // Convert `variable.subscripts` into the same format so that we can do an array lookup,
+  // for example if this separated variable instance is x[_a2,_b1], this will be:
+  //   [_a2,_b1]
+  const lhsComboString = variable.subscripts.map(subId => `[${subId}]`).join('')
+
+  // Find the index of the combination that matches `variable.subscripts`
+  const constIndex = comboStrings.indexOf(lhsComboString)
+  if (constIndex < 0) {
+    throw new Error(`Failed to determine index of const list element for ${variable.refId}`)
+  }
+
+  // Determine the LHS and RHS of the const assignment
+  const lhsVarId = variable.varName
+  const lhsIndicesString = variable.subscripts.map(subId => `[${sub(subId).value}]`).join('')
+  const lhsRef = `${lhsVarId}${lhsIndicesString}`
+  const rhsConstValue = cdbl(parsedEqn.rhs.constants[constIndex].value)
+  return `  ${lhsRef} = ${rhsConstValue};`
+}

package/src/generate/gen-direct-const.js

@@ -0,0 +1,91 @@
+import XLSX from 'xlsx'
+
+import { cartesianProductOf } from '../_shared/helpers.js'
+import { indexInSepDim, isDimension, sub } from '../_shared/subscript.js'
+
+import { handleExcelOrCsvFile } from './direct-data-helpers.js'
+
+/**
+ * Generate code for a variable that uses `GET DIRECT CONSTANTS` to source constant values from an external
+ * file (in CSV or Excel format).
+ *
+ * @param {*} variable The `Variable` instance to process.
+ * @param {Map<string, any>} directData The mapping of dataset name used in a `GET DIRECT CONSTANTS` call (e.g.,
+ * `?data`) to the tabular data contained in the loaded data file.
+ * @param {string} modelDir The path to the directory containing the model (used for resolving data files).
+ * @return {string[]} An array of strings containing the generated C code for the variable,
+ * one string per line of code.
+ */
+export function generateDirectConstInit(variable, directData, modelDir) {
+  // Create a function that reads the CSV or XLS[X] content
+  let { file, tab, startCell } = variable.directConstArgs
+  let getCellValue = handleExcelOrCsvFile(file, tab, 'constants', directData, modelDir)
+
+  // Get C subscripts in text form for the LHS in normal order.
+  let lhsSubIds = variable.parsedEqn.lhs.varDef.subscriptRefs?.map(s => s.subId) || []
+  let modelDimNames = lhsSubIds.filter(s => isDimension(s))
+
+  // Generate offsets from the start cell in the table corresponding to LHS indices.
+  let cellOffsets = []
+  let cSubscripts = variable.subscripts.map(s => (isDimension(s) ? sub(s).value : [s]))
+  let lhsIndexSubscripts = cartesianProductOf(cSubscripts)
+
+  // Find the table cell offset for each LHS index tuple.
+  let lines = []
+  for (let indexSubscripts of lhsIndexSubscripts) {
+    let entry = [null, null]
+    for (let i = 0; i < variable.subscripts.length; i++) {
+      // LHS dimensions or indices in a separated dimension map to table cells.
+      let lhsSubscript = variable.subscripts[i]
+      if (isDimension(lhsSubscript) || indexInSepDim(lhsSubscript, variable)) {
+        // Consider the LHS index subscript at this position.
+        let indexSubscript = indexSubscripts[i]
+        let ind = sub(indexSubscript)
+        // Find the model subscript position corresponding to the LHS index subscript.
+        for (let iModelDim = 0; iModelDim < modelDimNames.length; iModelDim++) {
+          // Only fill an entry position once.
+          if (entry[iModelDim] === null) {
+            let modelDim = sub(modelDimNames[iModelDim])
+            if (modelDim.family === ind.family) {
+              // Set the numeric index for the model dimension in the cell offset entry.
+              // Use the position within the dimension to map subdimensions onto cell offsets.
+              let pos = modelDim.value.indexOf(indexSubscript)
+              // Vectors use a 2D cell offset that maps to columns in the first row.
+              // Tables use a 2D cell offset with the row or column matching the model dimension.
+              let entryRowOrCol = modelDimNames.length > 1 ? iModelDim : 1
+              entry[entryRowOrCol] = pos
+              break
+            }
+          }
+        }
+      }
+    }
+    // Replace unfilled entry positions with zero.
+    entry = entry.map(x => (x === null ? 0 : x))
+    // Read values by column first when the start cell ends with an asterisk.
+    // Ref: https://www.vensim.com/documentation/fn_get_direct_constants.html
+    if (startCell.endsWith('*')) {
+      entry.reverse()
+    }
+    cellOffsets.push(entry)
+  }
+
+  // Read tabular data into an indexed variable for each cell.
+  let numericSubscripts = lhsIndexSubscripts.map(idx => idx.map(s => sub(s).value))
+  let lhsSubscripts = numericSubscripts.map(s => s.reduce((a, v) => a.concat(`[${v}]`), ''))
+  let dataAddress = XLSX.utils.decode_cell(startCell.toUpperCase())
+  let startCol = dataAddress.c
+  let startRow = dataAddress.r
+  if (startCol < 0 || startRow < 0) {
+    throw new Error(`Failed to parse 'cell' argument for GET DIRECT CONSTANTS call for ${variable.refId}: ${startCell}`)
+  }
+  for (let i = 0; i < cellOffsets.length; i++) {
+    let rowOffset = cellOffsets[i][0] ? cellOffsets[i][0] : 0
+    let colOffset = cellOffsets[i][1] ? cellOffsets[i][1] : 0
+    let dataValue = getCellValue(startCol + colOffset, startRow + rowOffset)
+    let lhs = `${variable.varName}${lhsSubscripts[i] || ''}`
+    lines.push(`  ${lhs} = ${dataValue};`)
+  }
+
+  return lines
+}

package/src/generate/gen-equation.js

@@ -0,0 +1,273 @@
+import {
+  dimensionNames,
+  hasMapping,
+  isDimension,
+  isIndex,
+  isTrivialDimension,
+  normalizeSubscripts,
+  separatedVariableIndex,
+  sub
+} from '../_shared/subscript.js'
+import { generateConstListElement } from './gen-const-list.js'
+
+import { generateDirectConstInit } from './gen-direct-const.js'
+import { generateExpr } from './gen-expr.js'
+import { generateLookupsFromDirectData } from './gen-lookup-from-direct.js'
+import { generateLookupsFromExternalData } from './gen-lookup-from-external.js'
+import { generateLookupFromPoints } from './gen-lookup-from-points.js'
+
+import LoopIndexVars from './loop-index-vars.js'
+
+/**
+ * Generate C code for the given model equation.
+ *
+ * TODO: Variable type
+ * TODO: Define type for mode values
+ *
+ * @param {*} variable The `Variable` instance to process.
+ * @param {'decl' | 'init-constants' | 'init-lookups' | 'init-levels' | 'eval'} mode The code generation mode.
+ * @param {Map<string, any>} extData The map of datasets from external `.dat` files.
+ * @param {Map<string, any>} directData The mapping of dataset name used in a `GET DIRECT DATA` call (e.g.,
+ * `?data`) to the tabular data contained in the loaded data file.
+ * @param {string} modelDir The path to the directory containing the model (used for resolving data files).
+ * @return {string[]} An array of strings containing the generated C code for the variable,
+ * one string per line of code.
+ */
+export function generateEquation(variable, mode, extData, directData, modelDir) {
+  // Maps of LHS subscript families to loop index vars for lookup on the RHS
+  const loopIndexVars = new LoopIndexVars(['i', 'j', 'k', 'l', 'm'])
+  const arrayIndexVars = new LoopIndexVars(['u', 'v', 'w', 's', 't', 'f', 'g', 'h', 'o', 'p', 'q', 'r'])
+
+  // Generate the LHS variable reference code
+  const parsedEqn = variable.parsedEqn
+  const cLhs = cVarRefWithLhsSubscripts(variable, parsedEqn.lhs.varDef.varId, loopIndexVars)
+
+  // Include the original model equation in a comment that comes before the generated code
+  // for that equation
+  const modelFormula = variable.origModelFormula || variable.modelFormula
+  const comment = `  // ${variable.modelLHS} = ${modelFormula.replace(/\n/g, '')}`
+
+  // Apply special handling for const lists
+  if (parsedEqn.rhs.kind === 'const-list') {
+    if (mode !== 'init-constants' && mode !== 'eval') {
+      throw new Error(`Invalid code gen mode '${mode}' for const list variable ${variable.modelLHS}`)
+    }
+    // XXX: The legacy code gen emitted a comment before each init statement, so we will
+    // do the same for now to maintain compatibility
+    return [comment, generateConstListElement(variable, parsedEqn)]
+  }
+
+  // Emit direct constants individually without separating them first
+  if (variable.directConstArgs) {
+    const initCode = generateDirectConstInit(variable, directData, modelDir)
+    return [comment, ...initCode]
+  }
+
+  // Get the dimension IDs for the LHS variable
+  const dimIds = dimensionNames(variable.subscripts)
+
+  // Turn each dimension ID into a loop with a loop index variable.
+  // If the variable has no subscripts, nothing will be emitted here.
+  const openLoops = []
+  const closeLoops = []
+  for (const dimId of dimIds) {
+    const indexName = loopIndexVars.index(dimId)
+    const dimLength = sub(dimId).size
+    openLoops.push(`  for (size_t ${indexName} = 0; ${indexName} < ${dimLength}; ${indexName}++) {`)
+    closeLoops.push('  }')
+  }
+
+  // Apply special handling for data variables.  The data can be defined in one of three ways:
+  //   - as a set of explicit data points (stored in the `Variable` instance), or
+  //   - from an external file via a `GET DIRECT DATA` call, or
+  //   - from an external data file (i.e., a "normal" data variable)
+  if (variable.isData()) {
+    if (variable.points.length > 0) {
+      // The variable already has data points defined, so generate a new lookup using that data.
+      // Note that unlike the other lookup cases, this one needs to include loop open/close code
+      // if the variable is subscripted.
+      const lookupDef = generateLookupFromPoints(variable, mode, /*copy=*/ true, cLhs, loopIndexVars)
+      if (lookupDef.length > 0) {
+        return [comment, ...openLoops, ...lookupDef, ...closeLoops]
+      } else {
+        return []
+      }
+    } else if (variable.directDataArgs) {
+      // The data is referenced using a `GET DIRECT DATA` call; generate one or more lookups
+      // using the data defined in external files
+      return generateLookupsFromDirectData(variable, mode, directData, modelDir, cLhs)
+    } else {
+      // This is a "normal" data variable; generate one or more lookups using the data defined
+      // in external files
+      return generateLookupsFromExternalData(variable, mode, extData, cLhs)
+    }
+  }
+
+  // Apply special handling for lookup variables.  The data for lookup variables is already
+  // defined as a set of explicit data points (stored in the `Variable` instance).
+  if (variable.isLookup()) {
+    return generateLookupFromPoints(variable, mode, /*copy=*/ false, cLhs, loopIndexVars)
+  }
+
+  // Keep a buffer of code that will be included before the innermost loop
+  const preInnerLoopLines = []
+
+  // Keep a buffer of code that will be included before the generated primary formula
+  const preFormulaLines = []
+
+  // Keep a buffer of code that will be included after the generated primary formula
+  const postFormulaLines = []
+
+  // Keep track of marked dimensions
+  const markedDimIds = new Set()
+
+  // Generate code for an equation with an expression on the RHS
+  const genExprCtx = {
+    variable,
+    mode,
+    cLhs,
+    loopIndexVars,
+    arrayIndexVars,
+    resetMarkedDims: () => markedDimIds.clear(),
+    addMarkedDim: dimId => markedDimIds.add(dimId),
+    emitPreInnerLoop: s => preInnerLoopLines.push(s),
+    emitPreFormula: s => preFormulaLines.push(s),
+    emitPostFormula: s => postFormulaLines.push(s),
+    cVarRef: varRef => cVarRef(variable, varRef, markedDimIds, loopIndexVars, arrayIndexVars),
+    cVarRefWithLhsSubscripts: baseVarId => cVarRefWithLhsSubscripts(variable, baseVarId, loopIndexVars),
+    cVarIndex: subOrDimId => cVarIndex(variable, [subOrDimId], subOrDimId, markedDimIds, loopIndexVars, arrayIndexVars)
+  }
+  const cRhs = generateExpr(parsedEqn.rhs.expr, genExprCtx)
+  const formula = `  ${cLhs} = ${cRhs};`
+
+  // Insert the pre-inner loop code, if needed
+  if (preInnerLoopLines.length > 0) {
+    openLoops.splice(variable.subscripts.length - 1, 0, ...preInnerLoopLines)
+  }
+
+  // Combine all lines of comments and code into a single array
+  return [comment, ...openLoops, ...preFormulaLines, formula, ...postFormulaLines, ...closeLoops]
+}
+
+/**
+ * Return the C code for a subscripted reference to the given variable using the LHS subscripts
+ * of the given equation.
+ *
+ * @param {*} lhsVariable The LHS `Variable` instance.
+ * @param {*} baseVarId The base variable ID to which the subscript parts will be appended.
+ * @param {LoopIndexVars} loopIndexVars The loop index state.
+ * @return {string} The C variable reference.
+ */
+function cVarRefWithLhsSubscripts(lhsVariable, baseVarId, loopIndexVars) {
+  const lhsSubIds = lhsVariable.subscripts
+  const cSubParts = lhsSubIds.map(subId => {
+    if (isDimension(subId)) {
+      const i = loopIndexVars.index(subId)
+      if (isTrivialDimension(subId)) {
+        // When the dimension is trivial, we can simply emit e.g. `[i]` instead of `[_dim[i]]`
+        return `[${i}]`
+      } else {
+        return `[${subId}][${i}]`
+      }
+    } else {
+      return `[${sub(subId).value}]`
+    }
+  })
+  return `${baseVarId}${cSubParts.join('')}`
+}
+
+/**
+ * Return the C code for a RHS (possibly subscripted) variable reference.
+ *
+ * @param {*} lhsVariable The LHS `Variable` instance.
+ * @param {*} rhsVarRef The `VariableRef` used in a RHS expression.
+ * @param {Set<string>} markedDimIds The set of dimension IDs that are marked for use
+ * in an array function, for example `SUM(x[DimA!])`.
+ * @param {LoopIndexVars} loopIndexVars The loop index state.
+ * @param {LoopIndexVars} arrayIndexVars The loop index state used for array functions
+ * (that use marked dimensions).
+ * @returns {string} The C variable reference.
+ */
+function cVarRef(lhsVariable, rhsVarRef, markedDimIds, loopIndexVars, arrayIndexVars) {
+  if (rhsVarRef.subscriptRefs === undefined) {
+    // No subscripts, so return the base variable ID
+    return rhsVarRef.varId
+  }
+
+  // Normalize the RHS subscripts
+  let rhsSubIds
+  try {
+    // XXX: For now, strip the mark here (need to revisit this)
+    rhsSubIds = normalizeSubscripts(rhsVarRef.subscriptRefs.map(subRef => subRef.subId.replace('!', '')))
+  } catch (e) {
+    throw new Error(`normalizeSubscripts failed in rhsVarRef: refId=${lhsVariable.refId} error=${e}`)
+  }
+
+  // Determine the subscript code (array lookup) for each dimension.  For example, if
+  // the RHS variable reference in the model looks like `x[DimA]`, this will convert the
+  // `[DimA]` part to `[_dima[i]]` (or simply `[i]` if it is a "trivial" dimension).
+  const cSubParts = rhsSubIds.map(rhsSubId => {
+    return cVarIndex(lhsVariable, rhsSubIds, rhsSubId, markedDimIds, loopIndexVars, arrayIndexVars)
+  })
+
+  return `${rhsVarRef.varId}${cSubParts.map(part => `[${part}]`).join('')}`
+}
+
+/**
+ * Return the C code for indexing into a subscripted variable.
+ *
+ * @param {*} lhsVariable The LHS `Variable` instance.
+ * @param {string[]} rhsSubIds The set of all subscript or dimension IDs used on the RHS.
+ * @param {string} rhsSubId The specific subscript or dimension ID being evaluated.
+ * @param {Set<string>} markedDimIds The set of dimension IDs that are marked for use
+ * in an array function, for example `SUM(x[DimA!])`.
+ * @param {LoopIndexVars} loopIndexVars The loop index state.
+ * @param {LoopIndexVars} arrayIndexVars The loop index state used for array functions
+ * (that use marked dimensions).
+ * @returns {string} The C variable reference.
+ */
+function cVarIndex(lhsVariable, rhsSubIds, rhsSubId, markedDimIds, loopIndexVars, arrayIndexVars) {
+  if (isIndex(rhsSubId)) {
+    // This is a specific subscript (i.e., an index); dereference the array using the index
+    // number of the subscript
+    return `${sub(rhsSubId).value}`
+  }
+
+  // Otherwise, this is a dimension. Get the corresponding loop index variable used
+  // in the "for" loop.
+  let indexName
+  if (markedDimIds.has(rhsSubId)) {
+    // This is a marked dimension as used in an array function (e.g., `SUM`), so use
+    // the name of the array loop index variable
+    indexName = arrayIndexVars.index(rhsSubId)
+  } else {
+    // Use the single index name for a separated variable if it exists
+    const separatedIndexName = separatedVariableIndex(rhsSubId, lhsVariable, rhsSubIds)
+    if (separatedIndexName) {
+      return `${sub(separatedIndexName).value}`
+    }
+
+    // See if we need to apply a mapping because the RHS dim is not found on the LHS
+    const found = lhsVariable.subscripts.findIndex(lhsSubId => sub(lhsSubId).family === sub(rhsSubId).family)
+    if (found < 0) {
+      // Find the mapping from the RHS subscript to a LHS subscript
+      for (const lhsSubId of lhsVariable.subscripts) {
+        if (hasMapping(rhsSubId, lhsSubId)) {
+          indexName = loopIndexVars.index(lhsSubId)
+          return `__map${rhsSubId}${lhsSubId}[${indexName}]`
+        }
+      }
+    }
+
+    // There is no mapping, so use the loop index for this dim family on the LHS
+    indexName = loopIndexVars.index(rhsSubId)
+  }
+
+  // Dereference the array using the corresponding loop index variable
+  if (isTrivialDimension(rhsSubId)) {
+    // When the dimension is trivial, we can emit e.g. `[i]` instead of `[_dim[i]]`
+    return `${indexName}`
+  } else {
+    return `${rhsSubId}[${indexName}]`
+  }
+}