@sdeverywhere/compile 0.7.28 → 0.7.29

package/package.json

@@ -1,11 +1,11 @@
 {
   "name": "@sdeverywhere/compile",
-  "version": "0.7.28",
+  "version": "0.7.29",
   "description": "The core Vensim to C compiler for the SDEverywhere tool suite.",
   "type": "module",
   "main": "./src/index.js",
   "dependencies": {
-    "@sdeverywhere/parse": "^0.1.3",
+    "@sdeverywhere/parse": "^0.1.4",
     "byline": "^5.0.0",
     "csv-parse": "^5.3.3",
     "ramda": "^0.27.0",

package/src/_shared/helpers.js

@@ -127,10 +127,10 @@ export let listConcat = (a, x, addSpaces = false) => {
 }
 // Convert a number or string into a C double constant string.
 // A blank string is converted to zero, following Excel.
-// A string that cannot be converted throws an exception.
+// A string that cannot be converted throws an error.
 export let cdbl = x => {
   function throwError() {
-    throw new Error(`ERROR: cannot convert "${x}" to a number`)
+    throw new Error(`Cannot convert "${x}" to a number`)
   }
   let s = '0.0'
   if (typeof x === 'number') {

package/src/generate/direct-data-helpers.js

@@ -48,7 +48,16 @@ function handleExcelWorkbook(fileOrTag, workbook, tab, dataKind, dataSource) {
     if (sheet) {
       return (c, r) => {
         let cell = sheet[XLSX.utils.encode_cell({ c, r })]
-        return cell != null ? cdbl(cell.v) : null
+        if (cell == null || cell.v === '') {
+          return null
+        }
+        try {
+          return cdbl(cell.v)
+        } catch (_error) {
+          // Return null when the cell value cannot be converted to a number;
+          // the caller will treat this as the end of data.
+          return null
+        }
       }
     } else {
       throw new Error(`Direct ${dataKind} worksheet ${tab} in ${dataSource} ${fileOrTag} not found`)
@@ -72,13 +81,16 @@ function handleCsvFile(file, dataPathname, delimiter, dataKind) {
   let data = readCsv(dataPathname, delimiter)
   if (data) {
     return (c, r) => {
-      let value = '0.0'
+      if (data[r] == null || data[r][c] == null || data[r][c] === '') {
+        return null
+      }
       try {
-        value = data[r] != null && data[r][c] != null ? cdbl(data[r][c]) : null
-      } catch (error) {
-        console.error(`${error.message} in ${dataPathname}`)
+        return cdbl(data[r][c])
+      } catch (_error) {
+        // Return null when the cell value cannot be converted to a number;
+        // the caller will treat this as the end of data.
+        return null
       }
-      return value
     }
   } else {
     throw new Error(`Direct ${dataKind} file ${file} could not be read`)

package/src/generate/gen-code.js

@@ -17,8 +17,9 @@ import { generateJS } from './gen-code-js.js'
  * @param {Map<string, any>} opts.directData The mapping of dataset name used in a
  * `GET DIRECT DATA` call (e.g., `?data`) to the tabular data contained in the loaded
  * data file.
- * @param {string} opts.modelDirname The path to the directory containing the model
- * (used for resolving data files for `GET DIRECT SUBSCRIPT`).
+ * @param {string} opts.modelDirname The absolute path to the directory containing data
+ * (dat, xlsx, csv) files that are referenced by the model.  This path is used for
+ * resolving data files for `GET DIRECT SUBSCRIPT` calls.
  * @returns A string containing the generated code.
  */
 export function generateCode(parsedModel, opts) {

package/src/generate/gen-direct-const.js

@@ -82,7 +82,12 @@ export function generateDirectConstInit(variable, directData, modelDir) {
   for (let i = 0; i < cellOffsets.length; i++) {
     let rowOffset = cellOffsets[i][0] ? cellOffsets[i][0] : 0
     let colOffset = cellOffsets[i][1] ? cellOffsets[i][1] : 0
+    // Use 0.0 as a fallback when the cell is missing, empty, or contains a non-numeric value.
+    // (Vensim raises an error in this case, but SDE has historically tolerated invalid cells.)
     let dataValue = getCellValue(startCol + colOffset, startRow + rowOffset)
+    if (dataValue == null) {
+      dataValue = '0.0'
+    }
     let lhs = `${variable.varName}${lhsSubscripts[i] || ''}`
     lines.push(`  ${lhs} = ${dataValue};`)
   }

package/src/generate/gen-equation.js

@@ -116,6 +116,12 @@ export function generateEquation(variable, mode, extData, directData, modelDir,
     // Emit decl/init code for the lookup
     const lookupDef = generateLookupFromPoints(variable, mode, /*copy=*/ false, cLhs, loopIndexVars, outFormat)
     if (lookupDef.length > 0) {
+      if (mode === 'decl') {
+        // When declaring a lookup, even if the lookup variable includes dimensions (i.e., is
+        // partially apply-to-all), the data variable declarations should not be inside for loops,
+        // so we omit them in this case
+        return [...lookupDef]
+      }
       return [...openLoops, ...lookupDef, ...closeLoops]
     } else {
       return []

package/src/generate/gen-expr.js

@@ -400,8 +400,11 @@ function generateFunctionCall(callExpr, ctx) {
     }
 
     case '_GET_DIRECT_CONSTANTS':
+    case '_GET_XLS_CONSTANTS':
     case '_GET_DIRECT_DATA':
+    case '_GET_XLS_DATA':
     case '_GET_DIRECT_LOOKUPS':
+    case '_GET_XLS_LOOKUPS':
       // These functions are handled at a higher level, so we should not get here
       throw new Error(`Unexpected function '${fnId}' in code gen for '${ctx.variable.modelLHS}'`)
 
@@ -970,6 +973,7 @@ function visitVariableRefs(expr, onVarRef) {
       break
 
     case 'lookup-call':
+      visitVariableRefs(expr.varRef, onVarRef)
       visitVariableRefs(expr.arg, onVarRef)
       break
 

package/src/generate/gen-lookup-from-direct.js

@@ -89,14 +89,21 @@ function generateDirectDataLookup(varLhs, getCellValue, timeRowOrCol, startCell,
     }
   }
 
-  let timeValue = getCellValue(timeCol, timeRow)
-  let dataValue = getCellValue(dataCol, dataRow)
-  while (timeValue != null && dataValue != null) {
-    lookupData = listConcat(lookupData, `${timeValue}, ${dataValue}`, true)
-    lookupSize++
+  // Read time/value pairs, matching Vensim's behavior:
+  //   - Stop reading when the first non-numeric time value is encountered.  This
+  //     allows additional content (e.g., labels) to follow the data in the row or column.
+  //   - Skip pairs with a non-numeric data value, but continue reading subsequent pairs.
+  while (true) {
+    const timeValue = getCellValue(timeCol, timeRow)
+    if (timeValue == null) {
+      break
+    }
+    const dataValue = getCellValue(dataCol, dataRow)
+    if (dataValue != null) {
+      lookupData = listConcat(lookupData, `${timeValue}, ${dataValue}`, true)
+      lookupSize++
+    }
     nextCell()
-    dataValue = getCellValue(dataCol, dataRow)
-    timeValue = getCellValue(timeCol, timeRow)
   }
   if (lookupSize === 0) {
     throw new Error(`Empty lookup data array for ${varLhs}`)

package/src/model/read-equations-expand.js

@@ -0,0 +1,72 @@
+import { indexNamesForSubscript } from '../_shared/subscript.js'
+
+/**
+ * Given the array of LHS subscript/dimension IDs (already mapped to correspond to the
+ * RHS positions) and a set of RHS variable instances, return the refIds of the RHS
+ * instances whose subscript combinations overlap with the LHS combinations at every
+ * position.
+ *
+ * Conceptually, this is equivalent to checking whether any combination in the LHS
+ * cartesian product matches any combination in a given RHS instance's cartesian
+ * product.  But because positions in a cartesian product are independent, we only
+ * need to check that each position has at least one index in common between the
+ * LHS and RHS index sets.  This reduces the complexity of the check from
+ * O(product of dimension sizes) to O(sum of dimension sizes).
+ *
+ * For example, suppose DimA={A1,A2} and DimB={B1,B2}, the LHS accesses `[DimA,DimB]`,
+ * and we want to check whether it overlaps with a RHS variable instance `_x[_dima,_b1]`.
+ * The full cartesian products look like this:
+ *   LHS combos: { (A1,B1), (A1,B2), (A2,B1), (A2,B2) }
+ *   RHS combos: { (A1,B1), (A2,B1) }
+ * The two sets share (A1,B1) and (A2,B1), so there is a match.  But we don't need
+ * to enumerate either set — we can check each position independently:
+ *   position 0:  LHS {A1,A2} ∩ RHS {A1,A2} = {A1,A2}  (non-empty)
+ *   position 1:  LHS {B1,B2} ∩ RHS {B1}    = {B1}     (non-empty)
+ * Every position has at least one index in common, so we know a full-combo match
+ * must exist (pick any shared index at each position, e.g., (A2,B1), and it is in
+ * both products).  Conversely, if any position has an empty intersection, no full
+ * combo can match — for example, if instead the LHS accessed `[A1,DimB]` (a specific
+ * index at position 0) and the RHS instance were `_x[_a2,_dimb]`, position 0 would
+ * give LHS {A1} ∩ RHS {A2} = ∅ and we could stop immediately.
+ *
+ * @param {string[]} mappedLhsSubIds The array of LHS subscript/dimension IDs at each
+ * position, mapped to correspond to the RHS variable reference positions.
+ * @param {{ subscripts: string[], refId: string }[]} rhsVarInstances The array of RHS
+ * variable instances to filter.
+ * @returns {string[]} A sorted array of refIds for the RHS instances whose subscripts
+ * overlap with the LHS at every position.
+ */
+export function matchingRhsRefIds(mappedLhsSubIds, rhsVarInstances) {
+  // Build a Set of LHS index names for each position for quick lookup
+  const lhsIndexSets = mappedLhsSubIds.map(id => new Set(indexNamesForSubscript(id)))
+
+  // For each RHS variable instance, check if there is overlap at every subscript
+  // position between the LHS and RHS index sets
+  const rhsRefIds = []
+  for (const rhsVarInstance of rhsVarInstances) {
+    let matches = true
+    for (let i = 0; i < rhsVarInstance.subscripts.length; i++) {
+      const rhsIndices = indexNamesForSubscript(rhsVarInstance.subscripts[i])
+      let hasOverlap = false
+      for (const id of rhsIndices) {
+        if (lhsIndexSets[i].has(id)) {
+          hasOverlap = true
+          break
+        }
+      }
+      if (!hasOverlap) {
+        matches = false
+        break
+      }
+    }
+    if (matches) {
+      rhsRefIds.push(rhsVarInstance.refId)
+    }
+  }
+
+  // Return the sorted array of relevant refIds
+  // TODO: Sorting is not essential here, but the legacy reader sorted so we will keep
+  // that behavior now to avoid invalidating tests.  Later we should remove this `sort`
+  // call and update the tests accordingly.
+  return rhsRefIds.sort()
+}

package/src/model/read-equations.js

@@ -1,8 +1,8 @@
 import { parseVensimModel } from '@sdeverywhere/parse'
 
-import { canonicalName, cartesianProductOf, newDepreciationVarName, newFixedDelayVarName } from '../_shared/helpers.js'
+import { canonicalName, newDepreciationVarName, newFixedDelayVarName } from '../_shared/helpers.js'
 
-import { hasMapping, indexNamesForSubscript, isDimension, isIndex, sub } from '../_shared/subscript.js'
+import { hasMapping, isDimension, isIndex, sub } from '../_shared/subscript.js'
 
 import Model from './model.js'
 import { generateDelayVariables } from './read-equation-fn-delay.js'
@@ -11,6 +11,7 @@ import { generateNpvVariables } from './read-equation-fn-npv.js'
 import { generateSmoothVariables } from './read-equation-fn-smooth.js'
 import { generateTrendVariables } from './read-equation-fn-trend.js'
 import { generateLookup } from './read-equation-fn-with-lookup.js'
+import { matchingRhsRefIds } from './read-equations-expand.js'
 import { readVariables } from './read-variables.js'
 
 class Context {
@@ -522,7 +523,8 @@ function visitFunctionCall(v, callExpr, context) {
         generateGameVariables(v, callExpr, context)
         break
 
-      case '_GET_DIRECT_CONSTANTS': {
+      case '_GET_DIRECT_CONSTANTS':
+      case '_GET_XLS_CONSTANTS':
         validateCallDepth(callExpr, context)
         validateCallArgs(callExpr, 3)
         validateCallArgType(callExpr, 0, 'string')
@@ -536,10 +538,11 @@ function visitFunctionCall(v, callExpr, context) {
           startCell: callExpr.args[2].text
         }
         break
-      }
 
       case '_GET_DIRECT_DATA':
+      case '_GET_XLS_DATA':
       case '_GET_DIRECT_LOOKUPS':
+      case '_GET_XLS_LOOKUPS':
         validateCallDepth(callExpr, context)
         validateCallArgs(callExpr, 4)
         validateCallArgType(callExpr, 0, 'string')
@@ -967,21 +970,26 @@ function expandedRefIdsForVar(lhsVariable, rhsBaseRefId, rhsSubIds) {
   // it must be non-apply-to-all.  The goal now is to determine which instances (refIds) are
   // relevant for the given `lhsVariable` context.
   //
-  // First, get all combinations of the LHS subscripts that map to the subscripts/dimensions
-  // in the RHS variable reference.  For example:
+  // First, determine the set of LHS subscript indices accessed at each position of the RHS
+  // variable reference.  For example:
   //   y[DimA,DimB,DimC] :EXCEPT: [DimA,DimB,C1] = x[DimA,DimC,DimB]
   // In this case the `DimC` on the RHS is only "accessed" by `C2` from the LHS, so we would
-  // build an array of strings representing the possible subset of combinations, like this:
-  //   _a1,_c2,_b1
-  //   _a1,_c2,_b2
-  //   _a2,_c2,_b1
-  //   _a2,_c2,_b2
+  // build a per-position set of accessed indices, like this:
+  //   position 0 (DimA on RHS): { _a1, _a2 }
+  //   position 1 (DimC on RHS): { _c2 }
+  //   position 2 (DimB on RHS): { _b1, _b2 }
+  //
+  // Then, for each RHS variable instance, check whether every subscript position has at
+  // least one index in common between the LHS set and the indices that the RHS instance
+  // accepts at that position.  If so, add the RHS `refId` to the array of variables
+  // referenced by the LHS.
   //
-  // Then, for each RHS variable instance:
-  //   - get all combinations of RHS subscripts that can be accepted by that RHS instance
-  //     (build an array of strings, e.g., ['_a1,_c1,_b1', '_a1,_c1,_b1', ...])
-  //   - see if any of the LHS subscript combos match any of the RHS subscript combos; if
-  //     so, then add the RHS `refId` to the array of variables referenced by the LHS
+  // Conceptually this is equivalent to checking whether any combination in the LHS
+  // cartesian product matches any combination in the RHS cartesian product, but we can
+  // avoid computing the cartesian products explicitly because positions in a cartesian
+  // product are independent: if every position has at least one element in common, then
+  // there exists a full combination that matches.  This reduces the complexity from
+  // O(product of dimension sizes) to O(sum of dimension sizes).
   //
   // In the following examples, suppose the referenced RHS variable is non-apply-to-all and
   // has two instances:
@@ -1012,43 +1020,18 @@ function expandedRefIdsForVar(lhsVariable, rhsBaseRefId, rhsSubIds) {
   //   _x[_dima,_c2,_dimb]
   //
 
-  // Step 1: Get all combinations of the LHS subscripts that map to the subscripts/dimensions
-  // in the RHS variable reference.  Here `rhsSubIds` is the array of parsed subscript/dimension
-  // IDs that appear in the RHS variable reference.  We figure out which LHS subscripts/dimensions
-  // are relevant for the RHS subscripts/dimensions given the context of the LHS variable (which
-  // may have been separated/expanded).
+  // Step 1: Resolve the LHS subscript/dimension at each position of the RHS variable
+  // reference.  Here `rhsSubIds` is the array of parsed subscript/dimension IDs that
+  // appear in the RHS variable reference.  We figure out which LHS subscripts/dimensions
+  // are relevant for the RHS subscripts/dimensions given the context of the LHS variable
+  // (which may have been separated/expanded).
   const lhsSubRefs = lhsVariable.parsedEqn.lhs.varDef.subscriptRefs
   const lhsSubIds = lhsSubRefs?.map(subRef => subRef.subId) || []
   const mappedLhsSubIds = rhsSubIds.map(rhsSubId => resolveRhsSubOrDim(lhsVariable, lhsSubIds, rhsSubId))
 
-  // Step 2: Build an array of mapped LHS subscript combos (one string of comma-separated
-  // subscript IDs for each combo)
-  const mappedLhsSubIdsPerPosition = mappedLhsSubIds.map(indexNamesForSubscript)
-  const mappedLhsCombos = cartesianProductOf(mappedLhsSubIdsPerPosition).map(combo => combo.join(','))
-
-  // Step 3: For each RHS variable instance, get all combinations of RHS subscripts that can
-  // be accepted by that particular RHS instance
-  const rhsRefIds = []
-  for (const rhsVarInstance of rhsVarInstances) {
-    // Build RHS subscript combos (one string of comma-separated subscript IDs for each combo)
-    const rhsVarInstanceSubIdsPerPosition = rhsVarInstance.subscripts.map(indexNamesForSubscript)
-    const rhsCombos = cartesianProductOf(rhsVarInstanceSubIdsPerPosition).map(combo => combo.join(','))
-
-    // See if any of the LHS subscript combos match any of the RHS subscript combos
-    for (const lhsCombo of mappedLhsCombos) {
-      if (rhsCombos.includes(lhsCombo)) {
-        // There was a match; add the refId and break out of the inner loop
-        rhsRefIds.push(rhsVarInstance.refId)
-        break
-      }
-    }
-  }
-
-  // Return the sorted array of relevant refIds
-  // TODO: Sorting is not essential here, but the legacy reader sorted so we will keep that
-  // behavior now to avoid invalidating tests.  Later we should remove this `sort` call and
-  // update the tests accordingly.
-  return rhsRefIds.sort()
+  // Step 2: Find the RHS variable instances whose subscripts overlap with the LHS
+  // subscripts at every position
+  return matchingRhsRefIds(mappedLhsSubIds, rhsVarInstances)
 }
 
 /**

package/src/model/read-variables.js

@@ -197,7 +197,13 @@ function subscriptPositionsToExpand(subIds, exceptSubIdSets, separationDims, rhs
 
     if (!expand) {
       // Direct data vars with subscripts are separated because we generate a lookup for each index
-      if (isDimension(subId) && (rhsText.includes('GET DIRECT DATA') || rhsText.includes('GET DIRECT LOOKUPS'))) {
+      if (
+        isDimension(subId) &&
+        (rhsText.includes('GET DIRECT DATA') ||
+          rhsText.includes('GET XLS DATA') ||
+          rhsText.includes('GET DIRECT LOOKUPS') ||
+          rhsText.includes('GET XLS LOOKUPS'))
+      ) {
         expand = true
       }
     }

package/src/parse-and-generate.js

@@ -32,8 +32,8 @@ import { generateCode } from './generate/gen-code.js'
  * @param {string[]} operations The set of operations to perform; can include 'generateC', 'generateJS',
  * 'printVarList', 'printRefIdTest', 'convertNames'.  If the array is empty, the model will be
  * read but no operation will be performed.
- * @param {string} modelDirname The absolute path to the directory containing the mdl file.
- * The dat and xlsx files referenced by the spec will be relative to this directory.
+ * @param {string} modelDirname The absolute path to the directory containing data (dat, xlsx, csv)
+ * files that are referenced by the model.  These files will be resolved relative to this directory.
  * @param {string} modelName The model name (without the mdl extension).
  * @param {string} buildDir The output directory where the C or list files will be written.
  * @param {string} [varname] The variable name passed to the 'sde causes' command.
@@ -134,8 +134,8 @@ export function printNames(namesPathname, operation) {
  *
  * @param {string} input The string containing the model text.
  * @param {string} modelKind The kind of model to parse, either 'vensim' or 'xmile'.
- * @param {string} modelDir The absolute path to the directory containing the mdl file.
- * The dat, xlsx, and csv files referenced by the model will be relative to this directory.
+ * @param {string} modelDir The absolute path to the directory containing data (dat, xlsx, csv)
+ * files that are referenced by the model.  These files will be resolved relative to this directory.
  * @param {Object} [options] The options that control parsing.
  * @param {boolean} options.sort Whether to sort definitions alphabetically in the preprocess step.
  * @return {*} A parsed tree representation of the model.
@@ -147,7 +147,7 @@ export function parseModel(input, modelKind, modelDir, options) {
     if (modelDir) {
       parseContext = {
         getDirectSubscripts(fileName, tabOrDelimiter, firstCell, lastCell /*, prefix*/) {
-          // Resolve the CSV file relative the model directory
+          // Resolve the CSV file relative to the model directory
           const csvPath = path.resolve(modelDir, fileName)
 
           // Read the subscripts from the CSV file