@sdeverywhere/compile 0.7.9 → 0.7.11

package/src/generate/gen-lookup-from-direct.js

@@ -0,0 +1,105 @@
+import * as R from 'ramda'
+import XLSX from 'xlsx'
+
+import { listConcat } from '../_shared/helpers.js'
+import { sub } from '../_shared/subscript.js'
+
+import { handleExcelOrCsvFile } from './direct-data-helpers.js'
+
+/**
+ * Generate code for a variable that uses `GET DIRECT DATA` to source data from an external file
+ * (in CSV or Excel format).
+ *
+ * @param {*} variable The `Variable` instance to process.
+ * @param {'decl' | 'init-lookups'} mode The code generation mode.
+ * @param {Map<string, any>} directData The mapping of dataset name used in a `GET DIRECT DATA` call (e.g.,
+ * `?data`) to the tabular data contained in the loaded data file.
+ * @param {string} modelDir The path to the directory containing the model (used for resolving data files).
+ * @param {string} varLhs The C code for the LHS variable reference.
+ * @return {string[]} An array of strings containing the generated C code for the variable,
+ * one string per line of code.
+ */
+export function generateLookupsFromDirectData(variable, mode, directData, modelDir, varLhs) {
+  if (mode === 'decl') {
+    // Nothing to emit in decl mode
+    return []
+  } else if (mode !== 'init-lookups') {
+    throw new Error(`Invalid code gen mode '${mode}' for data variable ${variable.modelLHS}`)
+  }
+
+  // Create a function that reads the CSV or XLS[X] content
+  const { file, tab, timeRowOrCol, startCell } = variable.directDataArgs
+  const getCellValue = handleExcelOrCsvFile(file, tab, 'data', directData, modelDir)
+
+  // If direct data exists for this variable, copy it from the workbook into one or more lookups
+  let indexNum = 0
+  if (!R.isEmpty(variable.separationDims)) {
+    // Generate a lookup for a separated index in the variable's dimension.
+    if (variable.separationDims.length > 1) {
+      console.error(`WARNING: direct data variable ${variable.varName} separated on more than one dimension`)
+    }
+    let dimName = variable.separationDims[0]
+    for (let subscript of variable.subscripts) {
+      if (sub(subscript).family === dimName) {
+        // Use the index value in the subscript family when that is the separation dimension.
+        indexNum = sub(subscript).value
+        break
+      }
+      if (sub(dimName).value.includes(subscript)) {
+        // Look up the index when the separation dimension is a subdimension.
+        indexNum = sub(dimName).value.indexOf(subscript)
+        break
+      }
+    }
+  }
+  return [generateDirectDataLookup(varLhs, getCellValue, timeRowOrCol, startCell, indexNum)]
+}
+
+function generateDirectDataLookup(varLhs, getCellValue, timeRowOrCol, startCell, indexNum) {
+  // Read a row or column of data as (time, value) pairs from the worksheet.
+  // The cell(c,r) function wraps data access by column and row.
+  let lookupData = ''
+  let lookupSize = 0
+  let dataAddress = XLSX.utils.decode_cell(startCell.toUpperCase())
+  let dataCol = dataAddress.c
+  let dataRow = dataAddress.r
+  if (dataCol < 0 || dataRow < 0) {
+    throw new Error(`Failed to parse 'cell' argument for GET DIRECT {DATA,LOOKUPS} call for ${varLhs}: ${startCell}`)
+  }
+
+  let timeCol, timeRow, nextCell
+  if (isNaN(parseInt(timeRowOrCol))) {
+    // Time values are in a column.
+    timeCol = XLSX.utils.decode_col(timeRowOrCol.toUpperCase())
+    timeRow = dataRow
+    dataCol += indexNum
+    nextCell = () => {
+      dataRow++
+      timeRow++
+    }
+  } else {
+    // Time values are in a row.
+    timeCol = dataCol
+    timeRow = XLSX.utils.decode_row(timeRowOrCol)
+    dataRow += indexNum
+    nextCell = () => {
+      dataCol++
+      timeCol++
+    }
+  }
+
+  let timeValue = getCellValue(timeCol, timeRow)
+  let dataValue = getCellValue(dataCol, dataRow)
+  while (timeValue != null && dataValue != null) {
+    lookupData = listConcat(lookupData, `${timeValue}, ${dataValue}`, true)
+    lookupSize++
+    nextCell()
+    dataValue = getCellValue(dataCol, dataRow)
+    timeValue = getCellValue(timeCol, timeRow)
+  }
+  if (lookupSize === 0) {
+    throw new Error(`Empty lookup data array for ${varLhs}`)
+  }
+
+  return `  ${varLhs} = __new_lookup(${lookupSize}, /*copy=*/true, (double[]){ ${lookupData} });`
+}

package/src/generate/gen-lookup-from-external.js

@@ -0,0 +1,115 @@
+import * as R from 'ramda'
+
+import { cartesianProductOf, permutationsOf } from '../_shared/helpers.js'
+import { isDimension, sub } from '../_shared/subscript.js'
+import { pointsString } from './gen-lookup-from-points.js'
+
+/**
+ * Generate code for a data variable whose data is sourced from an external file (in DAT format).
+ *
+ * @param {*} variable The `Variable` instance to process.
+ * @param {'decl' | 'init-lookups'} mode The code generation mode.
+ * @param {Map<string, any>} extData The map of datasets from external `.dat` files.
+ * @param {string} varLhs The C code for the LHS variable reference.
+ * @return {string[]} An array of strings containing the generated C code for the variable,
+ * one string per line of code.
+ */
+export function generateLookupsFromExternalData(variable, mode, extData, varLhs) {
+  if (mode !== 'decl' && mode !== 'init-lookups') {
+    throw new Error(`Invalid code gen mode '${mode}' for data variable ${variable.modelLHS}`)
+  }
+
+  // If there is external data for this variable, copy it from an external file to a lookup.
+  // Just like in `generateLookupFromPoints`, we declare static arrays to hold the data points in
+  // the first pass ("decl" mode), then initialize each `Lookup` using that data in the second pass
+  // ("init" mode).
+  const newLookup = (name, lhs, data, subscriptIndexes) => {
+    if (!data) {
+      throw new Error(`Data for ${name} not found in external data sources`)
+    }
+
+    if (data.size === 0) {
+      throw new Error(`Empty lookup data array for ${lhs}`)
+    }
+
+    const dataName = variable.varName + '_data_' + R.map(i => `_${i}_`, subscriptIndexes).join('')
+    if (mode === 'decl') {
+      // In decl mode, declare a static data array that will be used to create the associated `Lookup`
+      // at init time
+      const points = pointsString(Array.from(data.entries()))
+      return `double ${dataName}[${data.size * 2}] = { ${points} };`
+    } else if (mode === 'init-lookups') {
+      // In init mode, create the `Lookup`, passing in a pointer to the static data array declared in decl mode.
+      return `  ${lhs} = __new_lookup(${data.size}, /*copy=*/false, ${dataName});`
+    } else {
+      return []
+    }
+  }
+
+  // There are three common cases that we handle:
+  //  - variable has no subscripts (C variable _thing = _thing from dat file)
+  //  - variable has subscript(s) (C variable with index _thing[0] = _thing[_subscript] from dat file)
+  //  - variable has dimension(s) (C variable in for loop, _thing[i] = _thing[_subscript_i] from dat file)
+
+  if (!variable.subscripts || variable.subscripts.length === 0) {
+    // No subscripts
+    const data = extData.get(variable.varName)
+    return [newLookup(variable.varName, varLhs, data, [])]
+  }
+
+  if (variable.subscripts.length === 1 && !isDimension(variable.subscripts[0])) {
+    // There is exactly one subscript
+    const subscript = variable.subscripts[0]
+    const nameInDat = `${variable.varName}[${subscript}]`
+    const data = extData.get(nameInDat)
+    const subIndex = sub(subscript).value
+    return [newLookup(nameInDat, varLhs, data, [subIndex])]
+  }
+
+  if (!R.all(s => isDimension(s), variable.subscripts)) {
+    // We don't yet handle the case where there are more than one subscript or a mix of
+    // subscripts and dimensions
+    // TODO: Remove this restriction
+    throw new Error(`Data variable ${variable.varName} has >= 2 subscripts; not yet handled`)
+  }
+
+  // At this point, we know that we have one or more dimensions; compute all combinations
+  // of the dimensions that we will iterate over
+  const lines = []
+  const allDims = R.map(s => sub(s).value, variable.subscripts)
+  const dimTuples = cartesianProductOf(allDims)
+  for (const dims of dimTuples) {
+    // Note: It appears that the dat file can have the subscripts in a different order
+    // than what SDE uses when declaring the C array.  If we don't find data for one
+    // order, we try the other possible permutations.
+    const dimNamePermutations = permutationsOf(dims)
+    let nameInDat, data
+    for (const dimNames of dimNamePermutations) {
+      nameInDat = `${variable.varName}[${dimNames.join(',')}]`
+      data = extData.get(nameInDat)
+      if (data) {
+        break
+      }
+    }
+    if (!data) {
+      // We currently treat this as a warning, not an error, since there can sometimes be
+      // datasets that are a sparse matrix, i.e., data is not defined for certain dimensions.
+      // For these cases, the lookup will not be initialized (the Lookup pointer will remain
+      // NULL, and any calls to `LOOKUP` will return `:NA:`.
+      if (mode === 'decl') {
+        console.error(`WARNING: Data for ${nameInDat} not found in external data sources`)
+      }
+      continue
+    }
+
+    const subscriptIndexes = R.map(dim => sub(dim).value, dims)
+    const varSubscripts = R.map(index => `[${index}]`, subscriptIndexes).join('')
+    const lhs = `${variable.varName}${varSubscripts}`
+    const lookup = newLookup(nameInDat, lhs, data, subscriptIndexes)
+    if (lookup) {
+      lines.push(lookup)
+    }
+  }
+
+  return lines
+}

package/src/generate/gen-lookup-from-points.js

@@ -0,0 +1,85 @@
+import { cdbl } from '../_shared/helpers.js'
+import { isDimension, isTrivialDimension, sub } from '../_shared/subscript.js'
+
+/**
+ * Generate code for a data or lookup variable that contains an explicit set of points.
+ *
+ * @param {*} variable The `Variable` instance to process.
+ * @param {'decl' | 'init-lookups'} mode The code generation mode.
+ * @param {boolean} copy If false, a static data array will be used (good for larger data sets).
+ * If true, the data will inlined and copied when initializing the lookup (good for smaller data sets).
+ * @param {string} varLhs The C code for the LHS variable reference.
+ * @param {LoopIndexVars} loopIndexVars The loop index state.
+ * @return {string[]} An array of strings containing the generated C code for the variable,
+ * one string per line of code.
+ */
+export function generateLookupFromPoints(variable, mode, copy, varLhs, loopIndexVars) {
+  if (variable.points.length === 0) {
+    throw new Error(`Empty lookup data array for ${variable.modelLHS}`)
+  }
+
+  if (copy) {
+    // Inline the data points and copy them when initializing the lookup
+    if (mode === 'decl') {
+      // Nothing to emit in decl mode
+      return []
+    } else if (mode === 'init-lookups') {
+      // In init mode, generate a new lookup using the data points from the variable
+      const points = pointsString(variable.points)
+      return [`  ${varLhs} = __new_lookup(${variable.points.length}, /*copy=*/true, (double[]){ ${points} });`]
+    }
+  } else {
+    // Construct the name of the data array, which is based on the associated lookup var name,
+    // with any subscripts tacked on the end.
+    const dataName = variable.varName + '_data_' + generateLookupDataName(variable.subscripts, loopIndexVars)
+    if (mode === 'decl') {
+      // In decl mode, declare a static data array that will be used to create the associated `Lookup`
+      // at init time. Using static arrays is better for code size, helps us avoid creating a copy of
+      // the data in memory, and seems to perform much better when compiled to wasm when compared to the
+      // previous approach that used varargs + copying, especially on constrained (e.g. iOS) devices.
+      const points = pointsString(variable.points)
+      return [`double ${dataName}[${variable.points.length * 2}] = { ${points} };`]
+    } else if (mode === 'init-lookups') {
+      // In init mode, create the `Lookup`, passing in a pointer to the static data array declared earlier.
+      // TODO: Make use of the lookup range
+      return [`  ${varLhs} = __new_lookup(${variable.points.length}, /*copy=*/false, ${dataName});`]
+    }
+  }
+
+  throw new Error(`Invalid code gen mode '${mode}' for lookup ${variable.modelLHS}`)
+}
+
+/**
+ * Return a string containing a comma separated list of [x,y] pairs from the given array of points.
+ *
+ * @param {number[][]} points The array of [x,y] tuples.
+ * @return {string} The string containing the comma separated point values.
+ */
+export function pointsString(points) {
+  return points.map(p => `${cdbl(p[0])}, ${cdbl(p[1])}`).join(', ')
+}
+
+/**
+ * Return a C name for the static data array associated with a lookup variable.
+ *
+ * @param {string[]} subIds The array of subscript IDs.
+ * @param {LoopIndexVars} loopIndexVars The loop index state.
+ * @return {string} The C array name.
+ */
+function generateLookupDataName(subIds, loopIndexVars) {
+  return subIds
+    .map(subId => {
+      if (isDimension(subId)) {
+        const i = loopIndexVars.index(subId)
+        if (isTrivialDimension(subId)) {
+          // When the dimension is trivial, we can simply emit e.g. `[i]` instead of `[_dim[i]]`
+          return `_${i}_`
+        } else {
+          return `_${subId}_${i}_`
+        }
+      } else {
+        return `_${sub(subId).value}_`
+      }
+    })
+    .join('')
+}

package/src/model/equation-reader.js

@@ -22,10 +22,11 @@ import {
 import {
   extractMarkedDims,
   indexNamesForSubscript,
+  isDimension,
+  isIndex,
   normalizeSubscripts,
   separatedVariableIndex,
-  sub,
-  isDimension
+  sub
 } from '../_shared/subscript.js'
 import ModelReader from '../parse/model-reader.js'
 import { createParser } from '../parse/parser.js'
@@ -190,31 +191,36 @@ export default class EquationReader extends ModelReader {
         startCell: args[2]
       }
     } else if (fn === '_IF_THEN_ELSE') {
-      // Evaluate the condition expression of the `IF THEN ELSE`.  If it resolves
-      // to a compile-time constant, we only need to visit one branch, which means
-      // that no references will be recorded for the other branch, therefore allowing
-      // it to be skipped in the unused reference elimination phase and during the
-      // final code generation phase.
-      const condText = ctx.expr(0).getText()
-      const exprReader = new ExprReader()
-      const condExpr = exprReader.read(condText)
-      if (condExpr.constantValue !== undefined) {
-        // Record the conditional expression and its constant value so that
-        // it can be accessed later by EquationGen.  We need to record it
-        // this way because any variables referenced by the expression may
-        // be removed during the unused reference elimination phase.
-        Model.addConstantExpr(condText, condExpr.constantValue)
-        if (condExpr.constantValue !== 0) {
-          // Only visit the "if true" branch
-          this.setArgIndex(1)
-          ctx.expr(1).accept(this)
+      if (process.env.SDE_NONPUBLIC_REDUCE_VARIABLES !== '0') {
+        // Evaluate the condition expression of the `IF THEN ELSE`.  If it resolves
+        // to a compile-time constant, we only need to visit one branch, which means
+        // that no references will be recorded for the other branch, therefore allowing
+        // it to be skipped in the unused reference elimination phase and during the
+        // final code generation phase.
+        const condText = ctx.expr(0).getText()
+        const exprReader = new ExprReader()
+        const condExpr = exprReader.read(condText)
+        if (condExpr.constantValue !== undefined) {
+          // Record the conditional expression and its constant value so that
+          // it can be accessed later by EquationGen.  We need to record it
+          // this way because any variables referenced by the expression may
+          // be removed during the unused reference elimination phase.
+          Model.addConstantExpr(condText, condExpr.constantValue)
+          if (condExpr.constantValue !== 0) {
+            // Only visit the "if true" branch
+            this.setArgIndex(1)
+            ctx.expr(1).accept(this)
+          } else {
+            // Only visit the "if false" branch
+            this.setArgIndex(2)
+            ctx.expr(2).accept(this)
+          }
         } else {
-          // Only visit the "if false" branch
-          this.setArgIndex(2)
-          ctx.expr(2).accept(this)
+          // Visit the condition and both branches
+          super.visitExprList(ctx)
         }
       } else {
-        // Visit the condition and both branches
+        // Optimization is disabled, visit the condition and both branches
         super.visitExprList(ctx)
       }
     } else {
@@ -238,8 +244,8 @@ export default class EquationReader extends ModelReader {
     // Get the var name of a variable in a call and save it as a reference.
     let id = ctx.Id().getText()
     let varName = canonicalName(id)
-    // Do not add a dimension name as a reference.
-    if (!isDimension(varName)) {
+    // Do not add a dimension or index name as a reference.
+    if (!isDimension(varName) && !isIndex(varName)) {
       let fn = this.currentFunctionName()
       this.refId = varName
       this.expandedRefIds = []

package/src/model/model.js

@@ -2,7 +2,7 @@ import B from 'bufx'
 import yaml from 'js-yaml'
 import * as R from 'ramda'
 
-import { decanonicalize, isIterable, listConcat, strlist, vlog, vsort } from '../_shared/helpers.js'
+import { canonicalName, decanonicalize, isIterable, listConcat, strlist, vlog, vsort } from '../_shared/helpers.js'
 import {
   addIndex,
   allAliases,
@@ -17,6 +17,10 @@ import {
 import { createParser } from '../parse/parser.js'
 
 import EquationReader from './equation-reader.js'
+import { readEquation } from './read-equations.js'
+import { readDimensionDefs } from './read-subscripts.js'
+import { readVariables as readVariables2 } from './read-variables.js'
+import { reduceVariables } from './reduce-variables.js'
 import SubscriptRangeReader from './subscript-range-reader.js'
 import toposort from './toposort.js'
 import VarNameReader from './var-name-reader.js'
@@ -38,14 +42,65 @@ const PRINT_INIT_GRAPH = false
 const PRINT_AUX_GRAPH = false
 const PRINT_LEVEL_GRAPH = false
 
-function read(parseTree, spec, extData, directData, modelDirname) {
+// XXX: This is needed for tests due to variables being in module-level storage
+function resetModelState() {
+  variables.length = 0
+  inputVars.length = 0
+  variablesByName.clear()
+  constantExprs.clear()
+  nonAtoANames = Object.create(null)
+}
+
+/**
+ * Read the given model parse tree and resolve all subscript and variable/equation definitions.
+ *
+ * Note that this function currently does not return anything and instead stores the parsed subscript
+ * definitions in the `subscript` module and the parsed/analyzed variables in this module.
+ *
+ * TODO: FIX TYPE
+ * @param {*} parsedModel The parsed model structure.
+ * @param {*} spec The parsed `spec.json` object.
+ * @param {Map<string, any>} extData The map of datasets from external `.dat` files.
+ * @param {Map<string, any>} directData The mapping of dataset name used in a `GET DIRECT DATA`
+ * call (e.g., `?data`) to the tabular data contained in the loaded data file.
+ * @param {string} modelDirname The path to the directory containing the model (used for resolving data
+ * files for `GET DIRECT SUBSCRIPT`).
+ * @param {*} opts An optional object used by tests to stop the read process after a specific phase.
+ */
+function read(parsedModel, spec, extData, directData, modelDirname, opts) {
   // Some arrays need to be separated into variables with individual indices to
   // prevent eval cycles. They are manually added to the spec file.
   let specialSeparationDims = spec.specialSeparationDims
-  // Subscript ranges must be defined before reading variables that use them.
-  readSubscriptRanges(parseTree, spec.dimensionFamilies, spec.indexFamilies, modelDirname)
+
+  // Dimensions must be defined before reading variables that use them.
+  if (parsedModel.kind === 'vensim-legacy') {
+    readSubscriptRanges(parsedModel.parseTree, modelDirname)
+  } else {
+    readDimensionDefs(parsedModel, modelDirname)
+  }
+  if (opts?.stopAfterReadSubscripts) return
+  resolveDimensions(spec.dimensionFamilies)
+  if (opts?.stopAfterResolveSubscripts) return
+
   // Read variables from the model parse tree.
-  readVariables(parseTree, specialSeparationDims, directData)
+  if (parsedModel.kind === 'vensim-legacy') {
+    // TODO: directData is actually unused in VariableReader
+    readVariables(parsedModel.parseTree, specialSeparationDims, directData)
+  } else {
+    // Read the variables
+    const vars = readVariables2(parsedModel, specialSeparationDims)
+
+    // Include a placeholder variable for the exogenous `Time` variable
+    const timeVar = new Variable(null)
+    timeVar.modelLHS = 'Time'
+    timeVar.varName = '_time'
+    vars.push(timeVar)
+
+    // Add the variables to the `Model`
+    vars.forEach(addVariable)
+  }
+  if (opts?.stopAfterReadVariables) return
+
   if (spec) {
     // If the spec file contains `input/outputVarNames` (with full Vensim variable names)
     // convert those to C names first.  Otherwise, use `input/outputNames` which are already
@@ -61,21 +116,51 @@ function read(parseTree, spec, extData, directData, modelDirname) {
       inputVars = spec.inputVars
     }
   }
+
   // Analyze model equations to fill in more details about variables.
-  analyze()
+  analyze(parsedModel.kind, spec?.inputVars, opts)
+  if (opts?.stopAfterAnalyze) return
+
   // Check that all input and output vars in the spec actually exist in the model.
   checkSpecVars(spec, extData)
+
   // Remove variables that are not referenced by an input or output variable.
   removeUnusedVariables(spec)
+
   // Resolve duplicate declarations by converting to one variable type.
   resolveDuplicateDeclarations()
 }
-function readSubscriptRanges(tree, dimensionFamilies, indexFamilies, modelDirname) {
+
+/**
+ * Read subscript ranges from the given model.
+ *
+ * Note that this function currently does not return anything and instead stores the parsed subscript
+ * range definitions in the `subscript` module.
+ *
+ * @param {import('../parse/parser.js').VensimModelParseTree} parseTree The Vensim parse tree.
+ * @param {string} modelDirname The path to the directory containing the model (used for resolving data
+ * files for `GET DIRECT SUBSCRIPT`).
+ */
+function readSubscriptRanges(parseTree, modelDirname) {
   // Read subscript ranges from the model.
   let subscriptRangeReader = new SubscriptRangeReader(modelDirname)
-  subscriptRangeReader.visitModel(tree)
+  subscriptRangeReader.visitModel(parseTree)
+}
+
+/**
+ * Process the previously read subscript/dimension definitions (stored in the `subscript` module) to
+ * resolve aliases, families, and indices.
+ *
+ * Note that this function currently does not return anything and only updates the set of dimension
+ * and subscript definitions in the `subscript` module.
+ *
+ * @param {Object.<string, string>} dimensionFamilies The optional mapping of dimension name to family name
+ * as provided in a `spec.json` file.
+ */
+function resolveDimensions(dimensionFamilies) {
   let allDims = allDimensions()
-  // Expand dimensions that appeared in subscript range definitions into indices.
+
+  // Expand dimensions that appeared in dimension definitions into subscripts/indices.
   // Repeat until there are only indices in dimension values.
   let dimFoundInValue
   do {
@@ -94,7 +179,7 @@ function readSubscriptRanges(tree, dimensionFamilies, indexFamilies, modelDirnam
     }
   } while (dimFoundInValue)
 
-  // Fill in subscript aliases from their model families.
+  // Fill in dimension aliases from their model families.
   for (let dim of allAliases()) {
     if (dim.value === '') {
       let refDim = sub(dim.family)
@@ -204,6 +289,21 @@ function readSubscriptRanges(tree, dimensionFamilies, indexFamilies, modelDirnam
     }
   }
 }
+
+/**
+ * Read equations from the given model and generate `Variable` instances for all variables that
+ * are encountered while parsing.
+ *
+ * Note that this function currently does not return anything and instead stores the parsed
+ * variable definitions in the `model` module.
+ *
+ * @param {import('../parse/parser.js').VensimModelParseTree} tree The Vensim parse tree.
+ * @param {Object.<string, string>} specialSeparationDims The variable names that need to be
+ * separated because of circular references.  A mapping from "C" variable name to "C" dimension
+ * name to separate on.
+ * @param {Map<string, any>} directData The mapping of dataset name used in a `GET DIRECT DATA`
+ * call (e.g., `?data`) to the tabular data contained in the loaded data file.
+ */
 function readVariables(tree, specialSeparationDims, directData) {
   // Read all variables in the model parse tree.
   // This populates the variables table with basic information for each variable
@@ -216,26 +316,44 @@ function readVariables(tree, specialSeparationDims, directData) {
   v.varName = '_time'
   addVariable(v)
 }
-function analyze() {
+function analyze(parsedModelKind, inputVars, opts) {
   // Analyze the RHS of each equation in stages after all the variables are read.
   // Find non-apply-to-all vars that are defined with more than one equation.
   findNonAtoAVars()
+
   // Set the refId for each variable. Only non-apply-to-all vars include subscripts in the refId.
   setRefIds()
+
+  // If enabled, reduce expressions used in variable definitions.
+  if (parsedModelKind !== 'vensim-legacy') {
+    if (opts?.reduceVariables !== false && process.env.SDE_NONPUBLIC_REDUCE_VARIABLES !== '0') {
+      let reduceMode = opts?.reduceVariables || process.env.SDE_NONPUBLIC_REDUCE_VARIABLES || 'default'
+      reduceVariables(variables, inputVars || [], reduceMode)
+    }
+  }
+  if (opts?.stopAfterReduceVariables === true) return
+
   // Read the RHS to list the refIds of vars that are referenced and set the var type.
-  readEquations()
+  if (parsedModelKind === 'vensim-legacy') {
+    readEquations()
+  } else {
+    variables.forEach(readEquation)
+  }
 }
 
 function checkSpecVars(spec, extData) {
-  // Look up each var in the spec and issue and error message if it does not exist.
+  // Look up each var in the spec and issue and throw error if it does not exist.
 
   function check(varNames, specType) {
     if (isIterable(varNames)) {
       for (let varName of varNames) {
+        // TODO: This code as written does not check variables that include subscripts, but
+        // we should check those as well (and make sure that subscripts or indices are not
+        // out of the valid range)
         if (!R.contains('[', varName)) {
           if (!varWithRefId(varName)) {
             // Look for a variable in external data.
-            if (extData.has(varName)) {
+            if (extData?.has(varName)) {
               // console.error(`found ${specType} ${varName} in extData`)
               // Copy data from an external file to an equation that does a lookup.
               let lookup = R.reduce(
@@ -246,7 +364,9 @@ function checkSpecVars(spec, extData) {
               let modelEquation = `${decanonicalize(varName)} = WITH LOOKUP(Time, (${lookup}))`
               addEquation(modelEquation)
             } else {
-              console.error(`${specType} variable ${varName} not found in the model or external data sources`)
+              throw new Error(
+                `The ${specType} variable ${varName} was declared in spec.json, but no matching variable was found in the model or external data sources`
+              )
             }
           }
         }
@@ -585,7 +705,7 @@ function varWithRefId(refId) {
       }
     }
     if (!refVar) {
-      vlog('ERROR: no var found for refId', refId)
+      // vlog('ERROR: no var found for refId', refId)
     }
   }
   return refVar
@@ -675,7 +795,29 @@ function vensimName(cVarName) {
 function cName(vensimVarName) {
   // Convert a Vensim variable name to a C name.
   // This function requires model analysis to be completed first when the variable has subscripts.
-  return new VarNameReader().read(vensimVarName)
+  if (process.env.SDE_NONPUBLIC_USE_NEW_PARSE !== '1') {
+    // TODO: For now we use the legacy VarNameReader when the old parser is active; this
+    // code will be removed once the old parser is removed
+    return new VarNameReader().read(vensimVarName)
+  }
+  // Split the variable name from the subscripts
+  let matches = vensimVarName.match(/([^[]+)(?:\[([^\]]+)\])?/)
+  if (!matches) {
+    throw new Error(`Invalid variable name '${vensimVarName}' found when converting to C representation`)
+  }
+  let cVarName = canonicalName(matches[1])
+  if (matches[2]) {
+    // The variable name includes subscripts, so split them into individual IDs
+    let cSubIds = matches[2].split(',').map(x => canonicalName(x))
+    cSubIds = normalizeSubscripts(cSubIds)
+    // If a subscript is an index, convert it to an index number to match Vensim data exports
+    let cSubIdParts = cSubIds.map(cSubId => {
+      return isIndex(cSubId) ? `[${sub(cSubId).value}]` : `[${cSubId}]`
+    })
+    // Append the subscript parts to the base variable name to create the full reference
+    cVarName += cSubIdParts.join('')
+  }
+  return cVarName
 }
 function isInputVar(varName) {
   // Return true if the given variable (in canonical form) is included in the list of
@@ -1146,6 +1288,7 @@ export default {
   read,
   refIdForVar,
   refIdsWithName,
+  resetModelState,
   splitRefId,
   variables,
   varIndexInfo,