@sdeverywhere/compile 0.7.9 → 0.7.11

package/src/model/read-subscripts.js

@@ -0,0 +1,77 @@
+import XLSX from 'xlsx'
+
+import { readCsv } from '../_shared/helpers.js'
+import { Subscript } from '../_shared/subscript.js'
+
+/**
+ * Read the dimension definitions from the given model.
+ *
+ * @param {*} parsedModel TODO: Use ParsedVensimModel type here
+ */
+export function readDimensionDefs(parsedModel) {
+  // Read and process all dimension definitions from the parsed model
+  for (const dimensionDef of parsedModel.root.dimensions) {
+    const dimName = dimensionDef.dimName
+    const familyName = dimensionDef.familyName
+    if (dimensionDef.subscriptRefs.length > 0) {
+      // This is a normal dimension definition
+      const subNames = dimensionDef.subscriptRefs.map(ref => ref.subName)
+      const mappings = dimensionDef.subscriptMappings.map(mapping => {
+        // Convert from the AST representation (`SubscriptMapping`) to the structure used
+        // by the compiler
+        return {
+          toDim: mapping.toDimName,
+          value: mapping.subscriptRefs.map(ref => ref.subName)
+        }
+      })
+      Subscript(dimName, subNames, familyName, mappings)
+    } else {
+      // This is an alias (for example, `DimA <-> DimB`)
+      // XXX: The legacy Vensim parser set `modelValue` to an empty string (instead of an
+      // empty array) in the case of a `<->` alias, so we will do the same here for now.
+      // Once we remove the legacy parsing code we could fix the tests to expect an empty
+      // array instead of empty string.
+      Subscript(dimName, '', familyName, [])
+    }
+  }
+}
+
+/**
+ * Read the subscripts for a `GET DIRECT SUBSCRIPT` call in a dimension definition.
+ *
+ * @param {string} filePath The absolute
+ * @param {string} tabOrDelimiter
+ * @param {string} firstCell The name of the first cell.
+ * @param {string} lastCell The name of the last cell.
+ */
+export function getDirectSubscripts(fileName, tabOrDelimiter, firstCell, lastCell) {
+  // If lastCell is a column letter, scan the column, else scan the row
+  const dataAddress = XLSX.utils.decode_cell(firstCell.toUpperCase())
+  let col = dataAddress.c
+  let row = dataAddress.r
+  if (col < 0 || row < 0) {
+    throw new Error(`Failed to parse 'firstcell' argument for GET DIRECT SUBSCRIPT call: ${firstCell}`)
+  }
+  let nextCell
+  if (isNaN(parseInt(lastCell))) {
+    nextCell = () => row++
+  } else {
+    nextCell = () => col++
+  }
+
+  // Read subscript names from the CSV file at the given position
+  // TODO: We currently only support reading from CSV files, but Vensim also allows for
+  // Excel files for `GET DIRECT SUBSCRIPT`, so we should add support for those here
+  const data = readCsv(fileName, tabOrDelimiter)
+  const subNames = []
+  if (data) {
+    let subName = data[row][col]
+    while (subName != null) {
+      subNames.push(subName)
+      nextCell()
+      subName = data[row] != null ? data[row][col] : null
+    }
+  }
+
+  return subNames
+}

package/src/model/read-variables.js

@@ -0,0 +1,264 @@
+import * as R from 'ramda'
+
+import { toPrettyString } from '@sdeverywhere/parse'
+
+import { cartesianProductOf } from '../_shared/helpers.js'
+import {
+  isDimension,
+  isIndex,
+  isSubdimension,
+  normalizeSubscripts,
+  sub,
+  subscriptsMatch
+} from '../_shared/subscript.js'
+
+import Variable from './variable.js'
+
+/**
+ * TODO: Docs
+ *
+ * @param {*} parsedModel TODO: Use ParsedVensimModel type here
+ * @param {Object.<string, string>} [specialSeparationDims] The variable names that need to be
+ * separated because of circular references.  A mapping from "C" variable name to "C" dimension
+ * name to separate on.
+ * @returns {*} An array containing all `Variable` instances that were generated from
+ * the model equations.
+ */
+export function readVariables(parsedModel, specialSeparationDims) {
+  const variables = []
+
+  // Add one or more `Variable` definitions for each parsed equation
+  for (const eqn of parsedModel.root.equations) {
+    variables.push(...variablesForEquation(eqn, specialSeparationDims || {}))
+  }
+
+  return variables
+}
+
+/**
+ * Process a single parsed equation and return one or more `Variable` definitions
+ * that were derived from the given equation.
+ *
+ * TODO: Types
+ *
+ * @param {*} eqn The parsed equation.
+ * @param {Object.<string, string>} specialSeparationDims The variable names that need to be
+ * separated because of circular references.
+ * @returns {*} An array containing all `Variable` instances that were generated from
+ * the given equation.
+ */
+function variablesForEquation(eqn, specialSeparationDims) {
+  // Start a new variable defined by this equation
+  const variable = new Variable(null)
+
+  // Fill in the LHS details
+  const lhs = eqn.lhs.varDef
+  const baseVarId = lhs.varId
+  let lhsText
+  if (lhs.subscriptRefs?.length > 0) {
+    // Note that we use the original order of subscripts here, not the "normalized"
+    // order as below.  (This is how the legacy parser worked, so we will preserve
+    // that behavior for now.)
+    const subNames = lhs.subscriptRefs.map(sub => sub.subName)
+    let exceptPart
+    if (lhs.exceptSubscriptRefSets) {
+      const exceptSets = lhs.exceptSubscriptRefSets.map(exceptSubRefs => {
+        const exceptSubNames = exceptSubRefs.map(sub => sub.subName)
+        return `[${exceptSubNames.join(',')}]`
+      })
+      exceptPart = `:EXCEPT:${exceptSets.join(',')}`
+    } else {
+      exceptPart = ''
+    }
+    lhsText = `${lhs.varName}[${subNames.join(',')}]${exceptPart}`
+  } else {
+    lhsText = lhs.varName
+  }
+  variable.modelLHS = lhsText
+  variable.varName = baseVarId
+
+  // Fill in the RHS details
+  let rhsText
+  if (eqn.rhs.kind === 'expr') {
+    rhsText = toPrettyString(eqn.rhs.expr, { compact: true })
+  } else if (eqn.rhs.kind === 'const-list') {
+    rhsText = eqn.rhs.text
+  } else {
+    rhsText = ''
+    if (eqn.rhs.kind === 'data') {
+      // The legacy parser sets the variable's varType to 'data' at this stage,
+      // so we will do the same
+      variable.varType = 'data'
+    }
+  }
+  variable.modelFormula = rhsText
+  variable.parsedEqn = eqn
+
+  // If the variable is subscripted, expand on the LHS subscripts
+  let expansions = []
+  if (lhs.subscriptRefs?.length > 0) {
+    // XXX: We use `normalizeSubscripts` here so that we are compatible with
+    // the legacy parser.  It normalizes by putting the subscripts in alphabetical
+    // order by family.  This approach to ordering may lead to issues in cases where
+    // there are multiple dimensions used that resolve to the same family, so we
+    // should revisit this.
+    const subIds = normalizeSubscripts(lhs.subscriptRefs.map(ref => ref.subId))
+    const exceptSubIdSets = []
+    if (lhs.exceptSubscriptRefSets?.length > 0) {
+      for (const exceptSubRefs of lhs.exceptSubscriptRefSets) {
+        exceptSubIdSets.push(normalizeSubscripts(exceptSubRefs.map(ref => ref.subId)))
+      }
+    }
+
+    // Determine which positions we will expand
+    let positionsToExpand
+    if (eqn.rhs.kind === 'const-list') {
+      // For const lists, we expand on all dimensions (unconditionally)
+      positionsToExpand = subIds.map(subId => isDimension(subId))
+    } else {
+      // For other equations, look at the different subscripts and the RHS to determine
+      // which positions to expand.  Note that `specialSeparationDims` in the spec file
+      // can be a single string or an array of strings.
+      let separationDims = specialSeparationDims[baseVarId] || []
+      if (!Array.isArray(separationDims)) {
+        separationDims = [separationDims]
+      }
+      positionsToExpand = subscriptPositionsToExpand(subIds, exceptSubIdSets, separationDims, variable.modelFormula)
+    }
+
+    // Expand on LHS subscripts
+    expansions = computeExpansions(baseVarId, subIds, exceptSubIdSets, positionsToExpand)
+  }
+
+  if (expansions.length === 0) {
+    // Generate a single variable defined by the equation
+    return [variable]
+  } else {
+    // Generate variables expanded over subscripts to the model
+    const variables = []
+    for (const expansion of expansions) {
+      const v = new Variable(null)
+      v.varName = baseVarId
+      v.modelLHS = lhsText
+      v.modelFormula = rhsText
+      v.parsedEqn = eqn
+      v.subscripts = expansion.subIds
+      v.separationDims = expansion.separationDimIds
+      variables.push(v)
+    }
+    return variables
+  }
+}
+
+/**
+ * Return an array of boolean flags that indicate whether the variable should be
+ * expanded over the subscripts for that position.
+ *
+ * TODO: Use correct types here
+ *
+ * @param {*} subIds The list of subscripts appearing on the LHS in normalized order.
+ * @param {*} exceptSubIdSets An array of subscript lists from the :EXCEPT: clause.
+ * @param {string[]} separationDims The variable names that need to be separated for this
+ * variable because of circular references.
+ * @param {string} rhsText The text of the RHS of the variable definition.
+ * @returns {boolean[]} An array of boolean flags, one for each subscript position.
+ */
+function subscriptPositionsToExpand(subIds, exceptSubIdSets, separationDims, rhsText) {
+  // Decide whether we need to expand each subscript on the LHS.
+  // Construct an array of booleans in each subscript position.
+  const expandFlags = new Array(subIds.length).fill(false)
+
+  for (let i = 0; i < subIds.length; i++) {
+    const subId = subIds[i]
+    let expand = false
+
+    // Expand a subdimension and special separation dims in the LHS
+    if (isDimension(subId)) {
+      expand = isSubdimension(subId)
+      if (!expand) {
+        expand = separationDims.includes(subId)
+      }
+    }
+
+    if (!expand) {
+      // Direct data vars with subscripts are separated because we generate a lookup for each index
+      if (isDimension(subId) && (rhsText.includes('GET DIRECT DATA') || rhsText.includes('GET DIRECT LOOKUPS'))) {
+        expand = true
+      }
+    }
+
+    if (!expand) {
+      // Also expand on exception subscripts that are indices or subdimensions
+      for (const exceptSubIds of exceptSubIdSets) {
+        expand = isIndex(exceptSubIds[i]) || isSubdimension(exceptSubIds[i])
+        if (expand) {
+          break
+        }
+      }
+    }
+
+    expandFlags[i] = expand
+  }
+
+  return expandFlags
+}
+
+/**
+ * Return an array describing the set of variable expansions.
+ *
+ * TODO: Use correct types here
+ *
+ * @param {string} baseVarId The canonical base name of the variable.
+ * @param {*} subIds The list of subscripts appearing on the LHS in normalized order.
+ * @param {*} exceptSubIdSets An array of subscript lists from the :EXCEPT: clause.
+ * @param {*} positionsToExpand An array of boolean flags, one for each subscript position.
+ * @returns {*} An array of objects containing the `subIds` and `separationDims` for each expansion.
+ */
+function computeExpansions(baseVarId, subIds, exceptSubIdSets, positionsToExpand) {
+  // Construct an array with an array at each subscript position. If the subscript is expanded at that position,
+  // it will become an array of indices. Otherwise, it remains an index or dimension as a single-valued array.
+  const expandedSubIdsPerPosition = []
+  const separationDimIds = []
+  for (let i = 0; i < subIds.length; i++) {
+    const subId = subIds[i]
+    let subIdsInExpansion
+    if (positionsToExpand[i]) {
+      separationDimIds.push(subId)
+      if (isDimension(subId)) {
+        subIdsInExpansion = sub(subId).value
+      }
+    }
+    expandedSubIdsPerPosition.push(subIdsInExpansion || [subId])
+  }
+
+  // Generate an array of fully expanded subscripts, which may be indices or dimensions
+  const expandedSubIdSets = cartesianProductOf(expandedSubIdsPerPosition)
+  const skipExpansion = expandedSubIds => {
+    // Check the subscripts against each set of except subscripts. Skip expansion if one of them matches.
+    const subsRange = R.range(0, expandedSubIds.length)
+    for (const exceptSubIds of exceptSubIdSets) {
+      if (expandedSubIds.length === exceptSubIds.length) {
+        if (R.all(i => subscriptsMatch(expandedSubIds[i], exceptSubIds[i]), subsRange)) {
+          return true
+        }
+      } else {
+        console.error(`WARNING: expandedSubIds length ≠ exceptSubIds length in ${baseVarId}`)
+      }
+    }
+    return false
+  }
+
+  const expansions = []
+  for (const expandedSubIds of expandedSubIdSets) {
+    // Skip expansions that match exception subscripts
+    if (!skipExpansion(expandedSubIds)) {
+      // Add a new expansion
+      expansions.push({
+        subIds: expandedSubIds,
+        separationDimIds
+      })
+    }
+  }
+
+  return expansions
+}

package/src/model/reduce-variables.js

@@ -0,0 +1,166 @@
+import { reduceConditionals, reduceExpr, toPrettyString } from '@sdeverywhere/parse'
+
+import Model from './model.js'
+
+/**
+ * Process the parsed `Equation` instances associated with all variables in the model and attempt to
+ * reduce expressions through constant folding and simplifying trivial operations and function calls.
+ *
+ * @param {*} variables The array of `Variable` instances.
+ * @param {string[]} inputVarIds The array of IDs for the configured input variables, which need to be
+ * preserved (will not be reduced/eliminated).
+ * @param {'default' | 'aggressive'} mode The reduction method to use.
+ */
+export function reduceVariables(variables, inputVarIds, mode) {
+  const baseVarIdForRefId = refId => {
+    return refId.split('[')[0]
+  }
+
+  const refIdForVarRef = varRef => {
+    if (varRef.subscriptRefs?.length > 0) {
+      const subIds = varRef.subscriptRefs.map(subRef => subRef.subId)
+      return `${varRef.varId}[${subIds.join(',')}]`
+    } else {
+      return varRef.varId
+    }
+  }
+
+  // TODO: For now, we will look up input variables by base varId (ignoring any subscripts)
+  // to avoid any confusion
+  const inputBaseVarIdsSet = new Set()
+  for (const inputVarId of inputVarIds) {
+    inputBaseVarIdsSet.add(baseVarIdForRefId(inputVarId))
+  }
+
+  // Keep track of which variables are currently being reduced in order to detect cycles
+  // let currentLhsBaseVarId
+  const activelyReducingRefIds = new Set()
+
+  const reduceVariable = v => {
+    if (v.reduced) {
+      // The variable has already been visited/reduced
+      return
+    }
+
+    // For now, only attempt to reduce variables that have an "expr" equation
+    // TODO: Handle const lists too
+    if (v.parsedEqn?.rhs?.kind !== 'expr') {
+      v.reduced = true
+      return
+    }
+
+    // Add this variable to the set of active ones
+    // TODO: Allow cycle if the current LHS is referenced on the RHS
+    // const baseVarId = v.parsedEqn.lhs.varId
+    // if (baseVarId !== currentLhsBaseVarId) {
+    if (activelyReducingRefIds.has(v.refId)) {
+      throw new Error(`Cycle detected when reducing variables: ${v.refId}`)
+    }
+    // }
+    activelyReducingRefIds.add(v.refId)
+
+    // We currently have two options for reducing variables.  The less aggressive
+    // one only reduces conditionals (the default for now, for compatibility with
+    // the legacy reader), and the more aggressive one (enabled via environment
+    // variable) performs constant folding and simplifies trivial expressions and
+    // function calls.
+    let reduce
+    if (mode === 'aggressive') {
+      reduce = reduceExpr
+    } else {
+      reduce = reduceConditionals
+    }
+
+    // Reduce the expression
+    const parsedRhsExpr = v.parsedEqn.rhs.expr
+    const reducedRhsExpr = reduce(parsedRhsExpr, {
+      resolveVarRef
+    })
+
+    // Save reduced RHS as a new `parsedEqn`
+    const newEqn = {
+      lhs: v.parsedEqn.lhs,
+      rhs: {
+        kind: 'expr',
+        expr: reducedRhsExpr
+      }
+    }
+    v.parsedEqn = newEqn
+
+    // TODO: Ideally we would leave the `modelFormula` untouched so that when we generate the
+    // comment above the generated code, it would show the original equation instead of the
+    // reduced one.  But the tests currently rely on `modelFormula` being updated after we
+    // reduce the RHS, so we save the original formula to `origModelFormula` and use that when
+    // generating the code comment, and save the reduced formula to `modelFormula`.
+    const reducedRhsText = toPrettyString(reducedRhsExpr, { compact: true })
+    v.origModelFormula = v.modelFormula
+    v.modelFormula = reducedRhsText
+
+    // Set a flag to indicate that the variable has been visited and reduced
+    v.reduced = true
+
+    // Remove this variable from the set of active ones
+    activelyReducingRefIds.delete(v.refId)
+  }
+
+  const resolveVarRef = varRef => {
+    // If we look up by refId, it will fail if it's a separated var.  If it's a normal
+    // unsubscripted or apply-to-all var, then refId lookup should return one var.
+    // TODO: This approach needs to be revisited
+    const refId = refIdForVarRef(varRef)
+    const refVar = Model.varWithRefId(refId)
+    if (refVar) {
+      // XXX: For now, don't try to reduce variables like this:
+      //   x[DimA] = 1
+      // This is because `generateSmoothVariables` doesn't work well in cases
+      // where some arguments have subscripts and some don't.  So for example,
+      // if the "init" arg is subscripted but is defined as a constant, we would
+      // reduce it to a constant, but `generateSmoothVariables` doesn't know how
+      // to mix.
+      if (refVar.subscripts !== undefined && refVar.subscripts.length > 0) {
+        return undefined
+      }
+
+      // In "aggressive" mode, attempt to reduce the referenced variable, otherwise
+      // use the parsed RHS expression as is
+      if (mode === 'aggressive') {
+        reduceVariable(refVar)
+      }
+
+      // If the attempt was successful, the reduced expression will be defined here
+      const reducedExpr = refVar.parsedEqn?.rhs?.expr
+      if (reducedExpr) {
+        // Only substitute if the reduced expression resolves to a simple constant and
+        // the variable is not configured as an input (that can override the constant
+        // value at runtime).  Otherwise we will keep the variable reference intact.
+        // TODO: Are there other cases where we should do a substitution?  Like when
+        // the expression reduces to a single 'variable-ref'?
+        const refBaseVarId = baseVarIdForRefId(refId)
+        if (reducedExpr.kind === 'number' && !inputBaseVarIdsSet.has(refBaseVarId)) {
+          return reducedExpr
+        } else {
+          return undefined
+        }
+      } else {
+        return undefined
+      }
+    } else {
+      // We couldn't find the variable by refId, so find all variables with that base varId
+      // TODO: Revisit this
+      // const refVars = Model.varsWithName(varRef.varId)
+      // if (refVars.length === 1) {
+      //   // When there's a single variable, we can use that
+      //   reduceVariable(refVars[0])
+      //   return refVars[0].parsedEqn?.rhs?.expr
+      // } else {
+      //   return undefined
+      // }
+      return undefined
+    }
+  }
+
+  // Visit each variable and attempt to reduce it
+  for (const v of variables) {
+    reduceVariable(v)
+  }
+}

package/src/model/subscript-range-reader.js

@@ -115,9 +115,12 @@ export default class SubscriptRangeReader extends ModelReader {
     let lastCell = args[3]
     // let prefix = args[4]
     // If lastCell is a column letter, scan the column, else scan the row.
-    let dataAddress = XLSX.utils.decode_cell(firstCell)
+    let dataAddress = XLSX.utils.decode_cell(firstCell.toUpperCase())
     let col = dataAddress.c
     let row = dataAddress.r
+    if (col < 0 || row < 0) {
+      throw new Error(`Failed to parse 'firstcell' argument for GET DIRECT SUBSCRIPT call: ${firstCell}`)
+    }
     let nextCell
     if (isNaN(parseInt(lastCell))) {
       nextCell = () => row++

package/src/model/variable.js

@@ -3,8 +3,13 @@ export default class Variable {
     // The equation rule context allows us to generate code by visiting the parse tree.
     this.eqnCtx = eqnCtx
     // Save both sides of the equation text in the model for documentation purposes.
-    this.modelLHS = eqnCtx ? eqnCtx.lhs().getText() : ''
-    this.modelFormula = this.formula(eqnCtx)
+    if (eqnCtx) {
+      this.modelLHS = eqnCtx.lhs().getText()
+      this.modelFormula = this.formula(eqnCtx)
+    } else {
+      this.modelLHS = ''
+      this.modelFormula = ''
+    }
     // An equation defines a variable with a var name, saved in canonical form here.
     this.varName = ''
     // Subscripts are canonical dimension or index names on the LHS in normal order.

package/src/parse/parser.js

@@ -3,13 +3,17 @@
 import antlr4 from 'antlr4'
 import { ModelLexer, ModelParser } from 'antlr4-vensim'
 
+/**
+ * @typedef {object} VensimModelParseTree
+ */
+
 /**
  * Create a `ModelParser` for the given model text, which can be the
  * contents of an entire `mdl` file, or a portion of one (e.g., an
  * expression or definition).
  *
- * @param input The string containing the model text.
- * @return A `ModelParser` from which a parse tree can be obtained.
+ * @param {string} input The string containing the model text.
+ * @return {ModelParser} A `ModelParser` from which a parse tree can be obtained.
  */
 export function createParser(input) {
   let chars = new antlr4.InputStream(input)
@@ -23,8 +27,8 @@ export function createParser(input) {
 /**
  * Read the given model text and return a parse tree.
  *
- * @param input The string containing the model text.
- * @return A parse tree representation of the model.
+ * @param {string} input The string containing the model text.
+ * @return {VensimModelParseTree} A parse tree representation of the model.
  */
 export function parseModel(input) {
   let parser = createParser(input)