@sdeverywhere/cli 0.7.0

package/src/sde-bundle.js

@@ -0,0 +1,54 @@
+// Copyright (c) 2022 Climate Interactive / New Venture Fund
+
+import path from 'path'
+
+import { build as runBuild } from '@sdeverywhere/build'
+
+export let command = 'bundle [options]'
+export let describe = 'build and bundle a model as specified by a config file'
+export let builder = {
+  config: {
+    describe: 'path to the config file (defaults to `sde.config.js` in the current directory)',
+    type: 'string',
+    alias: 'c'
+  },
+  verbose: {
+    describe: 'enable verbose log messages',
+    type: 'boolean'
+  }
+}
+export let handler = argv => {
+  bundle(argv.config, argv.verbose)
+}
+export let bundle = async (configPath, verbose) => {
+  const logLevels = ['error', 'info']
+  if (verbose) {
+    logLevels.push('verbose')
+  }
+  const srcDir = new URL('.', import.meta.url).pathname
+  const sdeDir = path.resolve(srcDir, '..')
+  const sdeCmdPath = path.resolve(srcDir, 'main.js')
+  const result = await runBuild('production', {
+    config: configPath,
+    logLevels,
+    sdeDir,
+    sdeCmdPath
+  })
+  if (result.isOk()) {
+    // Exit with the specified code
+    console.log()
+    process.exit(result.exitCode)
+  } else {
+    // Exit with a non-zero code if any step failed
+    console.error(`ERROR: ${result.error.message}\n`)
+    process.exit(1)
+  }
+}
+
+export default {
+  command,
+  describe,
+  builder,
+  handler,
+  bundle
+}

package/src/sde-causes.js

@@ -0,0 +1,35 @@
+import { generateCode, parseModel, preprocessModel } from '@sdeverywhere/compile'
+
+import { modelPathProps, parseSpec } from './utils.js'
+
+let command = 'causes [options] <model> <c_varname>'
+let describe = 'print dependencies for a C variable name'
+let builder = {
+  spec: {
+    describe: 'pathname of the I/O specification JSON file',
+    type: 'string',
+    alias: 's'
+  }
+}
+let handler = argv => {
+  causes(argv.model, argv.c_varname, argv)
+}
+let causes = (model, varname, opts) => {
+  // Get the model name and directory from the model argument.
+  let { modelDirname, modelPathname } = modelPathProps(model)
+  let extData = new Map()
+  let directData = new Map()
+  let spec = parseSpec(opts.spec)
+  // Preprocess model text into parser input.
+  let input = preprocessModel(modelPathname, spec)
+  // Parse the model to get variable and subscript information.
+  let parseTree = parseModel(input)
+  let operation = 'printRefGraph'
+  generateCode(parseTree, { spec, operation, extData, directData, modelDirname, varname })
+}
+export default {
+  command,
+  describe,
+  builder,
+  handler
+}

package/src/sde-clean.js

@@ -0,0 +1,33 @@
+import path from 'path'
+import sh from 'shelljs'
+
+let command = 'clean [options]'
+let describe = 'clean out the build and output directories for a model'
+let builder = {
+  modeldir: {
+    describe: 'model directory',
+    type: 'string',
+    alias: 'm'
+  }
+}
+let handler = argv => {
+  clean(argv)
+}
+let clean = opts => {
+  // Remove the directories generated by SDEverywhere.
+  // Default to the current directory if no model directory is given.
+  let buildDirname = path.join(opts.modeldir || '.', 'build')
+  let outputDirname = path.join(opts.modeldir || '.', 'output')
+  let silentState = sh.config.silent
+  sh.config.silent = true
+  sh.rm('-r', buildDirname)
+  sh.rm('-r', outputDirname)
+  sh.config.silent = silentState
+}
+export default {
+  command,
+  describe,
+  builder,
+  handler,
+  clean
+}

package/src/sde-compare.js

@@ -0,0 +1,106 @@
+import { existsSync } from 'fs'
+
+import { pr } from 'bufx'
+import R from 'ramda'
+
+import { readDat } from '@sdeverywhere/compile'
+
+// The epsilon value determines the required precision for value comparisons.
+let ε = 1e-5
+
+export let command = 'compare [options] <vensimlog> <sdelog>'
+export let describe = 'compare Vensim and SDEverywhere log files in DAT format'
+export let builder = {
+  precision: {
+    describe: 'precision to which values must agree (default 1e-5)',
+    type: 'number',
+    alias: 'p'
+  },
+  name: {
+    describe: 'single Vensim variable name to compare',
+    type: 'string',
+    alias: 'n'
+  },
+  times: {
+    describe: 'limit comparisons to times separated by spaces',
+    type: 'array',
+    alias: 't'
+  }
+}
+export let handler = argv => {
+  compare(argv.vensimlog, argv.sdelog, argv)
+}
+export let compare = async (vensimfile, sdefile, opts) => {
+  if (!existsSync(vensimfile)) {
+    throw new Error(`Vensim DAT file not found: ${vensimfile}`)
+  }
+  if (!existsSync(sdefile)) {
+    throw new Error(`SDEverywhere DAT file not found: ${sdefile}`)
+  }
+
+  let vensimLog = await readDat(vensimfile)
+  let sdeLog = await readDat(sdefile)
+
+  if (vensimLog.size === 0) {
+    throw new Error(`Vensim DAT file did not contain data: ${vensimfile}`)
+  }
+  if (sdeLog.size === 0) {
+    throw new Error(`SDEverywhere DAT file did not contain data: ${sdefile}`)
+  }
+
+  if (opts.precision) {
+    ε = opts.precision
+  }
+
+  let noDATDifference = true
+  for (let varName of vensimLog.keys()) {
+    let sdeValues = sdeLog.get(varName)
+    // Ignore variables that are not found in the SDE log file.
+    if (sdeValues && (!opts.name || varName === opts.name)) {
+      let vensimValue = undefined
+      let vensimValues = vensimLog.get(varName)
+      // Filter on time t, the key in the values list.
+      for (let t of vensimValues.keys()) {
+        if (!opts.times || R.find(time => isEqual(time, t), opts.times)) {
+          // In Vensim log files, const vars only have one value at the initial time.
+          if (vensimValues.size > 1 || !vensimValue) {
+            vensimValue = vensimValues.get(t)
+          }
+          let sdeValue = sdeValues.get(t)
+          let diff = difference(sdeValue, vensimValue)
+          if (diff > ε) {
+            let diffPct = (diff * 100).toFixed(6)
+            pr(`${varName} time=${t.toFixed(2)} vensim=${vensimValue} sde=${sdeValue} diff=${diffPct}%`)
+            noDATDifference = false
+          }
+        }
+      }
+    }
+  }
+  if (noDATDifference) {
+    pr(`Data were the same for ${vensimfile} and ${sdefile}`)
+  }
+}
+let isZero = value => {
+  return Math.abs(value) < ε
+}
+let difference = (x, y) => {
+  let diff = 0
+  if (isZero(x) || isZero(y)) {
+    diff = Math.abs(x - y)
+  } else {
+    diff = Math.abs(1 - x / y)
+  }
+  return diff
+}
+let isEqual = (x, y) => {
+  return difference(x, y) < ε
+}
+
+export default {
+  command,
+  describe,
+  builder,
+  handler,
+  compare
+}

package/src/sde-compile.js

@@ -0,0 +1,57 @@
+import fs from 'fs-extra'
+import path from 'path'
+import sh from 'shelljs'
+
+import { buildDir, execCmd, modelPathProps } from './utils.js'
+
+export let command = 'compile [options] <model>'
+export let describe = 'compile the generated model to an executable file'
+export let builder = {
+  builddir: {
+    describe: 'build directory',
+    type: 'string',
+    alias: 'b'
+  }
+}
+export let handler = argv => {
+  compile(argv.model, argv)
+}
+export let compile = (model, opts) => {
+  let { modelDirname, modelName } = modelPathProps(model)
+  // Ensure the build directory exists.
+  let buildDirname = buildDir(opts.builddir, modelDirname)
+  // Link SDEverywhere C source files into the build directory.
+  linkCSourceFiles(modelDirname, buildDirname)
+  // Run make to compile the model C code.
+  let silentState = sh.config.silent
+  sh.config.silent = true
+  sh.pushd(buildDirname)
+  let exitCode = execCmd(`make P=${modelName}`)
+  sh.popd()
+  sh.config.silent = silentState
+  if (exitCode > 0) {
+    process.exit(exitCode)
+  }
+  return 0
+}
+export default {
+  command,
+  describe,
+  builder,
+  handler,
+  compile
+}
+
+let linkCSourceFiles = (modelDirname, buildDirname) => {
+  let cDirname = path.join(new URL('.', import.meta.url).pathname, 'c')
+  sh.ls(cDirname).forEach(filename => {
+    // If a C source file is present in the model directory, link to it instead
+    // as an override.
+    let srcPathname = path.join(modelDirname, filename)
+    if (!fs.existsSync(srcPathname)) {
+      srcPathname = path.join(cDirname, filename)
+    }
+    let dstPathname = path.join(buildDirname, filename)
+    fs.ensureSymlinkSync(srcPathname, dstPathname)
+  })
+}

package/src/sde-dev.js

@@ -0,0 +1,52 @@
+// Copyright (c) 2022 Climate Interactive / New Venture Fund
+
+import path from 'path'
+
+import { build as runBuild } from '@sdeverywhere/build'
+
+export let command = 'dev [options]'
+export let describe = 'run a model in a live development environment'
+export let builder = {
+  config: {
+    describe: 'path to the config file (defaults to `sde.config.js` in the current directory)',
+    type: 'string',
+    alias: 'c'
+  },
+  verbose: {
+    describe: 'enable verbose log messages',
+    type: 'boolean'
+  }
+}
+export let handler = argv => {
+  dev(argv.config, argv.verbose)
+}
+export let dev = async (configPath, verbose) => {
+  const logLevels = ['error', 'info']
+  if (verbose) {
+    logLevels.push('verbose')
+  }
+  const srcDir = new URL('.', import.meta.url).pathname
+  const sdeDir = path.resolve(srcDir, '..')
+  const sdeCmdPath = path.resolve(srcDir, 'main.js')
+  const result = await runBuild('development', {
+    config: configPath,
+    logLevels,
+    sdeDir,
+    sdeCmdPath
+  })
+
+  // Exit with a non-zero code if any step failed, otherwise keep the
+  // builder process alive
+  if (!result.isOk()) {
+    console.error(`ERROR: ${result.error.message}\n`)
+    process.exit(1)
+  }
+}
+
+export default {
+  command,
+  describe,
+  builder,
+  handler,
+  dev
+}

package/src/sde-exec.js

@@ -0,0 +1,48 @@
+import path from 'path'
+
+import { buildDir, execCmd, modelPathProps, outputDir } from './utils.js'
+
+export let command = 'exec [options] <model>'
+export let describe = 'execute the model and capture its output to a file'
+export let builder = {
+  builddir: {
+    describe: 'build directory',
+    type: 'string',
+    alias: 'b'
+  },
+  outfile: {
+    describe: 'output pathname',
+    type: 'string',
+    alias: 'o'
+  }
+}
+export let handler = argv => {
+  exec(argv.model, argv)
+}
+export let exec = (model, opts) => {
+  let { modelDirname, modelName } = modelPathProps(model)
+  // Ensure the build and output directories exist.
+  let buildDirname = buildDir(opts.builddir, modelDirname)
+  let outputDirname = outputDir(opts.outfile, modelDirname)
+  // Run the model and capture output in the model directory.
+  let modelCmd = `${buildDirname}/${modelName}`
+  let outputPathname
+  if (opts.outfile) {
+    outputPathname = opts.outfile
+  } else {
+    outputPathname = path.join(outputDirname, `${modelName}.txt`)
+  }
+  let exitCode = execCmd(`${modelCmd} >${outputPathname}`)
+  if (exitCode > 0) {
+    process.exit(exitCode)
+  }
+  return 0
+}
+
+export default {
+  command,
+  describe,
+  builder,
+  handler,
+  exec
+}

package/src/sde-flatten.js

@@ -0,0 +1,195 @@
+import path from 'path'
+
+import B from 'bufx'
+
+import { preprocessModel } from '@sdeverywhere/compile'
+
+import { buildDir, modelPathProps } from './utils.js'
+
+const command = 'flatten [options] <outmodel>'
+
+// TODO: Set to false for now to keep it hidden from the help menu, since
+// this is an experimental feature
+//const describe = 'flatten submodels into a single preprocessed mdl file'
+const describe = false
+
+const builder = {
+  builddir: {
+    describe: 'build directory',
+    type: 'string',
+    alias: 'b'
+  },
+  inputs: {
+    describe: 'input mdl files',
+    demand: true,
+    type: 'array'
+  }
+}
+
+const handler = argv => {
+  flatten(argv.outmodel, argv.inputs, argv)
+}
+
+const flatten = async (outFile, inFiles, opts) => {
+  // Ensure the build directory exists
+  // TODO: Since the input mdl files can technically exist in more than
+  // one directory, we don't want to guess, so just use the current directory
+  // if one isn't provided on the command line
+  const buildDirname = buildDir(opts.builddir, '.')
+
+  // Extract the equations and declarations from each parent or submodel file
+  // and store them into a map keyed by source file
+  const fileDecls = {}
+  for (const inFile of inFiles) {
+    // Get the path and short name of the input mdl file
+    const inModelProps = modelPathProps(inFile)
+
+    // Preprocess the mdl file and extract the equations
+    const decls = []
+    preprocessModel(inModelProps.modelPathname, undefined, 'genc', false, decls)
+
+    // Associate each declaration with the name of the model from which it came
+    for (const decl of decls) {
+      decl.sourceModelName = inModelProps.modelName
+    }
+
+    fileDecls[inModelProps.modelName] = decls
+  }
+
+  // Collapse so that we have only one equation or declaration per key.
+  // Equations that are defined in a different submodel will appear like
+  // a data variable in the model file where it is used (i.e., no equals
+  // sign).  If we see an equation, that takes priority over a "data"
+  // declaration.
+  const collapsed = {}
+  let hasErrors = false
+  for (const modelName of Object.keys(fileDecls)) {
+    const decls = fileDecls[modelName]
+    for (const decl of decls) {
+      const existingDecl = collapsed[decl.key]
+      if (existingDecl) {
+        // We've already seen a declaration with this key (from another file)
+        if (decl.kind === existingDecl.kind) {
+          if (decl.kind === 'sub') {
+            // The subscript keys are the same, so see if the contents are the same
+            if (decl.processedDecl !== existingDecl.processedDecl) {
+              // TODO: If we see two subscript declarations that differ, it's
+              // probably because a submodel uses a simplified set of dimension
+              // mappings, while the parent includes a superset of mappings.
+              // We should add more checks here, like see if the mappings in one
+              // are a subset of the other, but for now, we will emit a warning
+              // and just take the longer of the two.
+              let longerDecl
+              if (decl.processedDecl.length > existingDecl.processedDecl.length) {
+                longerDecl = decl
+              } else {
+                longerDecl = existingDecl
+              }
+              collapsed[decl.key] = longerDecl
+              let warnMsg = 'Subscript declarations are different; '
+              warnMsg += `will use the one from '${longerDecl.sourceModelName}.mdl' because it is longer`
+              console.warn('----------')
+              console.warn(`\nWARNING: ${warnMsg}:`)
+              console.warn(`\nIn ${existingDecl.sourceModelName}.mdl:`)
+              console.warn(`${existingDecl.processedDecl}`)
+              console.warn(`\nIn ${decl.sourceModelName}.mdl:`)
+              console.warn(`${decl.processedDecl}\n`)
+            }
+          } else if (decl.kind === 'eqn') {
+            // The equation keys are the same, so see if the contents are the same
+            if (decl.processedDecl !== existingDecl.processedDecl) {
+              hasErrors = true
+              console.error('----------')
+              console.error(`\nERROR: Differing equations:`)
+              console.error(`\nIn ${existingDecl.sourceModelName}.mdl:`)
+              console.error(`${existingDecl.processedDecl}`)
+              console.error(`\nIn ${decl.sourceModelName}.mdl:`)
+              console.error(`${decl.processedDecl}\n`)
+            }
+          }
+        } else if (decl.kind === 'eqn' && existingDecl.kind === 'decl') {
+          // Replace the declaration with this equation
+          collapsed[decl.key] = decl
+        }
+      } else {
+        // We haven't seen this key yet, so save the declaration
+        collapsed[decl.key] = decl
+      }
+    }
+  }
+
+  // XXX: For any subscripted (non-equation) declarations that remain, see if
+  // there are any corresponding equations.  This can happen in the case of
+  // subscripted variables that are defined in multiple parts in one submodel,
+  // but are declared using the full set in the consuming model.
+  // For example, in the submodel:
+  //   Variable[SubscriptA_Subset1] = ... ~~|
+  //   Variable[SubscriptA_Subset2] = ... ~~|
+  // In the consuming model:
+  //   Variable[SubscriptA_All] ~~|
+  // In this case, if we see Variable[SubscriptA_All], we can remove it and
+  // use the other `Variable` definitions.
+  for (const key of Object.keys(collapsed)) {
+    const decl = collapsed[key]
+    if (decl.kind === 'decl') {
+      const declKeyWithoutSubscript = key.split('[')[0]
+
+      // See if there is another equation that has the same key (minus subscript)
+      const otherDeclsThatMatch = []
+      for (const otherDecl of Object.values(collapsed)) {
+        if (otherDecl.kind === 'eqn') {
+          const otherDeclKeyWithoutSubscript = otherDecl.key.split('[')[0]
+          if (declKeyWithoutSubscript === otherDeclKeyWithoutSubscript) {
+            otherDeclsThatMatch.push(otherDecl)
+          }
+        }
+      }
+
+      if (otherDeclsThatMatch.length > 0) {
+        // Remove this declaration and emit a warning
+        delete collapsed[key]
+        console.warn('----------')
+        console.warn(`\nWARNING: Skipping declaration:`)
+        console.warn(`\nIn ${decl.sourceModelName}.mdl:`)
+        console.warn(`${decl.processedDecl}`)
+        console.warn(`\nThe following equations are assumed to provide the necessary data:`)
+        for (const otherDecl of otherDeclsThatMatch) {
+          console.warn(`\nIn ${otherDecl.sourceModelName}.mdl:`)
+          console.warn(`${otherDecl.processedDecl}`)
+        }
+        console.warn()
+      }
+    }
+  }
+
+  // Exit with a non-zero error code if there are any conflicting declarations
+  if (hasErrors) {
+    process.exit(1)
+  }
+
+  // Sort the declarations alphabetically by LHS variable name
+  const sorted = Object.values(collapsed).sort((a, b) => {
+    return a.key < b.key ? -1 : a.key > b.key ? 1 : 0
+  })
+
+  // Build a single buffer containing the sorted declarations
+  B.open('pp')
+  const ENCODING = '{UTF-8}'
+  B.emitLine(ENCODING, 'pp')
+  B.emitLine('', 'pp')
+  for (const decl of sorted) {
+    B.emitLine(`${decl.processedDecl}\n`, 'pp')
+  }
+
+  // Write the flattened mdl file to the build directory
+  const outModelProps = modelPathProps(outFile)
+  let outputPathname = path.join(buildDirname, `${outModelProps.modelName}.mdl`)
+  B.writeBuf(outputPathname, 'pp')
+}
+
+export default {
+  command,
+  describe,
+  builder,
+  handler
+}

package/src/sde-generate.js

@@ -0,0 +1,86 @@
+import path from 'path'
+import B from 'bufx'
+
+import { parseAndGenerate, preprocessModel } from '@sdeverywhere/compile'
+
+import { buildDir, modelPathProps, parseSpec } from './utils.js'
+
+export let command = 'generate [options] <model>'
+export let describe = 'generate model code'
+export let builder = {
+  genc: {
+    describe: 'generate C code for the model',
+    type: 'boolean'
+  },
+  list: {
+    describe: 'list model variables',
+    type: 'boolean',
+    alias: 'l'
+  },
+  preprocess: {
+    describe: 'write a preprocessed model that runs in Vensim',
+    type: 'boolean',
+    alias: 'p'
+  },
+  analysis: {
+    describe: 'write a nonexecutable preprocessed model for analysis',
+    type: 'boolean',
+    alias: 'a'
+  },
+  spec: {
+    describe: 'pathname of the I/O specification JSON file',
+    type: 'string',
+    alias: 's'
+  },
+  builddir: {
+    describe: 'build directory',
+    type: 'string',
+    alias: 'b'
+  },
+  refidtest: {
+    describe: 'test reference ids',
+    type: 'boolean',
+    alias: 'r'
+  }
+}
+export let handler = argv => {
+  generate(argv.model, argv)
+}
+
+export let generate = async (model, opts) => {
+  // Get the model name and directory from the model argument.
+  let { modelDirname, modelName, modelPathname } = modelPathProps(model)
+  // Ensure the build directory exists.
+  let buildDirname = buildDir(opts.builddir, modelDirname)
+  // Preprocess model text into parser input. Stop now if that's all we're doing.
+  let spec = parseSpec(opts.spec)
+  // Produce a runnable model with the "genc" and "preprocess" options.
+  let profile = opts.analysis ? 'analysis' : 'genc'
+  // Write the preprocessed model and removals if the option is "analysis" or "preprocess".
+  let writeFiles = opts.analysis || opts.preprocess
+  let input = preprocessModel(modelPathname, spec, profile, writeFiles)
+  if (writeFiles) {
+    let outputPathname = path.join(buildDirname, `${modelName}.mdl`)
+    B.write(input, outputPathname)
+    process.exit(0)
+  }
+  // Parse the model and generate code. If no operation is specified, the code generator will
+  // read the model and do nothing else. This is required for the list operation.
+  let operation = ''
+  if (opts.genc) {
+    operation = 'generateC'
+  } else if (opts.list) {
+    operation = 'printVarList'
+  } else if (opts.refidtest) {
+    operation = 'printRefIdTest'
+  }
+  await parseAndGenerate(input, spec, operation, modelDirname, modelName, buildDirname)
+}
+
+export default {
+  command,
+  describe,
+  builder,
+  handler,
+  generate
+}