@sdeverywhere/cli 0.7.27 → 0.7.29

package/README.md

@@ -97,7 +97,10 @@ sde bundle [--config sde.config.js] [--verbose]
 sde generate --list {model}
 ```
 
-#### Preprocess a model to remove macros and tabbed arrays to removals.txt
+#### Preprocess a model to remove macros and tabbed arrays
+
+_NOTE:_ For most users it is not necessary to run the preprocessor manually
+because the transpiler runs the preprocessor automatically prior to parsing.
 
 ```
 sde generate --preprocess {model}
@@ -204,21 +207,6 @@ There is an example in the `directdata` sample model.
 }
 ```
 
-#### Specify equations to remove from the model
-
-When SDEverywhere cannot handle a certain Vensim construct yet, you will need to remove equations that use the construct from the model, convert it by hand into something that SDEverywhere can handle, and then insert it back into the model.
-To have the preprocessor remove equations from the model into a `removals.txt` file, specify substrings to match in the equation in the `removalKeys` section.
-Macros and `TABBED ARRAY` equations are already automatically removed by the preprocessor.
-
-For instance, you could key on the variable name in the equation definition.
-
-```json
-"removalKeys": [
-  "varname1 =",
-  "varname2 ="
-]
-```
-
 #### Generating, compiling, running, and testing the C code
 
 To generate C code using the `--spec` argument, enter the following command:
@@ -260,15 +248,6 @@ That is, the values are separated by spaces, and each pair has an index number,
 The zero-based index maps into a static array of input variable pointers held in the function.
 These are used to set the value directly into the static `double` variable in the generated code.
 
-#### Inserting a file into the model
-
-Some constructs like macros are not supported by SDEverywhere.
-They are removed from the model by the preprocessor into the `removals.txt` file.
-You can edit these constructs into a form that SDEverywhere supports and insert them back into the model.
-Create a file called `mdl-edits.txt` in the model directory with the constructs to insert.
-For instance, manually expand macros and place them into the `mdl-edits.txt` file.
-The preprocessor will read this file and insert its contents unchanged into the beginning of the model.
-
 ## License
 
 SDEverywhere is distributed under the MIT license. See `LICENSE` for more details.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sdeverywhere/cli",
-  "version": "0.7.27",
+  "version": "0.7.29",
   "description": "Contains the `sde` command line interface for the SDEverywhere tool suite.",
   "type": "module",
   "files": [
@@ -11,8 +11,8 @@
     "sde": "src/main.js"
   },
   "dependencies": {
-    "@sdeverywhere/build": "^0.3.6",
-    "@sdeverywhere/compile": "^0.7.20",
+    "@sdeverywhere/build": "^0.3.7",
+    "@sdeverywhere/compile": "^0.7.22",
     "bufx": "^1.0.5",
     "byline": "^5.0.0",
     "ramda": "^0.27.0",

package/src/c/main.c

@@ -2,7 +2,7 @@
 
 int main(int argc, char** argv) {
   // TODO make the input buffer size dynamic
-  char inputs[1500];
+  char inputs[500000];
   // When true, output data without newlines or a header, suitable for embedding reference data.
   bool raw_output = false;
   // When true, suppress data output when using PR* macros.

package/src/c/makefile

@@ -1,6 +1,6 @@
 # Do "export P={c-file-basename}" before running this makefile.
 OBJECTS=main.o vensim.o model.o macros.o
-CFLAGS=-Wall -O3
+CFLAGS=-Wall -O2
 LDLIBS=
 
 # In case of a Linux build we need to add

package/src/c/vensim.c

@@ -56,25 +56,79 @@ double _ZIDZ(double a, double b) {
 Lookup* __new_lookup(size_t size, bool copy, double* data) {
   // Make a new Lookup with "size" number of pairs given in x, y order in a flattened list.
   Lookup* lookup = malloc(sizeof(Lookup));
-  lookup->n = size;
-  lookup->inverted_data = NULL;
-  lookup->data_is_owned = copy;
+  lookup->original_size = size;
+  lookup->original_data_is_owned = copy;
   if (copy) {
-    // Copy array into the lookup data.
-    lookup->data = malloc(sizeof(double) * 2 * size);
-    memcpy(lookup->data, data, size * 2 * sizeof(double));
+    // Copy the given lookup data into an internally managed buffer.
+    size_t data_length_in_bytes = size * 2 * sizeof(double);
+    lookup->original_data = malloc(data_length_in_bytes);
+    memcpy(lookup->original_data, data, data_length_in_bytes);
   } else {
     // Store a pointer to the lookup data (assumed to be static or owned elsewhere).
-    lookup->data = data;
+    lookup->original_data = data;
   }
+  // Set the original data as "active".
+  lookup->active_data = lookup->original_data;
+  lookup->active_size = lookup->original_size;
+  // Set `dynamic_data` to NULL initially; it will be allocated on demand if lookup
+  // data is overridden at runtime using `__set_lookup`.
+  lookup->dynamic_data = NULL;
+  lookup->dynamic_data_length = 0;
+  lookup->dynamic_size = 0;
+  // Set `inverted_data` to NULL initially; it will be allocated on demand in case
+  // of `LOOKUP INVERT` calls.
+  lookup->inverted_data = NULL;
+  // Set the cached "last" values to the initial values.
   lookup->last_input = DBL_MAX;
   lookup->last_hit_index = 0;
   return lookup;
 }
+void __set_lookup(Lookup* lookup, size_t size, double* data) {
+  // Set new data for the given `Lookup`.  If `data` is NULL, the original data that was
+  // supplied to the `__new_lookup` call will be restored as the "active" data.  Otherwise,
+  // `data` will be copied to an internal data buffer, which will be the "active" data.
+  // If `size` is greater than the size passed to previous calls, the internal data buffer
+  // will be grown as needed.
+  if (lookup == NULL) {
+    return;
+  }
+  if (data != NULL) {
+    // Allocate or grow the internal buffer as needed.
+    size_t data_length_in_elems = size * 2;
+    size_t data_length_in_bytes = data_length_in_elems * sizeof(double);
+    if (data_length_in_elems > lookup->dynamic_data_length) {
+      lookup->dynamic_data = malloc(data_length_in_bytes);
+      lookup->dynamic_data_length = data_length_in_elems;
+    }
+    // Copy the given lookup data into the internally managed buffer.
+    lookup->dynamic_size = size;
+    if (data_length_in_bytes > 0) {
+      memcpy(lookup->dynamic_data, data, data_length_in_bytes);
+    }
+    // Set the dynamic data as the "active" data.
+    lookup->active_data = lookup->dynamic_data;
+    lookup->active_size = lookup->dynamic_size;
+  } else {
+    // Restore the original data as the "active" data.
+    lookup->active_data = lookup->original_data;
+    lookup->active_size = lookup->original_size;
+  }
+  // Clear the cached inverted data, if needed.
+  if (lookup->inverted_data) {
+    free(lookup->inverted_data);
+    lookup->inverted_data = NULL;
+  }
+  // Reset the cached "last" values to the initial values.
+  lookup->last_input = DBL_MAX;
+  lookup->last_hit_index = 0;
+}
 void __delete_lookup(Lookup* lookup) {
   if (lookup) {
-    if (lookup->data && lookup->data_is_owned) {
-      free(lookup->data);
+    if (lookup->original_data && lookup->original_data_is_owned) {
+      free(lookup->original_data);
+    }
+    if (lookup->dynamic_data) {
+      free(lookup->dynamic_data);
     }
     if (lookup->inverted_data) {
       free(lookup->inverted_data);
@@ -84,8 +138,8 @@ void __delete_lookup(Lookup* lookup) {
 }
 void __print_lookup(Lookup* lookup) {
   if (lookup) {
-    for (size_t i = 0; i < lookup->n; i++) {
-      printf("(%g, %g)\n", *(lookup->data + 2 * i), *(lookup->data + 2 * i + 1));
+    for (size_t i = 0; i < lookup->active_size; i++) {
+      printf("(%g, %g)\n", *(lookup->active_data + 2 * i), *(lookup->active_data + 2 * i + 1));
     }
   }
 }
@@ -94,12 +148,12 @@ double __lookup(Lookup* lookup, double input, bool use_inverted_data, LookupMode
   // Interpolate the y value from an array of (x,y) pairs.
   // NOTE: The x values are assumed to be monotonically increasing.
 
-  if (lookup == NULL || lookup->n == 0) {
+  if (lookup == NULL || lookup->active_size == 0) {
     return _NA_;
   }
 
-  const double* data = use_inverted_data ? lookup->inverted_data : lookup->data;
-  const size_t max = (lookup->n) * 2;
+  const double* data = use_inverted_data ? lookup->inverted_data : lookup->active_data;
+  const size_t max = (lookup->active_size) * 2;
 
   // Use the cached values for improved lookup performance, except in the case
   // of `LOOKUP INVERT` (since it may not be accurate if calls flip back and forth
@@ -166,12 +220,12 @@ double __get_data_between_times(Lookup* lookup, double input, LookupMode mode) {
   // Interpolate the y value from an array of (x,y) pairs.
   // NOTE: The x values are assumed to be monotonically increasing.
 
-  if (lookup == NULL || lookup->n == 0) {
+  if (lookup == NULL || lookup->active_size == 0) {
     return _NA_;
   }
 
-  const double* data = lookup->data;
-  const size_t n = lookup->n;
+  const double* data = lookup->active_data;
+  const size_t n = lookup->active_size;
   const size_t max = n * 2;
 
   switch (mode) {
@@ -248,10 +302,10 @@ double __get_data_between_times(Lookup* lookup, double input, LookupMode mode) {
 double _LOOKUP_INVERT(Lookup* lookup, double y) {
   if (lookup->inverted_data == NULL) {
     // Invert the matrix and cache it.
-    lookup->inverted_data = malloc(sizeof(double) * 2 * lookup->n);
-    double* pLookup = lookup->data;
+    lookup->inverted_data = malloc(sizeof(double) * 2 * lookup->active_size);
+    double* pLookup = lookup->active_data;
     double* pInvert = lookup->inverted_data;
-    for (size_t i = 0; i < lookup->n; i++) {
+    for (size_t i = 0; i < lookup->active_size; i++) {
       *pInvert++ = *(pLookup + 1);
       *pInvert++ = *pLookup;
       pLookup += 2;
@@ -261,12 +315,12 @@ double _LOOKUP_INVERT(Lookup* lookup, double y) {
 }
 
 double _GAME(Lookup* lookup, double default_value) {
-  if (lookup == NULL || lookup->n <= 0) {
+  if (lookup == NULL || lookup->active_size == 0) {
     // The lookup is NULL or empty, so return the default value
     return default_value;
   }
 
-  double x0 = lookup->data[0];
+  double x0 = lookup->active_data[0];
   if (_time < x0) {
     // The current time is earlier than the first data point, so return the
     // default value

package/src/c/vensim.h

@@ -54,15 +54,44 @@ typedef enum {
 } LookupMode;
 
 typedef struct {
-  double* data;
-  size_t n;
+  // The pointer to the active data buffer.  This will be the same as either
+  // `original_data` or `dynamic_data`, depending on whether the lookup data
+  // is overridden at runtime using `__set_lookup`.
+  double* active_data;
+  // The size (i.e., number of pairs) of the active data.
+  size_t active_size;
+
+  // The pointer to the original data buffer.
+  double* original_data;
+  // The size (i.e., number of pairs) of the original data.
+  size_t original_size;
+  // Whether the `original_data` is owned by this `Lookup` instance.  If `true`,
+  // the `original_data` buffer will be freed by `__delete_lookup`.
+  bool original_data_is_owned;
+
+  // The pointer to the dynamic data buffer.  This will be NULL initially,
+  // and the buffer will be allocated (or grown) by `__set_lookup`.
+  double* dynamic_data;
+  // The size (i.e., number of pairs) of the dynamic data.
+  size_t dynamic_size;
+  // The number of elements in the dynamic data buffer.  The buffer will grow
+  // as needed, so this can be greater than `2 * dynamic_size`.
+  size_t dynamic_data_length;
+
+  // The inverted version of the active data buffer.  This is allocated on demand
+  // only in the case of `LOOKUP INVERT` function calls.
   double* inverted_data;
-  bool data_is_owned;
+
+  // The input value for the last hit.  This is cached for performance so that we
+  // can reduce the amount of linear searching in the common case where `LOOKUP`
+  // input values are monotonically increasing.
   double last_input;
+  // The index for the last hit (see `last_input`).
   size_t last_hit_index;
 } Lookup;
 
 Lookup* __new_lookup(size_t size, bool copy, double* data);
+void __set_lookup(Lookup* lookup, size_t size, double* data);
 void __delete_lookup(Lookup* lookup);
 void __print_lookup(Lookup* lookup);
 

package/src/sde-build.js

@@ -43,7 +43,8 @@ export let build = async (model, opts) => {
     }
   } catch (e) {
     // Exit with a non-zero error code if any step failed
-    console.error(`ERROR: ${e.message}\n`)
+    console.error(e)
+    console.error()
     process.exit(1)
   }
 }

package/src/sde-causes.js

@@ -1,4 +1,6 @@
-import { generateCode, parseModel, preprocessModel } from '@sdeverywhere/compile'
+import { readFileSync } from 'fs'
+
+import { parseAndGenerate } from '@sdeverywhere/compile'
 
 import { modelPathProps, parseSpec } from './utils.js'
 
@@ -14,22 +16,13 @@ let builder = {
 let handler = argv => {
   causes(argv.model, argv.c_varname, argv)
 }
-let causes = (model, varname, opts) => {
+let causes = async (model, varname, opts) => {
   // Get the model name and directory from the model argument.
-  let { modelDirname, modelPathname } = modelPathProps(model)
-  let extData = new Map()
-  let directData = new Map()
+  let { modelDirname, modelPathname, modelName } = modelPathProps(model)
   let spec = parseSpec(opts.spec)
-  // Preprocess model text into parser input.
-  // TODO: The legacy `parseModel` function previously required the `preprocessModel`
-  // step to be performed first, but the new `parseModel` runs the preprocessor
-  // implicitly, so we can remove this step (and can simplify this code to use
-  // `parseAndGenerate` instead)
-  let input = preprocessModel(modelPathname, spec)
   // Parse the model to get variable and subscript information.
-  let parsedModel = parseModel(input, modelDirname)
-  let operations = ['printRefGraph']
-  generateCode(parsedModel, { spec, operations, extData, directData, modelDirname, varname })
+  let input = readFileSync(modelPathname, 'utf8')
+  await parseAndGenerate(input, spec, ['printRefGraph'], modelDirname, modelName, '', varname)
 }
 export default {
   command,

package/src/sde-flatten.js

@@ -1,8 +1,9 @@
+import { readFileSync } from 'fs'
 import path from 'path'
 
 import B from 'bufx'
 
-import { preprocessModel } from '@sdeverywhere/compile'
+import { preprocessVensimModel } from '@sdeverywhere/compile'
 
 import { buildDir, modelPathProps } from './utils.js'
 
@@ -44,16 +45,23 @@ const flatten = async (outFile, inFiles, opts) => {
     // Get the path and short name of the input mdl file
     const inModelProps = modelPathProps(inFile)
 
-    // Preprocess the mdl file and extract the equations
-    const decls = []
-    preprocessModel(inModelProps.modelPathname, undefined, 'runnable', false, decls)
+    // Read the mdl file content
+    const mdlContent = readFileSync(inModelProps.modelPathname, 'utf8')
 
-    // Associate each declaration with the name of the model from which it came
-    for (const decl of decls) {
-      decl.sourceModelName = inModelProps.modelName
+    // Preprocess the mdl file and extract the equations and subscript range definitions
+    const { defs } = preprocessVensimModel(mdlContent)
+
+    for (const def of defs) {
+      // Associate each definition with the name of the model from which it came
+      def.sourceModelName = inModelProps.modelName
+
+      // The object contains the preprocessed definition in a `def` property; copy this
+      // to a `processedDecl` property for clarity (since the existing code used that
+      // property name).
+      def.processedDecl = def.def
     }
 
-    fileDecls[inModelProps.modelName] = decls
+    fileDecls[inModelProps.modelName] = defs
   }
 
   // Collapse so that we have only one equation or declaration per key.
@@ -176,9 +184,8 @@ const flatten = async (outFile, inFiles, opts) => {
   B.open('pp')
   const ENCODING = '{UTF-8}'
   B.emitLine(ENCODING, 'pp')
-  B.emitLine('', 'pp')
   for (const decl of sorted) {
-    B.emitLine(`${decl.processedDecl}\n`, 'pp')
+    B.emitLine(`\n${decl.processedDecl}`, 'pp')
   }
 
   // Write the flattened mdl file to the build directory

package/src/sde-generate.js

@@ -1,7 +1,8 @@
 import path from 'path'
+import { readFileSync } from 'fs'
 import B from 'bufx'
 
-import { parseAndGenerate, preprocessModel } from '@sdeverywhere/compile'
+import { parseAndGenerate, preprocessVensimModel } from '@sdeverywhere/compile'
 
 import { buildDir, modelPathProps, parseSpec } from './utils.js'
 
@@ -28,11 +29,6 @@ export let builder = {
     type: 'boolean',
     alias: 'p'
   },
-  analysis: {
-    describe: 'write a nonexecutable preprocessed model for analysis',
-    type: 'boolean',
-    alias: 'a'
-  },
   spec: {
     describe: 'pathname of the I/O specification JSON file',
     type: 'string',
@@ -64,16 +60,13 @@ export let generate = async (model, opts) => {
   let { modelDirname, modelName, modelPathname } = modelPathProps(model)
   // Ensure the build directory exists.
   let buildDirname = buildDir(opts.builddir, modelDirname)
-  // Preprocess model text into parser input. Stop now if that's all we're doing.
   let spec = parseSpec(opts.spec)
-  // Produce a runnable model with the "runnable" and "preprocess" options.
-  let profile = opts.analysis ? 'analysis' : 'runnable'
-  // Write the preprocessed model and removals if the option is "analysis" or "preprocess".
-  let writeFiles = opts.analysis || opts.preprocess
-  let input = preprocessModel(modelPathname, spec, profile, writeFiles)
-  if (writeFiles) {
+  let mdlContent = readFileSync(modelPathname, 'utf8')
+  if (opts.preprocess) {
+    // Only run the preprocessor.
+    let preprocessed = preprocessModel(mdlContent)
     let outputPathname = path.join(buildDirname, `${modelName}.mdl`)
-    B.write(input, outputPathname)
+    B.write(preprocessed, outputPathname)
     process.exit(0)
   }
   // Parse the model and generate code. If no operation is specified, the code generator will
@@ -94,7 +87,7 @@ export let generate = async (model, opts) => {
   if (opts.refidtest) {
     operations.push('printRefIdTest')
   }
-  await parseAndGenerate(input, spec, operations, modelDirname, modelName, buildDirname)
+  await parseAndGenerate(mdlContent, spec, operations, modelDirname, modelName, buildDirname)
 }
 
 export default {
@@ -104,3 +97,29 @@ export default {
   handler,
   generate
 }
+
+/**
+ * Read and preprocess the given Vensim model content.
+ *
+ * @param {string} mdlContent The mdl content.
+ * @return {string} The preprocessed mdl text.
+ */
+function preprocessModel(mdlContent) {
+  // Run the preprocessor
+  const { defs } = preprocessVensimModel(mdlContent)
+
+  // Sort the definitions alphabetically by key.  This mainly exists for compatibility
+  // with the legacy `sde generate --preprocess` command, which was changed in #55 to
+  // sort definitions alphabetically.
+  defs.sort((a, b) => {
+    return a.key < b.key ? -1 : a.key > b.key ? 1 : 0
+  })
+
+  // Join the preprocessed definitions into a single string
+  let text = '{UTF-8}\n'
+  for (const def of defs) {
+    text += `\n${def.def}\n`
+  }
+
+  return text
+}

package/src/sde-names.js

@@ -1,4 +1,6 @@
-import { parseAndGenerate, preprocessModel, printNames } from '@sdeverywhere/compile'
+import { readFileSync } from 'fs'
+
+import { parseAndGenerate, printNames } from '@sdeverywhere/compile'
 
 import { modelPathProps, parseSpec } from './utils.js'
 
@@ -26,9 +28,8 @@ let names = async (model, namesPathname, opts) => {
   // Get the model name and directory from the model argument.
   let { modelDirname, modelPathname, modelName } = modelPathProps(model)
   let spec = parseSpec(opts.spec)
-  // Preprocess model text into parser input.
-  let input = preprocessModel(modelPathname, spec)
   // Parse the model to get variable and subscript information.
+  let input = readFileSync(modelPathname, 'utf8')
   await parseAndGenerate(input, spec, ['convertNames'], modelDirname, modelName, '')
   // Read each variable name from the names file and convert it.
   printNames(namesPathname, opts.toc ? 'to-c' : 'to-vensim')

package/src/sde-test.js

@@ -66,7 +66,8 @@ export let test = async (model, opts) => {
     }
   } catch (e) {
     // Exit with a non-zero error code if any step failed
-    console.error(`ERROR: ${e.message}\n`)
+    console.error(e)
+    console.error()
     process.exit(1)
   }
 }