@rhinostone/swig 2.0.1 → 2.2.0

package/lib/async/pre-walker.js

@@ -0,0 +1,279 @@
+var utils = require('../utils');
+
+/*!
+ * Makes a string safe for a regular expression. Mirrors lib/parser.js.
+ * @private
+ */
+function escapeRegExp(str) {
+  return str.replace(/[\-\/\\\^$*+?.()|\[\]{}]/g, '\\$&');
+}
+
+/*!
+ * Build the splitter regex from the controls trio. Mirrors the regex that
+ * parser.parse() builds at parse-time so the pre-walker chunks the same
+ * way the real parser would.
+ * @private
+ */
+function buildSplitter(controls) {
+  var anyChar = '[\\s\\S]*?',
+    varOpen = escapeRegExp(controls.varControls[0]),
+    varClose = escapeRegExp(controls.varControls[1]),
+    tagOpen = escapeRegExp(controls.tagControls[0]),
+    tagClose = escapeRegExp(controls.tagControls[1]),
+    cmtOpen = escapeRegExp(controls.cmtControls[0]),
+    cmtClose = escapeRegExp(controls.cmtControls[1]);
+  return new RegExp(
+    '(' +
+      tagOpen + anyChar + tagClose + '|' +
+      varOpen + anyChar + varClose + '|' +
+      cmtOpen + anyChar + cmtClose +
+    ')'
+  );
+}
+
+/*!
+ * Strip tag controls and optional whitespace-control markers from a tag
+ * chunk, returning the trimmed tag body (e.g. `extends "x.html"`).
+ * @private
+ */
+function stripTagBody(chunk, tagOpen, tagClose) {
+  var body = chunk.substr(tagOpen.length, chunk.length - tagOpen.length - tagClose.length);
+  if (body.charAt(0) === '-') {
+    body = body.substr(1);
+  }
+  if (body.charAt(body.length - 1) === '-') {
+    body = body.substr(0, body.length - 1);
+  }
+  return body.replace(/^\s+|\s+$/g, '');
+}
+
+/**
+ * Scan template source for static `{% extends|include|import|from "..." %}`
+ * targets. Pure function; performs no I/O.
+ *
+ * The scanner mirrors the real parser's chunk-splitter so it agrees on
+ * chunk boundaries even under non-default control characters. Dynamic
+ * paths (`{% extends parent_var %}`) and tag bodies whose first token
+ * isn't a string literal are silently skipped — they remain on the sync
+ * path, which throws appropriately at parse time.
+ *
+ * @example
+ * preWalker.scan('{% extends "layout.html" %}{% include "x" %}', {
+ *   varControls: ['{{', '}}'],
+ *   tagControls: ['{%', '%}'],
+ *   cmtControls: ['{#', '#}'],
+ *   rawTag: 'raw',
+ *   keywords: ['extends', 'include', 'import']
+ * });
+ * // => [
+ * //   { kind: 'extends', path: 'layout.html' },
+ * //   { kind: 'include', path: 'x' }
+ * // ]
+ *
+ * @param  {string} source
+ * @param  {object} opts
+ * @param  {array}  opts.varControls  e.g. <code>['{{', '}}']</code>.
+ * @param  {array}  opts.tagControls  e.g. <code>['{%', '%}']</code>.
+ * @param  {array}  opts.cmtControls  e.g. <code>['{#', '#}']</code>.
+ * @param  {string} opts.rawTag       Tag name that opens verbatim regions
+ *                                    (<code>raw</code> for native swig).
+ * @param  {array}  opts.keywords     Keywords whose first quoted argument is
+ *                                    a template path. Native swig:
+ *                                    <code>['extends', 'include', 'import']</code>.
+ * @return {array}                    List of <code>{ kind, path }</code> entries.
+ */
+exports.scan = function (source, opts) {
+  source = source.replace(/\r\n/g, '\n');
+
+  var splitter = buildSplitter(opts),
+    tagOpen = opts.tagControls[0],
+    tagClose = opts.tagControls[1],
+    rawTag = opts.rawTag,
+    endRawTag = 'end' + rawTag,
+    keywordRegex = new RegExp(
+      '^(' + opts.keywords.join('|') + ')\\s+["\\\']([^"\\\']+)["\\\']'
+    ),
+    chunks = source.split(splitter),
+    results = [],
+    inRaw = false,
+    i,
+    chunk,
+    body,
+    name,
+    m;
+
+  for (i = 0; i < chunks.length; i += 1) {
+    chunk = chunks[i];
+    if (typeof chunk !== 'string' || !chunk) {
+      continue;
+    }
+
+    if (!utils.startsWith(chunk, tagOpen) || !utils.endsWith(chunk, tagClose)) {
+      continue;
+    }
+
+    body = stripTagBody(chunk, tagOpen, tagClose);
+    name = body.split(/\s+/)[0];
+
+    if (name === rawTag) {
+      inRaw = true;
+      continue;
+    }
+    if (name === endRawTag) {
+      inRaw = false;
+      continue;
+    }
+    if (inRaw) {
+      continue;
+    }
+
+    m = keywordRegex.exec(body);
+    if (m) {
+      results.push({ kind: m[1], path: m[2] });
+    }
+  }
+
+  return results;
+};
+
+/**
+ * Walk the dependency graph asynchronously starting from <var>entryPath</var>.
+ *
+ * Repeatedly loads, scans, and resolves child template paths in parallel
+ * via the user's async loader, until the dep graph closes. Returns a
+ * Promise resolving to a populated <code>{ resolvedPath: source }</code>
+ * map suitable for backing a memory loader.
+ *
+ * Cycles in the graph are tolerated — once a path is in the map or
+ * pending, subsequent enqueue requests are dropped. The synchronous
+ * renderer's existing circular-extends guard handles cycles at parse
+ * time on the second pass.
+ *
+ * @example
+ * preWalker.walk('/abs/entry.html', userLoader, scanOpts).then(function (memMap) {
+ *   // memMap = { '/abs/entry.html': '...', '/abs/layout.html': '...', ... }
+ * });
+ *
+ * @param  {string}  entryPath  Resolved path of the entry template.
+ * @param  {object}  loader     User loader. Must expose:
+ *                              <code>resolve(to, from)</code> (sync, returns
+ *                              string) and
+ *                              <code>load(id, cb)</code> (async, calls
+ *                              <code>cb(err, source)</code>).
+ * @param  {object}  scanOpts   Pass-through to {@link scan}.
+ * @return {Promise}            Resolves to the populated memory map.
+ */
+exports.walk = function (entryPath, loader, scanOpts) {
+  var memMap = {};
+  var pending = {};
+
+  return new Promise(function (resolve, reject) {
+    var inFlight = 0;
+    var queue = [];
+    var hasError = false;
+
+    function enqueue(path) {
+      if (memMap.hasOwnProperty(path) || pending[path]) {
+        return;
+      }
+      pending[path] = true;
+      queue.push(path);
+    }
+
+    function drain() {
+      while (queue.length > 0 && !hasError) {
+        var path = queue.shift();
+        inFlight += 1;
+        startLoad(path);
+      }
+      if (inFlight === 0 && !hasError && queue.length === 0) {
+        resolve(memMap);
+      }
+    }
+
+    function startLoad(resolvedPath) {
+      loader.load(resolvedPath, function (err, src) {
+        if (hasError) {
+          return;
+        }
+        if (err) {
+          hasError = true;
+          reject(err);
+          return;
+        }
+        if (typeof src !== 'string') {
+          hasError = true;
+          reject(new Error('Async loader returned non-string source for "' + resolvedPath + '"'));
+          return;
+        }
+        memMap[resolvedPath] = src;
+
+        var targets;
+        try {
+          targets = exports.scan(src, scanOpts);
+        } catch (e) {
+          hasError = true;
+          reject(e);
+          return;
+        }
+
+        var i, resolvedChild;
+        for (i = 0; i < targets.length; i += 1) {
+          try {
+            resolvedChild = loader.resolve(targets[i].path, resolvedPath);
+          } catch (e) {
+            hasError = true;
+            reject(e);
+            return;
+          }
+          enqueue(resolvedChild);
+        }
+
+        inFlight -= 1;
+        drain();
+      });
+    }
+
+    enqueue(entryPath);
+    drain();
+  });
+};
+
+/**
+ * Build a sync memory wrapper around a pre-populated
+ * <code>{ resolvedPath: source }</code> map. Delegates <code>resolve</code>
+ * to the user loader so cache keys match what the pre-walker produced.
+ *
+ * @example
+ * var mem = preWalker.makeMemoryWrapper(userLoader, memMap);
+ * swig.options.loader = mem;
+ * swig.renderFile('/abs/entry.html', locals);  // sync, hits memMap
+ *
+ * @param  {object} userLoader  Original async loader (used for resolve).
+ * @param  {object} memMap      Pre-populated source map.
+ * @return {object}             A loader exposing <code>resolve</code> and
+ *                              <code>load</code>.
+ */
+exports.makeMemoryWrapper = function (userLoader, memMap) {
+  return {
+    resolve: function (to, from) {
+      return userLoader.resolve(to, from);
+    },
+    load: function (id, cb) {
+      var src = memMap[id];
+      if (typeof src !== 'string') {
+        var err = new Error('Pre-walked map missing path: "' + id + '"');
+        if (cb) {
+          cb(err);
+          return;
+        }
+        throw err;
+      }
+      if (cb) {
+        cb(null, src);
+        return;
+      }
+      return src;
+    }
+  };
+};

package/lib/filters.js

@@ -117,26 +117,33 @@ exports["default"] = function (input, def) {
  * @param  {string} [type='html']   If you pass the string js in as the type, output will be escaped so that it is safe for JavaScript execution.
  * @return {string}         Escaped string.
  */
+function escapeHtmlRest(ch) {
+  return ch === '<' ? '&lt;' : ch === '>' ? '&gt;' : ch === '"' ? '&quot;' : '&#39;';
+}
+
 exports.escape = function (input, type) {
-  var out = iterateFilter.apply(exports.escape, arguments),
-    inp = input,
-    i = 0,
-    code;
+  var t, inp, out, i, code;
 
-  if (out !== undefined) {
-    return out;
+  if (input === null || input === undefined) {
+    return input;
   }
 
-  if (typeof input !== 'string') {
+  t = typeof input;
+
+  if (t !== 'string') {
+    if (t === 'object') {
+      out = iterateFilter.apply(exports.escape, arguments);
+      if (out !== undefined) {
+        return out;
+      }
+    }
     return input;
   }
 
-  out = '';
-
-  switch (type) {
-  case 'js':
-    inp = inp.replace(/\\/g, '\\u005C');
-    for (i; i < inp.length; i += 1) {
+  if (type === 'js') {
+    inp = input.replace(/\\/g, '\\u005C');
+    out = '';
+    for (i = 0; i < inp.length; i += 1) {
       code = inp.charCodeAt(i);
       if (code < 32) {
         code = code.toString(16).toUpperCase();
@@ -154,14 +161,10 @@ exports.escape = function (input, type) {
       .replace(/\=/g, '\\u003D')
       .replace(/-/g, '\\u002D')
       .replace(/;/g, '\\u003B');
-
-  default:
-    return inp.replace(/&(?!amp;|lt;|gt;|quot;|#39;)/g, '&amp;')
-      .replace(/</g, '&lt;')
-      .replace(/>/g, '&gt;')
-      .replace(/"/g, '&quot;')
-      .replace(/'/g, '&#39;');
   }
+
+  return input.replace(/&(?!amp;|lt;|gt;|quot;|#39;)/g, '&amp;')
+    .replace(/[<>"']/g, escapeHtmlRest);
 };
 exports.e = exports.escape;
 

package/lib/swig.js

@@ -4,16 +4,17 @@ var utils = require('./utils'),
   parser = require('./parser'),
   dateformatter = require('./dateformatter'),
   loaders = require('./loaders'),
+  preWalker = require('./async/pre-walker'),
   engine = require('@rhinostone/swig-core/lib/engine');
 
 /**
  * Swig version number as a string.
  * @example
- * if (swig.version === "2.0.1") { ... }
+ * if (swig.version === "2.2.0") { ... }
  *
  * @type {String}
  */
-exports.version = "2.0.1";
+exports.version = "2.2.0";
 
 /**
  * Swig Options Object. This object can be passed to many of the API-level Swig methods to control various aspects of the engine. All keys are optional.
@@ -177,6 +178,158 @@ exports.Swig = function (opts) {
       utils.throwError(err, null, options.filename);
     }
   });
+
+  var self = this;
+
+  function buildScanOpts() {
+    return {
+      varControls: self.options.varControls,
+      tagControls: self.options.tagControls,
+      cmtControls: self.options.cmtControls,
+      rawTag: 'raw',
+      keywords: ['extends', 'include', 'import']
+    };
+  }
+
+  /**
+   * Render a template file asynchronously, supporting async loaders.
+   *
+   * Pre-walks <var>extends</var> / <var>include</var> / <var>import</var>
+   * targets in parallel via the user loader, populates an in-memory map,
+   * then runs the existing sync render pipeline against the populated map.
+   * Dynamic paths (e.g. <code>{% extends parent_var %}</code>) are not
+   * pre-resolved and will throw at render time as they would on the sync
+   * path.
+   *
+   * @deprecated since 2.2.0 — use {@link Swig#renderFile} with a loader that
+   *   sets <code>loader.async === true</code>. The async-codegen dispatch
+   *   handles dynamic include paths the pre-walker cannot. This method will
+   *   be removed in 3.0.
+   *
+   * @example
+   * swig.setDefaults({ loader: myAsyncLoader });
+   * swig.renderFileAsync('page.html', { name: 'world' }, function (err, output) {
+   *   if (err) { return done(err); }
+   *   res.end(output);
+   * });
+   *
+   * @param  {string}   pathName  Template path; resolved via the active loader.
+   * @param  {object}   [locals]  Locals to render with.
+   * @param  {Function} cb        Node-style callback <code>(err, output)</code>.
+   * @return {undefined}
+   */
+  this.renderFileAsync = function (pathName, locals, cb) {
+    if (typeof locals === 'function') {
+      cb = locals;
+      locals = undefined;
+    }
+
+    var loader = self.options.loader;
+    var entry;
+
+    try {
+      entry = loader.resolve(pathName);
+    } catch (e) {
+      cb(e);
+      return;
+    }
+
+    preWalker.walk(entry, loader, buildScanOpts()).then(function (memMap) {
+      var memWrapper = preWalker.makeMemoryWrapper(loader, memMap);
+      var origLoader = self.options.loader;
+      self.options.loader = memWrapper;
+      var output, error;
+      try {
+        output = self.renderFile(entry, locals);
+      } catch (e) {
+        error = e;
+      }
+      self.options.loader = origLoader;
+      if (error) {
+        cb(error);
+        return;
+      }
+      cb(null, output);
+    }, function (err) {
+      cb(err);
+    });
+  };
+
+  /**
+   * Compile a template file asynchronously, supporting async loaders.
+   *
+   * Same pre-walk / memory-wrapper / sync-pipeline shape as
+   * {@link Swig#renderFileAsync}. Returns the compiled function (via
+   * <var>cb</var>) that takes a locals object and yields a rendered
+   * string. The returned function captures the pre-walked memory map and
+   * temporarily swaps the loader on each call, so subsequent runtime
+   * <var>include</var>s resolve correctly without re-running the pre-walk.
+   *
+   * @deprecated since 2.2.0 — use {@link Swig#compileFile} with
+   *   <code>options.codegenMode === 'async'</code> on a loader that sets
+   *   <code>loader.async === true</code>. The returned compiled function
+   *   yields a <code>Promise&lt;{output, exports}&gt;</code> instead of a
+   *   string. This method will be removed in 3.0.
+   *
+   * @example
+   * swig.compileFileAsync('page.html', {}, function (err, fn) {
+   *   if (err) { return done(err); }
+   *   res.end(fn({ name: 'world' }));
+   * });
+   *
+   * @param  {string}   pathName  Template path.
+   * @param  {object}   [options] Compilation options.
+   * @param  {Function} cb        Node-style callback <code>(err, fn)</code>.
+   * @return {undefined}
+   */
+  this.compileFileAsync = function (pathName, options, cb) {
+    if (typeof options === 'function') {
+      cb = options;
+      options = {};
+    }
+
+    var loader = self.options.loader;
+    var entry;
+
+    try {
+      entry = loader.resolve(pathName);
+    } catch (e) {
+      cb(e);
+      return;
+    }
+
+    preWalker.walk(entry, loader, buildScanOpts()).then(function (memMap) {
+      var memWrapper = preWalker.makeMemoryWrapper(loader, memMap);
+      var origLoader = self.options.loader;
+      self.options.loader = memWrapper;
+      var compiled, error;
+      try {
+        compiled = self.compileFile(entry, options);
+      } catch (e) {
+        error = e;
+      }
+      self.options.loader = origLoader;
+      if (error) {
+        cb(error);
+        return;
+      }
+      var wrapped = function (locals) {
+        var origInner = self.options.loader;
+        self.options.loader = memWrapper;
+        try {
+          var output = compiled(locals);
+          self.options.loader = origInner;
+          return output;
+        } catch (e) {
+          self.options.loader = origInner;
+          throw e;
+        }
+      };
+      cb(null, wrapped);
+    }, function (err) {
+      cb(err);
+    });
+  };
 };
 
 /*!
@@ -190,8 +343,10 @@ exports.parseFile = defaultInstance.parseFile;
 exports.precompile = defaultInstance.precompile;
 exports.compile = defaultInstance.compile;
 exports.compileFile = defaultInstance.compileFile;
+exports.compileFileAsync = defaultInstance.compileFileAsync;
 exports.render = defaultInstance.render;
 exports.renderFile = defaultInstance.renderFile;
+exports.renderFileAsync = defaultInstance.renderFileAsync;
 exports.run = defaultInstance.run;
 exports.invalidateCache = defaultInstance.invalidateCache;
 exports.loaders = loaders;

package/lib/tags/import.js

@@ -1,4 +1,5 @@
 var utils = require('../utils'),
+  ir = require('@rhinostone/swig-core/lib/ir'),
   backend = require('@rhinostone/swig-core/lib/backend');
 
 // CVE-2023-25345: prototype-chain properties that must not be used as import
@@ -26,7 +27,19 @@ var _dangerousProps = require('@rhinostone/swig-core/lib/security').dangerousPro
  * @param {literal}     as        Literally, "as".
  * @param {literal}     varname   Local-accessible object name to assign the macros to.
  */
-exports.compile = function (compiler, args) {
+exports.compile = function (compiler, args, content, parents, options) {
+  // Phase 2 (#T22): async-codegen branch. Parse stashed `[{path}, alias]`
+  // (no macro pre-render in async mode); emit IRImportDeferred so the
+  // backend's `_swig.getTemplate` + `.exports` bind happens at runtime.
+  if (options && options.codegenMode === 'async') {
+    var asyncAlias = args[args.length - 1];
+    var asyncPath = args[0].path;
+    return ir.importDeferred(
+      ir.literal('string', asyncPath),
+      asyncAlias,
+      options.filename || ''
+    );
+  }
   var ctx = args.pop(),
     allMacros = utils.map(args, function (arg) {
       return arg.name;
@@ -56,37 +69,45 @@ exports.parse = function (str, line, parser, types, stack, opts, swig) {
   var compiler = require('../parser').compile,
     parseOpts = { resolveFrom: opts.filename },
     compileOpts = utils.extend({}, opts, parseOpts),
-    tokens,
+    isAsync = !!(opts && opts.codegenMode === 'async'),
+    importPath,
     ctx;
 
   parser.on(types.STRING, function (token) {
     var self = this;
-    if (!tokens) {
-      tokens = swig.parseFile(token.match.replace(/^("|')|("|')$/g, ''), parseOpts).tokens;
-      utils.each(tokens, function (token) {
-        var out = '',
-          macroName;
-        if (!token || token.name !== 'macro' || !token.compile) {
-          return;
-        }
-        macroName = token.args[0];
-        // Phase 2 (#T15): macro.compile now returns an IRMacro node
-        // rather than a JS source string. Render it through the shared
-        // backend so import.js still gets the JS source it performs
-        // regex-surgery on for namespace-prefixing. The +'\n' trailing
-        // newline matches the pre-Phase-2 compile output exactly.
-        out += backend.compile([token.compile(compiler, token.args, token.content, [], compileOpts)], [], compileOpts) + '\n';
-        self.out.push({compiled: out, name: macroName});
-      });
+    if (importPath !== undefined) {
+      throw new Error('Unexpected string ' + token.match + ' on line ' + line + '.');
+    }
+    importPath = token.match.replace(/^("|')|("|')$/g, '');
+
+    if (isAsync) {
+      // Async mode: skip the sync parseFile + macro pre-render. Stash
+      // just the path; compile() emits IRImportDeferred.
+      self.out.push({ path: importPath });
       return;
     }
 
-    throw new Error('Unexpected string ' + token.match + ' on line ' + line + '.');
+    var tokens = swig.parseFile(importPath, parseOpts).tokens;
+    utils.each(tokens, function (token) {
+      var out = '',
+        macroName;
+      if (!token || token.name !== 'macro' || !token.compile) {
+        return;
+      }
+      macroName = token.args[0];
+      // Phase 2 (#T15): macro.compile now returns an IRMacro node
+      // rather than a JS source string. Render it through the shared
+      // backend so import.js still gets the JS source it performs
+      // regex-surgery on for namespace-prefixing. The +'\n' trailing
+      // newline matches the pre-Phase-2 compile output exactly.
+      out += backend.compile([token.compile(compiler, token.args, token.content, [], compileOpts)], [], compileOpts) + '\n';
+      self.out.push({compiled: out, name: macroName});
+    });
   });
 
   parser.on(types.VAR, function (token) {
     var self = this;
-    if (!tokens || ctx) {
+    if (importPath === undefined || ctx) {
       throw new Error('Unexpected variable "' + token.match + '" on line ' + line + '.');
     }
 

package/lib/tags/include.js

@@ -48,6 +48,10 @@ exports.compile = function (compiler, args, content, parents, options, blockName
     }
   }
 
+  if (options && options.codegenMode === 'async') {
+    return ir.includeDeferred(file, w || undefined, !!onlyCtx, !!ignoreMissing, parentFile);
+  }
+
   return ir.include(file, w || undefined, !!onlyCtx, !!ignoreMissing, parentFile);
 };
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rhinostone/swig",
-  "version": "2.0.1",
+  "version": "2.2.0",
   "description": "A simple, powerful, and extendable templating engine for node.js and browsers, similar to Django, Jinja2, and Twig.",
   "keywords": [
     "template",
@@ -21,7 +21,7 @@
     "Rhinostone <contact@gina.io>"
   ],
   "dependencies": {
-    "@rhinostone/swig-core": "2.0.1",
+    "@rhinostone/swig-core": "2.2.0",
     "terser": "^5.46.1",
     "yargs": "^17.7.2"
   },