@rhinostone/swig-twig 2.0.1 → 2.2.0

package/lib/async/pre-walker.js

@@ -0,0 +1,267 @@
+var utils = require('@rhinostone/swig-core/lib/utils');
+
+/*!
+ * Makes a string safe for a regular expression. Mirrors swig-twig's parser.
+ * @private
+ */
+function escapeRegExp(str) {
+  return str.replace(/[\-\/\\\^$*+?.()|\[\]{}]/g, '\\$&');
+}
+
+/*!
+ * Build the splitter regex from the controls trio. Mirrors the regex the
+ * Twig parser builds at parse-time so the pre-walker chunks the same way
+ * the real parser would.
+ * @private
+ */
+function buildSplitter(controls) {
+  var anyChar = '[\\s\\S]*?',
+    varOpen = escapeRegExp(controls.varControls[0]),
+    varClose = escapeRegExp(controls.varControls[1]),
+    tagOpen = escapeRegExp(controls.tagControls[0]),
+    tagClose = escapeRegExp(controls.tagControls[1]),
+    cmtOpen = escapeRegExp(controls.cmtControls[0]),
+    cmtClose = escapeRegExp(controls.cmtControls[1]);
+  return new RegExp(
+    '(' +
+      tagOpen + anyChar + tagClose + '|' +
+      varOpen + anyChar + varClose + '|' +
+      cmtOpen + anyChar + cmtClose +
+    ')'
+  );
+}
+
+/*!
+ * Strip tag controls and optional whitespace-control markers from a tag
+ * chunk, returning the trimmed tag body (e.g. <code>extends "x.html"</code>).
+ * @private
+ */
+function stripTagBody(chunk, tagOpen, tagClose) {
+  var body = chunk.substr(tagOpen.length, chunk.length - tagOpen.length - tagClose.length);
+  if (body.charAt(0) === '-') {
+    body = body.substr(1);
+  }
+  if (body.charAt(body.length - 1) === '-') {
+    body = body.substr(0, body.length - 1);
+  }
+  return body.replace(/^\s+|\s+$/g, '');
+}
+
+/**
+ * Scan Twig template source for static <code>{% extends|include|import|from "..." %}</code>
+ * targets. Pure function; performs no I/O.
+ *
+ * The scanner mirrors the Twig parser's chunk-splitter so it agrees on
+ * chunk boundaries even under non-default control characters. Dynamic
+ * paths (<code>{% extends parent_var %}</code>) and tag bodies whose
+ * first token isn't a string literal are silently skipped — they remain
+ * on the sync path, which throws appropriately at parse time.
+ *
+ * @example
+ * preWalker.scan('{% from "macros.html" import foo %}', {
+ *   varControls: ['{{', '}}'],
+ *   tagControls: ['{%', '%}'],
+ *   cmtControls: ['{#', '#}'],
+ *   rawTag: 'verbatim',
+ *   keywords: ['extends', 'include', 'import', 'from']
+ * });
+ * // => [{ kind: 'from', path: 'macros.html' }]
+ *
+ * @param  {string} source
+ * @param  {object} opts
+ * @param  {array}  opts.varControls
+ * @param  {array}  opts.tagControls
+ * @param  {array}  opts.cmtControls
+ * @param  {string} opts.rawTag       Tag name that opens verbatim regions
+ *                                    (<code>verbatim</code> for Twig).
+ * @param  {array}  opts.keywords     Keywords whose first quoted argument
+ *                                    is a template path. Twig:
+ *                                    <code>['extends', 'include', 'import',
+ *                                    'from']</code>.
+ * @return {array}                    List of <code>{ kind, path }</code> entries.
+ */
+exports.scan = function (source, opts) {
+  source = source.replace(/\r\n/g, '\n');
+
+  var splitter = buildSplitter(opts),
+    tagOpen = opts.tagControls[0],
+    tagClose = opts.tagControls[1],
+    rawTag = opts.rawTag,
+    endRawTag = 'end' + rawTag,
+    keywordRegex = new RegExp(
+      '^(' + opts.keywords.join('|') + ')\\s+["\\\']([^"\\\']+)["\\\']'
+    ),
+    chunks = source.split(splitter),
+    results = [],
+    inRaw = false,
+    i,
+    chunk,
+    body,
+    name,
+    m;
+
+  for (i = 0; i < chunks.length; i += 1) {
+    chunk = chunks[i];
+    if (typeof chunk !== 'string' || !chunk) {
+      continue;
+    }
+
+    if (!utils.startsWith(chunk, tagOpen) || !utils.endsWith(chunk, tagClose)) {
+      continue;
+    }
+
+    body = stripTagBody(chunk, tagOpen, tagClose);
+    name = body.split(/\s+/)[0];
+
+    if (name === rawTag) {
+      inRaw = true;
+      continue;
+    }
+    if (name === endRawTag) {
+      inRaw = false;
+      continue;
+    }
+    if (inRaw) {
+      continue;
+    }
+
+    m = keywordRegex.exec(body);
+    if (m) {
+      results.push({ kind: m[1], path: m[2] });
+    }
+  }
+
+  return results;
+};
+
+/**
+ * Walk the dependency graph asynchronously starting from <var>entryPath</var>.
+ *
+ * Repeatedly loads, scans, and resolves child template paths in parallel
+ * via the user's async loader, until the dep graph closes. Returns a
+ * Promise resolving to a populated <code>{ resolvedPath: source }</code>
+ * map suitable for backing a memory loader.
+ *
+ * Cycles in the graph are tolerated — once a path is in the map or
+ * pending, subsequent enqueue requests are dropped. The synchronous
+ * renderer's existing circular-extends guard handles cycles at parse
+ * time on the second pass.
+ *
+ * @param  {string}  entryPath  Resolved path of the entry template.
+ * @param  {object}  loader     User loader. Must expose:
+ *                              <code>resolve(to, from)</code> (sync, returns
+ *                              string) and
+ *                              <code>load(id, cb)</code> (async, calls
+ *                              <code>cb(err, source)</code>).
+ * @param  {object}  scanOpts   Pass-through to {@link scan}.
+ * @return {Promise}            Resolves to the populated memory map.
+ */
+exports.walk = function (entryPath, loader, scanOpts) {
+  var memMap = {};
+  var pending = {};
+
+  return new Promise(function (resolve, reject) {
+    var inFlight = 0;
+    var queue = [];
+    var hasError = false;
+
+    function enqueue(path) {
+      if (memMap.hasOwnProperty(path) || pending[path]) {
+        return;
+      }
+      pending[path] = true;
+      queue.push(path);
+    }
+
+    function drain() {
+      while (queue.length > 0 && !hasError) {
+        var path = queue.shift();
+        inFlight += 1;
+        startLoad(path);
+      }
+      if (inFlight === 0 && !hasError && queue.length === 0) {
+        resolve(memMap);
+      }
+    }
+
+    function startLoad(resolvedPath) {
+      loader.load(resolvedPath, function (err, src) {
+        if (hasError) {
+          return;
+        }
+        if (err) {
+          hasError = true;
+          reject(err);
+          return;
+        }
+        if (typeof src !== 'string') {
+          hasError = true;
+          reject(new Error('Async loader returned non-string source for "' + resolvedPath + '"'));
+          return;
+        }
+        memMap[resolvedPath] = src;
+
+        var targets;
+        try {
+          targets = exports.scan(src, scanOpts);
+        } catch (e) {
+          hasError = true;
+          reject(e);
+          return;
+        }
+
+        var i, resolvedChild;
+        for (i = 0; i < targets.length; i += 1) {
+          try {
+            resolvedChild = loader.resolve(targets[i].path, resolvedPath);
+          } catch (e) {
+            hasError = true;
+            reject(e);
+            return;
+          }
+          enqueue(resolvedChild);
+        }
+
+        inFlight -= 1;
+        drain();
+      });
+    }
+
+    enqueue(entryPath);
+    drain();
+  });
+};
+
+/**
+ * Build a sync memory wrapper around a pre-populated
+ * <code>{ resolvedPath: source }</code> map. Delegates <code>resolve</code>
+ * to the user loader so cache keys match what the pre-walker produced.
+ *
+ * @param  {object} userLoader  Original async loader (used for resolve).
+ * @param  {object} memMap      Pre-populated source map.
+ * @return {object}             A loader exposing <code>resolve</code> and
+ *                              <code>load</code>.
+ */
+exports.makeMemoryWrapper = function (userLoader, memMap) {
+  return {
+    resolve: function (to, from) {
+      return userLoader.resolve(to, from);
+    },
+    load: function (id, cb) {
+      var src = memMap[id];
+      if (typeof src !== 'string') {
+        var err = new Error('Pre-walked map missing path: "' + id + '"');
+        if (cb) {
+          cb(err);
+          return;
+        }
+        throw err;
+      }
+      if (cb) {
+        cb(null, src);
+        return;
+      }
+      return src;
+    }
+  };
+};

package/lib/filters.js

@@ -278,26 +278,33 @@ exports.raw.safe = true;
  * @param  {string} [type='html']  Pass `'js'` for JavaScript-safe escaping.
  * @return {string}
  */
+function escapeHtmlRest(ch) {
+  return ch === '<' ? '&lt;' : ch === '>' ? '&gt;' : ch === '"' ? '&quot;' : '&#39;';
+}
+
 exports.escape = function (input, type) {
-  var out = iterateFilter.apply(exports.escape, arguments),
-    inp = input,
-    i = 0,
-    code;
+  var t, inp, out, i, code;
 
-  if (out !== undefined) {
-    return out;
+  if (input === null || input === undefined) {
+    return input;
   }
 
-  if (typeof input !== 'string') {
+  t = typeof input;
+
+  if (t !== 'string') {
+    if (t === 'object') {
+      out = iterateFilter.apply(exports.escape, arguments);
+      if (out !== undefined) {
+        return out;
+      }
+    }
     return input;
   }
 
-  out = '';
-
-  switch (type) {
-  case 'js':
-    inp = inp.replace(/\\/g, '\\u005C');
-    for (i; i < inp.length; i += 1) {
+  if (type === 'js') {
+    inp = input.replace(/\\/g, '\\u005C');
+    out = '';
+    for (i = 0; i < inp.length; i += 1) {
       code = inp.charCodeAt(i);
       if (code < 32) {
         code = code.toString(16).toUpperCase();
@@ -315,14 +322,10 @@ exports.escape = function (input, type) {
       .replace(/\=/g, '\\u003D')
       .replace(/-/g, '\\u002D')
       .replace(/;/g, '\\u003B');
-
-  default:
-    return inp.replace(/&(?!amp;|lt;|gt;|quot;|#39;)/g, '&amp;')
-      .replace(/</g, '&lt;')
-      .replace(/>/g, '&gt;')
-      .replace(/"/g, '&quot;')
-      .replace(/'/g, '&#39;');
   }
+
+  return input.replace(/&(?!amp;|lt;|gt;|quot;|#39;)/g, '&amp;')
+    .replace(/[<>"']/g, escapeHtmlRest);
 };
 exports.e = exports.escape;
 

package/lib/index.js

@@ -14,7 +14,8 @@ var utils = require('@rhinostone/swig-core/lib/utils'),
   parser = require('./parser'),
   _tags = require('./tags'),
   _filters = require('./filters'),
-  _tests = require('./tests');
+  _tests = require('./tests'),
+  preWalker = require('./async/pre-walker');
 
 exports.name = 'twig';
 
@@ -163,6 +164,157 @@ exports.Twig = function (opts) {
   utils.each(_tests, function (fn, name) {
     self.setExtension('_test_' + name, fn);
   });
+
+  function buildScanOpts() {
+    return {
+      varControls: self.options.varControls,
+      tagControls: self.options.tagControls,
+      cmtControls: self.options.cmtControls,
+      rawTag: 'verbatim',
+      keywords: ['extends', 'include', 'import', 'from']
+    };
+  }
+
+  /**
+   * Render a Twig template file asynchronously, supporting async loaders.
+   *
+   * Pre-walks <code>extends</code> / <code>include</code> /
+   * <code>import</code> / <code>from</code> targets in parallel via the
+   * user loader, populates an in-memory map, then runs the existing sync
+   * render pipeline against the populated map. Dynamic paths
+   * (<code>{% extends parent_var %}</code>) are not pre-resolved and will
+   * throw at render time as they would on the sync path.
+   *
+   * @deprecated since 2.2.0 — use {@link Twig#renderFile} with a loader that
+   *   sets <code>loader.async === true</code>. The async-codegen dispatch
+   *   handles dynamic include paths the pre-walker cannot. This method will
+   *   be removed in 3.0.
+   *
+   * @example
+   * twig.setDefaults({ loader: myAsyncLoader });
+   * twig.renderFileAsync('page.twig', { name: 'world' }, function (err, output) {
+   *   if (err) { return done(err); }
+   *   res.end(output);
+   * });
+   *
+   * @param  {string}   pathName  Template path; resolved via the active loader.
+   * @param  {object}   [locals]  Locals to render with.
+   * @param  {Function} cb        Node-style callback <code>(err, output)</code>.
+   * @return {undefined}
+   */
+  this.renderFileAsync = function (pathName, locals, cb) {
+    if (typeof locals === 'function') {
+      cb = locals;
+      locals = undefined;
+    }
+
+    var loader = self.options.loader;
+    var entry;
+
+    try {
+      entry = loader.resolve(pathName);
+    } catch (e) {
+      cb(e);
+      return;
+    }
+
+    preWalker.walk(entry, loader, buildScanOpts()).then(function (memMap) {
+      var memWrapper = preWalker.makeMemoryWrapper(loader, memMap);
+      var origLoader = self.options.loader;
+      self.options.loader = memWrapper;
+      var output, error;
+      try {
+        output = self.renderFile(entry, locals);
+      } catch (e) {
+        error = e;
+      }
+      self.options.loader = origLoader;
+      if (error) {
+        cb(error);
+        return;
+      }
+      cb(null, output);
+    }, function (err) {
+      cb(err);
+    });
+  };
+
+  /**
+   * Compile a Twig template file asynchronously, supporting async loaders.
+   *
+   * Same pre-walk / memory-wrapper / sync-pipeline shape as
+   * {@link Twig#renderFileAsync}. Returns the compiled function (via
+   * <var>cb</var>) that takes a locals object and yields a rendered
+   * string. The returned function captures the pre-walked memory map and
+   * temporarily swaps the loader on each call, so subsequent runtime
+   * <code>include</code>s resolve correctly without re-running the
+   * pre-walk.
+   *
+   * @deprecated since 2.2.0 — use {@link Twig#compileFile} with
+   *   <code>options.codegenMode === 'async'</code> on a loader that sets
+   *   <code>loader.async === true</code>. The returned compiled function
+   *   yields a <code>Promise&lt;{output, exports}&gt;</code> instead of a
+   *   string. This method will be removed in 3.0.
+   *
+   * @example
+   * twig.compileFileAsync('page.twig', {}, function (err, fn) {
+   *   if (err) { return done(err); }
+   *   res.end(fn({ name: 'world' }));
+   * });
+   *
+   * @param  {string}   pathName  Template path.
+   * @param  {object}   [options] Compilation options.
+   * @param  {Function} cb        Node-style callback <code>(err, fn)</code>.
+   * @return {undefined}
+   */
+  this.compileFileAsync = function (pathName, options, cb) {
+    if (typeof options === 'function') {
+      cb = options;
+      options = {};
+    }
+
+    var loader = self.options.loader;
+    var entry;
+
+    try {
+      entry = loader.resolve(pathName);
+    } catch (e) {
+      cb(e);
+      return;
+    }
+
+    preWalker.walk(entry, loader, buildScanOpts()).then(function (memMap) {
+      var memWrapper = preWalker.makeMemoryWrapper(loader, memMap);
+      var origLoader = self.options.loader;
+      self.options.loader = memWrapper;
+      var compiled, error;
+      try {
+        compiled = self.compileFile(entry, options);
+      } catch (e) {
+        error = e;
+      }
+      self.options.loader = origLoader;
+      if (error) {
+        cb(error);
+        return;
+      }
+      var wrapped = function (locals) {
+        var origInner = self.options.loader;
+        self.options.loader = memWrapper;
+        try {
+          var output = compiled(locals);
+          self.options.loader = origInner;
+          return output;
+        } catch (e) {
+          self.options.loader = origInner;
+          throw e;
+        }
+      };
+      cb(null, wrapped);
+    }, function (err) {
+      cb(err);
+    });
+  };
 };
 
 /*!
@@ -176,8 +328,10 @@ exports.parseFile = defaultInstance.parseFile;
 exports.precompile = defaultInstance.precompile;
 exports.compile = defaultInstance.compile;
 exports.compileFile = defaultInstance.compileFile;
+exports.compileFileAsync = defaultInstance.compileFileAsync;
 exports.render = defaultInstance.render;
 exports.renderFile = defaultInstance.renderFile;
+exports.renderFileAsync = defaultInstance.renderFileAsync;
 exports.run = defaultInstance.run;
 exports.invalidateCache = defaultInstance.invalidateCache;
 

package/lib/tags/from.js

@@ -30,6 +30,7 @@
  */
 
 var utils = require('@rhinostone/swig-core/lib/utils');
+var ir = require('@rhinostone/swig-core/lib/ir');
 var backend = require('@rhinostone/swig-core/lib/backend');
 var _dangerousProps = require('@rhinostone/swig-core/lib/security').dangerousProps;
 
@@ -135,11 +136,20 @@ exports.parse = function (str, line, parser, types, stack, opts, swig, token) {
     consume();
   }
 
+  var path = pathTok.match.replace(/^['"]|['"]$/g, '');
+
+  if (opts && opts.codegenMode === 'async') {
+    // Phase 2 (#T22): async mode skips parse-time parseFile + macro
+    // pre-render. compile() emits IRFromImportDeferred; runtime resolves
+    // the template via _swig.getTemplate and binds each entry on _ctx.
+    token.args = [{ path: path, entries: entries }];
+    return true;
+  }
+
   if (!swig || typeof swig.parseFile !== 'function') {
     utils.throwError('"from" tag requires an engine context with a loader', line, opts.filename);
   }
 
-  var path = pathTok.match.replace(/^['"]|['"]$/g, '');
   var parseOpts = { resolveFrom: opts.filename };
   var compileOpts = utils.extend({}, opts, parseOpts);
   var parsed = swig.parseFile(path, parseOpts);
@@ -195,7 +205,26 @@ exports.parse = function (str, line, parser, types, stack, opts, swig, token) {
  *                  `_ctx.<aliasName>`. Backend lifts it into
  *                  `IRLegacyJS`.
  */
-exports.compile = function (compiler, args) {
+exports.compile = function (compiler, args, content, parents, options) {
+  // Phase 2 (#T22): async-codegen branch. Parse stashed a single bundle
+  // `[{path, entries: [{origName, aliasName}, ...]}]` in async mode (no
+  // macro pre-render); emit IRFromImportDeferred so the backend's
+  // `_swig.getTemplate` + per-entry `_ctx.<bind>` assignment happens at
+  // runtime.
+  if (options && options.codegenMode === 'async') {
+    var bundle = args[0];
+    var imports = utils.map(bundle.entries, function (e) {
+      return {
+        name: e.origName,
+        alias: e.aliasName === e.origName ? null : e.aliasName
+      };
+    });
+    return ir.fromImportDeferred(
+      ir.literal('string', bundle.path),
+      imports,
+      options.filename || ''
+    );
+  }
   var allOrigNames = utils.map(args, function (arg) { return arg.origName; }).join('|');
   var replacements = utils.map(args, function (arg) {
     return {

package/lib/tags/import.js

@@ -31,6 +31,7 @@
  */
 
 var utils = require('@rhinostone/swig-core/lib/utils');
+var ir = require('@rhinostone/swig-core/lib/ir');
 var backend = require('@rhinostone/swig-core/lib/backend');
 var _dangerousProps = require('@rhinostone/swig-core/lib/security').dangerousProps;
 
@@ -109,11 +110,20 @@ exports.parse = function (str, line, parser, types, stack, opts, swig, token) {
     utils.throwError('Unexpected token "' + peek().match + '" after alias in "import" tag', line, opts.filename);
   }
 
+  var path = pathTok.match.replace(/^['"]|['"]$/g, '');
+
+  if (opts && opts.codegenMode === 'async') {
+    // Phase 2 (#T22): async mode skips the parse-time parseFile + macro
+    // pre-render. compile() emits IRImportDeferred; runtime resolves the
+    // template via _swig.getTemplate and binds .exports under the alias.
+    token.args = [path, aliasTok.match];
+    return true;
+  }
+
   if (!swig || typeof swig.parseFile !== 'function') {
     utils.throwError('"import" tag requires an engine context with a loader', line, opts.filename);
   }
 
-  var path = pathTok.match.replace(/^['"]|['"]$/g, '');
   var parseOpts = { resolveFrom: opts.filename };
   var compileOpts = utils.extend({}, opts, parseOpts);
   var parsed = swig.parseFile(path, parseOpts);
@@ -145,7 +155,17 @@ exports.parse = function (str, line, parser, types, stack, opts, swig, token) {
  * @return {string} JS source that initialises `_ctx.<alias>` and
  *                  assigns every imported macro into it.
  */
-exports.compile = function (compiler, args) {
+exports.compile = function (compiler, args, content, parents, options) {
+  // Phase 2 (#T22): async-codegen branch. Parse stashed `[path, alias]`
+  // in async mode (no macro pre-render); emit IRImportDeferred so the
+  // backend's `_swig.getTemplate` + `.exports` bind happens at runtime.
+  if (options && options.codegenMode === 'async') {
+    return ir.importDeferred(
+      ir.literal('string', args[0]),
+      args[args.length - 1],
+      options.filename || ''
+    );
+  }
   var ctx = args.pop();
   var allMacros = utils.map(args, function (arg) { return arg.name; }).join('|');
   var out = '_ctx.' + ctx + ' = {};\n  var _output = "";\n';

package/lib/tags/include.js

@@ -163,8 +163,17 @@ function sliceTrim(tokens, start, end, types) {
  * context emission, isolated-vs-merged selector, resolveFrom, optional
  * try/catch for ignoreMissing).
  *
- * @return {object} IRInclude node from `token.irExpr`.
+ * In async codegen mode (`options.codegenMode === 'async'`), derive an
+ * `IRIncludeDeferred` from the same fields so the backend routes through
+ * the `_swig.getTemplate` + `await` deferred-resolution path instead of
+ * the sync `_swig.compileFile` call.
+ *
+ * @return {object} IRInclude or IRIncludeDeferred node.
  */
 exports.compile = function (compiler, args, content, parents, options, blockName, token) {
+  if (options && options.codegenMode === 'async') {
+    var i = token.irExpr;
+    return ir.includeDeferred(i.path, i.context, i.isolated, i.ignoreMissing, i.resolveFrom);
+  }
   return token.irExpr;
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rhinostone/swig-twig",
-  "version": "2.0.1",
+  "version": "2.2.0",
   "description": "Twig-syntax frontend for the @rhinostone/swig-core template engine. Part of the @rhinostone/swig multi-flavor family.",
   "keywords": [
     "template",
@@ -22,7 +22,7 @@
     "node": ">=12"
   },
   "peerDependencies": {
-    "@rhinostone/swig-core": "2.0.1"
+    "@rhinostone/swig-core": "2.2.0"
   },
   "publishConfig": {
     "access": "public"