@rhinostone/swig-twig 2.0.1 → 2.1.0

package/lib/async/pre-walker.js

@@ -0,0 +1,267 @@
+var utils = require('@rhinostone/swig-core/lib/utils');
+
+/*!
+ * Makes a string safe for a regular expression. Mirrors swig-twig's parser.
+ * @private
+ */
+function escapeRegExp(str) {
+  return str.replace(/[\-\/\\\^$*+?.()|\[\]{}]/g, '\\$&');
+}
+
+/*!
+ * Build the splitter regex from the controls trio. Mirrors the regex the
+ * Twig parser builds at parse-time so the pre-walker chunks the same way
+ * the real parser would.
+ * @private
+ */
+function buildSplitter(controls) {
+  var anyChar = '[\\s\\S]*?',
+    varOpen = escapeRegExp(controls.varControls[0]),
+    varClose = escapeRegExp(controls.varControls[1]),
+    tagOpen = escapeRegExp(controls.tagControls[0]),
+    tagClose = escapeRegExp(controls.tagControls[1]),
+    cmtOpen = escapeRegExp(controls.cmtControls[0]),
+    cmtClose = escapeRegExp(controls.cmtControls[1]);
+  return new RegExp(
+    '(' +
+      tagOpen + anyChar + tagClose + '|' +
+      varOpen + anyChar + varClose + '|' +
+      cmtOpen + anyChar + cmtClose +
+    ')'
+  );
+}
+
+/*!
+ * Strip tag controls and optional whitespace-control markers from a tag
+ * chunk, returning the trimmed tag body (e.g. <code>extends "x.html"</code>).
+ * @private
+ */
+function stripTagBody(chunk, tagOpen, tagClose) {
+  var body = chunk.substr(tagOpen.length, chunk.length - tagOpen.length - tagClose.length);
+  if (body.charAt(0) === '-') {
+    body = body.substr(1);
+  }
+  if (body.charAt(body.length - 1) === '-') {
+    body = body.substr(0, body.length - 1);
+  }
+  return body.replace(/^\s+|\s+$/g, '');
+}
+
+/**
+ * Scan Twig template source for static <code>{% extends|include|import|from "..." %}</code>
+ * targets. Pure function; performs no I/O.
+ *
+ * The scanner mirrors the Twig parser's chunk-splitter so it agrees on
+ * chunk boundaries even under non-default control characters. Dynamic
+ * paths (<code>{% extends parent_var %}</code>) and tag bodies whose
+ * first token isn't a string literal are silently skipped — they remain
+ * on the sync path, which throws appropriately at parse time.
+ *
+ * @example
+ * preWalker.scan('{% from "macros.html" import foo %}', {
+ *   varControls: ['{{', '}}'],
+ *   tagControls: ['{%', '%}'],
+ *   cmtControls: ['{#', '#}'],
+ *   rawTag: 'verbatim',
+ *   keywords: ['extends', 'include', 'import', 'from']
+ * });
+ * // => [{ kind: 'from', path: 'macros.html' }]
+ *
+ * @param  {string} source
+ * @param  {object} opts
+ * @param  {array}  opts.varControls
+ * @param  {array}  opts.tagControls
+ * @param  {array}  opts.cmtControls
+ * @param  {string} opts.rawTag       Tag name that opens verbatim regions
+ *                                    (<code>verbatim</code> for Twig).
+ * @param  {array}  opts.keywords     Keywords whose first quoted argument
+ *                                    is a template path. Twig:
+ *                                    <code>['extends', 'include', 'import',
+ *                                    'from']</code>.
+ * @return {array}                    List of <code>{ kind, path }</code> entries.
+ */
+exports.scan = function (source, opts) {
+  source = source.replace(/\r\n/g, '\n');
+
+  var splitter = buildSplitter(opts),
+    tagOpen = opts.tagControls[0],
+    tagClose = opts.tagControls[1],
+    rawTag = opts.rawTag,
+    endRawTag = 'end' + rawTag,
+    keywordRegex = new RegExp(
+      '^(' + opts.keywords.join('|') + ')\\s+["\\\']([^"\\\']+)["\\\']'
+    ),
+    chunks = source.split(splitter),
+    results = [],
+    inRaw = false,
+    i,
+    chunk,
+    body,
+    name,
+    m;
+
+  for (i = 0; i < chunks.length; i += 1) {
+    chunk = chunks[i];
+    if (typeof chunk !== 'string' || !chunk) {
+      continue;
+    }
+
+    if (!utils.startsWith(chunk, tagOpen) || !utils.endsWith(chunk, tagClose)) {
+      continue;
+    }
+
+    body = stripTagBody(chunk, tagOpen, tagClose);
+    name = body.split(/\s+/)[0];
+
+    if (name === rawTag) {
+      inRaw = true;
+      continue;
+    }
+    if (name === endRawTag) {
+      inRaw = false;
+      continue;
+    }
+    if (inRaw) {
+      continue;
+    }
+
+    m = keywordRegex.exec(body);
+    if (m) {
+      results.push({ kind: m[1], path: m[2] });
+    }
+  }
+
+  return results;
+};
+
+/**
+ * Walk the dependency graph asynchronously starting from <var>entryPath</var>.
+ *
+ * Repeatedly loads, scans, and resolves child template paths in parallel
+ * via the user's async loader, until the dep graph closes. Returns a
+ * Promise resolving to a populated <code>{ resolvedPath: source }</code>
+ * map suitable for backing a memory loader.
+ *
+ * Cycles in the graph are tolerated — once a path is in the map or
+ * pending, subsequent enqueue requests are dropped. The synchronous
+ * renderer's existing circular-extends guard handles cycles at parse
+ * time on the second pass.
+ *
+ * @param  {string}  entryPath  Resolved path of the entry template.
+ * @param  {object}  loader     User loader. Must expose:
+ *                              <code>resolve(to, from)</code> (sync, returns
+ *                              string) and
+ *                              <code>load(id, cb)</code> (async, calls
+ *                              <code>cb(err, source)</code>).
+ * @param  {object}  scanOpts   Pass-through to {@link scan}.
+ * @return {Promise}            Resolves to the populated memory map.
+ */
+exports.walk = function (entryPath, loader, scanOpts) {
+  var memMap = {};
+  var pending = {};
+
+  return new Promise(function (resolve, reject) {
+    var inFlight = 0;
+    var queue = [];
+    var hasError = false;
+
+    function enqueue(path) {
+      if (memMap.hasOwnProperty(path) || pending[path]) {
+        return;
+      }
+      pending[path] = true;
+      queue.push(path);
+    }
+
+    function drain() {
+      while (queue.length > 0 && !hasError) {
+        var path = queue.shift();
+        inFlight += 1;
+        startLoad(path);
+      }
+      if (inFlight === 0 && !hasError && queue.length === 0) {
+        resolve(memMap);
+      }
+    }
+
+    function startLoad(resolvedPath) {
+      loader.load(resolvedPath, function (err, src) {
+        if (hasError) {
+          return;
+        }
+        if (err) {
+          hasError = true;
+          reject(err);
+          return;
+        }
+        if (typeof src !== 'string') {
+          hasError = true;
+          reject(new Error('Async loader returned non-string source for "' + resolvedPath + '"'));
+          return;
+        }
+        memMap[resolvedPath] = src;
+
+        var targets;
+        try {
+          targets = exports.scan(src, scanOpts);
+        } catch (e) {
+          hasError = true;
+          reject(e);
+          return;
+        }
+
+        var i, resolvedChild;
+        for (i = 0; i < targets.length; i += 1) {
+          try {
+            resolvedChild = loader.resolve(targets[i].path, resolvedPath);
+          } catch (e) {
+            hasError = true;
+            reject(e);
+            return;
+          }
+          enqueue(resolvedChild);
+        }
+
+        inFlight -= 1;
+        drain();
+      });
+    }
+
+    enqueue(entryPath);
+    drain();
+  });
+};
+
+/**
+ * Build a sync memory wrapper around a pre-populated
+ * <code>{ resolvedPath: source }</code> map. Delegates <code>resolve</code>
+ * to the user loader so cache keys match what the pre-walker produced.
+ *
+ * @param  {object} userLoader  Original async loader (used for resolve).
+ * @param  {object} memMap      Pre-populated source map.
+ * @return {object}             A loader exposing <code>resolve</code> and
+ *                              <code>load</code>.
+ */
+exports.makeMemoryWrapper = function (userLoader, memMap) {
+  return {
+    resolve: function (to, from) {
+      return userLoader.resolve(to, from);
+    },
+    load: function (id, cb) {
+      var src = memMap[id];
+      if (typeof src !== 'string') {
+        var err = new Error('Pre-walked map missing path: "' + id + '"');
+        if (cb) {
+          cb(err);
+          return;
+        }
+        throw err;
+      }
+      if (cb) {
+        cb(null, src);
+        return;
+      }
+      return src;
+    }
+  };
+};

package/lib/index.js

@@ -14,7 +14,8 @@ var utils = require('@rhinostone/swig-core/lib/utils'),
   parser = require('./parser'),
   _tags = require('./tags'),
   _filters = require('./filters'),
-  _tests = require('./tests');
+  _tests = require('./tests'),
+  preWalker = require('./async/pre-walker');
 
 exports.name = 'twig';
 
@@ -163,6 +164,146 @@ exports.Twig = function (opts) {
   utils.each(_tests, function (fn, name) {
     self.setExtension('_test_' + name, fn);
   });
+
+  function buildScanOpts() {
+    return {
+      varControls: self.options.varControls,
+      tagControls: self.options.tagControls,
+      cmtControls: self.options.cmtControls,
+      rawTag: 'verbatim',
+      keywords: ['extends', 'include', 'import', 'from']
+    };
+  }
+
+  /**
+   * Render a Twig template file asynchronously, supporting async loaders.
+   *
+   * Pre-walks <code>extends</code> / <code>include</code> /
+   * <code>import</code> / <code>from</code> targets in parallel via the
+   * user loader, populates an in-memory map, then runs the existing sync
+   * render pipeline against the populated map. Dynamic paths
+   * (<code>{% extends parent_var %}</code>) are not pre-resolved and will
+   * throw at render time as they would on the sync path.
+   *
+   * @example
+   * twig.setDefaults({ loader: myAsyncLoader });
+   * twig.renderFileAsync('page.twig', { name: 'world' }, function (err, output) {
+   *   if (err) { return done(err); }
+   *   res.end(output);
+   * });
+   *
+   * @param  {string}   pathName  Template path; resolved via the active loader.
+   * @param  {object}   [locals]  Locals to render with.
+   * @param  {Function} cb        Node-style callback <code>(err, output)</code>.
+   * @return {undefined}
+   */
+  this.renderFileAsync = function (pathName, locals, cb) {
+    if (typeof locals === 'function') {
+      cb = locals;
+      locals = undefined;
+    }
+
+    var loader = self.options.loader;
+    var entry;
+
+    try {
+      entry = loader.resolve(pathName);
+    } catch (e) {
+      cb(e);
+      return;
+    }
+
+    preWalker.walk(entry, loader, buildScanOpts()).then(function (memMap) {
+      var memWrapper = preWalker.makeMemoryWrapper(loader, memMap);
+      var origLoader = self.options.loader;
+      self.options.loader = memWrapper;
+      var output, error;
+      try {
+        output = self.renderFile(entry, locals);
+      } catch (e) {
+        error = e;
+      }
+      self.options.loader = origLoader;
+      if (error) {
+        cb(error);
+        return;
+      }
+      cb(null, output);
+    }, function (err) {
+      cb(err);
+    });
+  };
+
+  /**
+   * Compile a Twig template file asynchronously, supporting async loaders.
+   *
+   * Same pre-walk / memory-wrapper / sync-pipeline shape as
+   * {@link Twig#renderFileAsync}. Returns the compiled function (via
+   * <var>cb</var>) that takes a locals object and yields a rendered
+   * string. The returned function captures the pre-walked memory map and
+   * temporarily swaps the loader on each call, so subsequent runtime
+   * <code>include</code>s resolve correctly without re-running the
+   * pre-walk.
+   *
+   * @example
+   * twig.compileFileAsync('page.twig', {}, function (err, fn) {
+   *   if (err) { return done(err); }
+   *   res.end(fn({ name: 'world' }));
+   * });
+   *
+   * @param  {string}   pathName  Template path.
+   * @param  {object}   [options] Compilation options.
+   * @param  {Function} cb        Node-style callback <code>(err, fn)</code>.
+   * @return {undefined}
+   */
+  this.compileFileAsync = function (pathName, options, cb) {
+    if (typeof options === 'function') {
+      cb = options;
+      options = {};
+    }
+
+    var loader = self.options.loader;
+    var entry;
+
+    try {
+      entry = loader.resolve(pathName);
+    } catch (e) {
+      cb(e);
+      return;
+    }
+
+    preWalker.walk(entry, loader, buildScanOpts()).then(function (memMap) {
+      var memWrapper = preWalker.makeMemoryWrapper(loader, memMap);
+      var origLoader = self.options.loader;
+      self.options.loader = memWrapper;
+      var compiled, error;
+      try {
+        compiled = self.compileFile(entry, options);
+      } catch (e) {
+        error = e;
+      }
+      self.options.loader = origLoader;
+      if (error) {
+        cb(error);
+        return;
+      }
+      var wrapped = function (locals) {
+        var origInner = self.options.loader;
+        self.options.loader = memWrapper;
+        try {
+          var output = compiled(locals);
+          self.options.loader = origInner;
+          return output;
+        } catch (e) {
+          self.options.loader = origInner;
+          throw e;
+        }
+      };
+      cb(null, wrapped);
+    }, function (err) {
+      cb(err);
+    });
+  };
 };
 
 /*!
@@ -176,8 +317,10 @@ exports.parseFile = defaultInstance.parseFile;
 exports.precompile = defaultInstance.precompile;
 exports.compile = defaultInstance.compile;
 exports.compileFile = defaultInstance.compileFile;
+exports.compileFileAsync = defaultInstance.compileFileAsync;
 exports.render = defaultInstance.render;
 exports.renderFile = defaultInstance.renderFile;
+exports.renderFileAsync = defaultInstance.renderFileAsync;
 exports.run = defaultInstance.run;
 exports.invalidateCache = defaultInstance.invalidateCache;
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rhinostone/swig-twig",
-  "version": "2.0.1",
+  "version": "2.1.0",
   "description": "Twig-syntax frontend for the @rhinostone/swig-core template engine. Part of the @rhinostone/swig multi-flavor family.",
   "keywords": [
     "template",
@@ -22,7 +22,7 @@
     "node": ">=12"
   },
   "peerDependencies": {
-    "@rhinostone/swig-core": "2.0.1"
+    "@rhinostone/swig-core": "2.1.0"
   },
   "publishConfig": {
     "access": "public"