@rhinostone/swig-twig 2.0.0 → 2.1.0

package/lib/async/pre-walker.js

@@ -0,0 +1,267 @@
+var utils = require('@rhinostone/swig-core/lib/utils');
+
+/*!
+ * Makes a string safe for a regular expression. Mirrors swig-twig's parser.
+ * @private
+ */
+function escapeRegExp(str) {
+  return str.replace(/[\-\/\\\^$*+?.()|\[\]{}]/g, '\\$&');
+}
+
+/*!
+ * Build the splitter regex from the controls trio. Mirrors the regex the
+ * Twig parser builds at parse-time so the pre-walker chunks the same way
+ * the real parser would.
+ * @private
+ */
+function buildSplitter(controls) {
+  var anyChar = '[\\s\\S]*?',
+    varOpen = escapeRegExp(controls.varControls[0]),
+    varClose = escapeRegExp(controls.varControls[1]),
+    tagOpen = escapeRegExp(controls.tagControls[0]),
+    tagClose = escapeRegExp(controls.tagControls[1]),
+    cmtOpen = escapeRegExp(controls.cmtControls[0]),
+    cmtClose = escapeRegExp(controls.cmtControls[1]);
+  return new RegExp(
+    '(' +
+      tagOpen + anyChar + tagClose + '|' +
+      varOpen + anyChar + varClose + '|' +
+      cmtOpen + anyChar + cmtClose +
+    ')'
+  );
+}
+
+/*!
+ * Strip tag controls and optional whitespace-control markers from a tag
+ * chunk, returning the trimmed tag body (e.g. <code>extends "x.html"</code>).
+ * @private
+ */
+function stripTagBody(chunk, tagOpen, tagClose) {
+  var body = chunk.substr(tagOpen.length, chunk.length - tagOpen.length - tagClose.length);
+  if (body.charAt(0) === '-') {
+    body = body.substr(1);
+  }
+  if (body.charAt(body.length - 1) === '-') {
+    body = body.substr(0, body.length - 1);
+  }
+  return body.replace(/^\s+|\s+$/g, '');
+}
+
+/**
+ * Scan Twig template source for static <code>{% extends|include|import|from "..." %}</code>
+ * targets. Pure function; performs no I/O.
+ *
+ * The scanner mirrors the Twig parser's chunk-splitter so it agrees on
+ * chunk boundaries even under non-default control characters. Dynamic
+ * paths (<code>{% extends parent_var %}</code>) and tag bodies whose
+ * first token isn't a string literal are silently skipped — they remain
+ * on the sync path, which throws appropriately at parse time.
+ *
+ * @example
+ * preWalker.scan('{% from "macros.html" import foo %}', {
+ *   varControls: ['{{', '}}'],
+ *   tagControls: ['{%', '%}'],
+ *   cmtControls: ['{#', '#}'],
+ *   rawTag: 'verbatim',
+ *   keywords: ['extends', 'include', 'import', 'from']
+ * });
+ * // => [{ kind: 'from', path: 'macros.html' }]
+ *
+ * @param  {string} source
+ * @param  {object} opts
+ * @param  {array}  opts.varControls
+ * @param  {array}  opts.tagControls
+ * @param  {array}  opts.cmtControls
+ * @param  {string} opts.rawTag       Tag name that opens verbatim regions
+ *                                    (<code>verbatim</code> for Twig).
+ * @param  {array}  opts.keywords     Keywords whose first quoted argument
+ *                                    is a template path. Twig:
+ *                                    <code>['extends', 'include', 'import',
+ *                                    'from']</code>.
+ * @return {array}                    List of <code>{ kind, path }</code> entries.
+ */
+exports.scan = function (source, opts) {
+  source = source.replace(/\r\n/g, '\n');
+
+  var splitter = buildSplitter(opts),
+    tagOpen = opts.tagControls[0],
+    tagClose = opts.tagControls[1],
+    rawTag = opts.rawTag,
+    endRawTag = 'end' + rawTag,
+    keywordRegex = new RegExp(
+      '^(' + opts.keywords.join('|') + ')\\s+["\\\']([^"\\\']+)["\\\']'
+    ),
+    chunks = source.split(splitter),
+    results = [],
+    inRaw = false,
+    i,
+    chunk,
+    body,
+    name,
+    m;
+
+  for (i = 0; i < chunks.length; i += 1) {
+    chunk = chunks[i];
+    if (typeof chunk !== 'string' || !chunk) {
+      continue;
+    }
+
+    if (!utils.startsWith(chunk, tagOpen) || !utils.endsWith(chunk, tagClose)) {
+      continue;
+    }
+
+    body = stripTagBody(chunk, tagOpen, tagClose);
+    name = body.split(/\s+/)[0];
+
+    if (name === rawTag) {
+      inRaw = true;
+      continue;
+    }
+    if (name === endRawTag) {
+      inRaw = false;
+      continue;
+    }
+    if (inRaw) {
+      continue;
+    }
+
+    m = keywordRegex.exec(body);
+    if (m) {
+      results.push({ kind: m[1], path: m[2] });
+    }
+  }
+
+  return results;
+};
+
+/**
+ * Walk the dependency graph asynchronously starting from <var>entryPath</var>.
+ *
+ * Repeatedly loads, scans, and resolves child template paths in parallel
+ * via the user's async loader, until the dep graph closes. Returns a
+ * Promise resolving to a populated <code>{ resolvedPath: source }</code>
+ * map suitable for backing a memory loader.
+ *
+ * Cycles in the graph are tolerated — once a path is in the map or
+ * pending, subsequent enqueue requests are dropped. The synchronous
+ * renderer's existing circular-extends guard handles cycles at parse
+ * time on the second pass.
+ *
+ * @param  {string}  entryPath  Resolved path of the entry template.
+ * @param  {object}  loader     User loader. Must expose:
+ *                              <code>resolve(to, from)</code> (sync, returns
+ *                              string) and
+ *                              <code>load(id, cb)</code> (async, calls
+ *                              <code>cb(err, source)</code>).
+ * @param  {object}  scanOpts   Pass-through to {@link scan}.
+ * @return {Promise}            Resolves to the populated memory map.
+ */
+exports.walk = function (entryPath, loader, scanOpts) {
+  var memMap = {};
+  var pending = {};
+
+  return new Promise(function (resolve, reject) {
+    var inFlight = 0;
+    var queue = [];
+    var hasError = false;
+
+    function enqueue(path) {
+      if (memMap.hasOwnProperty(path) || pending[path]) {
+        return;
+      }
+      pending[path] = true;
+      queue.push(path);
+    }
+
+    function drain() {
+      while (queue.length > 0 && !hasError) {
+        var path = queue.shift();
+        inFlight += 1;
+        startLoad(path);
+      }
+      if (inFlight === 0 && !hasError && queue.length === 0) {
+        resolve(memMap);
+      }
+    }
+
+    function startLoad(resolvedPath) {
+      loader.load(resolvedPath, function (err, src) {
+        if (hasError) {
+          return;
+        }
+        if (err) {
+          hasError = true;
+          reject(err);
+          return;
+        }
+        if (typeof src !== 'string') {
+          hasError = true;
+          reject(new Error('Async loader returned non-string source for "' + resolvedPath + '"'));
+          return;
+        }
+        memMap[resolvedPath] = src;
+
+        var targets;
+        try {
+          targets = exports.scan(src, scanOpts);
+        } catch (e) {
+          hasError = true;
+          reject(e);
+          return;
+        }
+
+        var i, resolvedChild;
+        for (i = 0; i < targets.length; i += 1) {
+          try {
+            resolvedChild = loader.resolve(targets[i].path, resolvedPath);
+          } catch (e) {
+            hasError = true;
+            reject(e);
+            return;
+          }
+          enqueue(resolvedChild);
+        }
+
+        inFlight -= 1;
+        drain();
+      });
+    }
+
+    enqueue(entryPath);
+    drain();
+  });
+};
+
+/**
+ * Build a sync memory wrapper around a pre-populated
+ * <code>{ resolvedPath: source }</code> map. Delegates <code>resolve</code>
+ * to the user loader so cache keys match what the pre-walker produced.
+ *
+ * @param  {object} userLoader  Original async loader (used for resolve).
+ * @param  {object} memMap      Pre-populated source map.
+ * @return {object}             A loader exposing <code>resolve</code> and
+ *                              <code>load</code>.
+ */
+exports.makeMemoryWrapper = function (userLoader, memMap) {
+  return {
+    resolve: function (to, from) {
+      return userLoader.resolve(to, from);
+    },
+    load: function (id, cb) {
+      var src = memMap[id];
+      if (typeof src !== 'string') {
+        var err = new Error('Pre-walked map missing path: "' + id + '"');
+        if (cb) {
+          cb(err);
+          return;
+        }
+        throw err;
+      }
+      if (cb) {
+        cb(null, src);
+        return;
+      }
+      return src;
+    }
+  };
+};

package/lib/index.js

@@ -14,7 +14,8 @@ var utils = require('@rhinostone/swig-core/lib/utils'),
   parser = require('./parser'),
   _tags = require('./tags'),
   _filters = require('./filters'),
-  _tests = require('./tests');
+  _tests = require('./tests'),
+  preWalker = require('./async/pre-walker');
 
 exports.name = 'twig';
 
@@ -163,6 +164,146 @@ exports.Twig = function (opts) {
   utils.each(_tests, function (fn, name) {
     self.setExtension('_test_' + name, fn);
   });
+
+  function buildScanOpts() {
+    return {
+      varControls: self.options.varControls,
+      tagControls: self.options.tagControls,
+      cmtControls: self.options.cmtControls,
+      rawTag: 'verbatim',
+      keywords: ['extends', 'include', 'import', 'from']
+    };
+  }
+
+  /**
+   * Render a Twig template file asynchronously, supporting async loaders.
+   *
+   * Pre-walks <code>extends</code> / <code>include</code> /
+   * <code>import</code> / <code>from</code> targets in parallel via the
+   * user loader, populates an in-memory map, then runs the existing sync
+   * render pipeline against the populated map. Dynamic paths
+   * (<code>{% extends parent_var %}</code>) are not pre-resolved and will
+   * throw at render time as they would on the sync path.
+   *
+   * @example
+   * twig.setDefaults({ loader: myAsyncLoader });
+   * twig.renderFileAsync('page.twig', { name: 'world' }, function (err, output) {
+   *   if (err) { return done(err); }
+   *   res.end(output);
+   * });
+   *
+   * @param  {string}   pathName  Template path; resolved via the active loader.
+   * @param  {object}   [locals]  Locals to render with.
+   * @param  {Function} cb        Node-style callback <code>(err, output)</code>.
+   * @return {undefined}
+   */
+  this.renderFileAsync = function (pathName, locals, cb) {
+    if (typeof locals === 'function') {
+      cb = locals;
+      locals = undefined;
+    }
+
+    var loader = self.options.loader;
+    var entry;
+
+    try {
+      entry = loader.resolve(pathName);
+    } catch (e) {
+      cb(e);
+      return;
+    }
+
+    preWalker.walk(entry, loader, buildScanOpts()).then(function (memMap) {
+      var memWrapper = preWalker.makeMemoryWrapper(loader, memMap);
+      var origLoader = self.options.loader;
+      self.options.loader = memWrapper;
+      var output, error;
+      try {
+        output = self.renderFile(entry, locals);
+      } catch (e) {
+        error = e;
+      }
+      self.options.loader = origLoader;
+      if (error) {
+        cb(error);
+        return;
+      }
+      cb(null, output);
+    }, function (err) {
+      cb(err);
+    });
+  };
+
+  /**
+   * Compile a Twig template file asynchronously, supporting async loaders.
+   *
+   * Same pre-walk / memory-wrapper / sync-pipeline shape as
+   * {@link Twig#renderFileAsync}. Returns the compiled function (via
+   * <var>cb</var>) that takes a locals object and yields a rendered
+   * string. The returned function captures the pre-walked memory map and
+   * temporarily swaps the loader on each call, so subsequent runtime
+   * <code>include</code>s resolve correctly without re-running the
+   * pre-walk.
+   *
+   * @example
+   * twig.compileFileAsync('page.twig', {}, function (err, fn) {
+   *   if (err) { return done(err); }
+   *   res.end(fn({ name: 'world' }));
+   * });
+   *
+   * @param  {string}   pathName  Template path.
+   * @param  {object}   [options] Compilation options.
+   * @param  {Function} cb        Node-style callback <code>(err, fn)</code>.
+   * @return {undefined}
+   */
+  this.compileFileAsync = function (pathName, options, cb) {
+    if (typeof options === 'function') {
+      cb = options;
+      options = {};
+    }
+
+    var loader = self.options.loader;
+    var entry;
+
+    try {
+      entry = loader.resolve(pathName);
+    } catch (e) {
+      cb(e);
+      return;
+    }
+
+    preWalker.walk(entry, loader, buildScanOpts()).then(function (memMap) {
+      var memWrapper = preWalker.makeMemoryWrapper(loader, memMap);
+      var origLoader = self.options.loader;
+      self.options.loader = memWrapper;
+      var compiled, error;
+      try {
+        compiled = self.compileFile(entry, options);
+      } catch (e) {
+        error = e;
+      }
+      self.options.loader = origLoader;
+      if (error) {
+        cb(error);
+        return;
+      }
+      var wrapped = function (locals) {
+        var origInner = self.options.loader;
+        self.options.loader = memWrapper;
+        try {
+          var output = compiled(locals);
+          self.options.loader = origInner;
+          return output;
+        } catch (e) {
+          self.options.loader = origInner;
+          throw e;
+        }
+      };
+      cb(null, wrapped);
+    }, function (err) {
+      cb(err);
+    });
+  };
 };
 
 /*!
@@ -176,8 +317,10 @@ exports.parseFile = defaultInstance.parseFile;
 exports.precompile = defaultInstance.precompile;
 exports.compile = defaultInstance.compile;
 exports.compileFile = defaultInstance.compileFile;
+exports.compileFileAsync = defaultInstance.compileFileAsync;
 exports.render = defaultInstance.render;
 exports.renderFile = defaultInstance.renderFile;
+exports.renderFileAsync = defaultInstance.renderFileAsync;
 exports.run = defaultInstance.run;
 exports.invalidateCache = defaultInstance.invalidateCache;
 

package/lib/lexer.js

@@ -221,7 +221,7 @@ var rules = [
   {
     type: TYPES.NUMBER,
     regex: [
-      /^[+\-]?\d+(\.\d+)?/
+      /^\d+(\.\d+)?/
     ]
   },
   {

package/lib/parser.js

@@ -515,8 +515,16 @@ exports.parse = function (swig, source, opts, tags, filters) {
       escapeRegExp(cmtOpen) + anyChar + escapeRegExp(cmtClose) +
       ')'
   );
-  var tagStrip = new RegExp('^' + escapeRegExp(tagOpen) + '\\s*|\\s*' + escapeRegExp(tagClose) + '$', 'g');
-  var varStrip = new RegExp('^' + escapeRegExp(varOpen) + '\\s*|\\s*' + escapeRegExp(varClose) + '$', 'g');
+  // Twig/Jinja2 whitespace-control. `{{- … -}}` / `{%- … -%}` strip
+  // surrounding whitespace; the `-?` lives only adjacent to the open /
+  // close marker (post-#T23 shape — drop the inner `-?` after `\s*` so
+  // `{{ -5 }}` doesn't have its expression-`-` eaten as a strip marker).
+  var tagStrip = new RegExp('^' + escapeRegExp(tagOpen) + '-?\\s*|\\s*-?' + escapeRegExp(tagClose) + '$', 'g');
+  var varStrip = new RegExp('^' + escapeRegExp(varOpen) + '-?\\s*|\\s*-?' + escapeRegExp(varClose) + '$', 'g');
+  var tagStripBefore = new RegExp('^' + escapeRegExp(tagOpen) + '-');
+  var tagStripAfter = new RegExp('-' + escapeRegExp(tagClose) + '$');
+  var varStripBefore = new RegExp('^' + escapeRegExp(varOpen) + '-');
+  var varStripAfter = new RegExp('-' + escapeRegExp(varClose) + '$');
 
   var line = 1;
   var stack = [];
@@ -524,6 +532,29 @@ exports.parse = function (swig, source, opts, tags, filters) {
   var tokens = [];
   var blocks = {};
   var inVerbatim = false;
+  // Carries `-}}` / `-%}` strip-after intent across the chunk boundary.
+  // Consumed by the next text chunk (leading whitespace stripped, flag
+  // reset). Mirrors native lib/parser.js's closure-scoped `stripNext`.
+  var stripNext = false;
+
+  /**
+   * If the previous token is a Text IR node, strip its trailing
+   * whitespace in-place. No-op for non-Text tokens.
+   *
+   * Mirrors lib/parser.js's stripPrevToken — same one-level-deep
+   * limitation: a `{%- endif %}` only strips the trailing whitespace of
+   * the last child of the immediately enclosing tag, not deeper.
+   *
+   * @param  {object} token IR node (typed), possibly a Text node.
+   * @return {object}       Same node; mutated when `type === 'Text'`.
+   * @private
+   */
+  function stripPrevToken(token) {
+    if (token && token.type === 'Text' && typeof token.value === 'string') {
+      token.value = token.value.replace(/\s*$/, '');
+    }
+    return token;
+  }
 
   /**
    * Build an IROutput node for a `{{ … }}` chunk.
@@ -628,13 +659,17 @@ exports.parse = function (swig, source, opts, tags, filters) {
   }
 
   utils.each(source.split(splitter), function (chunk) {
-    var token, lines;
+    var token, lines, stripPrev, prevToken, prevChildToken;
 
     if (!chunk) { return; }
 
     if (!inVerbatim && utils.startsWith(chunk, varOpen) && utils.endsWith(chunk, varClose)) {
+      stripPrev = varStripBefore.test(chunk);
+      stripNext = varStripAfter.test(chunk);
       token = parseVariable(chunk.replace(varStrip, ''), line);
     } else if (utils.startsWith(chunk, tagOpen) && utils.endsWith(chunk, tagClose)) {
+      stripPrev = tagStripBefore.test(chunk);
+      stripNext = tagStripAfter.test(chunk);
       token = parseTag(chunk.replace(tagStrip, ''), line);
       if (token) {
         if (token.name === 'extends') {
@@ -654,9 +689,28 @@ exports.parse = function (swig, source, opts, tags, filters) {
       line += lines ? lines.length : 0;
       return;
     } else {
+      if (stripNext) {
+        chunk = chunk.replace(/^\s*/, '');
+        stripNext = false;
+      }
       token = ir.text(chunk);
     }
 
+    // `{{-` / `{%-` strips the previous text chunk's trailing whitespace.
+    // Mirrors lib/parser.js: pop tokens.last; if it's a Text node strip
+    // it directly, else if it carries `.content` (a tag with body) drill
+    // one level into its last child. One-level-deep — matches native.
+    if (stripPrev && tokens.length) {
+      prevToken = tokens.pop();
+      if (prevToken && prevToken.type === 'Text') {
+        prevToken = stripPrevToken(prevToken);
+      } else if (prevToken && prevToken.content && prevToken.content.length) {
+        prevChildToken = stripPrevToken(prevToken.content.pop());
+        prevToken.content.push(prevChildToken);
+      }
+      tokens.push(prevToken);
+    }
+
     if (token) {
       if (stack.length) {
         stack[stack.length - 1].content.push(token);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rhinostone/swig-twig",
-  "version": "2.0.0",
+  "version": "2.1.0",
   "description": "Twig-syntax frontend for the @rhinostone/swig-core template engine. Part of the @rhinostone/swig multi-flavor family.",
   "keywords": [
     "template",
@@ -22,7 +22,7 @@
     "node": ">=12"
   },
   "peerDependencies": {
-    "@rhinostone/swig-core": "2.0.0"
+    "@rhinostone/swig-core": "2.1.0"
   },
   "publishConfig": {
     "access": "public"