@rhinostone/swig-jinja2 2.5.0

package/lib/index.js

@@ -0,0 +1,344 @@
+/**
+ * @rhinostone/swig-jinja2 — Jinja2 frontend for the @rhinostone/swig family.
+ *
+ * End-to-end render wiring (Path A): the package exposes a Jinja2
+ * constructor + default instance via `engine.install(self, frontend)` from
+ * @rhinostone/swig-core, so callers can `render(source, locals)` /
+ * `renderFile(path, locals, cb)` directly against Jinja2 syntax.
+ */
+
+var utils = require('@rhinostone/swig-core/lib/utils'),
+  engine = require('@rhinostone/swig-core/lib/engine'),
+  loaders = require('@rhinostone/swig-core/lib/loaders'),
+  dateformatter = require('@rhinostone/swig-core/lib/dateformatter'),
+  parser = require('./parser'),
+  _tags = require('./tags'),
+  _filters = require('./filters'),
+  _tests = require('./tests'),
+  preWalker = require('./async/pre-walker');
+
+exports.name = 'jinja2';
+
+/**
+ * Expression-level parser — Pratt-style recursive descent that consumes
+ * Jinja2 lexer tokens and returns swig-core IRExpr nodes, plus the
+ * top-level `parse(swig, source, opts, tags, filters)` splitter.
+ *
+ * @type {object}
+ */
+exports.parser = parser;
+
+/**
+ * Built-in Jinja2 tag registry.
+ *
+ * @type {object}
+ */
+exports.tags = _tags;
+
+/**
+ * Built-in Jinja2 filter catalog.
+ *
+ * @type {object}
+ */
+exports.filters = _filters;
+
+/**
+ * Template loaders re-exported from swig-core.
+ *
+ * @type {object}
+ */
+exports.loaders = loaders;
+
+var defaultOptions = {
+    autoescape: true,
+    varControls: ['{{', '}}'],
+    tagControls: ['{%', '%}'],
+    cmtControls: ['{#', '#}'],
+    locals: {},
+    cache: 'memory',
+    loader: loaders.fs()
+  },
+  defaultInstance;
+
+/**
+ * Validate the Jinja2 options object.
+ *
+ * @param  {?object} options Jinja2 options object.
+ * @return {undefined}      Throws on malformed input.
+ * @private
+ */
+function validateOptions(options) {
+  if (!options) {
+    return;
+  }
+
+  utils.each(['varControls', 'tagControls', 'cmtControls'], function (key) {
+    if (!options.hasOwnProperty(key)) {
+      return;
+    }
+    if (!utils.isArray(options[key]) || options[key].length !== 2) {
+      throw new Error('Option "' + key + '" must be an array containing 2 different control strings.');
+    }
+    if (options[key][0] === options[key][1]) {
+      throw new Error('Option "' + key + '" open and close controls must not be the same.');
+    }
+    utils.each(options[key], function (a, i) {
+      if (a.length < 2) {
+        throw new Error('Option "' + key + '" ' + ((i) ? 'open ' : 'close ') + 'control must be at least 2 characters. Saw "' + a + '" instead.');
+      }
+    });
+  });
+
+  if (options.hasOwnProperty('cache')) {
+    if (options.cache && options.cache !== 'memory') {
+      if (!options.cache.get || !options.cache.set) {
+        throw new Error('Invalid cache option ' + JSON.stringify(options.cache) + ' found. Expected "memory" or { get: function (key) { ... }, set: function (key, value) { ... } }.');
+      }
+    }
+  }
+  if (options.hasOwnProperty('loader')) {
+    if (options.loader) {
+      if (!options.loader.load || !options.loader.resolve) {
+        throw new Error('Invalid loader option ' + JSON.stringify(options.loader) + ' found. Expected { load: function (pathname, cb) { ... }, resolve: function (to, from) { ... } }.');
+      }
+    }
+  }
+}
+
+/**
+ * Set defaults for the base and all new Jinja2 environments.
+ *
+ * @param  {object} [options={}] Jinja2 options object.
+ * @return {undefined}
+ */
+exports.setDefaults = function (options) {
+  validateOptions(options);
+  defaultInstance.options = utils.extend(defaultInstance.options, options);
+};
+
+/**
+ * Set the default TimeZone offset for date formatting via the date filter.
+ * Mutates the shared dateformatter's tzOffset — affects every frontend
+ * (native swig + swig-jinja2) because both consume the same module instance.
+ *
+ * @param  {number} offset Offset from GMT, in minutes (west of GMT).
+ * @return {undefined}
+ */
+exports.setDefaultTZOffset = function (offset) {
+  dateformatter.tzOffset = offset;
+};
+
+/**
+ * Create a new, separate Jinja2 compile/render environment.
+ *
+ * @example
+ * var jinja2 = require('@rhinostone/swig-jinja2');
+ * var myjinja2 = new jinja2.Jinja2({ autoescape: false });
+ * myjinja2.render('Hello {{ name }}', { locals: { name: 'world' }});
+ *
+ * @param  {object} [opts={}] Jinja2 options object.
+ * @return {object}           New Jinja2 environment.
+ */
+exports.Jinja2 = function (opts) {
+  var self = this;
+  validateOptions(opts);
+  this.options = utils.extend({}, defaultOptions, opts || {});
+  this.cache = {};
+  this.extensions = {};
+
+  engine.install(this, {
+    parser: parser,
+    tags: _tags,
+    filters: _filters,
+    validateOptions: validateOptions,
+    onCompileError: function (err, options) {
+      utils.throwError(err, null, options.filename);
+    }
+  });
+
+  // Register Jinja2 `is <name>` runtime helpers as `_ext._test_<name>`.
+  // setExtension attaches them to this instance's `extensions` map, so a
+  // consumer can override any test per-instance with their own
+  // `setExtension('_test_<name>', fn)` after construction.
+  utils.each(_tests, function (fn, name) {
+    self.setExtension('_test_' + name, fn);
+  });
+
+  function buildScanOpts() {
+    return {
+      varControls: self.options.varControls,
+      tagControls: self.options.tagControls,
+      cmtControls: self.options.cmtControls,
+      rawTag: 'raw',
+      keywords: ['extends', 'include', 'import', 'from']
+    };
+  }
+
+  /**
+   * Render a Jinja2 template file asynchronously, supporting async loaders.
+   *
+   * Pre-walks <code>extends</code> / <code>include</code> /
+   * <code>import</code> / <code>from</code> targets in parallel via the
+   * user loader, populates an in-memory map, then runs the existing sync
+   * render pipeline against the populated map. Dynamic paths
+   * (<code>{% extends parent_var %}</code>) are not pre-resolved and will
+   * throw at render time as they would on the sync path.
+   *
+   * @deprecated since 2.5.0 — use {@link Jinja2#renderFile} with a loader
+   *   that sets <code>loader.async === true</code>. The async-codegen
+   *   dispatch handles dynamic include paths the pre-walker cannot. This
+   *   method will be removed in 3.0.
+   *
+   * @example
+   * jinja2.setDefaults({ loader: myAsyncLoader });
+   * jinja2.renderFileAsync('page.html', { name: 'world' }, function (err, output) {
+   *   if (err) { return done(err); }
+   *   res.end(output);
+   * });
+   *
+   * @param  {string}   pathName  Template path; resolved via the active loader.
+   * @param  {object}   [locals]  Locals to render with.
+   * @param  {Function} cb        Node-style callback <code>(err, output)</code>.
+   * @return {undefined}
+   */
+  this.renderFileAsync = function (pathName, locals, cb) {
+    if (typeof locals === 'function') {
+      cb = locals;
+      locals = undefined;
+    }
+
+    var loader = self.options.loader;
+    var entry;
+
+    try {
+      entry = loader.resolve(pathName);
+    } catch (e) {
+      cb(e);
+      return;
+    }
+
+    preWalker.walk(entry, loader, buildScanOpts()).then(function (memMap) {
+      var memWrapper = preWalker.makeMemoryWrapper(loader, memMap);
+      var origLoader = self.options.loader;
+      self.options.loader = memWrapper;
+      var output, error;
+      try {
+        output = self.renderFile(entry, locals);
+      } catch (e) {
+        error = e;
+      }
+      self.options.loader = origLoader;
+      if (error) {
+        cb(error);
+        return;
+      }
+      cb(null, output);
+    }, function (err) {
+      cb(err);
+    });
+  };
+
+  /**
+   * Compile a Jinja2 template file asynchronously, supporting async loaders.
+   *
+   * Same pre-walk / memory-wrapper / sync-pipeline shape as
+   * {@link Jinja2#renderFileAsync}. Returns the compiled function (via
+   * <var>cb</var>) that takes a locals object and yields a rendered
+   * string. The returned function captures the pre-walked memory map and
+   * temporarily swaps the loader on each call, so subsequent runtime
+   * <code>include</code>s resolve correctly without re-running the
+   * pre-walk.
+   *
+   * @deprecated since 2.5.0 — use {@link Jinja2#compileFile} with
+   *   <code>options.codegenMode === 'async'</code> on a loader that sets
+   *   <code>loader.async === true</code>. The returned compiled function
+   *   yields a <code>Promise&lt;{output, exports}&gt;</code> instead of a
+   *   string. This method will be removed in 3.0.
+   *
+   * @example
+   * jinja2.compileFileAsync('page.html', {}, function (err, fn) {
+   *   if (err) { return done(err); }
+   *   res.end(fn({ name: 'world' }));
+   * });
+   *
+   * @param  {string}   pathName  Template path.
+   * @param  {object}   [options] Compilation options.
+   * @param  {Function} cb        Node-style callback <code>(err, fn)</code>.
+   * @return {undefined}
+   */
+  this.compileFileAsync = function (pathName, options, cb) {
+    if (typeof options === 'function') {
+      cb = options;
+      options = {};
+    }
+
+    var loader = self.options.loader;
+    var entry;
+
+    try {
+      entry = loader.resolve(pathName);
+    } catch (e) {
+      cb(e);
+      return;
+    }
+
+    preWalker.walk(entry, loader, buildScanOpts()).then(function (memMap) {
+      var memWrapper = preWalker.makeMemoryWrapper(loader, memMap);
+      var origLoader = self.options.loader;
+      self.options.loader = memWrapper;
+      var compiled, error;
+      try {
+        compiled = self.compileFile(entry, options);
+      } catch (e) {
+        error = e;
+      }
+      self.options.loader = origLoader;
+      if (error) {
+        cb(error);
+        return;
+      }
+      var wrapped = function (locals) {
+        var origInner = self.options.loader;
+        self.options.loader = memWrapper;
+        try {
+          var output = compiled(locals);
+          self.options.loader = origInner;
+          return output;
+        } catch (e) {
+          self.options.loader = origInner;
+          throw e;
+        }
+      };
+      cb(null, wrapped);
+    }, function (err) {
+      cb(err);
+    });
+  };
+};
+
+/*!
+ * Export methods publicly via the default instance.
+ */
+defaultInstance = new exports.Jinja2();
+exports.setFilter = defaultInstance.setFilter;
+exports.setTag = defaultInstance.setTag;
+exports.setExtension = defaultInstance.setExtension;
+exports.parseFile = defaultInstance.parseFile;
+exports.precompile = defaultInstance.precompile;
+exports.compile = defaultInstance.compile;
+exports.compileFile = defaultInstance.compileFile;
+exports.compileFileAsync = defaultInstance.compileFileAsync;
+exports.render = defaultInstance.render;
+exports.renderFile = defaultInstance.renderFile;
+exports.renderFileAsync = defaultInstance.renderFileAsync;
+exports.run = defaultInstance.run;
+exports.invalidateCache = defaultInstance.invalidateCache;
+
+/**
+ * Express 3/4 compatibility alias.
+ *
+ * @example
+ * app.engine('jinja2', require('@rhinostone/swig-jinja2').__express);
+ * app.set('view engine', 'jinja2');
+ */
+exports.__express = defaultInstance.renderFile;

package/lib/lexer.js

@@ -0,0 +1,305 @@
+var utils = require('@rhinostone/swig-core/lib/utils');
+var TYPES = require('./tokentypes');
+
+/**
+ * A Jinja2 lexer token.
+ *
+ * @typedef {object} LexerToken
+ * @property {string} match  The string that was matched (post-replace).
+ * @property {number} type   Jinja2 token type enum value.
+ * @property {number} length Length of the input chunk consumed.
+ */
+
+/*!
+ * Jinja2 lexer rule table — the swig-shared token subset.
+ *
+ * The Jinja2-only operators (`~` concat, `**` power, `//` floor-division,
+ * `is` / `is not` tests) land in subsequent commits; until then they fall
+ * through to the unknown-token throw (fail-close). Jinja2 has no range
+ * (`..`), null-coalescing (`??`), ternary (`?:`), or `#{}` string-
+ * interpolation operators, so this table will never grow those rules.
+ *
+ * Rule ordering constraints worth the call-out:
+ *
+ *   - FILTER / FILTEREMPTY / FUNCTION above VAR — `|name(` and `name(`
+ *     must be consumed as filter / function tokens before VAR's
+ *     `^[a-zA-Z_$]\w*` pattern gobbles the bare identifier.
+ *   - LOGIC (`and` / `or`), NOT (`not`), BOOL (`true` / `false`), and the
+ *     COMPARATOR `in\s` keyword above VAR — same reason: bake the keyword
+ *     sequence into the operator token rather than emit an identifier.
+ *
+ * Rules are tried in order; first match wins. Patterns are anchored at
+ * start-of-string because the consumer slices `str` before each dispatch.
+ */
+var rules = [
+  {
+    type: TYPES.WHITESPACE,
+    regex: [
+      /^\s+/
+    ]
+  },
+  {
+    type: TYPES.STRING,
+    regex: [
+      /^""/,
+      /^".*?[^\\]"/,
+      /^''/,
+      /^'.*?[^\\]'/
+    ]
+  },
+  {
+    type: TYPES.FILTER,
+    regex: [
+      /^\|\s*(\w+)\(/
+    ],
+    idx: 1
+  },
+  {
+    type: TYPES.FILTEREMPTY,
+    regex: [
+      /^\|\s*(\w+)/
+    ],
+    idx: 1
+  },
+  {
+    type: TYPES.FUNCTIONEMPTY,
+    regex: [
+      /^\s*(\w+)\(\)/
+    ],
+    idx: 1
+  },
+  {
+    type: TYPES.FUNCTION,
+    regex: [
+      /^\s*(\w+)\(/
+    ],
+    idx: 1
+  },
+  {
+    type: TYPES.PARENOPEN,
+    regex: [
+      /^\(/
+    ]
+  },
+  {
+    type: TYPES.PARENCLOSE,
+    regex: [
+      /^\)/
+    ]
+  },
+  {
+    type: TYPES.COMMA,
+    regex: [
+      /^,/
+    ]
+  },
+  {
+    type: TYPES.LOGIC,
+    regex: [
+      /^(&&|\|\|)\s*/,
+      /^(and|or)\s+/
+    ],
+    idx: 1,
+    replace: {
+      'and': '&&',
+      'or': '||'
+    }
+  },
+  {
+    type: TYPES.COMPARATOR,
+    regex: [
+      /^(===|==|\!==|\!=|<=|<|>=|>|in\s)\s*/
+    ],
+    idx: 1
+  },
+  {
+    type: TYPES.ASSIGNMENT,
+    regex: [
+      /^(=|\+=|-=|\*=|\/=)/
+    ]
+  },
+  {
+    type: TYPES.NOT,
+    regex: [
+      /^\!\s*/,
+      /^not\s+/
+    ],
+    replace: {
+      'not': '!'
+    }
+  },
+  {
+    type: TYPES.BOOL,
+    regex: [
+      /^(true|false)\s+/,
+      /^(true|false)$/
+    ],
+    idx: 1
+  },
+  {
+    // ISNOT above IS above VAR — the `is` keyword would otherwise be
+    // gobbled by VAR's `^[a-zA-Z_$]\w*` pattern. ISNOT above IS because
+    // `is not` must be consumed as a single token, not IS + NOT. The `\b`
+    // word boundary keeps identifiers like `island` from matching.
+    type: TYPES.ISNOT,
+    regex: [
+      /^is\s+not\b/
+    ]
+  },
+  {
+    type: TYPES.IS,
+    regex: [
+      /^is\b/
+    ]
+  },
+  {
+    // The dotted-path interior segment uses `\w+` (not `\w*`) so a future
+    // operator whose first char is `.` cannot be absorbed as a zero-width
+    // interior segment. Jinja2 has no `..` range today, but the tighter
+    // form is the defensive default and matches the Twig sibling lexer.
+    type: TYPES.VAR,
+    regex: [
+      /^[a-zA-Z_$]\w*((\.\$?\w+)+)?/,
+      /^[a-zA-Z_$]\w*/
+    ]
+  },
+  {
+    type: TYPES.BRACKETOPEN,
+    regex: [
+      /^\[/
+    ]
+  },
+  {
+    type: TYPES.BRACKETCLOSE,
+    regex: [
+      /^\]/
+    ]
+  },
+  {
+    type: TYPES.CURLYOPEN,
+    regex: [
+      /^\{/
+    ]
+  },
+  {
+    type: TYPES.COLON,
+    regex: [
+      /^\:/
+    ]
+  },
+  {
+    type: TYPES.CURLYCLOSE,
+    regex: [
+      /^\}/
+    ]
+  },
+  {
+    type: TYPES.DOTKEY,
+    regex: [
+      /^\.(\w+)/
+    ],
+    idx: 1
+  },
+  {
+    type: TYPES.NUMBER,
+    regex: [
+      /^\d+(\.\d+)?/
+    ]
+  },
+  {
+    type: TYPES.TILDE,
+    regex: [
+      /^~/
+    ]
+  },
+  {
+    // Above OPERATOR so `**` wins over a bare `*`; first-match-wins would
+    // otherwise lex `**` as two OPERATOR tokens.
+    type: TYPES.POWER,
+    regex: [
+      /^\*\*/
+    ]
+  },
+  {
+    // Above OPERATOR so `//` wins over a bare `/`. Note JS has no
+    // floor-division operator (`a // b` is a comment), so the parser
+    // lowers FLOORDIV to Math.floor(a / b) rather than emitting `//`.
+    type: TYPES.FLOORDIV,
+    regex: [
+      /^\/\//
+    ]
+  },
+  {
+    type: TYPES.OPERATOR,
+    regex: [
+      /^(\+|\-|\/|\*|%)/
+    ]
+  }
+];
+
+exports.types = TYPES;
+
+/**
+ * Match the next token at the start of `str`.
+ *
+ * Throws via utils.throwError when no rule matches — including every
+ * Jinja2-only operator until its rule lands. The throw is opaque (no
+ * line / file info); the Jinja2 frontend's onCompileError callback
+ * attaches filename + line per the swig-core / frontend seam rule.
+ *
+ * @param  {string}     str Input slice starting at the unconsumed offset.
+ * @return {LexerToken}     Matched token.
+ * @throws {Error}          When no rule matches.
+ * @private
+ */
+function reader(str) {
+  var matched;
+
+  utils.some(rules, function (rule) {
+    return utils.some(rule.regex, function (regex) {
+      var match = str.match(regex),
+        normalized;
+
+      if (!match) {
+        return;
+      }
+
+      normalized = match[rule.idx || 0].replace(/\s*$/, '');
+      normalized = (rule.hasOwnProperty('replace') && rule.replace.hasOwnProperty(normalized)) ? rule.replace[normalized] : normalized;
+
+      matched = {
+        match: normalized,
+        type: rule.type,
+        length: match[0].length
+      };
+      return true;
+    });
+  });
+
+  if (!matched) {
+    utils.throwError('Unexpected token "' + str.charAt(0) + '" in Jinja2 expression');
+  }
+
+  return matched;
+}
+
+/**
+ * Tokenize a Jinja2 expression string.
+ *
+ * @param  {string}            str Expression source (the contents of
+ *                                 `{{ … }}` or `{% … %}` minus the
+ *                                 control delimiters and tag name).
+ * @return {Array.<LexerToken>}    Sequence of matched tokens.
+ * @throws {Error}                 On the first unrecognised character.
+ */
+exports.read = function (str) {
+  var offset = 0,
+    tokens = [],
+    match;
+  while (offset < str.length) {
+    match = reader(str.substring(offset));
+    offset += match.length;
+    tokens.push(match);
+  }
+  return tokens;
+};