ohm-js 17.2.1 → 17.4.0

package/dist/ohm-extras.js

@@ -12,14 +12,14 @@
 
   function abstract(optMethodName) {
     const methodName = optMethodName || '';
-    return function() {
+    return function () {
       throw new Error(
-          'this method ' +
+        'this method ' +
           methodName +
           ' is abstract! ' +
           '(it has no implementation in class ' +
           this.constructor.name +
-          ')',
+          ')'
       );
     };
   }
@@ -112,11 +112,11 @@
     this.strings = [];
   }
 
-  StringBuffer.prototype.append = function(str) {
+  StringBuffer.prototype.append = function (str) {
     this.strings.push(str);
   };
 
-  StringBuffer.prototype.contents = function() {
+  StringBuffer.prototype.contents = function () {
     return this.strings.join('');
   };
 
@@ -140,9 +140,9 @@
         case 'x':
           return escapeUnicode(s.slice(2, 4));
         case 'u':
-          return s.charAt(2) === '{' ?
-            escapeUnicode(s.slice(3, -1)) :
-            escapeUnicode(s.slice(2, 6));
+          return s.charAt(2) === '{'
+            ? escapeUnicode(s.slice(3, -1))
+            : escapeUnicode(s.slice(2, 6));
         default:
           return s.charAt(1);
       }
@@ -168,7 +168,7 @@
         typeName = typeof obj;
       }
       return typeName + ': ' + JSON.stringify(String(obj));
-    } catch (e) {
+    } catch {
       return baseToString;
     }
   }
@@ -334,9 +334,9 @@
       // Get the next line.
       const nextLineEndOffset = str.indexOf('\n', lineEndOffset + 1);
       nextLine =
-        nextLineEndOffset === -1 ?
-          str.slice(lineEndOffset) :
-          str.slice(lineEndOffset, nextLineEndOffset);
+        nextLineEndOffset === -1
+          ? str.slice(lineEndOffset)
+          : str.slice(lineEndOffset, nextLineEndOffset);
       // Strip leading and trailing EOL char(s).
       nextLine = nextLine.replace(/^\r?\n/, '').replace(/\r$/, '');
     }
@@ -448,11 +448,11 @@
       this._shapes = config.shapes;
       this._getTag = config.getTag;
 
-      this.Adapter = function(thing, family) {
+      this.Adapter = function (thing, family) {
         this._adaptee = thing;
         this._family = family;
       };
-      this.Adapter.prototype.valueOf = function() {
+      this.Adapter.prototype.valueOf = function () {
         throw new Error('heeey!');
       };
       this.operations = {};
@@ -481,15 +481,15 @@
         assert(k in this._getChildren, "Unrecognized action name '" + k + "'");
         const action = dict[k];
         assert(
-            typeof action === 'function',
-            "Key '" + k + "': expected function, got " + action,
+          typeof action === 'function',
+          "Key '" + k + "': expected function, got " + action
         );
         if (k in this._arities) {
           const expected = this._arities[k];
           const actual = dict[k].length;
           assert(
-              actual === expected,
-              "Action '" + k + "' has the wrong arity: expected " + expected + ', got ' + actual,
+            actual === expected,
+            "Action '" + k + "' has the wrong arity: expected " + expected + ', got ' + actual
           );
         }
       });
@@ -506,7 +506,7 @@
       };
 
       const family = this;
-      this.Adapter.prototype[name] = function(...args) {
+      this.Adapter.prototype[name] = function (...args) {
         const tag = family._getTag(this._adaptee);
         assert(tag in family._getChildren, "getTag returned unrecognized tag '" + tag + "'");
         assert(tag in actions, "No action for '" + tag + "' in operation '" + name + "'");
@@ -521,8 +521,8 @@
         const oldArgs = this.args;
         this.args = argsObj;
         const ans = actions[tag].apply(
-            this,
-            family._getChildren[tag](this._adaptee, family._wrap),
+          this,
+          family._getChildren[tag](this._adaptee, family._wrap)
         );
         this.args = oldArgs;
         return ans;
@@ -588,7 +588,7 @@
       const node = {
         type: ctorName,
       };
-      // eslint-disable-next-line guard-for-in
+
       for (const prop in propMap) {
         const mappedProp = mapping[ctorName] && mapping[ctorName][prop];
         if (typeof mappedProp === 'number') {
@@ -663,36 +663,60 @@
     return g.createSemantics().addOperation('toAST(mapping)', defaultOperation);
   }
 
-  // These are just categories that are used in ES5/ES2015.
-  // The full list of Unicode categories is here: http://www.fileformat.info/info/unicode/category/index.htm.
-  const UnicodeCategories = {
-    // Letters
-    Lu: /\p{Lu}/u,
-    Ll: /\p{Ll}/u,
-    Lt: /\p{Lt}/u,
-    Lm: /\p{Lm}/u,
-    Lo: /\p{Lo}/u,
-
-    // Numbers
-    Nl: /\p{Nl}/u,
-    Nd: /\p{Nd}/u,
-
-    // Marks
-    Mn: /\p{Mn}/u,
-    Mc: /\p{Mc}/u,
-
-    // Punctuation, Connector
-    Pc: /\p{Pc}/u,
-
-    // Separator, Space
-    Zs: /\p{Zs}/u,
-
-    // These two are not real Unicode categories, but our useful for Ohm.
-    // L is a combination of all the letter categories.
-    // Ltmo is a combination of Lt, Lm, and Lo.
-    L: /\p{Letter}/u,
-    Ltmo: /\p{Lt}|\p{Lm}|\p{Lo}/u,
-  };
+  // The full list of categories from:
+  // https://www.unicode.org/Public/UCD/latest/ucd/extracted/DerivedGeneralCategory.txt.
+
+  const toRegExp = val => new RegExp(String.raw`\p{${val}}`, 'u');
+
+  /*
+    grep -v '^#' DerivedGeneralCategory.txt \
+      | cut -d';' -f2 \
+      | awk 'NF{print $1}' \
+      | sort -u \
+      | awk '{printf "\x27%s\x27,\n",$1}'
+   */
+
+  const UnicodeCategories = Object.fromEntries(
+    [
+      'Cc',
+      'Cf',
+      'Cn',
+      'Co',
+      'Cs',
+      'Ll',
+      'Lm',
+      'Lo',
+      'Lt',
+      'Lu',
+      'Mc',
+      'Me',
+      'Mn',
+      'Nd',
+      'Nl',
+      'No',
+      'Pc',
+      'Pd',
+      'Pe',
+      'Pf',
+      'Pi',
+      'Po',
+      'Ps',
+      'Sc',
+      'Sk',
+      'Sm',
+      'So',
+      'Zl',
+      'Zp',
+      'Zs',
+    ].map(cat => [cat, toRegExp(cat)])
+  );
+  UnicodeCategories['Ltmo'] = /\p{Lt}|\p{Lm}|\p{Lo}/u;
+
+  // We only support a few of these for now, but could add more later.
+  // See https://www.unicode.org/Public/UCD/latest/ucd/PropertyAliases.txt
+  const UnicodeBinaryProperties = Object.fromEntries(
+    ['XID_Start', 'XID_Continue', 'White_Space'].map(prop => [prop, toRegExp(prop)])
+  );
 
   // --------------------------------------------------------------------
   // Private stuff
@@ -873,10 +897,18 @@
   // Unicode character
 
   class UnicodeChar extends PExpr {
-    constructor(category) {
+    constructor(categoryOrProp) {
       super();
-      this.category = category;
-      this.pattern = UnicodeCategories[category];
+      this.categoryOrProp = categoryOrProp;
+      if (categoryOrProp in UnicodeCategories) {
+        this.pattern = UnicodeCategories[categoryOrProp];
+      } else if (categoryOrProp in UnicodeBinaryProperties) {
+        this.pattern = UnicodeBinaryProperties[categoryOrProp];
+      } else {
+        throw new Error(
+          `Invalid Unicode category or property name: ${JSON.stringify(categoryOrProp)}`
+        );
+      }
     }
   }
 
@@ -927,9 +959,9 @@
   // Undeclared grammar
 
   function undeclaredGrammar(grammarName, namespace, interval) {
-    const message = namespace ?
-      `Grammar ${grammarName} is not declared in namespace '${namespace}'` :
-      'Undeclared grammar ' + grammarName;
+    const message = namespace
+      ? `Grammar ${grammarName} is not declared in namespace '${namespace}'`
+      : 'Undeclared grammar ' + grammarName;
     return createError(message, interval);
   }
 
@@ -949,8 +981,8 @@
 
   function undeclaredRule(ruleName, grammarName, optInterval) {
     return createError(
-        'Rule ' + ruleName + ' is not declared in grammar ' + grammarName,
-        optInterval,
+      'Rule ' + ruleName + ' is not declared in grammar ' + grammarName,
+      optInterval
     );
   }
 
@@ -958,8 +990,8 @@
 
   function cannotOverrideUndeclaredRule(ruleName, grammarName, optSource) {
     return createError(
-        'Cannot override rule ' + ruleName + ' because it is not declared in ' + grammarName,
-        optSource,
+      'Cannot override rule ' + ruleName + ' because it is not declared in ' + grammarName,
+      optSource
     );
   }
 
@@ -967,8 +999,8 @@
 
   function cannotExtendUndeclaredRule(ruleName, grammarName, optSource) {
     return createError(
-        'Cannot extend rule ' + ruleName + ' because it is not declared in ' + grammarName,
-        optSource,
+      'Cannot extend rule ' + ruleName + ' because it is not declared in ' + grammarName,
+      optSource
     );
   }
 
@@ -987,14 +1019,14 @@
 
   function wrongNumberOfParameters(ruleName, expected, actual, source) {
     return createError(
-        'Wrong number of parameters for rule ' +
+      'Wrong number of parameters for rule ' +
         ruleName +
         ' (expected ' +
         expected +
         ', got ' +
         actual +
         ')',
-        source,
+      source
     );
   }
 
@@ -1002,14 +1034,14 @@
 
   function wrongNumberOfArguments(ruleName, expected, actual, expr) {
     return createError(
-        'Wrong number of arguments for rule ' +
+      'Wrong number of arguments for rule ' +
         ruleName +
         ' (expected ' +
         expected +
         ', got ' +
         actual +
         ')',
-        expr,
+      expr
     );
   }
 
@@ -1017,8 +1049,8 @@
 
   function duplicateParameterNames(ruleName, duplicates, source) {
     return createError(
-        'Duplicate parameter names in rule ' + ruleName + ': ' + duplicates.join(', '),
-        source,
+      'Duplicate parameter names in rule ' + ruleName + ': ' + duplicates.join(', '),
+      source
     );
   }
 
@@ -1026,14 +1058,14 @@
 
   function invalidParameter(ruleName, expr) {
     return createError(
-        'Invalid parameter to rule ' +
+      'Invalid parameter to rule ' +
         ruleName +
         ': ' +
         expr +
         ' has arity ' +
         expr.getArity() +
         ', but parameter expressions must have arity 1',
-        expr.source,
+      expr.source
     );
   }
 
@@ -1045,8 +1077,8 @@
 
   function applicationOfSyntacticRuleFromLexicalContext(ruleName, applyExpr) {
     return createError(
-        'Cannot apply syntactic rule ' + ruleName + ' from here (inside a lexical context)',
-        applyExpr.source,
+      'Cannot apply syntactic rule ' + ruleName + ' from here (inside a lexical context)',
+      applyExpr.source
     );
   }
 
@@ -1055,9 +1087,9 @@
   function applySyntacticWithLexicalRuleApplication(applyExpr) {
     const {ruleName} = applyExpr;
     return createError(
-        `applySyntactic is for syntactic rules, but '${ruleName}' is a lexical rule. ` +
+      `applySyntactic is for syntactic rules, but '${ruleName}' is a lexical rule. ` +
         syntacticVsLexicalNote,
-        applyExpr.source,
+      applyExpr.source
     );
   }
 
@@ -1065,8 +1097,8 @@
 
   function unnecessaryExperimentalApplySyntactic(applyExpr) {
     return createError(
-        'applySyntactic is not required here (in a syntactic context)',
-        applyExpr.source,
+      'applySyntactic is not required here (in a syntactic context)',
+      applyExpr.source
     );
   }
 
@@ -1092,8 +1124,8 @@
     const digitIntervals = applyWrapper.children.slice(1, -1).map(d => d.source);
     const fullInterval = digitIntervals[0].coverageWith(...digitIntervals.slice(1));
     return createError(
-        `U+${fullInterval.contents} is not a valid Unicode code point`,
-        fullInterval,
+      `U+${fullInterval.contents} is not a valid Unicode code point`,
+      fullInterval
     );
   }
 
@@ -1111,8 +1143,8 @@
       "' (possible infinite loop)";
     if (applicationStack.length > 0) {
       const stackTrace = applicationStack
-          .map(app => new Apply(app.ruleName, app.args))
-          .join('\n');
+        .map(app => new Apply(app.ruleName, app.args))
+        .join('\n');
       message += '\nApplication stack (most recent application last):\n' + stackTrace;
     }
     return createError(message, kleeneExpr.expr.source);
@@ -1122,7 +1154,7 @@
 
   function inconsistentArity(ruleName, expected, actual, expr) {
     return createError(
-        'Rule ' +
+      'Rule ' +
         ruleName +
         ' involves an alternation which has inconsistent arity ' +
         '(expected ' +
@@ -1130,7 +1162,7 @@
         ', got ' +
         actual +
         ')',
-        expr.source,
+      expr.source
     );
   }
 
@@ -1145,12 +1177,12 @@
 
   function missingSemanticAction(ctorName, name, type, stack) {
     let stackTrace = stack
-        .slice(0, -1)
-        .map(info => {
-          const ans = '  ' + info[0].name + ' > ' + info[1];
-          return info.length === 3 ? ans + " for '" + info[2] + "'" : ans;
-        })
-        .join('\n');
+      .slice(0, -1)
+      .map(info => {
+        const ans = '  ' + info[0].name + ' > ' + info[1];
+        return info.length === 3 ? ans + " for '" + info[2] + "'" : ans;
+      })
+      .join('\n');
     stackTrace += '\n  ' + name + ' > ' + ctorName;
 
     let moreInfo = '';
@@ -1187,11 +1219,23 @@
 
   class Interval {
     constructor(sourceString, startIdx, endIdx) {
-      this.sourceString = sourceString;
+      // Store the full source in a non-enumerable property, so that when
+      // grammars and other objects are printed in the REPL, it's not
+      // cluttered with multiple copies of the same long string.
+      Object.defineProperty(this, '_sourceString', {
+        value: sourceString,
+        configurable: false,
+        enumerable: false,
+        writable: false,
+      });
       this.startIdx = startIdx;
       this.endIdx = endIdx;
     }
 
+    get sourceString() {
+      return this._sourceString;
+    }
+
     get contents() {
       if (this._contents === undefined) {
         this._contents = this.sourceString.slice(this.startIdx, this.endIdx);
@@ -1257,13 +1301,13 @@
         throw intervalSourcesDontMatch();
       }
       assert(
-          this.startIdx >= that.startIdx && this.endIdx <= that.endIdx,
-          'other interval does not cover this one',
+        this.startIdx >= that.startIdx && this.endIdx <= that.endIdx,
+        'other interval does not cover this one'
       );
       return new Interval(
-          this.sourceString,
-          this.startIdx - that.startIdx,
-          this.endIdx - that.startIdx,
+        this.sourceString,
+        this.startIdx - that.startIdx,
+        this.endIdx - that.startIdx
       );
     }
 
@@ -1282,7 +1326,7 @@
     }
   }
 
-  Interval.coverage = function(firstInterval, ...intervals) {
+  Interval.coverage = function (firstInterval, ...intervals) {
     let {startIdx, endIdx} = firstInterval;
     for (const interval of intervals) {
       if (interval.sourceString !== firstInterval.sourceString) {
@@ -1296,6 +1340,7 @@
   };
 
   const MAX_CHAR_CODE = 0xffff;
+  const MAX_CODE_POINT = 0x10ffff;
 
   class InputStream {
     constructor(source) {
@@ -1341,6 +1386,8 @@
 
           This is intended to be a locale-invariant comparison, which means it may not obey
           locale-specific expectations (e.g. "i" => "İ").
+
+          See also https://unicode.org/faq/casemap_charprop.html#casemap
          */
         for (idx = 0; idx < s.length; idx++) {
           const actual = this.next();
@@ -1375,13 +1422,13 @@
 
   class MatchResult {
     constructor(
-        matcher,
-        input,
-        startExpr,
-        cst,
-        cstOffset,
-        rightmostFailurePosition,
-        optRecordedFailures,
+      matcher,
+      input,
+      startExpr,
+      cst,
+      cstOffset,
+      rightmostFailurePosition,
+      optRecordedFailures
     ) {
       this.matcher = matcher;
       this.input = input;
@@ -1392,22 +1439,20 @@
       this._rightmostFailures = optRecordedFailures;
 
       if (this.failed()) {
-        /* eslint-disable no-invalid-this */
-        defineLazyProperty(this, 'message', function() {
+        defineLazyProperty(this, 'message', function () {
           const detail = 'Expected ' + this.getExpectedText();
           return (
             getLineAndColumnMessage(this.input, this.getRightmostFailurePosition()) + detail
           );
         });
-        defineLazyProperty(this, 'shortMessage', function() {
+        defineLazyProperty(this, 'shortMessage', function () {
           const detail = 'expected ' + this.getExpectedText();
           const errorInfo = getLineAndColumn(
-              this.input,
-              this.getRightmostFailurePosition(),
+            this.input,
+            this.getRightmostFailurePosition()
           );
           return 'Line ' + errorInfo.lineNum + ', col ' + errorInfo.colNum + ': ' + detail;
         });
-        /* eslint-enable no-invalid-this */
       }
     }
 
@@ -1436,9 +1481,9 @@
     }
 
     toString() {
-      return this.succeeded() ?
-        '[match succeeded]' :
-        '[match failed at position ' + this.getRightmostFailurePosition() + ']';
+      return this.succeeded()
+        ? '[match succeeded]'
+        : '[match failed at position ' + this.getRightmostFailurePosition() + ']';
     }
 
     // Return a string summarizing the expected contents of the input stream when
@@ -1504,14 +1549,14 @@
       const indexOfFirstInvolvedRule =
         applicationMemoKeyStack.indexOf(headApplication.toMemoKey()) + 1;
       const involvedApplicationMemoKeys = applicationMemoKeyStack.slice(
-          indexOfFirstInvolvedRule,
+        indexOfFirstInvolvedRule
       );
 
-      memoRec.isInvolved = function(applicationMemoKey) {
+      memoRec.isInvolved = function (applicationMemoKey) {
         return involvedApplicationMemoKeys.indexOf(applicationMemoKey) >= 0;
       };
 
-      memoRec.updateInvolvedApplicationMemoKeys = function() {
+      memoRec.updateInvolvedApplicationMemoKeys = function () {
         for (let idx = indexOfFirstInvolvedRule; idx < applicationMemoKeyStack.length; idx++) {
           const applicationMemoKey = applicationMemoKeyStack[idx];
           if (!this.isInvolved(applicationMemoKey)) {
@@ -1545,8 +1590,8 @@
       this.memo[memoKey] = memoRec;
       this.maxExaminedLength = Math.max(this.maxExaminedLength, memoRec.examinedLength);
       this.maxRightmostFailureOffset = Math.max(
-          this.maxRightmostFailureOffset,
-          memoRec.rightmostFailureOffset,
+        this.maxRightmostFailureOffset,
+        memoRec.rightmostFailureOffset
       );
       return memoRec;
     }
@@ -1568,8 +1613,8 @@
         } else {
           this.maxExaminedLength = Math.max(this.maxExaminedLength, memoRec.examinedLength);
           this.maxRightmostFailureOffset = Math.max(
-              this.maxRightmostFailureOffset,
-              memoRec.rightmostFailureOffset,
+            this.maxRightmostFailureOffset,
+            memoRec.rightmostFailureOffset
           );
         }
       });
@@ -1618,10 +1663,10 @@
     if (typeof obj === 'string') {
       // Replace non-printable characters with visible symbols.
       return obj
-          .replace(/ /g, DOT_OPERATOR)
-          .replace(/\t/g, SYMBOL_FOR_HORIZONTAL_TABULATION)
-          .replace(/\n/g, SYMBOL_FOR_LINE_FEED)
-          .replace(/\r/g, SYMBOL_FOR_CARRIAGE_RETURN);
+        .replace(/ /g, DOT_OPERATOR)
+        .replace(/\t/g, SYMBOL_FOR_HORIZONTAL_TABULATION)
+        .replace(/\n/g, SYMBOL_FOR_LINE_FEED)
+        .replace(/\r/g, SYMBOL_FOR_CARRIAGE_RETURN);
     }
     return String(obj);
   }
@@ -1652,13 +1697,13 @@
 
     cloneWithExpr(expr) {
       const ans = new Trace(
-          this.input,
-          this.pos,
-          this.pos2,
-          expr,
-          this.succeeded,
-          this.bindings,
-          this.children,
+        this.input,
+        this.pos,
+        this.pos2,
+        expr,
+        this.succeeded,
+        this.bindings,
+        this.children
       );
 
       ans.isHeadOfLeftRecursion = this.isHeadOfLeftRecursion;
@@ -1673,13 +1718,13 @@
     // Record the trace information for the terminating condition of the LR loop.
     recordLRTermination(ruleBodyTrace, value) {
       this.terminatingLREntry = new Trace(
-          this.input,
-          this.pos,
-          this.pos2,
-          this.expr,
-          false,
-          [value],
-          [ruleBodyTrace],
+        this.input,
+        this.pos,
+        this.pos2,
+        this.expr,
+        false,
+        [value],
+        [ruleBodyTrace]
       );
       this.terminatingLREntry.terminatesLR = true;
     }
@@ -1739,7 +1784,7 @@
         const ctorName = node.expr.constructor.name;
         // Don't print anything for Alt nodes.
         if (ctorName === 'Alt') {
-          return; // eslint-disable-line consistent-return
+          return;
         }
         sb.append(getInputExcerpt(node.input, node.pos, 10) + spaces(depth * 2 + 1));
         sb.append((node.succeeded ? CHECK_MARK : BALLOT_X) + ' ' + node.displayString);
@@ -1797,7 +1842,7 @@
     Terminal.prototype.allowsSkippingPrecedingSpace =
     Range.prototype.allowsSkippingPrecedingSpace =
     UnicodeChar.prototype.allowsSkippingPrecedingSpace =
-      function() {
+      function () {
         return true;
       };
 
@@ -1811,7 +1856,7 @@
     Not.prototype.allowsSkippingPrecedingSpace =
     Param.prototype.allowsSkippingPrecedingSpace =
     Seq.prototype.allowsSkippingPrecedingSpace =
-      function() {
+      function () {
         return false;
       };
 
@@ -1827,13 +1872,13 @@
 
   let lexifyCount;
 
-  PExpr.prototype.assertAllApplicationsAreValid = function(ruleName, grammar) {
+  PExpr.prototype.assertAllApplicationsAreValid = function (ruleName, grammar) {
     lexifyCount = 0;
     this._assertAllApplicationsAreValid(ruleName, grammar);
   };
 
   PExpr.prototype._assertAllApplicationsAreValid = abstract(
-      '_assertAllApplicationsAreValid',
+    '_assertAllApplicationsAreValid'
   );
 
   any._assertAllApplicationsAreValid =
@@ -1842,23 +1887,23 @@
     Range.prototype._assertAllApplicationsAreValid =
     Param.prototype._assertAllApplicationsAreValid =
     UnicodeChar.prototype._assertAllApplicationsAreValid =
-      function(ruleName, grammar) {
+      function (ruleName, grammar) {
         // no-op
       };
 
-  Lex.prototype._assertAllApplicationsAreValid = function(ruleName, grammar) {
+  Lex.prototype._assertAllApplicationsAreValid = function (ruleName, grammar) {
     lexifyCount++;
     this.expr._assertAllApplicationsAreValid(ruleName, grammar);
     lexifyCount--;
   };
 
-  Alt.prototype._assertAllApplicationsAreValid = function(ruleName, grammar) {
+  Alt.prototype._assertAllApplicationsAreValid = function (ruleName, grammar) {
     for (let idx = 0; idx < this.terms.length; idx++) {
       this.terms[idx]._assertAllApplicationsAreValid(ruleName, grammar);
     }
   };
 
-  Seq.prototype._assertAllApplicationsAreValid = function(ruleName, grammar) {
+  Seq.prototype._assertAllApplicationsAreValid = function (ruleName, grammar) {
     for (let idx = 0; idx < this.factors.length; idx++) {
       this.factors[idx]._assertAllApplicationsAreValid(ruleName, grammar);
     }
@@ -1867,14 +1912,14 @@
   Iter.prototype._assertAllApplicationsAreValid =
     Not.prototype._assertAllApplicationsAreValid =
     Lookahead.prototype._assertAllApplicationsAreValid =
-      function(ruleName, grammar) {
+      function (ruleName, grammar) {
         this.expr._assertAllApplicationsAreValid(ruleName, grammar);
       };
 
-  Apply.prototype._assertAllApplicationsAreValid = function(
-      ruleName,
-      grammar,
-      skipSyntacticCheck = false,
+  Apply.prototype._assertAllApplicationsAreValid = function (
+    ruleName,
+    grammar,
+    skipSyntacticCheck = false
   ) {
     const ruleInfo = grammar.rules[this.ruleName];
     const isContextSyntactic = isSyntactic(ruleName) && lexifyCount === 0;
@@ -1937,7 +1982,7 @@
   // --------------------------------------------------------------------
 
   PExpr.prototype.assertChoicesHaveUniformArity = abstract(
-      'assertChoicesHaveUniformArity',
+    'assertChoicesHaveUniformArity'
   );
 
   any.assertChoicesHaveUniformArity =
@@ -1947,11 +1992,11 @@
     Param.prototype.assertChoicesHaveUniformArity =
     Lex.prototype.assertChoicesHaveUniformArity =
     UnicodeChar.prototype.assertChoicesHaveUniformArity =
-      function(ruleName) {
+      function (ruleName) {
         // no-op
       };
 
-  Alt.prototype.assertChoicesHaveUniformArity = function(ruleName) {
+  Alt.prototype.assertChoicesHaveUniformArity = function (ruleName) {
     if (this.terms.length === 0) {
       return;
     }
@@ -1966,7 +2011,7 @@
     }
   };
 
-  Extend.prototype.assertChoicesHaveUniformArity = function(ruleName) {
+  Extend.prototype.assertChoicesHaveUniformArity = function (ruleName) {
     // Extend is a special case of Alt that's guaranteed to have exactly two
     // cases: [extensions, origBody].
     const actualArity = this.terms[0].getArity();
@@ -1976,25 +2021,25 @@
     }
   };
 
-  Seq.prototype.assertChoicesHaveUniformArity = function(ruleName) {
+  Seq.prototype.assertChoicesHaveUniformArity = function (ruleName) {
     for (let idx = 0; idx < this.factors.length; idx++) {
       this.factors[idx].assertChoicesHaveUniformArity(ruleName);
     }
   };
 
-  Iter.prototype.assertChoicesHaveUniformArity = function(ruleName) {
+  Iter.prototype.assertChoicesHaveUniformArity = function (ruleName) {
     this.expr.assertChoicesHaveUniformArity(ruleName);
   };
 
-  Not.prototype.assertChoicesHaveUniformArity = function(ruleName) {
+  Not.prototype.assertChoicesHaveUniformArity = function (ruleName) {
     // no-op (not required b/c the nested expr doesn't show up in the CST)
   };
 
-  Lookahead.prototype.assertChoicesHaveUniformArity = function(ruleName) {
+  Lookahead.prototype.assertChoicesHaveUniformArity = function (ruleName) {
     this.expr.assertChoicesHaveUniformArity(ruleName);
   };
 
-  Apply.prototype.assertChoicesHaveUniformArity = function(ruleName) {
+  Apply.prototype.assertChoicesHaveUniformArity = function (ruleName) {
     // The arities of the parameter expressions is required to be 1 by
     // `assertAllApplicationsAreValid()`.
   };
@@ -2004,7 +2049,7 @@
   // --------------------------------------------------------------------
 
   PExpr.prototype.assertIteratedExprsAreNotNullable = abstract(
-      'assertIteratedExprsAreNotNullable',
+    'assertIteratedExprsAreNotNullable'
   );
 
   any.assertIteratedExprsAreNotNullable =
@@ -2013,23 +2058,23 @@
     Range.prototype.assertIteratedExprsAreNotNullable =
     Param.prototype.assertIteratedExprsAreNotNullable =
     UnicodeChar.prototype.assertIteratedExprsAreNotNullable =
-      function(grammar) {
+      function (grammar) {
         // no-op
       };
 
-  Alt.prototype.assertIteratedExprsAreNotNullable = function(grammar) {
+  Alt.prototype.assertIteratedExprsAreNotNullable = function (grammar) {
     for (let idx = 0; idx < this.terms.length; idx++) {
       this.terms[idx].assertIteratedExprsAreNotNullable(grammar);
     }
   };
 
-  Seq.prototype.assertIteratedExprsAreNotNullable = function(grammar) {
+  Seq.prototype.assertIteratedExprsAreNotNullable = function (grammar) {
     for (let idx = 0; idx < this.factors.length; idx++) {
       this.factors[idx].assertIteratedExprsAreNotNullable(grammar);
     }
   };
 
-  Iter.prototype.assertIteratedExprsAreNotNullable = function(grammar) {
+  Iter.prototype.assertIteratedExprsAreNotNullable = function (grammar) {
     // Note: this is the implementation of this method for `Star` and `Plus` expressions.
     // It is overridden for `Opt` below.
     this.expr.assertIteratedExprsAreNotNullable(grammar);
@@ -2042,11 +2087,11 @@
     Not.prototype.assertIteratedExprsAreNotNullable =
     Lookahead.prototype.assertIteratedExprsAreNotNullable =
     Lex.prototype.assertIteratedExprsAreNotNullable =
-      function(grammar) {
+      function (grammar) {
         this.expr.assertIteratedExprsAreNotNullable(grammar);
       };
 
-  Apply.prototype.assertIteratedExprsAreNotNullable = function(grammar) {
+  Apply.prototype.assertIteratedExprsAreNotNullable = function (grammar) {
     this.args.forEach(arg => {
       arg.assertIteratedExprsAreNotNullable(grammar);
     });
@@ -2090,11 +2135,11 @@
     onlyChild() {
       if (this.numChildren() !== 1) {
         throw new Error(
-            'cannot get only child of a node of type ' +
+          'cannot get only child of a node of type ' +
             this.ctorName +
             ' (it has ' +
             this.numChildren() +
-            ' children)',
+            ' children)'
         );
       } else {
         return this.firstChild();
@@ -2104,7 +2149,7 @@
     firstChild() {
       if (this.hasNoChildren()) {
         throw new Error(
-            'cannot get first child of a ' + this.ctorName + ' node, which has no children',
+          'cannot get first child of a ' + this.ctorName + ' node, which has no children'
         );
       } else {
         return this.childAt(0);
@@ -2114,7 +2159,7 @@
     lastChild() {
       if (this.hasNoChildren()) {
         throw new Error(
-            'cannot get last child of a ' + this.ctorName + ' node, which has no children',
+          'cannot get last child of a ' + this.ctorName + ' node, which has no children'
         );
       } else {
         return this.childAt(this.numChildren() - 1);
@@ -2248,7 +2293,7 @@
   */
   PExpr.prototype.eval = abstract('eval'); // function(state) { ... }
 
-  any.eval = function(state) {
+  any.eval = function (state) {
     const {inputStream} = state;
     const origPos = inputStream.pos;
     const cp = inputStream.nextCodePoint();
@@ -2261,7 +2306,7 @@
     }
   };
 
-  end.eval = function(state) {
+  end.eval = function (state) {
     const {inputStream} = state;
     const origPos = inputStream.pos;
     if (inputStream.atEnd()) {
@@ -2273,7 +2318,7 @@
     }
   };
 
-  Terminal.prototype.eval = function(state) {
+  Terminal.prototype.eval = function (state) {
     const {inputStream} = state;
     const origPos = inputStream.pos;
     if (!inputStream.matchString(this.obj)) {
@@ -2285,7 +2330,7 @@
     }
   };
 
-  Range.prototype.eval = function(state) {
+  Range.prototype.eval = function (state) {
     const {inputStream} = state;
     const origPos = inputStream.pos;
 
@@ -2304,18 +2349,18 @@
     }
   };
 
-  Param.prototype.eval = function(state) {
+  Param.prototype.eval = function (state) {
     return state.eval(state.currentApplication().args[this.index]);
   };
 
-  Lex.prototype.eval = function(state) {
+  Lex.prototype.eval = function (state) {
     state.enterLexifiedContext();
     const ans = state.eval(this.expr);
     state.exitLexifiedContext();
     return ans;
   };
 
-  Alt.prototype.eval = function(state) {
+  Alt.prototype.eval = function (state) {
     for (let idx = 0; idx < this.terms.length; idx++) {
       if (state.eval(this.terms[idx])) {
         return true;
@@ -2324,7 +2369,7 @@
     return false;
   };
 
-  Seq.prototype.eval = function(state) {
+  Seq.prototype.eval = function (state) {
     for (let idx = 0; idx < this.factors.length; idx++) {
       const factor = this.factors[idx];
       if (!state.eval(factor)) {
@@ -2334,7 +2379,7 @@
     return true;
   };
 
-  Iter.prototype.eval = function(state) {
+  Iter.prototype.eval = function (state) {
     const {inputStream} = state;
     const origPos = inputStream.pos;
     const arity = this.getArity();
@@ -2356,8 +2401,8 @@
       numMatches++;
       const row = state._bindings.splice(state._bindings.length - arity, arity);
       const rowOffsets = state._bindingOffsets.splice(
-          state._bindingOffsets.length - arity,
-          arity,
+        state._bindingOffsets.length - arity,
+        arity
       );
       for (idx = 0; idx < row.length; idx++) {
         cols[idx].push(row[idx]);
@@ -2381,14 +2426,14 @@
     const isOptional = this instanceof Opt;
     for (idx = 0; idx < cols.length; idx++) {
       state._bindings.push(
-          new IterationNode(cols[idx], colOffsets[idx], matchLength, isOptional),
+        new IterationNode(cols[idx], colOffsets[idx], matchLength, isOptional)
       );
       state._bindingOffsets.push(offset);
     }
     return true;
   };
 
-  Not.prototype.eval = function(state) {
+  Not.prototype.eval = function (state) {
     /*
       TODO:
       - Right now we're just throwing away all of the failures that happen inside a `not`, and
@@ -2414,7 +2459,7 @@
     return true;
   };
 
-  Lookahead.prototype.eval = function(state) {
+  Lookahead.prototype.eval = function (state) {
     const {inputStream} = state;
     const origPos = inputStream.pos;
     if (state.eval(this.expr)) {
@@ -2425,7 +2470,7 @@
     }
   };
 
-  Apply.prototype.eval = function(state) {
+  Apply.prototype.eval = function (state) {
     const caller = state.currentApplication();
     const actuals = caller ? caller.args : [];
     const app = this.substituteParams(actuals);
@@ -2448,7 +2493,7 @@
     return app.reallyEval(state);
   };
 
-  Apply.prototype.handleCycle = function(state) {
+  Apply.prototype.handleCycle = function (state) {
     const posInfo = state.getCurrentPosInfo();
     const {currentLeftRecursion} = posInfo;
     const memoKey = this.toMemoKey();
@@ -2471,7 +2516,7 @@
     return state.useMemoizedResult(state.inputStream.pos, memoRec);
   };
 
-  Apply.prototype.reallyEval = function(state) {
+  Apply.prototype.reallyEval = function (state) {
     const {inputStream} = state;
     const origPos = inputStream.pos;
     const origPosInfo = state.getCurrentPosInfo();
@@ -2524,6 +2569,7 @@
       }
       if (memoRec) {
         memoRec.failuresAtRightmostPosition = state.cloneRecordedFailures();
+        memoRec.rightmostFailureOffset = state._getRightmostFailureOffset();
       }
     }
 
@@ -2541,8 +2587,8 @@
     // Fix the input stream's examinedLength -- it should be the maximum examined length
     // across all applications, not just this one.
     inputStream.examinedLength = Math.max(
-        inputStream.examinedLength,
-        origInputStreamExaminedLength,
+      inputStream.examinedLength,
+      origInputStreamExaminedLength
     );
 
     state.exitApplication(origPosInfo, value);
@@ -2550,7 +2596,7 @@
     return succeeded;
   };
 
-  Apply.prototype.evalOnce = function(expr, state) {
+  Apply.prototype.evalOnce = function (expr, state) {
     const {inputStream} = state;
     const origPos = inputStream.pos;
 
@@ -2565,7 +2611,7 @@
     }
   };
 
-  Apply.prototype.growSeedResult = function(body, state, origPos, lrMemoRec, newValue) {
+  Apply.prototype.growSeedResult = function (body, state, origPos, lrMemoRec, newValue) {
     if (!newValue) {
       return false;
     }
@@ -2583,13 +2629,13 @@
         // element in `state.trace`.
         const seedTrace = state.trace[state.trace.length - 1];
         lrMemoRec.traceEntry = new Trace(
-            state.input,
-            origPos,
-            inputStream.pos,
-            this,
-            true,
-            [newValue],
-            [seedTrace.clone()],
+          state.input,
+          origPos,
+          inputStream.pos,
+          this,
+          true,
+          [newValue],
+          [seedTrace.clone()]
         );
       }
       inputStream.pos = origPos;
@@ -2609,17 +2655,19 @@
     return lrMemoRec.value;
   };
 
-  UnicodeChar.prototype.eval = function(state) {
+  UnicodeChar.prototype.eval = function (state) {
     const {inputStream} = state;
     const origPos = inputStream.pos;
-    const ch = inputStream.next();
-    if (ch && this.pattern.test(ch)) {
-      state.pushBinding(new TerminalNode(ch.length), origPos);
-      return true;
-    } else {
-      state.processFailure(origPos, this);
-      return false;
+    const cp = inputStream.nextCodePoint();
+    if (cp !== undefined && cp <= MAX_CODE_POINT) {
+      const ch = String.fromCodePoint(cp);
+      if (this.pattern.test(ch)) {
+        state.pushBinding(new TerminalNode(ch.length), origPos);
+        return true;
+      }
     }
+    state.processFailure(origPos, this);
+    return false;
   };
 
   // --------------------------------------------------------------------
@@ -2635,17 +2683,17 @@
     Param.prototype.getArity =
     Apply.prototype.getArity =
     UnicodeChar.prototype.getArity =
-      function() {
+      function () {
         return 1;
       };
 
-  Alt.prototype.getArity = function() {
+  Alt.prototype.getArity = function () {
     // This is ok b/c all terms must have the same arity -- this property is
     // checked by the Grammar constructor.
     return this.terms.length === 0 ? 0 : this.terms[0].getArity();
   };
 
-  Seq.prototype.getArity = function() {
+  Seq.prototype.getArity = function () {
     let arity = 0;
     for (let idx = 0; idx < this.factors.length; idx++) {
       arity += this.factors[idx].getArity();
@@ -2653,15 +2701,15 @@
     return arity;
   };
 
-  Iter.prototype.getArity = function() {
+  Iter.prototype.getArity = function () {
     return this.expr.getArity();
   };
 
-  Not.prototype.getArity = function() {
+  Not.prototype.getArity = function () {
     return 0;
   };
 
-  Lookahead.prototype.getArity = Lex.prototype.getArity = function() {
+  Lookahead.prototype.getArity = Lex.prototype.getArity = function () {
     return this.expr.getArity();
   };
 
@@ -2684,38 +2732,38 @@
 
   PExpr.prototype.outputRecipe = abstract('outputRecipe');
 
-  any.outputRecipe = function(formals, grammarInterval) {
+  any.outputRecipe = function (formals, grammarInterval) {
     return ['any', getMetaInfo(this, grammarInterval)];
   };
 
-  end.outputRecipe = function(formals, grammarInterval) {
+  end.outputRecipe = function (formals, grammarInterval) {
     return ['end', getMetaInfo(this, grammarInterval)];
   };
 
-  Terminal.prototype.outputRecipe = function(formals, grammarInterval) {
+  Terminal.prototype.outputRecipe = function (formals, grammarInterval) {
     return ['terminal', getMetaInfo(this, grammarInterval), this.obj];
   };
 
-  Range.prototype.outputRecipe = function(formals, grammarInterval) {
+  Range.prototype.outputRecipe = function (formals, grammarInterval) {
     return ['range', getMetaInfo(this, grammarInterval), this.from, this.to];
   };
 
-  Param.prototype.outputRecipe = function(formals, grammarInterval) {
+  Param.prototype.outputRecipe = function (formals, grammarInterval) {
     return ['param', getMetaInfo(this, grammarInterval), this.index];
   };
 
-  Alt.prototype.outputRecipe = function(formals, grammarInterval) {
+  Alt.prototype.outputRecipe = function (formals, grammarInterval) {
     return ['alt', getMetaInfo(this, grammarInterval)].concat(
-        this.terms.map(term => term.outputRecipe(formals, grammarInterval)),
+      this.terms.map(term => term.outputRecipe(formals, grammarInterval))
     );
   };
 
-  Extend.prototype.outputRecipe = function(formals, grammarInterval) {
+  Extend.prototype.outputRecipe = function (formals, grammarInterval) {
     const extension = this.terms[0]; // [extension, original]
     return extension.outputRecipe(formals, grammarInterval);
   };
 
-  Splice.prototype.outputRecipe = function(formals, grammarInterval) {
+  Splice.prototype.outputRecipe = function (formals, grammarInterval) {
     const beforeTerms = this.terms.slice(0, this.expansionPos);
     const afterTerms = this.terms.slice(this.expansionPos + 1);
     return [
@@ -2726,9 +2774,9 @@
     ];
   };
 
-  Seq.prototype.outputRecipe = function(formals, grammarInterval) {
+  Seq.prototype.outputRecipe = function (formals, grammarInterval) {
     return ['seq', getMetaInfo(this, grammarInterval)].concat(
-        this.factors.map(factor => factor.outputRecipe(formals, grammarInterval)),
+      this.factors.map(factor => factor.outputRecipe(formals, grammarInterval))
     );
   };
 
@@ -2738,7 +2786,7 @@
     Not.prototype.outputRecipe =
     Lookahead.prototype.outputRecipe =
     Lex.prototype.outputRecipe =
-      function(formals, grammarInterval) {
+      function (formals, grammarInterval) {
         return [
           this.constructor.name.toLowerCase(),
           getMetaInfo(this, grammarInterval),
@@ -2746,7 +2794,7 @@
         ];
       };
 
-  Apply.prototype.outputRecipe = function(formals, grammarInterval) {
+  Apply.prototype.outputRecipe = function (formals, grammarInterval) {
     return [
       'app',
       getMetaInfo(this, grammarInterval),
@@ -2755,8 +2803,8 @@
     ];
   };
 
-  UnicodeChar.prototype.outputRecipe = function(formals, grammarInterval) {
-    return ['unicodeChar', getMetaInfo(this, grammarInterval), this.category];
+  UnicodeChar.prototype.outputRecipe = function (formals, grammarInterval) {
+    return ['unicodeChar', getMetaInfo(this, grammarInterval), this.categoryOrProp];
   };
 
   // --------------------------------------------------------------------
@@ -2776,18 +2824,18 @@
     Range.prototype.introduceParams =
     Param.prototype.introduceParams =
     UnicodeChar.prototype.introduceParams =
-      function(formals) {
+      function (formals) {
         return this;
       };
 
-  Alt.prototype.introduceParams = function(formals) {
+  Alt.prototype.introduceParams = function (formals) {
     this.terms.forEach((term, idx, terms) => {
       terms[idx] = term.introduceParams(formals);
     });
     return this;
   };
 
-  Seq.prototype.introduceParams = function(formals) {
+  Seq.prototype.introduceParams = function (formals) {
     this.factors.forEach((factor, idx, factors) => {
       factors[idx] = factor.introduceParams(formals);
     });
@@ -2798,12 +2846,12 @@
     Not.prototype.introduceParams =
     Lookahead.prototype.introduceParams =
     Lex.prototype.introduceParams =
-      function(formals) {
+      function (formals) {
         this.expr = this.expr.introduceParams(formals);
         return this;
       };
 
-  Apply.prototype.introduceParams = function(formals) {
+  Apply.prototype.introduceParams = function (formals) {
     const index = formals.indexOf(this.ruleName);
     if (index >= 0) {
       if (this.args.length > 0) {
@@ -2824,7 +2872,7 @@
   // --------------------------------------------------------------------
 
   // Returns `true` if this parsing expression may accept without consuming any input.
-  PExpr.prototype.isNullable = function(grammar) {
+  PExpr.prototype.isNullable = function (grammar) {
     return this._isNullable(grammar, Object.create(null));
   };
 
@@ -2835,15 +2883,15 @@
     Param.prototype._isNullable =
     Plus.prototype._isNullable =
     UnicodeChar.prototype._isNullable =
-      function(grammar, memo) {
+      function (grammar, memo) {
         return false;
       };
 
-  end._isNullable = function(grammar, memo) {
+  end._isNullable = function (grammar, memo) {
     return true;
   };
 
-  Terminal.prototype._isNullable = function(grammar, memo) {
+  Terminal.prototype._isNullable = function (grammar, memo) {
     if (typeof this.obj === 'string') {
       // This is an over-simplification: it's only correct if the input is a string. If it's an array
       // or an object, then the empty string parsing expression is not nullable.
@@ -2853,11 +2901,11 @@
     }
   };
 
-  Alt.prototype._isNullable = function(grammar, memo) {
+  Alt.prototype._isNullable = function (grammar, memo) {
     return this.terms.length === 0 || this.terms.some(term => term._isNullable(grammar, memo));
   };
 
-  Seq.prototype._isNullable = function(grammar, memo) {
+  Seq.prototype._isNullable = function (grammar, memo) {
     return this.factors.every(factor => factor._isNullable(grammar, memo));
   };
 
@@ -2865,15 +2913,15 @@
     Opt.prototype._isNullable =
     Not.prototype._isNullable =
     Lookahead.prototype._isNullable =
-      function(grammar, memo) {
+      function (grammar, memo) {
         return true;
       };
 
-  Lex.prototype._isNullable = function(grammar, memo) {
+  Lex.prototype._isNullable = function (grammar, memo) {
     return this.expr._isNullable(grammar, memo);
   };
 
-  Apply.prototype._isNullable = function(grammar, memo) {
+  Apply.prototype._isNullable = function (grammar, memo) {
     const key = this.toMemoKey();
     if (!Object.prototype.hasOwnProperty.call(memo, key)) {
       const {body} = grammar.rules[this.ruleName];
@@ -2902,19 +2950,19 @@
     Terminal.prototype.substituteParams =
     Range.prototype.substituteParams =
     UnicodeChar.prototype.substituteParams =
-      function(actuals) {
+      function (actuals) {
         return this;
       };
 
-  Param.prototype.substituteParams = function(actuals) {
+  Param.prototype.substituteParams = function (actuals) {
     return checkNotNull(actuals[this.index]);
   };
 
-  Alt.prototype.substituteParams = function(actuals) {
+  Alt.prototype.substituteParams = function (actuals) {
     return new Alt(this.terms.map(term => term.substituteParams(actuals)));
   };
 
-  Seq.prototype.substituteParams = function(actuals) {
+  Seq.prototype.substituteParams = function (actuals) {
     return new Seq(this.factors.map(factor => factor.substituteParams(actuals)));
   };
 
@@ -2922,11 +2970,11 @@
     Not.prototype.substituteParams =
     Lookahead.prototype.substituteParams =
     Lex.prototype.substituteParams =
-      function(actuals) {
+      function (actuals) {
         return new this.constructor(this.expr.substituteParams(actuals));
       };
 
-  Apply.prototype.substituteParams = function(actuals) {
+  Apply.prototype.substituteParams = function (actuals) {
     if (this.args.length === 0) {
       // Avoid making a copy of this application, as an optimization
       return this;
@@ -3000,15 +3048,15 @@
   // function(firstArgIndex, noDupCheck) { ... }
   PExpr.prototype.toArgumentNameList = abstract('toArgumentNameList');
 
-  any.toArgumentNameList = function(firstArgIndex, noDupCheck) {
+  any.toArgumentNameList = function (firstArgIndex, noDupCheck) {
     return ['any'];
   };
 
-  end.toArgumentNameList = function(firstArgIndex, noDupCheck) {
+  end.toArgumentNameList = function (firstArgIndex, noDupCheck) {
     return ['end'];
   };
 
-  Terminal.prototype.toArgumentNameList = function(firstArgIndex, noDupCheck) {
+  Terminal.prototype.toArgumentNameList = function (firstArgIndex, noDupCheck) {
     if (typeof this.obj === 'string' && /^[_a-zA-Z0-9]+$/.test(this.obj)) {
       // If this terminal is a valid suffix for a JS identifier, just prepend it with '_'
       return ['_' + this.obj];
@@ -3018,7 +3066,7 @@
     }
   };
 
-  Range.prototype.toArgumentNameList = function(firstArgIndex, noDupCheck) {
+  Range.prototype.toArgumentNameList = function (firstArgIndex, noDupCheck) {
     let argName = this.from + '_to_' + this.to;
     // If the `argName` is not valid then try to prepend a `_`.
     if (!isRestrictedJSIdentifier(argName)) {
@@ -3031,11 +3079,11 @@
     return [argName];
   };
 
-  Alt.prototype.toArgumentNameList = function(firstArgIndex, noDupCheck) {
+  Alt.prototype.toArgumentNameList = function (firstArgIndex, noDupCheck) {
     // `termArgNameLists` is an array of arrays where each row is the
     // argument name list that corresponds to a term in this alternation.
     const termArgNameLists = this.terms.map(term =>
-      term.toArgumentNameList(firstArgIndex, true),
+      term.toArgumentNameList(firstArgIndex, true)
     );
 
     const argumentNameList = [];
@@ -3055,7 +3103,7 @@
     return argumentNameList;
   };
 
-  Seq.prototype.toArgumentNameList = function(firstArgIndex, noDupCheck) {
+  Seq.prototype.toArgumentNameList = function (firstArgIndex, noDupCheck) {
     // Generate the argument name list, without worrying about duplicates.
     let argumentNameList = [];
     this.factors.forEach(factor => {
@@ -3071,44 +3119,44 @@
     return argumentNameList;
   };
 
-  Iter.prototype.toArgumentNameList = function(firstArgIndex, noDupCheck) {
+  Iter.prototype.toArgumentNameList = function (firstArgIndex, noDupCheck) {
     const argumentNameList = this.expr
-        .toArgumentNameList(firstArgIndex, noDupCheck)
-        .map(exprArgumentString =>
-        exprArgumentString[exprArgumentString.length - 1] === 's' ?
-          exprArgumentString + 'es' :
-          exprArgumentString + 's',
-        );
+      .toArgumentNameList(firstArgIndex, noDupCheck)
+      .map(exprArgumentString =>
+        exprArgumentString[exprArgumentString.length - 1] === 's'
+          ? exprArgumentString + 'es'
+          : exprArgumentString + 's'
+      );
     if (!noDupCheck) {
       resolveDuplicatedNames(argumentNameList);
     }
     return argumentNameList;
   };
 
-  Opt.prototype.toArgumentNameList = function(firstArgIndex, noDupCheck) {
+  Opt.prototype.toArgumentNameList = function (firstArgIndex, noDupCheck) {
     return this.expr.toArgumentNameList(firstArgIndex, noDupCheck).map(argName => {
       return 'opt' + argName[0].toUpperCase() + argName.slice(1);
     });
   };
 
-  Not.prototype.toArgumentNameList = function(firstArgIndex, noDupCheck) {
+  Not.prototype.toArgumentNameList = function (firstArgIndex, noDupCheck) {
     return [];
   };
 
   Lookahead.prototype.toArgumentNameList = Lex.prototype.toArgumentNameList =
-    function(firstArgIndex, noDupCheck) {
+    function (firstArgIndex, noDupCheck) {
       return this.expr.toArgumentNameList(firstArgIndex, noDupCheck);
     };
 
-  Apply.prototype.toArgumentNameList = function(firstArgIndex, noDupCheck) {
+  Apply.prototype.toArgumentNameList = function (firstArgIndex, noDupCheck) {
     return [this.ruleName];
   };
 
-  UnicodeChar.prototype.toArgumentNameList = function(firstArgIndex, noDupCheck) {
+  UnicodeChar.prototype.toArgumentNameList = function (firstArgIndex, noDupCheck) {
     return ['$' + firstArgIndex];
   };
 
-  Param.prototype.toArgumentNameList = function(firstArgIndex, noDupCheck) {
+  Param.prototype.toArgumentNameList = function (firstArgIndex, noDupCheck) {
     return ['param' + this.index];
   };
 
@@ -3121,7 +3169,7 @@
   // Returns a string representing the PExpr, for use as a UI label, etc.
   PExpr.prototype.toDisplayString = abstract('toDisplayString');
 
-  Alt.prototype.toDisplayString = Seq.prototype.toDisplayString = function() {
+  Alt.prototype.toDisplayString = Seq.prototype.toDisplayString = function () {
     if (this.source) {
       return this.source.trimmed().contents;
     }
@@ -3137,11 +3185,11 @@
     Terminal.prototype.toDisplayString =
     Range.prototype.toDisplayString =
     Param.prototype.toDisplayString =
-      function() {
+      function () {
         return this.toString();
       };
 
-  Apply.prototype.toDisplayString = function() {
+  Apply.prototype.toDisplayString = function () {
     if (this.args.length > 0) {
       const ps = this.args.map(arg => arg.toDisplayString());
       return this.ruleName + '<' + ps.join(',') + '>';
@@ -3150,8 +3198,8 @@
     }
   };
 
-  UnicodeChar.prototype.toDisplayString = function() {
-    return 'Unicode [' + this.category + '] character';
+  UnicodeChar.prototype.toDisplayString = function () {
+    return 'Unicode [' + this.categoryOrProp + '] character';
   };
 
   // --------------------------------------------------------------------
@@ -3254,34 +3302,34 @@
 
   PExpr.prototype.toFailure = abstract('toFailure');
 
-  any.toFailure = function(grammar) {
+  any.toFailure = function (grammar) {
     return new Failure(this, 'any object', 'description');
   };
 
-  end.toFailure = function(grammar) {
+  end.toFailure = function (grammar) {
     return new Failure(this, 'end of input', 'description');
   };
 
-  Terminal.prototype.toFailure = function(grammar) {
+  Terminal.prototype.toFailure = function (grammar) {
     return new Failure(this, this.obj, 'string');
   };
 
-  Range.prototype.toFailure = function(grammar) {
+  Range.prototype.toFailure = function (grammar) {
     // TODO: come up with something better
     return new Failure(this, JSON.stringify(this.from) + '..' + JSON.stringify(this.to), 'code');
   };
 
-  Not.prototype.toFailure = function(grammar) {
+  Not.prototype.toFailure = function (grammar) {
     const description =
       this.expr === any ? 'nothing' : 'not ' + this.expr.toFailure(grammar);
     return new Failure(this, description, 'description');
   };
 
-  Lookahead.prototype.toFailure = function(grammar) {
+  Lookahead.prototype.toFailure = function (grammar) {
     return this.expr.toFailure(grammar);
   };
 
-  Apply.prototype.toFailure = function(grammar) {
+  Apply.prototype.toFailure = function (grammar) {
     let {description} = grammar.rules[this.ruleName];
     if (!description) {
       const article = /^[aeiouAEIOU]/.test(this.ruleName) ? 'an' : 'a';
@@ -3290,23 +3338,23 @@
     return new Failure(this, description, 'description');
   };
 
-  UnicodeChar.prototype.toFailure = function(grammar) {
-    return new Failure(this, 'a Unicode [' + this.category + '] character', 'description');
+  UnicodeChar.prototype.toFailure = function (grammar) {
+    return new Failure(this, 'a Unicode [' + this.categoryOrProp + '] character', 'description');
   };
 
-  Alt.prototype.toFailure = function(grammar) {
+  Alt.prototype.toFailure = function (grammar) {
     const fs = this.terms.map(t => t.toFailure(grammar));
     const description = '(' + fs.join(' or ') + ')';
     return new Failure(this, description, 'description');
   };
 
-  Seq.prototype.toFailure = function(grammar) {
+  Seq.prototype.toFailure = function (grammar) {
     const fs = this.factors.map(f => f.toFailure(grammar));
     const description = '(' + fs.join(' ') + ')';
     return new Failure(this, description, 'description');
   };
 
-  Iter.prototype.toFailure = function(grammar) {
+  Iter.prototype.toFailure = function (grammar) {
     const description = '(' + this.expr.toFailure(grammar) + this.operator + ')';
     return new Failure(this, description, 'description');
   };
@@ -3324,55 +3372,55 @@
   */
   PExpr.prototype.toString = abstract('toString');
 
-  any.toString = function() {
+  any.toString = function () {
     return 'any';
   };
 
-  end.toString = function() {
+  end.toString = function () {
     return 'end';
   };
 
-  Terminal.prototype.toString = function() {
+  Terminal.prototype.toString = function () {
     return JSON.stringify(this.obj);
   };
 
-  Range.prototype.toString = function() {
+  Range.prototype.toString = function () {
     return JSON.stringify(this.from) + '..' + JSON.stringify(this.to);
   };
 
-  Param.prototype.toString = function() {
+  Param.prototype.toString = function () {
     return '$' + this.index;
   };
 
-  Lex.prototype.toString = function() {
+  Lex.prototype.toString = function () {
     return '#(' + this.expr.toString() + ')';
   };
 
-  Alt.prototype.toString = function() {
-    return this.terms.length === 1 ?
-      this.terms[0].toString() :
-      '(' + this.terms.map(term => term.toString()).join(' | ') + ')';
+  Alt.prototype.toString = function () {
+    return this.terms.length === 1
+      ? this.terms[0].toString()
+      : '(' + this.terms.map(term => term.toString()).join(' | ') + ')';
   };
 
-  Seq.prototype.toString = function() {
-    return this.factors.length === 1 ?
-      this.factors[0].toString() :
-      '(' + this.factors.map(factor => factor.toString()).join(' ') + ')';
+  Seq.prototype.toString = function () {
+    return this.factors.length === 1
+      ? this.factors[0].toString()
+      : '(' + this.factors.map(factor => factor.toString()).join(' ') + ')';
   };
 
-  Iter.prototype.toString = function() {
+  Iter.prototype.toString = function () {
     return this.expr + this.operator;
   };
 
-  Not.prototype.toString = function() {
+  Not.prototype.toString = function () {
     return '~' + this.expr;
   };
 
-  Lookahead.prototype.toString = function() {
+  Lookahead.prototype.toString = function () {
     return '&' + this.expr;
   };
 
-  Apply.prototype.toString = function() {
+  Apply.prototype.toString = function () {
     if (this.args.length > 0) {
       const ps = this.args.map(arg => arg.toString());
       return this.ruleName + '<' + ps.join(',') + '>';
@@ -3381,8 +3429,8 @@
     }
   };
 
-  UnicodeChar.prototype.toString = function() {
-    return '\\p{' + this.category + '}';
+  UnicodeChar.prototype.toString = function () {
+    return '\\p{' + this.categoryOrProp + '}';
   };
 
   class CaseInsensitiveTerminal extends PExpr {
@@ -3430,9 +3478,9 @@
 
     toFailure(grammar) {
       return new Failure(
-          this,
-          this.obj.toFailure(grammar) + ' (case-insensitive)',
-          'description',
+        this,
+        this.obj.toFailure(grammar) + ' (case-insensitive)',
+        'description'
       );
     }
 
@@ -3502,8 +3550,8 @@
       posInfo.exit();
 
       this.rightmostFailurePosition = Math.max(
-          this.rightmostFailurePosition,
-          this._rightmostFailurePositionStack.pop(),
+        this.rightmostFailurePosition,
+        this._rightmostFailurePositionStack.pop()
       );
 
       if (optNode) {
@@ -3639,9 +3687,9 @@
     }
 
     _getRightmostFailureOffset() {
-      return this.rightmostFailurePosition >= 0 ?
-        this.posToOffset(this.rightmostFailurePosition) :
-        -1;
+      return this.rightmostFailurePosition >= 0
+        ? this.posToOffset(this.rightmostFailurePosition)
+        : -1;
     }
 
     // Returns the memoized trace entry for `expr` at `pos`, if one exists, `null` otherwise.
@@ -3698,8 +3746,8 @@
       const memoRecRightmostFailurePosition =
         this.inputStream.pos + memoRec.rightmostFailureOffset;
       this.rightmostFailurePosition = Math.max(
-          this.rightmostFailurePosition,
-          memoRecRightmostFailurePosition,
+        this.rightmostFailurePosition,
+        memoRecRightmostFailurePosition
       );
       if (
         this.recordedFailures &&
@@ -3710,8 +3758,8 @@
       }
 
       this.inputStream.examinedLength = Math.max(
-          this.inputStream.examinedLength,
-          memoRec.examinedLength + origPos,
+        this.inputStream.examinedLength,
+        memoRec.examinedLength + origPos
       );
 
       if (memoRec.value) {
@@ -3789,7 +3837,7 @@
       let rightmostFailures;
       if (this.recordedFailures) {
         rightmostFailures = Object.keys(this.recordedFailures).map(
-            key => this.recordedFailures[key],
+          key => this.recordedFailures[key]
         );
       }
       const cst = this._bindings[0];
@@ -3797,13 +3845,13 @@
         cst.grammar = this.grammar;
       }
       return new MatchResult(
-          this.matcher,
-          this.input,
-          this.startExpr,
-          cst,
-          this._bindingOffsets[0],
-          this.rightmostFailurePosition,
-          rightmostFailures,
+        this.matcher,
+        this.input,
+        this.startExpr,
+        cst,
+        this._bindingOffsets[0],
+        this.rightmostFailurePosition,
+        rightmostFailures
       );
     }
 
@@ -4116,11 +4164,11 @@
       if (superSemantics) {
         if (!(grammar.equals(this.super.grammar) || grammar._inheritsFrom(this.super.grammar))) {
           throw new Error(
-              "Cannot extend a semantics for grammar '" +
+            "Cannot extend a semantics for grammar '" +
               this.super.grammar.name +
               "' for use with grammar '" +
               grammar.name +
-              "' (not a sub-grammar)",
+              "' (not a sub-grammar)"
           );
         }
         this.operations = Object.create(this.super.operations);
@@ -4129,7 +4177,7 @@
 
         // Assign unique symbols for each of the attributes inherited from the super-semantics so that
         // they are memoized independently.
-        // eslint-disable-next-line guard-for-in
+
         for (const attributeName in this.attributes) {
           Object.defineProperty(this.attributeKeys, attributeName, {
             value: uniqueId(attributeName),
@@ -4158,11 +4206,11 @@
     // Throws an exception if one or more of them doesn't.
     checkActionDicts() {
       let name;
-      // eslint-disable-next-line guard-for-in
+
       for (name in this.operations) {
         this.operations[name].checkActionDict(this.grammar);
       }
-      // eslint-disable-next-line guard-for-in
+
       for (name in this.attributes) {
         this.attributes[name].checkActionDict(this.grammar);
       }
@@ -4262,9 +4310,9 @@
       });
 
       const entry =
-        type === 'operation' ?
-          new Operation(name, formals, realActionDict, builtInDefault) :
-          new Attribute(name, realActionDict, builtInDefault);
+        type === 'operation'
+          ? new Operation(name, formals, realActionDict, builtInDefault)
+          : new Attribute(name, realActionDict, builtInDefault);
 
       // The following check is not strictly necessary (it will happen later anyway) but it's better
       // to catch errors early.
@@ -4280,7 +4328,7 @@
         // Check that the caller passed the correct number of arguments.
         if (arguments.length !== thisThing.formals.length) {
           throw new Error(
-              'Invalid number of arguments passed to ' +
+            'Invalid number of arguments passed to ' +
               name +
               ' ' +
               type +
@@ -4288,7 +4336,7 @@
               thisThing.formals.length +
               ', got ' +
               arguments.length +
-              ')',
+              ')'
           );
         }
 
@@ -4309,7 +4357,7 @@
 
       if (type === 'operation') {
         this.Wrapper.prototype[name] = doIt;
-        this.Wrapper.prototype[name].toString = function() {
+        this.Wrapper.prototype[name].toString = function () {
           return '[' + name + ' operation]';
         };
       } else {
@@ -4331,13 +4379,13 @@
 
       if (!(this.super && name in this.super[typePlural])) {
         throw new Error(
-            'Cannot extend ' +
+          'Cannot extend ' +
             type +
             " '" +
             name +
             "': did not inherit an " +
             type +
-            ' with that name',
+            ' with that name'
         );
       }
       if (hasOwnProperty(this[typePlural], name)) {
@@ -4354,9 +4402,9 @@
       });
 
       this[typePlural][name] =
-        type === 'operation' ?
-          new Operation(name, inheritedFormals, newActionDict) :
-          new Attribute(name, newActionDict);
+        type === 'operation'
+          ? new Operation(name, inheritedFormals, newActionDict)
+          : new Attribute(name, newActionDict);
 
       // The following check is not strictly necessary (it will happen later anyway) but it's better
       // to catch errors early.
@@ -4369,12 +4417,12 @@
       }
       if (name in this.operations) {
         throw new Error(
-            'Cannot add ' + type + " '" + name + "': an operation with that name already exists",
+          'Cannot add ' + type + " '" + name + "': an operation with that name already exists"
         );
       }
       if (name in this.attributes) {
         throw new Error(
-            'Cannot add ' + type + " '" + name + "': an attribute with that name already exists",
+          'Cannot add ' + type + " '" + name + "': an attribute with that name already exists"
         );
       }
     }
@@ -4400,8 +4448,8 @@
     }
 
     const r = Semantics.prototypeGrammar.match(
-        signature,
-      type === 'operation' ? 'OperationSignature' : 'AttributeSignature',
+      signature,
+      type === 'operation' ? 'OperationSignature' : 'AttributeSignature'
     );
     if (r.failed()) {
       throw new Error(r.message);
@@ -4411,7 +4459,7 @@
   }
 
   function newDefaultAction(type, name, doIt) {
-    return function(...children) {
+    return function (...children) {
       const thisThing = this._semantics.operations[name] || this._semantics.attributes[name];
       const args = thisThing.formals.map(formal => this.args[formal]);
 
@@ -4435,12 +4483,12 @@
   // Semantics instance. When that function is invoked with a CST node as an argument, it returns
   // a wrapper for that node which gives access to the operations and attributes provided by this
   // semantics.
-  Semantics.createSemantics = function(grammar, optSuperSemantics) {
+  Semantics.createSemantics = function (grammar, optSuperSemantics) {
     const s = new Semantics(
-        grammar,
-      optSuperSemantics !== undefined ?
-        optSuperSemantics :
-        Semantics.BuiltInSemantics._getSemantics(),
+      grammar,
+      optSuperSemantics !== undefined
+        ? optSuperSemantics
+        : Semantics.BuiltInSemantics._getSemantics()
     );
 
     // To enable clients to invoke a semantics like a function, return a function that acts as a proxy
@@ -4448,8 +4496,8 @@
     const proxy = function ASemantics(matchResult) {
       if (!(matchResult instanceof MatchResult)) {
         throw new TypeError(
-            'Semantics expected a MatchResult, but got ' +
-            unexpectedObjToString(matchResult),
+          'Semantics expected a MatchResult, but got ' +
+            unexpectedObjToString(matchResult)
         );
       }
       if (matchResult.failed()) {
@@ -4459,11 +4507,11 @@
       const cst = matchResult._cst;
       if (cst.grammar !== grammar) {
         throw new Error(
-            "Cannot use a MatchResult from grammar '" +
+          "Cannot use a MatchResult from grammar '" +
             cst.grammar.name +
             "' with a semantics for '" +
             grammar.name +
-            "'",
+            "'"
         );
       }
       const inputStream = new InputStream(matchResult.input);
@@ -4471,38 +4519,38 @@
     };
 
     // Forward public methods from the proxy to the semantics instance.
-    proxy.addOperation = function(signature, actionDict) {
+    proxy.addOperation = function (signature, actionDict) {
       s.addOperationOrAttribute('operation', signature, actionDict);
       return proxy;
     };
-    proxy.extendOperation = function(name, actionDict) {
+    proxy.extendOperation = function (name, actionDict) {
       s.extendOperationOrAttribute('operation', name, actionDict);
       return proxy;
     };
-    proxy.addAttribute = function(name, actionDict) {
+    proxy.addAttribute = function (name, actionDict) {
       s.addOperationOrAttribute('attribute', name, actionDict);
       return proxy;
     };
-    proxy.extendAttribute = function(name, actionDict) {
+    proxy.extendAttribute = function (name, actionDict) {
       s.extendOperationOrAttribute('attribute', name, actionDict);
       return proxy;
     };
-    proxy._getActionDict = function(operationOrAttributeName) {
+    proxy._getActionDict = function (operationOrAttributeName) {
       const action =
         s.operations[operationOrAttributeName] || s.attributes[operationOrAttributeName];
       if (!action) {
         throw new Error(
-            '"' +
+          '"' +
             operationOrAttributeName +
             '" is not a valid operation or attribute ' +
             'name in this semantics for "' +
             grammar.name +
-            '"',
+            '"'
         );
       }
       return action.actionDict;
     };
-    proxy._remove = function(operationOrAttributeName) {
+    proxy._remove = function (operationOrAttributeName) {
       let semantic;
       if (operationOrAttributeName in s.operations) {
         semantic = s.operations[operationOrAttributeName];
@@ -4514,16 +4562,16 @@
       delete s.Wrapper.prototype[operationOrAttributeName];
       return semantic;
     };
-    proxy.getOperationNames = function() {
+    proxy.getOperationNames = function () {
       return Object.keys(s.operations);
     };
-    proxy.getAttributeNames = function() {
+    proxy.getAttributeNames = function () {
       return Object.keys(s.attributes);
     };
-    proxy.getGrammar = function() {
+    proxy.getGrammar = function () {
       return s.grammar;
     };
-    proxy.toRecipe = function(semanticsOnly) {
+    proxy.toRecipe = function (semanticsOnly) {
       return s.toRecipe(semanticsOnly);
     };
 
@@ -4531,7 +4579,7 @@
     proxy.toString = s.toString.bind(s);
 
     // Returns the semantics for the proxy.
-    proxy._getSemantics = function() {
+    proxy._getSemantics = function () {
       return s;
     };
 
@@ -4623,8 +4671,8 @@
 
   function getSortedRuleValues(grammar) {
     return Object.keys(grammar.rules)
-        .sort()
-        .map(name => grammar.rules[name]);
+      .sort()
+      .map(name => grammar.rules[name]);
   }
 
   // Until ES2019, JSON was not a valid subset of JavaScript because U+2028 (line separator)
@@ -4645,11 +4693,11 @@
       if (optDefaultStartRule) {
         if (!(optDefaultStartRule in rules)) {
           throw new Error(
-              "Invalid start rule: '" +
+            "Invalid start rule: '" +
               optDefaultStartRule +
               "' is not a rule in grammar '" +
               name +
-              "'",
+              "'"
           );
         }
         this.defaultStartRule = optDefaultStartRule;
@@ -4720,7 +4768,6 @@
     _checkTopDownActionDict(what, name, actionDict) {
       const problems = [];
 
-      // eslint-disable-next-line guard-for-in
       for (const k in actionDict) {
         const v = actionDict[k];
         const isSpecialAction = SPECIAL_ACTION_NAMES.includes(k);
@@ -4750,10 +4797,10 @@
       if (problems.length > 0) {
         const prettyProblems = problems.map(problem => '- ' + problem);
         const error = new Error(
-            [
-              `Found errors in the action dictionary of the '${name}' ${what}:`,
-              ...prettyProblems,
-            ].join('\n'),
+          [
+            `Found errors in the action dictionary of the '${name}' ${what}:`,
+            ...prettyProblems,
+          ].join('\n')
         );
         error.problems = problems;
         throw error;
@@ -4766,9 +4813,9 @@
       // All special actions have an expected arity of 0, though all but _terminal
       // are expected to use the rest parameter syntax (e.g. `_iter(...children)`).
       // This is considered to have arity 0, i.e. `((...args) => {}).length` is 0.
-      return SPECIAL_ACTION_NAMES.includes(actionName) ?
-        0 :
-        this.rules[actionName].body.getArity();
+      return SPECIAL_ACTION_NAMES.includes(actionName)
+        ? 0
+        : this.rules[actionName].body.getArity();
     }
 
     _inheritsFrom(grammar) {
@@ -4859,7 +4906,7 @@
       sb.append('{');
 
       let first = true;
-      // eslint-disable-next-line guard-for-in
+
       for (const ruleName in this.rules) {
         const {body} = this.rules[ruleName];
         if (first) {
@@ -4906,10 +4953,10 @@
       if (formals.length !== app.args.length) {
         const {source} = this.rules[app.ruleName];
         throw wrongNumberOfParameters(
-            app.ruleName,
-            formals.length,
-            app.args.length,
-            source,
+          app.ruleName,
+          formals.length,
+          app.args.length,
+          source
         );
       }
       return app;
@@ -4928,63 +4975,63 @@
   // `digit`, and is implicitly the super-grammar of any grammar whose super-grammar
   // isn't specified.
   Grammar.ProtoBuiltInRules = new Grammar(
-      'ProtoBuiltInRules', // name
-      undefined, // supergrammar
-      {
-        any: {
-          body: any,
-          formals: [],
-          description: 'any character',
-          primitive: true,
-        },
-        end: {
-          body: end,
-          formals: [],
-          description: 'end of input',
-          primitive: true,
-        },
-
-        caseInsensitive: {
-          body: new CaseInsensitiveTerminal(new Param(0)),
-          formals: ['str'],
-          primitive: true,
-        },
-        lower: {
-          body: new UnicodeChar('Ll'),
-          formals: [],
-          description: 'a lowercase letter',
-          primitive: true,
-        },
-        upper: {
-          body: new UnicodeChar('Lu'),
-          formals: [],
-          description: 'an uppercase letter',
-          primitive: true,
-        },
-        // Union of Lt (titlecase), Lm (modifier), and Lo (other), i.e. any letter not in Ll or Lu.
-        unicodeLtmo: {
-          body: new UnicodeChar('Ltmo'),
-          formals: [],
-          description: 'a Unicode character in Lt, Lm, or Lo',
-          primitive: true,
-        },
-
-        // These rules are not truly primitive (they could be written in userland) but are defined
-        // here for bootstrapping purposes.
-        spaces: {
-          body: new Star(new Apply('space')),
-          formals: [],
-        },
-        space: {
-          body: new Range('\x00', ' '),
-          formals: [],
-          description: 'a space',
-        },
+    'ProtoBuiltInRules', // name
+    undefined, // supergrammar
+    {
+      any: {
+        body: any,
+        formals: [],
+        description: 'any character',
+        primitive: true,
+      },
+      end: {
+        body: end,
+        formals: [],
+        description: 'end of input',
+        primitive: true,
+      },
+
+      caseInsensitive: {
+        body: new CaseInsensitiveTerminal(new Param(0)),
+        formals: ['str'],
+        primitive: true,
+      },
+      lower: {
+        body: new UnicodeChar('Ll'),
+        formals: [],
+        description: 'a lowercase letter',
+        primitive: true,
+      },
+      upper: {
+        body: new UnicodeChar('Lu'),
+        formals: [],
+        description: 'an uppercase letter',
+        primitive: true,
+      },
+      // Union of Lt (titlecase), Lm (modifier), and Lo (other), i.e. any letter not in Ll or Lu.
+      unicodeLtmo: {
+        body: new UnicodeChar('Ltmo'),
+        formals: [],
+        description: 'a Unicode character in Lt, Lm, or Lo',
+        primitive: true,
       },
+
+      // These rules are not truly primitive (they could be written in userland) but are defined
+      // here for bootstrapping purposes.
+      spaces: {
+        body: new Star(new Apply('space')),
+        formals: [],
+      },
+      space: {
+        body: new Range('\x00', ' '),
+        formals: [],
+        description: 'a space',
+      },
+    }
   );
 
   // This method is called from main.js once Ohm has loaded.
-  Grammar.initApplicationParser = function(grammar, builderFn) {
+  Grammar.initApplicationParser = function (grammar, builderFn) {
     ohmGrammar$1 = grammar;
     buildGrammar$1 = builderFn;
   };
@@ -5012,7 +5059,7 @@
           // TODO: The conditional expression below is an ugly hack. It's kind of ok because
           // I doubt anyone will ever try to declare a grammar called `BuiltInRules`. Still,
           // we should try to find a better way to do this.
-          this.name === 'BuiltInRules' ? Grammar.ProtoBuiltInRules : Grammar.BuiltInRules,
+          this.name === 'BuiltInRules' ? Grammar.ProtoBuiltInRules : Grammar.BuiltInRules
         );
       }
       return this.superGrammar;
@@ -5080,10 +5127,10 @@
     // Creates a Grammar instance, and if it passes the sanity checks, returns it.
     build() {
       const grammar = new Grammar(
-          this.name,
-          this.ensureSuperGrammar(),
-          this.rules,
-          this.defaultStartRule,
+        this.name,
+        this.ensureSuperGrammar(),
+        this.rules,
+        this.defaultStartRule
       );
       // Initialize internal props that are inherited from the super grammar.
       grammar._matchStateInitializer = grammar.superGrammar._matchStateInitializer;
@@ -5184,7 +5231,7 @@
       if (superGrammar) {
         // `superGrammar` may be a recipe (i.e. an Array), or an actual grammar instance.
         gDecl.withSuperGrammar(
-          superGrammar instanceof Grammar ? superGrammar : this.fromRecipe(superGrammar),
+          superGrammar instanceof Grammar ? superGrammar : this.fromRecipe(superGrammar)
         );
       }
       if (defaultStartRule) {
@@ -5208,8 +5255,8 @@
         let source;
         if (gDecl.source && metaInfo && metaInfo.sourceInterval) {
           source = gDecl.source.subInterval(
-              metaInfo.sourceInterval[0],
-              metaInfo.sourceInterval[1] - metaInfo.sourceInterval[0],
+            metaInfo.sourceInterval[0],
+            metaInfo.sourceInterval[1] - metaInfo.sourceInterval[0]
           );
         }
         gDecl[action](ruleName, formals, body, description, source);
@@ -5304,7 +5351,7 @@
 
     app(ruleName, optParams) {
       if (optParams && optParams.length > 0) {
-        optParams = optParams.map(function(param) {
+        optParams = optParams.map(function (param) {
           return param instanceof PExpr ? param : this.fromRecipe(param);
         }, this);
       }
@@ -5316,10 +5363,10 @@
     // `this.currentDecl` and `this.currentRuleName` being set.
     splice(beforeTerms, afterTerms) {
       return new Splice(
-          this.currentDecl.superGrammar,
-          this.currentRuleName,
-          beforeTerms.map(term => this.fromRecipe(term)),
-          afterTerms.map(term => this.fromRecipe(term)),
+        this.currentDecl.superGrammar,
+        this.currentRuleName,
+        beforeTerms.map(term => this.fromRecipe(term)),
+        afterTerms.map(term => this.fromRecipe(term))
       );
     }
 
@@ -5460,10 +5507,10 @@
           });
 
           return new Splice(
-              decl.superGrammar,
-              currentRuleName,
-              beforeTerms,
-              afterTerms,
+            decl.superGrammar,
+            currentRuleName,
+            beforeTerms,
+            afterTerms
           ).withSource(this.source);
         } else {
           return builder.alt(...args).withSource(this.source);
@@ -5608,14 +5655,14 @@
     };
 
     Semantics.BuiltInSemantics = Semantics.createSemantics(builtInRules, null).addOperation(
-        'asIteration',
-        {
-          emptyListOf: actions.empty,
-          nonemptyListOf: actions.nonEmpty,
-          EmptyListOf: actions.empty,
-          NonemptyListOf: actions.nonEmpty,
-          _iter: actions.self,
-        },
+      'asIteration',
+      {
+        emptyListOf: actions.empty,
+        nonemptyListOf: actions.nonEmpty,
+        EmptyListOf: actions.empty,
+        NonemptyListOf: actions.nonEmpty,
+        _iter: actions.self,
+      }
     );
   }
 
@@ -5804,12 +5851,12 @@
   const newAnyBody = new Splice(BuiltInRules, 'any', [applyIndent, applyDedent], []);
 
   const IndentationSensitive = new Builder()
-      .newGrammar('IndentationSensitive')
-      .withSuperGrammar(BuiltInRules)
-      .define('indent', [], new Indentation(true), INDENT_DESCRIPTION, undefined, true)
-      .define('dedent', [], new Indentation(false), DEDENT_DESCRIPTION, undefined, true)
-      .extend('any', [], newAnyBody, 'any character', undefined)
-      .build();
+    .newGrammar('IndentationSensitive')
+    .withSuperGrammar(BuiltInRules)
+    .define('indent', [], new Indentation(true), INDENT_DESCRIPTION, undefined, true)
+    .define('dedent', [], new Indentation(false), DEDENT_DESCRIPTION, undefined, true)
+    .extend('any', [], newAnyBody, 'any character', undefined)
+    .build();
 
   Object.assign(IndentationSensitive, {
     _matchStateInitializer(state) {
@@ -5842,7 +5889,7 @@
         source = source.toString();
       } else {
         throw new TypeError(
-            'Expected string as first argument, got ' + unexpectedObjToString(source),
+          'Expected string as first argument, got ' + unexpectedObjToString(source)
         );
       }
     }
@@ -5998,10 +6045,60 @@
     return semantics(matchResult).examples();
   }
 
+  /*
+    To find iter nodes that are derived from the same repetition expression, we
+    look for adjacent iter nodes that have the same source interval and the same
+    number of children.
+
+    A few things to note:
+    - The children of `*` and `+` nodes can't be nullable, so the associated iter
+      nodes always consume some input, and therefore consecutive nodes that have
+      the same interval must come from the same repetition expression.
+    - We *could* mistake `a? b?` for (a b)?`, if neither of them comsume any input.
+      However, for the purposes of this module, those two cases are equivalent
+      anyways, since we only care about finding the correct order of the non-iter
+      nodes, and both interpretations yield the same order.
+   */
+  const isIterSibling = (refNode, n) => {
+    return (
+      n.isIteration() &&
+      n.source.startIdx === refNode.source.startIdx &&
+      n.source.endIdx === refNode.source.endIdx &&
+      n.children.length === refNode.children.length
+    );
+  };
+
+  function recoverSourceOrder(nodes, depth = 0) {
+    const ans = [];
+    for (let i = 0; i < nodes.length; i++) {
+      const n = nodes[i];
+      if (!n.isIteration()) {
+        ans.push(n);
+        continue;
+      }
+
+      // We found an iter node, now find its siblings.
+      const siblings = [n];
+      // Find the first node that's *not* part of the current list.
+      for (let j = i + 1; j < nodes.length && isIterSibling(n, nodes[j]); j++) {
+        siblings.push(nodes[j]);
+        i = j;
+      }
+      const cousins = [];
+      const numRows = siblings[0].children.length;
+      for (let row = 0; row < numRows; row++) {
+        cousins.push(...siblings.map(sib => sib.children[row]));
+      }
+      ans.push(...recoverSourceOrder(cousins, depth + 1));
+    }
+    return ans;
+  }
+
   exports.VisitorFamily = VisitorFamily;
   exports.extractExamples = extractExamples;
   exports.getLineAndColumn = getLineAndColumn;
   exports.getLineAndColumnMessage = getLineAndColumnMessage;
+  exports.recoverSourceOrder = recoverSourceOrder;
   exports.semanticsForToAST = semanticsForToAST;
   exports.toAST = toAST;