ohm-js 17.2.1 → 17.4.0

package/dist/ohm-extras.cjs

@@ -10,14 +10,14 @@ Object.defineProperty(exports, '__esModule', { value: true });
 
 function abstract(optMethodName) {
   const methodName = optMethodName || '';
-  return function() {
+  return function () {
     throw new Error(
-        'this method ' +
+      'this method ' +
         methodName +
         ' is abstract! ' +
         '(it has no implementation in class ' +
         this.constructor.name +
-        ')',
+        ')'
     );
   };
 }
@@ -110,11 +110,11 @@ function StringBuffer() {
   this.strings = [];
 }
 
-StringBuffer.prototype.append = function(str) {
+StringBuffer.prototype.append = function (str) {
   this.strings.push(str);
 };
 
-StringBuffer.prototype.contents = function() {
+StringBuffer.prototype.contents = function () {
   return this.strings.join('');
 };
 
@@ -138,9 +138,9 @@ function unescapeCodePoint(s) {
       case 'x':
         return escapeUnicode(s.slice(2, 4));
       case 'u':
-        return s.charAt(2) === '{' ?
-          escapeUnicode(s.slice(3, -1)) :
-          escapeUnicode(s.slice(2, 6));
+        return s.charAt(2) === '{'
+          ? escapeUnicode(s.slice(3, -1))
+          : escapeUnicode(s.slice(2, 6));
       default:
         return s.charAt(1);
     }
@@ -166,7 +166,7 @@ function unexpectedObjToString(obj) {
       typeName = typeof obj;
     }
     return typeName + ': ' + JSON.stringify(String(obj));
-  } catch (e) {
+  } catch {
     return baseToString;
   }
 }
@@ -332,9 +332,9 @@ function getLineAndColumn(str, offset) {
     // Get the next line.
     const nextLineEndOffset = str.indexOf('\n', lineEndOffset + 1);
     nextLine =
-      nextLineEndOffset === -1 ?
-        str.slice(lineEndOffset) :
-        str.slice(lineEndOffset, nextLineEndOffset);
+      nextLineEndOffset === -1
+        ? str.slice(lineEndOffset)
+        : str.slice(lineEndOffset, nextLineEndOffset);
     // Strip leading and trailing EOL char(s).
     nextLine = nextLine.replace(/^\r?\n/, '').replace(/\r$/, '');
   }
@@ -446,11 +446,11 @@ class VisitorFamily {
     this._shapes = config.shapes;
     this._getTag = config.getTag;
 
-    this.Adapter = function(thing, family) {
+    this.Adapter = function (thing, family) {
       this._adaptee = thing;
       this._family = family;
     };
-    this.Adapter.prototype.valueOf = function() {
+    this.Adapter.prototype.valueOf = function () {
       throw new Error('heeey!');
     };
     this.operations = {};
@@ -479,15 +479,15 @@ class VisitorFamily {
       assert(k in this._getChildren, "Unrecognized action name '" + k + "'");
       const action = dict[k];
       assert(
-          typeof action === 'function',
-          "Key '" + k + "': expected function, got " + action,
+        typeof action === 'function',
+        "Key '" + k + "': expected function, got " + action
       );
       if (k in this._arities) {
         const expected = this._arities[k];
         const actual = dict[k].length;
         assert(
-            actual === expected,
-            "Action '" + k + "' has the wrong arity: expected " + expected + ', got ' + actual,
+          actual === expected,
+          "Action '" + k + "' has the wrong arity: expected " + expected + ', got ' + actual
         );
       }
     });
@@ -504,7 +504,7 @@ class VisitorFamily {
     };
 
     const family = this;
-    this.Adapter.prototype[name] = function(...args) {
+    this.Adapter.prototype[name] = function (...args) {
       const tag = family._getTag(this._adaptee);
       assert(tag in family._getChildren, "getTag returned unrecognized tag '" + tag + "'");
       assert(tag in actions, "No action for '" + tag + "' in operation '" + name + "'");
@@ -519,8 +519,8 @@ class VisitorFamily {
       const oldArgs = this.args;
       this.args = argsObj;
       const ans = actions[tag].apply(
-          this,
-          family._getChildren[tag](this._adaptee, family._wrap),
+        this,
+        family._getChildren[tag](this._adaptee, family._wrap)
       );
       this.args = oldArgs;
       return ans;
@@ -586,7 +586,7 @@ const defaultOperation = {
     const node = {
       type: ctorName,
     };
-    // eslint-disable-next-line guard-for-in
+
     for (const prop in propMap) {
       const mappedProp = mapping[ctorName] && mapping[ctorName][prop];
       if (typeof mappedProp === 'number') {
@@ -661,36 +661,60 @@ function semanticsForToAST(g) {
   return g.createSemantics().addOperation('toAST(mapping)', defaultOperation);
 }
 
-// These are just categories that are used in ES5/ES2015.
-// The full list of Unicode categories is here: http://www.fileformat.info/info/unicode/category/index.htm.
-const UnicodeCategories = {
-  // Letters
-  Lu: /\p{Lu}/u,
-  Ll: /\p{Ll}/u,
-  Lt: /\p{Lt}/u,
-  Lm: /\p{Lm}/u,
-  Lo: /\p{Lo}/u,
-
-  // Numbers
-  Nl: /\p{Nl}/u,
-  Nd: /\p{Nd}/u,
-
-  // Marks
-  Mn: /\p{Mn}/u,
-  Mc: /\p{Mc}/u,
-
-  // Punctuation, Connector
-  Pc: /\p{Pc}/u,
-
-  // Separator, Space
-  Zs: /\p{Zs}/u,
-
-  // These two are not real Unicode categories, but our useful for Ohm.
-  // L is a combination of all the letter categories.
-  // Ltmo is a combination of Lt, Lm, and Lo.
-  L: /\p{Letter}/u,
-  Ltmo: /\p{Lt}|\p{Lm}|\p{Lo}/u,
-};
+// The full list of categories from:
+// https://www.unicode.org/Public/UCD/latest/ucd/extracted/DerivedGeneralCategory.txt.
+
+const toRegExp = val => new RegExp(String.raw`\p{${val}}`, 'u');
+
+/*
+  grep -v '^#' DerivedGeneralCategory.txt \
+    | cut -d';' -f2 \
+    | awk 'NF{print $1}' \
+    | sort -u \
+    | awk '{printf "\x27%s\x27,\n",$1}'
+ */
+
+const UnicodeCategories = Object.fromEntries(
+  [
+    'Cc',
+    'Cf',
+    'Cn',
+    'Co',
+    'Cs',
+    'Ll',
+    'Lm',
+    'Lo',
+    'Lt',
+    'Lu',
+    'Mc',
+    'Me',
+    'Mn',
+    'Nd',
+    'Nl',
+    'No',
+    'Pc',
+    'Pd',
+    'Pe',
+    'Pf',
+    'Pi',
+    'Po',
+    'Ps',
+    'Sc',
+    'Sk',
+    'Sm',
+    'So',
+    'Zl',
+    'Zp',
+    'Zs',
+  ].map(cat => [cat, toRegExp(cat)])
+);
+UnicodeCategories['Ltmo'] = /\p{Lt}|\p{Lm}|\p{Lo}/u;
+
+// We only support a few of these for now, but could add more later.
+// See https://www.unicode.org/Public/UCD/latest/ucd/PropertyAliases.txt
+const UnicodeBinaryProperties = Object.fromEntries(
+  ['XID_Start', 'XID_Continue', 'White_Space'].map(prop => [prop, toRegExp(prop)])
+);
 
 // --------------------------------------------------------------------
 // Private stuff
@@ -871,10 +895,18 @@ class Apply extends PExpr {
 // Unicode character
 
 class UnicodeChar extends PExpr {
-  constructor(category) {
+  constructor(categoryOrProp) {
     super();
-    this.category = category;
-    this.pattern = UnicodeCategories[category];
+    this.categoryOrProp = categoryOrProp;
+    if (categoryOrProp in UnicodeCategories) {
+      this.pattern = UnicodeCategories[categoryOrProp];
+    } else if (categoryOrProp in UnicodeBinaryProperties) {
+      this.pattern = UnicodeBinaryProperties[categoryOrProp];
+    } else {
+      throw new Error(
+        `Invalid Unicode category or property name: ${JSON.stringify(categoryOrProp)}`
+      );
+    }
   }
 }
 
@@ -925,9 +957,9 @@ function grammarSyntaxError(matchFailure) {
 // Undeclared grammar
 
 function undeclaredGrammar(grammarName, namespace, interval) {
-  const message = namespace ?
-    `Grammar ${grammarName} is not declared in namespace '${namespace}'` :
-    'Undeclared grammar ' + grammarName;
+  const message = namespace
+    ? `Grammar ${grammarName} is not declared in namespace '${namespace}'`
+    : 'Undeclared grammar ' + grammarName;
   return createError(message, interval);
 }
 
@@ -947,8 +979,8 @@ function grammarDoesNotSupportIncrementalParsing(grammar) {
 
 function undeclaredRule(ruleName, grammarName, optInterval) {
   return createError(
-      'Rule ' + ruleName + ' is not declared in grammar ' + grammarName,
-      optInterval,
+    'Rule ' + ruleName + ' is not declared in grammar ' + grammarName,
+    optInterval
   );
 }
 
@@ -956,8 +988,8 @@ function undeclaredRule(ruleName, grammarName, optInterval) {
 
 function cannotOverrideUndeclaredRule(ruleName, grammarName, optSource) {
   return createError(
-      'Cannot override rule ' + ruleName + ' because it is not declared in ' + grammarName,
-      optSource,
+    'Cannot override rule ' + ruleName + ' because it is not declared in ' + grammarName,
+    optSource
   );
 }
 
@@ -965,8 +997,8 @@ function cannotOverrideUndeclaredRule(ruleName, grammarName, optSource) {
 
 function cannotExtendUndeclaredRule(ruleName, grammarName, optSource) {
   return createError(
-      'Cannot extend rule ' + ruleName + ' because it is not declared in ' + grammarName,
-      optSource,
+    'Cannot extend rule ' + ruleName + ' because it is not declared in ' + grammarName,
+    optSource
   );
 }
 
@@ -985,14 +1017,14 @@ function duplicateRuleDeclaration(ruleName, grammarName, declGrammarName, optSou
 
 function wrongNumberOfParameters(ruleName, expected, actual, source) {
   return createError(
-      'Wrong number of parameters for rule ' +
+    'Wrong number of parameters for rule ' +
       ruleName +
       ' (expected ' +
       expected +
       ', got ' +
       actual +
       ')',
-      source,
+    source
   );
 }
 
@@ -1000,14 +1032,14 @@ function wrongNumberOfParameters(ruleName, expected, actual, source) {
 
 function wrongNumberOfArguments(ruleName, expected, actual, expr) {
   return createError(
-      'Wrong number of arguments for rule ' +
+    'Wrong number of arguments for rule ' +
       ruleName +
       ' (expected ' +
       expected +
       ', got ' +
       actual +
       ')',
-      expr,
+    expr
   );
 }
 
@@ -1015,8 +1047,8 @@ function wrongNumberOfArguments(ruleName, expected, actual, expr) {
 
 function duplicateParameterNames(ruleName, duplicates, source) {
   return createError(
-      'Duplicate parameter names in rule ' + ruleName + ': ' + duplicates.join(', '),
-      source,
+    'Duplicate parameter names in rule ' + ruleName + ': ' + duplicates.join(', '),
+    source
   );
 }
 
@@ -1024,14 +1056,14 @@ function duplicateParameterNames(ruleName, duplicates, source) {
 
 function invalidParameter(ruleName, expr) {
   return createError(
-      'Invalid parameter to rule ' +
+    'Invalid parameter to rule ' +
       ruleName +
       ': ' +
       expr +
       ' has arity ' +
       expr.getArity() +
       ', but parameter expressions must have arity 1',
-      expr.source,
+    expr.source
   );
 }
 
@@ -1043,8 +1075,8 @@ const syntacticVsLexicalNote =
 
 function applicationOfSyntacticRuleFromLexicalContext(ruleName, applyExpr) {
   return createError(
-      'Cannot apply syntactic rule ' + ruleName + ' from here (inside a lexical context)',
-      applyExpr.source,
+    'Cannot apply syntactic rule ' + ruleName + ' from here (inside a lexical context)',
+    applyExpr.source
   );
 }
 
@@ -1053,9 +1085,9 @@ function applicationOfSyntacticRuleFromLexicalContext(ruleName, applyExpr) {
 function applySyntacticWithLexicalRuleApplication(applyExpr) {
   const {ruleName} = applyExpr;
   return createError(
-      `applySyntactic is for syntactic rules, but '${ruleName}' is a lexical rule. ` +
+    `applySyntactic is for syntactic rules, but '${ruleName}' is a lexical rule. ` +
       syntacticVsLexicalNote,
-      applyExpr.source,
+    applyExpr.source
   );
 }
 
@@ -1063,8 +1095,8 @@ function applySyntacticWithLexicalRuleApplication(applyExpr) {
 
 function unnecessaryExperimentalApplySyntactic(applyExpr) {
   return createError(
-      'applySyntactic is not required here (in a syntactic context)',
-      applyExpr.source,
+    'applySyntactic is not required here (in a syntactic context)',
+    applyExpr.source
   );
 }
 
@@ -1090,8 +1122,8 @@ function invalidCodePoint(applyWrapper) {
   const digitIntervals = applyWrapper.children.slice(1, -1).map(d => d.source);
   const fullInterval = digitIntervals[0].coverageWith(...digitIntervals.slice(1));
   return createError(
-      `U+${fullInterval.contents} is not a valid Unicode code point`,
-      fullInterval,
+    `U+${fullInterval.contents} is not a valid Unicode code point`,
+    fullInterval
   );
 }
 
@@ -1109,8 +1141,8 @@ function kleeneExprHasNullableOperand(kleeneExpr, applicationStack) {
     "' (possible infinite loop)";
   if (applicationStack.length > 0) {
     const stackTrace = applicationStack
-        .map(app => new Apply(app.ruleName, app.args))
-        .join('\n');
+      .map(app => new Apply(app.ruleName, app.args))
+      .join('\n');
     message += '\nApplication stack (most recent application last):\n' + stackTrace;
   }
   return createError(message, kleeneExpr.expr.source);
@@ -1120,7 +1152,7 @@ function kleeneExprHasNullableOperand(kleeneExpr, applicationStack) {
 
 function inconsistentArity(ruleName, expected, actual, expr) {
   return createError(
-      'Rule ' +
+    'Rule ' +
       ruleName +
       ' involves an alternation which has inconsistent arity ' +
       '(expected ' +
@@ -1128,7 +1160,7 @@ function inconsistentArity(ruleName, expected, actual, expr) {
       ', got ' +
       actual +
       ')',
-      expr.source,
+    expr.source
   );
 }
 
@@ -1143,12 +1175,12 @@ function multipleErrors(errors) {
 
 function missingSemanticAction(ctorName, name, type, stack) {
   let stackTrace = stack
-      .slice(0, -1)
-      .map(info => {
-        const ans = '  ' + info[0].name + ' > ' + info[1];
-        return info.length === 3 ? ans + " for '" + info[2] + "'" : ans;
-      })
-      .join('\n');
+    .slice(0, -1)
+    .map(info => {
+      const ans = '  ' + info[0].name + ' > ' + info[1];
+      return info.length === 3 ? ans + " for '" + info[2] + "'" : ans;
+    })
+    .join('\n');
   stackTrace += '\n  ' + name + ' > ' + ctorName;
 
   let moreInfo = '';
@@ -1185,11 +1217,23 @@ function throwErrors(errors) {
 
 class Interval {
   constructor(sourceString, startIdx, endIdx) {
-    this.sourceString = sourceString;
+    // Store the full source in a non-enumerable property, so that when
+    // grammars and other objects are printed in the REPL, it's not
+    // cluttered with multiple copies of the same long string.
+    Object.defineProperty(this, '_sourceString', {
+      value: sourceString,
+      configurable: false,
+      enumerable: false,
+      writable: false,
+    });
     this.startIdx = startIdx;
     this.endIdx = endIdx;
   }
 
+  get sourceString() {
+    return this._sourceString;
+  }
+
   get contents() {
     if (this._contents === undefined) {
       this._contents = this.sourceString.slice(this.startIdx, this.endIdx);
@@ -1255,13 +1299,13 @@ class Interval {
       throw intervalSourcesDontMatch();
     }
     assert(
-        this.startIdx >= that.startIdx && this.endIdx <= that.endIdx,
-        'other interval does not cover this one',
+      this.startIdx >= that.startIdx && this.endIdx <= that.endIdx,
+      'other interval does not cover this one'
     );
     return new Interval(
-        this.sourceString,
-        this.startIdx - that.startIdx,
-        this.endIdx - that.startIdx,
+      this.sourceString,
+      this.startIdx - that.startIdx,
+      this.endIdx - that.startIdx
     );
   }
 
@@ -1280,7 +1324,7 @@ class Interval {
   }
 }
 
-Interval.coverage = function(firstInterval, ...intervals) {
+Interval.coverage = function (firstInterval, ...intervals) {
   let {startIdx, endIdx} = firstInterval;
   for (const interval of intervals) {
     if (interval.sourceString !== firstInterval.sourceString) {
@@ -1294,6 +1338,7 @@ Interval.coverage = function(firstInterval, ...intervals) {
 };
 
 const MAX_CHAR_CODE = 0xffff;
+const MAX_CODE_POINT = 0x10ffff;
 
 class InputStream {
   constructor(source) {
@@ -1339,6 +1384,8 @@ class InputStream {
 
         This is intended to be a locale-invariant comparison, which means it may not obey
         locale-specific expectations (e.g. "i" => "İ").
+
+        See also https://unicode.org/faq/casemap_charprop.html#casemap
        */
       for (idx = 0; idx < s.length; idx++) {
         const actual = this.next();
@@ -1373,13 +1420,13 @@ class InputStream {
 
 class MatchResult {
   constructor(
-      matcher,
-      input,
-      startExpr,
-      cst,
-      cstOffset,
-      rightmostFailurePosition,
-      optRecordedFailures,
+    matcher,
+    input,
+    startExpr,
+    cst,
+    cstOffset,
+    rightmostFailurePosition,
+    optRecordedFailures
   ) {
     this.matcher = matcher;
     this.input = input;
@@ -1390,22 +1437,20 @@ class MatchResult {
     this._rightmostFailures = optRecordedFailures;
 
     if (this.failed()) {
-      /* eslint-disable no-invalid-this */
-      defineLazyProperty(this, 'message', function() {
+      defineLazyProperty(this, 'message', function () {
         const detail = 'Expected ' + this.getExpectedText();
         return (
           getLineAndColumnMessage(this.input, this.getRightmostFailurePosition()) + detail
         );
       });
-      defineLazyProperty(this, 'shortMessage', function() {
+      defineLazyProperty(this, 'shortMessage', function () {
         const detail = 'expected ' + this.getExpectedText();
         const errorInfo = getLineAndColumn(
-            this.input,
-            this.getRightmostFailurePosition(),
+          this.input,
+          this.getRightmostFailurePosition()
         );
         return 'Line ' + errorInfo.lineNum + ', col ' + errorInfo.colNum + ': ' + detail;
       });
-      /* eslint-enable no-invalid-this */
     }
   }
 
@@ -1434,9 +1479,9 @@ class MatchResult {
   }
 
   toString() {
-    return this.succeeded() ?
-      '[match succeeded]' :
-      '[match failed at position ' + this.getRightmostFailurePosition() + ']';
+    return this.succeeded()
+      ? '[match succeeded]'
+      : '[match failed at position ' + this.getRightmostFailurePosition() + ']';
   }
 
   // Return a string summarizing the expected contents of the input stream when
@@ -1502,14 +1547,14 @@ class PosInfo {
     const indexOfFirstInvolvedRule =
       applicationMemoKeyStack.indexOf(headApplication.toMemoKey()) + 1;
     const involvedApplicationMemoKeys = applicationMemoKeyStack.slice(
-        indexOfFirstInvolvedRule,
+      indexOfFirstInvolvedRule
     );
 
-    memoRec.isInvolved = function(applicationMemoKey) {
+    memoRec.isInvolved = function (applicationMemoKey) {
       return involvedApplicationMemoKeys.indexOf(applicationMemoKey) >= 0;
     };
 
-    memoRec.updateInvolvedApplicationMemoKeys = function() {
+    memoRec.updateInvolvedApplicationMemoKeys = function () {
       for (let idx = indexOfFirstInvolvedRule; idx < applicationMemoKeyStack.length; idx++) {
         const applicationMemoKey = applicationMemoKeyStack[idx];
         if (!this.isInvolved(applicationMemoKey)) {
@@ -1543,8 +1588,8 @@ class PosInfo {
     this.memo[memoKey] = memoRec;
     this.maxExaminedLength = Math.max(this.maxExaminedLength, memoRec.examinedLength);
     this.maxRightmostFailureOffset = Math.max(
-        this.maxRightmostFailureOffset,
-        memoRec.rightmostFailureOffset,
+      this.maxRightmostFailureOffset,
+      memoRec.rightmostFailureOffset
     );
     return memoRec;
   }
@@ -1566,8 +1611,8 @@ class PosInfo {
       } else {
         this.maxExaminedLength = Math.max(this.maxExaminedLength, memoRec.examinedLength);
         this.maxRightmostFailureOffset = Math.max(
-            this.maxRightmostFailureOffset,
-            memoRec.rightmostFailureOffset,
+          this.maxRightmostFailureOffset,
+          memoRec.rightmostFailureOffset
         );
       }
     });
@@ -1616,10 +1661,10 @@ function asEscapedString(obj) {
   if (typeof obj === 'string') {
     // Replace non-printable characters with visible symbols.
     return obj
-        .replace(/ /g, DOT_OPERATOR)
-        .replace(/\t/g, SYMBOL_FOR_HORIZONTAL_TABULATION)
-        .replace(/\n/g, SYMBOL_FOR_LINE_FEED)
-        .replace(/\r/g, SYMBOL_FOR_CARRIAGE_RETURN);
+      .replace(/ /g, DOT_OPERATOR)
+      .replace(/\t/g, SYMBOL_FOR_HORIZONTAL_TABULATION)
+      .replace(/\n/g, SYMBOL_FOR_LINE_FEED)
+      .replace(/\r/g, SYMBOL_FOR_CARRIAGE_RETURN);
   }
   return String(obj);
 }
@@ -1650,13 +1695,13 @@ class Trace {
 
   cloneWithExpr(expr) {
     const ans = new Trace(
-        this.input,
-        this.pos,
-        this.pos2,
-        expr,
-        this.succeeded,
-        this.bindings,
-        this.children,
+      this.input,
+      this.pos,
+      this.pos2,
+      expr,
+      this.succeeded,
+      this.bindings,
+      this.children
     );
 
     ans.isHeadOfLeftRecursion = this.isHeadOfLeftRecursion;
@@ -1671,13 +1716,13 @@ class Trace {
   // Record the trace information for the terminating condition of the LR loop.
   recordLRTermination(ruleBodyTrace, value) {
     this.terminatingLREntry = new Trace(
-        this.input,
-        this.pos,
-        this.pos2,
-        this.expr,
-        false,
-        [value],
-        [ruleBodyTrace],
+      this.input,
+      this.pos,
+      this.pos2,
+      this.expr,
+      false,
+      [value],
+      [ruleBodyTrace]
     );
     this.terminatingLREntry.terminatesLR = true;
   }
@@ -1737,7 +1782,7 @@ class Trace {
       const ctorName = node.expr.constructor.name;
       // Don't print anything for Alt nodes.
       if (ctorName === 'Alt') {
-        return; // eslint-disable-line consistent-return
+        return;
       }
       sb.append(getInputExcerpt(node.input, node.pos, 10) + spaces(depth * 2 + 1));
       sb.append((node.succeeded ? CHECK_MARK : BALLOT_X) + ' ' + node.displayString);
@@ -1795,7 +1840,7 @@ any.allowsSkippingPrecedingSpace =
   Terminal.prototype.allowsSkippingPrecedingSpace =
   Range.prototype.allowsSkippingPrecedingSpace =
   UnicodeChar.prototype.allowsSkippingPrecedingSpace =
-    function() {
+    function () {
       return true;
     };
 
@@ -1809,7 +1854,7 @@ Alt.prototype.allowsSkippingPrecedingSpace =
   Not.prototype.allowsSkippingPrecedingSpace =
   Param.prototype.allowsSkippingPrecedingSpace =
   Seq.prototype.allowsSkippingPrecedingSpace =
-    function() {
+    function () {
       return false;
     };
 
@@ -1825,13 +1870,13 @@ awaitBuiltInRules(g => {
 
 let lexifyCount;
 
-PExpr.prototype.assertAllApplicationsAreValid = function(ruleName, grammar) {
+PExpr.prototype.assertAllApplicationsAreValid = function (ruleName, grammar) {
   lexifyCount = 0;
   this._assertAllApplicationsAreValid(ruleName, grammar);
 };
 
 PExpr.prototype._assertAllApplicationsAreValid = abstract(
-    '_assertAllApplicationsAreValid',
+  '_assertAllApplicationsAreValid'
 );
 
 any._assertAllApplicationsAreValid =
@@ -1840,23 +1885,23 @@ any._assertAllApplicationsAreValid =
   Range.prototype._assertAllApplicationsAreValid =
   Param.prototype._assertAllApplicationsAreValid =
   UnicodeChar.prototype._assertAllApplicationsAreValid =
-    function(ruleName, grammar) {
+    function (ruleName, grammar) {
       // no-op
     };
 
-Lex.prototype._assertAllApplicationsAreValid = function(ruleName, grammar) {
+Lex.prototype._assertAllApplicationsAreValid = function (ruleName, grammar) {
   lexifyCount++;
   this.expr._assertAllApplicationsAreValid(ruleName, grammar);
   lexifyCount--;
 };
 
-Alt.prototype._assertAllApplicationsAreValid = function(ruleName, grammar) {
+Alt.prototype._assertAllApplicationsAreValid = function (ruleName, grammar) {
   for (let idx = 0; idx < this.terms.length; idx++) {
     this.terms[idx]._assertAllApplicationsAreValid(ruleName, grammar);
   }
 };
 
-Seq.prototype._assertAllApplicationsAreValid = function(ruleName, grammar) {
+Seq.prototype._assertAllApplicationsAreValid = function (ruleName, grammar) {
   for (let idx = 0; idx < this.factors.length; idx++) {
     this.factors[idx]._assertAllApplicationsAreValid(ruleName, grammar);
   }
@@ -1865,14 +1910,14 @@ Seq.prototype._assertAllApplicationsAreValid = function(ruleName, grammar) {
 Iter.prototype._assertAllApplicationsAreValid =
   Not.prototype._assertAllApplicationsAreValid =
   Lookahead.prototype._assertAllApplicationsAreValid =
-    function(ruleName, grammar) {
+    function (ruleName, grammar) {
       this.expr._assertAllApplicationsAreValid(ruleName, grammar);
     };
 
-Apply.prototype._assertAllApplicationsAreValid = function(
-    ruleName,
-    grammar,
-    skipSyntacticCheck = false,
+Apply.prototype._assertAllApplicationsAreValid = function (
+  ruleName,
+  grammar,
+  skipSyntacticCheck = false
 ) {
   const ruleInfo = grammar.rules[this.ruleName];
   const isContextSyntactic = isSyntactic(ruleName) && lexifyCount === 0;
@@ -1935,7 +1980,7 @@ Apply.prototype._assertAllApplicationsAreValid = function(
 // --------------------------------------------------------------------
 
 PExpr.prototype.assertChoicesHaveUniformArity = abstract(
-    'assertChoicesHaveUniformArity',
+  'assertChoicesHaveUniformArity'
 );
 
 any.assertChoicesHaveUniformArity =
@@ -1945,11 +1990,11 @@ any.assertChoicesHaveUniformArity =
   Param.prototype.assertChoicesHaveUniformArity =
   Lex.prototype.assertChoicesHaveUniformArity =
   UnicodeChar.prototype.assertChoicesHaveUniformArity =
-    function(ruleName) {
+    function (ruleName) {
       // no-op
     };
 
-Alt.prototype.assertChoicesHaveUniformArity = function(ruleName) {
+Alt.prototype.assertChoicesHaveUniformArity = function (ruleName) {
   if (this.terms.length === 0) {
     return;
   }
@@ -1964,7 +2009,7 @@ Alt.prototype.assertChoicesHaveUniformArity = function(ruleName) {
   }
 };
 
-Extend.prototype.assertChoicesHaveUniformArity = function(ruleName) {
+Extend.prototype.assertChoicesHaveUniformArity = function (ruleName) {
   // Extend is a special case of Alt that's guaranteed to have exactly two
   // cases: [extensions, origBody].
   const actualArity = this.terms[0].getArity();
@@ -1974,25 +2019,25 @@ Extend.prototype.assertChoicesHaveUniformArity = function(ruleName) {
   }
 };
 
-Seq.prototype.assertChoicesHaveUniformArity = function(ruleName) {
+Seq.prototype.assertChoicesHaveUniformArity = function (ruleName) {
   for (let idx = 0; idx < this.factors.length; idx++) {
     this.factors[idx].assertChoicesHaveUniformArity(ruleName);
   }
 };
 
-Iter.prototype.assertChoicesHaveUniformArity = function(ruleName) {
+Iter.prototype.assertChoicesHaveUniformArity = function (ruleName) {
   this.expr.assertChoicesHaveUniformArity(ruleName);
 };
 
-Not.prototype.assertChoicesHaveUniformArity = function(ruleName) {
+Not.prototype.assertChoicesHaveUniformArity = function (ruleName) {
   // no-op (not required b/c the nested expr doesn't show up in the CST)
 };
 
-Lookahead.prototype.assertChoicesHaveUniformArity = function(ruleName) {
+Lookahead.prototype.assertChoicesHaveUniformArity = function (ruleName) {
   this.expr.assertChoicesHaveUniformArity(ruleName);
 };
 
-Apply.prototype.assertChoicesHaveUniformArity = function(ruleName) {
+Apply.prototype.assertChoicesHaveUniformArity = function (ruleName) {
   // The arities of the parameter expressions is required to be 1 by
   // `assertAllApplicationsAreValid()`.
 };
@@ -2002,7 +2047,7 @@ Apply.prototype.assertChoicesHaveUniformArity = function(ruleName) {
 // --------------------------------------------------------------------
 
 PExpr.prototype.assertIteratedExprsAreNotNullable = abstract(
-    'assertIteratedExprsAreNotNullable',
+  'assertIteratedExprsAreNotNullable'
 );
 
 any.assertIteratedExprsAreNotNullable =
@@ -2011,23 +2056,23 @@ any.assertIteratedExprsAreNotNullable =
   Range.prototype.assertIteratedExprsAreNotNullable =
   Param.prototype.assertIteratedExprsAreNotNullable =
   UnicodeChar.prototype.assertIteratedExprsAreNotNullable =
-    function(grammar) {
+    function (grammar) {
       // no-op
     };
 
-Alt.prototype.assertIteratedExprsAreNotNullable = function(grammar) {
+Alt.prototype.assertIteratedExprsAreNotNullable = function (grammar) {
   for (let idx = 0; idx < this.terms.length; idx++) {
     this.terms[idx].assertIteratedExprsAreNotNullable(grammar);
   }
 };
 
-Seq.prototype.assertIteratedExprsAreNotNullable = function(grammar) {
+Seq.prototype.assertIteratedExprsAreNotNullable = function (grammar) {
   for (let idx = 0; idx < this.factors.length; idx++) {
     this.factors[idx].assertIteratedExprsAreNotNullable(grammar);
   }
 };
 
-Iter.prototype.assertIteratedExprsAreNotNullable = function(grammar) {
+Iter.prototype.assertIteratedExprsAreNotNullable = function (grammar) {
   // Note: this is the implementation of this method for `Star` and `Plus` expressions.
   // It is overridden for `Opt` below.
   this.expr.assertIteratedExprsAreNotNullable(grammar);
@@ -2040,11 +2085,11 @@ Opt.prototype.assertIteratedExprsAreNotNullable =
   Not.prototype.assertIteratedExprsAreNotNullable =
   Lookahead.prototype.assertIteratedExprsAreNotNullable =
   Lex.prototype.assertIteratedExprsAreNotNullable =
-    function(grammar) {
+    function (grammar) {
       this.expr.assertIteratedExprsAreNotNullable(grammar);
     };
 
-Apply.prototype.assertIteratedExprsAreNotNullable = function(grammar) {
+Apply.prototype.assertIteratedExprsAreNotNullable = function (grammar) {
   this.args.forEach(arg => {
     arg.assertIteratedExprsAreNotNullable(grammar);
   });
@@ -2088,11 +2133,11 @@ class Node {
   onlyChild() {
     if (this.numChildren() !== 1) {
       throw new Error(
-          'cannot get only child of a node of type ' +
+        'cannot get only child of a node of type ' +
           this.ctorName +
           ' (it has ' +
           this.numChildren() +
-          ' children)',
+          ' children)'
       );
     } else {
       return this.firstChild();
@@ -2102,7 +2147,7 @@ class Node {
   firstChild() {
     if (this.hasNoChildren()) {
       throw new Error(
-          'cannot get first child of a ' + this.ctorName + ' node, which has no children',
+        'cannot get first child of a ' + this.ctorName + ' node, which has no children'
       );
     } else {
       return this.childAt(0);
@@ -2112,7 +2157,7 @@ class Node {
   lastChild() {
     if (this.hasNoChildren()) {
       throw new Error(
-          'cannot get last child of a ' + this.ctorName + ' node, which has no children',
+        'cannot get last child of a ' + this.ctorName + ' node, which has no children'
       );
     } else {
       return this.childAt(this.numChildren() - 1);
@@ -2246,7 +2291,7 @@ class IterationNode extends Node {
 */
 PExpr.prototype.eval = abstract('eval'); // function(state) { ... }
 
-any.eval = function(state) {
+any.eval = function (state) {
   const {inputStream} = state;
   const origPos = inputStream.pos;
   const cp = inputStream.nextCodePoint();
@@ -2259,7 +2304,7 @@ any.eval = function(state) {
   }
 };
 
-end.eval = function(state) {
+end.eval = function (state) {
   const {inputStream} = state;
   const origPos = inputStream.pos;
   if (inputStream.atEnd()) {
@@ -2271,7 +2316,7 @@ end.eval = function(state) {
   }
 };
 
-Terminal.prototype.eval = function(state) {
+Terminal.prototype.eval = function (state) {
   const {inputStream} = state;
   const origPos = inputStream.pos;
   if (!inputStream.matchString(this.obj)) {
@@ -2283,7 +2328,7 @@ Terminal.prototype.eval = function(state) {
   }
 };
 
-Range.prototype.eval = function(state) {
+Range.prototype.eval = function (state) {
   const {inputStream} = state;
   const origPos = inputStream.pos;
 
@@ -2302,18 +2347,18 @@ Range.prototype.eval = function(state) {
   }
 };
 
-Param.prototype.eval = function(state) {
+Param.prototype.eval = function (state) {
   return state.eval(state.currentApplication().args[this.index]);
 };
 
-Lex.prototype.eval = function(state) {
+Lex.prototype.eval = function (state) {
   state.enterLexifiedContext();
   const ans = state.eval(this.expr);
   state.exitLexifiedContext();
   return ans;
 };
 
-Alt.prototype.eval = function(state) {
+Alt.prototype.eval = function (state) {
   for (let idx = 0; idx < this.terms.length; idx++) {
     if (state.eval(this.terms[idx])) {
       return true;
@@ -2322,7 +2367,7 @@ Alt.prototype.eval = function(state) {
   return false;
 };
 
-Seq.prototype.eval = function(state) {
+Seq.prototype.eval = function (state) {
   for (let idx = 0; idx < this.factors.length; idx++) {
     const factor = this.factors[idx];
     if (!state.eval(factor)) {
@@ -2332,7 +2377,7 @@ Seq.prototype.eval = function(state) {
   return true;
 };
 
-Iter.prototype.eval = function(state) {
+Iter.prototype.eval = function (state) {
   const {inputStream} = state;
   const origPos = inputStream.pos;
   const arity = this.getArity();
@@ -2354,8 +2399,8 @@ Iter.prototype.eval = function(state) {
     numMatches++;
     const row = state._bindings.splice(state._bindings.length - arity, arity);
     const rowOffsets = state._bindingOffsets.splice(
-        state._bindingOffsets.length - arity,
-        arity,
+      state._bindingOffsets.length - arity,
+      arity
     );
     for (idx = 0; idx < row.length; idx++) {
       cols[idx].push(row[idx]);
@@ -2379,14 +2424,14 @@ Iter.prototype.eval = function(state) {
   const isOptional = this instanceof Opt;
   for (idx = 0; idx < cols.length; idx++) {
     state._bindings.push(
-        new IterationNode(cols[idx], colOffsets[idx], matchLength, isOptional),
+      new IterationNode(cols[idx], colOffsets[idx], matchLength, isOptional)
     );
     state._bindingOffsets.push(offset);
   }
   return true;
 };
 
-Not.prototype.eval = function(state) {
+Not.prototype.eval = function (state) {
   /*
     TODO:
     - Right now we're just throwing away all of the failures that happen inside a `not`, and
@@ -2412,7 +2457,7 @@ Not.prototype.eval = function(state) {
   return true;
 };
 
-Lookahead.prototype.eval = function(state) {
+Lookahead.prototype.eval = function (state) {
   const {inputStream} = state;
   const origPos = inputStream.pos;
   if (state.eval(this.expr)) {
@@ -2423,7 +2468,7 @@ Lookahead.prototype.eval = function(state) {
   }
 };
 
-Apply.prototype.eval = function(state) {
+Apply.prototype.eval = function (state) {
   const caller = state.currentApplication();
   const actuals = caller ? caller.args : [];
   const app = this.substituteParams(actuals);
@@ -2446,7 +2491,7 @@ Apply.prototype.eval = function(state) {
   return app.reallyEval(state);
 };
 
-Apply.prototype.handleCycle = function(state) {
+Apply.prototype.handleCycle = function (state) {
   const posInfo = state.getCurrentPosInfo();
   const {currentLeftRecursion} = posInfo;
   const memoKey = this.toMemoKey();
@@ -2469,7 +2514,7 @@ Apply.prototype.handleCycle = function(state) {
   return state.useMemoizedResult(state.inputStream.pos, memoRec);
 };
 
-Apply.prototype.reallyEval = function(state) {
+Apply.prototype.reallyEval = function (state) {
   const {inputStream} = state;
   const origPos = inputStream.pos;
   const origPosInfo = state.getCurrentPosInfo();
@@ -2522,6 +2567,7 @@ Apply.prototype.reallyEval = function(state) {
     }
     if (memoRec) {
       memoRec.failuresAtRightmostPosition = state.cloneRecordedFailures();
+      memoRec.rightmostFailureOffset = state._getRightmostFailureOffset();
     }
   }
 
@@ -2539,8 +2585,8 @@ Apply.prototype.reallyEval = function(state) {
   // Fix the input stream's examinedLength -- it should be the maximum examined length
   // across all applications, not just this one.
   inputStream.examinedLength = Math.max(
-      inputStream.examinedLength,
-      origInputStreamExaminedLength,
+    inputStream.examinedLength,
+    origInputStreamExaminedLength
   );
 
   state.exitApplication(origPosInfo, value);
@@ -2548,7 +2594,7 @@ Apply.prototype.reallyEval = function(state) {
   return succeeded;
 };
 
-Apply.prototype.evalOnce = function(expr, state) {
+Apply.prototype.evalOnce = function (expr, state) {
   const {inputStream} = state;
   const origPos = inputStream.pos;
 
@@ -2563,7 +2609,7 @@ Apply.prototype.evalOnce = function(expr, state) {
   }
 };
 
-Apply.prototype.growSeedResult = function(body, state, origPos, lrMemoRec, newValue) {
+Apply.prototype.growSeedResult = function (body, state, origPos, lrMemoRec, newValue) {
   if (!newValue) {
     return false;
   }
@@ -2581,13 +2627,13 @@ Apply.prototype.growSeedResult = function(body, state, origPos, lrMemoRec, newVa
       // element in `state.trace`.
       const seedTrace = state.trace[state.trace.length - 1];
       lrMemoRec.traceEntry = new Trace(
-          state.input,
-          origPos,
-          inputStream.pos,
-          this,
-          true,
-          [newValue],
-          [seedTrace.clone()],
+        state.input,
+        origPos,
+        inputStream.pos,
+        this,
+        true,
+        [newValue],
+        [seedTrace.clone()]
       );
     }
     inputStream.pos = origPos;
@@ -2607,17 +2653,19 @@ Apply.prototype.growSeedResult = function(body, state, origPos, lrMemoRec, newVa
   return lrMemoRec.value;
 };
 
-UnicodeChar.prototype.eval = function(state) {
+UnicodeChar.prototype.eval = function (state) {
   const {inputStream} = state;
   const origPos = inputStream.pos;
-  const ch = inputStream.next();
-  if (ch && this.pattern.test(ch)) {
-    state.pushBinding(new TerminalNode(ch.length), origPos);
-    return true;
-  } else {
-    state.processFailure(origPos, this);
-    return false;
+  const cp = inputStream.nextCodePoint();
+  if (cp !== undefined && cp <= MAX_CODE_POINT) {
+    const ch = String.fromCodePoint(cp);
+    if (this.pattern.test(ch)) {
+      state.pushBinding(new TerminalNode(ch.length), origPos);
+      return true;
+    }
   }
+  state.processFailure(origPos, this);
+  return false;
 };
 
 // --------------------------------------------------------------------
@@ -2633,17 +2681,17 @@ any.getArity =
   Param.prototype.getArity =
   Apply.prototype.getArity =
   UnicodeChar.prototype.getArity =
-    function() {
+    function () {
       return 1;
     };
 
-Alt.prototype.getArity = function() {
+Alt.prototype.getArity = function () {
   // This is ok b/c all terms must have the same arity -- this property is
   // checked by the Grammar constructor.
   return this.terms.length === 0 ? 0 : this.terms[0].getArity();
 };
 
-Seq.prototype.getArity = function() {
+Seq.prototype.getArity = function () {
   let arity = 0;
   for (let idx = 0; idx < this.factors.length; idx++) {
     arity += this.factors[idx].getArity();
@@ -2651,15 +2699,15 @@ Seq.prototype.getArity = function() {
   return arity;
 };
 
-Iter.prototype.getArity = function() {
+Iter.prototype.getArity = function () {
   return this.expr.getArity();
 };
 
-Not.prototype.getArity = function() {
+Not.prototype.getArity = function () {
   return 0;
 };
 
-Lookahead.prototype.getArity = Lex.prototype.getArity = function() {
+Lookahead.prototype.getArity = Lex.prototype.getArity = function () {
   return this.expr.getArity();
 };
 
@@ -2682,38 +2730,38 @@ function getMetaInfo(expr, grammarInterval) {
 
 PExpr.prototype.outputRecipe = abstract('outputRecipe');
 
-any.outputRecipe = function(formals, grammarInterval) {
+any.outputRecipe = function (formals, grammarInterval) {
   return ['any', getMetaInfo(this, grammarInterval)];
 };
 
-end.outputRecipe = function(formals, grammarInterval) {
+end.outputRecipe = function (formals, grammarInterval) {
   return ['end', getMetaInfo(this, grammarInterval)];
 };
 
-Terminal.prototype.outputRecipe = function(formals, grammarInterval) {
+Terminal.prototype.outputRecipe = function (formals, grammarInterval) {
   return ['terminal', getMetaInfo(this, grammarInterval), this.obj];
 };
 
-Range.prototype.outputRecipe = function(formals, grammarInterval) {
+Range.prototype.outputRecipe = function (formals, grammarInterval) {
   return ['range', getMetaInfo(this, grammarInterval), this.from, this.to];
 };
 
-Param.prototype.outputRecipe = function(formals, grammarInterval) {
+Param.prototype.outputRecipe = function (formals, grammarInterval) {
   return ['param', getMetaInfo(this, grammarInterval), this.index];
 };
 
-Alt.prototype.outputRecipe = function(formals, grammarInterval) {
+Alt.prototype.outputRecipe = function (formals, grammarInterval) {
   return ['alt', getMetaInfo(this, grammarInterval)].concat(
-      this.terms.map(term => term.outputRecipe(formals, grammarInterval)),
+    this.terms.map(term => term.outputRecipe(formals, grammarInterval))
   );
 };
 
-Extend.prototype.outputRecipe = function(formals, grammarInterval) {
+Extend.prototype.outputRecipe = function (formals, grammarInterval) {
   const extension = this.terms[0]; // [extension, original]
   return extension.outputRecipe(formals, grammarInterval);
 };
 
-Splice.prototype.outputRecipe = function(formals, grammarInterval) {
+Splice.prototype.outputRecipe = function (formals, grammarInterval) {
   const beforeTerms = this.terms.slice(0, this.expansionPos);
   const afterTerms = this.terms.slice(this.expansionPos + 1);
   return [
@@ -2724,9 +2772,9 @@ Splice.prototype.outputRecipe = function(formals, grammarInterval) {
   ];
 };
 
-Seq.prototype.outputRecipe = function(formals, grammarInterval) {
+Seq.prototype.outputRecipe = function (formals, grammarInterval) {
   return ['seq', getMetaInfo(this, grammarInterval)].concat(
-      this.factors.map(factor => factor.outputRecipe(formals, grammarInterval)),
+    this.factors.map(factor => factor.outputRecipe(formals, grammarInterval))
   );
 };
 
@@ -2736,7 +2784,7 @@ Star.prototype.outputRecipe =
   Not.prototype.outputRecipe =
   Lookahead.prototype.outputRecipe =
   Lex.prototype.outputRecipe =
-    function(formals, grammarInterval) {
+    function (formals, grammarInterval) {
       return [
         this.constructor.name.toLowerCase(),
         getMetaInfo(this, grammarInterval),
@@ -2744,7 +2792,7 @@ Star.prototype.outputRecipe =
       ];
     };
 
-Apply.prototype.outputRecipe = function(formals, grammarInterval) {
+Apply.prototype.outputRecipe = function (formals, grammarInterval) {
   return [
     'app',
     getMetaInfo(this, grammarInterval),
@@ -2753,8 +2801,8 @@ Apply.prototype.outputRecipe = function(formals, grammarInterval) {
   ];
 };
 
-UnicodeChar.prototype.outputRecipe = function(formals, grammarInterval) {
-  return ['unicodeChar', getMetaInfo(this, grammarInterval), this.category];
+UnicodeChar.prototype.outputRecipe = function (formals, grammarInterval) {
+  return ['unicodeChar', getMetaInfo(this, grammarInterval), this.categoryOrProp];
 };
 
 // --------------------------------------------------------------------
@@ -2774,18 +2822,18 @@ any.introduceParams =
   Range.prototype.introduceParams =
   Param.prototype.introduceParams =
   UnicodeChar.prototype.introduceParams =
-    function(formals) {
+    function (formals) {
       return this;
     };
 
-Alt.prototype.introduceParams = function(formals) {
+Alt.prototype.introduceParams = function (formals) {
   this.terms.forEach((term, idx, terms) => {
     terms[idx] = term.introduceParams(formals);
   });
   return this;
 };
 
-Seq.prototype.introduceParams = function(formals) {
+Seq.prototype.introduceParams = function (formals) {
   this.factors.forEach((factor, idx, factors) => {
     factors[idx] = factor.introduceParams(formals);
   });
@@ -2796,12 +2844,12 @@ Iter.prototype.introduceParams =
   Not.prototype.introduceParams =
   Lookahead.prototype.introduceParams =
   Lex.prototype.introduceParams =
-    function(formals) {
+    function (formals) {
       this.expr = this.expr.introduceParams(formals);
       return this;
     };
 
-Apply.prototype.introduceParams = function(formals) {
+Apply.prototype.introduceParams = function (formals) {
   const index = formals.indexOf(this.ruleName);
   if (index >= 0) {
     if (this.args.length > 0) {
@@ -2822,7 +2870,7 @@ Apply.prototype.introduceParams = function(formals) {
 // --------------------------------------------------------------------
 
 // Returns `true` if this parsing expression may accept without consuming any input.
-PExpr.prototype.isNullable = function(grammar) {
+PExpr.prototype.isNullable = function (grammar) {
   return this._isNullable(grammar, Object.create(null));
 };
 
@@ -2833,15 +2881,15 @@ any._isNullable =
   Param.prototype._isNullable =
   Plus.prototype._isNullable =
   UnicodeChar.prototype._isNullable =
-    function(grammar, memo) {
+    function (grammar, memo) {
       return false;
     };
 
-end._isNullable = function(grammar, memo) {
+end._isNullable = function (grammar, memo) {
   return true;
 };
 
-Terminal.prototype._isNullable = function(grammar, memo) {
+Terminal.prototype._isNullable = function (grammar, memo) {
   if (typeof this.obj === 'string') {
     // This is an over-simplification: it's only correct if the input is a string. If it's an array
     // or an object, then the empty string parsing expression is not nullable.
@@ -2851,11 +2899,11 @@ Terminal.prototype._isNullable = function(grammar, memo) {
   }
 };
 
-Alt.prototype._isNullable = function(grammar, memo) {
+Alt.prototype._isNullable = function (grammar, memo) {
   return this.terms.length === 0 || this.terms.some(term => term._isNullable(grammar, memo));
 };
 
-Seq.prototype._isNullable = function(grammar, memo) {
+Seq.prototype._isNullable = function (grammar, memo) {
   return this.factors.every(factor => factor._isNullable(grammar, memo));
 };
 
@@ -2863,15 +2911,15 @@ Star.prototype._isNullable =
   Opt.prototype._isNullable =
   Not.prototype._isNullable =
   Lookahead.prototype._isNullable =
-    function(grammar, memo) {
+    function (grammar, memo) {
       return true;
     };
 
-Lex.prototype._isNullable = function(grammar, memo) {
+Lex.prototype._isNullable = function (grammar, memo) {
   return this.expr._isNullable(grammar, memo);
 };
 
-Apply.prototype._isNullable = function(grammar, memo) {
+Apply.prototype._isNullable = function (grammar, memo) {
   const key = this.toMemoKey();
   if (!Object.prototype.hasOwnProperty.call(memo, key)) {
     const {body} = grammar.rules[this.ruleName];
@@ -2900,19 +2948,19 @@ any.substituteParams =
   Terminal.prototype.substituteParams =
   Range.prototype.substituteParams =
   UnicodeChar.prototype.substituteParams =
-    function(actuals) {
+    function (actuals) {
       return this;
     };
 
-Param.prototype.substituteParams = function(actuals) {
+Param.prototype.substituteParams = function (actuals) {
   return checkNotNull(actuals[this.index]);
 };
 
-Alt.prototype.substituteParams = function(actuals) {
+Alt.prototype.substituteParams = function (actuals) {
   return new Alt(this.terms.map(term => term.substituteParams(actuals)));
 };
 
-Seq.prototype.substituteParams = function(actuals) {
+Seq.prototype.substituteParams = function (actuals) {
   return new Seq(this.factors.map(factor => factor.substituteParams(actuals)));
 };
 
@@ -2920,11 +2968,11 @@ Iter.prototype.substituteParams =
   Not.prototype.substituteParams =
   Lookahead.prototype.substituteParams =
   Lex.prototype.substituteParams =
-    function(actuals) {
+    function (actuals) {
       return new this.constructor(this.expr.substituteParams(actuals));
     };
 
-Apply.prototype.substituteParams = function(actuals) {
+Apply.prototype.substituteParams = function (actuals) {
   if (this.args.length === 0) {
     // Avoid making a copy of this application, as an optimization
     return this;
@@ -2998,15 +3046,15 @@ function resolveDuplicatedNames(argumentNameList) {
 // function(firstArgIndex, noDupCheck) { ... }
 PExpr.prototype.toArgumentNameList = abstract('toArgumentNameList');
 
-any.toArgumentNameList = function(firstArgIndex, noDupCheck) {
+any.toArgumentNameList = function (firstArgIndex, noDupCheck) {
   return ['any'];
 };
 
-end.toArgumentNameList = function(firstArgIndex, noDupCheck) {
+end.toArgumentNameList = function (firstArgIndex, noDupCheck) {
   return ['end'];
 };
 
-Terminal.prototype.toArgumentNameList = function(firstArgIndex, noDupCheck) {
+Terminal.prototype.toArgumentNameList = function (firstArgIndex, noDupCheck) {
   if (typeof this.obj === 'string' && /^[_a-zA-Z0-9]+$/.test(this.obj)) {
     // If this terminal is a valid suffix for a JS identifier, just prepend it with '_'
     return ['_' + this.obj];
@@ -3016,7 +3064,7 @@ Terminal.prototype.toArgumentNameList = function(firstArgIndex, noDupCheck) {
   }
 };
 
-Range.prototype.toArgumentNameList = function(firstArgIndex, noDupCheck) {
+Range.prototype.toArgumentNameList = function (firstArgIndex, noDupCheck) {
   let argName = this.from + '_to_' + this.to;
   // If the `argName` is not valid then try to prepend a `_`.
   if (!isRestrictedJSIdentifier(argName)) {
@@ -3029,11 +3077,11 @@ Range.prototype.toArgumentNameList = function(firstArgIndex, noDupCheck) {
   return [argName];
 };
 
-Alt.prototype.toArgumentNameList = function(firstArgIndex, noDupCheck) {
+Alt.prototype.toArgumentNameList = function (firstArgIndex, noDupCheck) {
   // `termArgNameLists` is an array of arrays where each row is the
   // argument name list that corresponds to a term in this alternation.
   const termArgNameLists = this.terms.map(term =>
-    term.toArgumentNameList(firstArgIndex, true),
+    term.toArgumentNameList(firstArgIndex, true)
   );
 
   const argumentNameList = [];
@@ -3053,7 +3101,7 @@ Alt.prototype.toArgumentNameList = function(firstArgIndex, noDupCheck) {
   return argumentNameList;
 };
 
-Seq.prototype.toArgumentNameList = function(firstArgIndex, noDupCheck) {
+Seq.prototype.toArgumentNameList = function (firstArgIndex, noDupCheck) {
   // Generate the argument name list, without worrying about duplicates.
   let argumentNameList = [];
   this.factors.forEach(factor => {
@@ -3069,44 +3117,44 @@ Seq.prototype.toArgumentNameList = function(firstArgIndex, noDupCheck) {
   return argumentNameList;
 };
 
-Iter.prototype.toArgumentNameList = function(firstArgIndex, noDupCheck) {
+Iter.prototype.toArgumentNameList = function (firstArgIndex, noDupCheck) {
   const argumentNameList = this.expr
-      .toArgumentNameList(firstArgIndex, noDupCheck)
-      .map(exprArgumentString =>
-      exprArgumentString[exprArgumentString.length - 1] === 's' ?
-        exprArgumentString + 'es' :
-        exprArgumentString + 's',
-      );
+    .toArgumentNameList(firstArgIndex, noDupCheck)
+    .map(exprArgumentString =>
+      exprArgumentString[exprArgumentString.length - 1] === 's'
+        ? exprArgumentString + 'es'
+        : exprArgumentString + 's'
+    );
   if (!noDupCheck) {
     resolveDuplicatedNames(argumentNameList);
   }
   return argumentNameList;
 };
 
-Opt.prototype.toArgumentNameList = function(firstArgIndex, noDupCheck) {
+Opt.prototype.toArgumentNameList = function (firstArgIndex, noDupCheck) {
   return this.expr.toArgumentNameList(firstArgIndex, noDupCheck).map(argName => {
     return 'opt' + argName[0].toUpperCase() + argName.slice(1);
   });
 };
 
-Not.prototype.toArgumentNameList = function(firstArgIndex, noDupCheck) {
+Not.prototype.toArgumentNameList = function (firstArgIndex, noDupCheck) {
   return [];
 };
 
 Lookahead.prototype.toArgumentNameList = Lex.prototype.toArgumentNameList =
-  function(firstArgIndex, noDupCheck) {
+  function (firstArgIndex, noDupCheck) {
     return this.expr.toArgumentNameList(firstArgIndex, noDupCheck);
   };
 
-Apply.prototype.toArgumentNameList = function(firstArgIndex, noDupCheck) {
+Apply.prototype.toArgumentNameList = function (firstArgIndex, noDupCheck) {
   return [this.ruleName];
 };
 
-UnicodeChar.prototype.toArgumentNameList = function(firstArgIndex, noDupCheck) {
+UnicodeChar.prototype.toArgumentNameList = function (firstArgIndex, noDupCheck) {
   return ['$' + firstArgIndex];
 };
 
-Param.prototype.toArgumentNameList = function(firstArgIndex, noDupCheck) {
+Param.prototype.toArgumentNameList = function (firstArgIndex, noDupCheck) {
   return ['param' + this.index];
 };
 
@@ -3119,7 +3167,7 @@ Param.prototype.toArgumentNameList = function(firstArgIndex, noDupCheck) {
 // Returns a string representing the PExpr, for use as a UI label, etc.
 PExpr.prototype.toDisplayString = abstract('toDisplayString');
 
-Alt.prototype.toDisplayString = Seq.prototype.toDisplayString = function() {
+Alt.prototype.toDisplayString = Seq.prototype.toDisplayString = function () {
   if (this.source) {
     return this.source.trimmed().contents;
   }
@@ -3135,11 +3183,11 @@ any.toDisplayString =
   Terminal.prototype.toDisplayString =
   Range.prototype.toDisplayString =
   Param.prototype.toDisplayString =
-    function() {
+    function () {
       return this.toString();
     };
 
-Apply.prototype.toDisplayString = function() {
+Apply.prototype.toDisplayString = function () {
   if (this.args.length > 0) {
     const ps = this.args.map(arg => arg.toDisplayString());
     return this.ruleName + '<' + ps.join(',') + '>';
@@ -3148,8 +3196,8 @@ Apply.prototype.toDisplayString = function() {
   }
 };
 
-UnicodeChar.prototype.toDisplayString = function() {
-  return 'Unicode [' + this.category + '] character';
+UnicodeChar.prototype.toDisplayString = function () {
+  return 'Unicode [' + this.categoryOrProp + '] character';
 };
 
 // --------------------------------------------------------------------
@@ -3252,34 +3300,34 @@ class Failure {
 
 PExpr.prototype.toFailure = abstract('toFailure');
 
-any.toFailure = function(grammar) {
+any.toFailure = function (grammar) {
   return new Failure(this, 'any object', 'description');
 };
 
-end.toFailure = function(grammar) {
+end.toFailure = function (grammar) {
   return new Failure(this, 'end of input', 'description');
 };
 
-Terminal.prototype.toFailure = function(grammar) {
+Terminal.prototype.toFailure = function (grammar) {
   return new Failure(this, this.obj, 'string');
 };
 
-Range.prototype.toFailure = function(grammar) {
+Range.prototype.toFailure = function (grammar) {
   // TODO: come up with something better
   return new Failure(this, JSON.stringify(this.from) + '..' + JSON.stringify(this.to), 'code');
 };
 
-Not.prototype.toFailure = function(grammar) {
+Not.prototype.toFailure = function (grammar) {
   const description =
     this.expr === any ? 'nothing' : 'not ' + this.expr.toFailure(grammar);
   return new Failure(this, description, 'description');
 };
 
-Lookahead.prototype.toFailure = function(grammar) {
+Lookahead.prototype.toFailure = function (grammar) {
   return this.expr.toFailure(grammar);
 };
 
-Apply.prototype.toFailure = function(grammar) {
+Apply.prototype.toFailure = function (grammar) {
   let {description} = grammar.rules[this.ruleName];
   if (!description) {
     const article = /^[aeiouAEIOU]/.test(this.ruleName) ? 'an' : 'a';
@@ -3288,23 +3336,23 @@ Apply.prototype.toFailure = function(grammar) {
   return new Failure(this, description, 'description');
 };
 
-UnicodeChar.prototype.toFailure = function(grammar) {
-  return new Failure(this, 'a Unicode [' + this.category + '] character', 'description');
+UnicodeChar.prototype.toFailure = function (grammar) {
+  return new Failure(this, 'a Unicode [' + this.categoryOrProp + '] character', 'description');
 };
 
-Alt.prototype.toFailure = function(grammar) {
+Alt.prototype.toFailure = function (grammar) {
   const fs = this.terms.map(t => t.toFailure(grammar));
   const description = '(' + fs.join(' or ') + ')';
   return new Failure(this, description, 'description');
 };
 
-Seq.prototype.toFailure = function(grammar) {
+Seq.prototype.toFailure = function (grammar) {
   const fs = this.factors.map(f => f.toFailure(grammar));
   const description = '(' + fs.join(' ') + ')';
   return new Failure(this, description, 'description');
 };
 
-Iter.prototype.toFailure = function(grammar) {
+Iter.prototype.toFailure = function (grammar) {
   const description = '(' + this.expr.toFailure(grammar) + this.operator + ')';
   return new Failure(this, description, 'description');
 };
@@ -3322,55 +3370,55 @@ Iter.prototype.toFailure = function(grammar) {
 */
 PExpr.prototype.toString = abstract('toString');
 
-any.toString = function() {
+any.toString = function () {
   return 'any';
 };
 
-end.toString = function() {
+end.toString = function () {
   return 'end';
 };
 
-Terminal.prototype.toString = function() {
+Terminal.prototype.toString = function () {
   return JSON.stringify(this.obj);
 };
 
-Range.prototype.toString = function() {
+Range.prototype.toString = function () {
   return JSON.stringify(this.from) + '..' + JSON.stringify(this.to);
 };
 
-Param.prototype.toString = function() {
+Param.prototype.toString = function () {
   return '$' + this.index;
 };
 
-Lex.prototype.toString = function() {
+Lex.prototype.toString = function () {
   return '#(' + this.expr.toString() + ')';
 };
 
-Alt.prototype.toString = function() {
-  return this.terms.length === 1 ?
-    this.terms[0].toString() :
-    '(' + this.terms.map(term => term.toString()).join(' | ') + ')';
+Alt.prototype.toString = function () {
+  return this.terms.length === 1
+    ? this.terms[0].toString()
+    : '(' + this.terms.map(term => term.toString()).join(' | ') + ')';
 };
 
-Seq.prototype.toString = function() {
-  return this.factors.length === 1 ?
-    this.factors[0].toString() :
-    '(' + this.factors.map(factor => factor.toString()).join(' ') + ')';
+Seq.prototype.toString = function () {
+  return this.factors.length === 1
+    ? this.factors[0].toString()
+    : '(' + this.factors.map(factor => factor.toString()).join(' ') + ')';
 };
 
-Iter.prototype.toString = function() {
+Iter.prototype.toString = function () {
   return this.expr + this.operator;
 };
 
-Not.prototype.toString = function() {
+Not.prototype.toString = function () {
   return '~' + this.expr;
 };
 
-Lookahead.prototype.toString = function() {
+Lookahead.prototype.toString = function () {
   return '&' + this.expr;
 };
 
-Apply.prototype.toString = function() {
+Apply.prototype.toString = function () {
   if (this.args.length > 0) {
     const ps = this.args.map(arg => arg.toString());
     return this.ruleName + '<' + ps.join(',') + '>';
@@ -3379,8 +3427,8 @@ Apply.prototype.toString = function() {
   }
 };
 
-UnicodeChar.prototype.toString = function() {
-  return '\\p{' + this.category + '}';
+UnicodeChar.prototype.toString = function () {
+  return '\\p{' + this.categoryOrProp + '}';
 };
 
 class CaseInsensitiveTerminal extends PExpr {
@@ -3428,9 +3476,9 @@ class CaseInsensitiveTerminal extends PExpr {
 
   toFailure(grammar) {
     return new Failure(
-        this,
-        this.obj.toFailure(grammar) + ' (case-insensitive)',
-        'description',
+      this,
+      this.obj.toFailure(grammar) + ' (case-insensitive)',
+      'description'
     );
   }
 
@@ -3500,8 +3548,8 @@ class MatchState {
     posInfo.exit();
 
     this.rightmostFailurePosition = Math.max(
-        this.rightmostFailurePosition,
-        this._rightmostFailurePositionStack.pop(),
+      this.rightmostFailurePosition,
+      this._rightmostFailurePositionStack.pop()
     );
 
     if (optNode) {
@@ -3637,9 +3685,9 @@ class MatchState {
   }
 
   _getRightmostFailureOffset() {
-    return this.rightmostFailurePosition >= 0 ?
-      this.posToOffset(this.rightmostFailurePosition) :
-      -1;
+    return this.rightmostFailurePosition >= 0
+      ? this.posToOffset(this.rightmostFailurePosition)
+      : -1;
   }
 
   // Returns the memoized trace entry for `expr` at `pos`, if one exists, `null` otherwise.
@@ -3696,8 +3744,8 @@ class MatchState {
     const memoRecRightmostFailurePosition =
       this.inputStream.pos + memoRec.rightmostFailureOffset;
     this.rightmostFailurePosition = Math.max(
-        this.rightmostFailurePosition,
-        memoRecRightmostFailurePosition,
+      this.rightmostFailurePosition,
+      memoRecRightmostFailurePosition
     );
     if (
       this.recordedFailures &&
@@ -3708,8 +3756,8 @@ class MatchState {
     }
 
     this.inputStream.examinedLength = Math.max(
-        this.inputStream.examinedLength,
-        memoRec.examinedLength + origPos,
+      this.inputStream.examinedLength,
+      memoRec.examinedLength + origPos
     );
 
     if (memoRec.value) {
@@ -3787,7 +3835,7 @@ class MatchState {
     let rightmostFailures;
     if (this.recordedFailures) {
       rightmostFailures = Object.keys(this.recordedFailures).map(
-          key => this.recordedFailures[key],
+        key => this.recordedFailures[key]
       );
     }
     const cst = this._bindings[0];
@@ -3795,13 +3843,13 @@ class MatchState {
       cst.grammar = this.grammar;
     }
     return new MatchResult(
-        this.matcher,
-        this.input,
-        this.startExpr,
-        cst,
-        this._bindingOffsets[0],
-        this.rightmostFailurePosition,
-        rightmostFailures,
+      this.matcher,
+      this.input,
+      this.startExpr,
+      cst,
+      this._bindingOffsets[0],
+      this.rightmostFailurePosition,
+      rightmostFailures
     );
   }
 
@@ -4114,11 +4162,11 @@ class Semantics {
     if (superSemantics) {
       if (!(grammar.equals(this.super.grammar) || grammar._inheritsFrom(this.super.grammar))) {
         throw new Error(
-            "Cannot extend a semantics for grammar '" +
+          "Cannot extend a semantics for grammar '" +
             this.super.grammar.name +
             "' for use with grammar '" +
             grammar.name +
-            "' (not a sub-grammar)",
+            "' (not a sub-grammar)"
         );
       }
       this.operations = Object.create(this.super.operations);
@@ -4127,7 +4175,7 @@ class Semantics {
 
       // Assign unique symbols for each of the attributes inherited from the super-semantics so that
       // they are memoized independently.
-      // eslint-disable-next-line guard-for-in
+
       for (const attributeName in this.attributes) {
         Object.defineProperty(this.attributeKeys, attributeName, {
           value: uniqueId(attributeName),
@@ -4156,11 +4204,11 @@ class Semantics {
   // Throws an exception if one or more of them doesn't.
   checkActionDicts() {
     let name;
-    // eslint-disable-next-line guard-for-in
+
     for (name in this.operations) {
       this.operations[name].checkActionDict(this.grammar);
     }
-    // eslint-disable-next-line guard-for-in
+
     for (name in this.attributes) {
       this.attributes[name].checkActionDict(this.grammar);
     }
@@ -4260,9 +4308,9 @@ class Semantics {
     });
 
     const entry =
-      type === 'operation' ?
-        new Operation(name, formals, realActionDict, builtInDefault) :
-        new Attribute(name, realActionDict, builtInDefault);
+      type === 'operation'
+        ? new Operation(name, formals, realActionDict, builtInDefault)
+        : new Attribute(name, realActionDict, builtInDefault);
 
     // The following check is not strictly necessary (it will happen later anyway) but it's better
     // to catch errors early.
@@ -4278,7 +4326,7 @@ class Semantics {
       // Check that the caller passed the correct number of arguments.
       if (arguments.length !== thisThing.formals.length) {
         throw new Error(
-            'Invalid number of arguments passed to ' +
+          'Invalid number of arguments passed to ' +
             name +
             ' ' +
             type +
@@ -4286,7 +4334,7 @@ class Semantics {
             thisThing.formals.length +
             ', got ' +
             arguments.length +
-            ')',
+            ')'
         );
       }
 
@@ -4307,7 +4355,7 @@ class Semantics {
 
     if (type === 'operation') {
       this.Wrapper.prototype[name] = doIt;
-      this.Wrapper.prototype[name].toString = function() {
+      this.Wrapper.prototype[name].toString = function () {
         return '[' + name + ' operation]';
       };
     } else {
@@ -4329,13 +4377,13 @@ class Semantics {
 
     if (!(this.super && name in this.super[typePlural])) {
       throw new Error(
-          'Cannot extend ' +
+        'Cannot extend ' +
           type +
           " '" +
           name +
           "': did not inherit an " +
           type +
-          ' with that name',
+          ' with that name'
       );
     }
     if (hasOwnProperty(this[typePlural], name)) {
@@ -4352,9 +4400,9 @@ class Semantics {
     });
 
     this[typePlural][name] =
-      type === 'operation' ?
-        new Operation(name, inheritedFormals, newActionDict) :
-        new Attribute(name, newActionDict);
+      type === 'operation'
+        ? new Operation(name, inheritedFormals, newActionDict)
+        : new Attribute(name, newActionDict);
 
     // The following check is not strictly necessary (it will happen later anyway) but it's better
     // to catch errors early.
@@ -4367,12 +4415,12 @@ class Semantics {
     }
     if (name in this.operations) {
       throw new Error(
-          'Cannot add ' + type + " '" + name + "': an operation with that name already exists",
+        'Cannot add ' + type + " '" + name + "': an operation with that name already exists"
       );
     }
     if (name in this.attributes) {
       throw new Error(
-          'Cannot add ' + type + " '" + name + "': an attribute with that name already exists",
+        'Cannot add ' + type + " '" + name + "': an attribute with that name already exists"
       );
     }
   }
@@ -4398,8 +4446,8 @@ function parseSignature(signature, type) {
   }
 
   const r = Semantics.prototypeGrammar.match(
-      signature,
-    type === 'operation' ? 'OperationSignature' : 'AttributeSignature',
+    signature,
+    type === 'operation' ? 'OperationSignature' : 'AttributeSignature'
   );
   if (r.failed()) {
     throw new Error(r.message);
@@ -4409,7 +4457,7 @@ function parseSignature(signature, type) {
 }
 
 function newDefaultAction(type, name, doIt) {
-  return function(...children) {
+  return function (...children) {
     const thisThing = this._semantics.operations[name] || this._semantics.attributes[name];
     const args = thisThing.formals.map(formal => this.args[formal]);
 
@@ -4433,12 +4481,12 @@ function newDefaultAction(type, name, doIt) {
 // Semantics instance. When that function is invoked with a CST node as an argument, it returns
 // a wrapper for that node which gives access to the operations and attributes provided by this
 // semantics.
-Semantics.createSemantics = function(grammar, optSuperSemantics) {
+Semantics.createSemantics = function (grammar, optSuperSemantics) {
   const s = new Semantics(
-      grammar,
-    optSuperSemantics !== undefined ?
-      optSuperSemantics :
-      Semantics.BuiltInSemantics._getSemantics(),
+    grammar,
+    optSuperSemantics !== undefined
+      ? optSuperSemantics
+      : Semantics.BuiltInSemantics._getSemantics()
   );
 
   // To enable clients to invoke a semantics like a function, return a function that acts as a proxy
@@ -4446,8 +4494,8 @@ Semantics.createSemantics = function(grammar, optSuperSemantics) {
   const proxy = function ASemantics(matchResult) {
     if (!(matchResult instanceof MatchResult)) {
       throw new TypeError(
-          'Semantics expected a MatchResult, but got ' +
-          unexpectedObjToString(matchResult),
+        'Semantics expected a MatchResult, but got ' +
+          unexpectedObjToString(matchResult)
       );
     }
     if (matchResult.failed()) {
@@ -4457,11 +4505,11 @@ Semantics.createSemantics = function(grammar, optSuperSemantics) {
     const cst = matchResult._cst;
     if (cst.grammar !== grammar) {
       throw new Error(
-          "Cannot use a MatchResult from grammar '" +
+        "Cannot use a MatchResult from grammar '" +
           cst.grammar.name +
           "' with a semantics for '" +
           grammar.name +
-          "'",
+          "'"
       );
     }
     const inputStream = new InputStream(matchResult.input);
@@ -4469,38 +4517,38 @@ Semantics.createSemantics = function(grammar, optSuperSemantics) {
   };
 
   // Forward public methods from the proxy to the semantics instance.
-  proxy.addOperation = function(signature, actionDict) {
+  proxy.addOperation = function (signature, actionDict) {
     s.addOperationOrAttribute('operation', signature, actionDict);
     return proxy;
   };
-  proxy.extendOperation = function(name, actionDict) {
+  proxy.extendOperation = function (name, actionDict) {
     s.extendOperationOrAttribute('operation', name, actionDict);
     return proxy;
   };
-  proxy.addAttribute = function(name, actionDict) {
+  proxy.addAttribute = function (name, actionDict) {
     s.addOperationOrAttribute('attribute', name, actionDict);
     return proxy;
   };
-  proxy.extendAttribute = function(name, actionDict) {
+  proxy.extendAttribute = function (name, actionDict) {
     s.extendOperationOrAttribute('attribute', name, actionDict);
     return proxy;
   };
-  proxy._getActionDict = function(operationOrAttributeName) {
+  proxy._getActionDict = function (operationOrAttributeName) {
     const action =
       s.operations[operationOrAttributeName] || s.attributes[operationOrAttributeName];
     if (!action) {
       throw new Error(
-          '"' +
+        '"' +
           operationOrAttributeName +
           '" is not a valid operation or attribute ' +
           'name in this semantics for "' +
           grammar.name +
-          '"',
+          '"'
       );
     }
     return action.actionDict;
   };
-  proxy._remove = function(operationOrAttributeName) {
+  proxy._remove = function (operationOrAttributeName) {
     let semantic;
     if (operationOrAttributeName in s.operations) {
       semantic = s.operations[operationOrAttributeName];
@@ -4512,16 +4560,16 @@ Semantics.createSemantics = function(grammar, optSuperSemantics) {
     delete s.Wrapper.prototype[operationOrAttributeName];
     return semantic;
   };
-  proxy.getOperationNames = function() {
+  proxy.getOperationNames = function () {
     return Object.keys(s.operations);
   };
-  proxy.getAttributeNames = function() {
+  proxy.getAttributeNames = function () {
     return Object.keys(s.attributes);
   };
-  proxy.getGrammar = function() {
+  proxy.getGrammar = function () {
     return s.grammar;
   };
-  proxy.toRecipe = function(semanticsOnly) {
+  proxy.toRecipe = function (semanticsOnly) {
     return s.toRecipe(semanticsOnly);
   };
 
@@ -4529,7 +4577,7 @@ Semantics.createSemantics = function(grammar, optSuperSemantics) {
   proxy.toString = s.toString.bind(s);
 
   // Returns the semantics for the proxy.
-  proxy._getSemantics = function() {
+  proxy._getSemantics = function () {
     return s;
   };
 
@@ -4621,8 +4669,8 @@ const SPECIAL_ACTION_NAMES = ['_iter', '_terminal', '_nonterminal', '_default'];
 
 function getSortedRuleValues(grammar) {
   return Object.keys(grammar.rules)
-      .sort()
-      .map(name => grammar.rules[name]);
+    .sort()
+    .map(name => grammar.rules[name]);
 }
 
 // Until ES2019, JSON was not a valid subset of JavaScript because U+2028 (line separator)
@@ -4643,11 +4691,11 @@ class Grammar {
     if (optDefaultStartRule) {
       if (!(optDefaultStartRule in rules)) {
         throw new Error(
-            "Invalid start rule: '" +
+          "Invalid start rule: '" +
             optDefaultStartRule +
             "' is not a rule in grammar '" +
             name +
-            "'",
+            "'"
         );
       }
       this.defaultStartRule = optDefaultStartRule;
@@ -4718,7 +4766,6 @@ class Grammar {
   _checkTopDownActionDict(what, name, actionDict) {
     const problems = [];
 
-    // eslint-disable-next-line guard-for-in
     for (const k in actionDict) {
       const v = actionDict[k];
       const isSpecialAction = SPECIAL_ACTION_NAMES.includes(k);
@@ -4748,10 +4795,10 @@ class Grammar {
     if (problems.length > 0) {
       const prettyProblems = problems.map(problem => '- ' + problem);
       const error = new Error(
-          [
-            `Found errors in the action dictionary of the '${name}' ${what}:`,
-            ...prettyProblems,
-          ].join('\n'),
+        [
+          `Found errors in the action dictionary of the '${name}' ${what}:`,
+          ...prettyProblems,
+        ].join('\n')
       );
       error.problems = problems;
       throw error;
@@ -4764,9 +4811,9 @@ class Grammar {
     // All special actions have an expected arity of 0, though all but _terminal
     // are expected to use the rest parameter syntax (e.g. `_iter(...children)`).
     // This is considered to have arity 0, i.e. `((...args) => {}).length` is 0.
-    return SPECIAL_ACTION_NAMES.includes(actionName) ?
-      0 :
-      this.rules[actionName].body.getArity();
+    return SPECIAL_ACTION_NAMES.includes(actionName)
+      ? 0
+      : this.rules[actionName].body.getArity();
   }
 
   _inheritsFrom(grammar) {
@@ -4857,7 +4904,7 @@ class Grammar {
     sb.append('{');
 
     let first = true;
-    // eslint-disable-next-line guard-for-in
+
     for (const ruleName in this.rules) {
       const {body} = this.rules[ruleName];
       if (first) {
@@ -4904,10 +4951,10 @@ class Grammar {
     if (formals.length !== app.args.length) {
       const {source} = this.rules[app.ruleName];
       throw wrongNumberOfParameters(
-          app.ruleName,
-          formals.length,
-          app.args.length,
-          source,
+        app.ruleName,
+        formals.length,
+        app.args.length,
+        source
       );
     }
     return app;
@@ -4926,63 +4973,63 @@ class Grammar {
 // `digit`, and is implicitly the super-grammar of any grammar whose super-grammar
 // isn't specified.
 Grammar.ProtoBuiltInRules = new Grammar(
-    'ProtoBuiltInRules', // name
-    undefined, // supergrammar
-    {
-      any: {
-        body: any,
-        formals: [],
-        description: 'any character',
-        primitive: true,
-      },
-      end: {
-        body: end,
-        formals: [],
-        description: 'end of input',
-        primitive: true,
-      },
-
-      caseInsensitive: {
-        body: new CaseInsensitiveTerminal(new Param(0)),
-        formals: ['str'],
-        primitive: true,
-      },
-      lower: {
-        body: new UnicodeChar('Ll'),
-        formals: [],
-        description: 'a lowercase letter',
-        primitive: true,
-      },
-      upper: {
-        body: new UnicodeChar('Lu'),
-        formals: [],
-        description: 'an uppercase letter',
-        primitive: true,
-      },
-      // Union of Lt (titlecase), Lm (modifier), and Lo (other), i.e. any letter not in Ll or Lu.
-      unicodeLtmo: {
-        body: new UnicodeChar('Ltmo'),
-        formals: [],
-        description: 'a Unicode character in Lt, Lm, or Lo',
-        primitive: true,
-      },
-
-      // These rules are not truly primitive (they could be written in userland) but are defined
-      // here for bootstrapping purposes.
-      spaces: {
-        body: new Star(new Apply('space')),
-        formals: [],
-      },
-      space: {
-        body: new Range('\x00', ' '),
-        formals: [],
-        description: 'a space',
-      },
+  'ProtoBuiltInRules', // name
+  undefined, // supergrammar
+  {
+    any: {
+      body: any,
+      formals: [],
+      description: 'any character',
+      primitive: true,
+    },
+    end: {
+      body: end,
+      formals: [],
+      description: 'end of input',
+      primitive: true,
+    },
+
+    caseInsensitive: {
+      body: new CaseInsensitiveTerminal(new Param(0)),
+      formals: ['str'],
+      primitive: true,
+    },
+    lower: {
+      body: new UnicodeChar('Ll'),
+      formals: [],
+      description: 'a lowercase letter',
+      primitive: true,
+    },
+    upper: {
+      body: new UnicodeChar('Lu'),
+      formals: [],
+      description: 'an uppercase letter',
+      primitive: true,
+    },
+    // Union of Lt (titlecase), Lm (modifier), and Lo (other), i.e. any letter not in Ll or Lu.
+    unicodeLtmo: {
+      body: new UnicodeChar('Ltmo'),
+      formals: [],
+      description: 'a Unicode character in Lt, Lm, or Lo',
+      primitive: true,
     },
+
+    // These rules are not truly primitive (they could be written in userland) but are defined
+    // here for bootstrapping purposes.
+    spaces: {
+      body: new Star(new Apply('space')),
+      formals: [],
+    },
+    space: {
+      body: new Range('\x00', ' '),
+      formals: [],
+      description: 'a space',
+    },
+  }
 );
 
 // This method is called from main.js once Ohm has loaded.
-Grammar.initApplicationParser = function(grammar, builderFn) {
+Grammar.initApplicationParser = function (grammar, builderFn) {
   ohmGrammar$1 = grammar;
   buildGrammar$1 = builderFn;
 };
@@ -5010,7 +5057,7 @@ class GrammarDecl {
         // TODO: The conditional expression below is an ugly hack. It's kind of ok because
         // I doubt anyone will ever try to declare a grammar called `BuiltInRules`. Still,
         // we should try to find a better way to do this.
-        this.name === 'BuiltInRules' ? Grammar.ProtoBuiltInRules : Grammar.BuiltInRules,
+        this.name === 'BuiltInRules' ? Grammar.ProtoBuiltInRules : Grammar.BuiltInRules
       );
     }
     return this.superGrammar;
@@ -5078,10 +5125,10 @@ class GrammarDecl {
   // Creates a Grammar instance, and if it passes the sanity checks, returns it.
   build() {
     const grammar = new Grammar(
-        this.name,
-        this.ensureSuperGrammar(),
-        this.rules,
-        this.defaultStartRule,
+      this.name,
+      this.ensureSuperGrammar(),
+      this.rules,
+      this.defaultStartRule
     );
     // Initialize internal props that are inherited from the super grammar.
     grammar._matchStateInitializer = grammar.superGrammar._matchStateInitializer;
@@ -5182,7 +5229,7 @@ class Builder {
     if (superGrammar) {
       // `superGrammar` may be a recipe (i.e. an Array), or an actual grammar instance.
       gDecl.withSuperGrammar(
-        superGrammar instanceof Grammar ? superGrammar : this.fromRecipe(superGrammar),
+        superGrammar instanceof Grammar ? superGrammar : this.fromRecipe(superGrammar)
       );
     }
     if (defaultStartRule) {
@@ -5206,8 +5253,8 @@ class Builder {
       let source;
       if (gDecl.source && metaInfo && metaInfo.sourceInterval) {
         source = gDecl.source.subInterval(
-            metaInfo.sourceInterval[0],
-            metaInfo.sourceInterval[1] - metaInfo.sourceInterval[0],
+          metaInfo.sourceInterval[0],
+          metaInfo.sourceInterval[1] - metaInfo.sourceInterval[0]
         );
       }
       gDecl[action](ruleName, formals, body, description, source);
@@ -5302,7 +5349,7 @@ class Builder {
 
   app(ruleName, optParams) {
     if (optParams && optParams.length > 0) {
-      optParams = optParams.map(function(param) {
+      optParams = optParams.map(function (param) {
         return param instanceof PExpr ? param : this.fromRecipe(param);
       }, this);
     }
@@ -5314,10 +5361,10 @@ class Builder {
   // `this.currentDecl` and `this.currentRuleName` being set.
   splice(beforeTerms, afterTerms) {
     return new Splice(
-        this.currentDecl.superGrammar,
-        this.currentRuleName,
-        beforeTerms.map(term => this.fromRecipe(term)),
-        afterTerms.map(term => this.fromRecipe(term)),
+      this.currentDecl.superGrammar,
+      this.currentRuleName,
+      beforeTerms.map(term => this.fromRecipe(term)),
+      afterTerms.map(term => this.fromRecipe(term))
     );
   }
 
@@ -5458,10 +5505,10 @@ function buildGrammar(match, namespace, optOhmGrammarForTesting) {
         });
 
         return new Splice(
-            decl.superGrammar,
-            currentRuleName,
-            beforeTerms,
-            afterTerms,
+          decl.superGrammar,
+          currentRuleName,
+          beforeTerms,
+          afterTerms
         ).withSource(this.source);
       } else {
         return builder.alt(...args).withSource(this.source);
@@ -5606,14 +5653,14 @@ function initBuiltInSemantics(builtInRules) {
   };
 
   Semantics.BuiltInSemantics = Semantics.createSemantics(builtInRules, null).addOperation(
-      'asIteration',
-      {
-        emptyListOf: actions.empty,
-        nonemptyListOf: actions.nonEmpty,
-        EmptyListOf: actions.empty,
-        NonemptyListOf: actions.nonEmpty,
-        _iter: actions.self,
-      },
+    'asIteration',
+    {
+      emptyListOf: actions.empty,
+      nonemptyListOf: actions.nonEmpty,
+      EmptyListOf: actions.empty,
+      NonemptyListOf: actions.nonEmpty,
+      _iter: actions.self,
+    }
   );
 }
 
@@ -5802,12 +5849,12 @@ const applyDedent = new Apply('dedent');
 const newAnyBody = new Splice(BuiltInRules, 'any', [applyIndent, applyDedent], []);
 
 const IndentationSensitive = new Builder()
-    .newGrammar('IndentationSensitive')
-    .withSuperGrammar(BuiltInRules)
-    .define('indent', [], new Indentation(true), INDENT_DESCRIPTION, undefined, true)
-    .define('dedent', [], new Indentation(false), DEDENT_DESCRIPTION, undefined, true)
-    .extend('any', [], newAnyBody, 'any character', undefined)
-    .build();
+  .newGrammar('IndentationSensitive')
+  .withSuperGrammar(BuiltInRules)
+  .define('indent', [], new Indentation(true), INDENT_DESCRIPTION, undefined, true)
+  .define('dedent', [], new Indentation(false), DEDENT_DESCRIPTION, undefined, true)
+  .extend('any', [], newAnyBody, 'any character', undefined)
+  .build();
 
 Object.assign(IndentationSensitive, {
   _matchStateInitializer(state) {
@@ -5840,7 +5887,7 @@ function grammars$1(source, optNamespace) {
       source = source.toString();
     } else {
       throw new TypeError(
-          'Expected string as first argument, got ' + unexpectedObjToString(source),
+        'Expected string as first argument, got ' + unexpectedObjToString(source)
       );
     }
   }
@@ -5996,9 +6043,59 @@ function extractExamples(grammarsDef) {
   return semantics(matchResult).examples();
 }
 
+/*
+  To find iter nodes that are derived from the same repetition expression, we
+  look for adjacent iter nodes that have the same source interval and the same
+  number of children.
+
+  A few things to note:
+  - The children of `*` and `+` nodes can't be nullable, so the associated iter
+    nodes always consume some input, and therefore consecutive nodes that have
+    the same interval must come from the same repetition expression.
+  - We *could* mistake `a? b?` for (a b)?`, if neither of them comsume any input.
+    However, for the purposes of this module, those two cases are equivalent
+    anyways, since we only care about finding the correct order of the non-iter
+    nodes, and both interpretations yield the same order.
+ */
+const isIterSibling = (refNode, n) => {
+  return (
+    n.isIteration() &&
+    n.source.startIdx === refNode.source.startIdx &&
+    n.source.endIdx === refNode.source.endIdx &&
+    n.children.length === refNode.children.length
+  );
+};
+
+function recoverSourceOrder(nodes, depth = 0) {
+  const ans = [];
+  for (let i = 0; i < nodes.length; i++) {
+    const n = nodes[i];
+    if (!n.isIteration()) {
+      ans.push(n);
+      continue;
+    }
+
+    // We found an iter node, now find its siblings.
+    const siblings = [n];
+    // Find the first node that's *not* part of the current list.
+    for (let j = i + 1; j < nodes.length && isIterSibling(n, nodes[j]); j++) {
+      siblings.push(nodes[j]);
+      i = j;
+    }
+    const cousins = [];
+    const numRows = siblings[0].children.length;
+    for (let row = 0; row < numRows; row++) {
+      cousins.push(...siblings.map(sib => sib.children[row]));
+    }
+    ans.push(...recoverSourceOrder(cousins, depth + 1));
+  }
+  return ans;
+}
+
 exports.VisitorFamily = VisitorFamily;
 exports.extractExamples = extractExamples;
 exports.getLineAndColumn = getLineAndColumn;
 exports.getLineAndColumnMessage = getLineAndColumnMessage;
+exports.recoverSourceOrder = recoverSourceOrder;
 exports.semanticsForToAST = semanticsForToAST;
 exports.toAST = toAST;