ohm-js 16.3.0-dev.unicode-code-point-escape → 16.3.2

package/extras/semantics-toAST.js

@@ -1,13 +1,5 @@
 'use strict';
 
-// --------------------------------------------------------------------
-// Imports
-// --------------------------------------------------------------------
-
-const pexprs = require('../src/pexprs');
-const MatchResult = require('../src/MatchResult');
-const Grammar = require('../src/Grammar');
-
 // --------------------------------------------------------------------
 // Operations
 // --------------------------------------------------------------------
@@ -23,11 +15,6 @@ const defaultOperation = {
 
     // without customization
     if (!Object.prototype.hasOwnProperty.call(mapping, ctorName)) {
-      // intermediate node
-      if (this._node instanceof pexprs.Alt || this._node instanceof pexprs.Apply) {
-        return children[0].toAST(mapping);
-      }
-
       // lexical rule
       if (this.isLexical()) {
         return this.sourceString;
@@ -111,8 +98,8 @@ const defaultOperation = {
 // The optional `mapping` parameter can be used to customize how the nodes of the CST
 // are mapped to the AST (see /doc/extras.md#toastmatchresult-mapping).
 function toAST(res, mapping) {
-  if (!(res instanceof MatchResult) || res.failed()) {
-    throw new Error('toAST() expects a succesfull MatchResult as first parameter');
+  if (typeof res.failed !== 'function' || res.failed()) {
+    throw new Error('toAST() expects a succesful MatchResult as first parameter');
   }
 
   mapping = Object.assign({}, mapping);
@@ -130,7 +117,7 @@ function toAST(res, mapping) {
 
 // Returns a semantics containg the toAST(mapping) operation for the given grammar g.
 function semanticsForToAST(g) {
-  if (!(g instanceof Grammar)) {
+  if (typeof g.createSemantics !== 'function') {
     throw new Error('semanticsToAST() expects a Grammar as parameter');
   }
 

package/index.d.ts

@@ -309,13 +309,6 @@ declare namespace ohm {
      */
     isOptional: boolean;
 
-    /**
-     * For a terminal node, the raw value that was consumed from the
-     * input stream.
-     * @deprecated Use `sourceString` instead.
-     */
-    primitiveValue: string;
-
     /**
      * In addition to the properties defined above, within a given
      * semantics, every node also has a method/property corresponding to

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ohm-js",
-  "version": "16.3.0-dev.unicode-code-point-escape",
+  "version": "16.3.2",
   "description": "An object-oriented language for parsing and pattern matching",
   "repository": "https://github.com/harc/ohm",
   "keywords": [
@@ -15,7 +15,7 @@
     "rapid",
     "prototyping"
   ],
-  "homepage": "https://ohmlang.github.io/",
+  "homepage": "https://ohmjs.org",
   "bugs": "https://github.com/harc/ohm/issues",
   "main": "index.js",
   "module": "dist/ohm.esm.js",
@@ -59,33 +59,35 @@
     "Jason Merrill <jwmerrill@gmail.com>",
     "Ray Toal <rtoal@lmu.edu>",
     "Yoshiki Ohshima <Yoshiki.Ohshima@acm.org>",
+    "megabuz <3299889+megabuz@users.noreply.github.com>",
     "stagas <gstagas@gmail.com>",
     "Jonathan Edwards <JonathanMEdwards@gmail.com>",
     "Milan Lajtoš <milan.lajtos@me.com>",
     "Neil Jewers <njjewers@uwaterloo.ca>",
-    "megabuz <3299889+megabuz@users.noreply.github.com>",
     "Mike Niebling <(none)>",
-    "sfinnie <scott.finnie@gmail.com>",
-    "Justin Chase <justin.m.chase@gmail.com>",
+    "AngryPowman <angrypowman@qq.com>",
+    "Patrick Dubroy <patrick@sourcegraph.com>",
+    "Leslie Ying <acetophore@users.noreply.github.com>",
     "Pierre Donias <pierre.donias@gmail.com>",
+    "Justin Chase <justin.m.chase@gmail.com>",
     "Ian Harris <ian@fofgof.xyz>",
-    "Daniel Tomlinson <DanielTomlinson@me.com>",
     "Stan Rozenraukh <stan@stanistan.com>",
     "Stephan Seidt <stephan.seidt@gmail.com>",
     "Steve Phillips <steve@tryingtobeawesome.com>",
     "Szymon Kaliski <kaliskiszymon@gmail.com>",
     "Thomas Nyberg <tomnyberg@gmail.com>",
-    "Casey Olson <casey.m.olson@gmail.com>",
+    "Daniel Tomlinson <DanielTomlinson@me.com>",
     "Vse Mozhet Byt <vsemozhetbyt@gmail.com>",
     "Wil Chung <10446+iamwilhelm@users.noreply.github.com>",
-    "Arthur Carabott <arthurc@gmail.com>",
+    "Casey Olson <casey.m.olson@gmail.com>",
     "abego <ub@abego-software.de>",
     "acslk <d_vd415@hotmail.com>",
     "codeZeilen <codeZeilen@users.noreply.github.com>",
-    "AngryPowman <angrypowman@qq.com>",
+    "kassadin <kassadin@foxmail.com>",
+    "Arthur Carabott <arthurc@gmail.com>",
+    "owch <bowenrainyday@gmail.com>",
     "Luca Guzzon <luca.guzzon@gmail.com>",
-    "Leslie Ying <acetophore@users.noreply.github.com>",
-    "owch <bowenrainyday@gmail.com>"
+    "sfinnie <scott.finnie@gmail.com>"
   ],
   "dependencies": {},
   "devDependencies": {

package/src/CaseInsensitiveTerminal.js

@@ -35,7 +35,7 @@ class CaseInsensitiveTerminal extends PExpr {
       state.processFailure(origPos, this);
       return false;
     } else {
-      state.pushBinding(new TerminalNode(state.grammar, matchStr), origPos);
+      state.pushBinding(new TerminalNode(matchStr.length), origPos);
       return true;
     }
   }

package/src/MatchState.js

@@ -219,7 +219,7 @@ MatchState.prototype = {
   // Returns the memoized trace entry for `expr` at `pos`, if one exists, `null` otherwise.
   getMemoizedTraceEntry(pos, expr) {
     const posInfo = this.memoTable[pos];
-    if (posInfo && expr.ruleName) {
+    if (posInfo && expr instanceof pexprs.Apply) {
       const memoRec = posInfo.memo[expr.toMemoKey()];
       if (memoRec && memoRec.traceEntry) {
         const entry = memoRec.traceEntry.cloneWithExpr(expr);
@@ -361,11 +361,15 @@ MatchState.prototype = {
           key => this.recordedFailures[key]
       );
     }
+    const cst = this._bindings[0];
+    if (cst) {
+      cst.grammar = this.grammar;
+    }
     return new MatchResult(
         this.matcher,
         this.input,
         this.startExpr,
-        this._bindings[0],
+        cst,
         this._bindingOffsets[0],
         this.rightmostFailurePosition,
         rightmostFailures

package/src/Semantics.js

@@ -47,11 +47,6 @@ class Wrapper {
     return '[semantics wrapper for ' + this._node.grammar.name + ']';
   }
 
-  // This is used by ohm editor to display a node wrapper appropriately.
-  toJSON() {
-    return this.toString();
-  }
-
   _forgetMemoizedResultFor(attributeName) {
     // Remove the memoized attribute from the cstNode and all its children.
     delete this._node[this._semantics.attributeKeys[attributeName]];
@@ -131,7 +126,7 @@ class Wrapper {
     const childWrappers = optChildWrappers || [];
 
     const childNodes = childWrappers.map(c => c._node);
-    const iter = new IterationNode(this._node.grammar, childNodes, [], -1, false);
+    const iter = new IterationNode(childNodes, [], -1, false);
 
     const wrapper = this._semantics.wrap(iter, null, null);
     wrapper._childWrappers = childWrappers;
@@ -158,18 +153,6 @@ class Wrapper {
     return this._node.numChildren();
   }
 
-  // Returns the primitive value of this CST node, if it's a terminal node. Otherwise,
-  // throws an exception.
-  // DEPRECATED: Use `sourceString` instead.
-  get primitiveValue() {
-    if (this.isTerminal()) {
-      return this._node.primitiveValue;
-    }
-    throw new TypeError(
-        "tried to access the 'primitiveValue' attribute of a non-terminal CST node"
-    );
-  }
-
   // Returns the contents of the input stream consumed by this CST node.
   get sourceString() {
     return this.source.contents;

package/src/common.js

@@ -40,7 +40,7 @@ exports.abstract = function(optMethodName) {
 
 exports.assert = function(cond, message) {
   if (!cond) {
-    throw new Error(message);
+    throw new Error(message || 'Assertion failed');
   }
 };
 
@@ -134,24 +134,9 @@ exports.StringBuffer.prototype.contents = function() {
   return this.strings.join('');
 };
 
-// Character escaping and unescaping
-
-exports.escapeChar = function(c, optDelim) {
-  const charCode = c.charCodeAt(0);
-  if ((c === '"' || c === "'") && optDelim && c !== optDelim) {
-    return c;
-  } else if (charCode < 128) {
-    return escapeStringFor[charCode];
-  } else if (128 <= charCode && charCode < 256) {
-    return '\\x' + exports.padLeft(charCode.toString(16), 2, '0');
-  } else {
-    return '\\u' + exports.padLeft(charCode.toString(16), 4, '0');
-  }
-};
-
 const escapeUnicode = str => String.fromCodePoint(parseInt(str, 16));
 
-exports.unescapeChar = function(s) {
+exports.unescapeCodePoint = function(s) {
   if (s.charAt(0) === '\\') {
     switch (s.charAt(1)) {
       case 'b':

package/src/errors.js

@@ -4,9 +4,9 @@
 // Imports
 // --------------------------------------------------------------------
 
-const pexprs = require('./pexprs-main');
-
+const {assert} = require('./common');
 const Namespace = require('./Namespace');
+const pexprs = require('./pexprs-main');
 
 // --------------------------------------------------------------------
 // Private stuff
@@ -136,7 +136,7 @@ function wrongNumberOfArguments(ruleName, expected, actual, expr) {
       ', got ' +
       actual +
       ')',
-      expr.source
+      expr
   );
 }
 
@@ -209,6 +209,21 @@ function multipleSuperSplices(expr) {
   return createError("'...' can appear at most once in a rule body", expr.source);
 }
 
+// Unicode code point escapes
+
+function invalidCodePoint(applyWrapper) {
+  const node = applyWrapper._node;
+  assert(node && node.isNonterminal() && node.ctorName === 'escapeChar_unicodeCodePoint');
+
+  // Get an interval that covers all of the hex digits.
+  const digitIntervals = applyWrapper.children.slice(1, -1).map(d => d.source);
+  const fullInterval = digitIntervals[0].coverageWith(...digitIntervals.slice(1));
+  return createError(
+      `U+${fullInterval.contents} is not a valid Unicode code point`,
+      fullInterval
+  );
+}
+
 // ----------------- Kleene operators -----------------
 
 function kleeneExprHasNullableOperand(kleeneExpr, applicationStack) {
@@ -314,6 +329,7 @@ module.exports = {
   inconsistentArity,
   incorrectArgumentType,
   intervalSourcesDontMatch,
+  invalidCodePoint,
   invalidConstructorCall,
   invalidParameter,
   grammarSyntaxError,

package/src/main.js

@@ -226,8 +226,15 @@ function buildGrammar(match, namespace, optOhmGrammarForTesting) {
       return c.visit();
     },
 
-    terminalChar(_) {
-      return common.unescapeChar(this.sourceString);
+    escapeChar(c) {
+      try {
+        return common.unescapeCodePoint(this.sourceString);
+      } catch (err) {
+        if (err instanceof RangeError && err.message.startsWith('Invalid code point ')) {
+          throw errors.invalidCodePoint(c);
+        }
+        throw err; // Rethrow
+      }
     },
 
     NonemptyListOf(x, _, xs) {

package/src/nodes.js

@@ -2,20 +2,19 @@
 
 const common = require('./common');
 
-// Ensures that the deprecation warning for `primitiveValue` only appears once.
-let didWarnForPrimitiveValue = false;
-
 // --------------------------------------------------------------------
 // Private stuff
 // --------------------------------------------------------------------
 
 class Node {
-  constructor(grammar, ctorName, matchLength) {
-    this.grammar = grammar;
-    this.ctorName = ctorName;
+  constructor(matchLength) {
     this.matchLength = matchLength;
   }
 
+  get ctorName() {
+    throw new Error('subclass responsibility');
+  }
+
   numChildren() {
     return this.children ? this.children.length : 0;
   }
@@ -109,52 +108,38 @@ class Node {
   isOptional() {
     return false;
   }
-
-  toJSON() {
-    return {[this.ctorName]: this.children};
-  }
 }
 
 // Terminals
 
 class TerminalNode extends Node {
-  constructor(grammar, value) {
-    const matchLength = value ? value.length : 0;
-    super(grammar, '_terminal', matchLength);
-    this._value = value;
+  get ctorName() {
+    return '_terminal';
   }
 
   isTerminal() {
     return true;
   }
 
-  toJSON() {
-    return {[this.ctorName]: this._value};
-  }
-
   get primitiveValue() {
-    if (!didWarnForPrimitiveValue) {
-      // eslint-disable-next-line no-console
-      console.warn(
-          'Warning: primitiveValue is deprecated and will be removed in a future version of Ohm. ' +
-          'Use sourceString instead.'
-      );
-      didWarnForPrimitiveValue = true;
-    }
-
-    return this._value;
+    throw new Error('The `primitiveValue` property was removed in Ohm v17.');
   }
 }
 
 // Nonterminals
 
 class NonterminalNode extends Node {
-  constructor(grammar, ruleName, children, childOffsets, matchLength) {
-    super(grammar, ruleName, matchLength);
+  constructor(ruleName, children, childOffsets, matchLength) {
+    super(matchLength);
+    this.ruleName = ruleName;
     this.children = children;
     this.childOffsets = childOffsets;
   }
 
+  get ctorName() {
+    return this.ruleName;
+  }
+
   isNonterminal() {
     return true;
   }
@@ -171,13 +156,17 @@ class NonterminalNode extends Node {
 // Iterations
 
 class IterationNode extends Node {
-  constructor(grammar, children, childOffsets, matchLength, isOptional) {
-    super(grammar, '_iter', matchLength);
+  constructor(children, childOffsets, matchLength, isOptional) {
+    super(matchLength);
     this.children = children;
     this.childOffsets = childOffsets;
     this.optional = isOptional;
   }
 
+  get ctorName() {
+    return '_iter';
+  }
+
   isIteration() {
     return true;
   }

package/src/ohm-grammar.ohm

@@ -102,7 +102,8 @@ Ohm {
     | "\\n"                                      -- lineFeed
     | "\\r"                                      -- carriageReturn
     | "\\t"                                      -- tab
-    | "\\u{" hexDigit+ "}"                       -- unicodeCodePoint
+    | "\\u{" hexDigit hexDigit? hexDigit?
+             hexDigit? hexDigit? hexDigit? "}"   -- unicodeCodePoint
     | "\\u" hexDigit hexDigit hexDigit hexDigit  -- unicodeEscape
     | "\\x" hexDigit hexDigit                    -- hexEscape
 

package/src/pexprs-eval.js

@@ -41,7 +41,7 @@ pexprs.any.eval = function(state) {
   const origPos = inputStream.pos;
   const ch = inputStream.next();
   if (ch) {
-    state.pushBinding(new TerminalNode(state.grammar, ch), origPos);
+    state.pushBinding(new TerminalNode(ch.length), origPos);
     return true;
   } else {
     state.processFailure(origPos, this);
@@ -53,7 +53,7 @@ pexprs.end.eval = function(state) {
   const {inputStream} = state;
   const origPos = inputStream.pos;
   if (inputStream.atEnd()) {
-    state.pushBinding(new TerminalNode(state.grammar, undefined), origPos);
+    state.pushBinding(new TerminalNode(0), origPos);
     return true;
   } else {
     state.processFailure(origPos, this);
@@ -68,7 +68,7 @@ pexprs.Terminal.prototype.eval = function(state) {
     state.processFailure(origPos, this);
     return false;
   } else {
-    state.pushBinding(new TerminalNode(state.grammar, this.obj), origPos);
+    state.pushBinding(new TerminalNode(this.obj.length), origPos);
     return true;
   }
 };
@@ -77,13 +77,14 @@ pexprs.Range.prototype.eval = function(state) {
   const {inputStream} = state;
   const origPos = inputStream.pos;
 
-  const cp =
-    this.from.length > 1 || this.to.length > 1 ?
-      inputStream.nextCodePoint() :
-      inputStream.nextCharCode();
+  // A range can operate in one of two modes: matching a single, 16-bit _code unit_,
+  // or matching a _code point_. (Code points over 0xFFFF take up two 16-bit code units.)
+  const cp = this.matchCodePoint ? inputStream.nextCodePoint() : inputStream.nextCharCode();
 
+  // Always compare by code point value to get the correct result in all scenarios.
+  // Note that for strings of length 1, codePointAt(0) and charPointAt(0) are equivalent.
   if (cp !== undefined && this.from.codePointAt(0) <= cp && cp <= this.to.codePointAt(0)) {
-    state.pushBinding(new TerminalNode(state.grammar, String.fromCodePoint(cp)), origPos);
+    state.pushBinding(new TerminalNode(String.fromCodePoint(cp).length), origPos);
     return true;
   } else {
     state.processFailure(origPos, this);
@@ -168,7 +169,7 @@ pexprs.Iter.prototype.eval = function(state) {
   const isOptional = this instanceof pexprs.Opt;
   for (idx = 0; idx < cols.length; idx++) {
     state._bindings.push(
-        new IterationNode(state.grammar, cols[idx], colOffsets[idx], matchLength, isOptional)
+        new IterationNode(cols[idx], colOffsets[idx], matchLength, isOptional)
     );
     state._bindingOffsets.push(offset);
   }
@@ -343,13 +344,8 @@ pexprs.Apply.prototype.evalOnce = function(expr, state) {
     const arity = expr.getArity();
     const bindings = state._bindings.splice(state._bindings.length - arity, arity);
     const offsets = state._bindingOffsets.splice(state._bindingOffsets.length - arity, arity);
-    return new NonterminalNode(
-        state.grammar,
-        this.ruleName,
-        bindings,
-        offsets,
-        inputStream.pos - origPos
-    );
+    const matchLength = inputStream.pos - origPos;
+    return new NonterminalNode(this.ruleName, bindings, offsets, matchLength);
   } else {
     return false;
   }
@@ -404,7 +400,7 @@ pexprs.UnicodeChar.prototype.eval = function(state) {
   const origPos = inputStream.pos;
   const ch = inputStream.next();
   if (ch && this.pattern.test(ch)) {
-    state.pushBinding(new TerminalNode(state.grammar, ch), origPos);
+    state.pushBinding(new TerminalNode(ch.length), origPos);
     return true;
   } else {
     state.processFailure(origPos, this);

package/src/pexprs-main.js

@@ -53,6 +53,9 @@ class Range extends PExpr {
     super();
     this.from = from;
     this.to = to;
+    // If either `from` or `to` is made up of multiple code units, then
+    // the range should consume a full code point, not a single code unit.
+    this.matchCodePoint = from.length > 1 || to.length > 1;
   }
 }