ohm-js 16.2.0 → 16.3.1

package/extras/semantics-toAST.js

@@ -1,13 +1,5 @@
 'use strict';
 
-// --------------------------------------------------------------------
-// Imports
-// --------------------------------------------------------------------
-
-const pexprs = require('../src/pexprs');
-const MatchResult = require('../src/MatchResult');
-const Grammar = require('../src/Grammar');
-
 // --------------------------------------------------------------------
 // Operations
 // --------------------------------------------------------------------
@@ -23,11 +15,6 @@ const defaultOperation = {
 
     // without customization
     if (!Object.prototype.hasOwnProperty.call(mapping, ctorName)) {
-      // intermediate node
-      if (this._node instanceof pexprs.Alt || this._node instanceof pexprs.Apply) {
-        return children[0].toAST(mapping);
-      }
-
       // lexical rule
       if (this.isLexical()) {
         return this.sourceString;
@@ -111,8 +98,8 @@ const defaultOperation = {
 // The optional `mapping` parameter can be used to customize how the nodes of the CST
 // are mapped to the AST (see /doc/extras.md#toastmatchresult-mapping).
 function toAST(res, mapping) {
-  if (!(res instanceof MatchResult) || res.failed()) {
-    throw new Error('toAST() expects a succesfull MatchResult as first parameter');
+  if (typeof res.failed !== 'function' || res.failed()) {
+    throw new Error('toAST() expects a succesful MatchResult as first parameter');
   }
 
   mapping = Object.assign({}, mapping);
@@ -130,7 +117,7 @@ function toAST(res, mapping) {
 
 // Returns a semantics containg the toAST(mapping) operation for the given grammar g.
 function semanticsForToAST(g) {
-  if (!(g instanceof Grammar)) {
+  if (typeof g.createSemantics !== 'function') {
     throw new Error('semanticsToAST() expects a Grammar as parameter');
   }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ohm-js",
-  "version": "16.2.0",
+  "version": "16.3.1",
   "description": "An object-oriented language for parsing and pattern matching",
   "repository": "https://github.com/harc/ohm",
   "keywords": [
@@ -15,7 +15,7 @@
     "rapid",
     "prototyping"
   ],
-  "homepage": "https://ohmlang.github.io/",
+  "homepage": "https://ohmjs.org",
   "bugs": "https://github.com/harc/ohm/issues",
   "main": "index.js",
   "module": "dist/ohm.esm.js",
@@ -59,33 +59,35 @@
     "Jason Merrill <jwmerrill@gmail.com>",
     "Ray Toal <rtoal@lmu.edu>",
     "Yoshiki Ohshima <Yoshiki.Ohshima@acm.org>",
+    "megabuz <3299889+megabuz@users.noreply.github.com>",
     "stagas <gstagas@gmail.com>",
     "Jonathan Edwards <JonathanMEdwards@gmail.com>",
     "Milan Lajtoš <milan.lajtos@me.com>",
     "Neil Jewers <njjewers@uwaterloo.ca>",
-    "megabuz <3299889+megabuz@users.noreply.github.com>",
     "Mike Niebling <(none)>",
-    "sfinnie <scott.finnie@gmail.com>",
-    "Justin Chase <justin.m.chase@gmail.com>",
+    "AngryPowman <angrypowman@qq.com>",
+    "Patrick Dubroy <patrick@sourcegraph.com>",
+    "Leslie Ying <acetophore@users.noreply.github.com>",
     "Pierre Donias <pierre.donias@gmail.com>",
+    "Justin Chase <justin.m.chase@gmail.com>",
     "Ian Harris <ian@fofgof.xyz>",
-    "Daniel Tomlinson <DanielTomlinson@me.com>",
     "Stan Rozenraukh <stan@stanistan.com>",
     "Stephan Seidt <stephan.seidt@gmail.com>",
     "Steve Phillips <steve@tryingtobeawesome.com>",
     "Szymon Kaliski <kaliskiszymon@gmail.com>",
     "Thomas Nyberg <tomnyberg@gmail.com>",
-    "Casey Olson <casey.m.olson@gmail.com>",
+    "Daniel Tomlinson <DanielTomlinson@me.com>",
     "Vse Mozhet Byt <vsemozhetbyt@gmail.com>",
     "Wil Chung <10446+iamwilhelm@users.noreply.github.com>",
-    "Arthur Carabott <arthurc@gmail.com>",
+    "Casey Olson <casey.m.olson@gmail.com>",
     "abego <ub@abego-software.de>",
     "acslk <d_vd415@hotmail.com>",
     "codeZeilen <codeZeilen@users.noreply.github.com>",
-    "AngryPowman <angrypowman@qq.com>",
+    "kassadin <kassadin@foxmail.com>",
+    "Arthur Carabott <arthurc@gmail.com>",
+    "owch <bowenrainyday@gmail.com>",
     "Luca Guzzon <luca.guzzon@gmail.com>",
-    "Leslie Ying <acetophore@users.noreply.github.com>",
-    "owch <bowenrainyday@gmail.com>"
+    "sfinnie <scott.finnie@gmail.com>"
   ],
   "dependencies": {},
   "devDependencies": {

package/src/InputStream.js

@@ -29,6 +29,21 @@ InputStream.prototype = {
     return ans;
   },
 
+  nextCharCode() {
+    const nextChar = this.next();
+    return nextChar && nextChar.charCodeAt(0);
+  },
+
+  nextCodePoint() {
+    const cp = this.source.slice(this.pos++).codePointAt(0);
+    // If the code point is beyond plane 0, it takes up two characters.
+    if (cp > 0xffff) {
+      this.pos += 1;
+    }
+    this.examinedLength = Math.max(this.examinedLength, this.pos);
+    return cp;
+  },
+
   matchString(s, optIgnoreCase) {
     let idx;
     if (optIgnoreCase) {

package/src/common.js

@@ -40,7 +40,7 @@ exports.abstract = function(optMethodName) {
 
 exports.assert = function(cond, message) {
   if (!cond) {
-    throw new Error(message);
+    throw new Error(message || 'Assertion failed');
   }
 };
 
@@ -134,22 +134,9 @@ exports.StringBuffer.prototype.contents = function() {
   return this.strings.join('');
 };
 
-// Character escaping and unescaping
+const escapeUnicode = str => String.fromCodePoint(parseInt(str, 16));
 
-exports.escapeChar = function(c, optDelim) {
-  const charCode = c.charCodeAt(0);
-  if ((c === '"' || c === "'") && optDelim && c !== optDelim) {
-    return c;
-  } else if (charCode < 128) {
-    return escapeStringFor[charCode];
-  } else if (128 <= charCode && charCode < 256) {
-    return '\\x' + exports.padLeft(charCode.toString(16), 2, '0');
-  } else {
-    return '\\u' + exports.padLeft(charCode.toString(16), 4, '0');
-  }
-};
-
-exports.unescapeChar = function(s) {
+exports.unescapeCodePoint = function(s) {
   if (s.charAt(0) === '\\') {
     switch (s.charAt(1)) {
       case 'b':
@@ -165,9 +152,11 @@ exports.unescapeChar = function(s) {
       case 'v':
         return '\v';
       case 'x':
-        return String.fromCharCode(parseInt(s.substring(2, 4), 16));
+        return escapeUnicode(s.slice(2, 4));
       case 'u':
-        return String.fromCharCode(parseInt(s.substring(2, 6), 16));
+        return s.charAt(2) === '{' ?
+          escapeUnicode(s.slice(3, -1)) :
+          escapeUnicode(s.slice(2, 6));
       default:
         return s.charAt(1);
     }

package/src/errors.js

@@ -4,9 +4,9 @@
 // Imports
 // --------------------------------------------------------------------
 
-const pexprs = require('./pexprs-main');
-
+const {assert} = require('./common');
 const Namespace = require('./Namespace');
+const pexprs = require('./pexprs-main');
 
 // --------------------------------------------------------------------
 // Private stuff
@@ -136,7 +136,7 @@ function wrongNumberOfArguments(ruleName, expected, actual, expr) {
       ', got ' +
       actual +
       ')',
-      expr.source
+      expr
   );
 }
 
@@ -209,6 +209,21 @@ function multipleSuperSplices(expr) {
   return createError("'...' can appear at most once in a rule body", expr.source);
 }
 
+// Unicode code point escapes
+
+function invalidCodePoint(applyWrapper) {
+  const node = applyWrapper._node;
+  assert(node && node.isNonterminal() && node.ctorName === 'escapeChar_unicodeCodePoint');
+
+  // Get an interval that covers all of the hex digits.
+  const digitIntervals = applyWrapper.children.slice(1, -1).map(d => d.source);
+  const fullInterval = digitIntervals[0].coverageWith(...digitIntervals.slice(1));
+  return createError(
+      `U+${fullInterval.contents} is not a valid Unicode code point`,
+      fullInterval
+  );
+}
+
 // ----------------- Kleene operators -----------------
 
 function kleeneExprHasNullableOperand(kleeneExpr, applicationStack) {
@@ -314,6 +329,7 @@ module.exports = {
   inconsistentArity,
   incorrectArgumentType,
   intervalSourcesDontMatch,
+  invalidCodePoint,
   invalidConstructorCall,
   invalidParameter,
   grammarSyntaxError,

package/src/main.js

@@ -226,12 +226,15 @@ function buildGrammar(match, namespace, optOhmGrammarForTesting) {
       return c.visit();
     },
 
-    terminalChar(_) {
-      return common.unescapeChar(this.sourceString);
-    },
-
-    escapeChar(_) {
-      return this.sourceString;
+    escapeChar(c) {
+      try {
+        return common.unescapeCodePoint(this.sourceString);
+      } catch (err) {
+        if (err instanceof RangeError && err.message.startsWith('Invalid code point ')) {
+          throw errors.invalidCodePoint(c);
+        }
+        throw err; // Rethrow
+      }
     },
 
     NonemptyListOf(x, _, xs) {

package/src/ohm-grammar.ohm

@@ -92,7 +92,7 @@ Ohm {
 
   terminalChar
     = escapeChar
-    | ~"\\" ~"\"" ~"\n" any
+      | ~"\\" ~"\"" ~"\n" "\u{0}".."\u{10FFFF}"
 
   escapeChar  (an escape sequence)
     = "\\\\"                                     -- backslash
@@ -102,6 +102,8 @@ Ohm {
     | "\\n"                                      -- lineFeed
     | "\\r"                                      -- carriageReturn
     | "\\t"                                      -- tab
+    | "\\u{" hexDigit hexDigit? hexDigit?
+             hexDigit? hexDigit? hexDigit? "}"   -- unicodeCodePoint
     | "\\u" hexDigit hexDigit hexDigit hexDigit  -- unicodeEscape
     | "\\x" hexDigit hexDigit                    -- hexEscape
 

package/src/pexprs-eval.js

@@ -76,9 +76,15 @@ pexprs.Terminal.prototype.eval = function(state) {
 pexprs.Range.prototype.eval = function(state) {
   const {inputStream} = state;
   const origPos = inputStream.pos;
-  const ch = inputStream.next();
-  if (ch && this.from <= ch && ch <= this.to) {
-    state.pushBinding(new TerminalNode(state.grammar, ch), origPos);
+
+  // A range can operate in one of two modes: matching a single, 16-bit _code unit_,
+  // or matching a _code point_. (Code points over 0xFFFF take up two 16-bit code units.)
+  const cp = this.matchCodePoint ? inputStream.nextCodePoint() : inputStream.nextCharCode();
+
+  // Always compare by code point value to get the correct result in all scenarios.
+  // Note that for strings of length 1, codePointAt(0) and charPointAt(0) are equivalent.
+  if (cp !== undefined && this.from.codePointAt(0) <= cp && cp <= this.to.codePointAt(0)) {
+    state.pushBinding(new TerminalNode(state.grammar, String.fromCodePoint(cp)), origPos);
     return true;
   } else {
     state.processFailure(origPos, this);

package/src/pexprs-main.js

@@ -53,6 +53,9 @@ class Range extends PExpr {
     super();
     this.from = from;
     this.to = to;
+    // If either `from` or `to` is made up of multiple code units, then
+    // the range should consume a full code point, not a single code unit.
+    this.matchCodePoint = from.length > 1 || to.length > 1;
   }
 }