ohm-js 16.2.0-pre.esm → 16.3.0-dev.unicode-code-point-escape

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ohm-js",
-  "version": "16.2.0-pre.esm",
+  "version": "16.3.0-dev.unicode-code-point-escape",
   "description": "An object-oriented language for parsing and pattern matching",
   "repository": "https://github.com/harc/ohm",
   "keywords": [

package/src/InputStream.js

@@ -29,6 +29,21 @@ InputStream.prototype = {
     return ans;
   },
 
+  nextCharCode() {
+    const nextChar = this.next();
+    return nextChar && nextChar.charCodeAt(0);
+  },
+
+  nextCodePoint() {
+    const cp = this.source.slice(this.pos++).codePointAt(0);
+    // If the code point is beyond plane 0, it takes up two characters.
+    if (cp > 0xffff) {
+      this.pos += 1;
+    }
+    this.examinedLength = Math.max(this.examinedLength, this.pos);
+    return cp;
+  },
+
   matchString(s, optIgnoreCase) {
     let idx;
     if (optIgnoreCase) {

package/src/common.js

@@ -149,6 +149,8 @@ exports.escapeChar = function(c, optDelim) {
   }
 };
 
+const escapeUnicode = str => String.fromCodePoint(parseInt(str, 16));
+
 exports.unescapeChar = function(s) {
   if (s.charAt(0) === '\\') {
     switch (s.charAt(1)) {
@@ -165,9 +167,11 @@ exports.unescapeChar = function(s) {
       case 'v':
         return '\v';
       case 'x':
-        return String.fromCharCode(parseInt(s.substring(2, 4), 16));
+        return escapeUnicode(s.slice(2, 4));
       case 'u':
-        return String.fromCharCode(parseInt(s.substring(2, 6), 16));
+        return s.charAt(2) === '{' ?
+          escapeUnicode(s.slice(3, -1)) :
+          escapeUnicode(s.slice(2, 6));
       default:
         return s.charAt(1);
     }

package/src/main.js

@@ -230,10 +230,6 @@ function buildGrammar(match, namespace, optOhmGrammarForTesting) {
       return common.unescapeChar(this.sourceString);
     },
 
-    escapeChar(_) {
-      return this.sourceString;
-    },
-
     NonemptyListOf(x, _, xs) {
       return [x.visit()].concat(xs.children.map(c => c.visit()));
     },

package/src/ohm-grammar.ohm

@@ -92,7 +92,7 @@ Ohm {
 
   terminalChar
     = escapeChar
-    | ~"\\" ~"\"" ~"\n" any
+      | ~"\\" ~"\"" ~"\n" "\u{0}".."\u{10FFFF}"
 
   escapeChar  (an escape sequence)
     = "\\\\"                                     -- backslash
@@ -102,6 +102,7 @@ Ohm {
     | "\\n"                                      -- lineFeed
     | "\\r"                                      -- carriageReturn
     | "\\t"                                      -- tab
+    | "\\u{" hexDigit+ "}"                       -- unicodeCodePoint
     | "\\u" hexDigit hexDigit hexDigit hexDigit  -- unicodeEscape
     | "\\x" hexDigit hexDigit                    -- hexEscape
 

package/src/pexprs-eval.js

@@ -76,9 +76,14 @@ pexprs.Terminal.prototype.eval = function(state) {
 pexprs.Range.prototype.eval = function(state) {
   const {inputStream} = state;
   const origPos = inputStream.pos;
-  const ch = inputStream.next();
-  if (ch && this.from <= ch && ch <= this.to) {
-    state.pushBinding(new TerminalNode(state.grammar, ch), origPos);
+
+  const cp =
+    this.from.length > 1 || this.to.length > 1 ?
+      inputStream.nextCodePoint() :
+      inputStream.nextCharCode();
+
+  if (cp !== undefined && this.from.codePointAt(0) <= cp && cp <= this.to.codePointAt(0)) {
+    state.pushBinding(new TerminalNode(state.grammar, String.fromCodePoint(cp)), origPos);
     return true;
   } else {
     state.processFailure(origPos, this);