n3 3.0.0-alpha.7 → 3.0.0-alpha.9

package/src/N3Lexer.js

@@ -2,7 +2,35 @@
 import namespaces from './IRIs';
 
 const { xsd } = namespaces;
-const SPACE = 0x20, TAB = 0x09, LF = 0x0A, CR = 0x0D, HASH = 0x23;
+const SPACE = 0x20, TAB = 0x09, LF = 0x0A, CR = 0x0D, HASH = 0x23,
+    DOT = 0x2E, COLON = 0x3A, ZERO = 0x30, NINE = 0x39,
+    PERCENT = 0x25, BACKSLASH = 0x5C;
+
+// Whitespace as matched by `\s`
+function isWhitespace(charCode) {
+  return charCode === SPACE || charCode >= TAB && charCode <= CR ||
+    charCode >= 0xA0 && (charCode === 0xA0 || charCode === 0x1680 ||
+      charCode >= 0x2000 && charCode <= 0x200A || charCode === 0x2028 || charCode === 0x2029 ||
+      charCode === 0x202F || charCode === 0x205F || charCode === 0x3000 || charCode === 0xFEFF);
+}
+// Characters that can directly follow a name: whitespace and punctuation
+// (the lookahead `[,;!\^\s#()\[\]\{\}"'<>]` of the former regular expressions)
+const asciiDelimiters = new Uint8Array(0x80);
+for (const char of ',;!^#()[]{}"\'<> \t\n\v\f\r')
+  asciiDelimiters[char.charCodeAt(0)] = 1;
+function isDelimiter(charCode) {
+  return charCode < 0x80 ? asciiDelimiters[charCode] === 1 : isWhitespace(charCode);
+}
+// Whether a name can end before the given position: it must be followed by
+// a delimiter, optionally after a dot. At the end of finished input, it can always end.
+function canEndName(input, pos, inputFinished) {
+  let charCode = input.charCodeAt(pos);
+  if (charCode === DOT)
+    charCode = input.charCodeAt(++pos);
+  if (pos >= input.length)
+    return inputFinished;
+  return isDelimiter(charCode);
+}
 
 // Fixed escape sequences allowed in string literals (ECHAR)
 const stringEscapeReplacements = {
@@ -16,9 +44,105 @@ const localNameEscapeReplacements = {
   '=': '=', '/': '/', '?': '?', '#': '#', '@': '@', '%': '%',
 };
 const illegalIriChars = /[\x00-\x20<>\\"\{\}\|\^\`]/;
-// Characters that cannot occur in a prefixed name, not even escaped
-// (global, so that testAt searches the rest of the input from a position)
-const nonPrefixedNameChar = /[\s<>"{}|^`]/g;
+
+// Character classes of names, as bit flags for ASCII characters
+const PREFIX_START = 1, // PN_CHARS_BASE
+    LOCAL_START = 2,      // PN_CHARS_U, digits, and colon
+    NAME_CHAR = 4,        // PN_CHARS
+    LOCAL_CHAR = 8,       // PN_CHARS and colon
+    LOCAL_ESCAPE = 16;    // characters that can be escaped in local names (PN_LOCAL_ESC)
+const asciiNameClasses = new Uint8Array(0x80);
+for (let charCode = 0; charCode < 0x80; charCode++) {
+  const char = String.fromCharCode(charCode);
+  const letter = char >= 'A' && char <= 'Z' || char >= 'a' && char <= 'z';
+  const nameChar = letter || char === '_' || char === '-' || char >= '0' && char <= '9';
+  asciiNameClasses[charCode] =
+    (letter ? PREFIX_START : 0) |
+    (nameChar && char !== '-' || char === ':' ? LOCAL_START : 0) |
+    (nameChar ? NAME_CHAR : 0) |
+    (nameChar || char === ':' ? LOCAL_CHAR : 0) |
+    (char in localNameEscapeReplacements ? LOCAL_ESCAPE : 0);
+}
+// Returns the length (0, 1, or 2 code units) of the character at the given position
+// if it is in the given name character class
+function nameCharLength(input, pos, charClass) {
+  const charCode = input.charCodeAt(pos);
+  if (charCode < 0x80)
+    return (asciiNameClasses[charCode] & charClass) !== 0 ? 1 : 0;
+  // Characters from U+10000 to U+EFFFF consist of a surrogate pair
+  if (charCode >= 0xD800 && charCode <= 0xDB7F) {
+    const low = input.charCodeAt(pos + 1);
+    return low >= 0xDC00 && low <= 0xDFFF ? 2 : 0;
+  }
+  // PN_CHARS has some characters that PN_CHARS_BASE does not
+  if (charClass >= NAME_CHAR && (charCode === 0xB7 || charCode >= 0x300 && charCode <= 0x36F ||
+                                 charCode === 0x203F || charCode === 0x2040))
+    return 1;
+  // PN_CHARS_BASE
+  return charCode >= 0xC0 && charCode <= 0x1FFF && charCode !== 0xD7 && charCode !== 0xF7 &&
+           (charCode < 0x300 || charCode >= 0x370 && charCode !== 0x37E) ||
+         charCode >= 0x200C && charCode <= 0x200D || charCode >= 0x2070 && charCode <= 0x218F ||
+         charCode >= 0x2C00 && charCode <= 0x2FEF || charCode >= 0x3001 && charCode <= 0xD7FF ||
+         charCode >= 0xF900 && charCode <= 0xFDCF || charCode >= 0xFDF0 && charCode <= 0xFFFD ? 1 : 0;
+}
+function isLocalEscape(charCode) {
+  return charCode < 0x80 && (asciiNameClasses[charCode] & LOCAL_ESCAPE) !== 0;
+}
+function isHexDigit(charCode) {
+  return charCode >= ZERO && charCode <= NINE ||
+    charCode >= 0x41 && charCode <= 0x46 || charCode >= 0x61 && charCode <= 0x66;
+}
+// Returns the end of the prefix (PN_PREFIX) at the given position,
+// which can contain single dots, but not start or end with one
+function skipPrefix(input, pos) {
+  let length = nameCharLength(input, pos, PREFIX_START);
+  while (length !== 0) {
+    pos += length;
+    const next = input.charCodeAt(pos) === DOT ? pos + 1 : pos;
+    // Most names are ASCII, so look those up without a call
+    const charCode = input.charCodeAt(next);
+    length = charCode < 0x80 ? (asciiNameClasses[charCode] & NAME_CHAR) !== 0 ? 1 : 0 :
+      nameCharLength(input, next, NAME_CHAR);
+    if (length !== 0)
+      pos = next;
+  }
+  return pos;
+}
+// Returns the end of the local name (PN_LOCAL) at the given position,
+// which can contain dots, but not start or end with one
+function skipLocalName(input, pos) {
+  let end = pos, charClass = LOCAL_START;
+  while (true) {
+    // Most names are ASCII, so look those up without a call
+    const charCode = input.charCodeAt(pos);
+    let length = charCode < 0x80 ? (asciiNameClasses[charCode] & charClass) !== 0 ? 1 : 0 :
+      nameCharLength(input, pos, charClass);
+    // Percent-encoded character (PERCENT)
+    if (length === 0 && charCode === PERCENT &&
+        isHexDigit(input.charCodeAt(pos + 1)) && isHexDigit(input.charCodeAt(pos + 2)))
+      length = 3;
+    // Escaped character (PN_LOCAL_ESC)
+    else if (length === 0 && charCode === BACKSLASH && isLocalEscape(input.charCodeAt(pos + 1)))
+      length = 2;
+    if (length !== 0) {
+      end = pos += length;
+      charClass = LOCAL_CHAR;
+    }
+    // Dots are allowed after the first character, but not at the end
+    else if (charCode === DOT && charClass === LOCAL_CHAR)
+      pos++;
+    else
+      return end;
+  }
+}
+// Returns the end of the prefixed name at the given position, or -1 if there is none
+function skipPrefixedName(input, pos, inputFinished) {
+  const colon = skipPrefix(input, pos);
+  if (input.charCodeAt(colon) !== COLON)
+    return -1;
+  const end = skipLocalName(input, colon + 1);
+  return canEndName(input, end, inputFinished) ? end : -1;
+}
 
 // A valid code point is a Unicode scalar value: at most U+10FFFF and not a surrogate
 function isValidCodePoint(charCode) {
@@ -75,13 +199,11 @@ export default class N3Lexer {
     this._simpleQuotedString = /"([^"\\\r\n]*)"(?=[^"])/y; // string without escape sequences
     this._simpleApostropheString = /'([^'\\\r\n]*)'(?=[^'])/y;
     this._langcode = /@([a-z]+(?:-[a-z0-9]+)*)(?=[^a-z0-9])/iy;
-    this._prefix = /((?:[A-Za-z\xc0-\xd6\xd8-\xf6\xf8-\u02ff\u0370-\u037d\u037f-\u1fff\u200c\u200d\u2070-\u218f\u2c00-\u2fef\u3001-\ud7ff\uf900-\ufdcf\ufdf0-\ufffd]|[\ud800-\udb7f][\udc00-\udfff])(?:\.?[\-0-9A-Z_a-z\xb7\xc0-\xd6\xd8-\xf6\xf8-\u037d\u037f-\u1fff\u200c\u200d\u203f\u2040\u2070-\u218f\u2c00-\u2fef\u3001-\ud7ff\uf900-\ufdcf\ufdf0-\ufffd]|[\ud800-\udb7f][\udc00-\udfff])*)?:(?=[#\s<])/y;
-    this._prefixed = /((?:[A-Za-z\xc0-\xd6\xd8-\xf6\xf8-\u02ff\u0370-\u037d\u037f-\u1fff\u200c\u200d\u2070-\u218f\u2c00-\u2fef\u3001-\ud7ff\uf900-\ufdcf\ufdf0-\ufffd]|[\ud800-\udb7f][\udc00-\udfff])(?:\.?[\-0-9A-Z_a-z\xb7\xc0-\xd6\xd8-\xf6\xf8-\u037d\u037f-\u1fff\u200c\u200d\u203f\u2040\u2070-\u218f\u2c00-\u2fef\u3001-\ud7ff\uf900-\ufdcf\ufdf0-\ufffd]|[\ud800-\udb7f][\udc00-\udfff])*)?:((?:(?:[0-9:A-Z_a-z\xc0-\xd6\xd8-\xf6\xf8-\u02ff\u0370-\u037d\u037f-\u1fff\u200c\u200d\u2070-\u218f\u2c00-\u2fef\u3001-\ud7ff\uf900-\ufdcf\ufdf0-\ufffd]|[\ud800-\udb7f][\udc00-\udfff]|%[0-9a-fA-F]{2}|\\[!#-\/;=?\-@_~])(?:(?:[\.\-0-9:A-Z_a-z\xb7\xc0-\xd6\xd8-\xf6\xf8-\u037d\u037f-\u1fff\u200c\u200d\u203f\u2040\u2070-\u218f\u2c00-\u2fef\u3001-\ud7ff\uf900-\ufdcf\ufdf0-\ufffd]|[\ud800-\udb7f][\udc00-\udfff]|%[0-9a-fA-F]{2}|\\[!#-\/;=?\-@_~])*(?:[\-0-9:A-Z_a-z\xb7\xc0-\xd6\xd8-\xf6\xf8-\u037d\u037f-\u1fff\u200c\u200d\u203f\u2040\u2070-\u218f\u2c00-\u2fef\u3001-\ud7ff\uf900-\ufdcf\ufdf0-\ufffd]|[\ud800-\udb7f][\udc00-\udfff]|%[0-9a-fA-F]{2}|\\[!#-\/;=?\-@_~]))?)?)(?:[ \t]+|(?=\.?[,;!\^\s#()\[\]\{\}"'<>]))/y;
     this._variable = /\?(?:(?:[A-Z_a-z\xc0-\xd6\xd8-\xf6\xf8-\u02ff\u0370-\u037d\u037f-\u1fff\u200c\u200d\u2070-\u218f\u2c00-\u2fef\u3001-\ud7ff\uf900-\ufdcf\ufdf0-\ufffd]|[\ud800-\udb7f][\udc00-\udfff])(?:[\-0-9:A-Z_a-z\xb7\xc0-\xd6\xd8-\xf6\xf8-\u037d\u037f-\u1fff\u200c\u200d\u203f\u2040\u2070-\u218f\u2c00-\u2fef\u3001-\ud7ff\uf900-\ufdcf\ufdf0-\ufffd]|[\ud800-\udb7f][\udc00-\udfff])*)(?=[.,;!\^\s#()\[\]\{\}"'<>])/y;
     this._blank = /_:((?:[0-9A-Z_a-z\xc0-\xd6\xd8-\xf6\xf8-\u02ff\u0370-\u037d\u037f-\u1fff\u200c\u200d\u2070-\u218f\u2c00-\u2fef\u3001-\ud7ff\uf900-\ufdcf\ufdf0-\ufffd]|[\ud800-\udb7f][\udc00-\udfff])(?:\.?[\-0-9A-Z_a-z\xb7\xc0-\xd6\xd8-\xf6\xf8-\u037d\u037f-\u1fff\u200c\u200d\u203f\u2040\u2070-\u218f\u2c00-\u2fef\u3001-\ud7ff\uf900-\ufdcf\ufdf0-\ufffd]|[\ud800-\udb7f][\udc00-\udfff])*)(?:[ \t]+|(?=\.?[,;:!\^\s#()\[\]\{\}"'<>]))/y;
     this._number = /[\-+]?(?:(\d+\.\d*|\.?\d+)[eE][\-+]?\d+|(?=\.?\d)\d*(?:(\.)\d+)?)(?=\.?[,;:!\^\s#()\[\]\{\}"'<>])/y;
     this._boolean = /(?:true|false)(?=[.,;!\^\s#()\[\]\{\}"'<>])/y;
-    this._atKeyword = /@[a-z]+(?=[\s#<:"'])/iy;
+    this._atKeyword = /@[a-z]+(?=[\s#<:"'.])/iy;
     this._keyword = /(?:PREFIX|BASE|VERSION|GRAPH)(?=[\s#<"'])/iy;
     this._n3Verb = /(?:has|is|of)(?=[\s#()\[\]\{\}"'<>?_+\-0-9])/y;
     this._n3Id = /id(?=[\s#<])/y;
@@ -116,7 +238,9 @@ export default class N3Lexer {
         if (!/^[a-z]+$/i.test(name) || reservedWords.test(name))
           throw new Error(`Invalid directive name: "${name}"`);
       }
-      this._directive = new RegExp(`(?:${options.directives.join('|')})(?=[\\s#<])`, 'iy');
+      // Like VERSION, directives are case-sensitive in N-Triples and N-Quads
+      const names = this._lineMode ? options.directives.map(name => name.toUpperCase()) : options.directives;
+      this._directive = new RegExp(`(?:${names.join('|')})(?=[\\s#<])`, this._lineMode ? 'y' : 'iy');
       this._directiveMaxLength = Math.max(...options.directives.map(name => name.length));
       // The first characters of directive names, so other words skip the regular expression
       this._directiveStarts = options.directives.map(name => name[0].toLowerCase() + name[0].toUpperCase()).join('');
@@ -501,9 +625,14 @@ export default class N3Lexer {
       // Some first characters do not allow an immediate decision, so inspect more
       if (inconclusive) {
         // Try to find a prefix
+        let end;
         if ((this._previousMarker === '@prefix' || this._previousMarker === 'PREFIX') &&
-            (match = execAt(this._prefix, input, pos)))
-          type = 'prefix', value = match[1] || '';
+            input.charCodeAt(end = skipPrefix(input, pos)) === COLON &&
+            end + 1 < input.length && (input[end + 1] === '#' || input[end + 1] === '<' ||
+                                       isWhitespace(input.charCodeAt(end + 1)))) {
+          type = 'prefix', value = input.slice(pos, end);
+          matchLength = end + 1 - pos;
+        }
         // Try to find an additional directive keyword
         // (at the end of the input, only a short final word can be one)
         else if (this._directive !== null && this._directiveStarts.includes(firstChar) &&
@@ -512,13 +641,13 @@ export default class N3Lexer {
                  (match = execAtEnd(this._directive, input, pos))))
           type = match[0].toUpperCase();
         // Try to find a prefixed name. Since it can contain (but not end with) a dot,
-        // we always need a non-dot character before deciding it is a prefixed name.
-        // Therefore, try inserting a space if we're at the end of the input.
-        else if ((match = execAt(this._prefixed, input, pos)) ||
-                 inputFinished && (match = execAtEnd(this._prefixed, input, pos))) {
-          type = 'prefixed', prefix = match[1] || '';
-          value = this._unescape(match[2], localNameEscapeReplacements);
-          lexicalLength = prefix.length + match[2].length + 1;
+        // we always need a non-dot character before deciding it is a prefixed name,
+        // except at the end of the input.
+        else if (!this._lineMode && (end = skipPrefixedName(input, pos, inputFinished)) >= 0) {
+          const colon = input.indexOf(':', pos);
+          type = 'prefixed', prefix = input.slice(pos, colon);
+          value = this._unescape(input.slice(colon + 1, end), localNameEscapeReplacements);
+          matchLength = end - pos;
         }
       }
 
@@ -605,30 +734,21 @@ export default class N3Lexer {
       return null;
 
     // Most verb boundaries cannot be part of a prefix, so keep the common path fast.
+    // U+1680 and U+FEFF are whitespace to the regular expression but name characters.
     const next = input[pos + verb[0].length];
-    if (next !== '-' && next !== '_' && (next < '0' || next > '9'))
+    if (next !== '-' && next !== '_' && (next < '0' || next > '9') &&
+        next !== '\u1680' && next !== '\ufeff')
       return verb;
 
     // A prefix can start with a verb and continue with characters that are also
     // valid verb boundaries. Prefer the longer prefixed name when it is complete.
-    if (execAt(this._prefixed, input, pos))
-      return null;
-    // Appending to the input only matters when a prefixed name could run up to
-    // its end, which a character that cannot occur in prefixed names rules out.
-    // This avoids copying the rest of the document for every such verb.
-    if (testAt(nonPrefixedNameChar, input, pos))
-      return verb;
-    if (execAtEnd(this._prefixed, input, pos))
+    if (skipPrefixedName(input, pos, true) >= 0)
       return null;
 
-    // If a stream chunk ends partway through such a prefix, wait for the colon
-    // instead of prematurely emitting the verb. Appending ": " lets the prefix
-    // grammar determine whether all input seen so far can be a complete prefix.
-    if (!inputFinished) {
-      const prefix = execAt(this._prefix, `${input.slice(pos)}: `, 0);
-      if (prefix)
-        return null;
-    }
+    // If a stream chunk ends partway through such a prefix,
+    // wait for the colon instead of prematurely emitting the verb.
+    if (!inputFinished && skipPrefix(input, pos) === input.length)
+      return null;
     return verb;
   }
 
@@ -758,6 +878,27 @@ export default class N3Lexer {
     return err;
   }
 
+  // ### `_startTokenization` resets the lexer state for a new input
+  _startTokenization() {
+    this._line = 1;
+    this._linePosition = 0;
+    this._previousMarker = undefined;
+    this.previousToken = undefined;
+    this._literalClosingPos = 0;
+    this._input = undefined;
+    // Deferred tokenization and stream events can outlive their invocation.
+    // Ignore them once a later call takes ownership of the lexer state.
+    return this._tokenization = {};
+  }
+
+  // ### `_tokenizeString` synchronously emits the tokens of a complete string through the callback,
+  // so that the caller can consume each token without the lexer collecting them all first
+  _tokenizeString(input, callback) {
+    this._startTokenization();
+    this._input = this._readStartingBom(input);
+    this._tryTokenizeToEnd(callback, true);
+  }
+
   // ### Strips off any starting UTF BOM mark.
   _readStartingBom(input) {
     if (input.startsWith('\ufeff')) {
@@ -775,15 +916,7 @@ export default class N3Lexer {
   // Separator whitespace counts towards the next token's start, outside either range.
   // Multiline tokens also have endLine; their end column is relative to that line.
   tokenize(input, callback) {
-    // Deferred tokenization and stream events can outlive their invocation.
-    // Ignore them once a later call takes ownership of the lexer state.
-    const tokenization = this._tokenization = {};
-    this._line = 1;
-    this._linePosition = 0;
-    this._previousMarker = undefined;
-    this.previousToken = undefined;
-    this._literalClosingPos = 0;
-    this._input = undefined;
+    const tokenization = this._startTokenization();
 
     // If the input is a string, continuously emit tokens through the callback until the end
     if (typeof input === 'string') {

package/src/N3Parser.js

@@ -1683,6 +1683,10 @@ export default class N3Parser {
     // The read callback is the next function to be executed when a token arrives.
     // We start reading in the top context.
     this._readCallback = this._readBeforeTopContext;
+    // A parse that failed part-way can have left scopes and a statement open
+    this._contextStack = [];
+    this._graph = this._subject = this._predicate = this._object = null;
+    this._tripleTerm = this._reifier = null;
     this._sparqlStyle = false;
     this._prefixes = Object.create(null);
     this._prefixes._ = this._blankNodePrefix ? this._blankNodePrefix.substr(2)
@@ -1702,14 +1706,14 @@ export default class N3Parser {
     this._emptyFormula = false;
     this._tripleTermDepth = 0;
 
-    let readToken = token => {
+    const readGrammarToken = token => {
       return this._readCallback = this._readCallback(token);
     };
+    let readToken = readGrammarToken;
 
     // Comments bypass the grammar, but participate in the token lifecycle.
     if (onComment || this._lexer.comments) {
       this._lexer.comments = true;
-      const readGrammarToken = readToken;
       readToken = token => {
         if (token.type !== 'comment')
           return readGrammarToken(token);
@@ -1744,7 +1748,26 @@ export default class N3Parser {
       const quads = [];
       let error;
       this._callback = (e, t) => { e ? (error = e) : t && quads.push(t); };
-      this._lexer.tokenize(input).every(readToken);
+      const lexer = this._lexer;
+      // Without callbacks, nothing observes the parse before the whole document
+      // has lexed, so tokens can be parsed as they are lexed instead of being
+      // collected into an array first. A lexer that replaces the built-in
+      // tokenize keeps going through its own implementation.
+      if (!onPrefix && !onVersion && !onDirective && !onComment && !onToken && !onTokenEnd &&
+          lexer.tokenize === N3Lexer.prototype.tokenize && typeof lexer._tokenizeString === 'function') {
+        // A lexical error used to stop the parse before any token was read,
+        // so undo base declarations that were read before it
+        const base = [this._base, this._basePath, this._baseRoot, this._baseScheme];
+        lexer._tokenizeString(input, (e, token) => {
+          if (e) {
+            [this._base, this._basePath, this._baseRoot, this._baseScheme] = base;
+            this._callback(e), this._callback = noop;
+          }
+          else if (this._readCallback) readToken(token);
+        });
+      }
+      else
+        lexer.tokenize(input).every(readToken);
       if (error) throw error;
       return quads;
     }

package/src/N3Reasoner.js

@@ -34,6 +34,7 @@ export default class N3Reasoner {
       if (!store._addToIndex(graphItem.subjects,   subject,   predicate, object)) return;
       store._addToIndex(graphItem.predicates, predicate, object,    subject);
       store._addToIndex(graphItem.objects,    object,    subject,   predicate);
+      this._indexed++;
     }
     // Count genuinely new derivations and fail past the budget. The check comes
     // after all three indexes are updated, so a caught error leaves the store
@@ -175,7 +176,7 @@ export default class N3Reasoner {
   }
 
   reason(rules) {
-    this._derivations = 0;
+    this._derivations = this._indexed = 0;
     if (!Array.isArray(rules)) {
       rules = getRulesFromDataset(rules);
     }
@@ -242,9 +243,10 @@ export default class N3Reasoner {
       }
     }
     finally {
-      // Invalidate the cached size even if a derivation budget was exceeded,
-      // so a caught budget error leaves the store fully consistent.
-      this._store._size = null;
+      // Count the quads added directly to the indexes, even if a derivation
+      // budget was exceeded, so a caught budget error leaves the store consistent
+      if (this._store._size !== null)
+        this._store._size += this._indexed;
     }
   }
 }