n3 2.13.1 → 2.13.3

package/lib/N3Lexer.js

@@ -136,7 +136,7 @@ class N3Lexer {
         if (!(key in lineModeRegExps) && this[key] instanceof RegExp) this[key] = invalidRegExp;
       }
       // The only keyword in N-Triples and N-Quads is VERSION, which is case-sensitive
-      this._keyword = /VERSION(?=[\s#<])/y;
+      this._keyword = /VERSION(?=[\s#<"])/y;
     }
     // When not in line mode, enable N3 functionality by default
     else {
@@ -737,6 +737,27 @@ class N3Lexer {
     return err;
   }
 
+  // ### `_startTokenization` resets the lexer state for a new input
+  _startTokenization() {
+    this._line = 1;
+    this._linePosition = 0;
+    this._previousMarker = undefined;
+    this.previousToken = undefined;
+    this._literalClosingPos = 0;
+    this._input = undefined;
+    // Deferred tokenization and stream events can outlive their invocation.
+    // Ignore them once a later call takes ownership of the lexer state.
+    return this._tokenization = {};
+  }
+
+  // ### `_tokenizeString` synchronously emits the tokens of a complete string through the callback,
+  // so that the caller can consume each token without the lexer collecting them all first
+  _tokenizeString(input, callback) {
+    this._startTokenization();
+    this._input = this._readStartingBom(input);
+    this._tryTokenizeToEnd(callback, true);
+  }
+
   // ### Strips off any starting UTF BOM mark.
   _readStartingBom(input) {
     if (input.startsWith('\ufeff')) {
@@ -754,15 +775,7 @@ class N3Lexer {
   // Separator whitespace counts towards the next token's start, outside either range.
   // Multiline tokens also have endLine; their end column is relative to that line.
   tokenize(input, callback) {
-    // Deferred tokenization and stream events can outlive their invocation.
-    // Ignore them once a later call takes ownership of the lexer state.
-    const tokenization = this._tokenization = {};
-    this._line = 1;
-    this._linePosition = 0;
-    this._previousMarker = undefined;
-    this.previousToken = undefined;
-    this._literalClosingPos = 0;
-    this._input = undefined;
+    const tokenization = this._startTokenization();
 
     // If the input is a string, continuously emit tokens through the callback until the end
     if (typeof input === 'string') {

package/lib/N3Parser.js

@@ -1554,6 +1554,10 @@ class N3Parser {
     // The read callback is the next function to be executed when a token arrives.
     // We start reading in the top context.
     this._readCallback = this._readBeforeTopContext;
+    // A parse that failed part-way can have left scopes and a statement open
+    this._contextStack = [];
+    this._graph = this._subject = this._predicate = this._object = null;
+    this._tripleTerm = this._reifier = null;
     this._sparqlStyle = false;
     this._prefixes = Object.create(null);
     this._prefixes._ = this._blankNodePrefix ? this._blankNodePrefix.substr(2) : `b${blankNodePrefix++}_`;
@@ -1570,14 +1574,14 @@ class N3Parser {
     this._quantifiedChanges = null;
     this._emptyFormula = false;
     this._tripleTermDepth = 0;
-    let readToken = token => {
+    const readGrammarToken = token => {
       return this._readCallback = this._readCallback(token);
     };
+    let readToken = readGrammarToken;
 
     // Comments bypass the grammar, but participate in the token lifecycle.
     if (onComment || this._lexer.comments) {
       this._lexer.comments = true;
-      const readGrammarToken = readToken;
       readToken = token => {
         if (token.type !== 'comment') return readGrammarToken(token);
         if (onComment) onComment(token.value);
@@ -1611,7 +1615,22 @@ class N3Parser {
       this._callback = (e, t) => {
         e ? error = e : t && quads.push(t);
       };
-      this._lexer.tokenize(input).every(readToken);
+      const lexer = this._lexer;
+      // Without callbacks, nothing observes the parse before the whole document
+      // has lexed, so tokens can be parsed as they are lexed instead of being
+      // collected into an array first. A lexer that replaces the built-in
+      // tokenize keeps going through its own implementation.
+      if (!onPrefix && !onVersion && !onDirective && !onComment && !onToken && !onTokenEnd && lexer.tokenize === _N3Lexer.default.prototype.tokenize && typeof lexer._tokenizeString === 'function') {
+        // A lexical error used to stop the parse before any token was read,
+        // so undo base declarations that were read before it
+        const base = [this._base, this._basePath, this._baseRoot, this._baseScheme];
+        lexer._tokenizeString(input, (e, token) => {
+          if (e) {
+            [this._base, this._basePath, this._baseRoot, this._baseScheme] = base;
+            this._callback(e), this._callback = noop;
+          } else if (this._readCallback) readToken(token);
+        });
+      } else lexer.tokenize(input).every(readToken);
       if (error) throw error;
       return quads;
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "n3",
-  "version": "2.13.1",
+  "version": "2.13.3",
   "description": "Lightning fast, asynchronous, streaming Turtle / N3 / RDF library.",
   "author": "Ruben Verborgh <ruben.verborgh@gmail.com>",
   "keywords": [

package/src/N3Lexer.js

@@ -102,7 +102,7 @@ export default class N3Lexer {
           this[key] = invalidRegExp;
       }
       // The only keyword in N-Triples and N-Quads is VERSION, which is case-sensitive
-      this._keyword = /VERSION(?=[\s#<])/y;
+      this._keyword = /VERSION(?=[\s#<"])/y;
     }
     // When not in line mode, enable N3 functionality by default
     else {
@@ -758,6 +758,27 @@ export default class N3Lexer {
     return err;
   }
 
+  // ### `_startTokenization` resets the lexer state for a new input
+  _startTokenization() {
+    this._line = 1;
+    this._linePosition = 0;
+    this._previousMarker = undefined;
+    this.previousToken = undefined;
+    this._literalClosingPos = 0;
+    this._input = undefined;
+    // Deferred tokenization and stream events can outlive their invocation.
+    // Ignore them once a later call takes ownership of the lexer state.
+    return this._tokenization = {};
+  }
+
+  // ### `_tokenizeString` synchronously emits the tokens of a complete string through the callback,
+  // so that the caller can consume each token without the lexer collecting them all first
+  _tokenizeString(input, callback) {
+    this._startTokenization();
+    this._input = this._readStartingBom(input);
+    this._tryTokenizeToEnd(callback, true);
+  }
+
   // ### Strips off any starting UTF BOM mark.
   _readStartingBom(input) {
     if (input.startsWith('\ufeff')) {
@@ -775,15 +796,7 @@ export default class N3Lexer {
   // Separator whitespace counts towards the next token's start, outside either range.
   // Multiline tokens also have endLine; their end column is relative to that line.
   tokenize(input, callback) {
-    // Deferred tokenization and stream events can outlive their invocation.
-    // Ignore them once a later call takes ownership of the lexer state.
-    const tokenization = this._tokenization = {};
-    this._line = 1;
-    this._linePosition = 0;
-    this._previousMarker = undefined;
-    this.previousToken = undefined;
-    this._literalClosingPos = 0;
-    this._input = undefined;
+    const tokenization = this._startTokenization();
 
     // If the input is a string, continuously emit tokens through the callback until the end
     if (typeof input === 'string') {

package/src/N3Parser.js

@@ -1682,6 +1682,10 @@ export default class N3Parser {
     // The read callback is the next function to be executed when a token arrives.
     // We start reading in the top context.
     this._readCallback = this._readBeforeTopContext;
+    // A parse that failed part-way can have left scopes and a statement open
+    this._contextStack = [];
+    this._graph = this._subject = this._predicate = this._object = null;
+    this._tripleTerm = this._reifier = null;
     this._sparqlStyle = false;
     this._prefixes = Object.create(null);
     this._prefixes._ = this._blankNodePrefix ? this._blankNodePrefix.substr(2)
@@ -1701,14 +1705,14 @@ export default class N3Parser {
     this._emptyFormula = false;
     this._tripleTermDepth = 0;
 
-    let readToken = token => {
+    const readGrammarToken = token => {
       return this._readCallback = this._readCallback(token);
     };
+    let readToken = readGrammarToken;
 
     // Comments bypass the grammar, but participate in the token lifecycle.
     if (onComment || this._lexer.comments) {
       this._lexer.comments = true;
-      const readGrammarToken = readToken;
       readToken = token => {
         if (token.type !== 'comment')
           return readGrammarToken(token);
@@ -1743,7 +1747,26 @@ export default class N3Parser {
       const quads = [];
       let error;
       this._callback = (e, t) => { e ? (error = e) : t && quads.push(t); };
-      this._lexer.tokenize(input).every(readToken);
+      const lexer = this._lexer;
+      // Without callbacks, nothing observes the parse before the whole document
+      // has lexed, so tokens can be parsed as they are lexed instead of being
+      // collected into an array first. A lexer that replaces the built-in
+      // tokenize keeps going through its own implementation.
+      if (!onPrefix && !onVersion && !onDirective && !onComment && !onToken && !onTokenEnd &&
+          lexer.tokenize === N3Lexer.prototype.tokenize && typeof lexer._tokenizeString === 'function') {
+        // A lexical error used to stop the parse before any token was read,
+        // so undo base declarations that were read before it
+        const base = [this._base, this._basePath, this._baseRoot, this._baseScheme];
+        lexer._tokenizeString(input, (e, token) => {
+          if (e) {
+            [this._base, this._basePath, this._baseRoot, this._baseScheme] = base;
+            this._callback(e), this._callback = noop;
+          }
+          else if (this._readCallback) readToken(token);
+        });
+      }
+      else
+        lexer.tokenize(input).every(readToken);
       if (error) throw error;
       return quads;
     }