n3 2.7.11 → 2.7.12

package/lib/N3Lexer.js

@@ -92,7 +92,7 @@ class N3Lexer {
     this._n3Verb = /^(?:has|is|of)(?=[\s#()\[\]\{\}"'<>?_+\-0-9])/;
     this._n3Id = /^id(?=[\s#<])/;
     this._shortPredicates = /^a(?=[\s#()\[\]\{\}"'<>])/;
-    this._commentLine = /^[ \t]*#([^\n\r]*)(?:\r\n|\n|\r)[ \t]*/;
+    this._commentLine = /^[ \t]*#([^\n\r]*)(?:\r\n|\n|\r)([ \t]*)/;
     this._whitespace = /^[ \t]+/;
     options = options || {};
 
@@ -123,7 +123,7 @@ class N3Lexer {
   _tokenizeToEnd(callback, inputFinished) {
     // Continue parsing as far as possible; the loop will return eventually
     let input = this._input;
-    let currentLineLength = input.length;
+    let currentLineLength = this._linePosition + input.length;
     while (true) {
       // Consume one separator line at a time, including its following indentation.
       while (true) {
@@ -137,28 +137,43 @@ class N3Lexer {
         if (charCode === HASH) {
           const comment = this._commentLine.exec(input);
           if (comment) {
-            if (this.comments) emitToken('comment', comment[1], '', this._line, comment[0].length);
-            input = input.slice(comment[0].length);
-            currentLineLength = input.length;
+            const commentLength = comment[0].length;
+            // Keep a trailing CR buffered in case the next chunk starts with LF.
+            if (!inputFinished && commentLength === input.length && input.charCodeAt(commentLength - 1) === CR) {
+              this._linePosition = currentLineLength - input.length;
+              return this._input = input;
+            }
+            if (this.comments) emitComment(comment[1], this._line, separatorLength);
+            input = input.slice(commentLength);
+            currentLineLength = input.length + comment[2].length;
             this._line++;
           } else {
             // A comment without a line ending stays buffered until EOF.
             input = input.slice(separatorLength);
-            if (!inputFinished) return this._input = input;
-            if (this.comments) emitToken('comment', input.slice(1), '', this._line, input.length);
+            if (!inputFinished) {
+              this._linePosition = currentLineLength - input.length;
+              return this._input = input;
+            }
+            if (this.comments) emitComment(input.slice(1), this._line, 0);
             input = '';
             break;
           }
         } else if (charCode === LF || charCode === CR) {
+          // A CR at the end of a chunk may still be followed by LF.
+          if (!inputFinished && charCode === CR && separatorLength + 1 === input.length) {
+            this._linePosition = currentLineLength - input.length;
+            return this._input = input;
+          }
           separatorLength += charCode === CR && input.charCodeAt(separatorLength + 1) === LF ? 2 : 1;
-          // Indentation is part of the same separator match as the newline.
+          // Indentation is consumed with the newline, but belongs to the next line's columns.
+          let indentationLength = 0;
           const next = input.charCodeAt(separatorLength);
           if (next === SPACE || next === TAB) {
             const following = input.charCodeAt(separatorLength + 1);
-            separatorLength += following === SPACE || following === TAB ? this._whitespace.exec(input.slice(separatorLength))[0].length : 1;
+            indentationLength = following === SPACE || following === TAB ? this._whitespace.exec(input.slice(separatorLength))[0].length : 1;
           }
-          input = input.slice(separatorLength);
-          currentLineLength = input.length;
+          input = input.slice(separatorLength + indentationLength);
+          currentLineLength = input.length + indentationLength;
           this._line++;
         } else {
           if (separatorLength !== 0) input = input.slice(separatorLength);
@@ -170,6 +185,7 @@ class N3Lexer {
           input = null;
           emitToken('eof', '', '', this._line, 0);
         }
+        this._linePosition = currentLineLength;
         return this._input = input;
       }
 
@@ -181,6 +197,8 @@ class N3Lexer {
         prefix = '',
         match = null,
         matchLength = 0,
+        lexicalLength = 0,
+        finalLineLength = 0,
         inconclusive = false;
       switch (firstChar) {
         case '^':
@@ -207,12 +225,16 @@ class N3Lexer {
         // Fall through in case the type is an IRI
         case '<':
           // Try to find a full IRI without escape sequences
-          if (match = this._unescapedIri.exec(input)) type = 'IRI', value = match[1];
+          if (match = this._unescapedIri.exec(input)) {
+            type = 'IRI', value = match[1];
+            lexicalLength = match[1].length + 2;
+          }
           // Try to find a full IRI with escape sequences
           else if (match = this._iri.exec(input)) {
             value = this._unescape(match[1], stringEscapeReplacements);
             if (value === null || illegalIriChars.test(value)) return reportSyntaxError(this);
             type = 'IRI';
+            lexicalLength = match[1].length + 2;
           }
           // Try to find a triple term
           else if (input.length > 2 && input[1] === '<' && input[2] === '(') type = '<<(', matchLength = 3;
@@ -234,7 +256,10 @@ class N3Lexer {
           // Try to find a blank node. Since it can contain (but not end with) a dot,
           // we always need a non-dot character before deciding it is a blank node.
           // Therefore, try inserting a space if we're at the end of the input.
-          if ((match = this._blank.exec(input)) || inputFinished && (match = this._blank.exec(`${input} `))) type = 'blank', prefix = '_', value = match[1];
+          if ((match = this._blank.exec(input)) || inputFinished && (match = this._blank.exec(`${input} `))) {
+            type = 'blank', prefix = '_', value = match[1];
+            lexicalLength = match[1].length + 2;
+          }
           break;
         case '"':
           // Try to find a literal without escape sequences
@@ -243,7 +268,8 @@ class N3Lexer {
           else {
             ({
               value,
-              matchLength
+              matchLength,
+              finalLineLength
             } = this._parseLiteral(input));
             if (value === null) return reportSyntaxError(this);
           }
@@ -260,7 +286,8 @@ class N3Lexer {
             else {
               ({
                 value,
-                matchLength
+                matchLength,
+                finalLineLength
               } = this._parseLiteral(input));
               if (value === null) return reportSyntaxError(this);
             }
@@ -406,7 +433,11 @@ class N3Lexer {
         // Try to find a prefixed name. Since it can contain (but not end with) a dot,
         // we always need a non-dot character before deciding it is a prefixed name.
         // Therefore, try inserting a space if we're at the end of the input.
-        else if ((match = this._prefixed.exec(input)) || inputFinished && (match = this._prefixed.exec(`${input} `))) type = 'prefixed', prefix = match[1] || '', value = this._unescape(match[2], localNameEscapeReplacements);
+        else if ((match = this._prefixed.exec(input)) || inputFinished && (match = this._prefixed.exec(`${input} `))) {
+          type = 'prefixed', prefix = match[1] || '';
+          value = this._unescape(match[2], localNameEscapeReplacements);
+          lexicalLength = prefix.length + match[2].length + 1;
+        }
       }
 
       // A type token is special: it can only be emitted after an IRI or prefixed name is read
@@ -428,19 +459,49 @@ class N3Lexer {
         // We could be in streaming mode, and then we just wait for more input to arrive.
         // Otherwise, a syntax error has occurred in the input.
         // One exception: error on an unaccounted linebreak (= not inside a triple-quoted literal).
-        if (inputFinished || !/^'''|^"""/.test(input) && /\n|\r/.test(input)) return reportSyntaxError(this);else return this._input = input;
+        if (inputFinished || !/^'''|^"""/.test(input) && /\n|\r/.test(input)) return reportSyntaxError(this);else {
+          this._linePosition = currentLineLength - input.length;
+          return this._input = input;
+        }
       }
 
       // Emit the parsed token
+      // Consumption includes separator whitespace; lexicalLength excludes it
+      // and any synthetic EOF space. slice below clamps consumption to the input.
       const length = matchLength || match[0].length;
-      const token = emitToken(type, value, prefix, line, length);
+      let token;
+      if (finalLineLength) {
+        token = {
+          type,
+          value,
+          prefix,
+          line,
+          start: currentLineLength - input.length,
+          end: finalLineLength,
+          endLine: this._line
+        };
+        callback(null, token);
+      } else token = emitToken(type, value, prefix, line, lexicalLength || length);
       this.previousToken = token;
       this._previousMarker = type;
 
       // Advance to next part to tokenize
       input = input.slice(length);
+      if (finalLineLength) currentLineLength = input.length + finalLineLength;
     }
 
+    // Emits a comment at its exact position within matched whitespace.
+    function emitComment(value, line, offset) {
+      const start = currentLineLength - input.length + offset;
+      callback(null, {
+        type: 'comment',
+        value,
+        prefix: '',
+        line,
+        start,
+        end: start + value.length + 1
+      });
+    }
     // Emits the token through the callback
     function emitToken(type, value, prefix, line, length) {
       const start = input ? currentLineLength - input.length : currentLineLength;
@@ -539,15 +600,18 @@ class N3Lexer {
         // means these are actual, non-escaped closing quotes
         if (backslashCount % 2 === 0) {
           // Extract and unescape the value
-          const raw = input.substring(openingLength, closingPos);
-          const lines = raw.split(/\r\n|\r|\n/).length - 1;
+          const raw = input.substring(openingLength, closingPos),
+            lines = raw.split(/\r\n|\r|\n/),
+            lineCount = lines.length - 1;
           const matchLength = closingPos + openingLength;
           // Only triple-quoted strings can be multi-line
-          if (openingLength === 1 && lines !== 0 || openingLength === 3 && this._lineMode) break;
-          this._line += lines;
+          if (openingLength === 1 && lineCount !== 0 || openingLength === 3 && this._lineMode) break;
+          this._line += lineCount;
+          const finalLineLength = lineCount === 0 ? 0 : lines[lines.length - 1].length + openingLength;
           return {
             value: this._unescape(raw, stringEscapeReplacements),
-            matchLength
+            matchLength,
+            finalLineLength
           };
         }
         closingPos++;
@@ -556,7 +620,8 @@ class N3Lexer {
     }
     return {
       value: '',
-      matchLength: 0
+      matchLength: 0,
+      finalLineLength: 0
     };
   }
 
@@ -574,18 +639,30 @@ class N3Lexer {
 
   // ### Strips off any starting UTF BOM mark.
   _readStartingBom(input) {
-    return input.startsWith('\ufeff') ? input.slice(1) : input;
+    if (input.startsWith('\ufeff')) {
+      this._linePosition = 1;
+      return input.slice(1);
+    }
+    return input;
   }
 
   // ## Public methods
 
   // ### `tokenize` starts the transformation of an N3 document into an array of tokens.
   // The input can be a string or a stream.
+  // Token ranges use one-based lines and zero-based, end-exclusive UTF-16 columns.
+  // Separator whitespace counts towards the next token's start, outside either range.
+  // Multiline tokens also have endLine; their end column is relative to that line.
   tokenize(input, callback) {
     // Deferred tokenization and stream events can outlive their invocation.
     // Ignore them once a later call takes ownership of the lexer state.
     const tokenization = this._tokenization = {};
     this._line = 1;
+    this._linePosition = 0;
+    this._previousMarker = undefined;
+    this.previousToken = undefined;
+    this._literalClosingPos = 0;
+    this._input = undefined;
 
     // If the input is a string, continuously emit tokens through the callback until the end
     if (typeof input === 'string') {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "n3",
-  "version": "2.7.11",
+  "version": "2.7.12",
   "description": "Lightning fast, asynchronous, streaming Turtle / N3 / RDF library.",
   "author": "Ruben Verborgh <ruben.verborgh@gmail.com>",
   "keywords": [

package/src/N3Lexer.js

@@ -57,7 +57,7 @@ export default class N3Lexer {
     this._n3Verb = /^(?:has|is|of)(?=[\s#()\[\]\{\}"'<>?_+\-0-9])/;
     this._n3Id = /^id(?=[\s#<])/;
     this._shortPredicates = /^a(?=[\s#()\[\]\{\}"'<>])/;
-    this._commentLine = /^[ \t]*#([^\n\r]*)(?:\r\n|\n|\r)[ \t]*/;
+    this._commentLine = /^[ \t]*#([^\n\r]*)(?:\r\n|\n|\r)([ \t]*)/;
     this._whitespace = /^[ \t]+/;
     options = options || {};
 
@@ -89,7 +89,7 @@ export default class N3Lexer {
   _tokenizeToEnd(callback, inputFinished) {
     // Continue parsing as far as possible; the loop will return eventually
     let input = this._input;
-    let currentLineLength = input.length;
+    let currentLineLength = this._linePosition + input.length;
     while (true) {
       // Consume one separator line at a time, including its following indentation.
       while (true) {
@@ -103,34 +103,49 @@ export default class N3Lexer {
         if (charCode === HASH) {
           const comment = this._commentLine.exec(input);
           if (comment) {
+            const commentLength = comment[0].length;
+            // Keep a trailing CR buffered in case the next chunk starts with LF.
+            if (!inputFinished && commentLength === input.length &&
+                input.charCodeAt(commentLength - 1) === CR) {
+              this._linePosition = currentLineLength - input.length;
+              return this._input = input;
+            }
             if (this.comments)
-              emitToken('comment', comment[1], '', this._line, comment[0].length);
-            input = input.slice(comment[0].length);
-            currentLineLength = input.length;
+              emitComment(comment[1], this._line, separatorLength);
+            input = input.slice(commentLength);
+            currentLineLength = input.length + comment[2].length;
             this._line++;
           }
           else {
             // A comment without a line ending stays buffered until EOF.
             input = input.slice(separatorLength);
-            if (!inputFinished)
+            if (!inputFinished) {
+              this._linePosition = currentLineLength - input.length;
               return this._input = input;
+            }
             if (this.comments)
-              emitToken('comment', input.slice(1), '', this._line, input.length);
+              emitComment(input.slice(1), this._line, 0);
             input = '';
             break;
           }
         }
         else if (charCode === LF || charCode === CR) {
+          // A CR at the end of a chunk may still be followed by LF.
+          if (!inputFinished && charCode === CR && separatorLength + 1 === input.length) {
+            this._linePosition = currentLineLength - input.length;
+            return this._input = input;
+          }
           separatorLength += charCode === CR && input.charCodeAt(separatorLength + 1) === LF ? 2 : 1;
-          // Indentation is part of the same separator match as the newline.
+          // Indentation is consumed with the newline, but belongs to the next line's columns.
+          let indentationLength = 0;
           const next = input.charCodeAt(separatorLength);
           if (next === SPACE || next === TAB) {
             const following = input.charCodeAt(separatorLength + 1);
-            separatorLength += following === SPACE || following === TAB ?
+            indentationLength = following === SPACE || following === TAB ?
               this._whitespace.exec(input.slice(separatorLength))[0].length : 1;
           }
-          input = input.slice(separatorLength);
-          currentLineLength = input.length;
+          input = input.slice(separatorLength + indentationLength);
+          currentLineLength = input.length + indentationLength;
           this._line++;
         }
         else {
@@ -144,13 +159,15 @@ export default class N3Lexer {
           input = null;
           emitToken('eof', '', '', this._line, 0);
         }
+        this._linePosition = currentLineLength;
         return this._input = input;
       }
 
       // Look for specific token types based on the first character
       const line = this._line, firstChar = input[0];
       let type = '', value = '', prefix = '',
-          match = null, matchLength = 0, inconclusive = false;
+          match = null, matchLength = 0, lexicalLength = 0,
+          finalLineLength = 0, inconclusive = false;
       switch (firstChar) {
       case '^':
         // We need at least 3 tokens lookahead to distinguish ^^<IRI> and ^^pre:fixed
@@ -177,14 +194,17 @@ export default class N3Lexer {
         // Fall through in case the type is an IRI
       case '<':
         // Try to find a full IRI without escape sequences
-        if (match = this._unescapedIri.exec(input))
+        if (match = this._unescapedIri.exec(input)) {
           type = 'IRI', value = match[1];
+          lexicalLength = match[1].length + 2;
+        }
         // Try to find a full IRI with escape sequences
         else if (match = this._iri.exec(input)) {
           value = this._unescape(match[1], stringEscapeReplacements);
           if (value === null || illegalIriChars.test(value))
             return reportSyntaxError(this);
           type = 'IRI';
+          lexicalLength = match[1].length + 2;
         }
         // Try to find a triple term
         else if (input.length > 2 && input[1] === '<' && input[2] === '(')
@@ -214,8 +234,10 @@ export default class N3Lexer {
         // we always need a non-dot character before deciding it is a blank node.
         // Therefore, try inserting a space if we're at the end of the input.
         if ((match = this._blank.exec(input)) ||
-            inputFinished && (match = this._blank.exec(`${input} `)))
+            inputFinished && (match = this._blank.exec(`${input} `))) {
           type = 'blank', prefix = '_', value = match[1];
+          lexicalLength = match[1].length + 2;
+        }
         break;
 
       case '"':
@@ -224,7 +246,7 @@ export default class N3Lexer {
           value = match[1];
         // Try to find a literal wrapped in three pairs of quotes
         else {
-          ({ value, matchLength } = this._parseLiteral(input));
+          ({ value, matchLength, finalLineLength } = this._parseLiteral(input));
           if (value === null)
             return reportSyntaxError(this);
         }
@@ -241,7 +263,7 @@ export default class N3Lexer {
             value = match[1];
           // Try to find a literal wrapped in three pairs of quotes
           else {
-            ({ value, matchLength } = this._parseLiteral(input));
+            ({ value, matchLength, finalLineLength } = this._parseLiteral(input));
             if (value === null)
               return reportSyntaxError(this);
           }
@@ -436,8 +458,11 @@ export default class N3Lexer {
         // we always need a non-dot character before deciding it is a prefixed name.
         // Therefore, try inserting a space if we're at the end of the input.
         else if ((match = this._prefixed.exec(input)) ||
-                 inputFinished && (match = this._prefixed.exec(`${input} `)))
-          type = 'prefixed', prefix = match[1] || '', value = this._unescape(match[2], localNameEscapeReplacements);
+                 inputFinished && (match = this._prefixed.exec(`${input} `))) {
+          type = 'prefixed', prefix = match[1] || '';
+          value = this._unescape(match[2], localNameEscapeReplacements);
+          lexicalLength = prefix.length + match[2].length + 1;
+        }
       }
 
       // A type token is special: it can only be emitted after an IRI or prefixed name is read
@@ -456,20 +481,44 @@ export default class N3Lexer {
         // One exception: error on an unaccounted linebreak (= not inside a triple-quoted literal).
         if (inputFinished || (!/^'''|^"""/.test(input) && /\n|\r/.test(input)))
           return reportSyntaxError(this);
-        else
+        else {
+          this._linePosition = currentLineLength - input.length;
           return this._input = input;
+        }
       }
 
       // Emit the parsed token
+      // Consumption includes separator whitespace; lexicalLength excludes it
+      // and any synthetic EOF space. slice below clamps consumption to the input.
       const length = matchLength || match[0].length;
-      const token = emitToken(type, value, prefix, line, length);
+      let token;
+      if (finalLineLength) {
+        token = {
+          type, value, prefix, line,
+          start: currentLineLength - input.length,
+          end: finalLineLength, endLine: this._line,
+        };
+        callback(null, token);
+      }
+      else
+        token = emitToken(type, value, prefix, line, lexicalLength || length);
       this.previousToken = token;
       this._previousMarker = type;
 
       // Advance to next part to tokenize
       input = input.slice(length);
+      if (finalLineLength)
+        currentLineLength = input.length + finalLineLength;
     }
 
+    // Emits a comment at its exact position within matched whitespace.
+    function emitComment(value, line, offset) {
+      const start = currentLineLength - input.length + offset;
+      callback(null, {
+        type: 'comment', value, prefix: '', line,
+        start, end: start + value.length + 1,
+      });
+    }
     // Emits the token through the callback
     function emitToken(type, value, prefix, line, length) {
       const start = input ? currentLineLength - input.length : currentLineLength;
@@ -568,21 +617,23 @@ export default class N3Lexer {
         // means these are actual, non-escaped closing quotes
         if (backslashCount % 2 === 0) {
           // Extract and unescape the value
-          const raw = input.substring(openingLength, closingPos);
-          const lines = raw.split(/\r\n|\r|\n/).length - 1;
+          const raw = input.substring(openingLength, closingPos),
+              lines = raw.split(/\r\n|\r|\n/),
+              lineCount = lines.length - 1;
           const matchLength = closingPos + openingLength;
           // Only triple-quoted strings can be multi-line
-          if (openingLength === 1 && lines !== 0 ||
+          if (openingLength === 1 && lineCount !== 0 ||
               openingLength === 3 && this._lineMode)
             break;
-          this._line += lines;
-          return { value: this._unescape(raw, stringEscapeReplacements), matchLength };
+          this._line += lineCount;
+          const finalLineLength = lineCount === 0 ? 0 : lines[lines.length - 1].length + openingLength;
+          return { value: this._unescape(raw, stringEscapeReplacements), matchLength, finalLineLength };
         }
         closingPos++;
       }
       this._literalClosingPos = input.length - openingLength + 1;
     }
-    return { value: '', matchLength: 0 };
+    return { value: '', matchLength: 0, finalLineLength: 0 };
   }
 
   // ### `_syntaxError` creates a syntax error for the given issue
@@ -599,18 +650,30 @@ export default class N3Lexer {
 
   // ### Strips off any starting UTF BOM mark.
   _readStartingBom(input) {
-    return input.startsWith('\ufeff') ? input.slice(1) : input;
+    if (input.startsWith('\ufeff')) {
+      this._linePosition = 1;
+      return input.slice(1);
+    }
+    return input;
   }
 
   // ## Public methods
 
   // ### `tokenize` starts the transformation of an N3 document into an array of tokens.
   // The input can be a string or a stream.
+  // Token ranges use one-based lines and zero-based, end-exclusive UTF-16 columns.
+  // Separator whitespace counts towards the next token's start, outside either range.
+  // Multiline tokens also have endLine; their end column is relative to that line.
   tokenize(input, callback) {
     // Deferred tokenization and stream events can outlive their invocation.
     // Ignore them once a later call takes ownership of the lexer state.
     const tokenization = this._tokenization = {};
     this._line = 1;
+    this._linePosition = 0;
+    this._previousMarker = undefined;
+    this.previousToken = undefined;
+    this._literalClosingPos = 0;
+    this._input = undefined;
 
     // If the input is a string, continuously emit tokens through the callback until the end
     if (typeof input === 'string') {