n3 2.7.11 → 2.8.0

package/lib/N3Lexer.js

@@ -92,7 +92,7 @@ class N3Lexer {
     this._n3Verb = /^(?:has|is|of)(?=[\s#()\[\]\{\}"'<>?_+\-0-9])/;
     this._n3Id = /^id(?=[\s#<])/;
     this._shortPredicates = /^a(?=[\s#()\[\]\{\}"'<>])/;
-    this._commentLine = /^[ \t]*#([^\n\r]*)(?:\r\n|\n|\r)[ \t]*/;
+    this._commentLine = /^[ \t]*#([^\n\r]*)(?:\r\n|\n|\r)([ \t]*)/;
     this._whitespace = /^[ \t]+/;
     options = options || {};
 
@@ -123,7 +123,7 @@ class N3Lexer {
   _tokenizeToEnd(callback, inputFinished) {
     // Continue parsing as far as possible; the loop will return eventually
     let input = this._input;
-    let currentLineLength = input.length;
+    let currentLineLength = this._linePosition + input.length;
     while (true) {
       // Consume one separator line at a time, including its following indentation.
       while (true) {
@@ -137,28 +137,43 @@ class N3Lexer {
         if (charCode === HASH) {
           const comment = this._commentLine.exec(input);
           if (comment) {
-            if (this.comments) emitToken('comment', comment[1], '', this._line, comment[0].length);
-            input = input.slice(comment[0].length);
-            currentLineLength = input.length;
+            const commentLength = comment[0].length;
+            // Keep a trailing CR buffered in case the next chunk starts with LF.
+            if (!inputFinished && commentLength === input.length && input.charCodeAt(commentLength - 1) === CR) {
+              this._linePosition = currentLineLength - input.length;
+              return this._input = input;
+            }
+            if (this.comments) emitComment(comment[1], this._line, separatorLength);
+            input = input.slice(commentLength);
+            currentLineLength = input.length + comment[2].length;
             this._line++;
           } else {
             // A comment without a line ending stays buffered until EOF.
             input = input.slice(separatorLength);
-            if (!inputFinished) return this._input = input;
-            if (this.comments) emitToken('comment', input.slice(1), '', this._line, input.length);
+            if (!inputFinished) {
+              this._linePosition = currentLineLength - input.length;
+              return this._input = input;
+            }
+            if (this.comments) emitComment(input.slice(1), this._line, 0);
             input = '';
             break;
           }
         } else if (charCode === LF || charCode === CR) {
+          // A CR at the end of a chunk may still be followed by LF.
+          if (!inputFinished && charCode === CR && separatorLength + 1 === input.length) {
+            this._linePosition = currentLineLength - input.length;
+            return this._input = input;
+          }
           separatorLength += charCode === CR && input.charCodeAt(separatorLength + 1) === LF ? 2 : 1;
-          // Indentation is part of the same separator match as the newline.
+          // Indentation is consumed with the newline, but belongs to the next line's columns.
+          let indentationLength = 0;
           const next = input.charCodeAt(separatorLength);
           if (next === SPACE || next === TAB) {
             const following = input.charCodeAt(separatorLength + 1);
-            separatorLength += following === SPACE || following === TAB ? this._whitespace.exec(input.slice(separatorLength))[0].length : 1;
+            indentationLength = following === SPACE || following === TAB ? this._whitespace.exec(input.slice(separatorLength))[0].length : 1;
           }
-          input = input.slice(separatorLength);
-          currentLineLength = input.length;
+          input = input.slice(separatorLength + indentationLength);
+          currentLineLength = input.length + indentationLength;
           this._line++;
         } else {
           if (separatorLength !== 0) input = input.slice(separatorLength);
@@ -170,6 +185,7 @@ class N3Lexer {
           input = null;
           emitToken('eof', '', '', this._line, 0);
         }
+        this._linePosition = currentLineLength;
         return this._input = input;
       }
 
@@ -181,6 +197,8 @@ class N3Lexer {
         prefix = '',
         match = null,
         matchLength = 0,
+        lexicalLength = 0,
+        finalLineLength = 0,
         inconclusive = false;
       switch (firstChar) {
         case '^':
@@ -207,12 +225,16 @@ class N3Lexer {
         // Fall through in case the type is an IRI
         case '<':
           // Try to find a full IRI without escape sequences
-          if (match = this._unescapedIri.exec(input)) type = 'IRI', value = match[1];
+          if (match = this._unescapedIri.exec(input)) {
+            type = 'IRI', value = match[1];
+            lexicalLength = match[1].length + 2;
+          }
           // Try to find a full IRI with escape sequences
           else if (match = this._iri.exec(input)) {
             value = this._unescape(match[1], stringEscapeReplacements);
             if (value === null || illegalIriChars.test(value)) return reportSyntaxError(this);
             type = 'IRI';
+            lexicalLength = match[1].length + 2;
           }
           // Try to find a triple term
           else if (input.length > 2 && input[1] === '<' && input[2] === '(') type = '<<(', matchLength = 3;
@@ -234,7 +256,10 @@ class N3Lexer {
           // Try to find a blank node. Since it can contain (but not end with) a dot,
           // we always need a non-dot character before deciding it is a blank node.
           // Therefore, try inserting a space if we're at the end of the input.
-          if ((match = this._blank.exec(input)) || inputFinished && (match = this._blank.exec(`${input} `))) type = 'blank', prefix = '_', value = match[1];
+          if ((match = this._blank.exec(input)) || inputFinished && (match = this._blank.exec(`${input} `))) {
+            type = 'blank', prefix = '_', value = match[1];
+            lexicalLength = match[1].length + 2;
+          }
           break;
         case '"':
           // Try to find a literal without escape sequences
@@ -243,7 +268,8 @@ class N3Lexer {
           else {
             ({
               value,
-              matchLength
+              matchLength,
+              finalLineLength
             } = this._parseLiteral(input));
             if (value === null) return reportSyntaxError(this);
           }
@@ -260,7 +286,8 @@ class N3Lexer {
             else {
               ({
                 value,
-                matchLength
+                matchLength,
+                finalLineLength
               } = this._parseLiteral(input));
               if (value === null) return reportSyntaxError(this);
             }
@@ -406,7 +433,11 @@ class N3Lexer {
         // Try to find a prefixed name. Since it can contain (but not end with) a dot,
         // we always need a non-dot character before deciding it is a prefixed name.
         // Therefore, try inserting a space if we're at the end of the input.
-        else if ((match = this._prefixed.exec(input)) || inputFinished && (match = this._prefixed.exec(`${input} `))) type = 'prefixed', prefix = match[1] || '', value = this._unescape(match[2], localNameEscapeReplacements);
+        else if ((match = this._prefixed.exec(input)) || inputFinished && (match = this._prefixed.exec(`${input} `))) {
+          type = 'prefixed', prefix = match[1] || '';
+          value = this._unescape(match[2], localNameEscapeReplacements);
+          lexicalLength = prefix.length + match[2].length + 1;
+        }
       }
 
       // A type token is special: it can only be emitted after an IRI or prefixed name is read
@@ -428,19 +459,49 @@ class N3Lexer {
         // We could be in streaming mode, and then we just wait for more input to arrive.
         // Otherwise, a syntax error has occurred in the input.
         // One exception: error on an unaccounted linebreak (= not inside a triple-quoted literal).
-        if (inputFinished || !/^'''|^"""/.test(input) && /\n|\r/.test(input)) return reportSyntaxError(this);else return this._input = input;
+        if (inputFinished || !/^'''|^"""/.test(input) && /\n|\r/.test(input)) return reportSyntaxError(this);else {
+          this._linePosition = currentLineLength - input.length;
+          return this._input = input;
+        }
       }
 
       // Emit the parsed token
+      // Consumption includes separator whitespace; lexicalLength excludes it
+      // and any synthetic EOF space. slice below clamps consumption to the input.
       const length = matchLength || match[0].length;
-      const token = emitToken(type, value, prefix, line, length);
+      let token;
+      if (finalLineLength) {
+        token = {
+          type,
+          value,
+          prefix,
+          line,
+          start: currentLineLength - input.length,
+          end: finalLineLength,
+          endLine: this._line
+        };
+        callback(null, token);
+      } else token = emitToken(type, value, prefix, line, lexicalLength || length);
       this.previousToken = token;
       this._previousMarker = type;
 
       // Advance to next part to tokenize
       input = input.slice(length);
+      if (finalLineLength) currentLineLength = input.length + finalLineLength;
     }
 
+    // Emits a comment at its exact position within matched whitespace.
+    function emitComment(value, line, offset) {
+      const start = currentLineLength - input.length + offset;
+      callback(null, {
+        type: 'comment',
+        value,
+        prefix: '',
+        line,
+        start,
+        end: start + value.length + 1
+      });
+    }
     // Emits the token through the callback
     function emitToken(type, value, prefix, line, length) {
       const start = input ? currentLineLength - input.length : currentLineLength;
@@ -539,15 +600,18 @@ class N3Lexer {
         // means these are actual, non-escaped closing quotes
         if (backslashCount % 2 === 0) {
           // Extract and unescape the value
-          const raw = input.substring(openingLength, closingPos);
-          const lines = raw.split(/\r\n|\r|\n/).length - 1;
+          const raw = input.substring(openingLength, closingPos),
+            lines = raw.split(/\r\n|\r|\n/),
+            lineCount = lines.length - 1;
           const matchLength = closingPos + openingLength;
           // Only triple-quoted strings can be multi-line
-          if (openingLength === 1 && lines !== 0 || openingLength === 3 && this._lineMode) break;
-          this._line += lines;
+          if (openingLength === 1 && lineCount !== 0 || openingLength === 3 && this._lineMode) break;
+          this._line += lineCount;
+          const finalLineLength = lineCount === 0 ? 0 : lines[lines.length - 1].length + openingLength;
           return {
             value: this._unescape(raw, stringEscapeReplacements),
-            matchLength
+            matchLength,
+            finalLineLength
           };
         }
         closingPos++;
@@ -556,7 +620,8 @@ class N3Lexer {
     }
     return {
       value: '',
-      matchLength: 0
+      matchLength: 0,
+      finalLineLength: 0
     };
   }
 
@@ -574,18 +639,30 @@ class N3Lexer {
 
   // ### Strips off any starting UTF BOM mark.
   _readStartingBom(input) {
-    return input.startsWith('\ufeff') ? input.slice(1) : input;
+    if (input.startsWith('\ufeff')) {
+      this._linePosition = 1;
+      return input.slice(1);
+    }
+    return input;
   }
 
   // ## Public methods
 
   // ### `tokenize` starts the transformation of an N3 document into an array of tokens.
   // The input can be a string or a stream.
+  // Token ranges use one-based lines and zero-based, end-exclusive UTF-16 columns.
+  // Separator whitespace counts towards the next token's start, outside either range.
+  // Multiline tokens also have endLine; their end column is relative to that line.
   tokenize(input, callback) {
     // Deferred tokenization and stream events can outlive their invocation.
     // Ignore them once a later call takes ownership of the lexer state.
     const tokenization = this._tokenization = {};
     this._line = 1;
+    this._linePosition = 0;
+    this._previousMarker = undefined;
+    this.previousToken = undefined;
+    this._literalClosingPos = 0;
+    this._input = undefined;
 
     // If the input is a string, continuously emit tokens through the callback until the end
     if (typeof input === 'string') {

package/lib/N3Reasoner.js

@@ -34,10 +34,17 @@ class N3Reasoner {
     this._maxPremiseDepth = options.maxPremiseDepth === undefined ? Infinity : options.maxPremiseDepth;
   }
   _add(subject, predicate, object, graphItem, c, cb) {
-    // Only add to the remaining indexes if there is not already a value in the index
-    if (!this._store._addToIndex(graphItem.subjects, subject, predicate, object)) return;
-    this._store._addToIndex(graphItem.predicates, predicate, object, subject);
-    this._store._addToIndex(graphItem.objects, object, subject, predicate);
+    const store = this._store;
+    // Observing views must see each derivation before indexing
+    // (stores from other N3 versions have no observers)
+    if (store._observers) {
+      if (!store._addQuad(subject, predicate, object, this._graphId)) return;
+    } else {
+      // Only add to the remaining indexes if there is not already a value in the index
+      if (!store._addToIndex(graphItem.subjects, subject, predicate, object)) return;
+      store._addToIndex(graphItem.predicates, predicate, object, subject);
+      store._addToIndex(graphItem.objects, object, subject, predicate);
+    }
     // Count genuinely new derivations and fail past the budget. The check comes
     // after all three indexes are updated, so a caught error leaves the store
     // in a consistent state (the reasoning result is merely incomplete).
@@ -224,6 +231,8 @@ class N3Reasoner {
     const graphs = this._store._getGraphs();
     try {
       for (const graphId in graphs) {
+        // Observers compare numeric graph identifiers
+        this._graphId = Number(graphId);
         this._reasonGraphNaive(rules, graphs[graphId]);
       }
     } finally {