n3 2.7.11 → 2.8.0

package/src/N3Lexer.js

@@ -57,7 +57,7 @@ export default class N3Lexer {
     this._n3Verb = /^(?:has|is|of)(?=[\s#()\[\]\{\}"'<>?_+\-0-9])/;
     this._n3Id = /^id(?=[\s#<])/;
     this._shortPredicates = /^a(?=[\s#()\[\]\{\}"'<>])/;
-    this._commentLine = /^[ \t]*#([^\n\r]*)(?:\r\n|\n|\r)[ \t]*/;
+    this._commentLine = /^[ \t]*#([^\n\r]*)(?:\r\n|\n|\r)([ \t]*)/;
     this._whitespace = /^[ \t]+/;
     options = options || {};
 
@@ -89,7 +89,7 @@ export default class N3Lexer {
   _tokenizeToEnd(callback, inputFinished) {
     // Continue parsing as far as possible; the loop will return eventually
     let input = this._input;
-    let currentLineLength = input.length;
+    let currentLineLength = this._linePosition + input.length;
     while (true) {
       // Consume one separator line at a time, including its following indentation.
       while (true) {
@@ -103,34 +103,49 @@ export default class N3Lexer {
         if (charCode === HASH) {
           const comment = this._commentLine.exec(input);
           if (comment) {
+            const commentLength = comment[0].length;
+            // Keep a trailing CR buffered in case the next chunk starts with LF.
+            if (!inputFinished && commentLength === input.length &&
+                input.charCodeAt(commentLength - 1) === CR) {
+              this._linePosition = currentLineLength - input.length;
+              return this._input = input;
+            }
             if (this.comments)
-              emitToken('comment', comment[1], '', this._line, comment[0].length);
-            input = input.slice(comment[0].length);
-            currentLineLength = input.length;
+              emitComment(comment[1], this._line, separatorLength);
+            input = input.slice(commentLength);
+            currentLineLength = input.length + comment[2].length;
             this._line++;
           }
           else {
             // A comment without a line ending stays buffered until EOF.
             input = input.slice(separatorLength);
-            if (!inputFinished)
+            if (!inputFinished) {
+              this._linePosition = currentLineLength - input.length;
               return this._input = input;
+            }
             if (this.comments)
-              emitToken('comment', input.slice(1), '', this._line, input.length);
+              emitComment(input.slice(1), this._line, 0);
             input = '';
             break;
           }
         }
         else if (charCode === LF || charCode === CR) {
+          // A CR at the end of a chunk may still be followed by LF.
+          if (!inputFinished && charCode === CR && separatorLength + 1 === input.length) {
+            this._linePosition = currentLineLength - input.length;
+            return this._input = input;
+          }
           separatorLength += charCode === CR && input.charCodeAt(separatorLength + 1) === LF ? 2 : 1;
-          // Indentation is part of the same separator match as the newline.
+          // Indentation is consumed with the newline, but belongs to the next line's columns.
+          let indentationLength = 0;
           const next = input.charCodeAt(separatorLength);
           if (next === SPACE || next === TAB) {
             const following = input.charCodeAt(separatorLength + 1);
-            separatorLength += following === SPACE || following === TAB ?
+            indentationLength = following === SPACE || following === TAB ?
               this._whitespace.exec(input.slice(separatorLength))[0].length : 1;
           }
-          input = input.slice(separatorLength);
-          currentLineLength = input.length;
+          input = input.slice(separatorLength + indentationLength);
+          currentLineLength = input.length + indentationLength;
           this._line++;
         }
         else {
@@ -144,13 +159,15 @@ export default class N3Lexer {
           input = null;
           emitToken('eof', '', '', this._line, 0);
         }
+        this._linePosition = currentLineLength;
         return this._input = input;
       }
 
       // Look for specific token types based on the first character
       const line = this._line, firstChar = input[0];
       let type = '', value = '', prefix = '',
-          match = null, matchLength = 0, inconclusive = false;
+          match = null, matchLength = 0, lexicalLength = 0,
+          finalLineLength = 0, inconclusive = false;
       switch (firstChar) {
       case '^':
         // We need at least 3 tokens lookahead to distinguish ^^<IRI> and ^^pre:fixed
@@ -177,14 +194,17 @@ export default class N3Lexer {
         // Fall through in case the type is an IRI
       case '<':
         // Try to find a full IRI without escape sequences
-        if (match = this._unescapedIri.exec(input))
+        if (match = this._unescapedIri.exec(input)) {
           type = 'IRI', value = match[1];
+          lexicalLength = match[1].length + 2;
+        }
         // Try to find a full IRI with escape sequences
         else if (match = this._iri.exec(input)) {
           value = this._unescape(match[1], stringEscapeReplacements);
           if (value === null || illegalIriChars.test(value))
             return reportSyntaxError(this);
           type = 'IRI';
+          lexicalLength = match[1].length + 2;
         }
         // Try to find a triple term
         else if (input.length > 2 && input[1] === '<' && input[2] === '(')
@@ -214,8 +234,10 @@ export default class N3Lexer {
         // we always need a non-dot character before deciding it is a blank node.
         // Therefore, try inserting a space if we're at the end of the input.
         if ((match = this._blank.exec(input)) ||
-            inputFinished && (match = this._blank.exec(`${input} `)))
+            inputFinished && (match = this._blank.exec(`${input} `))) {
           type = 'blank', prefix = '_', value = match[1];
+          lexicalLength = match[1].length + 2;
+        }
         break;
 
       case '"':
@@ -224,7 +246,7 @@ export default class N3Lexer {
           value = match[1];
         // Try to find a literal wrapped in three pairs of quotes
         else {
-          ({ value, matchLength } = this._parseLiteral(input));
+          ({ value, matchLength, finalLineLength } = this._parseLiteral(input));
           if (value === null)
             return reportSyntaxError(this);
         }
@@ -241,7 +263,7 @@ export default class N3Lexer {
             value = match[1];
           // Try to find a literal wrapped in three pairs of quotes
           else {
-            ({ value, matchLength } = this._parseLiteral(input));
+            ({ value, matchLength, finalLineLength } = this._parseLiteral(input));
             if (value === null)
               return reportSyntaxError(this);
           }
@@ -436,8 +458,11 @@ export default class N3Lexer {
         // we always need a non-dot character before deciding it is a prefixed name.
         // Therefore, try inserting a space if we're at the end of the input.
         else if ((match = this._prefixed.exec(input)) ||
-                 inputFinished && (match = this._prefixed.exec(`${input} `)))
-          type = 'prefixed', prefix = match[1] || '', value = this._unescape(match[2], localNameEscapeReplacements);
+                 inputFinished && (match = this._prefixed.exec(`${input} `))) {
+          type = 'prefixed', prefix = match[1] || '';
+          value = this._unescape(match[2], localNameEscapeReplacements);
+          lexicalLength = prefix.length + match[2].length + 1;
+        }
       }
 
       // A type token is special: it can only be emitted after an IRI or prefixed name is read
@@ -456,20 +481,44 @@ export default class N3Lexer {
         // One exception: error on an unaccounted linebreak (= not inside a triple-quoted literal).
         if (inputFinished || (!/^'''|^"""/.test(input) && /\n|\r/.test(input)))
           return reportSyntaxError(this);
-        else
+        else {
+          this._linePosition = currentLineLength - input.length;
           return this._input = input;
+        }
       }
 
       // Emit the parsed token
+      // Consumption includes separator whitespace; lexicalLength excludes it
+      // and any synthetic EOF space. slice below clamps consumption to the input.
       const length = matchLength || match[0].length;
-      const token = emitToken(type, value, prefix, line, length);
+      let token;
+      if (finalLineLength) {
+        token = {
+          type, value, prefix, line,
+          start: currentLineLength - input.length,
+          end: finalLineLength, endLine: this._line,
+        };
+        callback(null, token);
+      }
+      else
+        token = emitToken(type, value, prefix, line, lexicalLength || length);
       this.previousToken = token;
       this._previousMarker = type;
 
       // Advance to next part to tokenize
       input = input.slice(length);
+      if (finalLineLength)
+        currentLineLength = input.length + finalLineLength;
     }
 
+    // Emits a comment at its exact position within matched whitespace.
+    function emitComment(value, line, offset) {
+      const start = currentLineLength - input.length + offset;
+      callback(null, {
+        type: 'comment', value, prefix: '', line,
+        start, end: start + value.length + 1,
+      });
+    }
     // Emits the token through the callback
     function emitToken(type, value, prefix, line, length) {
       const start = input ? currentLineLength - input.length : currentLineLength;
@@ -568,21 +617,23 @@ export default class N3Lexer {
         // means these are actual, non-escaped closing quotes
         if (backslashCount % 2 === 0) {
           // Extract and unescape the value
-          const raw = input.substring(openingLength, closingPos);
-          const lines = raw.split(/\r\n|\r|\n/).length - 1;
+          const raw = input.substring(openingLength, closingPos),
+              lines = raw.split(/\r\n|\r|\n/),
+              lineCount = lines.length - 1;
           const matchLength = closingPos + openingLength;
           // Only triple-quoted strings can be multi-line
-          if (openingLength === 1 && lines !== 0 ||
+          if (openingLength === 1 && lineCount !== 0 ||
               openingLength === 3 && this._lineMode)
             break;
-          this._line += lines;
-          return { value: this._unescape(raw, stringEscapeReplacements), matchLength };
+          this._line += lineCount;
+          const finalLineLength = lineCount === 0 ? 0 : lines[lines.length - 1].length + openingLength;
+          return { value: this._unescape(raw, stringEscapeReplacements), matchLength, finalLineLength };
         }
         closingPos++;
       }
       this._literalClosingPos = input.length - openingLength + 1;
     }
-    return { value: '', matchLength: 0 };
+    return { value: '', matchLength: 0, finalLineLength: 0 };
   }
 
   // ### `_syntaxError` creates a syntax error for the given issue
@@ -599,18 +650,30 @@ export default class N3Lexer {
 
   // ### Strips off any starting UTF BOM mark.
   _readStartingBom(input) {
-    return input.startsWith('\ufeff') ? input.slice(1) : input;
+    if (input.startsWith('\ufeff')) {
+      this._linePosition = 1;
+      return input.slice(1);
+    }
+    return input;
   }
 
   // ## Public methods
 
   // ### `tokenize` starts the transformation of an N3 document into an array of tokens.
   // The input can be a string or a stream.
+  // Token ranges use one-based lines and zero-based, end-exclusive UTF-16 columns.
+  // Separator whitespace counts towards the next token's start, outside either range.
+  // Multiline tokens also have endLine; their end column is relative to that line.
   tokenize(input, callback) {
     // Deferred tokenization and stream events can outlive their invocation.
     // Ignore them once a later call takes ownership of the lexer state.
     const tokenization = this._tokenization = {};
     this._line = 1;
+    this._linePosition = 0;
+    this._previousMarker = undefined;
+    this.previousToken = undefined;
+    this._literalClosingPos = 0;
+    this._input = undefined;
 
     // If the input is a string, continuously emit tokens through the callback until the end
     if (typeof input === 'string') {

package/src/N3Reasoner.js

@@ -23,10 +23,18 @@ export default class N3Reasoner {
   }
 
   _add(subject, predicate, object, graphItem, c, cb) {
-    // Only add to the remaining indexes if there is not already a value in the index
-    if (!this._store._addToIndex(graphItem.subjects,   subject,   predicate, object)) return;
-    this._store._addToIndex(graphItem.predicates, predicate, object,    subject);
-    this._store._addToIndex(graphItem.objects,    object,    subject,   predicate);
+    const store = this._store;
+    // Observing views must see each derivation before indexing
+    // (stores from other N3 versions have no observers)
+    if (store._observers) {
+      if (!store._addQuad(subject, predicate, object, this._graphId)) return;
+    }
+    else {
+      // Only add to the remaining indexes if there is not already a value in the index
+      if (!store._addToIndex(graphItem.subjects,   subject,   predicate, object)) return;
+      store._addToIndex(graphItem.predicates, predicate, object,    subject);
+      store._addToIndex(graphItem.objects,    object,    subject,   predicate);
+    }
     // Count genuinely new derivations and fail past the budget. The check comes
     // after all three indexes are updated, so a caught error leaves the store
     // in a consistent state (the reasoning result is merely incomplete).
@@ -228,6 +236,8 @@ export default class N3Reasoner {
     const graphs = this._store._getGraphs();
     try {
       for (const graphId in graphs) {
+        // Observers compare numeric graph identifiers
+        this._graphId = Number(graphId);
         this._reasonGraphNaive(rules, graphs[graphId]);
       }
     }