n3 2.11.6 → 2.11.8

package/lib/N3Lexer.js

@@ -4,7 +4,6 @@ Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.default = void 0;
-var _buffer = require("buffer");
 var _IRIs = _interopRequireDefault(require("./IRIs"));
 function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
 // **N3Lexer** tokenizes N3 documents.
@@ -94,6 +93,9 @@ function isSeparatorCode(code) {
   return code === SPACE || code === TAB || code === LF || code === CR || code === HASH;
 }
 
+// Unfinished input in a stream up to this length is tokenized again with every chunk
+const MIN_RESCAN_LENGTH = 1024;
+
 // ## Constructor
 class N3Lexer {
   constructor(options) {
@@ -721,31 +723,39 @@ class N3Lexer {
     }
     // Otherwise, the input must be a stream
     else {
-      this._pendingBuffer = null;
+      let decoder,
+        retryLength = 0;
       if (typeof input.setEncoding === 'function') input.setEncoding('utf8');
       // Adds the data chunk to the buffer and parses as far as possible
       input.on('data', data => {
         if (this._tokenization === tokenization && this._input !== null && data.length !== 0) {
-          // Prepend any previous pending writes
-          if (this._pendingBuffer) {
-            data = _buffer.Buffer.concat([this._pendingBuffer, data]);
-            this._pendingBuffer = null;
-          }
-          // Hold if the buffer ends in an incomplete unicode sequence
-          if (data[data.length - 1] & 0x80) {
-            this._pendingBuffer = data;
+          // Decode bytes, keeping an incomplete trailing character for the next chunk
+          if (typeof data !== 'string') {
+            decoder = decoder || new TextDecoder('utf-8', {
+              ignoreBOM: true
+            });
+            if (!(data = decoder.decode(data, {
+              stream: true
+            }))) return;
           }
-          // Otherwise, tokenize as far as possible
-          else {
-            // Only read a BOM at the start
-            if (typeof this._input === 'undefined') this._input = this._readStartingBom(typeof data === 'string' ? data : data.toString());else this._input += data;
+          // Only read a BOM at the start
+          if (typeof this._input === 'undefined') this._input = this._readStartingBom(data);else this._input += data;
+          // Tokenize as far as possible. When a previous attempt left a long unfinished token,
+          // wait until the buffered input has doubled, so the token is not rescanned for every chunk.
+          if (this._input.length >= retryLength) {
             this._tokenizeToEnd(callback, false);
+            retryLength = this._input !== null && this._input.length > MIN_RESCAN_LENGTH ? 2 * this._input.length : 0;
           }
         }
       });
       // Parses until the end
       input.on('end', () => {
-        if (this._tokenization === tokenization && typeof this._input === 'string') this._tokenizeToEnd(callback, true);
+        if (this._tokenization === tokenization && this._input !== null) {
+          // Decode any incomplete character left at the end
+          const rest = decoder ? decoder.decode() : '';
+          if (rest) this._input = typeof this._input === 'string' ? this._input + rest : rest;
+          if (typeof this._input === 'string') this._tokenizeToEnd(callback, true);
+        }
       });
       input.on('error', error => {
         if (this._tokenization === tokenization) callback(error);

package/lib/N3Writer.js

@@ -188,7 +188,7 @@ class N3Writer {
     // Escape special characters
     if (this._escape.test(iri)) iri = iri.replace(this._escapeAll, this._characterReplacer);
     // Try to represent the IRI as prefixed name, unless no prefixes were added
-    const prefixMatch = this._hasPrefixes ? this._prefixRegex.exec(iri) : null;
+    const prefixMatch = this._hasPrefixes ? (this._prefixRegex || this._createPrefixRegex()).exec(iri) : null;
     return !prefixMatch ? `<${iri}>` : !prefixMatch[1] ? iri : this._prefixIRIs[prefixMatch[1]] + prefixMatch[2];
   }
 
@@ -319,22 +319,28 @@ class N3Writer {
       this._prefixPatterns[iri] = [(0, _Util.escapeRegex)(iri), (0, _Util.escapeRegex)(prefix)];
       this._write(`@prefix ${prefix} <${iri}>.\n`);
     }
-    // Recreate the prefix matcher
+    // Recreate the prefix matcher when it is next needed, so that adding
+    // prefixes one by one does not rebuild it for every prefix
     if (hasPrefixes) {
       this._hasPrefixes = true;
-      let IRIlist = '',
-        prefixList = '';
-      for (const prefixIRI in this._prefixPatterns) {
-        const [IRIpattern, prefixPattern] = this._prefixPatterns[prefixIRI];
-        IRIlist += IRIlist ? `|${IRIpattern}` : IRIpattern;
-        prefixList += prefixList ? `|${prefixPattern}` : prefixPattern;
-      }
-      this._prefixRegex = new RegExp(`^(?:${prefixList})[^/]*$|` + `^(${IRIlist})([_a-zA-Z0-9](?:\\.?[\\-_a-zA-Z0-9])*)$`);
+      this._prefixRegex = null;
     }
     // End a prefix block with a newline
     this._write(hasPrefixes ? '\n' : '', done);
   }
 
+  // ### `_createPrefixRegex` creates the matcher for the current prefixes
+  _createPrefixRegex() {
+    let IRIlist = '',
+      prefixList = '';
+    for (const prefixIRI in this._prefixPatterns) {
+      const [IRIpattern, prefixPattern] = this._prefixPatterns[prefixIRI];
+      IRIlist += IRIlist ? `|${IRIpattern}` : IRIpattern;
+      prefixList += prefixList ? `|${prefixPattern}` : prefixPattern;
+    }
+    return this._prefixRegex = new RegExp(`^(?:${prefixList})[^/]*$|` + `^(${IRIlist})([_a-zA-Z0-9](?:\\.?[\\-_a-zA-Z0-9])*)$`);
+  }
+
   // ### `blank` creates a blank node with the given content
   blank(predicate, object) {
     let children = predicate,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "n3",
-  "version": "2.11.6",
+  "version": "2.11.8",
   "description": "Lightning fast, asynchronous, streaming Turtle / N3 / RDF library.",
   "author": "Ruben Verborgh <ruben.verborgh@gmail.com>",
   "keywords": [

package/src/N3Lexer.js

@@ -1,5 +1,4 @@
 // **N3Lexer** tokenizes N3 documents.
-import { Buffer } from 'buffer';
 import namespaces from './IRIs';
 
 const { xsd } = namespaces;
@@ -59,6 +58,9 @@ function isSeparatorCode(code) {
   return code === SPACE || code === TAB || code === LF || code === CR || code === HASH;
 }
 
+// Unfinished input in a stream up to this length is tokenized again with every chunk
+const MIN_RESCAN_LENGTH = 1024;
+
 // ## Constructor
 export default class N3Lexer {
   constructor(options) {
@@ -747,36 +749,42 @@ export default class N3Lexer {
     }
     // Otherwise, the input must be a stream
     else {
-      this._pendingBuffer = null;
+      let decoder, retryLength = 0;
       if (typeof input.setEncoding === 'function')
         input.setEncoding('utf8');
       // Adds the data chunk to the buffer and parses as far as possible
       input.on('data', data => {
         if (this._tokenization === tokenization && this._input !== null && data.length !== 0) {
-          // Prepend any previous pending writes
-          if (this._pendingBuffer) {
-            data = Buffer.concat([this._pendingBuffer, data]);
-            this._pendingBuffer = null;
-          }
-          // Hold if the buffer ends in an incomplete unicode sequence
-          if (data[data.length - 1] & 0x80) {
-            this._pendingBuffer = data;
+          // Decode bytes, keeping an incomplete trailing character for the next chunk
+          if (typeof data !== 'string') {
+            decoder = decoder || new TextDecoder('utf-8', { ignoreBOM: true });
+            if (!(data = decoder.decode(data, { stream: true })))
+              return;
           }
-          // Otherwise, tokenize as far as possible
-          else {
-            // Only read a BOM at the start
-            if (typeof this._input === 'undefined')
-              this._input = this._readStartingBom(typeof data === 'string' ? data : data.toString());
-            else
-              this._input += data;
+          // Only read a BOM at the start
+          if (typeof this._input === 'undefined')
+            this._input = this._readStartingBom(data);
+          else
+            this._input += data;
+          // Tokenize as far as possible. When a previous attempt left a long unfinished token,
+          // wait until the buffered input has doubled, so the token is not rescanned for every chunk.
+          if (this._input.length >= retryLength) {
             this._tokenizeToEnd(callback, false);
+            retryLength = this._input !== null && this._input.length > MIN_RESCAN_LENGTH ?
+              2 * this._input.length : 0;
           }
         }
       });
       // Parses until the end
       input.on('end', () => {
-        if (this._tokenization === tokenization && typeof this._input === 'string')
-          this._tokenizeToEnd(callback, true);
+        if (this._tokenization === tokenization && this._input !== null) {
+          // Decode any incomplete character left at the end
+          const rest = decoder ? decoder.decode() : '';
+          if (rest)
+            this._input = typeof this._input === 'string' ? this._input + rest : rest;
+          if (typeof this._input === 'string')
+            this._tokenizeToEnd(callback, true);
+        }
       });
       input.on('error', error => {
         if (this._tokenization === tokenization)

package/src/N3Writer.js

@@ -189,7 +189,7 @@ export default class N3Writer {
     if (this._escape.test(iri))
       iri = iri.replace(this._escapeAll, this._characterReplacer);
     // Try to represent the IRI as prefixed name, unless no prefixes were added
-    const prefixMatch = this._hasPrefixes ? this._prefixRegex.exec(iri) : null;
+    const prefixMatch = this._hasPrefixes ? (this._prefixRegex || this._createPrefixRegex()).exec(iri) : null;
     return !prefixMatch ? `<${iri}>` :
            (!prefixMatch[1] ? iri : this._prefixIRIs[prefixMatch[1]] + prefixMatch[2]);
   }
@@ -333,22 +333,28 @@ export default class N3Writer {
       this._prefixPatterns[iri] = [escapeRegex(iri), escapeRegex(prefix)];
       this._write(`@prefix ${prefix} <${iri}>.\n`);
     }
-    // Recreate the prefix matcher
+    // Recreate the prefix matcher when it is next needed, so that adding
+    // prefixes one by one does not rebuild it for every prefix
     if (hasPrefixes) {
       this._hasPrefixes = true;
-      let IRIlist = '', prefixList = '';
-      for (const prefixIRI in this._prefixPatterns) {
-        const [IRIpattern, prefixPattern] = this._prefixPatterns[prefixIRI];
-        IRIlist += IRIlist ? `|${IRIpattern}` : IRIpattern;
-        prefixList += prefixList ? `|${prefixPattern}` : prefixPattern;
-      }
-      this._prefixRegex = new RegExp(`^(?:${prefixList})[^/]*$|` +
-                                     `^(${IRIlist})([_a-zA-Z0-9](?:\\.?[\\-_a-zA-Z0-9])*)$`);
+      this._prefixRegex = null;
     }
     // End a prefix block with a newline
     this._write(hasPrefixes ? '\n' : '', done);
   }
 
+  // ### `_createPrefixRegex` creates the matcher for the current prefixes
+  _createPrefixRegex() {
+    let IRIlist = '', prefixList = '';
+    for (const prefixIRI in this._prefixPatterns) {
+      const [IRIpattern, prefixPattern] = this._prefixPatterns[prefixIRI];
+      IRIlist += IRIlist ? `|${IRIpattern}` : IRIpattern;
+      prefixList += prefixList ? `|${prefixPattern}` : prefixPattern;
+    }
+    return this._prefixRegex = new RegExp(`^(?:${prefixList})[^/]*$|` +
+                                          `^(${IRIlist})([_a-zA-Z0-9](?:\\.?[\\-_a-zA-Z0-9])*)$`);
+  }
+
   // ### `blank` creates a blank node with the given content
   blank(predicate, object) {
     let children = predicate, child, length;