n3 2.2.9 → 2.2.11

package/lib/N3Store.js

@@ -14,14 +14,22 @@ function _interopRequireWildcard(e, t) { if ("function" == typeof WeakMap) var r
 // **N3Store** objects store N3 quads by graph in memory.
 
 const ITERATOR = Symbol('iter');
+const SIZE = Symbol('size');
 function hasInIndex(index0, key0, key1, key2) {
   const index1 = index0 && index0[key0];
   const index2 = index1 && index1[key1];
   return !!index2 && key2 in index2;
 }
 function merge(target, source, depth = 4) {
-  if (depth === 0) return Object.assign(target, source);
-  for (const key in source) target[key] = merge(target[key] || Object.create(null), source[key], depth - 1);
+  let size = target[SIZE] || 0;
+  for (const key in source) {
+    if (!(key in target)) {
+      size++;
+      target[key] = depth === 0 ? null : merge(Object.create(null), source[key], depth - 1);
+    } else if (depth !== 0) target[key] = merge(target[key], source[key], depth - 1);
+  }
+  // Depth 2 is the level of the `subjects`, `predicates`, and `objects` indexes.
+  if (depth <= 2) target[SIZE] = size;
   return target;
 }
 
@@ -33,13 +41,16 @@ function merge(target, source, depth = 4) {
  * *not* be set as the value for an index.
  */
 function intersect(s1, s2, depth = 4) {
-  let target = false;
+  let target = false,
+    size = 0;
+  if (depth <= 2 && s2[SIZE] < s1[SIZE]) [s1, s2] = [s2, s1];
   for (const key in s1) {
     if (key in s2) {
       const intersection = depth === 0 ? null : intersect(s1[key], s2[key], depth - 1);
       if (intersection !== false) {
         target = target || Object.create(null);
         target[key] = intersection;
+        size++;
       }
       // Depth 3 is the 'subjects', 'predicates' and 'objects' keys.
       // If the 'subjects' index is empty, so will the 'predicates' and 'objects' index.
@@ -48,6 +59,9 @@ function intersect(s1, s2, depth = 4) {
       }
     }
   }
+
+  // Depth 2 is the level of the `subjects`, `predicates`, and `objects` indexes.
+  if (depth <= 2 && target) target[SIZE] = size;
   return target;
 }
 
@@ -59,18 +73,21 @@ function intersect(s1, s2, depth = 4) {
  * *not* be set as the value for an index.
  */
 function difference(s1, s2, depth = 4) {
-  let target = false;
+  let target = false,
+    size = 0;
   for (const key in s1) {
     // When the key is not in the index, then none of the triples defined by s1[key] are
     // in s2 and so we want to copy them over to the resultant store.
     if (!(key in s2)) {
       target = target || Object.create(null);
       target[key] = depth === 0 ? null : merge({}, s1[key], depth - 1);
+      size++;
     } else if (depth !== 0) {
       const diff = difference(s1[key], s2[key], depth - 1);
       if (diff !== false) {
         target = target || Object.create(null);
         target[key] = diff;
+        size++;
       }
       // Depth 3 is the 'subjects', 'predicates' and 'objects' keys.
       // If the 'subjects' index is empty, so will the 'predicates' and 'objects' index.
@@ -79,6 +96,9 @@ function difference(s1, s2, depth = 4) {
       }
     }
   }
+
+  // Depth 2 is the level of the `subjects`, `predicates`, and `objects` indexes.
+  if (depth <= 2 && target) target[SIZE] = size;
   return target;
 }
 
@@ -178,7 +198,7 @@ class N3Store {
     size = 0;
     const graphs = this._graphs;
     let subjects, subject;
-    for (const graphKey in graphs) for (const subjectKey in subjects = graphs[graphKey].subjects) for (const predicateKey in subject = subjects[subjectKey]) size += Object.keys(subject[predicateKey]).length;
+    for (const graphKey in graphs) for (const subjectKey in subjects = graphs[graphKey].subjects) for (const predicateKey in subject = subjects[subjectKey]) size += subject[predicateKey][SIZE];
     return this._size = size;
   }
 
@@ -187,12 +207,27 @@ class N3Store {
   // ### `_addToIndex` adds a quad to a three-layered index.
   // Returns if the index has changed, if the entry did not already exist.
   _addToIndex(index0, key0, key1, key2) {
-    // Create layers as necessary
-    const index1 = index0[key0] || (index0[key0] = {});
-    const index2 = index1[key1] || (index1[key1] = {});
+    // Create layers as necessary, maintaining their entry counters
+    let index1 = index0[key0];
+    if (!index1) {
+      index0[key0] = index1 = {
+        [SIZE]: 0
+      };
+      index0[SIZE]++;
+    }
+    let index2 = index1[key1];
+    if (!index2) {
+      index1[key1] = index2 = {
+        [SIZE]: 0
+      };
+      index1[SIZE]++;
+    }
     // Setting the key to _any_ value signals the presence of the quad
     const existed = key2 in index2;
-    if (!existed) index2[key2] = null;
+    if (!existed) {
+      index2[key2] = null;
+      index2[SIZE]++;
+    }
     return !existed;
   }
 
@@ -203,11 +238,13 @@ class N3Store {
       index2 = index1[key1];
     delete index2[key2];
 
-    // Remove intermediary index layers if they are empty
-    for (const key in index2) return;
+    // Remove intermediary index layers if they are empty,
+    // which the entry counters detect in constant time
+    if (--index2[SIZE] !== 0) return;
     delete index1[key1];
-    for (const key in index1) return;
+    if (--index1[SIZE] !== 0) return;
     delete index0[key0];
+    index0[SIZE]--;
   }
 
   // ### `_findInIndex` finds a set of quads in a three-layered index.
@@ -312,7 +349,7 @@ class N3Store {
             // If a key is specified, count the quad if it exists
             if (key2) key2 in index2 && count++;
             // Otherwise, count all quads
-            else count += Object.keys(index2).length;
+            else count += index2[SIZE];
           }
         }
       }
@@ -365,9 +402,15 @@ class N3Store {
     // Create the graph if it doesn't exist yet
     if (!graphItem) {
       graphItem = this._graphs[graph] = {
-        subjects: {},
-        predicates: {},
-        objects: {}
+        subjects: {
+          [SIZE]: 0
+        },
+        predicates: {
+          [SIZE]: 0
+        },
+        objects: {
+          [SIZE]: 0
+        }
       };
       // Freezing a graph helps subsequent `add` performance,
       // and properties will never be modified anyway
@@ -454,8 +497,7 @@ class N3Store {
     if (this._size !== null) this._size--;
 
     // Remove the graph if it is empty
-    for (subject in graphItem.subjects) return true;
-    delete graphs[graph];
+    if (graphItem.subjects[SIZE] === 0) delete graphs[graph];
     return true;
   }
 
@@ -1062,7 +1104,8 @@ exports.default = N3Store;
 function indexMatch(index, ids, depth = 0) {
   const ind = ids[depth];
   if (ind && !(ind in index)) return false;
-  let target = false;
+  let target = false,
+    size = 0;
   for (const key in ind ? {
     [ind]: index[ind]
   } : index) {
@@ -1070,8 +1113,10 @@ function indexMatch(index, ids, depth = 0) {
     if (result !== false) {
       target = target || Object.create(null);
       target[key] = result;
+      size++;
     }
   }
+  if (target) target[SIZE] = size;
   return target;
 }
 

package/lib/N3StreamWriter.js

@@ -9,6 +9,9 @@ var _N3Writer = _interopRequireDefault(require("./N3Writer"));
 function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
 // **N3StreamWriter** serializes a quad stream into a text stream.
 
+const MIN_CHUNK_SIZE = 16 * 1024;
+const DEFAULT_FLUSH_DELAY_MS = 20;
+
 // ## Constructor
 class N3StreamWriter extends _readableStream.Transform {
   constructor(options) {
@@ -17,27 +20,77 @@ class N3StreamWriter extends _readableStream.Transform {
       writableObjectMode: true
     });
 
+    // Coalesce serialized fragments into larger stream chunks
+    this._buffer = '';
+
+    // Flush partial chunks after a bounded delay
+    this._flushTimer = null;
+    this._flushDelay = options && options.flushDelay !== undefined ? options.flushDelay : DEFAULT_FLUSH_DELAY_MS;
+
     // Set up writer with a dummy stream object
     const writer = this._writer = new _N3Writer.default({
-      write: (quad, encoding, callback) => {
-        this.push(quad);
+      write: (chunk, encoding, callback) => {
+        this._buffer += chunk;
+        if (this._buffer.length >= MIN_CHUNK_SIZE) this._pushBuffer();else if (this._flushTimer === null) this._armFlushTimer();
         callback && callback();
       },
       end: callback => {
+        this._pushBuffer();
         this.push(null);
         callback && callback();
       }
     }, options);
 
-    // Implement Transform methods on top of writer
+    // Flush buffered output before serialization errors
+    let pendingDone = null;
+    const quadDone = error => {
+      const done = pendingDone;
+      pendingDone = null;
+      if (error) this._pushBuffer();
+      done(error);
+    };
     this._transform = (quad, encoding, done) => {
-      writer.addQuad(quad, done);
+      pendingDone = done;
+      writer.addQuad(quad, quadDone);
     };
     this._flush = done => {
       writer.end(done);
     };
   }
 
+  // ### `_pushBuffer` flushes coalesced output to the stream queue
+  _pushBuffer() {
+    this._clearFlushTimer();
+    if (this._buffer !== '') {
+      this.push(this._buffer);
+      this._buffer = '';
+    }
+  }
+
+  // ### `_armFlushTimer` schedules a partial-chunk flush
+  _armFlushTimer() {
+    this._flushTimer = setTimeout(() => {
+      this._flushTimer = null;
+      this._pushBuffer();
+    }, this._flushDelay);
+    // Browser timers do not implement `unref`
+    this._flushTimer.unref && this._flushTimer.unref();
+  }
+
+  // ### `_clearFlushTimer` cancels a scheduled flush
+  _clearFlushTimer() {
+    if (this._flushTimer !== null) {
+      clearTimeout(this._flushTimer);
+      this._flushTimer = null;
+    }
+  }
+
+  // ### `_destroy` cancels a scheduled flush, so it cannot fire afterwards
+  _destroy(error, callback) {
+    this._clearFlushTimer();
+    super._destroy(error, callback);
+  }
+
   // ### Serializes a stream of quads
   import(stream) {
     stream.on('data', quad => {
@@ -47,6 +100,7 @@ class N3StreamWriter extends _readableStream.Transform {
       this.end();
     });
     stream.on('error', error => {
+      this._pushBuffer();
       this.emit('error', error);
     });
     stream.on('prefix', (prefix, iri) => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "n3",
-  "version": "2.2.9",
+  "version": "2.2.11",
   "description": "Lightning fast, asynchronous, streaming Turtle / N3 / RDF library.",
   "author": "Ruben Verborgh <ruben.verborgh@gmail.com>",
   "keywords": [

package/src/N3Store.js

@@ -6,6 +6,7 @@ import { isDefaultGraph } from './N3Util';
 import N3Writer from './N3Writer';
 
 const ITERATOR = Symbol('iter');
+const SIZE = Symbol('size');
 
 function hasInIndex(index0, key0, key1, key2) {
   const index1 = index0 && index0[key0];
@@ -14,11 +15,18 @@ function hasInIndex(index0, key0, key1, key2) {
 }
 
 function merge(target, source, depth = 4) {
-  if (depth === 0)
-    return Object.assign(target, source);
-
-  for (const key in source)
-    target[key] = merge(target[key] || Object.create(null), source[key], depth - 1);
+  let size = target[SIZE] || 0;
+  for (const key in source) {
+    if (!(key in target)) {
+      size++;
+      target[key] = depth === 0 ? null : merge(Object.create(null), source[key], depth - 1);
+    }
+    else if (depth !== 0)
+      target[key] = merge(target[key], source[key], depth - 1);
+  }
+  // Depth 2 is the level of the `subjects`, `predicates`, and `objects` indexes.
+  if (depth <= 2)
+    target[SIZE] = size;
 
   return target;
 }
@@ -31,7 +39,10 @@ function merge(target, source, depth = 4) {
  * *not* be set as the value for an index.
  */
 function intersect(s1, s2, depth = 4) {
-  let target = false;
+  let target = false, size = 0;
+
+  if (depth <= 2 && s2[SIZE] < s1[SIZE])
+    [s1, s2] = [s2, s1];
 
   for (const key in s1) {
     if (key in s2) {
@@ -39,6 +50,7 @@ function intersect(s1, s2, depth = 4) {
       if (intersection !== false) {
         target = target || Object.create(null);
         target[key] = intersection;
+        size++;
       }
       // Depth 3 is the 'subjects', 'predicates' and 'objects' keys.
       // If the 'subjects' index is empty, so will the 'predicates' and 'objects' index.
@@ -48,6 +60,10 @@ function intersect(s1, s2, depth = 4) {
     }
   }
 
+  // Depth 2 is the level of the `subjects`, `predicates`, and `objects` indexes.
+  if (depth <= 2 && target)
+    target[SIZE] = size;
+
   return target;
 }
 
@@ -59,7 +75,7 @@ function intersect(s1, s2, depth = 4) {
  * *not* be set as the value for an index.
  */
 function difference(s1, s2, depth = 4) {
-  let target = false;
+  let target = false, size = 0;
 
   for (const key in s1) {
     // When the key is not in the index, then none of the triples defined by s1[key] are
@@ -67,12 +83,14 @@ function difference(s1, s2, depth = 4) {
     if (!(key in s2)) {
       target = target || Object.create(null);
       target[key] = depth === 0 ? null : merge({}, s1[key], depth - 1);
+      size++;
     }
     else if (depth !== 0) {
       const diff = difference(s1[key], s2[key], depth - 1);
       if (diff !== false) {
         target = target || Object.create(null);
         target[key] = diff;
+        size++;
       }
       // Depth 3 is the 'subjects', 'predicates' and 'objects' keys.
       // If the 'subjects' index is empty, so will the 'predicates' and 'objects' index.
@@ -82,6 +100,10 @@ function difference(s1, s2, depth = 4) {
     }
   }
 
+  // Depth 2 is the level of the `subjects`, `predicates`, and `objects` indexes.
+  if (depth <= 2 && target)
+    target[SIZE] = size;
+
   return target;
 }
 
@@ -200,7 +222,7 @@ export default class N3Store {
     for (const graphKey in graphs)
       for (const subjectKey in (subjects = graphs[graphKey].subjects))
         for (const predicateKey in (subject = subjects[subjectKey]))
-          size += Object.keys(subject[predicateKey]).length;
+          size += subject[predicateKey][SIZE];
     return this._size = size;
   }
 
@@ -209,13 +231,23 @@ export default class N3Store {
   // ### `_addToIndex` adds a quad to a three-layered index.
   // Returns if the index has changed, if the entry did not already exist.
   _addToIndex(index0, key0, key1, key2) {
-    // Create layers as necessary
-    const index1 = index0[key0] || (index0[key0] = {});
-    const index2 = index1[key1] || (index1[key1] = {});
+    // Create layers as necessary, maintaining their entry counters
+    let index1 = index0[key0];
+    if (!index1) {
+      index0[key0] = index1 = { [SIZE]: 0 };
+      index0[SIZE]++;
+    }
+    let index2 = index1[key1];
+    if (!index2) {
+      index1[key1] = index2 = { [SIZE]: 0 };
+      index1[SIZE]++;
+    }
     // Setting the key to _any_ value signals the presence of the quad
     const existed = key2 in index2;
-    if (!existed)
+    if (!existed) {
       index2[key2] = null;
+      index2[SIZE]++;
+    }
     return !existed;
   }
 
@@ -225,11 +257,13 @@ export default class N3Store {
     const index1 = index0[key0], index2 = index1[key1];
     delete index2[key2];
 
-    // Remove intermediary index layers if they are empty
-    for (const key in index2) return;
+    // Remove intermediary index layers if they are empty,
+    // which the entry counters detect in constant time
+    if (--index2[SIZE] !== 0) return;
     delete index1[key1];
-    for (const key in index1) return;
+    if (--index1[SIZE] !== 0) return;
     delete index0[key0];
+    index0[SIZE]--;
   }
 
   // ### `_findInIndex` finds a set of quads in a three-layered index.
@@ -332,7 +366,7 @@ export default class N3Store {
             // If a key is specified, count the quad if it exists
             if (key2) (key2 in index2) && count++;
             // Otherwise, count all quads
-            else count += Object.keys(index2).length;
+            else count += index2[SIZE];
           }
         }
       }
@@ -384,7 +418,11 @@ export default class N3Store {
     let graphItem = this._graphs[graph];
     // Create the graph if it doesn't exist yet
     if (!graphItem) {
-      graphItem = this._graphs[graph] = { subjects: {}, predicates: {}, objects: {} };
+      graphItem = this._graphs[graph] = {
+        subjects: { [SIZE]: 0 },
+        predicates: { [SIZE]: 0 },
+        objects: { [SIZE]: 0 },
+      };
       // Freezing a graph helps subsequent `add` performance,
       // and properties will never be modified anyway
       Object.freeze(graphItem);
@@ -469,8 +507,8 @@ export default class N3Store {
     if (this._size !== null) this._size--;
 
     // Remove the graph if it is empty
-    for (subject in graphItem.subjects) return true;
-    delete graphs[graph];
+    if (graphItem.subjects[SIZE] === 0)
+      delete graphs[graph];
     return true;
   }
 
@@ -1130,15 +1168,18 @@ function indexMatch(index, ids, depth = 0) {
   if (ind && !(ind in index))
     return false;
 
-  let target = false;
+  let target = false, size = 0;
   for (const key in (ind ? { [ind]: index[ind] } : index)) {
     const result = depth === 2 ? null : indexMatch(index[key], ids, depth + 1);
 
     if (result !== false) {
       target = target || Object.create(null);
       target[key] = result;
+      size++;
     }
   }
+  if (target)
+    target[SIZE] = size;
   return target;
 }
 

package/src/N3StreamWriter.js

@@ -2,27 +2,89 @@
 import { Transform } from 'readable-stream';
 import N3Writer from './N3Writer';
 
+const MIN_CHUNK_SIZE = 16 * 1024;
+const DEFAULT_FLUSH_DELAY_MS = 20;
+
 // ## Constructor
 export default class N3StreamWriter extends Transform {
   constructor(options) {
     super({ encoding: 'utf8', writableObjectMode: true });
 
+    // Coalesce serialized fragments into larger stream chunks
+    this._buffer = '';
+
+    // Flush partial chunks after a bounded delay
+    this._flushTimer = null;
+    this._flushDelay = options && options.flushDelay !== undefined ?
+      options.flushDelay : DEFAULT_FLUSH_DELAY_MS;
+
     // Set up writer with a dummy stream object
     const writer = this._writer = new N3Writer({
-      write: (quad, encoding, callback) => { this.push(quad); callback && callback(); },
-      end: callback => { this.push(null); callback && callback(); },
+      write: (chunk, encoding, callback) => {
+        this._buffer += chunk;
+        if (this._buffer.length >= MIN_CHUNK_SIZE)
+          this._pushBuffer();
+        else if (this._flushTimer === null)
+          this._armFlushTimer();
+        callback && callback();
+      },
+      end: callback => { this._pushBuffer(); this.push(null); callback && callback(); },
     }, options);
 
-    // Implement Transform methods on top of writer
-    this._transform = (quad, encoding, done) => { writer.addQuad(quad, done); };
+    // Flush buffered output before serialization errors
+    let pendingDone = null;
+    const quadDone = error => {
+      const done = pendingDone;
+      pendingDone = null;
+      if (error)
+        this._pushBuffer();
+      done(error);
+    };
+    this._transform = (quad, encoding, done) => {
+      pendingDone = done;
+      writer.addQuad(quad, quadDone);
+    };
     this._flush = done => { writer.end(done); };
   }
 
+  // ### `_pushBuffer` flushes coalesced output to the stream queue
+  _pushBuffer() {
+    this._clearFlushTimer();
+    if (this._buffer !== '') {
+      this.push(this._buffer);
+      this._buffer = '';
+    }
+  }
+
+  // ### `_armFlushTimer` schedules a partial-chunk flush
+  _armFlushTimer() {
+    this._flushTimer = setTimeout(() => {
+      this._flushTimer = null;
+      this._pushBuffer();
+    }, this._flushDelay);
+    // Browser timers do not implement `unref`
+    this._flushTimer.unref && this._flushTimer.unref();
+  }
+
+  // ### `_clearFlushTimer` cancels a scheduled flush
+  _clearFlushTimer() {
+    if (this._flushTimer !== null) {
+      clearTimeout(this._flushTimer);
+      this._flushTimer = null;
+    }
+  }
+
+  // ### `_destroy` cancels a scheduled flush, so it cannot fire afterwards
+  _destroy(error, callback) {
+    this._clearFlushTimer();
+    super._destroy(error, callback);
+  }
+
 // ### Serializes a stream of quads
   import(stream) {
     stream.on('data',   quad => { this.write(quad); });
     stream.on('end',    () => { this.end(); });
-    stream.on('error',  error => { this.emit('error', error); });
+    stream.on('error',  error => { this._pushBuffer(); this.emit('error', error); });
     stream.on('prefix', (prefix, iri) => { this._writer.addPrefix(prefix, iri); });
     return this;
   }