@electerm/nedb 1.8.0

package/lib/persistence.js

@@ -0,0 +1,314 @@
+/**
+ * Handle every persistence-related task
+ * The interface Datastore expects to be implemented is
+ * * Persistence.loadDatabase(callback) and callback has signature err
+ * * Persistence.persistNewState(newDocs, callback) where newDocs is an array of documents and callback has signature err
+ */
+
+var storage = require('./storage')
+  , path = require('path')
+  , model = require('./model')
+  , async = require('./async')
+  , customUtils = require('./customUtils')
+  , Index = require('./indexes')
+  ;
+
+
+/**
+ * Create a new Persistence object for database options.db
+ * @param {Datastore} options.db
+ * @param {Boolean} options.nodeWebkitAppName Optional, specify the name of your NW app if you want options.filename to be relative to the directory where
+ *                                            Node Webkit stores application data such as cookies and local storage (the best place to store data in my opinion)
+ */
+function Persistence (options) {
+  var i, j, randomString;
+
+  this.db = options.db;
+  this.inMemoryOnly = this.db.inMemoryOnly;
+  this.filename = this.db.filename;
+  this.corruptAlertThreshold = options.corruptAlertThreshold !== undefined ? options.corruptAlertThreshold : 0.1;
+
+  if (!this.inMemoryOnly && this.filename && this.filename.charAt(this.filename.length - 1) === '~') {
+    throw new Error("The datafile name can't end with a ~, which is reserved for crash safe backup files");
+  }
+
+  // After serialization and before deserialization hooks with some basic sanity checks
+  if (options.afterSerialization && !options.beforeDeserialization) {
+    throw new Error("Serialization hook defined but deserialization hook undefined, cautiously refusing to start NeDB to prevent dataloss");
+  }
+  if (!options.afterSerialization && options.beforeDeserialization) {
+    throw new Error("Serialization hook undefined but deserialization hook defined, cautiously refusing to start NeDB to prevent dataloss");
+  }
+  this.afterSerialization = options.afterSerialization || function (s) { return s; };
+  this.beforeDeserialization = options.beforeDeserialization || function (s) { return s; };
+  for (i = 1; i < 30; i += 1) {
+    for (j = 0; j < 10; j += 1) {
+      randomString = customUtils.uid(i);
+      if (this.beforeDeserialization(this.afterSerialization(randomString)) !== randomString) {
+        throw new Error("beforeDeserialization is not the reverse of afterSerialization, cautiously refusing to start NeDB to prevent dataloss");
+      }
+    }
+  }
+
+  // For NW apps, store data in the same directory where NW stores application data
+  if (this.filename && options.nodeWebkitAppName) {
+    console.log("==================================================================");
+    console.log("WARNING: The nodeWebkitAppName option is deprecated");
+    console.log("To get the path to the directory where Node Webkit stores the data");
+    console.log("for your app, use the internal nw.gui module like this");
+    console.log("require('nw.gui').App.dataPath");
+    console.log("See https://github.com/rogerwang/node-webkit/issues/500");
+    console.log("==================================================================");
+    this.filename = Persistence.getNWAppFilename(options.nodeWebkitAppName, this.filename);
+  }
+};
+
+
+/**
+ * Check if a directory exists and create it on the fly if it is not the case
+ * cb is optional, signature: err
+ */
+Persistence.ensureDirectoryExists = function (dir, cb) {
+  var callback = cb || function () {}
+    ;
+
+  storage.mkdirp(dir).then(function() { return callback(); }).catch(callback);
+};
+
+
+
+
+/**
+ * Return the path the datafile if the given filename is relative to the directory where Node Webkit stores
+ * data for this application. Probably the best place to store data
+ */
+Persistence.getNWAppFilename = function (appName, relativeFilename) {
+  var home;
+
+  switch (process.platform) {
+    case 'win32':
+    case 'win64':
+      home = process.env.LOCALAPPDATA || process.env.APPDATA;
+      if (!home) { throw new Error("Couldn't find the base application data folder"); }
+      home = path.join(home, appName);
+      break;
+    case 'darwin':
+      home = process.env.HOME;
+      if (!home) { throw new Error("Couldn't find the base application data directory"); }
+      home = path.join(home, 'Library', 'Application Support', appName);
+      break;
+    case 'linux':
+      home = process.env.HOME;
+      if (!home) { throw new Error("Couldn't find the base application data directory"); }
+      home = path.join(home, '.config', appName);
+      break;
+    default:
+      throw new Error("Can't use the Node Webkit relative path for platform " + process.platform);
+      break;
+  }
+
+  return path.join(home, 'nedb-data', relativeFilename);
+}
+
+
+/**
+ * Persist cached database
+ * This serves as a compaction function since the cache always contains only the number of documents in the collection
+ * while the data file is append-only so it may grow larger
+ * @param {Function} cb Optional callback, signature: err
+ */
+Persistence.prototype.persistCachedDatabase = function (cb) {
+  var callback = cb || function () {}
+    , toPersist = ''
+    , self = this
+    ;
+
+  if (this.inMemoryOnly) { return callback(null); }
+
+  this.db.getAllData().forEach(function (doc) {
+    toPersist += self.afterSerialization(model.serialize(doc)) + '\n';
+  });
+  Object.keys(this.db.indexes).forEach(function (fieldName) {
+    if (fieldName != "_id") {   // The special _id index is managed by datastore.js, the others need to be persisted
+      toPersist += self.afterSerialization(model.serialize({ $$indexCreated: { fieldName: fieldName, unique: self.db.indexes[fieldName].unique, sparse: self.db.indexes[fieldName].sparse }})) + '\n';
+    }
+  });
+
+  storage.crashSafeWriteFile(this.filename, toPersist, function (err) {
+    if (err) { return callback(err); }
+    self.db.emit('compaction.done');
+    return callback(null);
+  });
+};
+
+
+/**
+ * Queue a rewrite of the datafile
+ */
+Persistence.prototype.compactDatafile = function () {
+  this.db.executor.push({ this: this, fn: this.persistCachedDatabase, arguments: [] });
+};
+
+
+/**
+ * Set automatic compaction every interval ms
+ * @param {Number} interval in milliseconds, with an enforced minimum of 5 seconds
+ */
+Persistence.prototype.setAutocompactionInterval = function (interval) {
+  var self = this
+    , minInterval = 5000
+    , realInterval = Math.max(interval || 0, minInterval)
+    ;
+
+  this.stopAutocompaction();
+
+  this.autocompactionIntervalId = setInterval(function () {
+    self.compactDatafile();
+  }, realInterval);
+};
+
+
+/**
+ * Stop autocompaction (do nothing if autocompaction was not running)
+ */
+Persistence.prototype.stopAutocompaction = function () {
+  if (this.autocompactionIntervalId) { clearInterval(this.autocompactionIntervalId); }
+};
+
+
+/**
+ * Persist new state for the given newDocs (can be insertion, update or removal)
+ * Use an append-only format
+ * @param {Array} newDocs Can be empty if no doc was updated/removed
+ * @param {Function} cb Optional, signature: err
+ */
+Persistence.prototype.persistNewState = function (newDocs, cb) {
+  var self = this
+    , toPersist = ''
+    , callback = cb || function () {}
+    ;
+
+  // In-memory only datastore
+  if (self.inMemoryOnly) { return callback(null); }
+
+  newDocs.forEach(function (doc) {
+    toPersist += self.afterSerialization(model.serialize(doc)) + '\n';
+  });
+
+  if (toPersist.length === 0) { return callback(null); }
+
+  storage.appendFile(self.filename, toPersist, 'utf8', function (err) {
+    return callback(err);
+  });
+};
+
+
+/**
+ * From a database's raw data, return the corresponding
+ * machine understandable collection
+ */
+Persistence.prototype.treatRawData = function (rawData) {
+  var data = rawData.split('\n')
+    , dataById = {}
+    , tdata = []
+    , i
+    , indexes = {}
+    , corruptItems = -1   // Last line of every data file is usually blank so not really corrupt
+    ;
+
+  for (i = 0; i < data.length; i += 1) {
+    var doc;
+
+    try {
+      doc = model.deserialize(this.beforeDeserialization(data[i]));
+      if (doc._id) {
+        if (doc.$$deleted === true) {
+          delete dataById[doc._id];
+        } else {
+          dataById[doc._id] = doc;
+        }
+      } else if (doc.$$indexCreated && doc.$$indexCreated.fieldName != undefined) {
+        indexes[doc.$$indexCreated.fieldName] = doc.$$indexCreated;
+      } else if (typeof doc.$$indexRemoved === "string") {
+        delete indexes[doc.$$indexRemoved];
+      }
+    } catch (e) {
+      corruptItems += 1;
+    }
+  }
+
+  // A bit lenient on corruption
+  if (data.length > 0 && corruptItems / data.length > this.corruptAlertThreshold) {
+    throw new Error("More than " + Math.floor(100 * this.corruptAlertThreshold) + "% of the data file is corrupt, the wrong beforeDeserialization hook may be used. Cautiously refusing to start NeDB to prevent dataloss");
+  }
+
+  Object.keys(dataById).forEach(function (k) {
+    tdata.push(dataById[k]);
+  });
+
+  return { data: tdata, indexes: indexes };
+};
+
+
+/**
+ * Load the database
+ * 1) Create all indexes
+ * 2) Insert all data
+ * 3) Compact the database
+ * This means pulling data out of the data file or creating it if it doesn't exist
+ * Also, all data is persisted right away, which has the effect of compacting the database file
+ * This operation is very quick at startup for a big collection (60ms for ~10k docs)
+ * @param {Function} cb Optional callback, signature: err
+ */
+Persistence.prototype.loadDatabase = function (cb) {
+  var callback = cb || function () {}
+    , self = this
+    ;
+
+  self.db.resetIndexes();
+
+  // In-memory only datastore
+  if (self.inMemoryOnly) { return callback(null); }
+
+  async.waterfall([
+    function (cb) {
+      Persistence.ensureDirectoryExists(path.dirname(self.filename), function (err) {
+        storage.ensureDatafileIntegrity(self.filename, function (err) {
+          storage.readFile(self.filename, 'utf8', function (err, rawData) {
+            if (err) { return cb(err); }
+
+            try {
+              var treatedData = self.treatRawData(rawData);
+            } catch (e) {
+              return cb(e);
+            }
+
+            // Recreate all indexes in the datafile
+            Object.keys(treatedData.indexes).forEach(function (key) {
+              self.db.indexes[key] = new Index(treatedData.indexes[key]);
+            });
+
+            // Fill cached database (i.e. all indexes) with data
+            try {
+              self.db.resetIndexes(treatedData.data);
+            } catch (e) {
+              self.db.resetIndexes();   // Rollback any index which didn't fail
+              return cb(e);
+            }
+
+            self.db.persistence.persistCachedDatabase(cb);
+          });
+        });
+      });
+    }
+  ], function (err) {
+       if (err) { return callback(err); }
+
+       self.db.executor.processBuffer();
+       return callback(null);
+     });
+};
+
+
+// Interface
+module.exports = Persistence;

package/lib/storage.js

@@ -0,0 +1,136 @@
+/**
+ * Way data is stored for this database
+ * For a Node.js/Node Webkit database it's the file system
+ * For a browser-side database it's localforage which chooses the best option depending on user browser (IndexedDB then WebSQL then localStorage)
+ *
+ * This version is the Node.js/Node Webkit version
+ * It's essentially fs, mkdirp and crash safe write and read functions
+ */
+
+var fs = require('fs')
+  , mkdirp = require('mkdirp')
+  , async = require('./async')
+  , path = require('path')
+  , storage = {}
+  ;
+
+storage.exists = fs.exists;
+storage.rename = fs.rename;
+storage.writeFile = fs.writeFile;
+storage.unlink = fs.unlink;
+storage.appendFile = fs.appendFile;
+storage.readFile = fs.readFile;
+storage.mkdirp = mkdirp;
+
+
+/**
+ * Explicit name ...
+ */
+storage.ensureFileDoesntExist = function (file, callback) {
+  storage.exists(file, function (exists) {
+    if (!exists) { return callback(null); }
+
+    storage.unlink(file, function (err) { return callback(err); });
+  });
+};
+
+
+/**
+ * Flush data in OS buffer to storage if corresponding option is set
+ * @param {String} options.filename
+ * @param {Boolean} options.isDir Optional, defaults to false
+ * If options is a string, it is assumed that the flush of the file (not dir) called options was requested
+ */
+storage.flushToStorage = function (options, callback) {
+  var filename, flags;
+  if (typeof options === 'string') {
+    filename = options;
+    flags = 'r+';
+  } else {
+    filename = options.filename;
+    flags = options.isDir ? 'r' : 'r+';
+  }
+
+  // Windows can't fsync (FlushFileBuffers) directories. We can live with this as it cannot cause 100% dataloss
+  // except in the very rare event of the first time database is loaded and a crash happens
+  if (flags === 'r' && (process.platform === 'win32' || process.platform === 'win64')) { return callback(null); }
+
+  fs.open(filename, flags, function (err, fd) {
+    if (err) { return callback(err); }
+    fs.fsync(fd, function (errFS) {
+      fs.close(fd, function (errC) {
+        if (errFS || errC) {
+          var e = new Error('Failed to flush to storage');
+          e.errorOnFsync = errFS;
+          e.errorOnClose = errC;
+          return callback(e);
+        } else {
+          return callback(null);
+        }
+      });
+    });
+  });
+};
+
+
+/**
+ * Fully write or rewrite the datafile, immune to crashes during the write operation (data will not be lost)
+ * @param {String} filename
+ * @param {String} data
+ * @param {Function} cb Optional callback, signature: err
+ */
+storage.crashSafeWriteFile = function (filename, data, cb) {
+  var callback = cb || function () {}
+    , tempFilename = filename + '~';
+
+  async.waterfall([
+    async.apply(storage.flushToStorage, { filename: path.dirname(filename), isDir: true })
+  , function (cb) {
+      storage.exists(filename, function (exists) {
+        if (exists) {
+          storage.flushToStorage(filename, function (err) { return cb(err); });
+        } else {
+          return cb();
+        }
+      });
+    }
+  , function (cb) {
+      storage.writeFile(tempFilename, data, function (err) { return cb(err); });
+    }
+  , async.apply(storage.flushToStorage, tempFilename)
+  , function (cb) {
+      storage.rename(tempFilename, filename, function (err) { return cb(err); });
+    }
+  , async.apply(storage.flushToStorage, { filename: path.dirname(filename), isDir: true })
+  ], function (err) { return callback(err); })
+};
+
+
+/**
+ * Ensure the datafile contains all the data, even if there was a crash during a full file write
+ * @param {String} filename
+ * @param {Function} callback signature: err
+ */
+storage.ensureDatafileIntegrity = function (filename, callback) {
+  var tempFilename = filename + '~';
+
+  storage.exists(filename, function (filenameExists) {
+    // Write was successful
+    if (filenameExists) { return callback(null); }
+
+    storage.exists(tempFilename, function (oldFilenameExists) {
+      // New database
+      if (!oldFilenameExists) {
+        return storage.writeFile(filename, '', 'utf8', function (err) { callback(err); });
+      }
+
+      // Write failed, use old version
+      storage.rename(tempFilename, filename, function (err) { return callback(err); });
+    });
+  });
+};
+
+
+
+// Interface
+module.exports = storage;

package/lib/underscore.js

@@ -0,0 +1,177 @@
+/**
+ * Minimal underscore.js replacement with only the functions needed by nedb
+ * This replaces the full underscore library to reduce dependencies
+ */
+
+/**
+ * Returns the intersection of arrays - elements that appear in all arrays
+ * @param {Array} array - First array
+ * @param {...Array} arrays - Additional arrays to intersect with
+ * @returns {Array} - Array of elements that appear in all input arrays
+ */
+function intersection(array) {
+  if (!Array.isArray(array)) return [];
+  
+  var result = [];
+  var argsLength = arguments.length;
+  
+  for (var i = 0; i < array.length; i++) {
+    var item = array[i];
+    var included = true;
+    
+    // Check if item exists in all other arrays
+    for (var j = 1; j < argsLength; j++) {
+      var otherArray = arguments[j];
+      if (!Array.isArray(otherArray) || otherArray.indexOf(item) === -1) {
+        included = false;
+        break;
+      }
+    }
+    
+    // Add to result if not already there and exists in all arrays
+    if (included && result.indexOf(item) === -1) {
+      result.push(item);
+    }
+  }
+  
+  return result;
+}
+
+/**
+ * Extracts property values from an array of objects
+ * @param {Array} array - Array of objects
+ * @param {String} property - Property name to extract
+ * @returns {Array} - Array of extracted property values
+ */
+function pluck(array, property) {
+  if (!Array.isArray(array)) return [];
+  
+  var result = [];
+  for (var i = 0; i < array.length; i++) {
+    if (array[i] && array[i].hasOwnProperty(property)) {
+      result.push(array[i][property]);
+    }
+  }
+  return result;
+}
+
+/**
+ * Returns a copy of object with specified keys omitted
+ * @param {Object} obj - Source object
+ * @param {...String} keys - Keys to omit
+ * @returns {Object} - New object without the omitted keys
+ */
+function omit(obj) {
+  if (typeof obj !== 'object' || obj === null) return {};
+  
+  var result = {};
+  var keysToOmit = Array.prototype.slice.call(arguments, 1);
+  
+  for (var key in obj) {
+    if (obj.hasOwnProperty(key) && keysToOmit.indexOf(key) === -1) {
+      result[key] = obj[key];
+    }
+  }
+  
+  return result;
+}
+
+/**
+ * Checks if object has the given key as a direct property
+ * @param {Object} obj - Object to check
+ * @param {String} key - Key to check for
+ * @returns {Boolean} - True if object has the key
+ */
+function has(obj, key) {
+  return obj != null && Object.prototype.hasOwnProperty.call(obj, key);
+}
+
+/**
+ * Creates a new array with results of calling provided function on every element
+ * @param {Array} array - Array to map over
+ * @param {Function} iteratee - Function to call for each element
+ * @returns {Array} - New array with mapped values
+ */
+function map(array, iteratee) {
+  if (!Array.isArray(array) || typeof iteratee !== 'function') return [];
+  
+  var result = [];
+  for (var i = 0; i < array.length; i++) {
+    result.push(iteratee(array[i], i, array));
+  }
+  return result;
+}
+
+/**
+ * Creates a new array with all elements that pass the test implemented by provided function
+ * @param {Array} array - Array to filter
+ * @param {Function} predicate - Function to test each element
+ * @returns {Array} - New array with filtered elements
+ */
+function filter(array, predicate) {
+  if (!Array.isArray(array) || typeof predicate !== 'function') return [];
+  
+  var result = [];
+  for (var i = 0; i < array.length; i++) {
+    if (predicate(array[i], i, array)) {
+      result.push(array[i]);
+    }
+  }
+  return result;
+}
+
+/**
+ * Creates a duplicate-free version of an array
+ * @param {Array} array - Array to make unique
+ * @param {Function} iteratee - Optional function to compute uniqueness criterion
+ * @returns {Array} - New array with unique values
+ */
+function uniq(array, iteratee) {
+  if (!Array.isArray(array)) return [];
+  
+  var result = [];
+  var seen = [];
+  
+  for (var i = 0; i < array.length; i++) {
+    var value = array[i];
+    var computed = iteratee ? iteratee(value) : value;
+    
+    if (seen.indexOf(computed) === -1) {
+      seen.push(computed);
+      result.push(value);
+    }
+  }
+  
+  return result;
+}
+
+/**
+ * Checks if value is a function
+ * @param {*} value - Value to check
+ * @returns {Boolean} - True if value is a function
+ */
+function isFunction(value) {
+  return typeof value === 'function';
+}
+
+/**
+ * Checks if value is a boolean
+ * @param {*} value - Value to check  
+ * @returns {Boolean} - True if value is a boolean
+ */
+function isBoolean(value) {
+  return typeof value === 'boolean';
+}
+
+// Export the functions
+module.exports = {
+  intersection: intersection,
+  pluck: pluck,
+  omit: omit,
+  has: has,
+  map: map,
+  filter: filter,
+  uniq: uniq,
+  isFunction: isFunction,
+  isBoolean: isBoolean
+};

package/package.json

@@ -0,0 +1,52 @@
+{
+  "name": "@electerm/nedb",
+  "version": "1.8.0",
+  "author": "Louis Chatriot <louis.chatriot@gmail.com>",
+  "contributors": [
+    "Louis Chatriot"
+  ],
+  "description": "File-based embedded data store for node.js",
+  "keywords": [
+    "database",
+    "datastore",
+    "embedded"
+  ],
+  "homepage": "https://github.com/electerm/nedb",
+  "repository": {
+    "type": "git",
+    "url": "git+ssh://git@github.com/electerm/nedb.git"
+  },
+  "dependencies": {
+    "@yetzt/binary-search-tree": "^0.2.6",
+    "mkdirp": "^1.0.4"
+  },
+  "devDependencies": {
+    "async": "^3.2.0",
+    "chai": "^4.3.4",
+    "commander": "^7.2.0",
+    "exec-time": "^0.0.4",
+    "mocha": "^8.4.0",
+    "request": "^2.88.2",
+    "sinon": "^10.0.0",
+    "underscore": "^1.13.7"
+  },
+  "scripts": {
+    "test": "mocha --reporter spec --timeout 1000",
+    "pub": "npm publish --access public"
+  },
+  "files": [
+    "lib/",
+    "index.js",
+    "LICENSE",
+    "README.md"
+  ],
+  "main": "index",
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/electerm/nedb/issues"
+  },
+  "directories": {
+    "lib": "lib",
+    "test": "test"
+  }
+}