@harperfast/harper 5.0.0-alpha.10 → 5.0.0-beta.3

package/utility/lmdb/DeleteRecordsResponseObject.js

@@ -0,0 +1,25 @@
+'use strict';
+
+/**
+ * Response object from lmdb delete function
+ * @param {Array.<string|number>} deleted
+ * @param {Array.<string|number>} skipped
+ * @param {number} txnTime
+ * @param {Array.<Object>} originalRecords
+ */
+class DeleteRecordsResponseObject {
+	/**
+	 * @param {Array.<string|number>} deleted
+	 * @param {Array.<string|number>} skipped
+	 * @param {number} txnTime
+	 * @param {Array.<Object>} originalRecords
+	 */
+	constructor(deleted = [], skipped = [], txnTime = undefined, originalRecords = []) {
+		this.deleted = deleted;
+		this.skipped = skipped;
+		this.txn_time = txnTime;
+		this.original_records = originalRecords;
+	}
+}
+
+module.exports = DeleteRecordsResponseObject;

package/utility/lmdb/InsertRecordsResponseObject.js

@@ -0,0 +1,22 @@
+'use strict';
+
+/**
+ * Response object from lmdb insert function
+ * @param {Array.<string|number>} written_hashes
+ * @param {Array.<string|number>} skipped_hashes
+ * @param {number} txnTime
+ */
+class InsertRecordsResponseObject {
+	/**
+	 * @param {Array.<string|number>} written_hashes
+	 * @param {Array.<string|number>} skipped_hashes
+	 * @param {number} txnTime
+	 */
+	constructor(written_hashes = [], skipped_hashes = [], txnTime = undefined) {
+		this.written_hashes = written_hashes;
+		this.skipped_hashes = skipped_hashes;
+		this.txn_time = txnTime;
+	}
+}
+
+module.exports = InsertRecordsResponseObject;

package/utility/lmdb/OpenDBIObject.js

@@ -0,0 +1,31 @@
+'use strict';
+const envMngr = require('../environment/environmentManager.js');
+const terms = require('../../utility/hdbTerms.ts');
+const { RecordEncoder } = require('../../resources/RecordEncoder.ts');
+envMngr.initSync();
+
+const LMDB_CACHING = envMngr.get(terms.CONFIG_PARAMS.STORAGE_CACHING) !== false;
+
+/**
+ * Defines how a DBI will be created/opened
+ */
+class OpenDBIObject {
+	/**
+	 * @param {Boolean} dupSort - if the dbi allows duplicate keys
+	 * @param {Boolean} useVersions - if the dbi uses versions
+	 */
+	constructor(dupSort, isPrimary = false) {
+		this.dupSort = dupSort === true;
+		this.encoding = dupSort ? 'ordered-binary' : 'msgpack';
+		this.useVersions = isPrimary;
+		this.sharedStructuresKey = Symbol.for('structures');
+		if (isPrimary) {
+			this.cache = LMDB_CACHING && { validated: true };
+			this.randomAccessStructure = true;
+			this.freezeData = true;
+			this.encoder = { Encoder: RecordEncoder };
+		}
+	}
+}
+
+exports.OpenDBIObject = OpenDBIObject;

package/utility/lmdb/OpenEnvironmentObject.js

@@ -0,0 +1,41 @@
+'use strict';
+//Set initial map size to 1Gb
+// eslint-disable-next-line no-magic-numbers
+const MAP_SIZE = 1024 * 1024 * 1024;
+//allow up to 1,000 named data bases in an environment
+const MAX_DBS = 10000;
+const MAX_READERS = 2048;
+const envMngr = require('../environment/environmentManager.js');
+const terms = require('../../utility/hdbTerms.ts');
+envMngr.initSync();
+
+class OpenEnvironmentObject {
+	constructor(path, readOnly = false) {
+		this.path = path;
+		this.mapSize = MAP_SIZE;
+		this.maxDbs = MAX_DBS;
+		this.maxReaders = MAX_READERS;
+		this.sharedStructuresKey = Symbol.for('structures');
+		this.readOnly = readOnly;
+		this.trackMetrics = true;
+		this.eventTurnBatching = false; // event turn batching is not needed in Harper
+		this.noSync =
+			envMngr.get(terms.CONFIG_PARAMS.STORAGE_WRITEASYNC) === true ||
+			envMngr.get(terms.CONFIG_PARAMS.STORAGE_WRITEASYNC) === 'true' ||
+			envMngr.get(terms.CONFIG_PARAMS.STORAGE_WRITEASYNC) === 'TRUE';
+		//this.noFSAccess = true; // we might re-enable this if we want secure JS environments
+		// otherwise overlappingSync uses lmdb-js default, which is enabled on linux/mac, disabled on windows
+		if (envMngr.get(terms.CONFIG_PARAMS.STORAGE_OVERLAPPINGSYNC) !== undefined)
+			this.overlappingSync = envMngr.get(terms.CONFIG_PARAMS.STORAGE_OVERLAPPINGSYNC);
+		if (envMngr.get(terms.CONFIG_PARAMS.STORAGE_MAXFREESPACETOLOAD))
+			this.maxFreeSpaceToLoad = envMngr.get(terms.CONFIG_PARAMS.STORAGE_MAXFREESPACETOLOAD);
+		if (envMngr.get(terms.CONFIG_PARAMS.STORAGE_MAXFREESPACETORETAIN))
+			this.maxFreeSpaceToRetain = envMngr.get(terms.CONFIG_PARAMS.STORAGE_MAXFREESPACETORETAIN);
+		if (envMngr.get(terms.CONFIG_PARAMS.STORAGE_PAGESIZE))
+			this.pageSize = envMngr.get(terms.CONFIG_PARAMS.STORAGE_PAGESIZE);
+		this.noReadAhead = envMngr.get(terms.CONFIG_PARAMS.STORAGE_NOREADAHEAD);
+	}
+}
+
+module.exports = OpenEnvironmentObject;
+OpenEnvironmentObject.MAX_DBS = MAX_DBS;

package/utility/lmdb/UpdateRecordsResponseObject.js

@@ -0,0 +1,25 @@
+'use strict';
+
+/**
+ * Response object from lmdb update function
+ * @param {Array.<string|number>} written_hashes
+ * @param {Array.<string|number>} skipped_hashes
+ * @param {number} txnTime
+ * @param {Array.<Object>} originalRecords
+ */
+class UpdateRecordsResponseObject {
+	/**
+	 * @param {Array.<string|number>} written_hashes
+	 * @param {Array.<string|number>} skipped_hashes
+	 * @param {number} txnTime
+	 * @param {Array.<Object>} originalRecords
+	 */
+	constructor(written_hashes = [], skipped_hashes = [], txnTime = undefined, originalRecords = []) {
+		this.written_hashes = written_hashes;
+		this.skipped_hashes = skipped_hashes;
+		this.txn_time = txnTime;
+		this.original_records = originalRecords;
+	}
+}
+
+module.exports = UpdateRecordsResponseObject;

package/utility/lmdb/UpsertRecordsResponseObject.js

@@ -0,0 +1,22 @@
+'use strict';
+
+/**
+ * Response object from lmdb upsert function
+ * @param {Array.<string|number>} written_hashes
+ * @param {number} txnTime
+ * @param {Array.<Object>} originalRecords
+ */
+class UpsertRecordsResponseObject {
+	/**
+	 * @param {Array.<string|number>} written_hashes
+	 * @param {number} txnTime
+	 * @param {Array.<Object>} originalRecords
+	 */
+	constructor(written_hashes = [], txnTime = undefined, originalRecords = []) {
+		this.written_hashes = written_hashes;
+		this.txn_time = txnTime;
+		this.original_records = originalRecords;
+	}
+}
+
+module.exports = UpsertRecordsResponseObject;

package/utility/lmdb/cleanLMDBMap.js

@@ -0,0 +1,65 @@
+'use strict';
+
+const environmentUtility = require('./environmentUtility.js');
+const harperLogger = require('../logging/harper_logger.js');
+const LMDB_ERRORS = require('../errors/commonErrors.js').LMDB_ERRORS_ENUM;
+
+module.exports = cleanLMDBMap;
+
+/**
+ * this function strips away the cached environments from global when a schema item is removed
+ * @param msg
+ */
+async function cleanLMDBMap(msg) {
+	try {
+		if (global.lmdb_map !== undefined && msg.operation !== undefined) {
+			let keys = Object.keys(global.lmdb_map);
+			let cachedEnvironment = undefined;
+
+			switch (msg.operation) {
+				case 'drop_schema':
+					for (let x = 0; x < keys.length; x++) {
+						let key = keys[x];
+						if (key.startsWith(`${msg.schema}.`) || key.startsWith(`txn.${msg.schema}.`)) {
+							try {
+								await environmentUtility.closeEnvironment(global.lmdb_map[key]);
+							} catch (err) {
+								if (err.message !== LMDB_ERRORS.ENV_REQUIRED) {
+									throw err;
+								}
+							}
+						}
+					}
+					break;
+				case 'drop_table':
+					// eslint-disable-next-line no-case-declarations
+					let schemaTableName = `${msg.schema}.${msg.table}`;
+					// eslint-disable-next-line no-case-declarations
+					let txnSchemaTableName = `txn.${schemaTableName}`;
+					try {
+						await environmentUtility.closeEnvironment(global.lmdb_map[schemaTableName]);
+						await environmentUtility.closeEnvironment(global.lmdb_map[txnSchemaTableName]);
+					} catch (err) {
+						if (err.message !== LMDB_ERRORS.ENV_REQUIRED) {
+							throw err;
+						}
+					}
+					break;
+				case 'drop_attribute':
+					cachedEnvironment = global.lmdb_map[`${msg.schema}.${msg.table}`];
+					if (
+						cachedEnvironment !== undefined &&
+						typeof cachedEnvironment.dbis === 'object' &&
+						cachedEnvironment.dbis[`${msg.attribute}`] !== undefined
+					) {
+						delete cachedEnvironment.dbis[`${msg.attribute}`];
+					}
+					break;
+				default:
+					break;
+			}
+		}
+	} catch (e) {
+		harperLogger.error(e);
+	}
+}

package/utility/lmdb/commonUtility.js

@@ -0,0 +1,130 @@
+'use strict';
+
+const LMDB_ERRORS = require('../errors/commonErrors.js').LMDB_ERRORS_ENUM;
+// eslint-disable-next-line no-unused-vars
+const lmdb = require('lmdb');
+const lmdbTerms = require('./terms.js');
+
+const { OVERFLOW_MARKER, MAX_SEARCH_KEY_LENGTH } = lmdbTerms;
+const PRIMITIVES = ['number', 'string', 'symbol', 'boolean', 'bigint'];
+/**
+ * validates the env argument
+ * @param {lmdb.Transaction|lmdb.RootDatabase} env - environment object used thigh level to interact with all data in an
+ * environment
+ */
+function validateEnv(env) {
+	env = env?.primaryStore || env;
+	if (!env) {
+		throw new Error(LMDB_ERRORS.ENV_REQUIRED);
+	}
+}
+
+/**
+ * converts raw data to it's string version
+ * @param rawValue
+ * @returns {Number|String|null}
+ */
+function stringifyData(rawValue) {
+	if (rawValue === null || rawValue === undefined) {
+		return null;
+	}
+
+	let value;
+
+	try {
+		value = typeof rawValue === 'object' ? JSON.stringify(rawValue) : rawValue.toString();
+	} catch {
+		value = rawValue.toString();
+	}
+
+	return value;
+}
+
+/**
+ * takes a raw value and converts it to be written to LMDB. lmdb-store accepts primitives ('number', 'string', 'symbol', 'boolean', 'bigint', buffer) and array of primitives as keys.
+ * if it is anything else we convert to string
+ * @param {*} key - raw value which needs to be converted
+ * @returns {*}
+ */
+function convertKeyValueToWrite(key) {
+	//if this is a primitive return the value
+	if (key instanceof Date) {
+		return key.valueOf();
+	}
+	return key;
+}
+
+/**
+ * Return all the indexable values from an attribute, ready to be indexed
+ */
+function getIndexedValues(value, indexNulls) {
+	if (value === null) {
+		return indexNulls ? [null] : undefined;
+	}
+	if (value === undefined) {
+		return undefined;
+	}
+	if (PRIMITIVES.includes(typeof value)) {
+		if (value.length > MAX_SEARCH_KEY_LENGTH) {
+			return [value.slice(0, MAX_SEARCH_KEY_LENGTH) + OVERFLOW_MARKER];
+		}
+		return [value];
+	}
+	if (Array.isArray(value)) {
+		const values = [];
+		for (let i = 0, l = value.length; i < l; i++) {
+			let element = value[i];
+			if (PRIMITIVES.includes(typeof element)) {
+				if (element.length > MAX_SEARCH_KEY_LENGTH) {
+					values.push(element.slice(0, MAX_SEARCH_KEY_LENGTH) + OVERFLOW_MARKER);
+				} else {
+					values.push(element);
+				}
+			} else if (element === null && indexNulls) {
+				values.push(null);
+			} else if (element instanceof Date) {
+				values.push(element.getTime());
+			}
+		}
+		return values;
+	} else if (value instanceof Date) {
+		return [value.getTime()];
+	}
+	return undefined;
+}
+
+let lastTime = 0; // reported time used to ensure monotonic time.
+let startTime = 0; // the start time of the (current time relative to performance time counter)
+function adjustStartTime() {
+	// calculate the start time
+	// TODO: We may actually want to implement a gradual time shift if the clock time really changes substantially
+	// and for sub-millisecond updates, may want to average them so we can progressively narrow in on true time
+	startTime = Date.now() - performance.now();
+}
+adjustStartTime();
+// we periodically update our start time because clock time can drift (but we still ensure monotonic time)
+const TIME_ADJUSTMENT_INTERVAL = 60000;
+setInterval(adjustStartTime, TIME_ADJUSTMENT_INTERVAL).unref();
+/**
+ * A monotonic timestamp that is guaranteed to be higher than the last call to this function.
+ * Will use decimal microseconds as necessary to differentiate from previous calls without too much drift.
+ */
+function getNextMonotonicTime() {
+	let now = performance.now() + startTime;
+	if (now > lastTime) {
+		// current time is higher than last time, can safely return it
+		lastTime = now;
+		return now;
+	}
+	// otherwise, we MUST return a higher time than last time, so we increase the time and return it.
+	// increment by as small of count as possible, to minimize how far we are from clock time
+	lastTime += 0.000488;
+	return lastTime;
+}
+module.exports = {
+	validateEnv,
+	stringifyData,
+	convertKeyValueToWrite,
+	getNextMonotonicTime,
+	getIndexedValues,
+};

package/utility/lmdb/deleteUtility.js

@@ -0,0 +1,128 @@
+'use strict';
+
+const environmentUtil = require('./environmentUtility.js');
+const common = require('./commonUtility.js');
+const LMDB_ERRORS = require('../errors/commonErrors.js').LMDB_ERRORS_ENUM;
+const log = require('../logging/harper_logger.js');
+// eslint-disable-next-line no-unused-vars
+const lmdb = require('lmdb');
+const DeleteRecordsResponseObject = require('./DeleteRecordsResponseObject.js');
+const hdbTerms = require('../hdbTerms.ts');
+const UPDATED_TIME_ATTRIBUTE_NAME = hdbTerms.TIME_STAMP_NAMES_ENUM.UPDATED_TIME;
+
+/**
+ *  deletes rows and their entries in all indices
+ * @param {lmdb.RootDatabase} env - environment object used high level to interact with all data in an environment
+ * @param {String} hash_attribute - name of the hash_attribute for this environment
+ * @param {Array.<String>} ids - list of ids to delete
+ * @param {number} whenDeleted - The timestamp of the deletion
+ * @returns {Promise<DeleteRecordsResponseObject>}
+ */
+async function deleteRecords(env, hash_attribute, ids, whenDeleted) {
+	//validate
+	common.validateEnv(env);
+
+	if (hash_attribute === undefined) {
+		throw new Error(LMDB_ERRORS.HASH_ATTRIBUTE_REQUIRED);
+	}
+
+	if (!Array.isArray(ids)) {
+		if (ids === undefined) {
+			throw new Error(LMDB_ERRORS.IDS_REQUIRED);
+		}
+
+		throw new Error(LMDB_ERRORS.IDS_MUST_BE_ITERABLE);
+	}
+
+	try {
+		//open all dbis for this env
+		let allDbis = environmentUtil.listDBIs(env);
+		environmentUtil.initializeDBIs(env, hash_attribute, allDbis);
+		let deleted = new DeleteRecordsResponseObject();
+
+		//iterate records and process deletes
+		let hashValue;
+		let puts = [];
+		let keys = [];
+		for (let x = 0, length = ids.length; x < length; x++) {
+			try {
+				hashValue = ids[x];
+
+				//attempt to fetch the hash attribute value, this is the row.
+				let record = env.dbis[hash_attribute].get(hashValue);
+				//if it doesn't exist we skip & move to the next id
+				if (
+					!record ||
+					// of if the deletion timestamp is older than the current record, last-write wins
+					(whenDeleted && record[UPDATED_TIME_ATTRIBUTE_NAME] > whenDeleted)
+				) {
+					deleted.skipped.push(hashValue);
+					continue;
+				}
+
+				let promise = env.dbis[hash_attribute].ifVersion(hashValue, lmdb.IF_EXISTS, () => {
+					//always just delete the hash_attribute entry upfront
+					env.dbis[hash_attribute].remove(hashValue);
+
+					//iterate & delete the non-hash attribute entries
+					for (let y = 0; y < allDbis.length; y++) {
+						let attribute = allDbis[y];
+						if (!record.hasOwnProperty(attribute) || attribute === hash_attribute) {
+							continue;
+						}
+
+						let dbi = env.dbis[attribute];
+						let value = record[attribute];
+						if (value !== null && value !== undefined) {
+							try {
+								let values = common.getIndexedValues(value);
+								if (values) {
+									for (let i = 0, l = values.length; i < l; i++) {
+										dbi.remove(values[i], hashValue);
+									}
+								}
+							} catch {
+								log.warn(`cannot delete from attribute: ${attribute}, ${value}:${hashValue}`);
+							}
+						}
+					}
+				});
+				puts.push(promise);
+				keys.push(hashValue);
+				deleted.original_records.push(record);
+			} catch (e) {
+				log.warn(e);
+				deleted.skipped.push(hashValue);
+			}
+		}
+
+		let removeIndices = [];
+		let putResults = await Promise.all(puts);
+		for (let x = 0, length = putResults.length; x < length; x++) {
+			if (putResults[x] === true) {
+				deleted.deleted.push(keys[x]);
+			} else {
+				deleted.skipped.push(keys[x]);
+				removeIndices.push(x);
+			}
+		}
+
+		let offset = 0;
+		for (let x = 0; x < removeIndices.length; x++) {
+			let index = removeIndices[x];
+			deleted.original_records.splice(index - offset, 1);
+			//the offset needs to increase for every index we remove
+			offset++;
+		}
+
+		deleted.txn_time = common.getNextMonotonicTime();
+
+		return deleted;
+	} catch (e) {
+		throw e;
+	}
+}
+
+module.exports = {
+	deleteRecords,
+};