@harperfast/harper 5.0.0-alpha.10 → 5.0.0-beta.1

package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteRecords.js

@@ -0,0 +1,89 @@
+'use strict';
+
+const hdbUtils = require('../../../../utility/common_utils.js');
+const deleteUtility = require('../../../../utility/lmdb/deleteUtility.js');
+const environmentUtility = require('../../../../utility/lmdb/environmentUtility.js');
+const { getSchemaPath } = require('../lmdbUtility/initializePaths.js');
+const writeTransaction = require('../lmdbUtility/lmdbWriteTransaction.js');
+const logger = require('../../../../utility/logging/harper_logger.js');
+
+module.exports = lmdbDeleteRecords;
+
+/**
+ * Deletes a full table row at a certain hash.
+ * @param deleteObj
+ * @param writeToTxnLog {boolean}
+ */
+async function lmdbDeleteRecords(deleteObj, writeToTxnLog = true) {
+	let schemaTable = global.hdb_schema[deleteObj.schema][deleteObj.table];
+	let hash_attribute = schemaTable.hash_attribute;
+	if (hdbUtils.isEmpty(hash_attribute)) {
+		throw new Error(`could not retrieve hash attribute for schema:${deleteObj.schema} and table ${deleteObj.table}`);
+	}
+
+	//this would happen for SQL delete
+	if (hdbUtils.isEmptyOrZeroLength(deleteObj.hash_values) && !hdbUtils.isEmptyOrZeroLength(deleteObj.records)) {
+		//reintitialize hash_values since it is empty we are not sure if the variable has been set to empty array yet
+		deleteObj.hash_values = [];
+		for (let k = 0; k < deleteObj.records.length; k++) {
+			let hashValue = deleteObj.records[k][hash_attribute];
+			if (!hdbUtils.isEmpty(hashValue)) {
+				deleteObj.hash_values.push(hashValue);
+			}
+		}
+	}
+
+	if (hdbUtils.isEmptyOrZeroLength(deleteObj.hash_values)) {
+		return createDeleteResponse([], []);
+	} else if (!Array.isArray(deleteObj.hash_values)) {
+		throw new Error('hash_values must be an array');
+	}
+
+	//this is needed for clustering, right now clustering expects delete to have a records array and use that to get the hash_values.
+	if (hdbUtils.isEmptyOrZeroLength(deleteObj.records)) {
+		deleteObj.records = [];
+		for (let x = 0; x < deleteObj.hash_values.length; x++) {
+			deleteObj.records[x] = {
+				[hash_attribute]: deleteObj.hash_values[x],
+			};
+		}
+	}
+	let envBasePath = getSchemaPath(deleteObj.schema, deleteObj.table);
+	let environment = await environmentUtility.openEnvironment(envBasePath, deleteObj.table);
+
+	let response = await deleteUtility.deleteRecords(
+		environment,
+		hash_attribute,
+		deleteObj.hash_values,
+		deleteObj.__origin?.timestamp
+	);
+
+	try {
+		if (writeToTxnLog === true) {
+			await writeTransaction(deleteObj, response);
+		}
+	} catch (e) {
+		logger.error(`unable to write transaction due to ${e.message}`);
+	}
+
+	return createDeleteResponse(response.deleted, response.skipped, response.txn_time);
+}
+
+/**
+ * creates the response object for deletes based on the deleted & skipped hashes
+ * @param {[]} deleted - list of hash values successfully deleted
+ * @param {[]} skipped - list  of hash values which did not get deleted
+ * @param {number} txnTime - the transaction timestamp
+ * @returns {{skipped_hashes: [], deleted_hashes: [], message: string}}
+ */
+function createDeleteResponse(deleted, skipped, txnTime) {
+	let total = deleted.length + skipped.length;
+	let plural = total === 1 ? 'record' : 'records';
+
+	return {
+		message: `${deleted.length} of ${total} ${plural} successfully deleted`,
+		deleted_hashes: deleted,
+		skipped_hashes: skipped,
+		txn_time: txnTime,
+	};
+}

package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropAttribute.js

@@ -0,0 +1,109 @@
+'use strict';
+
+const SearchObject = require('../../../SearchObject.js');
+const DeleteObject = require('../../../DeleteObject.js');
+// eslint-disable-next-line no-unused-vars
+const DropAttributeObject = require('../../../DropAttributeObject.js');
+const hdbTerms = require('../../../../utility/hdbTerms.ts');
+const commonUtils = require('../../../../utility/common_utils.js');
+const environmentUtility = require('../../../../utility/lmdb/environmentUtility.js');
+const systemSchema = require('../../../../json/systemSchema.json');
+const searchByValue = require('./lmdbSearchByValue.js');
+const deleteRecords = require('./lmdbDeleteRecords.js');
+const { getSchemaPath } = require('../lmdbUtility/initializePaths.js');
+
+module.exports = lmdbDropAttribute;
+
+/**
+ * First deletes the attribute/dbi from lmdb then removes its record from system table
+ * @param {DropAttributeObject} dropAttributeObj
+ * @param {Boolean} [removeData] - optional, defaults to false
+ * @returns {undefined}
+ */
+async function lmdbDropAttribute(dropAttributeObj, removeData = true) {
+	let tableInfo;
+	if (dropAttributeObj.schema === hdbTerms.SYSTEM_SCHEMA_NAME) {
+		tableInfo = systemSchema[dropAttributeObj.table];
+	} else {
+		tableInfo = global.hdb_schema[dropAttributeObj.schema][dropAttributeObj.table];
+	}
+
+	//remove meta data
+	let deleteResults = await dropAttributeFromSystem(dropAttributeObj);
+	//drop dbi
+	let schemaPath = getSchemaPath(dropAttributeObj.schema, dropAttributeObj.table);
+	let env = await environmentUtility.openEnvironment(schemaPath, dropAttributeObj.table);
+
+	//in the scenario of drop table / schema we don't need to remove individual elements since we are removing entire environments
+	if (removeData === true) {
+		await removeAttributeFromAllObjects(dropAttributeObj, env, tableInfo.hash_attribute);
+	}
+
+	environmentUtility.dropDBI(env, dropAttributeObj.attribute);
+
+	return deleteResults;
+}
+
+/**
+ * iterates the hash attribute dbi and removes the attribute dropped
+ * @param {DropAttributeObject} dropAttributeObj
+ * @param {lmdb.RootDatabase} env
+ * @param {String} hash_attribute
+ */
+async function removeAttributeFromAllObjects(dropAttributeObj, env, hash_attribute) {
+	//get reference to hash attribute index (dbi)
+	let dbi = environmentUtility.openDBI(env, hash_attribute);
+
+	//declare just one promise for the bulk write, read the await below
+	let promise;
+	let attributeToDelete = dropAttributeObj.attribute;
+	//iterate the entire hash attribute index to remove the dropped attribute and update the entry
+	for (let { key, value, version } of dbi.getRange({ start: false, versions: true })) {
+		//delete the attribute being dropped from the record
+		let updatedValue = {};
+		for (let property in value) {
+			if (property !== attributeToDelete) updatedValue[property] = value[property];
+		}
+		// maintain the same version number as we re-save the data with the
+		// property dropped
+		promise = env.dbis[hash_attribute].put(key, updatedValue, version);
+	}
+	//since lmdb processes all promised writes in order we only need to wait for the last promise to execute to know all the previous ones have also finished
+	await promise;
+}
+
+/**
+ * Searches the system attributes table for attribute record then sends record to delete to be removed from system table.
+ * @param {DropAttributeObject} dropAttributeObj
+ * @returns {undefined}
+ */
+async function dropAttributeFromSystem(dropAttributeObj) {
+	let searchObj = new SearchObject(
+		hdbTerms.SYSTEM_SCHEMA_NAME,
+		hdbTerms.SYSTEM_TABLE_NAMES.ATTRIBUTE_TABLE_NAME,
+		hdbTerms.SYSTEM_DEFAULT_ATTRIBUTE_NAMES.ATTR_SCHEMA_TABLE_KEY,
+		`${dropAttributeObj.schema}.${dropAttributeObj.table}`,
+		undefined,
+		[hdbTerms.SYSTEM_DEFAULT_ATTRIBUTE_NAMES.ATTR_ID_KEY, hdbTerms.SYSTEM_DEFAULT_ATTRIBUTE_NAMES.ATTR_ATTRIBUTE_KEY]
+	);
+
+	let tableAttributes = Array.from(await searchByValue(searchObj));
+	let attribute = tableAttributes.filter(
+		(attr) => attr[hdbTerms.SYSTEM_DEFAULT_ATTRIBUTE_NAMES.ATTR_ATTRIBUTE_KEY] === dropAttributeObj.attribute
+	);
+	if (commonUtils.isEmptyOrZeroLength(attribute)) {
+		throw new Error(
+			`Attribute '${dropAttributeObj.attribute}' was not found in '${dropAttributeObj.schema}.${dropAttributeObj.table}'`
+		);
+	}
+
+	let id = attribute.map((attr) => attr[hdbTerms.SYSTEM_DEFAULT_ATTRIBUTE_NAMES.ATTR_ID_KEY]);
+
+	let deleteTableObj = new DeleteObject(
+		hdbTerms.SYSTEM_SCHEMA_NAME,
+		hdbTerms.SYSTEM_TABLE_NAMES.ATTRIBUTE_TABLE_NAME,
+		id
+	);
+
+	return deleteRecords(deleteTableObj);
+}

package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropSchema.js

@@ -0,0 +1,107 @@
+'use strict';
+
+const fs = require('fs-extra');
+const SearchObject = require('../../../SearchObject.js');
+const SearchByHashObject = require('../../../SearchByHashObject.js');
+const DeleteObject = require('../../../DeleteObject.js');
+const dropTable = require('./lmdbDropTable.js');
+const deleteRecords = require('./lmdbDeleteRecords.js');
+const getDataByHash = require('./lmdbGetDataByHash.js');
+const searchDataByValue = require('./lmdbSearchByValue.js');
+const hdbTerms = require('../../../../utility/hdbTerms.ts');
+const { getSchemaPath } = require('../lmdbUtility/initializePaths.js');
+const { handleHDBError, hdbErrors } = require('../../../../utility/errors/hdbError.js');
+const { HDB_ERROR_MSGS, HTTP_STATUS_CODES } = hdbErrors;
+
+module.exports = lmdbDropSchema;
+
+/**
+ * deletes all environment files under the schema folder, deletes all schema/table/attribute meta data from system
+ * @param dropSchemaObj
+ */
+async function lmdbDropSchema(dropSchemaObj) {
+	let deleteSchema;
+
+	try {
+		deleteSchema = await validateDropSchema(dropSchemaObj.schema);
+
+		//We search in system > hdbTable for tables with the schema to ensure we are deleting all schema datastores
+		const tableSearchObj = new SearchObject(
+			hdbTerms.SYSTEM_SCHEMA_NAME,
+			hdbTerms.SYSTEM_TABLE_NAMES.TABLE_TABLE_NAME,
+			hdbTerms.SYSTEM_DEFAULT_ATTRIBUTE_NAMES.ATTR_SCHEMA_KEY,
+			deleteSchema,
+			undefined,
+			[hdbTerms.SYSTEM_DEFAULT_ATTRIBUTE_NAMES.ATTR_NAME_KEY]
+		);
+
+		let tables = Array.from(await searchDataByValue(tableSearchObj));
+
+		for (let x = 0; x < tables.length; x++) {
+			const deleteTableObj = {
+				schema: deleteSchema,
+				table: tables[x].name,
+			};
+			try {
+				await dropTable(deleteTableObj);
+			} catch (e) {
+				//this message would get thrown for an environment that doesn't exist
+				if (e.message !== 'invalid environment') {
+					throw e;
+				}
+			}
+		}
+
+		//After all tables for schema are deleted, we can delete the schema
+		const deleteSchemaObj = new DeleteObject(
+			hdbTerms.SYSTEM_SCHEMA_NAME,
+			hdbTerms.SYSTEM_TABLE_NAMES.SCHEMA_TABLE_NAME,
+			[deleteSchema]
+		);
+
+		// Delete the schema from the system > hdbSchema datastore
+		await deleteRecords(deleteSchemaObj);
+		let schemaPath = getSchemaPath(deleteSchema);
+		await fs.remove(schemaPath);
+	} catch (err) {
+		throw err;
+	}
+}
+
+async function validateDropSchema(dropSchema) {
+	let searchObj = new SearchByHashObject(
+		hdbTerms.SYSTEM_SCHEMA_NAME,
+		hdbTerms.SYSTEM_TABLE_NAMES.SCHEMA_TABLE_NAME,
+		[dropSchema],
+		[hdbTerms.SYSTEM_DEFAULT_ATTRIBUTE_NAMES.ATTR_NAME_KEY]
+	);
+
+	let searchResult;
+	let deleteSchema;
+
+	try {
+		searchResult = Array.from(await getDataByHash(searchObj));
+	} catch (err) {
+		throw err;
+	}
+
+	// Data found by the search function should match the dropSchema
+	for (let [, schema] of searchResult) {
+		if (schema.name === dropSchema) {
+			deleteSchema = dropSchema;
+		}
+	}
+
+	if (!deleteSchema) {
+		throw handleHDBError(
+			new Error(),
+			HDB_ERROR_MSGS.SCHEMA_NOT_FOUND(dropSchema),
+			HTTP_STATUS_CODES.NOT_FOUND,
+			undefined,
+			undefined,
+			true
+		);
+	}
+
+	return deleteSchema;
+}

package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropTable.js

@@ -0,0 +1,137 @@
+'use strict';
+
+const SearchObject = require('../../../SearchObject.js');
+const DeleteObject = require('../../../DeleteObject.js');
+const searchByValue = require('./lmdbSearchByValue.js');
+const deleteRecords = require('./lmdbDeleteRecords.js');
+const hdbTerms = require('../../../../utility/hdbTerms.ts');
+const hdbUtils = require('../../../../utility/common_utils.js');
+const environmentUtility = require('../../../../utility/lmdb/environmentUtility.js');
+const { getTransactionAuditStorePath, getSchemaPath } = require('../lmdbUtility/initializePaths.js');
+const log = require('../../../../utility/logging/harper_logger.js');
+
+module.exports = lmdbDropTable;
+
+/**
+ * Calls drops the table, all of it's attribute & deletes the environment
+ * @param dropTableObj
+ */
+async function lmdbDropTable(dropTableObj) {
+	try {
+		if (
+			hdbUtils.isEmpty(global.hdb_schema[dropTableObj.schema]) ||
+			hdbUtils.isEmpty(global.hdb_schema[dropTableObj.schema][dropTableObj.table])
+		) {
+			throw new Error(`unknown schema:${dropTableObj.schema} and table ${dropTableObj.table}`);
+		}
+		await deleteAttributesFromSystem(dropTableObj);
+		await dropTableFromSystem(dropTableObj);
+
+		let schemaPath = getSchemaPath(dropTableObj.schema, dropTableObj.table);
+		try {
+			await environmentUtility.deleteEnvironment(schemaPath, dropTableObj.table);
+		} catch (e) {
+			if (e.message === 'invalid environment') {
+				log.warn(`cannot delete environment for ${dropTableObj.schema}.${dropTableObj.table}, environment not found`);
+			} else {
+				throw e;
+			}
+		}
+
+		try {
+			let transactionPath = getTransactionAuditStorePath(dropTableObj.schema, dropTableObj.table);
+			await environmentUtility.deleteEnvironment(transactionPath, dropTableObj.table, true);
+		} catch (e) {
+			if (e.message === 'invalid environment') {
+				log.warn(`cannot delete environment for ${dropTableObj.schema}.${dropTableObj.table}, environment not found`);
+			} else {
+				throw e;
+			}
+		}
+	} catch (err) {
+		throw err;
+	}
+}
+
+/**
+ *
+ * @param dropTableObj
+ * @returns {Promise<void>}
+ */
+async function deleteAttributesFromSystem(dropTableObj) {
+	let searchObj = new SearchObject(
+		hdbTerms.SYSTEM_SCHEMA_NAME,
+		hdbTerms.SYSTEM_TABLE_NAMES.ATTRIBUTE_TABLE_NAME,
+		hdbTerms.SYSTEM_DEFAULT_ATTRIBUTE_NAMES.ATTR_SCHEMA_TABLE_KEY,
+		`${dropTableObj.schema}.${dropTableObj.table}`,
+		undefined,
+		[hdbTerms.SYSTEM_DEFAULT_ATTRIBUTE_NAMES.ATTR_ID_KEY]
+	);
+
+	let searchResult = Array.from(await searchByValue(searchObj));
+
+	let deleteIds = [];
+	for (let x = 0; x < searchResult.length; x++) {
+		let entry = searchResult[x];
+		deleteIds.push(entry.id);
+	}
+
+	if (deleteIds.length === 0) {
+		return;
+	}
+
+	let deleteTableObj = new DeleteObject(
+		hdbTerms.SYSTEM_SCHEMA_NAME,
+		hdbTerms.SYSTEM_TABLE_NAMES.ATTRIBUTE_TABLE_NAME,
+		deleteIds
+	);
+
+	await deleteRecords(deleteTableObj);
+}
+
+/**
+ * Searches the system table for the table hash, then uses hash to delete table from system.
+ * @param dropTableObj
+ */
+async function dropTableFromSystem(dropTableObj) {
+	let searchObj = new SearchObject(
+		hdbTerms.SYSTEM_SCHEMA_NAME,
+		hdbTerms.SYSTEM_TABLE_NAMES.TABLE_TABLE_NAME,
+		hdbTerms.SYSTEM_DEFAULT_ATTRIBUTE_NAMES.ATTR_NAME_KEY,
+		dropTableObj.table,
+		undefined,
+		[
+			hdbTerms.SYSTEM_DEFAULT_ATTRIBUTE_NAMES.ATTR_NAME_KEY,
+			hdbTerms.SYSTEM_DEFAULT_ATTRIBUTE_NAMES.ATTR_SCHEMA_KEY,
+			hdbTerms.SYSTEM_DEFAULT_ATTRIBUTE_NAMES.ATTR_ID_KEY,
+		]
+	);
+	let searchResult;
+	let deleteTable;
+	try {
+		searchResult = Array.from(await searchByValue(searchObj));
+	} catch (err) {
+		throw err;
+	}
+
+	// Data found by the search function should match the dropTableObject
+	for (let x = 0; x < searchResult.length; x++) {
+		let item = searchResult[x];
+		if (item.name === dropTableObj.table && item.schema === dropTableObj.schema) {
+			deleteTable = item;
+		}
+	}
+
+	if (!deleteTable) {
+		throw new Error(`${dropTableObj.schema}.${dropTableObj.table} was not found`);
+	}
+
+	let deleteTableObj = new DeleteObject(hdbTerms.SYSTEM_SCHEMA_NAME, hdbTerms.SYSTEM_TABLE_NAMES.TABLE_TABLE_NAME, [
+		deleteTable.id,
+	]);
+	try {
+		await deleteRecords(deleteTableObj);
+	} catch (err) {
+		throw err;
+	}
+}

package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbFlush.js

@@ -0,0 +1,35 @@
+'use strict';
+
+const { getSchemaPath } = require('../lmdbUtility/initializePaths.js');
+const environmentUtility = require('../../../../utility/lmdb/environmentUtility.js');
+
+module.exports = {
+	flush,
+	resetReadTxn,
+};
+
+/**
+ * This is wrapper for sync/flush to disk
+ * @param schema
+ * @param table
+ * @returns {Promise<any>}
+ */
+async function flush(schema, table) {
+	let environment = await environmentUtility.openEnvironment(getSchemaPath(schema, table), table.toString());
+	return environment.flushed;
+}
+
+/**
+ * This is wrapper for resetting the current read transaction to ensure it is the very latest
+ * @param schema
+ * @param table
+ * @returns {void}
+ */
+async function resetReadTxn(schema, table) {
+	try {
+		let environment = await environmentUtility.openEnvironment(getSchemaPath(schema, table), table.toString());
+		environment.resetReadTxn();
+	} catch {
+		// if no environment, then the read txn can't be out of date!
+	}
+}

package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetBackup.js

@@ -0,0 +1,111 @@
+'use strict';
+
+const { Readable } = require('stream');
+const { getDatabases } = require('../../../../resources/databases.ts');
+const { readSync, openSync, createReadStream } = require('fs');
+const { open } = require('lmdb');
+const { OpenDBIObject } = require('../../../../utility/lmdb/OpenDBIObject.js');
+const OpenEnvironmentObject = require('../../../../utility/lmdb/OpenEnvironmentObject.js');
+const { AUDIT_STORE_OPTIONS } = require('../../../../resources/auditStore.ts');
+const { INTERNAL_DBIS_NAME, AUDIT_STORE_NAME } = require('../../../../utility/lmdb/terms.js');
+
+module.exports = getBackup;
+const META_SIZE = 32768;
+const DELAY_ITERATIONS = 100;
+/**
+ * function execute the readTransactionLog operation
+ * @param {GetBackupObject} getBackupObj
+ * @returns {Promise<[]>}
+ */
+async function getBackup(getBackupObj) {
+	const databaseName = getBackupObj.database || getBackupObj.schema || 'data';
+	const database = getDatabases()[databaseName];
+	const backupDate = new Date().toISOString();
+	let tables = getBackupObj.tables || (getBackupObj.table && [getBackupObj.table]);
+	if (tables) {
+		// if tables are specified, we have to copy the database with just the specified tables and then stream that
+		let tableClass = database[tables[0]];
+		if (!tableClass) throw new Error(`Can not find table ${tables[0]}`);
+		// we use the attribute store to drive this process, finding the right stores to duplicate
+		let attributeStore = tableClass.dbisDB;
+		let backupRoot = open({ noSync: true, maxDbs: OpenEnvironmentObject.MAX_DBS }); // open a temporary database (this
+		// will also cause it to
+		// close on completion)
+		let resolution;
+		let backupAttributeStore = backupRoot.openDB(INTERNAL_DBIS_NAME, new OpenDBIObject(false));
+		let readTxn = attributeStore.useReadTransaction();
+		let i = 0;
+		const copyDatabase = async function (storeName, options) {
+			options.encoding = 'binary'; // directly copy bytes
+			options.encoder = undefined;
+			let backupStore = backupRoot.openDB(storeName, options);
+			let sourceStore = attributeStore.openDB(storeName, options);
+			for (let { key, version, value } of sourceStore.getRange({
+				start: null,
+				transaction: readTxn,
+				versions: sourceStore.useVersions,
+			})) {
+				resolution = backupStore.put(key, value, version);
+				if (i++ % DELAY_ITERATIONS === 0) {
+					await new Promise((resolve) => setTimeout(resolve, 20));
+					if (readTxn.openTimer) readTxn.openTimer = 0; // reset any timer monitoring this
+				}
+			}
+		};
+		for (let { key, value: attributeInfo } of attributeStore.getRange({ transaction: readTxn, start: false })) {
+			if (tables.some((table) => key.startsWith?.(table + '/'))) {
+				// it is a store we need to copy
+				backupAttributeStore.put(key, attributeInfo);
+				const [, attribute] = key.split('/');
+				let isPrimaryKey = !attribute;
+				let options = new OpenDBIObject(!isPrimaryKey, isPrimaryKey);
+				await copyDatabase(key, options);
+			}
+		}
+		if (getBackupObj.include_audit) {
+			await copyDatabase(AUDIT_STORE_NAME, { ...AUDIT_STORE_OPTIONS });
+		}
+		await resolution;
+		let stream = createReadStream(backupRoot.path);
+		stream.headers = getHeaders();
+		stream.on('close', () => {
+			readTxn.done();
+			backupRoot.close(); // this should delete it
+		});
+		return stream;
+	}
+	const firstTable = database[Object.keys(database)[0]];
+	const store = firstTable.primaryStore;
+
+	let fd = openSync(store.path);
+	return store.transaction(() => {
+		let metaBuffers = Buffer.alloc(META_SIZE);
+		readSync(fd, metaBuffers, 0, META_SIZE); // sync, need to do this as fast as possible since we are in a write txn
+		store.resetReadTxn(); // make sure we are not using a cached read transaction, force a fresh one
+		let readTxn = store.useReadTransaction(); // this guarantees the current transaction is preserved in the backup
+		// renew is necessary because normally renew is actually lazily called on the next db operation, but
+		// we are not performing any db operations
+		readTxn.renew();
+		// create a file stream that starts after the meta area
+		let fileStream = createReadStream(null, { fd, start: META_SIZE });
+		let stream = new Readable.from(
+			(async function* () {
+				yield metaBuffers; // return the meta area that was frozen inside the write transaction
+				for await (const chunk of fileStream) {
+					if (readTxn.openTimer) readTxn.openTimer = 0; // reset any timer monitoring this
+					yield chunk;
+				}
+				readTxn.done(); // done with the read txn
+			})()
+		);
+		stream.headers = getHeaders();
+		return stream;
+	});
+	function getHeaders() {
+		const headers = new Map();
+		headers.set('content-type', 'application/octet-stream');
+		headers.set('content-disposition', `attachment; filename="${databaseName}"`);
+		headers.set('date', backupDate);
+		return headers;
+	}
+}

package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByHash.js

@@ -0,0 +1,28 @@
+'use strict';
+
+const searchUtility = require('../../../../utility/lmdb/searchUtility.js');
+const hashSearchInit = require('../lmdbUtility/initializeHashSearch.js');
+
+module.exports = lmdbGetDataByHash;
+
+/**
+ * fetches records by their hash values and returns a map of the results
+ * @param {SearchByHashObject} searchObject
+ */
+async function lmdbGetDataByHash(searchObject) {
+	let environment = await hashSearchInit(searchObject);
+	let transaction = environment.useReadTransaction();
+	transaction.database = environment;
+
+	const tableInfo = global.hdb_schema[searchObject.schema][searchObject.table];
+	try {
+		return searchUtility.batchSearchByHashToMap(
+			transaction,
+			tableInfo.hash_attribute,
+			searchObject.get_attributes,
+			searchObject.hash_values
+		);
+	} finally {
+		transaction.done();
+	}
+}

package/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByValue.js

@@ -0,0 +1,29 @@
+'use strict';
+
+const searchValidator = require('../../../../validation/searchValidator.js');
+const commonUtils = require('../../../../utility/common_utils.js');
+const hdbTerms = require('../../../../utility/hdbTerms.ts');
+const lmdbSearch = require('../lmdbUtility/lmdbSearch.js');
+
+module.exports = lmdbGetDataByValue;
+
+/**
+ * gets records by value returns a map of objects
+ * @param {SearchObject} searchObject
+ * @param {hdbTerms.VALUE_SEARCH_COMPARATORS} [comparator]
+ * @returns {{String|Number, Object}}
+ */
+function lmdbGetDataByValue(searchObject, comparator) {
+	let comparatorSearch = !commonUtils.isEmpty(comparator);
+	if (comparatorSearch && hdbTerms.VALUE_SEARCH_COMPARATORS_REVERSE_LOOKUP[comparator] === undefined) {
+		throw new Error(`Value search comparator - ${comparator} - is not valid`);
+	}
+
+	let validationError = searchValidator(searchObject, 'value');
+	if (validationError) {
+		throw validationError;
+	}
+
+	let returnMap = true;
+	return lmdbSearch.prepSearch(searchObject, comparator, returnMap);
+}