@harperfast/harper 5.0.25 → 5.1.0-beta.1

package/resources/DatabaseTransaction.ts

@@ -1,17 +1,17 @@
+import { cleanupUnusedBlobs } from './blob.ts';
 import { Transaction as LMDBTransaction } from 'lmdb';
-import { getNextMonotonicTime } from '../utility/lmdb/commonUtility.js';
-import { ServerError } from '../utility/errors/hdbError.js';
-import * as harperLogger from '../utility/logging/harper_logger.js';
+import { getNextMonotonicTime } from '../utility/lmdb/commonUtility.ts';
+import { ServerError } from '../utility/errors/hdbError.ts';
+import * as harperLogger from '../utility/logging/harper_logger.ts';
 import type { Context, Id } from './ResourceInterface.ts';
-import * as envMngr from '../utility/environment/environmentManager.js';
+import * as envMngr from '../utility/environment/environmentManager.ts';
 import { CONFIG_PARAMS } from '../utility/hdbTerms.ts';
-import { convertToMS } from '../utility/common_utils.js';
+import { convertToMS } from '../utility/common_utils.ts';
 import { when } from '../utility/when.ts';
 import { setTimeout as delay } from 'node:timers/promises';
 import { Transaction as RocksTransaction, type Store as RocksStore } from '@harperfast/rocksdb-js';
 import type { RootDatabaseKind } from './databases.ts';
 import type { Entry } from './RecordEncoder.ts';
-import { cleanupUnusedBlobs } from './blob.ts';
 
 const trackedTxns = new Set<DatabaseTransaction>();
 const MAX_OUTSTANDING_TXN_DURATION = convertToMS(envMngr.get(CONFIG_PARAMS.STORAGE_MAXTRANSACTIONQUEUETIME)) || 45000; // Allow write transactions to be queued for up to 25 seconds before we start rejecting them
@@ -44,20 +44,26 @@ export type CommitOptions = {
 type ReadTransaction = (LMDBTransaction | RocksTransaction) & {
 	openTimer?: number;
 	retryRisk?: number;
+	isDone?: boolean;
+	isCommitted?: boolean;
 };
 
 export type TransactionWrite = {
 	key: Id;
-	store: RootDatabaseKind;
+	store: any; // using any here because of circular dependency and complex RootDatabaseKind
 	invalidated?: boolean;
 	entry?: Partial<Entry>;
 	before?: () => void | Promise<void>;
 	beforeIntermediate?: () => void | Promise<void>;
-	commit?: (txnTime: number, existingEntry: Entry, retry: boolean, transaction: RocksTransaction) => void;
+	commit?: (txnTime: number, existingEntry: Partial<Entry>, retry: boolean, transaction: any) => void;
 	validate?: (txnTime: number) => void;
 	fullUpdate?: boolean;
 	saved?: boolean;
 	deferSave?: boolean;
+	nodeName?: string;
+	nodeId?: number;
+	promise?: Promise<any>;
+	result?: any;
 	// blobs that were pre-saved as part of this write; used to clean up files if the commit is skipped or aborted
 	savedBlobs?: Blob[];
 	// the commit handler's most recent decision: true means it took an early-return that left savedBlobs unreferenced.
@@ -95,7 +101,7 @@ export class DatabaseTransaction implements Transaction {
 		this.readTxnRefCount = (this.readTxnRefCount || 0) + 1;
 		this.timeout = txnExpiration; // reset the timeout
 		if (this.transaction) {
-			if (this.transaction.openTimer) this.transaction.openTimer = 0;
+			if ((this.transaction as any).openTimer) (this.transaction as any).openTimer = 0;
 			return this.transaction;
 		}
 		if (this.open !== TRANSACTION_STATE.OPEN) return; // can not start a new read transaction as there is no future commit that will take place, just have to allow the read to latest database state
@@ -110,7 +116,7 @@ export class DatabaseTransaction implements Transaction {
 		if (DEBUG_LONG_TXNS) {
 			this.stackTraces = [new StartedTransaction()];
 		}
-		if (this.transaction.openTimer) this.transaction.openTimer = 0;
+		if ((this.transaction as any).openTimer) (this.transaction as any).openTimer = 0;
 		trackedTxns.add(this);
 		return this.transaction;
 	}
@@ -157,7 +163,13 @@ export class DatabaseTransaction implements Transaction {
 		this.writes.push(operation);
 		if (!operation.deferSave) {
 			// Setting saved to false means to defer saving
-			this.save(operation);
+			const saveResult: any = this.save(operation);
+			if (saveResult?.then) {
+				// When the transaction is already committed (immediateCommit path), save() returns
+				// the commit promise. Propagate it so callers can await the actual write being
+				// committed rather than resolving before it is durable.
+				return saveResult.then(() => operation);
+			}
 		}
 		return operation;
 	}
@@ -184,7 +196,7 @@ export class DatabaseTransaction implements Transaction {
 		}
 		if (this.retries > 0) {
 			// This marks the Rocks transaction as a retry so we don't write the transaction log again
-			transaction.isRetry = true;
+			(transaction as any).isRetry = true;
 		}
 		if (!txnTime) txnTime = this.timestamp = transaction.getTimestamp();
 		if (reloadEntry || operation.entry === undefined) {
@@ -193,7 +205,7 @@ export class DatabaseTransaction implements Transaction {
 		if (!operation.saved) {
 			operation.saved = true;
 			// immediately execute in this transaction
-			if (operation.validate?.(txnTime) === false) {
+			if ((operation.validate?.(txnTime) as any) === false) {
 				operation.commit = () => {}; // noop if we try again
 				return;
 			}
@@ -215,7 +227,7 @@ export class DatabaseTransaction implements Transaction {
 		let transaction = options.transaction ?? this.transaction; // we need to preserve this transaction as we might to resurrect it if we have to retry
 		for (let i = 0; i < this.writes.length; i++) {
 			let operation = this.writes[i];
-			if (this.retries === 0 && operation.saved) continue;
+			if (!operation || (this.retries === 0 && operation.saved)) continue;
 			this.save(operation, transaction, i < this.validated);
 		}
 		this.validated = this.writes.length;
@@ -252,10 +264,18 @@ export class DatabaseTransaction implements Transaction {
 					trackedTxns.delete(this);
 					this.transaction = null; // clear transaction so any further operations operate immediately
 					if (transaction) {
+						this.writes = this.writes.filter((write) => write); // filter out removed entries
 						if (this.writes.length > 0) {
 							commitResolution = transaction.commit();
 						} else {
-							commitResolution = transaction.abort();
+							try {
+								commitResolution = transaction.abort();
+							} catch {
+								// The transaction has uncommitted writes that were already cleared from
+								// this.writes by a concurrent immediate-commit path (e.g. writes made with
+								// an explicitly-reused closed transaction). Those writes are handled by the
+								// concurrent commit, so there is nothing left to do here.
+							}
 						}
 					}
 				}
@@ -276,7 +296,7 @@ export class DatabaseTransaction implements Transaction {
 					const completions = [];
 					return commitResolution.then(
 						() => {
-							transaction.onCommit?.();
+							(transaction as any).onCommit?.();
 							if (this.next) {
 								completions.push(this.next.commit(options));
 							}
@@ -292,7 +312,7 @@ export class DatabaseTransaction implements Transaction {
 									completions.push(
 										confirmReplication(
 											databaseName,
-											lastWrite.store.getEntry(lastWrite.key).version,
+											(lastWrite.store.getEntry(lastWrite.key) as any).version,
 											this.replicatedConfirmation
 										)
 									);
@@ -335,6 +355,11 @@ export class DatabaseTransaction implements Transaction {
 						}
 					);
 				}
+				for (const write of this.writes) {
+					if (write?.skipped && write?.savedBlobs) cleanupUnusedBlobs(write.savedBlobs);
+				}
+				this.writes = [];
+				if (this.#context?.resourceCache) this.#context.resourceCache = null;
 				const txnResolution: CommitResolution = {
 					txnTime: this.timestamp,
 				};
@@ -342,18 +367,16 @@ export class DatabaseTransaction implements Transaction {
 					// now run any other transactions
 					options.timestamp = this.timestamp;
 					const nextResolution = this.next?.commit(options);
-					if (nextResolution?.then)
-						return nextResolution?.then((nextResolution) => ({
+					if ((nextResolution as any)?.then)
+						return (nextResolution as any)?.then((nextResolution) => ({
 							txnTime: this.timestamp,
 							next: nextResolution,
 						}));
-					txnResolution.next = nextResolution;
+					txnResolution.next = nextResolution as any;
 				}
 				return txnResolution;
 			},
 			(error) => {
-				// before/beforeIntermediate (e.g. blob save) failed; abort to clean up pre-saved blob files
-				// and the underlying transaction so it doesn't leak.
 				this.abort();
 				throw error;
 			}
@@ -362,7 +385,6 @@ export class DatabaseTransaction implements Transaction {
 	abort(): void {
 		while (this.readTxnsUsed > 0) this.doneReadTxn(); // release the read snapshot when we abort, we assume we don't need it
 		this.open = TRANSACTION_STATE.CLOSED;
-		// any blobs that were pre-saved as part of these writes will never be referenced; schedule deletion
 		for (const write of this.writes) {
 			if (write?.savedBlobs) cleanupUnusedBlobs(write.savedBlobs);
 		}
@@ -396,10 +418,11 @@ export class ImmediateTransaction extends DatabaseTransaction {
 		super();
 		this.db = db;
 	}
-	save(transaction: ImmediateTransaction) {
+	save(...args: any[]): any {
+		const transaction = args[0];
 		if (this.isCommitting) {
 			// if we are in the commit, do the save and force a reload so we get a read within the transaction
-			super.save(transaction, null, true);
+			super.save(transaction, null as any, true);
 		} else {
 			this.isCommitting = true;
 			return when(this.commit(), () => {
@@ -408,10 +431,15 @@ export class ImmediateTransaction extends DatabaseTransaction {
 		}
 	}
 
+	declare _timestamp: number;
+	// @ts-expect-error accessor overriding property
 	get timestamp() {
 		return this._timestamp || (this._timestamp = getNextMonotonicTime());
 	}
-	getReadTxn() {
+	set timestamp(value: number) {
+		this._timestamp = value;
+	}
+	getReadTxn(): any {
 		return; // no transaction means read latest
 	}
 }
@@ -422,10 +450,10 @@ function startMonitoringTxns() {
 	timer = setInterval(function () {
 		for (const txn of trackedTxns) {
 			if (txn.timeout <= 0) {
-				const url = txn.getContext()?.url;
+				const url = (txn.getContext() as any)?.url;
 				harperLogger.error(
 					`Transaction was open too long and has been committed, from table: ${
-						txn.db?.name + (url ? ' path: ' + url : '')
+						(txn.db as any)?.name + (url ? ' path: ' + url : '')
 					}`,
 					...(txn.startedFrom ? [`was started from ${txn.startedFrom.resourceName}.${txn.startedFrom.method}`] : []),
 					...(DEBUG_LONG_TXNS ? ['starting stack trace', txn.stackTraces] : [])
@@ -433,8 +461,8 @@ function startMonitoringTxns() {
 				// reset the transaction
 				try {
 					const result = txn.commit();
-					if (result?.then) {
-						result.catch((error) => {
+					if ((result as any)?.then) {
+						(result as any).catch((error) => {
 							harperLogger.debug?.(`Error committing timed out transaction: ${error.message}`);
 						});
 					}

package/resources/ErrorResource.ts

@@ -5,9 +5,10 @@ import type { Context } from './ResourceInterface.ts';
  * to access endpoints/resources that had an internal error in their configuration or setup. This helps ensure that
  * if there is a problem with a resource, it is immediately apparent and can be fixed.
  */
-export class ErrorResource implements Resource {
+export class ErrorResource extends Resource {
 	error: Error;
 	constructor(error: Error) {
+		super(null as any, null);
 		this.error = error;
 	}
 	isError = true;

package/resources/LMDBTransaction.ts

@@ -1,4 +1,3 @@
-import { Transaction as LMDBNativeTransaction } from 'lmdb';
 import {
 	DatabaseTransaction,
 	type CommitOptions,
@@ -6,8 +5,8 @@ import {
 	type CommitResolution,
 } from './DatabaseTransaction';
 import { cleanupUnusedBlobs } from './blob.ts';
-import { getNextMonotonicTime } from '../utility/lmdb/commonUtility.js';
-import * as harperLogger from '../utility/logging/harper_logger.js';
+import { getNextMonotonicTime } from '../utility/lmdb/commonUtility.ts';
+import * as harperLogger from '../utility/logging/harper_logger.ts';
 import type { Context } from './ResourceInterface.ts';
 import { Transaction as RocksTransaction } from '@harperfast/rocksdb-js';
 import type { RootDatabaseKind } from './databases.ts';
@@ -25,11 +24,6 @@ export function replicationConfirmation(callback) {
 	confirmReplication = callback;
 }
 
-type ReadTransaction = LMDBNativeTransaction & {
-	openTimer?: number;
-	retryRisk?: number;
-};
-
 export class LMDBTransaction extends DatabaseTransaction {
 	#context: Context;
 	writes: TransactionWrite[] = []; // the set of writes to commit if the conditions are met
@@ -37,33 +31,33 @@ export class LMDBTransaction extends DatabaseTransaction {
 	_timestamp = 0;
 	declare next: DatabaseTransaction;
 	declare stale: boolean;
-	overloadChecked: boolean;
+	declare overloadChecked: boolean;
 	open = TRANSACTION_STATE.OPEN;
 
-	getReadTxn(): ReadTransaction {
+	getReadTxn(): any {
 		// used optimistically
 		this.readTxnRefCount = (this.readTxnRefCount || 0) + 1;
 		this.timeout = txnExpiration; // reset the timeout
 		if (this.stale) this.stale = false;
 		if (this.readTxn) {
-			if (this.readTxn.openTimer) this.readTxn.openTimer = 0;
+			if ((this.readTxn as any).openTimer) (this.readTxn as any).openTimer = 0;
 			return this.readTxn;
 		}
 		if (this.open !== TRANSACTION_STATE.OPEN) return; // can not start a new read transaction as there is no future commit that will take place, just have to allow the read to latest database state
 
 		// Get a read transaction from lmdb-js; make sure we do this first, as it can fail, we don't want to leave the transaction in a bad state with readTxnsUsed > 0
-		this.readTxn = this.db.useReadTransaction();
+		this.readTxn = (this.db as any).useReadTransaction();
 
 		this.readTxnsUsed = 1;
-		if (this.readTxn.openTimer) this.readTxn.openTimer = 0;
-		trackedTxns.add(this);
+		if ((this.readTxn as any).openTimer) (this.readTxn as any).openTimer = 0;
+		trackedTxns.add(this as any);
 		return this.readTxn;
 	}
 
 	useReadTxn() {
 		this.getReadTxn();
 		if (this.readTxn) {
-			(this.readTxn as LMDBTransaction).use();
+			(this.readTxn as any).use();
 			this.readTxnsUsed++;
 		}
 		return this.readTxn;
@@ -74,10 +68,10 @@ export class LMDBTransaction extends DatabaseTransaction {
 		if (this.readTxn instanceof RocksTransaction) {
 			// TODO: Implement this for RocksDB
 		} else {
-			(this.readTxn as LMDBTransaction).done();
+			(this.readTxn as any).done();
 		}
 		if (--this.readTxnsUsed === 0) {
-			trackedTxns.delete(this);
+			trackedTxns.delete(this as any);
 			this.readTxn = null;
 		}
 	}
@@ -88,7 +82,7 @@ export class LMDBTransaction extends DatabaseTransaction {
 		}
 	}
 
-	addWrite(operation: TransactionWrite) {
+	addWrite(operation: TransactionWrite): any {
 		if (this.open === TRANSACTION_STATE.CLOSED) {
 			throw new Error('Can not use a transaction that is no longer open');
 		}
@@ -97,7 +91,13 @@ export class LMDBTransaction extends DatabaseTransaction {
 			// if the transaction is lingering, it is already committed, so we need to commit the write immediately
 			const immediateTxn = new ImmediateTransaction(this.db);
 			immediateTxn.addWrite(operation);
-			return immediateTxn.commit({});
+			const result = immediateTxn.commit({});
+			if (result?.then) {
+				operation.promise = result;
+			} else {
+				operation.result = result;
+			}
+			return result;
 		}
 
 		this.writes.push(operation); // standard path, add to current transaction
@@ -111,7 +111,8 @@ export class LMDBTransaction extends DatabaseTransaction {
 	/**
 	 * Resolves with information on the timestamp and success of the commit
 	 */
-	commit(options: CommitOptions = {}): Promise<CommitResolution> {
+	commit(options: CommitOptions = {}): any {
+		options = options || {};
 		let txnTime = this.timestamp;
 		if (!txnTime) txnTime = this.timestamp = options.timestamp || getNextMonotonicTime();
 		if (!options.timestamp) options.timestamp = txnTime;
@@ -190,7 +191,11 @@ export class LMDBTransaction extends DatabaseTransaction {
 						write.entry = write.store.getEntry(write.key);
 					}
 
-					const conditionResolution = write.store.ifVersion(write.key, write.entry?.version ?? null, nextCondition);
+					const conditionResolution = (write.store as any).ifVersion(
+						write.key,
+						write.entry?.version ?? null,
+						nextCondition
+					);
 					resolution = resolution || conditionResolution;
 				} else {
 					nextCondition();
@@ -249,7 +254,7 @@ export class LMDBTransaction extends DatabaseTransaction {
 							completions.push(
 								confirmReplication(
 									databaseName,
-									lastWrite.store.getEntry(lastWrite.key).localTime,
+									(lastWrite.store.getEntry(lastWrite.key) as any).localTime,
 									this.replicatedConfirmation
 								)
 							);
@@ -286,12 +291,12 @@ export class LMDBTransaction extends DatabaseTransaction {
 		if (this.next) {
 			// now run any other transactions
 			const nextResolution = this.next?.commit(options);
-			if (nextResolution?.then)
-				return nextResolution?.then((nextResolution) => ({
+			if ((nextResolution as any)?.then)
+				return (nextResolution as any)?.then((nextResolution) => ({
 					txnTime,
 					next: nextResolution,
 				}));
-			txnResolution.next = nextResolution;
+			txnResolution.next = nextResolution as any;
 		}
 		return txnResolution;
 	}
@@ -305,7 +310,7 @@ export class LMDBTransaction extends DatabaseTransaction {
 		// reset the transaction
 		this.writes = [];
 	}
-	save() {
+	save(..._args: any[]): any {
 		// noop for LMDB
 	}
 }
@@ -315,12 +320,16 @@ export class ImmediateTransaction extends LMDBTransaction {
 		super();
 		this.db = db;
 	}
-	save(_transaction: ImmediateTransaction, _isRetry = false) {
+	save(..._args: any[]): any {
 		return this.commit();
 	}
+	// @ts-expect-error accessor overriding property
 	get timestamp() {
 		return this._timestamp || (this._timestamp = getNextMonotonicTime());
 	}
+	set timestamp(value: number) {
+		this._timestamp = value;
+	}
 	getReadTxn() {
 		return; // no transaction means read latest
 	}
@@ -333,10 +342,10 @@ function startMonitoringTxns() {
 	timer = setInterval(function () {
 		for (const txn of trackedTxns) {
 			if (txn.timeout <= 0) {
-				const url = txn.getContext()?.url;
+				const url = (txn.getContext() as any)?.url;
 				harperLogger.error(
 					`Transaction was open too long and has been committed, from table: ${
-						txn.db?.name + (url ? ' path: ' + url : '')
+						(txn.db as any)?.name + (url ? ' path: ' + url : '')
 					}`
 				);
 				// reset the transaction

package/resources/RecordEncoder.ts

@@ -6,6 +6,7 @@
  */
 
 import { Encoder } from 'msgpackr';
+import { createStructon } from 'structon';
 import {
 	HAS_PREVIOUS_RESIDENCY_ID,
 	HAS_CURRENT_RESIDENCY_ID,
@@ -15,7 +16,7 @@ import {
 	ACTION_32_BIT,
 	HAS_ADDITIONAL_AUDIT_REFS as HAS_ADDITIONAL_AUDIT_REFS_AUDIT,
 } from './auditStore.ts';
-import * as harperLogger from '../utility/logging/harper_logger.js';
+import * as harperLogger from '../utility/logging/harper_logger.ts';
 import './blob.ts';
 import {
 	blobsWereEncoded,
@@ -29,6 +30,8 @@ import { getThisNodeId } from './nodeIdMapping.ts';
 import { recordAction } from './analytics/write.ts';
 import { RocksDatabase } from '@harperfast/rocksdb-js';
 import { when } from '../utility/when.ts';
+
+const StructonEncoder = createStructon(Encoder) as typeof Encoder;
 export type Entry = {
 	key: any;
 	value: any;
@@ -40,6 +43,7 @@ export type Entry = {
 	residencyId: number;
 	size: number;
 	deref?: () => any;
+	[METADATA]?: any;
 	additionalAuditRefs?: Array<{ version: number; nodeId: number }>;
 };
 
@@ -80,7 +84,10 @@ let timestampNextEncoding = 0,
 	additionalAuditRefsNextEncoding: Array<{ version: number; nodeId: number }> | undefined;
 // tracking metadata with a singleton works better than trying to alter response of getEntry/get and coordinating that across caching layers
 export let lastMetadata: Entry | null = null;
-export class RecordEncoder extends Encoder {
+export class RecordEncoder extends StructonEncoder {
+	rootStore: any;
+	declare saveStructures: any;
+	declare getStructures: any;
 	structureUpdate?: any;
 	isRocksDB: boolean;
 	name: string;
@@ -328,7 +335,7 @@ export class RecordEncoder extends Encoder {
 					additionalAuditRefs,
 					size: end - start,
 					value,
-				};
+				} as any;
 				if (this.isRocksDB) return lastMetadata;
 				return value;
 			} // else a normal entry
@@ -575,6 +582,7 @@ export function recordUpdater(store, tableId, auditStore) {
 			version: number;
 			instructedWrite?: boolean;
 			ifVersion?: number;
+			transaction?: any;
 		} = {
 			version: newVersion,
 			instructedWrite: timestampNextEncoding > 0,
@@ -590,7 +598,7 @@ export function recordUpdater(store, tableId, auditStore) {
 				metadataInNextEncoding |= HAS_RESIDENCY_ID;
 				extendedType |= HAS_CURRENT_RESIDENCY_ID;
 			} else residencyIdAtNextEncoding = 0;
-			const nodeId = options?.nodeId;
+			const nodeId = options?.nodeId ?? (audit ? getThisNodeId(auditStore) : undefined);
 			if (nodeId >= 0) {
 				nodeIdAtNextEncoding = nodeId;
 				metadataInNextEncoding |= HAS_NODE_ID;

package/resources/RequestTarget.ts

@@ -15,6 +15,8 @@ export class RequestTarget extends URLSearchParams {
 	/** Request best effort and returning synchronously */
 	declare syncAllowed?: boolean;
 
+	declare sync?: boolean;
+
 	/** Indicates that this is a request to query for collection of records */
 	isCollection?: boolean;
 	// these are query parameters