@harperfast/harper 5.2.8 → 5.2.10

package/resources/Table.ts

@@ -86,7 +86,7 @@ import { rebuildUpdateBefore } from './crdt.ts';
 import { appendHeader } from '../server/serverHelpers/Headers.ts';
 import fs from 'node:fs';
 import { Blob, deleteBlobsInObject, findBlobsInObject, startPreCommitBlobsForRecord } from './blob.ts';
-import { onStorageReclamation, getStorageSpaceStats } from '../server/storageReclamation.ts';
+import { onStorageReclamation, removeStorageReclamation, getStorageSpaceStats } from '../server/storageReclamation.ts';
 import { RequestTarget } from './RequestTarget.ts';
 import harperLogger from '../utility/logging/harper_logger.ts';
 import { throttle } from '../server/throttle.ts';
@@ -1564,7 +1564,12 @@ export function makeTable(options) {
 					if (removeTombstonedCatalog()) await dbisDb.committed;
 				}
 			} else {
-				// legacy table per database
+				// legacy table per database. The store to retire is this table's own audit store: nothing
+				// assigns `primaryStore.auditStore` — openAuditStore() assigns `rootStore.auditStore`, and
+				// this is the reference makeTable() was handed. Awaited so a pass suspended mid-removal has
+				// released the primary DBI before it is closed and unlinked.
+				await auditStore?.stopAuditCleanup?.();
+				removeStorageReclamation(primaryStore.path);
 				await primaryStore.close();
 				fs.unlinkSync(primaryStore.path);
 			}
@@ -3335,6 +3340,7 @@ export function makeTable(options) {
 			// objects. Entries are small and shallow; the clone is cheap next to the query.
 			conditions = cloneConditions(conditions);
 			let orderAlignedCondition;
+			let syntheticOrderCondition;
 			const filtered = {};
 
 			function prepareConditions(conditions: any[], operator: string) {
@@ -3473,7 +3479,7 @@ export function makeTable(options) {
 							// if it is indexed, we add a pseudo-condition to align with the natural sort order of the index.
 							// the primary key has no secondary index, but the primary store is itself keyed in
 							// primary-key order, so scanning it is already aligned with the sort
-							orderAlignedCondition = { ...sort, comparator: 'sort' };
+							orderAlignedCondition = syntheticOrderCondition = { ...sort, comparator: 'sort' };
 							conditions.push(orderAlignedCondition);
 						} else if (conditions.length === 0 && !target.allowFullScan)
 							throw handleHDBError(
@@ -3503,8 +3509,10 @@ export function makeTable(options) {
 						};
 					}
 				} else {
-					// if we had to add an aligned condition that isn't first, we remove it and do ordering later
-					if (orderAlignedCondition) conditions.splice(conditions.indexOf(orderAlignedCondition), 1);
+					// if we had to add an aligned condition that isn't first, we remove it and do ordering later —
+					// only the one we added; a caller's own condition on the sort attribute is still a filter
+					const syntheticIndex = syntheticOrderCondition ? conditions.indexOf(syntheticOrderCondition) : -1;
+					if (syntheticIndex >= 0) conditions.splice(syntheticIndex, 1);
 					postOrdering = sort;
 				}
 			}

package/resources/auditStore.ts

@@ -118,6 +118,13 @@ const warnedBodylessMints = new Set<string>();
 const warnedBodylessTables = new Set<number>();
 const FLOAT_TARGET = new Float64Array(1);
 const FLOAT_BUFFER = new Uint8Array(FLOAT_TARGET.buffer);
+
+/** Last resort on a detached path: a failing log sink must not itself become an unhandled rejection. */
+function warnContained(message: string, error: unknown) {
+	try {
+		harperLogger.warn(message, error);
+	} catch {}
+}
 let DEFAULT_AUDIT_CLEANUP_DELAY = 10000; // default delay of 10 seconds
 let timestampErrored = false;
 export function openAuditStore(rootStore) {
@@ -133,7 +140,10 @@ export function openAuditStore(rootStore) {
 		if (!auditStore) {
 			// this means we are creating a new audit store. Initialize with the last removed timestamp (we don't want to put this in legacy audit logs since we don't know if they have had deletions or not).
 			auditStore = rootStore.openDB(AUDIT_STORE_NAME, AUDIT_STORE_OPTIONS);
-			updateLastRemoved(auditStore, 1);
+			// this open path is synchronous, so nothing downstream can own the write's rejection
+			updateLastRemoved(auditStore, 1)?.catch?.((error) =>
+				warnContained('Error initializing the audit log last-removed marker', error)
+			);
 		}
 		const superGetRange = auditStore.getRange.bind(auditStore);
 		auditStore.getRange = function (options) {
@@ -164,7 +174,15 @@ export function openAuditStore(rootStore) {
 	let pendingCleanupResolve: (() => void) | null = null;
 	let lastCleanupResolution: Promise<void>;
 	let cleanupPriority = 0;
-	let auditCleanupDelay = DEFAULT_AUDIT_CLEANUP_DELAY;
+	auditStore.auditCleanupDelay = DEFAULT_AUDIT_CLEANUP_DELAY;
+	let cleanupStopped = false;
+	// a last-removed marker whose write failed, retried on later passes: dropping it would leave
+	// getLastRemoved() reporting a boundary the entries behind it have already been deleted past
+	let pendingLastRemoved: number | undefined;
+	const isRocksAuditStore = auditStore instanceof RocksTransactionLogStore;
+	// A pass yields, so the store can be closed underneath it. Every touch of the environment after a
+	// resume — cursor advance, cursor release, marker write, re-arm — has to re-check.
+	const storeClosing = () => auditStore.rootStore.status === 'closed' || auditStore.rootStore.status === 'closing';
 	onStorageReclamation(rootStore.path, (priority) => {
 		cleanupPriority = priority; // update the priority
 		if (priority) {
@@ -175,28 +193,23 @@ export function openAuditStore(rootStore) {
 	/**
 	 * Schedules a pass of the audit cleanup loop. The returned promise fulfills once the pass that
 	 * serves this call has finished, so callers (notably tests) can await completion instead of
-	 * guessing a delay. Two things it does not promise: a pass removes at most
+	 * guessing a delay. Two things it does not promise: an LMDB pass removes at most
 	 * MAX_DELETES_PER_CLEANUP entries before rescheduling itself, so fulfillment means "that pass
 	 * finished", not "the audit log is fully pruned"; and a pass that failed logs and fulfills rather
 	 * than rejecting, since this promise doubles as the loop's serialization barrier.
 	 */
 	function scheduleAuditCleanup(newCleanupDelay?: number): Promise<void> {
 		// Skip audit cleanup/purge in read-only mode
-		if (isReadOnlyMode()) return Promise.resolve();
-		if (auditStore instanceof RocksTransactionLogStore) {
-			auditStore.rootStore.purgeLogs({
-				before: Date.now() - auditRetention / (1 + cleanupPriority * cleanupPriority),
-			});
-			return Promise.resolve();
-		}
+		if (cleanupStopped || isReadOnlyMode()) return Promise.resolve();
 
-		if (newCleanupDelay) auditCleanupDelay = newCleanupDelay;
+		if (newCleanupDelay) auditStore.auditCleanupDelay = newCleanupDelay;
 		// the pass we are about to cancel has not started, so its callers are handed over to this one
 		const supersededResolve = pendingCleanupResolve;
 		clearTimeout(pendingCleanup);
 		const resolution = new Promise<void>((resolve) => {
 			pendingCleanupResolve = resolve;
-			pendingCleanup = setTimeout(async () => {
+			const runCleanupPass = async () => {
+				pendingCleanup = null;
 				pendingCleanupResolve = null; // started, so a later schedule can no longer cancel this pass
 				// claim the serialization slot before yielding: assigning it after the await lets every
 				// pass released by the same resolution run concurrently over the same range
@@ -204,67 +217,146 @@ export function openAuditStore(rootStore) {
 				lastCleanupResolution = resolution;
 				await previousCleanup;
 				// query for audit entries that are old
-				if (auditStore.rootStore.status === 'closed' || auditStore.rootStore.status === 'closing') {
+				if (cleanupStopped || storeClosing()) {
 					// nothing to clean up and nothing to reschedule, but leaving `resolution` pending would
 					// wedge the loop permanently: it is now the resolution every later pass awaits
 					resolve();
 					return;
 				}
+				const passCleanupPriority = cleanupPriority;
 				let deleted = 0;
 				let lastKey: any;
 				try {
-					for (const auditRecord of auditStore.getRange({
-						start: 1, // must not be zero or it will be interpreted as null and overlap with symbols in search
-						snapshot: false,
-						end: Date.now() - auditRetention / (1 + cleanupPriority * cleanupPriority), // remove up until the audit retention time, reducing audit retention time if cleanup is higher priority
-					})) {
+					if (isRocksAuditStore) {
+						auditStore.rootStore.purgeLogs({
+							before: Date.now() - auditRetention / (1 + passCleanupPriority * passCleanupPriority),
+						});
+					} else {
+						// Driven explicitly rather than with for-of: this loop suspends on the awaits below, and a
+						// close landing mid-pass closes the env under it. for-of calls next() before the body, so a
+						// check inside the body advances the cursor first — the guard has to precede every next().
+						const entries = auditStore
+							.getRange({
+								start: 1, // must not be zero or it will be interpreted as null and overlap with symbols in search
+								snapshot: false,
+								end: Date.now() - auditRetention / (1 + passCleanupPriority * passCleanupPriority), // remove up until the audit retention time, reducing audit retention time if cleanup is higher priority
+							})
+							[Symbol.iterator]();
 						try {
-							// awaited so a rejection (not just a synchronous throw) is caught here instead of
-							// escaping as an unhandled rejection once a later iteration's promise replaces this one
-							await removeAuditEntry(auditStore, auditRecord);
-						} catch (error) {
-							harperLogger.warn('Error removing audit entry', error);
-						}
-						lastKey = auditRecord.key;
-						await new Promise(setImmediate);
-						if (++deleted >= MAX_DELETES_PER_CLEANUP) {
-							// limit the amount we cleanup per event turn so we don't use too much memory/CPU
-							auditCleanupDelay = 10; // and keep trying very soon
-							break;
+							while (!cleanupStopped && !storeClosing()) {
+								const entry = entries.next();
+								if (entry.done) break;
+								const auditRecord = entry.value;
+								try {
+									// awaited so a rejection (not just a synchronous throw) is caught here instead of
+									// escaping as an unhandled rejection once a later iteration's promise replaces this one
+									await removeAuditEntry(auditStore, auditRecord);
+								} catch (error) {
+									harperLogger.warn('Error removing audit entry', error);
+								}
+								lastKey = auditRecord.key;
+								await new Promise(setImmediate);
+								if (++deleted >= MAX_DELETES_PER_CLEANUP) {
+									// limit the amount we cleanup per event turn so we don't use too much memory/CPU
+									auditStore.auditCleanupDelay = 10; // and keep trying very soon
+									break;
+								}
+							}
+						} finally {
+							// for-of released the underlying cursor on break; an explicit loop owes that itself.
+							// Skipped only once the root is closing: releasing a cursor into a closed env reaches
+							// native code, while a retirement that leaves the env open still owes the release.
+							if (!storeClosing()) entries.return?.();
 						}
 					}
 				} catch (error) {
-					// the timer callback is detached, so anything escaping here lands as an unhandled
-					// rejection instead of a log line — and skips the reschedule below with it
+					// a failed scan is a log line, not a retired loop: the bookkeeping and reschedule below
+					// still run
 					harperLogger.warn('Error during audit log cleanup', error);
 				} finally {
-					// resolve() first and unconditionally: updateLastRemoved() below can throw
-					// synchronously if the store transitions to closing/closed mid-pass, and a throw
-					// from the rest of this block must not skip settling `resolution` — that's the
-					// serialization barrier every later pass awaits, and never settling it wedges the
-					// cleanup loop for the life of the store.
-					resolve();
-					if (deleted === 0) {
-						// if we didn't delete anything, we can increase the delay (double until we get to one tenth of
-						// the retention time). Plain arithmetic, not `<<`/`>>`: those coerce to int32, so a
-						// sub-millisecond retention collapsed the delay to 0 permanently (0 << 1 is 0), and a
-						// retention over ~248 days grows it past 2^31 where halving it wraps negative.
-						auditCleanupDelay = Math.max(1, Math.min(auditCleanupDelay * 2, auditRetention / 10, MAX_CLEANUP_DELAY));
-					} else {
-						// if we did delete something, update our updates since timestamp
-						updateLastRemoved(auditStore, lastKey);
-						// and do updates faster
-						if (auditCleanupDelay > 100) auditCleanupDelay = auditCleanupDelay / 2;
+					try {
+						if (isRocksAuditStore) {
+							// eligibility only changes on rotation/flush, so the LMDB backoff — keyed on a per-entry
+							// delete count — would only rescan the same segments
+							auditStore.auditCleanupDelay = Math.max(
+								DEFAULT_AUDIT_CLEANUP_DELAY,
+								Math.min(auditRetention / (1 + cleanupPriority * cleanupPriority) / 10, MAX_CLEANUP_DELAY)
+							);
+						} else {
+							if (deleted === 0) {
+								// if we didn't delete anything, we can increase the delay (double until we get to one tenth of
+								// the retention time). Plain arithmetic, not `<<`/`>>`: those coerce to int32, so a
+								// sub-millisecond retention collapsed the delay to 0 permanently (0 << 1 is 0), and a
+								// retention over ~248 days grows it past 2^31 where halving it wraps negative.
+								auditStore.auditCleanupDelay = Math.max(
+									1,
+									Math.min(auditStore.auditCleanupDelay * 2, auditRetention / 10, MAX_CLEANUP_DELAY)
+								);
+							} else {
+								pendingLastRemoved = lastKey;
+								// and do updates faster
+								if (auditStore.auditCleanupDelay > 100) auditStore.auditCleanupDelay = auditStore.auditCleanupDelay / 2;
+							}
+							// skipped when the store was retired or closed mid-pass — this writes to the audit
+							// store — and carried to the next pass instead, so a failed write is not lost
+							if (pendingLastRemoved !== undefined && !cleanupStopped && !storeClosing()) {
+								const marker = pendingLastRemoved;
+								try {
+									// awaited so the barrier below covers the write, and so a rejection is logged
+									// here rather than escaping the detached timer callback
+									await updateLastRemoved(auditStore, marker);
+									if (pendingLastRemoved === marker) pendingLastRemoved = undefined;
+								} catch (error) {
+									harperLogger.warn('Error recording the last removed audit entry', error);
+								}
+							}
+						}
+					} finally {
+						// settled and re-armed whatever the bookkeeping above threw: this promise is both the
+						// serialization barrier every later pass awaits and the drain barrier
+						// stopAuditCleanup() hands its callers, so never settling it wedges both
+						resolve();
+						// both conjuncts are backstops, not the ownership rule — see DESIGN.md: the arming
+						// sites already restrict Rocks to the last worker
+						if (
+							!cleanupStopped &&
+							!storeClosing() &&
+							(!isRocksAuditStore || (getWorkerIndex() === getWorkerCount() - 1 && !pendingCleanupResolve))
+						) {
+							scheduleAuditCleanup();
+						}
 					}
-					scheduleAuditCleanup();
 				}
 				// we can run this pretty frequently since there is very little overhead to these queries
-			}, auditCleanupDelay).unref();
+			};
+			pendingCleanup = setTimeout(() => {
+				// nothing owns the timer callback's promise, so anything the pass lets escape — including a
+				// throw from the logging inside its own containment — would land as an unhandled rejection
+				runCleanupPass().catch((error) => {
+					warnContained('Error during audit log cleanup', error);
+					resolve();
+				});
+			}, auditStore.auditCleanupDelay).unref();
 		});
 		if (supersededResolve) resolution.then(supersededResolve);
 		return resolution;
 	}
 	auditStore.scheduleAuditCleanup = scheduleAuditCleanup;
+	/**
+	 * Retires the cleanup loop for good, and returns a drain barrier: the promise settles once the pass
+	 * that was already running has finished. Retirement alone only stops the loop admitting more work —
+	 * a pass suspended inside `await removeAuditEntry()` still has a write queued whose DBI the native
+	 * writer consumes later, so a caller that closes or unlinks stores must await this first. The
+	 * synchronous teardown paths cannot; the in-pass status checks are what covers them.
+	 */
+	auditStore.stopAuditCleanup = function (): Promise<void> {
+		cleanupStopped = true;
+		clearTimeout(pendingCleanup);
+		pendingCleanup = null;
+		pendingCleanupResolve?.();
+		pendingCleanupResolve = null;
+		return lastCleanupResolution ?? Promise.resolve();
+	};
 	if (getWorkerIndex() === getWorkerCount() - 1) {
 		scheduleAuditCleanup();
 	}
@@ -296,7 +388,7 @@ export function removeAuditEntry(auditStore: any, auditRecord: AuditRecord): Pro
 
 function updateLastRemoved(auditStore, lastKey) {
 	FLOAT_TARGET[0] = lastKey;
-	auditStore.put(Symbol.for('last-removed'), FLOAT_BUFFER);
+	return auditStore.put(Symbol.for('last-removed'), FLOAT_BUFFER);
 }
 
 export function getLastRemoved(auditStore) {