@harperfast/harper 5.3.0 → 5.3.1

package/resources/recordLock.ts

@@ -12,8 +12,10 @@ import { ClientError } from '../utility/errors/hdbError.ts';
  * Phase 1 layers cluster-wide exclusion on top without changing that: once the native key is held,
  * `Table.lock()` obtains a delegation for the key from its home node (see `recordLockCoordinator.ts`)
  * and calls `joinClusterRound()` on the handle. A live delegation serves repeat locks with no cluster
- * message at all. A node-scoped lock (`{ scope: 'node' }`), or any lock on a database with no
- * transport registered, skips the cluster step entirely and behaves exactly as it did in Phase 0.
+ * message at all. A node-scoped lock (`{ scope: 'node' }`) or a defaulted lock on a table declared
+ * `replicate: false` skips the cluster step entirely and behaves exactly as it did in Phase 0; so does
+ * a defaulted lock on a replicating table while no transport has ever been registered for the
+ * database — once one has been, its absence fails closed (`isClusterLockRequired`).
  */
 
 export const DEFAULT_LOCK_LEASE_MS = 30_000;
@@ -32,8 +34,9 @@ export interface RecordLockOptions {
 	/** A held lock outlives the acquiring transaction; it is released by `unlock()` or by its lease */
 	hold?: boolean;
 	/**
-	 * `'cluster'` (default) is exclusive across every participating node; `'node'` keeps Phase 0
-	 * semantics and is exclusive only across this node's worker threads.
+	 * `'cluster'` is exclusive across every participating node; `'node'` keeps Phase 0 semantics and is
+	 * exclusive only across this node's worker threads. Defaults to `'cluster'` on a replicating table
+	 * and to `'node'` on a table declared `replicate: false`, which has no cluster to lock across.
 	 */
 	scope?: 'cluster' | 'node';
 }
@@ -122,12 +125,15 @@ function requireDuration(name: string, value: unknown, fallback: number, min: nu
 	return value;
 }
 
-export function resolveLockOptions(options?: RecordLockOptions | null): ResolvedRecordLockOptions {
+export function resolveLockOptions(
+	options: RecordLockOptions | null | undefined,
+	tableReplicates: boolean
+): ResolvedRecordLockOptions {
 	if (options != null && typeof options !== 'object')
 		throw new ClientError(`Lock options must be an object, but received ${typeof options}`);
 	const hold = options?.hold ?? false;
 	if (typeof hold !== 'boolean') throw new ClientError(`Lock option hold must be a boolean, but received ${hold}`);
-	const scope = options?.scope ?? 'cluster';
+	const scope = options?.scope ?? (tableReplicates ? 'cluster' : 'node');
 	if (scope !== 'cluster' && scope !== 'node')
 		throw new ClientError(`Lock option scope must be 'cluster' or 'node', but received ${scope}`);
 	return {

package/resources/replayLogs.ts

@@ -13,7 +13,7 @@ import {
 	shouldAbortSlowReplay,
 	REPLAY_WALL_CLOCK_LIMIT_MS,
 } from './replayLogsGuards.ts';
-import { purgeAgedLogs } from './auditStore.ts';
+import { LOCAL_ONLY, purgeAgedLogs } from './auditStore.ts';
 import { get as envGet } from '../utility/environment/environmentManager.ts';
 import { CONFIG_PARAMS } from '../utility/hdbTerms.ts';
 
@@ -291,7 +291,14 @@ export function replayLogs(rootStore: RocksDatabase, tables: any, electedReplaye
 					transaction.isReplay = true;
 				}
 				context.transaction = transaction;
-				const options = { context, residencyId, nodeId, originatingOperation, version };
+				const options = {
+					context,
+					residencyId,
+					nodeId,
+					originatingOperation,
+					version,
+					localOnly: (extendedType & LOCAL_ONLY) !== 0,
+				};
 				writes++;
 				stagedWrites++;
 				switch (type) {

package/resources/scheduler/DESIGN.md

@@ -25,11 +25,12 @@ non-obvious part is what Harper's substrate does and does not offer for that:
   is taken by the legacy jobs subsystem). On constrained/directional replication topologies the
   lease may not reach every node; that limitation is inherent.
 - The alphabetical node-name tie-break deliberately mirrors replication's deterministic failover
-  convention (sorted node names in `subscriptionManager.ts`); the escalation ladder
+  convention (sorted node names in harper-pro `replication/subscriptionManager.ts`); the escalation ladder
   (`promotionWaitMs`) exists because a dead alphabetically-first node must not deadlock a
   leaderless cluster (each successive node waits one more `2 × watcher interval` rung).
-- Thread-once vs cluster-once are separate layers: `getWorkerIndex() === 0` gates to one worker per
-  node (correct in every threading mode incl. `threads: 0`); the lease gates across nodes.
+- Thread-once vs cluster-once are separate layers: `isApplicationPrimaryWorker()` (worker 0, or an
+  isolated application's dedicated worker, #2524) gates to one worker per node (correct in every
+  threading mode incl. `threads: 0`); the lease gates across nodes.
   `handleApplication` holds a cross-thread load lock with a 30s timeout, so the plugin only
   registers there — election, scheduling, and catch-up run async after.
 - Catch-up fires at most ONE missed occurrence per cron job, and a new job's `firstSeenAt` baseline

package/resources/scheduler/engine.ts

@@ -241,7 +241,36 @@ export function findMissedCronOccurrence(
  * repeatedly — a reload or redeploy replaces the component's whole job set.
  */
 export function registerComponentJobs(componentName: string, jobs: ScheduledJob[]): void {
-	unregisterComponentJobs(componentName);
+	if (isInternalName(componentName)) {
+		schedulerLogger.error?.(
+			`Component name '${componentName}' is reserved for Harper's own jobs; its jobs were not registered`
+		);
+		return;
+	}
+	installJobs(componentName, jobs);
+}
+
+/** Jobs Harper itself owns, under a reserved name no component can register or unregister. */
+export function registerInternalJobs(name: string, jobs: ScheduledJob[]): void {
+	installJobs(INTERNAL_PREFIX + name, jobs);
+}
+
+export function unregisterInternalJobs(name: string): void {
+	removeJobs(INTERNAL_PREFIX + name);
+}
+
+export function internalJobOwner(name: string): string {
+	return INTERNAL_PREFIX + name;
+}
+
+const INTERNAL_PREFIX = 'harper:';
+
+function isInternalName(componentName: string): boolean {
+	return componentName.startsWith(INTERNAL_PREFIX);
+}
+
+function installJobs(componentName: string, jobs: ScheduledJob[]): void {
+	removeJobs(componentName);
 	const jobMap = new Map<string, RegisteredJob>();
 	for (const job of jobs) {
 		jobMap.set(job.name, { ...job, running: false });
@@ -267,6 +296,10 @@ export function registerComponentJobs(componentName: string, jobs: ScheduledJob[
  * harmless and resolves on the next worker restart.
  */
 export function unregisterComponentJobs(componentName: string): void {
+	if (!isInternalName(componentName)) removeJobs(componentName);
+}
+
+function removeJobs(componentName: string): void {
 	const jobMap = jobsByComponent.get(componentName);
 	if (!jobMap) return;
 	for (const job of jobMap.values()) {
@@ -312,7 +345,7 @@ export function runFailoverCheckForTests(): Promise<void> {
 /** Reset all engine state and timers. Intended for tests. */
 export function stopSchedulerEngine(): void {
 	for (const componentName of [...jobsByComponent.keys()]) {
-		unregisterComponentJobs(componentName);
+		removeJobs(componentName);
 	}
 	if (heartbeatTimer) clearInterval(heartbeatTimer);
 	if (failoverWatcherTimer) clearInterval(failoverWatcherTimer);

package/resources/search.ts

@@ -129,7 +129,7 @@ export function executeConditions(
 			// matches — the index decides whether building it beats evaluating the predicate.
 			const candidateKeys =
 				pushdownIndex.candidateKeyFilter && siblings.length > 0
-					? planCandidateKeys(siblings, table, txn, !recordGuards)
+					? planCandidateKeys(siblings, table, txn, !recordGuards, Boolean(pushdownIndex.candidateKeyProbe))
 					: undefined;
 			// Execute a COPY carrying the pushed-down predicate, never mutating the caller's condition
 			// object (which may be reused across requests with different users / guards).
@@ -412,6 +412,16 @@ function composeRecordFilter(recordFilters, table, context): (primaryKey: Id, su
 export interface CandidateKeyPlan {
 	/** Planner estimate of the matching record count; the index sizes its work budget from it. */
 	estimatedCount: number;
+	/** Optional point-membership gate for exact conditions an index store can probe directly. */
+	probe?: {
+		/** Whether the probe covers every pushed-down condition and no opaque record guard remains. */
+		complete: boolean;
+		/** Planner estimate for the conditions this probe actually covers. */
+		estimatedCount: number;
+		/** Worst-case synchronous index reads performed by one call to `has`. */
+		maxReadsPerCandidate: number;
+		has(primaryKey: Id): boolean;
+	};
 	/**
 	 * The matching primary keys, with `complete` true when the set alone decides admission, so the
 	 * index may drop the traversal predicate. That needs every pushed-down condition covered, no
@@ -420,7 +430,13 @@ export interface CandidateKeyPlan {
 	 * superset that gates the predicate instead of replacing it. Null when even the narrowest term
 	 * overruns, which means the whole plan was not worth what it read.
 	 */
-	collect(maxKeys: number): { keys: CandidateKeySet; complete: boolean } | null;
+	collect(maxKeys: number): CandidateKeyCollection | null;
+	collect(maxKeys: number, cooperate: () => Promise<void>): Promise<CandidateKeyCollection | null>;
+}
+
+interface CandidateKeyCollection {
+	keys: CandidateKeySet;
+	complete: boolean;
 }
 
 /**
@@ -470,6 +486,9 @@ interface CandidateKeyScan {
 	index: any;
 	range: any;
 	estimatedCount: number;
+	predicateExact: boolean;
+	probeValue?: any;
+	canProbe?: boolean;
 }
 /** Scans whose UNION is one AND term; a single-element `scans` is a leaf condition. */
 interface CandidateKeyTerm {
@@ -498,7 +517,7 @@ const CANDIDATE_KEY_COMPARATORS = new Set([
  * Plan one leaf condition as an index range scan, or undefined when its matches cannot be read off
  * an index exactly, or when the scan's size cannot be estimated.
  */
-function planCandidateKeyScan(condition, table): CandidateKeyScan | undefined {
+function planCandidateKeyScan(condition, table, includePointProbe: boolean): CandidateKeyScan | undefined {
 	if (condition.negated) return undefined;
 	const attributeName = condition.attribute ?? condition[0];
 	if (typeof attributeName !== 'string' || attributeName === table.primaryKey) return undefined;
@@ -516,6 +535,16 @@ function planCandidateKeyScan(condition, table): CandidateKeyScan | undefined {
 		return undefined;
 	let value = condition[1] ?? condition.value;
 	if (value instanceof Date) value = value.getTime();
+	const attributeType = findAttribute(table.attributes, attributeName)?.type;
+	const predicateExact =
+		comparator !== 'equals' ||
+		value === null ||
+		typeof value === 'string' ||
+		typeof value === 'boolean' ||
+		attributeType === 'Int' ||
+		attributeType === 'Long' ||
+		attributeType === 'Float' ||
+		attributeType === 'BigInt';
 	let start;
 	let end;
 	let inclusiveEnd = false;
@@ -585,22 +614,67 @@ function planCandidateKeyScan(condition, table): CandidateKeyScan | undefined {
 		if (!(belowTrue >= 0)) return undefined;
 		estimatedCount += belowTrue;
 	}
-	return { index, range: { start, end, inclusiveEnd, exclusiveStart }, estimatedCount };
+	const canProbe = includePointProbe && comparator === 'equals' && typeof index.hasIndexEntry === 'function';
+	return {
+		index,
+		range: { start, end, inclusiveEnd, exclusiveStart },
+		estimatedCount,
+		predicateExact,
+		...(includePointProbe ? { probeValue: canProbe ? value : undefined, canProbe } : null),
+	};
 }
 
-function planCandidateKeys(conditions, table, transaction, guardFree: boolean): CandidateKeyPlan | undefined {
+function planCandidateKeys(
+	conditions,
+	table,
+	transaction,
+	guardFree: boolean,
+	includePointProbe: boolean
+): CandidateKeyPlan | undefined {
 	const terms: CandidateKeyTerm[] = [];
-	const planned = collectCandidateKeyTerms(conditions, table, terms) && guardFree;
+	const planned = collectCandidateKeyTerms(conditions, table, terms, includePointProbe) && guardFree;
 	if (terms.length === 0) return undefined;
 	const estimatedCount = terms.reduce((narrowest, term) => Math.min(narrowest, term.estimatedCount), Infinity);
-	return {
-		estimatedCount,
-		collect: (maxKeys) => collectCandidateKeys(terms, transaction, maxKeys, planned),
+	function collect(maxKeys: number): CandidateKeyCollection | null;
+	function collect(maxKeys: number, cooperate: () => Promise<void>): Promise<CandidateKeyCollection | null>;
+	function collect(maxKeys: number, cooperate?: () => Promise<void>) {
+		return cooperate
+			? collectCandidateKeysCooperatively(terms, transaction, maxKeys, planned, cooperate)
+			: collectCandidateKeys(terms, transaction, maxKeys, planned);
+	}
+	const plan: CandidateKeyPlan = { estimatedCount, collect };
+	if (!includePointProbe) return plan;
+	const probeTerms = terms.filter((term) => term.scans.every((scan) => scan.canProbe));
+	if (probeTerms.length === 0) return plan;
+	const probeEstimatedCount = probeTerms.reduce(
+		(narrowest, term) => Math.min(narrowest, term.estimatedCount),
+		Infinity
+	);
+	const maxProbeReadsPerCandidate = probeTerms.reduce((total, term) => total + term.scans.length, 0);
+	const probeOptions = { transaction };
+	plan.probe = {
+		complete: planned && probeTerms.length === terms.length,
+		estimatedCount: probeEstimatedCount,
+		maxReadsPerCandidate: maxProbeReadsPerCandidate,
+		has(primaryKey) {
+			for (const term of probeTerms) {
+				let matched = false;
+				for (const scan of term.scans) {
+					if (scan.index.hasIndexEntry(scan.probeValue, primaryKey, probeOptions)) {
+						matched = true;
+						break;
+					}
+				}
+				if (!matched) return false;
+			}
+			return true;
+		},
 	};
+	return plan;
 }
 
 /** Returns whether every condition was covered. */
-function collectCandidateKeyTerms(conditions, table, terms: CandidateKeyTerm[]): boolean {
+function collectCandidateKeyTerms(conditions, table, terms: CandidateKeyTerm[], includePointProbe: boolean): boolean {
 	let complete = true;
 	for (const condition of conditions) {
 		if (condition.conditions) {
@@ -609,7 +683,7 @@ function collectCandidateKeyTerms(conditions, table, terms: CandidateKeyTerm[]):
 				// that branch matches, and admission may over-admit but never omit.
 				const scans: CandidateKeyScan[] = [];
 				for (const child of condition.conditions) {
-					const scan = child.conditions ? undefined : planCandidateKeyScan(child, table);
+					const scan = child.conditions ? undefined : planCandidateKeyScan(child, table, includePointProbe);
 					if (!scan) {
 						complete = false;
 						break;
@@ -618,17 +692,22 @@ function collectCandidateKeyTerms(conditions, table, terms: CandidateKeyTerm[]):
 				}
 				if (scans.length === condition.conditions.length) {
 					terms.push({ scans, estimatedCount: scans.reduce((total, scan) => total + scan.estimatedCount, 0) });
+					if (scans.some((scan) => !scan.predicateExact)) complete = false;
 				}
-			} else if (!collectCandidateKeyTerms(condition.conditions, table, terms)) complete = false;
+			} else if (!collectCandidateKeyTerms(condition.conditions, table, terms, includePointProbe)) complete = false;
 			continue;
 		}
-		const scan = planCandidateKeyScan(condition, table);
-		if (scan) terms.push({ scans: [scan], estimatedCount: scan.estimatedCount });
-		else complete = false;
+		const scan = planCandidateKeyScan(condition, table, includePointProbe);
+		if (scan) {
+			terms.push({ scans: [scan], estimatedCount: scan.estimatedCount });
+			if (!scan.predicateExact) complete = false;
+		} else complete = false;
 	}
 	return complete;
 }
 
+const CANDIDATE_KEY_COLLECTION_YIELD_INTERVAL = 256;
+
 /**
  * Run the planned scans narrowest-estimate first, intersecting on storage key identity. `maxKeys`
  * bounds index entries READ, not keys retained — a wide later scan whose intersection is tiny still
@@ -641,7 +720,7 @@ function collectCandidateKeys(
 	transaction,
 	maxKeys: number,
 	planned: boolean
-): { keys: CandidateKeySet; complete: boolean } | null {
+): CandidateKeyCollection | null {
 	const ordered = terms.length > 1 ? terms.slice().sort((a, b) => a.estimatedCount - b.estimatedCount) : terms;
 	let matched: CandidateKeySet | undefined;
 	let complete = planned;
@@ -675,6 +754,65 @@ function collectCandidateKeys(
 	return { keys: matched!, complete };
 }
 
+function* cooperativeCandidateKeyCollectionSteps(
+	terms: CandidateKeyTerm[],
+	transaction,
+	maxKeys: number,
+	planned: boolean
+): Generator<void, CandidateKeyCollection | null> {
+	const ordered = terms.length > 1 ? terms.slice().sort((a, b) => a.estimatedCount - b.estimatedCount) : terms;
+	let matched: CandidateKeySet | undefined;
+	let complete = planned;
+	let scanned = 0;
+	for (const term of ordered) {
+		if (matched && term.estimatedCount > maxKeys - scanned) {
+			complete = false;
+			continue;
+		}
+		const next = new CandidateKeySet();
+		let overran = false;
+		for (const scan of term.scans) {
+			for (const { value: primaryKey } of scan.index.getRange({ ...scan.range, values: true, transaction })) {
+				if (++scanned > maxKeys) {
+					overran = true;
+					break;
+				}
+				if (!matched || matched.has(primaryKey)) next.add(primaryKey);
+				if (scanned % CANDIDATE_KEY_COLLECTION_YIELD_INTERVAL === 0) yield;
+			}
+			if (overran) break;
+		}
+		if (overran) {
+			if (!matched) return null;
+			complete = false;
+			break;
+		}
+		matched = next;
+		if (matched.size === 0) break;
+	}
+	return { keys: matched!, complete };
+}
+
+async function collectCandidateKeysCooperatively(
+	terms: CandidateKeyTerm[],
+	transaction,
+	maxKeys: number,
+	planned: boolean,
+	cooperate: () => Promise<void>
+): Promise<CandidateKeyCollection | null> {
+	const steps = cooperativeCandidateKeyCollectionSteps(terms, transaction, maxKeys, planned);
+	try {
+		let next = steps.next();
+		while (!next.done) {
+			await cooperate();
+			next = steps.next();
+		}
+		return next.value;
+	} finally {
+		steps.return(null);
+	}
+}
+
 /**
  * A record's index entries are not adjacent — the composite `[indexedValue, primaryKey]` key sorts
  * on the indexed value first — so comparing neighbours cannot collapse them and the scan has to

package/resources/transaction.ts

@@ -7,6 +7,7 @@ import {
 	type Transaction,
 } from './DatabaseTransaction.ts';
 import { AsyncLocalStorage } from 'async_hooks';
+import * as harperLogger from '../utility/logging/harper_logger.ts';
 
 export const contextStorage = new AsyncLocalStorage<Context>();
 
@@ -53,6 +54,13 @@ export function transaction<T>(
 	if (context.replicatedConfirmation) transaction.replicatedConfirmation = context.replicatedConfirmation;
 	if (context.sourceApply) transaction.sourceApply = true;
 	transaction.setContext(context);
+
+	// Cancellation belongs to the request (harper#2001): once its signal aborts, the chain refuses new
+	// writes and releases what it staged (DatabaseTransaction.admitRequestWrite). sourceApply is exempt,
+	// having no resume path (harper-pro#348); work that must outlive the client runs on a context without
+	// the signal (resources/DESIGN.md).
+	if (!transaction.sourceApply) transaction.requestSignal = context.signal;
+
 	let result;
 	try {
 		result =
@@ -68,18 +76,55 @@ export function transaction<T>(
 	return onComplete(result);
 	// when the transaction function completes, run this to commit the transaction
 	function onComplete(result) {
-		const committed = transaction.commit({ doneWriting: true });
+		let committed;
+		try {
+			committed = transaction.commit({ doneWriting: true });
+		} catch (error) {
+			return onCommitError(error, result);
+		}
 		if ((committed as any).then) {
-			return (committed as any).then(() => {
-				return result;
-			});
+			return (committed as any).then(
+				() => result,
+				(error) => onCommitError(error, result)
+			);
 		} else {
 			return result;
 		}
 	}
+	function onCommitError(error, result) {
+		try {
+			if (typeof result?.onDone === 'function') result.onDone();
+		} catch (cleanupError) {
+			harperLogger.debug?.('closing results after a failed commit', cleanupError);
+		}
+		abortAndThrow(error, false);
+	}
 	// if the transaction function throws an error, we abort
 	function onError(error) {
-		transaction.abort();
+		abortAndThrow(error, true);
+	}
+	function abortAndThrow(error, callbackThrew: boolean): never {
+		// A commit attempt that has not reached its native outcome owns its own teardown — a handler that
+		// fired txn.commit() without awaiting it can get here while it is still running, and aborting
+		// would clear the writes it is committing and abort the handle it is committing them through.
+		// Ownership of the scope still ends here, or that attempt would rotate the instance back OPEN with
+		// no wrapper left to commit or abort it; abandonScope() also defers the iterator cleanup below to
+		// the point where the attempt settles.
+		if (transaction.isChainCommitting()) {
+			transaction.abandonScope();
+		} else {
+			try {
+				// "retain only while read iterators still own the handle", the same rule
+				// abortAfterCommitError uses — so the two layers cannot undo each other one frame apart.
+				transaction.abort(true);
+			} catch (abortError) {
+				harperLogger.debug?.('aborting transaction after an error', abortError);
+			}
+			// Only when the callback threw: then nothing was returned, so no live response can own an iterator
+			// it opened, and the retained handle would wait on an onDone() nobody will call. A callback that
+			// completed may have handed an iterator out; the monitor reclaims it if that consumer abandons it.
+			if (callbackThrew) transaction.closeOwnedReadIterators();
+		}
 		throw error;
 	}
 }

package/resources/transactionBroadcast.ts

@@ -1,10 +1,33 @@
+import { basename } from 'node:path';
 import { warn } from '../utility/logging/harper_logger.js';
+import { DatabaseClosingError, DatabaseGenerationChangedError } from '../utility/errors/hdbError.ts';
 import { IterableEventQueue } from './IterableEventQueue.ts';
 import { keyArrayToString } from './Resources.ts';
 import type { Id } from './ResourceInterface.ts';
 
 const allSubscriptions = Object.create(null); // using it as a map that doesn't change much
 const allSameThreadSubscriptions = Object.create(null); // using it as a map that doesn't change much
+// The handle each database path was last opened with on this thread, kept as a token so a retired database's closed
+// store graph is not retained. A registration through any other handle would join the entry the current store's
+// commits drive while reading through that handle's closed stores.
+const HANDLE_TOKEN = Symbol('subscription-handle');
+const currentHandles = new Map<
+	string,
+	{ token: symbol; generationId: string | undefined; tracksGeneration: boolean }
+>();
+
+// A store with no generation (LMDB) is never replaced by a copy, so its reopen is only ever a close.
+function generationChanged(
+	generationId: string | null | undefined,
+	currentId: string | null | undefined,
+	tracksGeneration: boolean
+): boolean {
+	return tracksGeneration && !(generationId != null && generationId === currentId);
+}
+
+function closingError(auditStore: any, path: string): DatabaseClosingError {
+	return new DatabaseClosingError(auditStore?.rootStore?.databaseName ?? basename(path));
+}
 /**
  * This module/function is responsible for the main work of tracking subscriptions and listening for new transactions
  * that have occurred on any thread, and then reading through the transaction log to notify listeners. This is
@@ -24,6 +47,15 @@ export function addSubscription(table, key, listener?: (key) => any, startTime?:
 	if (!path) {
 		throw new Error('No path for table primary store');
 	}
+	const generationId = table.auditStore?.databaseGeneration?.id ?? null;
+	const current = currentHandles.get(path);
+	const replaced = current !== undefined && table.auditStore?.[HANDLE_TOKEN] !== current.token;
+	if (replaced || table.auditStore?.rootStore?.status === 'closed') {
+		if (options?.scope === 'full-database') return;
+		throw replaced && generationChanged(generationId, current.generationId, current.tracksGeneration)
+			? new DatabaseGenerationChangedError()
+			: closingError(table.auditStore, path);
+	}
 	if (options?.crossThreads === false) {
 		// we are only listening for commits on our own thread, so we use a separate subscriber and sequencer tracker
 		databaseSubscriptions = allSameThreadSubscriptions[path] || (allSameThreadSubscriptions[path] = []);
@@ -60,6 +92,7 @@ export function addSubscription(table, key, listener?: (key) => any, startTime?:
 		}
 	}
 	databaseSubscriptions.auditStore = table.auditStore;
+	databaseSubscriptions.generationId ??= generationId;
 	if (databaseSubscriptions.lastTxnTime == null) {
 		databaseSubscriptions.lastTxnTime = Date.now();
 	}
@@ -92,6 +125,42 @@ export function addSubscription(table, key, listener?: (key) => any, startTime?:
 	return subscription;
 }
 
+/**
+ * End every subscription this thread registered on the database before `auditStore` reopened it: its commit
+ * listener and its table's stores belong to the closed handle, so it can never deliver again. On a store that
+ * tracks generations, another or an unknown one must resynchronize; otherwise the retryable
+ * `DatabaseClosingError`. `auditStore` becomes the only handle `addSubscription` accepts on the path before
+ * any listener runs.
+ */
+export function endSubscriptionsFromEarlierHandles(auditStore: any, tracksGeneration: boolean): void {
+	const path = auditStore.rootStore.path;
+	const generationId = auditStore.databaseGeneration?.id;
+	const token = Symbol(basename(path));
+	auditStore[HANDLE_TOKEN] = token;
+	currentHandles.set(path, { token, generationId, tracksGeneration });
+	for (const registry of [allSubscriptions, allSameThreadSubscriptions]) {
+		const databaseSubscriptions = registry[path];
+		if (!databaseSubscriptions) continue;
+		delete registry[path];
+		const changed = generationChanged(databaseSubscriptions.generationId, generationId, tracksGeneration);
+		for (const tableId in databaseSubscriptions) {
+			const tableSubscriptions = databaseSubscriptions[tableId];
+			if (!(tableSubscriptions instanceof Map)) continue;
+			for (const keySubscriptions of tableSubscriptions.values()) {
+				for (const subscription of [...keySubscriptions]) {
+					try {
+						subscription.close(changed ? new DatabaseGenerationChangedError() : closingError(auditStore, path));
+					} catch (error) {
+						try {
+							warn(error);
+						} catch {}
+					}
+				}
+			}
+		}
+	}
+}
+
 /**
  * This is the class that is returned from subscribe calls and provide the interface to set a callback, end the
  * subscription and get the initial state.

package/security/DESIGN.md

@@ -29,7 +29,7 @@ Four constraints that look like choices but are not:
 3. **`createOperationToken`, not `createTokens`.** `createTokens` overwrites `hdb_user.refresh_token` as a side effect, so minting for CI would silently revoke whatever credential that user already held (#2018) — the exact problem this feature removes.
 4. **The role is the boundary; the per-policy `operations` allowlist only narrows it.** Least privilege is primarily the role of the user the policy names. A policy may _optionally_ carry an `operations` scope, which can only subtract from that role — never add to it. It is deliberately not merged into `permission.operations`: gate 2 in `operation_authorization.ts` treats an explicit listing of an SU-only operation as a deliberate grant, so reusing that field would _widen_ where this must only narrow. The scope is carried as a separate `tokenOperations` claim and intersected ahead of every early return, including the super_user bypass.
 
-   Its enforcement surface is the operations API and SQL (`verifyPerms` / `verifyPermsAST`) — **not** the application REST/GraphQL resource path, which authorizes through table-level `checkPermission` and does not consult the scope. A scoped token therefore still carries its role's full CRUD there, which is why the role has to be least-privilege on its own; the scope is defense in depth, not a substitute. Closing that gap is a follow-up on the same surface as CORE-3061. Because a second authorization mechanism beside roles is one more place for the two to disagree, whether to keep this at all is an open design question on #2173 rather than a settled constraint.
+   Its enforcement surface is the operations API and SQL (`verifyPerms` / `verifyPermsAST`) — **not** the application REST/GraphQL resource path, which authorizes through table-level `checkPermission` and does not consult the scope. A scoped token therefore still carries its role's full CRUD there, which is why the role has to be least-privilege on its own; the scope is defense in depth, not a substitute. Closing that gap is tracked in #2201. The scope shipped with #2173 and is documented in the operations API reference; whether to keep a second authorization mechanism beside roles at all — one more place for the two to disagree — was left open there.
 
    Naming `sql` in a scope grants the SQL interface, not unrestricted DML through it: a write statement additionally requires its matching data operation (`insert`/`update`/`delete`) in scope. That is what keeps `read_only` — which expands to include `sql` — from admitting a DELETE, given that `verifyPermsAST` returns early for a super_user before any table check runs.
 
@@ -99,32 +99,11 @@ early-returns (super_user, structure_user, system-table allowances): persisted r
 `super_user` with other permission keys, but inline roles can combine `structure_user` with an
 allowlist, and the gate ordering is what keeps unlisted schema ops unreachable.
 
-## TLS hot-reload: cert vs. private key follow two different propagation paths (`security/keys.ts`)
-
-A renewed **certificate** and a renewed **private key** reach a worker's live TLS secure context
-by completely separate routes, and the two must reconverge or HTTPS breaks on that worker.
-Certificates propagate through data: only the main thread watches the cert file (`isMainThread`
-guard in `loadCertificates`) and writes the new PEM into the `system.hdb_certificate` table; every
-worker is subscribed to that table and rebuilds its secure contexts (`updateTLS` inside
-`createTLSSelector`) on the notification. Private keys never touch the table — each worker watches
-its own key file (the private-key `loadAndWatch` has no `isMainThread` guard) and loads the PEM
-straight into its in-thread `privateKeys` map. `getPrivateKeyByName` reads that map first, so an
-already-built secure context has the key bytes baked in (`setCert`/`setKey` at build time); a later
-map update does not touch contexts already built.
-
-The hazard when a rotation changes **both**: the cert can win the race to a worker (table write +
-subscription) and trigger `updateTLS` before that worker has reloaded the matching key, producing a
-context that pairs the new cert with the old key — every handshake on it then fails, and nothing
-rebuilds it until the _next_ cert-table change. The fix: a private-key reload (`handlePrivateKeyReload`,
-the single sink for both the chokidar watcher and PR #1394's periodic poll) triggers a debounced
-rebuild of every live selector via the module-level `liveTLSRebuilders` set, so the worker reconverges
-on its own. Subtleties to preserve: the rotation guard (`previous !== undefined && previous !== key`)
-must skip both the initial load and identical-content reloads or watchers thrash; transient one-shot
-selectors (`getReplicationCert`) pass `liveReload=false` so they don't accumulate in the never-pruned
-set; and the cert subscription shares the same debounced `scheduleRebuild` (same 1500ms cadence), so
-its coalescing must stay a superset-safe no-op for the single-swap #586 case. Regression coverage:
-`integrationTests/security/cert-key-reload.test.ts` deterministically pins the cert-before-key ordering
-(it fails by design without the rebuild trigger); `cert-reload.test.ts` guards the cert-only #586 path.
+## TLS hot-reload publishes matching configured pairs (`security/keys.ts`)
+
+`loadCertificates` publishes a configured leaf only after `X509Certificate.checkPrivateKey` validates the exact PEM being stored. Only the main thread watches the certificate and key directories; its filename-filtered, shallow watches and five-minute safety poll share a cert/key fingerprint. An incomplete pair returns `false` so it is retried, with a pending-pair warning and an error after 30 seconds (immediate when the new certificate name has no non-self-signed record, then hourly). Restoring the last applied files after an aborted renewal re-enters the loader to clear the pending alarm. CA-only records remain independent of the leaf key. Workers use the `hdb_certificate` subscription as their reload trigger and read configured key files when building contexts; no worker key watcher or poll races that notification. Configured keys remain in `getPrivateKeys()` for key retrieval, but entries populated from files are refreshed for retrieval and rebuilds, retaining their cached value while the file is temporarily unreadable; explicit values that differ from the last file-derived key still take precedence. A directory that cannot be watched because it is not readable falls back to Node file polling, which remains armed while configured files are absent, independently of the optional five-minute safety poll. Enforced by the rename-order, per-worker, poll-disabled cases in `integrationTests/security/cert-key-reload.test.ts` and loader tests in `unitTests/security/keys.test.js`.
+
+An unrelated table notification during a key-first installation gap, or a second rotation between publication and a worker's key read, can still produce a transient mismatch; the trust-aware last-good retention below protects serving while the next complete pair propagates. The older-file guard still prevents disk from replacing a newer replicated certificate.
 
 **Publication is transactional (#2382).** `updateTLS` builds the entire replacement state —
 hostname→context map, CA map, and default candidate — into pass-local candidates and reconciles the
@@ -144,9 +123,9 @@ the old state (availability outranks the drop in that corner) while the failure
 row is a pass failure like any other (reported through the signature throttle, armed for retry) and
 its trust drops until it heals. A failed pass arms a
 self-retry on the shared debounce with a per-signature backoff (1.5s doubling to 5min) and
-signature-throttled logging; external triggers (table subscription, key reload) stay at the plain
-debounce. `loadAndWatch` latches its mtime before the callback for chokidar/poll dedupe, but rolls
-the latch back on a synchronous throw or a rejected callback promise (equality-guarded so a stale
+signature-throttled logging; external triggers (table subscription) stay at the plain
+debounce. `loadAndWatch` latches its cert/key fingerprint before the callback for chokidar/poll dedupe, but rolls
+the latch back on a false result, synchronous throw, or rejected callback promise (equality-guarded so a stale
 rejection cannot unlatch a newer reload) — the latch means "last successfully applied", so the
 periodic poll can heal a lost `hdb_certificate` write instead of deduplicating it forever.