@harperfast/harper 5.2.12 → 5.2.14

package/resources/Table.ts

@@ -49,6 +49,7 @@ import {
 	ServerError,
 	AccessViolation,
 	ValidationError,
+	appendErrorContext,
 	type ValidationIssue,
 } from '../utility/errors/hdbError.ts';
 import * as signalling from '../utility/signalling.ts';
@@ -139,6 +140,20 @@ const EVICTION_BATCH_SIZE = 100;
 // letting an unbounded number of open transactions (and their snapshots) accumulate.
 const MAX_INFLIGHT_EVICTION_BATCHES = 4;
 const CACHEABLE_STATUS_CODES = new Set([200, 203, 204, 206, 300, 301, 308, 404, 405, 410, 414, 501]);
+// Smallest forward sample `getRecordCount` will extrapolate a record rate from; below it the scan runs
+// to completion and reports an exact count.
+const MIN_ESTIMATOR_SAMPLE = 1_000;
+// Budget intervals the forward scan may spend before it must estimate rather than keep scanning.
+const MAX_ESTIMATE_CHECKPOINTS = 20;
+// A store estimate's `count`, or 0 when the store answered with a shape that cannot be trusted --
+// DESIGN.md's invariant for this API family is that such an answer degrades rather than poisons.
+function usableCount(estimate: any): number {
+	const { count, confidence } = estimate ?? {};
+	// `confidence` needs its own finiteness check, not just the range: `null >= 0 && null <= 1` is true
+	if (!Number.isFinite(count) || count < 0 || !Number.isFinite(confidence) || confidence < 0 || confidence > 1)
+		return 0;
+	return count;
+}
 envMngr.initSync();
 const LMDB_PREFETCH_WRITES = envMngr.get(CONFIG_PARAMS.STORAGE_PREFETCHWRITES);
 const LOCK_TIMEOUT = 10000;
@@ -1595,8 +1610,7 @@ export function makeTable(options) {
 					records: './', // an href to the records themselves
 					name: tableName,
 					database: databaseName,
-					auditSize:
-						auditStore instanceof RocksDatabase ? auditStore.getKeysCount() : auditStore?.getStats().entryCount,
+					auditSize: auditStore?.getStats().entryCount,
 					attributes,
 					recordCount: undefined,
 					estimatedRecordRange: undefined,
@@ -2581,7 +2595,10 @@ export function makeTable(options) {
 									// getRange yields ascending by audit-log key, so the first entry is the oldest retained.
 									// Mirror replicationConnection's retention check and the cleanup key basis (localTime ??
 									// version). Fall back to the nominal time-based purge floor when the log is empty/unavailable.
-									for (const entry of auditStore.getRange({ start: 1, log: options?.nodeId })) {
+									for (const entry of auditStore.getRange({
+										start: 1,
+										log: auditStore.logFor(options?.nodeId, options?.viaNodeId).name,
+									})) {
 										oldestRetainedAuditTime = entry.localTime ?? entry.version;
 										break;
 									}
@@ -2605,7 +2622,7 @@ export function makeTable(options) {
 							// A recommit of the same transaction survived that skip only because the old write batch
 							// still carried the put; a fresh-transaction replay (ERR_TRY_AGAIN) would drop the write.
 							if (isRocksDB && !stagedOwnAuditEntry && dedupVersionCouldBeRetained(txnTime)) {
-								const priorAudit = auditStore.get(txnTime, tableId, id, options?.nodeId);
+								const priorAudit = auditStore.get(txnTime, tableId, id, options?.nodeId, options?.viaNodeId);
 								if (
 									priorAudit &&
 									priorAudit.version === txnTime &&
@@ -2669,7 +2686,7 @@ export function makeTable(options) {
 							const isReDeliveredDuplicate = () => {
 								if (stagedOwnAuditEntry) return false;
 								if (!dedupVersionCouldBeRetained(txnTime)) return false; // pre-retention version — skip the end-of-log scan (best-effort; see above)
-								const duplicate = auditStore.get(txnTime, tableId, id, options?.nodeId);
+								const duplicate = auditStore.get(txnTime, tableId, id, options?.nodeId, options?.viaNodeId);
 								return (
 									duplicate &&
 									duplicate.version === txnTime &&
@@ -3135,6 +3152,7 @@ export function makeTable(options) {
 				commit: (txnTime, existingEntry, retry, transaction: any) => {
 					write.stagedEntry = undefined; // reset per round; set below once the removal is applied
 					write.superseded = false; // reset per round, as in the update path
+					write.skipped = false;
 					// what a preceding write in this transaction left for this key is what gets removed
 					// from the indices here, not the pre-transaction record (harper#1968)
 					const priorStagedOp = priorStagedWrite(write);
@@ -3153,6 +3171,20 @@ export function makeTable(options) {
 					if (precedesExistingVersion(txnTime, existingEntry, options?.nodeId) < 0) {
 						return;
 					}
+					const removalNodeId = options?.nodeId ?? 0; // this node's own id is 0
+					const stagedRemoval = { value: undefined, version: txnTime, nodeId: removalNodeId };
+					// re-delivered onto the removal it already made: writing again would append an audit entry
+					// that every peer forwards and re-logs in turn. On a recommit the kept write batch shows this write's own
+					// round-one tombstone as the existing entry, so a retry round repeats the first round's decision.
+					const redelivered = retry
+						? write.redelivered
+						: existingRecord == null && isSameRemoval(removalBefore(write, existingEntry), txnTime, removalNodeId);
+					write.redelivered = redelivered;
+					if (redelivered) {
+						write.stagedEntry = stagedRemoval;
+						write.skipped = true;
+						return;
+					}
 					updateIndices(id, existingRecord, null, transaction && { transaction });
 					if (audit || trackDeletes) {
 						updateRecord(
@@ -3176,7 +3208,7 @@ export function makeTable(options) {
 						// Only RocksDB's remove() takes an options object; on LMDB the 2nd arg is ifVersion, and its writes already join the batched txn.
 						removeEntry(primaryStore, existingEntry, isRocksDB && transaction ? { transaction } : undefined);
 					}
-					write.stagedEntry = { value: undefined }; // the key holds no record for the rest of this transaction
+					write.stagedEntry = stagedRemoval; // the key holds no record for the rest of this transaction
 					// the removal supersedes the nearest record an earlier write in this transaction stored
 					// (older ones were already marked by their staged successors), so its saved blobs are
 					// cleaned up post-commit unless its audit entry references them
@@ -4896,82 +4928,174 @@ export function makeTable(options) {
 			const exactCount = options?.exactCount;
 			const TIME_LIMIT = options?.timeLimit ?? 1000 / 2; // one second time limit, enforced by seeing if we are halfway through at 500ms
 			const start = performance.now();
-			// `entryCount` (the exact key count) is only needed once the scan blows the time budget --
-			// to decide whether to estimate and as the extrapolation base. On RocksDB it is a full
-			// key-only scan, so we defer it: tables that finish within budget (the common case) and
-			// `exact_count` requests never pay for it. `halfway`/`entryCount` stay 0 until first computed.
 			let entryCount = 0;
-			let halfway = 0;
-			let counted = false;
-			let completeForExact = false;
+			let remainderPhysical = 0;
+			let estimator;
+			// feature-detected per DESIGN.md's invariant for this API family; LMDB stores do not implement it
+			const canEstimate =
+				typeof primaryStore.createCountEstimator === 'function' && typeof primaryStore.estimateCount === 'function';
+			let estimatorFailed = false;
+			let warnedNoBase = false;
+			let checkpoints = 0;
+			let checkpointedEntries = 0;
 			let recordCount = 0;
 			let entriesScanned = 0;
+			let lastKey;
 			let limit: number;
-			for (const { value } of primaryStore.getRange({ start: true, lazy: true, snapshot: false })) {
+			let nextCheckAt = start + TIME_LIMIT;
+			for (const { key, value } of primaryStore.getRange({ start: true, lazy: true, snapshot: false })) {
 				if (value != null) recordCount++;
 				entriesScanned++;
+				lastKey = key;
 				await rest();
-				if (!exactCount && !completeForExact && performance.now() - start > TIME_LIMIT) {
-					if (!counted) {
-						counted = true;
-						entryCount = isRocksDB
-							? primaryStore.getKeysCount({ start: undefined })
-							: primaryStore.getStats().entryCount;
-						halfway = Math.floor(entryCount / 2);
+				// a table too small to reach the floor is small enough to finish exactly
+				if (exactCount || entriesScanned < MIN_ESTIMATOR_SAMPLE) continue;
+				const now = performance.now();
+				if (now <= nextCheckAt) continue;
+				nextCheckAt = now + TIME_LIMIT;
+				checkpoints++;
+				if (canEstimate && !estimatorFailed) {
+					try {
+						estimator ??= primaryStore.createCountEstimator({ start: true });
+						estimator.advance(lastKey, entriesScanned - checkpointedEntries);
+						checkpointedEntries = entriesScanned;
+						entryCount = usableCount(estimator.estimate());
+					} catch (error) {
+						// a store closing concurrently -- drop_table can, while this scan is parked in a yield
+						logger.debug?.('Count estimator unavailable, falling back to an exact scan', error);
+						estimatorFailed = true;
+						estimator = undefined;
+						entryCount = 0;
 					}
-					if (entriesScanned < halfway) {
-						// it is taking too long, so we will just take this sample and a sample from the end to estimate
-						limit = entriesScanned;
-						break;
+				} else if (!canEstimate) {
+					// `canEstimate` false is not the same as "LMDB": a RocksDB store whose native module predates
+					// the estimator API lands here too, and `RocksDatabase.getStats()` carries no `entryCount`.
+					try {
+						const stats = primaryStore.getStats?.();
+						entryCount = Number.isFinite(stats?.entryCount) && stats.entryCount > 0 ? stats.entryCount : 0;
+					} catch {
+						entryCount = 0;
+					}
+				}
+				if (!entryCount && canEstimate && !estimatorFailed) {
+					// Range estimates are block-granular and can report 0 for a store whose entries are still
+					// in the memtable. Without a base the escape cannot fire at all, so fall back to the
+					// whole-store property rather than silently walking the table.
+					try {
+						const wholeStore = primaryStore.getEstimatedKeyCount();
+						entryCount = Number.isFinite(wholeStore) && wholeStore > 0 ? wholeStore : 0;
+					} catch {
+						entryCount = 0;
+					}
+					if (!entryCount && !warnedNoBase) {
+						warnedNoBase = true;
+						logger.debug?.(`No usable key-count estimate for ${tableName}; counting records by full scan`);
+					}
+				}
+				// Zero is "no usable base": degrade to the exact scan. The checkpoint ceiling is what stops a
+				// base that keeps undershooting from holding the halfway test false forever and walking the
+				// whole table; the reverse sample is bounded by `limit` in turn.
+				if (
+					entryCount > 0 &&
+					(checkpoints >= MAX_ESTIMATE_CHECKPOINTS || entriesScanned < Math.floor(entryCount / 2))
+				) {
+					if (canEstimate) {
+						try {
+							const remaining = primaryStore.estimateCount({ start: lastKey, exclusiveStart: true });
+							// widened by its own reported untrustworthiness: block-granular, so it can land below
+							// the live count it is meant to bound
+							const remainingCount = usableCount(remaining);
+							remainderPhysical = remainingCount > 0 ? remainingCount * (2 - remaining.confidence) : 0;
+						} catch {
+							remainderPhysical = 0;
+						}
+						// A zero or unusable remainder is valid -- entries still in the memtable read as none
+						// through range statistics -- but it would leave `baseMax` resting on the sampled ends
+						// alone. The whole-store property is a separate, non-range source, so fall back to it.
+						if (!remainderPhysical) {
+							try {
+								const wholeStore = primaryStore.getEstimatedKeyCount();
+								if (Number.isFinite(wholeStore)) remainderPhysical = Math.max(wholeStore - entriesScanned, 0);
+							} catch {
+								remainderPhysical = 0;
+							}
+						}
 					}
-					// Past the halfway point already: finishing the scan for an exact count is cheaper
-					// than estimating. Set the flag so we stop re-evaluating the budget on each remaining iteration.
-					completeForExact = true;
+					limit = entriesScanned;
+					break;
 				}
 			}
 			if (limit) {
 				// in this case we are going to make an estimate of the table count using the first thousand
 				// entries and last thousand entries
 				const firstRecordCount = recordCount;
+				const firstKey = lastKey;
 				recordCount = 0;
 				// Bound the reverse scan explicitly. The getRange `limit` option is honored by lmdb-js but
 				// ignored by rocksdb-js; without this break the scan reads the whole table, so `recordRate`
 				// blows up to ~entryCount/(2*limit) and the estimate scales with entryCount^2 -- the source
 				// of the wildly inflated `record_count` (e.g. 20,000,000 for ~105k rows) on large RocksDB
-				// tables. The early-exit above guarantees limit < entryCount/2, so the two samples stay disjoint.
+				// tables.
 				let reverseScanned = 0;
-				for (const { value } of primaryStore.getRange({
+				// Sized independently of the forward scan. `entriesScanned` is whatever the forward pass
+				// covered before it escaped, and the checkpoint ceiling lets that run twenty budget
+				// intervals when the base keeps undershooting; matching it here would read that same count
+				// again and double the wall clock of the call this path exists to bound.
+				const reverseLimit = Math.min(limit, MIN_ESTIMATOR_SAMPLE);
+				// Disjointness is enforced against the forward scan's own last key rather than inferred from
+				// the base, which is an estimate that can overshoot by more than 2x.
+				let sampledWholeTable = false;
+				for (const { key, value } of primaryStore.getRange({
 					start: '\uffff',
 					reverse: true,
 					lazy: true,
-					limit,
+					limit: reverseLimit,
 					snapshot: false,
 				})) {
+					if (compareKeys(key, firstKey) <= 0) {
+						sampledWholeTable = true;
+						break;
+					}
 					if (value != null) recordCount++;
 					reverseScanned++;
 					await rest();
-					if (reverseScanned >= limit) break;
+					if (reverseScanned >= reverseLimit) break;
 				}
+				// the samples met, so between them they covered every entry
+				if (sampledWholeTable) return { recordCount: recordCount + firstRecordCount };
 				// Use the actual entries sampled, not limit*2: the reverse scan can yield fewer than `limit`
 				// (concurrent deletions under snapshot:false, or an overestimated entryCount), and counting
 				// those un-scanned slots would inflate the denominator and underestimate the rate.
-				const sampleSize = limit + reverseScanned;
-				const recordRate = (recordCount + firstRecordCount) / sampleSize;
-				const variance =
-					Math.pow((recordCount - firstRecordCount + 1) / limit / 2, 2) + // variance between samples
-					(recordRate * (1 - recordRate)) / sampleSize;
-				const sd = Math.max(Math.sqrt(variance) * entryCount, 1);
-				const estimatedRecordCount = Math.round(recordRate * entryCount);
-				// TODO: This uses a normal/Wald interval, but a binomial confidence interval is probably better calculated using
-				// Wilson score interval or Agresti-Coull interval (I think the latter is a little easier to calculate/implement).
-				const lowerCiLimit = Math.max(estimatedRecordCount - 1.96 * sd, recordCount + firstRecordCount);
-				const upperCiLimit = Math.min(estimatedRecordCount + 1.96 * sd, entryCount);
-				let significantUnit = Math.pow(10, Math.round(Math.log10(sd)));
-				if (significantUnit > estimatedRecordCount) significantUnit = significantUnit / 10;
-				recordCount = Math.round(estimatedRecordCount / significantUnit) * significantUnit;
+				const sampledRecords = recordCount + firstRecordCount;
+				const recordRate = sampledRecords / (limit + reverseScanned);
+				// Endpoints for the extrapolation base, spanning both ways an estimated base can be wrong:
+				// every remaining entry superseded (only what was sampled is live) through every remaining
+				// entry live (the uncalibrated physical count). Churn concentrated outside the sampled ends
+				// calibrates to nothing, so an interval derived from the estimator's confidence would sit
+				// narrowly around the wrong number. Both endpoints are themselves estimates on RocksDB, so
+				// this is a widened heuristic interval, not a guaranteed bound on the live count.
+				const baseMin = entriesScanned + reverseScanned;
+				const baseMax = Math.max(entriesScanned + remainderPhysical, entryCount, baseMin);
+				const estimatedRecordCount = Math.round(recordRate * Math.max(entryCount, baseMin));
+				// The samples counted these directly, and the entries between them can only add; everything
+				// outside the samples could be live. A statistical interval inside those endpoints would be
+				// narrowest exactly where the ends are least representative of the middle -- sampled ends
+				// that are all deletion entries give a rate of 0, collapsing an upper end to ~0 with live
+				// rows in between -- so the endpoints are the evidence itself.
+				const lower = sampledRecords;
+				const upper = Math.round(baseMax);
+				// Report only the precision the interval supports, but never so coarse a unit that the
+				// estimate rounds away: `baseMax` is physical and can exceed a calibrated estimate by
+				// orders of magnitude, which a single division cannot walk back.
+				let significantUnit = Math.pow(10, Math.round(Math.log10(Math.max((upper - lower) / 2, 1))));
+				while (significantUnit > estimatedRecordCount && significantUnit > 1) significantUnit /= 10;
+				recordCount = Math.min(
+					Math.max(Math.round(estimatedRecordCount / significantUnit) * significantUnit, lower),
+					upper
+				);
 				return {
 					recordCount,
-					estimatedRange: [Math.round(lowerCiLimit), Math.round(upperCiLimit)],
+					estimatedRange: [lower, upper],
 				};
 			}
 			return {
@@ -5857,6 +5981,23 @@ export function makeTable(options) {
 		return ids;
 	}
 
+	/**
+	 * What left a delete's key without a record: the nearest earlier write in the transaction that staged
+	 * state, else the stored entry. Only writes marked skipped are passed over, because an invalidate or
+	 * relocate stores a null stub without staging it.
+	 */
+	function removalBefore(write: any, existingEntry: Entry | undefined): Partial<Entry> | undefined {
+		for (let prior = write.priorWrite; prior; prior = prior.priorWrite) {
+			if (prior.stagedEntry) return prior.stagedEntry;
+			if (!prior.skipped) return;
+		}
+		if (!(existingEntry?.metadataFlags & INVALIDATED)) return existingEntry;
+	}
+
+	function isSameRemoval(entry: Partial<Entry> | undefined, txnTime: number, nodeId: number): boolean {
+		return entry != null && entry.value == null && entry.version === txnTime && (entry.nodeId ?? 0) === nodeId;
+	}
+
 	function precedesExistingVersion(txnTime: number, existingEntry: Partial<Entry>, nodeId?: number): number {
 		if (nodeId === undefined) {
 			nodeId = getThisNodeId(auditStore);
@@ -6089,25 +6230,34 @@ export function makeTable(options) {
 					}
 					resolve(resolvedEntry);
 				} catch (error) {
-					error.message += ` while resolving record ${id} for ${tableName}`;
-					if (
-						existingRecord &&
-						(((error.code === 'ECONNRESET' || error.code === 'ECONNREFUSED' || error.code === 'EAI_AGAIN') &&
-							!context?.mustRevalidate) ||
-							(context?.staleIfError &&
-								(error.statusCode === 500 ||
-									error.statusCode === 502 ||
-									error.statusCode === 503 ||
-									error.statusCode === 504)))
-					) {
-						// these are conditions under which we can use stale data after an error
-						resolve({
-							key: id,
-							version: existingVersion,
-							value: existingRecord,
-						} as any);
-						logger.trace?.(error.message, '(returned stale record)');
-					} else reject(error);
+					// A source may reject with anything at all, so deciding how to settle is itself
+					// fallible: `message` is not assignable on every error (a DOMException from
+					// AbortSignal.timeout), and a nullish rejection makes the reads below throw.
+					// Leaving this promise unsettled hangs the caller forever, so every path here
+					// has to end in resolve() or reject().
+					try {
+						appendErrorContext(error, ` while resolving record ${id} for ${tableName}`);
+						if (
+							existingRecord &&
+							(((error.code === 'ECONNRESET' || error.code === 'ECONNREFUSED' || error.code === 'EAI_AGAIN') &&
+								!context?.mustRevalidate) ||
+								(context?.staleIfError &&
+									(error.statusCode === 500 ||
+										error.statusCode === 502 ||
+										error.statusCode === 503 ||
+										error.statusCode === 504)))
+						) {
+							// these are conditions under which we can use stale data after an error
+							resolve({
+								key: id,
+								version: existingVersion,
+								value: existingRecord,
+							} as any);
+							logger.trace?.((error as Error)?.message, '(returned stale record)');
+						} else reject(error);
+					} catch (settlingError) {
+						reject(error ?? settlingError);
+					}
 					const resolveDuration = performance.now() - start;
 					recordAction(resolveDuration, 'cache-resolution', tableName, null, 'fail');
 					if (responseHeaders)

package/resources/replayLogs.ts

@@ -75,8 +75,9 @@ export function replayLogs(rootStore: RocksDatabase, tables: any): Promise<void>
 			tableById.set(table.tableId, table);
 		}
 		// replay all the logs
-		let transaction: DatabaseTransaction;
+		let transaction: DatabaseTransaction | undefined;
 		let lastTimestamp = 0;
+		let openCommitEnded = false;
 		let writes = 0;
 		let skipped = 0;
 		// Track forward progress so a backlog of unwritable entries can't grind the boot thread
@@ -110,8 +111,27 @@ export function replayLogs(rootStore: RocksDatabase, tables: any): Promise<void>
 				originatingOperation,
 				username,
 				extendedType,
+				endTxn,
 			} = auditRecord;
 			try {
+				// A replay transaction is one native commit ("Boot replay transactions" in DESIGN.md).
+				if (transaction && (openCommitEnded || lastTimestamp !== version)) {
+					const ending = transaction;
+					transaction = undefined;
+					try {
+						ending.directCommitSync();
+					} catch (error) {
+						logger.error('Error committing replay transaction', error);
+					}
+					// The wall-clock budget (harper#1316) is checked only here, between native commits with
+					// nothing staged, so aborting never tears a commit in half.
+					if (shouldAbortSlowReplay(performance.now() - replayStartTime, replayTimeoutMs)) {
+						logger.fatal(
+							`Aborting transaction-log replay in ${(rootStore as any).databaseName} database: replay has exceeded the wall-clock time limit (${writes} written, ${skipped} skipped). The transaction log contains a pathologically deep out-of-order write history that is too expensive to reconcile during boot (harper#1316). Re-clone this node from a healthy leader to recover the unreplayed data.`
+						);
+						break;
+					}
+				}
 				if (classifyAuditEntryForReplay(extendedType, tableId, true) === 'corrupt-header') {
 					skipped++;
 					noProgressRun++;
@@ -165,28 +185,9 @@ export function replayLogs(rootStore: RocksDatabase, tables: any): Promise<void>
 					warnedReplayHappening = true;
 					console.warn('Harper was not properly shutdown, replaying transaction logs to synchronize database');
 				}
-				if (lastTimestamp !== version) {
+				if (!transaction) {
 					lastTimestamp = version;
-					try {
-						// commit the last transaction since we are starting a new one
-						transaction?.directCommitSync();
-					} catch (error) {
-						logger.error('Error committing replay transaction', error);
-					}
-					// Abort if replay has exceeded the total wall-clock budget even while making progress
-					// (harper#1316, facet a). shouldAbortStalledReplay resets its counters on every write,
-					// so a slow-but-progressing replay (deep out-of-order audit chain walk per entry) can
-					// peg the boot thread indefinitely without tripping it. Checked only here, at a version
-					// boundary: the prior version's transaction was just committed in full and the new one
-					// is not yet staged, so aborting never tears a same-version (same source-transaction)
-					// write batch in half. Re-clone to recover the unreplayed remainder.
-					if (shouldAbortSlowReplay(performance.now() - replayStartTime, replayTimeoutMs)) {
-						logger.fatal(
-							`Aborting transaction-log replay in ${(rootStore as any).databaseName} database: replay has exceeded the wall-clock time limit (${writes} written, ${skipped} skipped). The transaction log contains a pathologically deep out-of-order write history that is too expensive to reconcile during boot (harper#1316). Re-clone this node from a healthy leader to recover the unreplayed data.`
-						);
-						transaction = undefined as any; // already committed above; nothing staged for the new version
-						break;
-					}
+					openCommitEnded = false;
 					transaction = new DatabaseTransaction();
 					transaction.db = primaryStore;
 					transaction.timestamp = version;
@@ -281,6 +282,8 @@ export function replayLogs(rootStore: RocksDatabase, tables: any): Promise<void>
 				logger.error(`Error writing from replay of log`, err, {
 					version,
 				});
+			} finally {
+				if (endTxn) openCommitEnded = true;
 			}
 		}
 		try {

package/resources/transactionBroadcast.ts

@@ -192,7 +192,7 @@ function notifyFromTransactionData(subscriptions, auditLogIterable?, allowYield
 					let ancestorLevel = 0;
 					do {
 						// we iterate through the key hierarchy, notifying all subscribers for each key,
-						// so for an id like resource/foo/bar, we notify subscribers for resource/foo/bar, resource/foo/, resource/foo, resource/, and resource
+						// so for an id like resource/foo/bar, we notify subscribers for resource/foo/bar, resource/foo/, resource/, and the root (null)
 						// this allows for efficient subscriptions to children ids/topics
 						const keySubscriptions = tableSubscriptions.get(matchingKey);
 						if (keySubscriptions) {
@@ -230,9 +230,9 @@ function notifyFromTransactionData(subscriptions, auditLogIterable?, allowYield
 						if (lastSlash !== matchingKey.length - 1) {
 							ancestorLevel++; // don't increase the ancestor level for this going from resource/ to resource
 						}
-						if (lastSlash > -1) {
-							matchingKey = matchingKey.slice(0, lastSlash + 1);
-						} else matchingKey = null;
+						// lastIndexOf clamps a negative fromIndex to 0, so '/' would otherwise yield itself forever
+						const parentKey = lastSlash > -1 ? matchingKey.slice(0, lastSlash + 1) : null;
+						matchingKey = parentKey === matchingKey ? null : parentKey;
 					} while (true);
 				}
 			} else if (auditRecord.type === 'reload') {

package/security/certificateVerification/index.ts

@@ -27,10 +27,14 @@ import { extractCertificateChain, extractRevocationUrls, bufferToPem } from './v
 import { getCachedCertificateVerificationConfig } from './verificationConfig.ts';
 import { verifyOCSP } from './ocspVerification.ts';
 import { verifyCRL } from './crlVerification.ts';
-import type { PeerCertificate, CertificateVerificationResult } from './types.ts';
+import { resolveTrustedIssuer } from './trustedIssuers.ts';
+import type { PeerCertificate, CertificateVerificationResult, FailureMode } from './types.ts';
 
 const logger = loggerWithTag('cert-verification');
 
+const warnedUnresolvedLeaves = new Set<string>();
+const MAX_WARNED_LEAVES = 10_000;
+
 /**
  * Verify certificate revocation status using OCSP and/or CRL
  * @param peerCertificate - Peer certificate object from TLS connection
@@ -52,14 +56,32 @@ export async function verifyCertificate(
 		return { valid: true, status: 'disabled', method: 'disabled' };
 	}
 
+	// ahead of the chain check so an explicitly disabled control can never reject
+	if (config.crl.enabled === false && config.ocsp.enabled === false) {
+		logger.debug?.('Both CRL and OCSP disabled - verification disabled');
+		return { valid: true, status: 'disabled', method: 'disabled' };
+	}
+
 	// Extract certificate chain
 	const certChain = extractCertificateChain(peerCertificate);
 	logger.trace?.(`Certificate chain length: ${certChain.length}`);
 
-	// Check if we have sufficient chain for verification (need certificate and issuer)
-	if (certChain.length < 2 || !certChain[0].issuer) {
-		logger.debug?.('Certificate chain insufficient for revocation checking - need certificate and issuer');
-		return { valid: true, status: 'no-issuer-cert', method: 'disabled' };
+	if (certChain.length === 0) {
+		return unresolvedIssuerResult(config.failureMode, peerCertificate, 'no certificate data');
+	}
+	// Revocation checking needs the issuer. The socket omits it on every resumed TLS session and on
+	// Node 26.8.0/26.8.1 (nodejs/node#65579); recover it from the CA set that authorized the client.
+	if (!certChain[0].issuer) {
+		const issuer = resolveTrustedIssuer(certChain[0].cert, peerCertificate.fingerprint256);
+		if (!issuer) {
+			return unresolvedIssuerResult(
+				config.failureMode,
+				peerCertificate,
+				'issuer is neither in the presented chain nor among the configured certificate authorities'
+			);
+		}
+		logger.debug?.('Issuer certificate resolved from the configured certificate authorities');
+		certChain[0].issuer = issuer;
 	}
 
 	// Extract certificate revocation URLs in single parse operation
@@ -117,17 +139,36 @@ export async function verifyCertificate(
 		logger.debug?.('Skipping OCSP - no responder URLs in certificate');
 	}
 
-	// All methods tried or skipped - determine failure handling
-	// If both methods are explicitly disabled (not just unavailable), treat as disabled
-	if (config.crl.enabled === false && config.ocsp.enabled === false) {
-		logger.debug?.('Both CRL and OCSP disabled - verification disabled');
-		return { valid: true, status: 'disabled', method: 'disabled' };
-	}
-
-	// Methods are enabled but unavailable - apply failure mode
+	// All methods tried or skipped - apply failure mode
 	if (config.failureMode === 'fail-closed') {
 		return { valid: false, status: 'no-verification-available', method: 'disabled' };
 	}
 
 	return { valid: true, status: 'verification-unavailable-allowed', method: 'disabled' };
 }
+
+/**
+ * Revocation status cannot be established without the issuer, so the configured failure mode
+ * decides: refuse under fail-closed, allow under fail-open. Either way this is a security control
+ * not running, so it is logged at warn (once per certificate) rather than debug.
+ */
+function unresolvedIssuerResult(
+	failureMode: FailureMode | undefined,
+	peerCertificate: PeerCertificate,
+	reason: string
+): CertificateVerificationResult {
+	const valid = failureMode !== 'fail-closed';
+	const leafKey = peerCertificate.fingerprint256 ?? peerCertificate.subject?.CN ?? '';
+	if (!warnedUnresolvedLeaves.has(leafKey)) {
+		if (warnedUnresolvedLeaves.size >= MAX_WARNED_LEAVES) warnedUnresolvedLeaves.clear();
+		warnedUnresolvedLeaves.add(leafKey);
+		logger.warn?.(
+			`Cannot check revocation status for client certificate ${peerCertificate.subject?.CN ?? 'unknown'}: ${reason}; ${
+				valid ? 'allowing connection (fail-open)' : 'rejecting connection (fail-closed)'
+			}`
+		);
+	} else {
+		logger.debug?.(`Cannot check revocation status for ${peerCertificate.subject?.CN ?? 'unknown'}: ${reason}`);
+	}
+	return { valid, status: 'no-issuer-cert', method: 'disabled' };
+}