@harperfast/harper 5.2.9 → 5.3.0-alpha.1

package/resources/replayLogsGuards.ts

@@ -132,16 +132,66 @@ export function shouldAbortSlowReplay(totalElapsedMs: number, timeLimitMs = REPL
 }
 
 /**
- * Wraps a transaction-log query iterator so a corrupt/torn frame ends that log's iteration
- * cleanly instead of escaping as an uncaughtException. rocksdb-js throws a bounded RangeError
- * when an entry's framing is broken; framing loss means the next entry can't be located, so the
- * frame marks end-of-log (entries before it were already yielded) and startup replay /
- * replication broadcast continue. `onCorruptFrame` fires once, latched — kept a callback (not a
- * direct log) so this module stays out of the Harper module graph and is unit-testable.
+ * A corrupt transaction-log frame from rocksdb-js. `resyncPosition` is set only when intact entries
+ * follow the break, which is what distinguishes lost entries from a merely truncated tail; its
+ * absence is also all that any rocksdb-js predating that field can report.
+ */
+export type CorruptFrameError = RangeError & {
+	logId?: number | null;
+	position?: number | null;
+	resyncPosition?: number | null;
+	unreadableBytes?: number | null;
+};
+
+/**
+ * Corrupt frames hit by one `getRange` call, shared by its per-log iterators.
+ *
+ * `truncatedVersions` is what a consumer has to act on: a break destroys the framing after the last
+ * entry the broken log yielded, and whether the entries it swallowed continued that entry's version
+ * is exactly what can no longer be read — UNLESS that last entry's own `endTxn` already closed its
+ * transaction, in which case nothing of it was left open to swallow. That version's transaction is
+ * therefore incomplete only when it wasn't already known-closed, and replay must discard it rather
+ * than commit the part of it that was still readable. A log that broke before yielding anything
+ * truncates no transaction.
+ *
+ * `midLogBreak` is coarser: true once any break in this range left intact entries unreadable behind
+ * it (see {@link isMidLogBreak}), regardless of which version they belonged to. A consumer that must
+ * never serve a range with lost entries — an elected branch replay publishing itself as a trustworthy
+ * point-in-time copy — rejects on this rather than on `truncatedVersions`, since a mid-log break can
+ * strand entries whose version this range never even attributes to one log (harper#2016, #2063).
+ */
+export interface CorruptFrameStop {
+	breaks: number;
+	truncatedVersions: Set<number>;
+	midLogBreak: boolean;
+}
+
+/**
+ * Whether a corrupt frame is a mid-log break — intact entries followed it, so entries were lost
+ * rather than the log merely being torn at its tail. Shared by the reporter below and by
+ * `RocksTransactionLogStore.getRange`'s `CorruptFrameStop.midLogBreak`, so the two can't drift on
+ * what counts as a mid-log break.
+ */
+export function isMidLogBreak(error: CorruptFrameError): boolean {
+	return error.resyncPosition != null;
+}
+
+/**
+ * Wraps a transaction-log query iterator so a corrupt/torn frame ends that log's iteration cleanly
+ * instead of escaping as an uncaughtException.
+ *
+ * Iteration stops at the break whether or not intact entries follow it. Resuming past a mid-log
+ * break would recover those entries, but a transaction log carries no frame-level transaction
+ * boundaries: replay groups equal-version entries into one source transaction, so skipping a frame
+ * inside such a group applies and checkpoints the surviving subset of a transaction that never
+ * committed that way at the source. Availability loss is visible and recoverable; a silently torn
+ * transaction is neither. Recovery is deferred until the engine can resume at a proven boundary
+ * (harper#2016, harper#2063); `resyncPosition` is used only to report the break as lost entries
+ * rather than as a truncated tail.
  */
 export function endIteratorOnCorruptFrame<T>(
 	iterator: Iterator<T>,
-	onCorruptFrame: (error: RangeError) => void
+	onCorruptFrame: (error: CorruptFrameError) => void
 ): IterableIterator<T> {
 	let stopped = false;
 	return {
@@ -157,7 +207,7 @@ export function endIteratorOnCorruptFrame<T>(
 				// version-dependent (1.4.2 added hex offsets). Anything else re-throws.
 				if (!(error instanceof RangeError)) throw error;
 				stopped = true;
-				onCorruptFrame(error);
+				onCorruptFrame(error as CorruptFrameError);
 				return { done: true, value: undefined };
 			}
 		},
@@ -175,3 +225,116 @@ export function endIteratorOnCorruptFrame<T>(
 		},
 	};
 }
+
+/**
+ * A corrupt frame, accumulated for the life of the process. The same frame is re-encountered by
+ * every reader until each consumer's resume cursor has passed it, so this is deduplicated by
+ * location and the repeats become a count.
+ */
+export interface CorruptFrameReport {
+	log: string;
+	logId?: number;
+	position?: number;
+	/** Intact entries followed the break, so entries were lost rather than merely truncated. */
+	midLog: boolean;
+	/** Extent of the unreadable region, when the engine reports it; 0 when nothing valid follows it. */
+	unreadableBytes: number;
+	firstSeen: number;
+	lastSeen: number;
+	occurrences: number;
+}
+
+/** Distinct break sites retained. One physical corruption yields one site, so this is generous. */
+export const MAX_CORRUPT_FRAME_REPORTS = 256;
+
+// Re-encounters move a site to the end, so the first key is the least recently seen. Full means
+// evicting that one rather than refusing a new site, which could not then be deduplicated.
+const corruptFrameReports = new Map<string, CorruptFrameReport>();
+let evictedCorruptFrameReports = 0;
+
+/**
+ * Every corrupt frame seen by this worker. This is not yet consumed by cluster/health status;
+ * callers must aggregate it across worker threads before exposing a node-wide signal.
+ *
+ * Mid-log breaks are immediately logged at error level even without that consumer.
+ */
+export function getCorruptFrameReports(): CorruptFrameReport[] {
+	return [...corruptFrameReports.values()];
+}
+
+/** Break sites evicted because {@link MAX_CORRUPT_FRAME_REPORTS} was reached. */
+export function getEvictedCorruptFrameReportCount(): number {
+	return evictedCorruptFrameReports;
+}
+
+export function clearCorruptFrameReports() {
+	corruptFrameReports.clear();
+	evictedCorruptFrameReports = 0;
+}
+
+// `logId`/`position` are absent on any rocksdb-js predating the resync support, and every break on
+// a stream would then collapse onto one key — folding genuinely different corruptions into a single
+// count that only ever logs once. The message carries the offset and file in text, so it separates
+// them when the fields can't.
+function corruptFrameKey(logName: string, error: CorruptFrameError): string {
+	const { logId, position } = error;
+	return logId != null && position != null
+		? `${logName}\u0000${logId}:${position}`
+		: `${logName}\u0000${error.message}`;
+}
+
+/**
+ * Records a corrupt frame and logs it once per distinct break.
+ *
+ * A mid-log break is an `error`, not a `warn`: entries after it were acknowledged and are now
+ * quarantined behind it. Severity follows the break's own shape, never this pass's outcome — every
+ * pass stops, so keying on that would report the #2063 signature with the benign torn-tail `warn`.
+ * A site first seen as a torn tail can escalate later, since only a rocksdb-js that reports
+ * `resyncPosition` can tell the two apart.
+ */
+export function createCorruptFrameReporter(logger: {
+	warn: (message: string, error?: unknown) => void;
+	error: (message: string, error?: unknown) => void;
+}) {
+	return (logName: string) => (error: CorruptFrameError) => {
+		const midLog = isMidLogBreak(error);
+		const unreadableBytes = error.unreadableBytes ?? 0;
+		const now = Date.now();
+		const key = corruptFrameKey(logName, error);
+		const existing = corruptFrameReports.get(key);
+		if (existing) {
+			corruptFrameReports.delete(key);
+			corruptFrameReports.set(key, existing);
+			existing.occurrences++;
+			existing.lastSeen = now;
+			if (!midLog || existing.midLog) return;
+			existing.midLog = true;
+			existing.unreadableBytes = unreadableBytes;
+		} else {
+			if (corruptFrameReports.size >= MAX_CORRUPT_FRAME_REPORTS) {
+				corruptFrameReports.delete(corruptFrameReports.keys().next().value);
+				evictedCorruptFrameReports++;
+			}
+			corruptFrameReports.set(key, {
+				log: logName,
+				logId: error.logId ?? undefined,
+				position: error.position ?? undefined,
+				midLog,
+				unreadableBytes,
+				firstSeen: now,
+				lastSeen: now,
+				occurrences: 1,
+			});
+		}
+		if (midLog) {
+			logger.error(
+				`Corrupt entry in transaction log "${logName}"; ${unreadableBytes} byte(s) are unreadable and the entries within them are lost. ` +
+					'Intact entries follow the break, but reading stops there rather than skipping the frame, which could tear a source transaction: ' +
+					'they are quarantined until the log is repaired or this node is re-cloned, and they are neither replayed nor replicated meanwhile.',
+				error
+			);
+		} else {
+			logger.warn(`Stopping transaction log "${logName}" at a corrupt entry during replay`, error);
+		}
+	};
+}

package/resources/search.ts

@@ -1,11 +1,12 @@
 import { ClientError, IndexRebuildingError, Violation } from '../utility/errors/hdbError.ts';
 import { OVERFLOW_MARKER, MAX_SEARCH_KEY_LENGTH, SEARCH_TYPES } from '../utility/lmdb/terms.ts';
 import { compareKeys, MAXIMUM_KEY, writeKey } from 'ordered-binary';
-import { SKIP } from '@harperfast/extended-iterable';
+import { SKIP, ExtendedIterable } from '@harperfast/extended-iterable';
 import { INVALIDATED, EVICTED, freezeRecord } from './Table.ts';
 import type { DirectCondition, Id } from './ResourceInterface.ts';
 import { RequestTarget } from './RequestTarget.ts';
 import { lastMetadata } from './RecordEncoder.ts';
+import { writeKeyId } from './DatabaseTransaction.ts';
 import { recordAction } from './analytics/write';
 import { RocksDatabase } from '@harperfast/rocksdb-js';
 
@@ -232,6 +233,41 @@ function composeRecordFilter(recordFilters, table, context): (primaryKey: Id) =>
 	};
 }
 
+/**
+ * A record's index entries are not adjacent — the composite `[indexedValue, primaryKey]` key sorts
+ * on the indexed value first — so comparing neighbours cannot collapse them and the scan has to
+ * remember what it already yielded (#2434).
+ *
+ * Heap cost is one key per distinct record in the scanned range, held for the life of the scan, and
+ * a page window does not cap it: sibling-condition and row filters run downstream, so a selective
+ * filter keeps the scan running and the set growing while the page fills. On a multi-million-row
+ * range that is hundreds of MB where main streamed at O(1). Accepted for #2434 over the bounded
+ * alternative, which costs a primary read per index entry.
+ */
+function distinctRecords(entries: any): AsyncIterable<Id> {
+	const distinct = new ExtendedIterable();
+	(distinct as any).iterate = (options) => {
+		// A non-scalar key decodes to a fresh instance per entry, so identity never matches it;
+		// `writeKeyId` is the store's own key equality. Its bytes get their own set because a scalar
+		// string id can legitimately carry exactly those bytes, and one set would fold the two —
+		// the collision `flattenKey` has, and the reason Table.ts prefixes its key ids.
+		const yieldedKeys = new Set();
+		const yieldedEncodedKeys = new Set();
+		// map rather than filter: filter clears `continueOnRecoverableError` for everything below it
+		return entries
+			.map((primaryKey) => {
+				const isEncoded = typeof primaryKey === 'object' && primaryKey !== null;
+				const yielded = isEncoded ? yieldedEncodedKeys : yieldedKeys;
+				const identity = isEncoded ? writeKeyId(primaryKey) : primaryKey;
+				if (yielded.has(identity)) return SKIP;
+				yielded.add(identity);
+				return primaryKey;
+			})
+			.iterate(options);
+	};
+	return distinct as any;
+}
+
 /**
  * Search for records or keys, based on the search condition, using an index if available
  * @param searchCondition
@@ -261,6 +297,9 @@ export function searchByIndex(
 		throw new ClientError(`Search condition for ${attribute_name} must have a value`);
 	}
 	let needFullScan;
+	// `[indexedValue, primaryKey]` is unique, so a scan that stays inside one indexed value cannot
+	// reach a record twice and skips the collapse below
+	let scansOneIndexedValue;
 	if (Array.isArray(attribute_name)) {
 		const firstAttributeName = attribute_name[0];
 		// get the potential relationship attribute
@@ -388,6 +427,7 @@ export function searchByIndex(
 				start = value;
 				end = value;
 				inclusiveEnd = true;
+				scansOneIndexedValue = true;
 				break;
 			case 'in':
 				// Phase 1: route through filter — index-merge optimization is a Phase 2 follow-up.
@@ -531,7 +571,7 @@ export function searchByIndex(
 			}
 			return loaded;
 		}
-		return index.getRange(rangeOptions).map(
+		const scanned = index.getRange(rangeOptions).map(
 			filter
 				? function ({ key, value }) {
 						let recordMatcher: any;
@@ -554,6 +594,14 @@ export function searchByIndex(
 					}
 				: ({ value }) => value
 		);
+		// Collapsing AFTER the map is required, not incidental: the condition's own filter tests one
+		// element at a time, so a record whose first entry fails it and whose second passes must
+		// still be reached. Collapsing first would settle that record on the failing entry and drop
+		// it. Scoped to attributes the schema declares multi-valued; a runtime array under an
+		// undeclared attribute repeats the same way but stays outside this boundary.
+		return scansOneIndexedValue || !findAttribute(Table.attributes, attribute_name)?.elements
+			? scanned
+			: distinctRecords(scanned);
 	} else {
 		return Table.primaryStore
 			.getRange(reverse ? { end: true, transaction, reverse: true } : { start: true, transaction })
@@ -1112,6 +1160,91 @@ export function filterByType(searchCondition, Table, context, filtered, isPrimar
 	}
 }
 
+/**
+ * Estimates a range comparator's entry count from the store's statistical estimate, blended with
+ * the table-fraction heuristic by the estimate's confidence so low-confidence estimates degrade
+ * to the old behavior. Returns undefined (caller falls back to the heuristic) when the store
+ * cannot estimate, the shape is unexpected, or the executed range wouldn't match this one.
+ */
+function estimateRangeCondition(table, condition, searchType, fraction) {
+	const attributeName = condition[0] ?? condition.attribute;
+	const isPrimaryKey = attributeName === table.primaryKey;
+	const store = isPrimaryKey ? table.primaryStore : table.indices[attributeName];
+	// LMDB-backed and custom index stores do not implement estimateCount
+	if (typeof store?.estimateCount !== 'function') return undefined;
+	let value = condition[1] ?? condition.value;
+	if (value instanceof Date) value = value.getTime();
+	let range;
+	switch (searchType) {
+		case 'lt':
+			// `start: true` mirrors searchByIndex: `true` sorts above `null`, so an indexNulls
+			// index's `[null, primaryKey]` entries are outside the executed range
+			range = { start: true, end: value };
+			break;
+		case 'le':
+			range = { start: true, end: value, inclusiveEnd: true };
+			break;
+		case 'gt':
+			range = { start: value, exclusiveStart: true };
+			break;
+		case 'ge':
+			range = { start: value };
+			break;
+		case 'between':
+		case 'gele':
+		case 'gelt':
+		case 'gtlt':
+		case 'gtle': {
+			if (!Array.isArray(value)) return undefined;
+			let [start, end] = value;
+			if (start instanceof Date) start = start.getTime();
+			if (end instanceof Date) end = end.getTime();
+			range = {
+				start,
+				end,
+				inclusiveEnd: searchType === 'between' || searchType === 'gele' || searchType === 'gtle',
+				exclusiveStart: searchType === 'gtlt' || searchType === 'gtle',
+			};
+			break;
+		}
+		case 'starts_with': {
+			const prefix = value?.toString();
+			if (!prefix) return undefined;
+			range = { start: prefix, end: getStringPrefixUpperBound(prefix) };
+			break;
+		}
+		case 'prefix': {
+			let start = Array.isArray(value) ? value : [value, null];
+			if (start[start.length - 1] != null) start = start.concat(null);
+			const end = start.slice(0);
+			end[end.length - 1] = MAXIMUM_KEY;
+			range = { start, end };
+			break;
+		}
+		default:
+			return undefined;
+	}
+	// Long string bounds get truncated + filtered at execution (searchByIndex), so the
+	// executed range is wider than this one; don't estimate what won't be iterated.
+	if (
+		(typeof range.start === 'string' && range.start.length > MAX_SEARCH_KEY_LENGTH) ||
+		(typeof range.end === 'string' && range.end.length > MAX_SEARCH_KEY_LENGTH)
+	) {
+		return undefined;
+	}
+	let count, confidence;
+	try {
+		({ count, confidence } = store.estimateCount(range) ?? {});
+	} catch {
+		// a concurrently closing/dropped store must degrade the plan, not fail the query
+		return undefined;
+	}
+	if (!Number.isFinite(count) || count < 0 || !Number.isFinite(confidence) || confidence < 0 || confidence > 1)
+		return undefined;
+	const heuristic = fraction * estimatedEntryCount(table.primaryStore) + 1;
+	return Math.max(1, Math.round(confidence * count + (1 - confidence) * heuristic));
+}
+
 export function estimateCondition(table) {
 	function estimateConditionForTable(condition) {
 		if (condition.estimated_count === undefined) {
@@ -1141,7 +1274,14 @@ export function estimateCondition(table) {
 			// skip if it is cached
 			let searchType = condition.comparator || condition.search_type;
 			searchType = ALTERNATE_COMPARATOR_NAMES[searchType] || searchType;
-			if (searchType === SEARCH_TYPES.EQUALS || !searchType) {
+			if (condition.negated) {
+				// a negated condition always executes as a full scan (searchByIndex forces
+				// needFullScan), so follow the filter-only convention used by contains/ends_with:
+				// estimate Infinity so its positive-range estimate can never win the driving-condition
+				// ordering. Short-circuit here rather than computing (and discarding) a positive-range
+				// estimate that would cross the native FFI boundary for nothing.
+				condition.estimated_count = Infinity;
+			} else if (searchType === SEARCH_TYPES.EQUALS || !searchType) {
 				const attribute_name = condition[0] ?? condition.attribute;
 				if (attribute_name == null || attribute_name === table.primaryKey) condition.estimated_count = 1;
 				else if (Array.isArray(attribute_name) && attribute_name.length > 1) {
@@ -1192,11 +1332,14 @@ export function estimateCondition(table) {
 				} else if (Array.isArray(condition.value)) {
 					condition.estimated_count = Infinity;
 				} else condition.estimated_count = Infinity;
-				// for range queries (betweens, startsWith, greater, etc.), just arbitrarily guess
 			} else if (searchType === 'starts_with' || searchType === 'prefix')
-				condition.estimated_count = STARTS_WITH_ESTIMATE * estimatedEntryCount(table.primaryStore) + 1;
+				condition.estimated_count =
+					estimateRangeCondition(table, condition, searchType, STARTS_WITH_ESTIMATE) ??
+					STARTS_WITH_ESTIMATE * estimatedEntryCount(table.primaryStore) + 1;
 			else if (searchType === 'between')
-				condition.estimated_count = BETWEEN_ESTIMATE * estimatedEntryCount(table.primaryStore) + 1;
+				condition.estimated_count =
+					estimateRangeCondition(table, condition, searchType, BETWEEN_ESTIMATE) ??
+					BETWEEN_ESTIMATE * estimatedEntryCount(table.primaryStore) + 1;
 			else if (searchType === 'sort') {
 				const attribute_name = condition[0] ?? condition.attribute;
 				const index = table.indices[attribute_name];
@@ -1211,7 +1354,10 @@ export function estimateCondition(table) {
 				if (index?.customIndex?.estimateCount)
 					// allow custom index to define its own estimation of counts
 					condition.estimated_count = index.customIndex.estimateCount(condition.value);
-				else condition.estimated_count = OPEN_RANGE_ESTIMATE * estimatedEntryCount(table.primaryStore) + 1;
+				else
+					condition.estimated_count =
+						estimateRangeCondition(table, condition, searchType, OPEN_RANGE_ESTIMATE) ??
+						OPEN_RANGE_ESTIMATE * estimatedEntryCount(table.primaryStore) + 1;
 			}
 			// we give a condition significantly more weight/preference if we will be ordering by it
 			if (typeof condition.descending === 'boolean') condition.estimated_count /= 2;
@@ -1397,6 +1543,7 @@ function parseBlock(query, expectedEnd) {
 						break;
 					case 'group-by':
 						recordError('group by is not implemented yet');
+						break;
 					case 'sort':
 						query.sort = toSortObject(args);
 						break;

package/resources/transactionBroadcast.ts

@@ -95,7 +95,7 @@ export function addSubscription(table, key, listener?: (key) => any, startTime?:
  * subscription and get the initial state.
  */
 class Subscription extends IterableEventQueue {
-	listener: (recordId: Id, auditEntry: any, localTime: number, beginTxn: boolean) => void;
+	listener: (recordId: Id, auditEntry: any, txnLogKey: number, beginTxn: boolean) => void;
 	subscriptions: any;
 	startTime?: number;
 	includeDescendants?: boolean;
@@ -178,10 +178,10 @@ function notifyFromTransactionData(subscriptions, auditLogIterable?, allowYield
 			}
 			if (result.done) break;
 			const auditRecord = result.value;
-			const timestamp: number = auditRecord.localTime ?? auditRecord.version;
+			const timestamp: number = auditRecord.txnLogKey;
 			subscriptions.lastTxnTime = timestamp;
 			// the transaction extent: RocksDB entries committed together share the log key (record
-			// versions may differ); LMDB's localTime is a per-entry audit key, so version delimits there
+			// versions may differ); LMDB's transaction-log key is per-entry, so version delimits there
 			const txnKey = auditStore.reusableIterable ? timestamp : auditRecord.version;
 			if (ACTIONS_OF_INTEREST.includes(auditRecord.type)) {
 				const tableSubscriptions = subscriptions[auditRecord.tableId];

package/security/auth.ts

@@ -7,13 +7,21 @@ import { v4 as uuid } from 'uuid';
 import * as env from '../utility/environment/environmentManager.ts';
 import { CONFIG_PARAMS, AUTH_AUDIT_STATUS, AUTH_AUDIT_TYPES } from '../utility/hdbTerms.ts';
 import harperLogger from '../utility/logging/harper_logger.ts';
-const { forComponent, AuthAuditLog } = harperLogger;
+const { forComponent, AuthAuditLog, errorForLog } = harperLogger;
 import serverHandlers from '../server/itc/serverHandlers.js';
 const { user } = serverHandlers;
-import { Headers, addVaryHeader } from '../server/serverHelpers/Headers.ts';
+import { Headers, addVaryHeader, SHARED_CACHE_OPTIN, PRIVATE_SCOPE } from '../server/serverHelpers/Headers.ts';
 import { convertToMS } from '../utility/common_utils.ts';
 import { verifyCertificate } from './certificateVerification/index.ts';
+import {
+	credentialRejectionError,
+	deferCredentialRejection,
+	getDeferredCredentialRejection,
+	isCredentialRejection,
+} from './deferredAuthentication.ts';
 import { serializeMessage } from '../server/serverHelpers/contentTypes.ts';
+import { hdbErrors } from '../utility/errors/hdbError.ts';
+const { AUTHENTICATION_ERROR_MSGS, HTTP_STATUS_CODES } = hdbErrors;
 const authLogger = forComponent('authentication');
 const { debug } = authLogger;
 const authEventLog = authLogger.withTag('auth-event');
@@ -50,10 +58,6 @@ const LOG_AUTH_FAILED = env.get(CONFIG_PARAMS.LOGGING_AUDITAUTHEVENTS_LOGFAILED)
 
 const DEFAULT_COOKIE_EXPIRES = 'Tue, 01 Oct 8307 19:33:20 GMT';
 
-// RFC 9111 cache-scope directives; boundaries on both sides so a token like `public-foo` doesn't match
-const SHARED_CACHE_OPTIN = /(^|[,\s])(public|s-maxage)($|[\s,;=])/i;
-const PRIVATE_SCOPE = /(^|[,\s])(private|no-store)($|[\s,;=])/i;
-
 let authorizationCache = new Map();
 server.onInvalidatedUser(() => {
 	// TODO: Eventually we probably want to be able to invalidate individual users
@@ -190,7 +194,13 @@ export async function authentication(request, nextHandler) {
 		if (request.user) {
 			// already authenticated
 		} else if (authorization) {
-			const cachedUser = authorizationCache.get(authorization);
+			let cachedUser = authorizationCache.get(authorization);
+			// A cached Bearer identity must not outlive its token: expiry is the only revocation
+			// mechanism for scoped tokens, so it has to be exact, not cache-TTL-fuzzy.
+			if (cachedUser?.authExpiresAt && cachedUser.authExpiresAt * 1000 <= Date.now()) {
+				authorizationCache.delete(authorization);
+				cachedUser = undefined;
+			}
 			if (cachedUser?.role) {
 				// Shallow-clone so verifyPerms's `role.permission = fullRolePerms` reassignment
 				// doesn't mutate the cache entry (defense-in-depth; operations and other
@@ -204,6 +214,7 @@ export async function authentication(request, nextHandler) {
 				const strategy = authorization.slice(0, spaceIndex);
 				const credentials = authorization.slice(spaceIndex + 1);
 				let username, password;
+				let credentialRejection;
 				try {
 					switch (strategy) {
 						case 'Basic':
@@ -231,12 +242,23 @@ export async function authentication(request, nextHandler) {
 											// API has its own logic for handling this
 											status: -1,
 										});
-									} catch {
+									} catch (refreshError) {
+										// Preserve refresh-validation faults; only a tagged rejection permits falling
+										// back to the original operation-token rejection.
+										if (!isCredentialRejection(refreshError)) throw refreshError;
 										throw error;
 									}
 								}
+								// A non-'invalid token' rejection (e.g. expired) must propagate, not fall
+								// through to caching an undefined user (which would read as anonymous).
+								throw error;
 							}
 							break;
+						default:
+							throw credentialRejectionError(
+								AUTHENTICATION_ERROR_MSGS.GENERIC_AUTH_FAIL,
+								HTTP_STATUS_CODES.UNAUTHORIZED
+							);
 					}
 				} catch (err) {
 					if (LOG_AUTH_FAILED) {
@@ -247,18 +269,31 @@ export async function authentication(request, nextHandler) {
 						}
 					}
 
-					return applyResponseHeaders({
-						status: 401,
-						body: serializeMessage({ error: err.message }, request),
-					});
+					const internalFault = !isCredentialRejection(err);
+					if (request.isOperationsServer || internalFault) {
+						if (internalFault) authLogger.error('Authentication failed internally', errorForLog(err));
+						return applyResponseHeaders({
+							status: 401,
+							body: serializeMessage(
+								{ error: internalFault ? AUTHENTICATION_ERROR_MSGS.GENERIC_AUTH_FAIL : err.message },
+								request
+							),
+						});
+					}
+					credentialRejection = err;
 				}
 
-				authorizationCache.set(authorization, newUser);
-				if (LOG_AUTH_SUCCESSFUL) authAuditLog(newUser.username, AUTH_AUDIT_STATUS.SUCCESS, strategy);
-				// Shallow-clone so verifyPerms's `role.permission = fullRolePerms` reassignment
-				// doesn't mutate the just-stored cache entry (defense-in-depth).
-				if (newUser?.role) {
-					newUser = { ...newUser, role: { ...newUser.role, permission: { ...newUser.role.permission } } };
+				if (credentialRejection) {
+					deferCredentialRejection(request, credentialRejection, strategy);
+				} else {
+					authorizationCache.set(authorization, newUser);
+					if (LOG_AUTH_SUCCESSFUL && newUser != null)
+						authAuditLog(newUser.username, AUTH_AUDIT_STATUS.SUCCESS, strategy);
+					// Shallow-clone so verifyPerms's `role.permission = fullRolePerms` reassignment
+					// doesn't mutate the just-stored cache entry (defense-in-depth).
+					if (newUser?.role) {
+						newUser = { ...newUser, role: { ...newUser.role, permission: { ...newUser.role.permission } } };
+					}
 				}
 			}
 
@@ -346,16 +381,19 @@ export async function authentication(request, nextHandler) {
 		if (!response) return response;
 		if (response.status === 401) {
 			wasUnauthorized = true;
-			if (
-				headers['user-agent']?.startsWith('Mozilla') &&
-				headers.accept?.startsWith('text/html') &&
-				resources.loginPath
-			) {
-				// on the web if we have a login page, default to redirecting to it
-				response.status = 302;
-				response.headers.set('Location', resources.loginPath(request));
-			} // the HTTP specified way of indicating HTTP authentication methods supported:
-			else response.headers.set('WWW-Authenticate', 'Basic');
+			// Downstream owns the challenge or redirect after Harper deferred the credential decision.
+			if (!getDeferredCredentialRejection(request)) {
+				if (
+					headers['user-agent']?.startsWith('Mozilla') &&
+					headers.accept?.startsWith('text/html') &&
+					resources.loginPath
+				) {
+					// on the web if we have a login page, default to redirecting to it
+					response.status = 302;
+					response.headers.set('Location', resources.loginPath(request));
+				} // the HTTP specified way of indicating HTTP authentication methods supported:
+				else response.headers.set('WWW-Authenticate', 'Basic');
+			}
 		}
 		return applyResponseHeaders(response);
 	} catch (error) {
@@ -367,7 +405,8 @@ export async function authentication(request, nextHandler) {
 		// if we are rejecting the credentials (401, possibly rewritten to a login redirect); such a
 		// response must never be stored by a shared cache and served to a different principal (#1565)
 		const rejectedAuth = response?.status === 401 || wasUnauthorized;
-		const identityDependent = !!request.user || rejectedAuth;
+		// A downstream response produced under the untouched credential remains identity-dependent (#1565).
+		const identityDependent = !!request.user || rejectedAuth || !!getDeferredCredentialRejection(request);
 		// with CORS enabled the response is origin-dependent — ACAO reflects the request Origin, and
 		// its absence when no Origin was sent is origin-dependent too — so a shared cache must
 		// partition on Origin either way (#1518)