@harperfast/harper 5.2.9 → 5.3.0-alpha.1

package/resources/defineTable.ts

@@ -27,6 +27,8 @@
 
 import { table, type Table } from './databases.ts';
 import { attributeToFragment } from './jsonSchemaTypes.ts';
+import type { RecordLockOptions } from './recordLock.ts';
+import type { WritableRecord } from './ResourceInterface.ts';
 
 // ─────────────────────────────────────────────────────────────────────────────
 // Field model — phantom-typed Field. Flags are set via GETTER PROPERTIES (no call):
@@ -235,6 +237,7 @@ interface TypedVerbs<S extends Shape> {
 	post(record: InsertOf<S>, context?: any): MaybePromise<unknown>;
 	patch(id: IdOf<S>, changes: PatchOf<S>, context?: any): MaybePromise<unknown>;
 	update(id: IdOf<S>, updates?: PatchOf<S>, context?: any): MaybePromise<InstanceOf<S>>;
+	lock(id: IdOf<S>, options?: RecordLockOptions, context?: any): Promise<WritableRecord<InstanceOf<S>>>;
 	delete(id: IdOf<S>, context?: any): MaybePromise<unknown>;
 	search(query?: any, context?: any): AsyncIterable<ReadVariant<S>>;
 	query(query?: any, context?: any): AsyncIterable<ReadVariant<S>>;
@@ -248,7 +251,7 @@ interface TypedVerbs<S extends Shape> {
  */
 export type TableHandle<S extends Shape = Shape> = Omit<
 	Table,
-	'get' | 'put' | 'post' | 'patch' | 'update' | 'delete' | 'search' | 'query'
+	'get' | 'put' | 'post' | 'patch' | 'update' | 'lock' | 'delete' | 'search' | 'query'
 > &
 	TypedVerbs<S> & {
 		// phantom projection carriers — discovery surface, zero runtime cost
@@ -400,8 +403,21 @@ function compileTypeDef(name: string, shape: Shape, options: DefineTableOptions)
  * index changes) through the same evolution path GraphQL reloads take.
  */
 export function defineTable<S extends Shape>(name: string, shape: S, options: DefineTableOptions = {}): TableHandle<S> {
+	return defineTableUsing(table, name, shape, options);
+}
+
+/**
+ * `defineTable` through a specific table factory: the one a branched application's scope hands out
+ * (`scopedTableFactory`), so the table lands in its branch. Internal -- the public entry is `defineTable`.
+ */
+export function defineTableUsing<S extends Shape>(
+	tableFactory: typeof table,
+	name: string,
+	shape: S,
+	options: DefineTableOptions = {}
+): TableHandle<S> {
 	const typeDef = compileTypeDef(name, shape, options);
-	const tableClass = table(typeDef);
+	const tableClass = tableFactory(typeDef);
 	typeDef.tableClass = tableClass;
 	return tableClass as TableHandle<S>;
 }

package/resources/graphql.ts

@@ -1,10 +1,9 @@
 import { dirname } from 'path';
 import { Script } from 'node:vm';
-import { table } from './databases.ts';
+import { scopedTableFactory, table } from './databases.ts';
 import { getWorkerIndex } from '../server/threads/manageThreads.js';
 import { Resources } from './Resources.ts';
 import type { NamedTypeNode, StringValueNode, ValueNode } from 'graphql';
-import { once } from 'node:events';
 import { ClientError } from '../utility/errors/hdbError.ts';
 import { attributeToFragment, type JsonSchemaFragment } from './jsonSchemaTypes.ts';
 import harperLogger from '../utility/logging/harper_logger.ts';
@@ -63,7 +62,7 @@ server.knownGraphQLDirectives.push(
  */
 export function handleApplication(scope: import('../components/Scope.ts').Scope) {
 	let initialLoadComplete = false;
-	const entryHandler = scope.handleEntry(async (entry) => {
+	scope.handleEntry(async (entry) => {
 		if (initialLoadComplete) {
 			scope.requestRestart();
 			return;
@@ -75,19 +74,56 @@ export function handleApplication(scope: import('../components/Scope.ts').Scope)
 			return;
 		}
 
-		await processGraphQLSchema((entry as any).contents, entry.urlPath, entry.absolutePath, scope.resources);
+		await processGraphQLSchema(
+			(entry as any).contents,
+			entry.urlPath,
+			entry.absolutePath,
+			scope.resources,
+			scopedTableFactory(scope.applicationScope?.branches),
+			scope.logger
+		);
 	});
-	const initialLoadPromise = once(entryHandler, 'initialLoadComplete');
-	initialLoadPromise.then(() => {
+	// Settles with the load's real outcome, so a diagnosed failure is not left for the loader's watchdog
+	// to report while it holds the plugin-wide load lock (#1917). The flag advances either way: a schema
+	// corrected afterwards belongs on the restart path.
+	return scope.waitForInitialLoads().finally(() => {
 		initialLoadComplete = true;
 	});
-	return initialLoadPromise;
 }
 
-async function processGraphQLSchema(gqlContent, urlPath, filePath, resources) {
+function describeLocation(loc: { startToken?: { line: number; column: number } } | undefined) {
+	const token = loc?.startToken;
+	return token ? `line ${token.line}, column ${token.column}` : 'an unknown location';
+}
+
+async function processGraphQLSchema(
+	gqlContent,
+	urlPath,
+	filePath,
+	resources,
+	declareTable: typeof table = table,
+	logger: { warn?: (...args: any[]) => void; error?: (...args: any[]) => void } = harperLogger
+) {
 	// lazy load the graphql package so we don't load it for users that don't use graphql
-	const { parse, Source, Kind } = await import('graphql');
-	const ast = parse(new Source(gqlContent.toString(), filePath));
+	const { parse, Source, Kind, specifiedDirectives } = await import('graphql');
+	let ast;
+	try {
+		ast = parse(new Source(gqlContent.toString(), filePath));
+	} catch (error) {
+		// GraphQLError.toString() is the only rendering carrying the source excerpt and caret; `.stack`,
+		// which the logger prefers, points at Harper internals. It stays out of the thrown message: a
+		// failed component's ErrorResource message becomes the REST problem title.
+		logger.error?.(`Invalid GraphQL schema in ${filePath}:\n${error}`);
+		throw new Error(`Invalid GraphQL schema${urlPath ? ` at ${urlPath}` : ''}`);
+	}
+	// Spec-defined and schema-declared directives are not unknown just because Harper ignores them.
+	const recognizedDirectives = new Set([
+		...server.knownGraphQLDirectives,
+		...specifiedDirectives.map((directive) => directive.name),
+	]);
+	for (const definition of ast.definitions) {
+		if (definition.kind === Kind.DIRECTIVE_DEFINITION) recognizedDirectives.add(definition.name.value);
+	}
 	const types = new Map();
 	const tables = [];
 	// we begin by iterating through the definitions in the AST to get the types and convert them
@@ -112,6 +148,7 @@ async function processGraphQLSchema(gqlContent, urlPath, filePath, resources) {
 						// @table is the canonical schema-defined declaration; pass the flag explicitly so
 						// the existing-Table re-assert in databases.ts::table() fires on every reload.
 						typeDef.schemaDefined = true;
+						typeDef.schemaRelationshipsDefined = true;
 						if (typeDef.schema) typeDef.database = typeDef.schema;
 						if (!typeDef.table) typeDef.table = typeName;
 						if (typeDef.audit) typeDef.audit = typeDef.audit !== 'false';
@@ -168,7 +205,10 @@ async function processGraphQLSchema(gqlContent, urlPath, filePath, resources) {
 					for (const directive of field.directives) {
 						const directiveName = directive.name.value;
 						if (directiveName === 'primaryKey') {
-							if (hasPrimaryKey) console.warn('Can not define two attributes as a primary key at', directive.loc);
+							if (hasPrimaryKey)
+								logger.warn?.(
+									`Can not define two attributes as a primary key, at ${describeLocation(directive.loc)} in ${filePath}`
+								);
 							else {
 								property.isPrimaryKey = true;
 								hasPrimaryKey = true;
@@ -244,8 +284,10 @@ async function processGraphQLSchema(gqlContent, urlPath, filePath, resources) {
 									authorizedRoles.push((arg.value as StringValueNode).value);
 								}
 							}
-						} else if (server.knownGraphQLDirectives.includes(directiveName)) {
-							console.warn(`@${directiveName} is an unknown directive, at`, directive.loc);
+						} else if (!recognizedDirectives.has(directiveName)) {
+							logger.warn?.(
+								`@${directiveName} is an unknown field directive, at ${describeLocation(directive.loc)} in ${filePath}`
+							);
 						}
 					}
 					// @embed targets a vector column and auto-indexes it with HNSW; resolved after all
@@ -302,19 +344,31 @@ async function processGraphQLSchema(gqlContent, urlPath, filePath, resources) {
 		} else if (property.type === 'array') connectPropertyType(property.elements);
 		else if (!PRIMITIVE_TYPES.includes(property.type)) {
 			if (getWorkerIndex() === 0)
-				console.error(
+				logger.error?.(
 					`The type ${property.type} is unknown at line ${property.location.line}, column ${property.location.column}, in ${filePath}`
 				);
 		}
 	}
 	for (const typeDef of types.values()) {
-		for (const property of typeDef.attributes) connectPropertyType(property);
+		for (const property of typeDef.attributes) {
+			connectPropertyType(property);
+			const relationshipType = property.definition || property.elements?.definition;
+			if (property.relationship && relationshipType?.table) {
+				Object.defineProperty(property, 'relationshipReference', {
+					value: {
+						database: relationshipType.database || 'data',
+						table: relationshipType.table,
+					},
+					configurable: true,
+				});
+			}
+		}
 	}
 	// any tables that are defined in the schema can now be registered
 	for (const typeDef of tables) {
 		// with graphql database definitions, this is a declaration that the table should exist and that it
 		// should be created if it does not exist
-		typeDef.tableClass = table(typeDef);
+		typeDef.tableClass = declareTable(typeDef);
 		if (getWorkerIndex() === 0) {
 			// Post-Phase-2: typeDef.properties is the canonical Record (no .find); read the Array form.
 			const pk = (typeDef.attributes as any[])?.find((p) => p.isPrimaryKey)?.name ?? 'id';

package/resources/indexes/HierarchicalNavigableSmallWorld.ts

@@ -209,6 +209,10 @@ export class HierarchicalNavigableSmallWorld {
 	// a value of 1 is extremely aggressive.
 	optimizeRouting = 0.5;
 	nodesVisitedCount = 0;
+	// Test seam: source of randomness for level assignment. Tests that assert exact result-set
+	// equality across search strategies replace this with a seeded PRNG so the graph shape is
+	// reproducible; greedy routing legitimately diverges on rare unlucky graphs otherwise.
+	random: () => number = Math.random;
 	// Visit-budget multiplier for predicate-aware traversal (#1241). Under-filled filtered searches
 	// stop after the resolved budget ef * filterExpansion visits; automatic search ef contributes at
 	// most AUTO_EF_MAX, while explicit schema/query ef remains authoritative. A filter that fills its
@@ -337,7 +341,7 @@ export class HierarchicalNavigableSmallWorld {
 			const storedScale = q ? q.scale : undefined;
 			let entryPoint = entryPointId && this.safeGetSync(entryPointId, options);
 			if (entryPoint == null) {
-				const level = Math.floor(-Math.log(Math.random()) * this.mL);
+				const level = Math.min(Math.floor(-Math.log(this.random()) * this.mL), MAX_LEVEL);
 				const node = {
 					vector: storedVector,
 					scale: storedScale,
@@ -358,7 +362,7 @@ export class HierarchicalNavigableSmallWorld {
 			}
 
 			// Generate random level for this new element
-			const level = oldNode.level ?? Math.min(Math.floor(-Math.log(Math.random()) * this.mL), MAX_LEVEL);
+			const level = oldNode.level ?? Math.min(Math.floor(-Math.log(this.random()) * this.mL), MAX_LEVEL);
 			let currentLevel = entryPoint.level;
 			if (level > currentLevel) {
 				// if we are at a higher level, make this the new entry point

package/resources/longLivedTransactions.ts

@@ -0,0 +1,360 @@
+import { isMainThread, threadId } from 'node:worker_threads';
+import { resolve } from 'node:path';
+import { registryStatus } from '@harperfast/rocksdb-js';
+import { logger } from '../utility/logging/logger.ts';
+import { CONFIG_PARAMS } from '../utility/hdbTerms.ts';
+import * as envMngr from '../utility/environment/environmentManager.ts';
+import { convertToMS, prettyDuration } from '../utility/common_utils.ts';
+
+const DEFAULT_REPORT_THRESHOLD_MS = 300000;
+const SWEEP_INTERVAL_MS = 60000;
+// One line per over-age handle would, against a large leak, block the main thread's socket routing
+// for as long as it took to format them. The per-handle backoff below rotates the cap over the whole
+// set across passes, so every holder is still named eventually.
+export const MAX_LONG_LIVED_TRANSACTION_REPORTS_PER_PASS = 10;
+const MAX_HOLDER_CANDIDATES = 3;
+// The sweep prunes against the handles it just enumerated; the attribution map has no such
+// enumeration, so its entries expire instead. Far longer than a sweep interval, so a still-live
+// holder keeps its backoff.
+const ATTRIBUTION_STATE_TTL_MS = 3600000;
+// The prune is O(map) and the monitor calls reportLongLivedHolder once per over-threshold transaction
+// per tick, so pruning per call is O(N^2) in exactly the leak this reporting exists to describe.
+const ATTRIBUTION_PRUNE_INTERVAL_MS = 60000;
+
+type RegistryStatusFn = typeof registryStatus;
+let registryStatusFn: RegistryStatusFn | undefined = typeof registryStatus === 'function' ? registryStatus : undefined;
+
+// Report state is keyed by database path AND native id: rocksdb-js allocates transaction ids per
+// database descriptor, so two databases routinely both have id 2, and a reopen restarts the counter.
+type ReportState = { nextReportAgeMs: number; lastAgeMs: number; touchedAt: number };
+const sweepReports = new Map<string, ReportState>();
+const attributionReports = new Map<string, ReportState>();
+let sweepTimer: NodeJS.Timeout | undefined;
+let invalidThresholdWarned = false;
+let missingDetailsWarned = false;
+let lastAttributionPruneAt = 0;
+let thresholdInEffectMs = -1;
+
+function reportKey(databasePath: string | undefined, nativeId: number, openedAt?: number): string {
+	return `${databasePath ?? '?'} ${nativeId}${openedAt == null ? '' : ` ${openedAt}`}`;
+}
+
+/**
+ * Record this observation of a handle and return its report state. A shrinking age means the id was
+ * reused by a reopened descriptor, so the previous holder's backoff must not suppress the new one.
+ */
+function observeHandle(
+	reports: Map<string, ReportState>,
+	key: string,
+	ageMs: number,
+	thresholdMs: number
+): ReportState {
+	let state = reports.get(key);
+	if (state && ageMs < state.lastAgeMs) state = undefined;
+	if (!state) {
+		state = { nextReportAgeMs: thresholdMs, lastAgeMs: ageMs, touchedAt: 0 };
+		reports.set(key, state);
+	}
+	state.lastAgeMs = ageMs;
+	state.touchedAt = Date.now();
+	return state;
+}
+
+/**
+ * Drop the accrued backoff when the configured threshold changes, so a handle already under
+ * observation is re-measured against the new one. Without this, `nextReportAgeMs` keeps whatever it
+ * was given at first observation: lowering the threshold during an incident would not bring a
+ * silent handle's first report forward, and raising it would not quiet one. Reporting each live
+ * holder once more at the new threshold is what an operator who just changed it is asking for.
+ */
+function rebaseIfThresholdChanged(thresholdMs: number): void {
+	if (thresholdMs === thresholdInEffectMs) return;
+	thresholdInEffectMs = thresholdMs;
+	sweepReports.clear();
+	attributionReports.clear();
+}
+
+/**
+ * Apply a live threshold at the start of every reporting pass, including one that finds no eligible
+ * holder. Otherwise a threshold raised above a holder's age leaves its old attribution backoff in
+ * place when the threshold is lowered again.
+ */
+export function rebaseLongLivedTransactionReports(thresholdMs: number): void {
+	rebaseIfThresholdChanged(thresholdMs);
+}
+
+/**
+ * Push this handle's next report out to twice its current age, so one held for hours produces a
+ * handful of lines rather than one per pass. Called only where a line was actually emitted: advancing
+ * it for a handle the per-pass cap skipped would silence that handle without ever naming it.
+ */
+function markReported(state: ReportState, ageMs: number): void {
+	state.nextReportAgeMs = ageMs * 2;
+}
+
+/**
+ * How old a native transaction handle must be before it is reported, in ms; 0 disables every surface
+ * here. Read per pass rather than cached so a live config reload takes effect. A value that cannot
+ * be a duration falls back to the default instead of silently disabling reporting.
+ */
+export function getReportThresholdMs(): number {
+	const configured = envMngr.get(CONFIG_PARAMS.STORAGE_LONGTRANSACTIONREPORTTHRESHOLD);
+	if (configured == null) return DEFAULT_REPORT_THRESHOLD_MS;
+	// convertToMS returns 0 for anything that is neither string nor number, which YAML hands us for
+	// `yes`/`no` (booleans) and for an object left by a config merge — indistinguishable from the
+	// documented 0 that disables reporting, so it would silently switch off what the operator configured.
+	const ms = typeof configured === 'string' || typeof configured === 'number' ? convertToMS(configured) : NaN;
+	if (ms === 0) return 0;
+	if (!Number.isFinite(ms) || ms < 0) {
+		if (!invalidThresholdWarned) {
+			invalidThresholdWarned = true;
+			logger.warn?.(
+				`Invalid storage.longTransactionReportThreshold "${configured}"; using default ${DEFAULT_REPORT_THRESHOLD_MS}ms`
+			);
+		}
+		return DEFAULT_REPORT_THRESHOLD_MS;
+	}
+	return ms;
+}
+
+/**
+ * Report every native transaction handle in the process-global rocksdb-js registry that has been open
+ * past the threshold — including handles no `DatabaseTransaction` owns, which is the one class of
+ * holder no other surface can see (harper#2471). The registry spans worker threads, so this runs on
+ * the main thread only; a per-thread sweep would report each handle once per worker.
+ */
+export function runLongLivedTransactionSweep(): void {
+	try {
+		const thresholdMs = getReportThresholdMs();
+		rebaseLongLivedTransactionReports(thresholdMs);
+		if (thresholdMs === 0 || !registryStatusFn) return;
+		const status = registryStatusFn();
+		const seen = new Set<string>();
+		const due: { path: string | undefined; id: number; ageMs: number; state: ReportState }[] = [];
+		let missingDetails = false;
+		for (const database of status) {
+			const details = database.transactionDetails;
+			if (!details) {
+				missingDetails = true;
+				continue;
+			}
+			for (const handle of details) {
+				if (handle.ageMs < thresholdMs) continue;
+				const key = reportKey(database.path, handle.id);
+				seen.add(key);
+				const state = observeHandle(sweepReports, key, handle.ageMs, thresholdMs);
+				if (handle.ageMs >= state.nextReportAgeMs)
+					due.push({ path: database.path, id: handle.id, ageMs: handle.ageMs, state });
+			}
+		}
+		for (const key of sweepReports.keys()) if (!seen.has(key)) sweepReports.delete(key);
+		if (missingDetails && !missingDetailsWarned) {
+			missingDetailsWarned = true;
+			logger.warn?.(
+				'Long-lived transaction reporting is degraded: the installed @harperfast/rocksdb-js does not expose registryStatus().transactionDetails (requires 2.8.0)'
+			);
+		}
+		if (due.length === 0) return;
+		due.sort((a, b) => b.ageMs - a.ageMs);
+		for (const handle of due.slice(0, MAX_LONG_LIVED_TRANSACTION_REPORTS_PER_PASS)) {
+			markReported(handle.state, handle.ageMs);
+			logger.warn?.(
+				`Long-lived RocksDB transaction handle: id ${handle.id} has been open for ${prettyDuration(handle.ageMs)} on database ${handle.path ?? '?'}. ` +
+					`A handle this old can hold write intents that park other writers' commits, and may pin a read snapshot that blocks version reclamation.`
+			);
+		}
+		if (due.length > MAX_LONG_LIVED_TRANSACTION_REPORTS_PER_PASS) {
+			logger.warn?.(
+				`${due.length - MAX_LONG_LIVED_TRANSACTION_REPORTS_PER_PASS} further RocksDB transaction handle(s) are open past the ${prettyDuration(thresholdMs)} reporting threshold and were not listed individually.`
+			);
+		}
+	} catch (error) {
+		logger.error?.('Error sweeping for long-lived transaction handles', error);
+	}
+}
+
+/**
+ * Start the periodic sweep. Armed unconditionally on the main thread — the pass itself honors a live
+ * threshold change, including one that disables it — and idempotent because the thread startup path
+ * that calls it can run more than once.
+ */
+export function startLongLivedTransactionReporting(): void {
+	if (!isMainThread || sweepTimer) return;
+	if (!registryStatusFn) {
+		logger.debug?.('Long-lived transaction reporting unavailable; @harperfast/rocksdb-js exposes no registryStatus()');
+		return;
+	}
+	sweepTimer = setInterval(runLongLivedTransactionSweep, SWEEP_INTERVAL_MS).unref();
+}
+
+export type LongLivedHolder = {
+	databasePath: string | undefined;
+	nativeId: number;
+	openedAt: number;
+	ageMs: number;
+	databaseName?: string;
+	tableName?: string;
+	// A thunk, not a number: the caller's count walks the whole staged write set, and the backoff below
+	// suppresses most ticks — the #2471 shape is a transaction holding a large write set for hours, so
+	// paying for that walk on every suppressed tick is the one cost this reporting must not add.
+	countPendingWrites: () => number;
+	states: string[];
+	timeoutBudget?: number;
+	startedFrom?: { resourceName: string; method: string };
+};
+
+export type LongLivedHolderReportBudget = {
+	remaining: number;
+	skipped: number;
+	thresholdMs: number;
+};
+
+export function createLongLivedHolderReportBudget(thresholdMs: number): LongLivedHolderReportBudget {
+	return { remaining: MAX_LONG_LIVED_TRANSACTION_REPORTS_PER_PASS, skipped: 0, thresholdMs };
+}
+
+export function finishLongLivedHolderReports(reportBudget: LongLivedHolderReportBudget): void {
+	if (reportBudget.skipped === 0) return;
+	try {
+		logger.warn?.(
+			`${reportBudget.skipped} further Harper transaction holder(s) are open past the ${prettyDuration(reportBudget.thresholdMs)} reporting threshold and were deferred.`
+		);
+	} catch {}
+}
+
+/**
+ * Report a handle this thread owns, with the attribution the registry sweep cannot supply. The native
+ * id is the join key between the two: the sweep names an id, this names who holds it and which state
+ * kept the open-transaction monitor from reaping it (harper#2471). Never throws — the monitor
+ * interval that calls it has no handler.
+ */
+export function reportLongLivedHolder(holder: LongLivedHolder, reportBudget?: LongLivedHolderReportBudget): void {
+	try {
+		if (!registryStatusFn) return;
+		const thresholdMs = reportBudget?.thresholdMs ?? getReportThresholdMs();
+		rebaseLongLivedTransactionReports(thresholdMs);
+		if (thresholdMs === 0) return;
+		const now = Date.now();
+		if (now - lastAttributionPruneAt >= ATTRIBUTION_PRUNE_INTERVAL_MS) {
+			lastAttributionPruneAt = now;
+			const expiredBefore = now - ATTRIBUTION_STATE_TTL_MS;
+			for (const [key, state] of attributionReports)
+				if (state.touchedAt < expiredBefore) attributionReports.delete(key);
+		}
+		const state = observeHandle(
+			attributionReports,
+			reportKey(holder.databasePath, holder.nativeId, holder.openedAt),
+			holder.ageMs,
+			thresholdMs
+		);
+		if (holder.ageMs < state.nextReportAgeMs) return;
+		if (reportBudget && reportBudget.remaining <= 0) {
+			reportBudget.skipped++;
+			return;
+		}
+		markReported(state, holder.ageMs);
+		if (reportBudget) reportBudget.remaining--;
+		const table = holder.tableName ? `.${holder.tableName}` : '';
+		logger.warn?.(
+			`Harper transaction has held RocksDB transaction ${holder.nativeId} for ${prettyDuration(holder.ageMs)} ` +
+				`on thread ${threadId}, database ${holder.databaseName ?? '?'}${table} (${holder.databasePath ?? '?'}), ` +
+				`holding ${holder.countPendingWrites()} staged write(s), state: ${holder.states.join('+')}` +
+				(holder.timeoutBudget ? `, open-transaction budget ${holder.timeoutBudget}ms` : '') +
+				(holder.startedFrom?.resourceName
+					? `, started from ${holder.startedFrom.resourceName}${holder.startedFrom.method ? '.' + holder.startedFrom.method : ''}`
+					: '') +
+				'. It is not being reaped by the open-transaction monitor; the state tells you why.'
+		);
+	} catch (error) {
+		logger.debug?.('Failed to report a long-lived transaction holder', error);
+	}
+}
+
+/**
+ * The live handles that could be holding the write intent a stuck commit is parked on, as a sentence
+ * to append to that commit's log. A handle on another database is offered too, labelled with its path:
+ * the verification table is one process-global slot array whose hash mixes in the database id, so a
+ * holder in `system` or `oauth` collides with — and parks — a `data` commit at the same rate two `data`
+ * keys do. Rank puts the stuck commit's own database first, then age. Age only ranks: a coordinated
+ * retry parks on whichever transaction holds the slot when the commit reaches it, which can be younger
+ * than the commit itself, so filtering by age could drop the real holder.
+ * Returns '' rather than throwing — the caller's own 503 must survive a failure here.
+ */
+export function describeHolderCandidates(
+	databasePath: string | undefined,
+	excludeNativeId: number | undefined
+): string {
+	if (!databasePath || !registryStatusFn) return '';
+	try {
+		if (getReportThresholdMs() === 0) return '';
+		const target = resolve(databasePath);
+		const candidates: { id: number; ageMs: number; sameDatabase: boolean; path: string | undefined }[] = [];
+		let targetSeen = false;
+		let targetHasDetails = false;
+		for (const database of registryStatusFn()) {
+			// An entry with no path can never be the target, and resolve() would throw on it — which,
+			// inside this loop, would drop the candidate list for every database rather than just this one.
+			const sameDatabase = database.path != null && resolve(database.path) === target;
+			const details = database.transactionDetails;
+			if (sameDatabase) {
+				targetSeen = true;
+				if (details) targetHasDetails = true;
+			}
+			if (!details) continue;
+			for (const handle of details)
+				if (!sameDatabase || handle.id !== excludeNativeId)
+					candidates.push({ id: handle.id, ageMs: handle.ageMs, sameDatabase, path: database.path });
+		}
+		if (!targetHasDetails) {
+			// A database with no other handles is normal; no entry at all means the join between a store's
+			// path and the registry's path has drifted, which is the one worth telling an operator apart.
+			logger.debug?.(
+				targetSeen
+					? `Registry entry for ${target} exposes no transactionDetails; only other databases' handles can be offered as holder candidates for its stuck commit`
+					: `No registry entry for ${target}; only other databases' handles can be offered as holder candidates for its stuck commit`
+			);
+		}
+		if (candidates.length === 0) return '';
+		candidates.sort((a, b) => Number(b.sameDatabase) - Number(a.sameDatabase) || b.ageMs - a.ageMs);
+		const shortlist = candidates.slice(0, MAX_HOLDER_CANDIDATES);
+		// Ranking this database first and then capping can bury the one candidate the cross-database
+		// search exists to surface: a busy database routinely has MAX_HOLDER_CANDIDATES handles of its
+		// own, so a foreign holder would only ever appear inside the "and N more" count. Give the oldest
+		// one the last slot instead.
+		if (shortlist.length === MAX_HOLDER_CANDIDATES && shortlist.every((candidate) => candidate.sameDatabase)) {
+			const foreign = candidates.find((candidate) => !candidate.sameDatabase);
+			if (foreign) shortlist[MAX_HOLDER_CANDIDATES - 1] = foreign;
+		}
+		const named = shortlist
+			.map(
+				(candidate) =>
+					`${candidate.id} (open ${prettyDuration(candidate.ageMs)}${candidate.sameDatabase ? '' : ` on ${candidate.path ?? '?'}`})`
+			)
+			.join(', ');
+		return (
+			` Live transaction handles, any of which could hold the write intent this commit is parked on ` +
+			`(the verification table is one process-global slot array, so a handle on another database can hold it), ` +
+			`this database first, oldest first: ${named}` +
+			(candidates.length > MAX_HOLDER_CANDIDATES ? `, and ${candidates.length - MAX_HOLDER_CANDIDATES} more.` : '.')
+		);
+	} catch {
+		return '';
+	}
+}
+
+export function setRegistryStatusForTests(fn?: RegistryStatusFn): void {
+	registryStatusFn = fn;
+}
+
+export function resetLongLivedTransactionReportsForTests(): void {
+	sweepReports.clear();
+	attributionReports.clear();
+	invalidThresholdWarned = false;
+	missingDetailsWarned = false;
+	lastAttributionPruneAt = 0;
+	thresholdInEffectMs = -1;
+	if (sweepTimer) {
+		clearInterval(sweepTimer);
+		sweepTimer = undefined;
+	}
+}