@harperfast/harper 5.2.13 → 5.3.0-beta.1

package/resources/replayLogs.ts

@@ -40,13 +40,36 @@ export function structuresWouldShrink(existing: any, updated: any): boolean {
 	return false;
 }
 
-export function replayLogs(rootStore: RocksDatabase, tables: any): Promise<void> {
-	if (!isMainThread) return; // ideally we don't do it like this, but for now this is predictable
-	return new Promise((resolve) => {
+/** The total wall-clock budget (ms) a replay may spend, as configured for this process. */
+export function replayTimeBudgetMs(): number {
+	const configuredReplayTimeout = Number(envGet(CONFIG_PARAMS.REPLICATION_REPLAYTIMEOUT));
+	return configuredReplayTimeout > 0 ? configuredReplayTimeout : REPLAY_WALL_CLOCK_LIMIT_MS;
+}
+
+/**
+ * `electedReplayer` marks a caller that has already won a cross-thread election for this store —
+ * the branch claim (branchDatabase.ts) — and so may replay off the main thread. It also makes the
+ * replay strict: the promise settles (never hangs), and a failure to apply the tail — a commit or
+ * write error, a stall or wall-clock abort — rejects instead of quietly serving a rewound store.
+ * Undecodable entries and a torn tail stay tolerated in both modes: a crash tears the last frame of
+ * a log by construction, and end-of-log is the designed reading of it (see replayLogsGuards.ts). A
+ * **mid-log** break is different — entries behind it were acknowledged and are now quarantined — so
+ * an elected replay rejects on one rather than publish a branch as a trustworthy point-in-time copy
+ * over known lost writes; boot replay still logs it and continues (harper#2016, harper#2063).
+ */
+export function replayLogs(rootStore: RocksDatabase, tables: any, electedReplayer?: boolean): Promise<void> {
+	if (!isMainThread && !electedReplayer) return Promise.resolve(); // ideally we don't do it like this, but for now this is predictable
+	return new Promise((resolve, reject) => {
 		const acquired = rootStore.tryLock('replayLogs', async () => {
 			resolve();
 		});
-		if (!acquired) return;
+		if (!acquired) {
+			// An elected replayer is the store's sole opener, so a held lock is a protocol violation;
+			// hanging its awaited promise would wedge the branch claim in CREATING. The boot-scan
+			// caller keeps the settle-on-unlock semantics (nothing awaits it).
+			if (electedReplayer) reject(new Error(`The replay lock for ${(rootStore as any).databaseName} is already held`));
+			return;
+		}
 		// Shed transaction-log files already older than the audit retention window before
 		// replaying. A node that crash-loops during recovery never reaches the steady-state
 		// cleanup loop, so without this its aged backlog only grows and enlarges each subsequent
@@ -75,9 +98,13 @@ export function replayLogs(rootStore: RocksDatabase, tables: any): Promise<void>
 			tableById.set(table.tableId, table);
 		}
 		// replay all the logs
-		let transaction: DatabaseTransaction;
+		let transaction: DatabaseTransaction | undefined;
 		let lastTimestamp = 0;
 		let writes = 0;
+		// Writes staged on the currently-open transaction, so a discard can be taken back out of `writes`.
+		let stagedWrites = 0;
+		// Records dropped because a corrupt frame truncated the transaction they belong to.
+		let discardedWrites = 0;
 		let skipped = 0;
 		// Track forward progress so a backlog of unwritable entries can't grind the boot thread
 		// forever (harper#1266). `noProgressRun` counts every entry processed without a successful
@@ -89,13 +116,33 @@ export function replayLogs(rootStore: RocksDatabase, tables: any): Promise<void>
 		let lastProgressTime = replayStartTime;
 		// Total wall-clock budget (ms) for replay, configurable via `replication.replayTimeout`;
 		// falls back to the 10-minute default (harper#1316).
-		const configuredReplayTimeout = Number(envGet(CONFIG_PARAMS.REPLICATION_REPLAYTIMEOUT));
-		const replayTimeoutMs = configuredReplayTimeout > 0 ? configuredReplayTimeout : REPLAY_WALL_CLOCK_LIMIT_MS;
+		const replayTimeoutMs = replayTimeBudgetMs();
+		// Set on the first unrecoverable failure in elected mode; rejects below instead of publishing
+		// a store the replay knows is incomplete.
+		let strictFailure: Error | undefined;
+		const strictAbort = (error: Error, staged: DatabaseTransaction | undefined): void => {
+			try {
+				staged?.abort();
+			} catch (abortError) {
+				logger.warn('Error aborting a strict replay transaction', abortError);
+			}
+			strictFailure = error;
+		};
 		const txnLog: RocksTransactionLogStore = (rootStore as any).auditStore;
-		for (const auditRecord of txnLog.getRange({ startFromLastFlushed: true, readUncommitted: true }) as any) {
+		const entries = txnLog.getRange({
+			startFromLastFlushed: true,
+			readUncommitted: true,
+			trackCorruptTransactions: true,
+		});
+		for (const auditRecord of entries as any) {
 			if (noProgressRun > 0 && shouldAbortStalledReplay(noProgressRun, performance.now() - lastProgressTime)) {
+				const stallDiagnostic = `Aborting transaction-log replay in ${(rootStore as any).databaseName} database: ${noProgressRun} consecutive audit entries with no successful write (${skipped} skipped as unrecoverable, ${writes} replayed so far). This backlog is making no forward progress and was blocking startup (harper#1266) — typically a peer transaction log whose values reference unresolvable shared structures (harper#1163), or a backlog for a dropped table.`;
+				if (electedReplayer) {
+					strictAbort(new Error(stallDiagnostic), transaction);
+					break;
+				}
 				logger.fatal(
-					`Aborting transaction-log replay in ${(rootStore as any).databaseName} database: ${noProgressRun} consecutive audit entries with no successful write (${skipped} skipped as unrecoverable, ${writes} replayed so far). This backlog is making no forward progress and was blocking startup (harper#1266) — typically a peer transaction log whose values reference unresolvable shared structures (harper#1163), or a backlog for a dropped table. Continuing boot without replaying the remainder; shed or relocate the oversized/undecodable peer transaction log(s), or re-clone this node, to recover the unreplayed data.`
+					`${stallDiagnostic} Continuing boot without replaying the remainder; shed or relocate the oversized/undecodable peer transaction log(s), or re-clone this node, to recover the unreplayed data.`
 				);
 				break;
 			}
@@ -105,6 +152,7 @@ export function replayLogs(rootStore: RocksDatabase, tables: any): Promise<void>
 				nodeId,
 				recordId,
 				version,
+				txnLogKey,
 				residencyId,
 				expiresAt,
 				originatingOperation,
@@ -117,6 +165,11 @@ export function replayLogs(rootStore: RocksDatabase, tables: any): Promise<void>
 					noProgressRun++;
 					continue;
 				}
+				if (type === 'evict') {
+					noProgressRun = 0;
+					lastProgressTime = performance.now();
+					continue;
+				}
 				const Table = tableById.get(tableId);
 				if (!Table) {
 					// Entry for a table this node no longer has (dropped/foreign). Not an
@@ -165,14 +218,35 @@ export function replayLogs(rootStore: RocksDatabase, tables: any): Promise<void>
 					warnedReplayHappening = true;
 					console.warn('Harper was not properly shutdown, replaying transaction logs to synchronize database');
 				}
-				if (lastTimestamp !== version) {
-					lastTimestamp = version;
+				// Transactions are delimited by the log key — which is also what `truncatedVersions` records —
+				// and each write is replayed at its own stored record version. The two differ for a source
+				// fill, and stamping such a record at the log key would move its version forward and make a
+				// later legitimate write in between look stale (harper#2411).
+				if (lastTimestamp !== txnLogKey) {
+					const torn = entries.corruptFrameStop.truncatedVersions.has(lastTimestamp);
+					lastTimestamp = txnLogKey;
 					try {
-						// commit the last transaction since we are starting a new one
-						transaction?.directCommitSync();
+						// commit the last transaction since we are starting a new one, unless a corrupt
+						// frame swallowed the rest of it — half of a source transaction must never become
+						// durable, so it is dropped whole and stays in the log for a repaired retry
+						if (torn) {
+							writes -= stagedWrites;
+							discardedWrites += stagedWrites;
+							transaction?.abort();
+						} else transaction?.directCommitSync();
 					} catch (error) {
-						logger.error('Error committing replay transaction', error);
+						// directCommitSync aborts and detaches its transaction on failure; no cleanup here.
+						// The torn branch already backed stagedWrites out of writes before this try, so
+						// only a failed COMMIT (never applied) needs it backed out here too — otherwise a
+						// commit failure left `writes` counting records that never actually landed.
+						if (!torn) writes -= stagedWrites;
+						if (electedReplayer) {
+							strictFailure = error;
+							break;
+						}
+						logger.error(`Error ${torn ? 'discarding a torn' : 'committing'} replay transaction`, error);
 					}
+					stagedWrites = 0;
 					// Abort if replay has exceeded the total wall-clock budget even while making progress
 					// (harper#1316, facet a). shouldAbortStalledReplay resets its counters on every write,
 					// so a slow-but-progressing replay (deep out-of-order audit chain walk per entry) can
@@ -181,15 +255,18 @@ export function replayLogs(rootStore: RocksDatabase, tables: any): Promise<void>
 					// is not yet staged, so aborting never tears a same-version (same source-transaction)
 					// write batch in half. Re-clone to recover the unreplayed remainder.
 					if (shouldAbortSlowReplay(performance.now() - replayStartTime, replayTimeoutMs)) {
-						logger.fatal(
-							`Aborting transaction-log replay in ${(rootStore as any).databaseName} database: replay has exceeded the wall-clock time limit (${writes} written, ${skipped} skipped). The transaction log contains a pathologically deep out-of-order write history that is too expensive to reconcile during boot (harper#1316). Re-clone this node from a healthy leader to recover the unreplayed data.`
-						);
-						transaction = undefined as any; // already committed above; nothing staged for the new version
+						const slowMessage = `Aborting transaction-log replay in ${(rootStore as any).databaseName} database: replay has exceeded the wall-clock time limit (${writes} written, ${skipped} skipped). The transaction log contains a pathologically deep out-of-order write history that is too expensive to reconcile during boot (harper#1316). Re-clone this node from a healthy leader to recover the unreplayed data.`;
+						transaction = undefined; // already committed above; nothing staged for the new version
+						if (electedReplayer) {
+							strictFailure = new Error(slowMessage);
+							break;
+						}
+						logger.fatal(slowMessage);
 						break;
 					}
 					transaction = new DatabaseTransaction();
 					transaction.db = primaryStore;
-					transaction.timestamp = version;
+					transaction.timestamp = txnLogKey;
 					// retries=1 routes operation.commit() through its retry path (no duplicate audit staging)
 					transaction.retries = 1;
 					// Explicit replay marker: skips schema validation (harper#1316) and makes save() stamp
@@ -198,8 +275,9 @@ export function replayLogs(rootStore: RocksDatabase, tables: any): Promise<void>
 					transaction.isReplay = true;
 				}
 				context.transaction = transaction;
-				const options = { context, residencyId, nodeId, originatingOperation };
+				const options = { context, residencyId, nodeId, originatingOperation, version };
 				writes++;
+				stagedWrites++;
 				switch (type) {
 					case 'put':
 						tableInstance._writeUpdate(recordId, record, true, options);
@@ -274,6 +352,10 @@ export function replayLogs(rootStore: RocksDatabase, tables: any): Promise<void>
 				noProgressRun = 0;
 				lastProgressTime = performance.now();
 			} catch (err) {
+				if (electedReplayer) {
+					strictAbort(err, transaction);
+					break;
+				}
 				// A write that threw made no forward progress either — count it toward the stall
 				// bound so a continuous stream of throwing writes can't grind the boot thread
 				// indefinitely (and the per-entry error log below can't spam unboundedly). harper#1266
@@ -283,18 +365,62 @@ export function replayLogs(rootStore: RocksDatabase, tables: any): Promise<void>
 				});
 			}
 		}
-		try {
-			transaction?.directCommitSync();
-		} catch (error) {
-			logger.error('Error committing replay transaction', error);
+		const finalTorn = entries.corruptFrameStop.truncatedVersions.has(lastTimestamp);
+		if (!strictFailure) {
+			try {
+				if (finalTorn) {
+					writes -= stagedWrites;
+					discardedWrites += stagedWrites;
+					transaction?.abort();
+				} else transaction?.directCommitSync();
+			} catch (error) {
+				// directCommitSync aborts and detaches its transaction on failure; no cleanup here.
+				// Mirrors the interior version-boundary catch above: a failed commit never applied, so
+				// it must not stay counted in `writes`.
+				if (!finalTorn) writes -= stagedWrites;
+				if (electedReplayer) strictFailure = error;
+				logger.error(`Error ${finalTorn ? 'discarding a torn' : 'committing'} replay transaction`, error);
+			}
+		}
+		// `breaks` also counts a torn tail — the designed, benign reading of a crash's last frame, and
+		// the reporter already logged it at `warn`. Only a mid-log break lost entries, so it alone earns
+		// this quarantine/repair summary; duplicating it for a torn tail would tell an operator to
+		// repair or re-clone the node on every ordinary unclean shutdown.
+		if (entries.corruptFrameStop.midLogBreak) {
+			logger.error(
+				`Transaction-log replay in ${(rootStore as any).databaseName} database stopped at a corrupt entry after replaying ${writes} records. Every entry after the break is quarantined — neither replayed nor replicated — and ${discardedWrites} record(s) of the transaction the break truncated were discarded rather than applied in part. Repair the transaction log or re-clone this node to recover them.`
+			);
+			// electedReplayer must not resolve past known loss — see the doc comment above. Only set
+			// strictFailure if nothing has already failed the replay for an unrelated reason, so this
+			// diagnostic never shadows a more specific commit/write error.
+			if (electedReplayer && !strictFailure) {
+				strictFailure = new Error(
+					`Elected replay in ${(rootStore as any).databaseName} database stopped at a mid-log corrupt transaction-log frame; entries behind the break were acknowledged and are now quarantined. Refusing to publish a branch over known lost writes — repair the transaction log or re-clone this node.`
+				);
+			}
 		}
 		if (writes > 0) logger.warn(`Replayed ${writes} records in ${(rootStore as any).databaseName} database`);
 		if (skipped > 0)
 			logger.warn(
 				`Skipped ${skipped} unrecoverable audit entries in ${(rootStore as any).databaseName} database during replay`
 			);
-		// we never actually release the lock because we only want to ever run one time
-		// rootStore.unlock('replayLogs');
+		if (strictFailure) {
+			// A failed strict replay must be re-runnable: the branch claim resets for a retry, so the
+			// lock must not be what survives to wedge it. Only a COMPLETED replay keeps the lock
+			// forever (the once-per-boot guarantee). Reject BEFORE unlocking: unlock wakes tryLock
+			// callbacks — including this holder's own resolve() — and the first settle must be the
+			// rejection, or a failed replay reads as success.
+			reject(strictFailure);
+			try {
+				rootStore.unlock('replayLogs');
+			} catch (unlockError) {
+				logger.warn('Error releasing the replay lock after a failed replay', unlockError);
+			}
+		} else {
+			// we never actually release the lock because we only want to ever run one time
+			// rootStore.unlock('replayLogs');
+			resolve();
+		}
 	});
 }
 function asBinary(buffer) {

package/resources/replayLogsGuards.ts

@@ -132,16 +132,66 @@ export function shouldAbortSlowReplay(totalElapsedMs: number, timeLimitMs = REPL
 }
 
 /**
- * Wraps a transaction-log query iterator so a corrupt/torn frame ends that log's iteration
- * cleanly instead of escaping as an uncaughtException. rocksdb-js throws a bounded RangeError
- * when an entry's framing is broken; framing loss means the next entry can't be located, so the
- * frame marks end-of-log (entries before it were already yielded) and startup replay /
- * replication broadcast continue. `onCorruptFrame` fires once, latched — kept a callback (not a
- * direct log) so this module stays out of the Harper module graph and is unit-testable.
+ * A corrupt transaction-log frame from rocksdb-js. `resyncPosition` is set only when intact entries
+ * follow the break, which is what distinguishes lost entries from a merely truncated tail; its
+ * absence is also all that any rocksdb-js predating that field can report.
+ */
+export type CorruptFrameError = RangeError & {
+	logId?: number | null;
+	position?: number | null;
+	resyncPosition?: number | null;
+	unreadableBytes?: number | null;
+};
+
+/**
+ * Corrupt frames hit by one `getRange` call, shared by its per-log iterators.
+ *
+ * `truncatedVersions` is what a consumer has to act on: a break destroys the framing after the last
+ * entry the broken log yielded, and whether the entries it swallowed continued that entry's version
+ * is exactly what can no longer be read — UNLESS that last entry's own `endTxn` already closed its
+ * transaction, in which case nothing of it was left open to swallow. That version's transaction is
+ * therefore incomplete only when it wasn't already known-closed, and replay must discard it rather
+ * than commit the part of it that was still readable. A log that broke before yielding anything
+ * truncates no transaction.
+ *
+ * `midLogBreak` is coarser: true once any break in this range left intact entries unreadable behind
+ * it (see {@link isMidLogBreak}), regardless of which version they belonged to. A consumer that must
+ * never serve a range with lost entries — an elected branch replay publishing itself as a trustworthy
+ * point-in-time copy — rejects on this rather than on `truncatedVersions`, since a mid-log break can
+ * strand entries whose version this range never even attributes to one log (harper#2016, #2063).
+ */
+export interface CorruptFrameStop {
+	breaks: number;
+	truncatedVersions: Set<number>;
+	midLogBreak: boolean;
+}
+
+/**
+ * Whether a corrupt frame is a mid-log break — intact entries followed it, so entries were lost
+ * rather than the log merely being torn at its tail. Shared by the reporter below and by
+ * `RocksTransactionLogStore.getRange`'s `CorruptFrameStop.midLogBreak`, so the two can't drift on
+ * what counts as a mid-log break.
+ */
+export function isMidLogBreak(error: CorruptFrameError): boolean {
+	return error.resyncPosition != null;
+}
+
+/**
+ * Wraps a transaction-log query iterator so a corrupt/torn frame ends that log's iteration cleanly
+ * instead of escaping as an uncaughtException.
+ *
+ * Iteration stops at the break whether or not intact entries follow it. Resuming past a mid-log
+ * break would recover those entries, but a transaction log carries no frame-level transaction
+ * boundaries: replay groups equal-version entries into one source transaction, so skipping a frame
+ * inside such a group applies and checkpoints the surviving subset of a transaction that never
+ * committed that way at the source. Availability loss is visible and recoverable; a silently torn
+ * transaction is neither. Recovery is deferred until the engine can resume at a proven boundary
+ * (harper#2016, harper#2063); `resyncPosition` is used only to report the break as lost entries
+ * rather than as a truncated tail.
  */
 export function endIteratorOnCorruptFrame<T>(
 	iterator: Iterator<T>,
-	onCorruptFrame: (error: RangeError) => void
+	onCorruptFrame: (error: CorruptFrameError) => void
 ): IterableIterator<T> {
 	let stopped = false;
 	return {
@@ -157,7 +207,7 @@ export function endIteratorOnCorruptFrame<T>(
 				// version-dependent (1.4.2 added hex offsets). Anything else re-throws.
 				if (!(error instanceof RangeError)) throw error;
 				stopped = true;
-				onCorruptFrame(error);
+				onCorruptFrame(error as CorruptFrameError);
 				return { done: true, value: undefined };
 			}
 		},
@@ -175,3 +225,116 @@ export function endIteratorOnCorruptFrame<T>(
 		},
 	};
 }
+
+/**
+ * A corrupt frame, accumulated for the life of the process. The same frame is re-encountered by
+ * every reader until each consumer's resume cursor has passed it, so this is deduplicated by
+ * location and the repeats become a count.
+ */
+export interface CorruptFrameReport {
+	log: string;
+	logId?: number;
+	position?: number;
+	/** Intact entries followed the break, so entries were lost rather than merely truncated. */
+	midLog: boolean;
+	/** Extent of the unreadable region, when the engine reports it; 0 when nothing valid follows it. */
+	unreadableBytes: number;
+	firstSeen: number;
+	lastSeen: number;
+	occurrences: number;
+}
+
+/** Distinct break sites retained. One physical corruption yields one site, so this is generous. */
+export const MAX_CORRUPT_FRAME_REPORTS = 256;
+
+// Re-encounters move a site to the end, so the first key is the least recently seen. Full means
+// evicting that one rather than refusing a new site, which could not then be deduplicated.
+const corruptFrameReports = new Map<string, CorruptFrameReport>();
+let evictedCorruptFrameReports = 0;
+
+/**
+ * Every corrupt frame seen by this worker. This is not yet consumed by cluster/health status;
+ * callers must aggregate it across worker threads before exposing a node-wide signal.
+ *
+ * Mid-log breaks are immediately logged at error level even without that consumer.
+ */
+export function getCorruptFrameReports(): CorruptFrameReport[] {
+	return [...corruptFrameReports.values()];
+}
+
+/** Break sites evicted because {@link MAX_CORRUPT_FRAME_REPORTS} was reached. */
+export function getEvictedCorruptFrameReportCount(): number {
+	return evictedCorruptFrameReports;
+}
+
+export function clearCorruptFrameReports() {
+	corruptFrameReports.clear();
+	evictedCorruptFrameReports = 0;
+}
+
+// `logId`/`position` are absent on any rocksdb-js predating the resync support, and every break on
+// a stream would then collapse onto one key — folding genuinely different corruptions into a single
+// count that only ever logs once. The message carries the offset and file in text, so it separates
+// them when the fields can't.
+function corruptFrameKey(logName: string, error: CorruptFrameError): string {
+	const { logId, position } = error;
+	return logId != null && position != null
+		? `${logName}\u0000${logId}:${position}`
+		: `${logName}\u0000${error.message}`;
+}
+
+/**
+ * Records a corrupt frame and logs it once per distinct break.
+ *
+ * A mid-log break is an `error`, not a `warn`: entries after it were acknowledged and are now
+ * quarantined behind it. Severity follows the break's own shape, never this pass's outcome — every
+ * pass stops, so keying on that would report the #2063 signature with the benign torn-tail `warn`.
+ * A site first seen as a torn tail can escalate later, since only a rocksdb-js that reports
+ * `resyncPosition` can tell the two apart.
+ */
+export function createCorruptFrameReporter(logger: {
+	warn: (message: string, error?: unknown) => void;
+	error: (message: string, error?: unknown) => void;
+}) {
+	return (logName: string) => (error: CorruptFrameError) => {
+		const midLog = isMidLogBreak(error);
+		const unreadableBytes = error.unreadableBytes ?? 0;
+		const now = Date.now();
+		const key = corruptFrameKey(logName, error);
+		const existing = corruptFrameReports.get(key);
+		if (existing) {
+			corruptFrameReports.delete(key);
+			corruptFrameReports.set(key, existing);
+			existing.occurrences++;
+			existing.lastSeen = now;
+			if (!midLog || existing.midLog) return;
+			existing.midLog = true;
+			existing.unreadableBytes = unreadableBytes;
+		} else {
+			if (corruptFrameReports.size >= MAX_CORRUPT_FRAME_REPORTS) {
+				corruptFrameReports.delete(corruptFrameReports.keys().next().value);
+				evictedCorruptFrameReports++;
+			}
+			corruptFrameReports.set(key, {
+				log: logName,
+				logId: error.logId ?? undefined,
+				position: error.position ?? undefined,
+				midLog,
+				unreadableBytes,
+				firstSeen: now,
+				lastSeen: now,
+				occurrences: 1,
+			});
+		}
+		if (midLog) {
+			logger.error(
+				`Corrupt entry in transaction log "${logName}"; ${unreadableBytes} byte(s) are unreadable and the entries within them are lost. ` +
+					'Intact entries follow the break, but reading stops there rather than skipping the frame, which could tear a source transaction: ' +
+					'they are quarantined until the log is repaired or this node is re-cloned, and they are neither replayed nor replicated meanwhile.',
+				error
+			);
+		} else {
+			logger.warn(`Stopping transaction log "${logName}" at a corrupt entry during replay`, error);
+		}
+	};
+}

package/resources/replicatedApplyFailure.ts

@@ -0,0 +1,77 @@
+import { onRemovedDB } from './databases.ts';
+import { logger } from '../utility/logging/logger.ts';
+import { contextStorage } from './transaction.ts';
+
+export interface ReplicatedApplyFailure {
+	readonly database: string;
+	readonly table?: string;
+	readonly nodeId: number;
+	readonly position: number;
+	readonly localTime?: number;
+	readonly error: unknown;
+}
+
+export type ReplicatedApplyFailureListener = (failure: ReplicatedApplyFailure) => void | Promise<void>;
+
+const listenersByDatabase = new Map<string, Set<ReplicatedApplyFailureListener>>();
+let watchingDatabaseRemoval = false;
+
+export function registerReplicatedApplyFailureListener(
+	database: string,
+	listener: ReplicatedApplyFailureListener
+): void {
+	if (!watchingDatabaseRemoval) {
+		onRemovedDB((database) => listenersByDatabase.delete(database));
+		watchingDatabaseRemoval = true;
+	}
+	let listeners = listenersByDatabase.get(database);
+	if (!listeners) listenersByDatabase.set(database, (listeners = new Set()));
+	listeners.add(listener);
+}
+
+export function unregisterReplicatedApplyFailureListener(
+	database: string,
+	listener: ReplicatedApplyFailureListener
+): void {
+	const listeners = listenersByDatabase.get(database);
+	listeners?.delete(listener);
+	if (listeners?.size === 0) listenersByDatabase.delete(database);
+}
+
+export async function notifyReplicatedApplyFailure(
+	database: string,
+	event: { table?: string; nodeId?: number; localTime?: number },
+	position: number | undefined,
+	error: unknown,
+	table?: string
+): Promise<void> {
+	if (listenersByDatabase.size === 0) return;
+	try {
+		if (typeof event?.nodeId !== 'number' || typeof position !== 'number') return;
+		const listeners = listenersByDatabase.get(database);
+		if (!listeners) return;
+		const failure: ReplicatedApplyFailure = Object.freeze({
+			database,
+			table: event.table ?? table,
+			nodeId: event.nodeId,
+			position,
+			localTime: event.localTime,
+			error,
+		});
+		for (const listener of [...listeners]) {
+			try {
+				await contextStorage.exit(() => listener(failure));
+			} catch (error) {
+				logListenerFailure(database, error);
+			}
+		}
+	} catch (error) {
+		logListenerFailure(database, error);
+	}
+}
+
+function logListenerFailure(database: string, error: unknown): void {
+	try {
+		logger.error?.('replicated apply failure listener failed', database, error);
+	} catch {}
+}

package/resources/scheduler/scheduler.ts

@@ -1,5 +1,5 @@
 import { isAbsolute, join } from 'node:path';
-import { getWorkerIndex } from '../../server/threads/manageThreads.js';
+import { isApplicationPrimaryWorker } from '../../server/threads/manageThreads.js';
 import { ClientError } from '../../utility/errors/hdbError.ts';
 import { convertToMS } from '../../utility/common_utils.ts';
 import harperLogger from '../../utility/logging/harper_logger.ts';
@@ -94,12 +94,12 @@ export async function handleApplication(scope): Promise<void> {
 	}
 
 	// Activation gates: one worker owns scheduling for the whole node
-	// (getWorkerIndex() === 0 is correct in every threading mode, including
+	// (the application's primary worker is correct in every threading mode, including
 	// threads:0 where the main thread acts as worker 0), and a deploy
 	// pre-flight validation scope must never touch the live engine — it can
 	// share a running component's identity, so registering from it would
-	// displace the real component's jobs (review finding).
-	if (getWorkerIndex() !== 0) {
+	// displace the real component's jobs.
+	if (!isApplicationPrimaryWorker(scope.applicationScope?.name)) {
 		schedulerLogger.debug?.('Scheduler config validated; activation skipped on non-primary worker');
 		return;
 	}