@harperfast/harper 5.3.0-beta.3 → 5.3.0-beta.4

package/components/ollama/index.ts

@@ -59,6 +59,15 @@ export interface OllamaBackendConfig {
  * support exists on some models but is uneven across the model catalog; we keep
  * the v1 portability guarantee honest and skip them here.
  */
+const CAPABILITIES: ModelCapabilities = Object.freeze({
+	embed: true,
+	generate: true,
+	stream: true,
+	tools: false,
+	adapters: false,
+	structuredOutput: true,
+});
+
 export class OllamaBackend implements ModelBackend {
 	readonly name = 'ollama';
 	readonly #origin: string;
@@ -74,7 +83,7 @@ export class OllamaBackend implements ModelBackend {
 	}
 
 	capabilities(): ModelCapabilities {
-		return { embed: true, generate: true, stream: true, tools: false, adapters: false };
+		return CAPABILITIES;
 	}
 
 	async embed(input: string | string[], opts: BackendOpts<EmbedOpts>): Promise<ModelCallResult<Float32Array[]>> {

package/components/openai/index.ts

@@ -19,6 +19,7 @@
 import { setEmbedding, setGenerative } from '../../resources/models/backendRegistry.ts';
 import {
 	assignFiniteTokenCount,
+	ChoiceScoringUnsupportedError,
 	composeSignal,
 	MAX_ERROR_BODY_BYTES,
 	normalizeOrigin,
@@ -31,6 +32,7 @@ import { ServerError } from '../../utility/errors/hdbError.ts';
 import harperLogger from '../../utility/logging/harper_logger.ts';
 import type {
 	BackendOpts,
+	ChoiceScores,
 	EmbedOpts,
 	GenerateChunk,
 	GenerateInput,
@@ -40,6 +42,7 @@ import type {
 	ModelBackend,
 	ModelCallResult,
 	ModelCapabilities,
+	ScoreChoicesOpts,
 	ToolCall,
 	ToolDef,
 	TokenUsage,
@@ -71,6 +74,14 @@ const MAX_TOOL_CALL_ACCUMULATOR_ENTRIES = 128;
 // cap (1 MiB) plus the 128-entry cap still allows ~128 MiB accumulated; this cap
 // keeps any single stream well-bounded. Real responses use tens of KB.
 const MAX_TOTAL_TOOL_CALL_ARGS_CHARS = 8 * 1024 * 1024; // 8 MiB
+// `top_logprobs` accepts at most 20, so one scoring request can cover at most 20 choices with a
+// single-letter label each; more must be declined (#2838).
+const MAX_SCORED_CHOICES = 20;
+const CHOICE_LABELS = 'ABCDEFGHIJKLMNOPQRST';
+// A scoring response is one content token with at most 20 alternatives, a few KiB. The general
+// 256 MiB cap is sized for embedding batches; an object schema scores its fields concurrently,
+// so a faulty endpoint must not be able to hold that much per call.
+const MAX_SCORE_RESPONSE_BODY_BYTES = 1 << 20; // 1 MiB
 
 const log = harperLogger.forComponent('openai').conditional;
 
@@ -97,12 +108,28 @@ export interface OpenAIBackendConfig {
  * - `generate` → `POST {baseUrl}/chat/completions` (always chat shape)
  * - `generateStream` → same with `stream: true`; consumes SSE wire format and
  *   yields `GenerateChunk` per delta.
+ * - `scoreChoices` → `POST {baseUrl}/chat/completions` with `logprobs` for one
+ *   token, the choices labelled `A`..`T` (#2838).
  *
  * Capabilities advertise `tools: true` — first backend with native tool-call
  * support. `adapters: false` — OpenAI doesn't expose LoRA adapter selection
  * externally. `toolMode: 'return'` (Phase 1 default) is supported end-to-end;
  * `toolMode: 'auto'` is reserved for #612.
  */
+const CAPABILITIES: ModelCapabilities = Object.freeze({
+	embed: true,
+	generate: true,
+	stream: true,
+	tools: true,
+	adapters: false,
+	scoreChoices: true,
+	structuredOutput: true,
+	maxScoredChoices: MAX_SCORED_CHOICES,
+});
+// The same shape for a model that refused `logprobs`; two frozen objects, so `capabilities()` allocates nothing.
+const { maxScoredChoices: _scoringLimit, ...UNLIMITED } = CAPABILITIES;
+const CAPABILITIES_WITHOUT_SCORING: ModelCapabilities = Object.freeze({ ...UNLIMITED, scoreChoices: false });
+
 export class OpenAIBackend implements ModelBackend {
 	readonly name = 'openai';
 	readonly #baseUrl: string;
@@ -115,6 +142,12 @@ export class OpenAIBackend implements ModelBackend {
 	// (o-series, gpt-5 family) reject `max_tokens` in favour of `max_completion_tokens`;
 	// OpenAI-compatible shims (vLLM, Ollama-compat, older gateways) only know `max_tokens`.
 	readonly #isNativeOpenAI: boolean;
+	// Models this instance has seen refuse `logprobs`: a 400 naming the parameter (OpenAI's
+	// reasoning models), or an answer with no log-probability field at all (a compatible endpoint
+	// that ignores the parameter). Scoring stays off for them until a reload builds a new
+	// instance, and `capabilities()` says so for the configured model, so a decision against it
+	// votes without an attempt. An empty completion is this call's problem, not the model's.
+	readonly #logprobsRejected = new Set<string>();
 
 	constructor(config: OpenAIBackendConfig = {}, fetchImpl: typeof fetch = fetch) {
 		this.#apiKey = requireCredential(config.apiKey, 'OpenAI', 'apiKey', OpenAIBackendError);
@@ -131,7 +164,9 @@ export class OpenAIBackend implements ModelBackend {
 	}
 
 	capabilities(): ModelCapabilities {
-		return { embed: true, generate: true, stream: true, tools: true, adapters: false };
+		return this.#defaultModel === undefined || !this.#logprobsRejected.has(this.#defaultModel)
+			? CAPABILITIES
+			: CAPABILITIES_WITHOUT_SCORING;
 	}
 
 	async embed(input: string | string[], opts: BackendOpts<EmbedOpts>): Promise<ModelCallResult<Float32Array[]>> {
@@ -242,7 +277,71 @@ export class OpenAIBackend implements ModelBackend {
 		}
 	}
 
-	async #post(path: string, body: object, callerSignal?: AbortSignal): Promise<Response> {
+	/**
+	 * Score `choices` from the first completion token's alternatives (#2838). The choices are
+	 * listed under single-letter labels and the model is asked for the letter alone, so every
+	 * choice is one token and the `top_logprobs` list can cover the whole set. What this model or
+	 * endpoint cannot do is declined with `ChoiceScoringUnsupportedError`, not failed: more than
+	 * 20 choices, a 400 that refuses `logprobs`, a response with no log-probabilities, no content
+	 * token or no alternatives, no label among them, or too much mass outside the list to know the
+	 * leading label. A malformed alternative is a failure like any other bad response.
+	 */
+	async scoreChoices(
+		input: GenerateInput,
+		choices: readonly string[],
+		opts: BackendOpts<ScoreChoicesOpts>
+	): Promise<ModelCallResult<ChoiceScores>> {
+		const model = opts.model ?? this.#defaultModel;
+		requireModel(model, 'scoreChoices', OpenAIBackendError);
+		if (choices.length < 1 || choices.length > MAX_SCORED_CHOICES)
+			throw new ChoiceScoringUnsupportedError(
+				`OpenAI scoreChoices covers 1 to ${MAX_SCORED_CHOICES} choices per call; got ${choices.length}`
+			);
+		if (this.#logprobsRejected.has(model))
+			throw new ChoiceScoringUnsupportedError(
+				`OpenAI model '${model}' rejects log-probabilities; scoring stays off for it until the backend is reloaded`
+			);
+		const body: Record<string, unknown> = {
+			model,
+			messages: [...normalizeMessages(input), { role: 'user', content: labeledChoices(choices) }],
+			stream: false,
+			logprobs: true,
+			top_logprobs: MAX_SCORED_CHOICES,
+		};
+		if (this.#isNativeOpenAI) body.max_completion_tokens = 1;
+		else body.max_tokens = 1;
+		const res = await this.#post('/chat/completions', body, opts.signal, (status, error) => {
+			if (!rejectsLogprobs(status, error)) return undefined;
+			this.#logprobsRejected.add(model);
+			return new ChoiceScoringUnsupportedError(`OpenAI model '${model}' rejects log-probabilities (HTTP ${status})`);
+		});
+		const data = await readBoundedJson<OpenAIChatResponse>(
+			res,
+			'OpenAI /chat/completions',
+			OpenAIBackendError,
+			MAX_SCORE_RESPONSE_BODY_BYTES
+		);
+		const choice = data.choices?.[0];
+		if (!choice) throw new OpenAIBackendError('OpenAI /chat/completions response missing choices[0]');
+		const usage: TokenUsage = {};
+		assignFiniteTokenCount(usage, 'promptTokens', data.usage?.prompt_tokens);
+		assignFiniteTokenCount(usage, 'completionTokens', data.usage?.completion_tokens);
+		// Only an answer that carries no log-probabilities shows the endpoint ignores the parameter;
+		// an empty or filtered completion says nothing about the model.
+		const answered = typeof choice.message?.content === 'string' && choice.message.content.length > 0;
+		const logLikelihoods = scoreLabels(choice.logprobs, choices.length, (reason, structural) => {
+			if (structural && answered) this.#logprobsRejected.add(model);
+			return new ChoiceScoringUnsupportedError(`OpenAI model '${model}' ${reason}`, usage);
+		});
+		return { status: 'completed', output: { logLikelihoods }, usage };
+	}
+
+	async #post(
+		path: string,
+		body: object,
+		callerSignal?: AbortSignal,
+		translate?: (status: number, error: OpenAIErrorEnvelope) => Error | undefined
+	): Promise<Response> {
 		const signal = composeSignal(callerSignal, this.#requestTimeoutMs);
 		const headers: Record<string, string> = {
 			'Content-Type': 'application/json',
@@ -262,35 +361,171 @@ export class OpenAIBackend implements ModelBackend {
 			// doesn't leak request content. Cap length defensively against a
 			// misbehaving compat shim. Falls back to status-only if the body
 			// isn't JSON or doesn't have the envelope.
-			throw new OpenAIBackendError(
-				`OpenAI ${path} returned HTTP ${res.status}${await readErrorSuffix(res)}`,
-				res.status
-			);
+			const error = await readErrorEnvelope(res);
+			const translated = translate?.(res.status, error);
+			if (translated) throw translated;
+			throw new OpenAIBackendError(`OpenAI ${path} returned HTTP ${res.status}${errorSuffix(error)}`, res.status);
 		}
 		return res;
 	}
 }
 
-async function readErrorSuffix(res: Response): Promise<string> {
+interface OpenAIErrorEnvelope {
+	message?: string;
+	param?: string;
+	code?: string;
+}
+
+async function readErrorEnvelope(res: Response): Promise<OpenAIErrorEnvelope> {
 	try {
-		const body = await readBoundedJson<{ error?: { message?: unknown; type?: unknown } }>(
+		const body = await readBoundedJson<{ error?: { message?: unknown; param?: unknown; code?: unknown } }>(
 			res,
 			'OpenAI error response',
 			OpenAIBackendError,
 			MAX_ERROR_BODY_BYTES
 		);
-		const message = body?.error?.message;
-		if (typeof message === 'string' && message.length > 0) {
-			const truncated =
-				message.length > MAX_UPSTREAM_ERROR_MESSAGE_CHARS
-					? message.slice(0, MAX_UPSTREAM_ERROR_MESSAGE_CHARS) + '…'
-					: message;
-			return `: ${truncated}`;
-		}
-		return '';
+		const error: OpenAIErrorEnvelope = {};
+		if (typeof body?.error?.message === 'string') error.message = body.error.message;
+		if (typeof body?.error?.param === 'string') error.param = body.error.param;
+		if (typeof body?.error?.code === 'string') error.code = body.error.code;
+		return error;
 	} catch {
-		return '';
+		return {};
+	}
+}
+
+function errorSuffix(error: OpenAIErrorEnvelope): string {
+	const message = error.message;
+	if (typeof message !== 'string' || message.length === 0) return '';
+	const truncated =
+		message.length > MAX_UPSTREAM_ERROR_MESSAGE_CHARS
+			? message.slice(0, MAX_UPSTREAM_ERROR_MESSAGE_CHARS) + '…'
+			: message;
+	return `: ${truncated}`;
+}
+
+/**
+ * Whether a 400 says this model or endpoint does not take log-probabilities at all, as opposed to
+ * rejecting this request for another reason. OpenAI names the parameter (`param: 'logprobs'`);
+ * a compatible shim tends to say so only in the message, which must then both name the parameter
+ * and say it is unsupported, so a request-specific complaint that merely mentions it does not
+ * switch scoring off.
+ */
+function rejectsLogprobs(status: number, error: OpenAIErrorEnvelope): boolean {
+	if (status !== 400) return false;
+	if (error.param === 'logprobs' || error.param === 'top_logprobs') return true;
+	const message = error.message ?? '';
+	return (
+		/\b(?:top_)?logprobs\b/i.test(message) &&
+		/\b(not supported|unsupported|not available|unavailable|disabled|not allowed|does not support|doesn't support)\b/i.test(
+			message
+		)
+	);
+}
+
+/**
+ * One line per choice under its label. A value that cannot sit on one line as itself (a line
+ * break or other control character, leading or trailing space, a quote or backslash, or nothing
+ * at all) is JSON-quoted, so no value can pose as a further option and values that differ only in
+ * whitespace stay distinct; a plain value never starts with a quote, so the two forms cannot
+ * collide.
+ */
+function labeledChoices(choices: readonly string[]): string {
+	const lines = ['Choose exactly one option and reply with its letter only.'];
+	choices.forEach((choice, i) => lines.push(`${CHOICE_LABELS[i]}. ${plainOrQuoted(choice)}`));
+	return lines.join('\n');
+}
+
+// Anything that cannot sit on one line as itself. Built from code points so the source carries
+// no raw line terminator, which would end a regex literal.
+const LINE_TERMINATORS = String.fromCharCode(0x2028, 0x2029);
+const NEEDS_QUOTING = new RegExp(`^$|^\\s|\\s$|["\\\\\\x00-\\x1f\\x7f${LINE_TERMINATORS}]`);
+const LINE_TERMINATOR = new RegExp(`[${LINE_TERMINATORS}]`, 'g');
+
+function plainOrQuoted(choice: string): string {
+	if (!NEEDS_QUOTING.test(choice)) return choice;
+	// JSON.stringify leaves the two Unicode line terminators unescaped; they must not break the line.
+	return JSON.stringify(choice).replace(LINE_TERMINATOR, (c) => `\\u${c.charCodeAt(0).toString(16)}`);
+}
+
+// How far below the smallest listed alternative an unlisted label sits when the list leaves no mass
+// over: finite, so the vector normalizes.
+const UNLISTED_MARGIN = Math.log(1e6);
+
+/**
+ * One log-likelihood per label from the first content token's alternatives. Spellings of the same
+ * letter (`A`, ` A`, `a.`) combine by log-sum-exp, so a label's score is the mass of its listed
+ * spellings; spellings outside the list are unknown, bounded together by the mass the list leaves
+ * over. A label with no listed spelling is placed only when that leftover is smaller than the
+ * leading observed label's mass, so a wholly unlisted label can never be the chosen value;
+ * otherwise the call is declined rather than guessed. Unlisted labels then share the leftover
+ * equally, so together they never exceed it and each stays below the leader; when the list leaves
+ * nothing over they sit a fixed margin below its smallest entry. Every placed score is still an estimate: an observed label's own unlisted
+ * spellings can add up to the leftover to it, so two observed labels closer than that may be
+ * ordered wrongly. No label at all means the model was not answering with a letter, and the call
+ * is declined. A malformed entry is a bad response, not a decline. Messages never quote tokens,
+ * which are upstream text.
+ *
+ * `unsupported(reason, structural)`: `structural` marks a response with no log-probability field
+ * at all, which the caller may remember for the model; a missing or empty alternatives list is
+ * this call's.
+ */
+function scoreLabels(
+	logprobs: OpenAIChatLogprobs | null | undefined,
+	count: number,
+	unsupported: (reason: string, structural: boolean) => Error
+): number[] {
+	if (logprobs == null || logprobs.content === undefined) throw unsupported('returned no log-probabilities', true);
+	const first = Array.isArray(logprobs.content) ? logprobs.content[0] : undefined;
+	if (!first) throw unsupported('returned no scored content token', false);
+	if (!Array.isArray(first.top_logprobs) || first.top_logprobs.length === 0)
+		throw unsupported('returned no alternatives for the scored token', false);
+	const seen = new Map<string, number>();
+	for (const alternative of [first, ...first.top_logprobs]) {
+		const token = alternative?.token;
+		const logprob = alternative?.logprob;
+		if (typeof token !== 'string' || typeof logprob !== 'number' || !Number.isFinite(logprob))
+			throw new OpenAIBackendError('OpenAI /chat/completions returned a malformed log-probability entry');
+		if (!seen.has(token)) seen.set(token, logprob);
 	}
+	const scores: Array<number | undefined> = new Array(count).fill(undefined);
+	let smallestListed = Infinity;
+	let listedMass = 0;
+	for (const [token, logprob] of seen) {
+		smallestListed = Math.min(smallestListed, logprob);
+		listedMass += Math.exp(logprob);
+		const index = labelIndex(token);
+		if (index === undefined || index >= count) continue;
+		const current = scores[index];
+		scores[index] = current === undefined ? logprob : logAddExp(current, logprob);
+	}
+	const observed = scores.filter((score): score is number => score !== undefined);
+	if (observed.length === 0) throw unsupported('did not answer with one of the choice labels', false);
+	if (observed.length === count) return observed;
+	const leftover = Math.max(0, 1 - listedMass);
+	if (leftover >= Math.exp(Math.max(...observed)))
+		throw unsupported('left too much probability outside its listed alternatives to know the leading choice', false);
+	// The leftover is every token outside the list, so it bounds the unlisted labels together; each
+	// takes an equal share of it, because one token's worth understates a label whose spellings are
+	// all unlisted. A share of nothing sits a fixed margin below the list.
+	const share = leftover / (count - observed.length);
+	const floor = share > 0 ? Math.log(share) : smallestListed - UNLISTED_MARGIN;
+	return scores.map((score) => score ?? floor);
+}
+
+function labelIndex(token: string): number | undefined {
+	const letter = token
+		.trim()
+		.replace(/^[("'[]+|[.)\]:"',]+$/g, '')
+		.toUpperCase();
+	if (letter.length !== 1) return undefined;
+	const index = CHOICE_LABELS.indexOf(letter);
+	return index === -1 ? undefined : index;
+}
+
+function logAddExp(a: number, b: number): number {
+	const max = Math.max(a, b);
+	return max + Math.log(Math.exp(a - max) + Math.exp(b - max));
 }
 
 /**
@@ -588,10 +823,22 @@ interface OpenAIChatResponse {
 	choices?: Array<{
 		message?: { role: string; content?: string | null; tool_calls?: OpenAIToolCall[] };
 		finish_reason?: string | null;
+		logprobs?: OpenAIChatLogprobs | null;
 	}>;
 	usage?: { prompt_tokens?: number; completion_tokens?: number; total_tokens?: number };
 }
 
+interface OpenAIChatLogprobs {
+	content?: OpenAITokenLogprob[] | null;
+}
+
+/** One scored token position; fields are `unknown` because a compatible endpoint's shape is not trusted. */
+interface OpenAITokenLogprob {
+	token?: unknown;
+	logprob?: unknown;
+	top_logprobs?: Array<{ token?: unknown; logprob?: unknown }> | null;
+}
+
 interface OpenAIToolCall {
 	id?: string;
 	type?: 'function';

package/components/operations.js

@@ -32,6 +32,7 @@ const { Resources } = require('../resources/Resources.ts');
 const {
 	Application,
 	prepareApplication,
+	componentPreparationBudgetMs,
 	ASIDE_STAGING_DIR,
 	DEPLOY_STAGING_DIR,
 	dropComponentDirectory,
@@ -451,6 +452,8 @@ async function packageComponent(req) {
  *
  * A no-op on the main thread, and the operations API deploys there — so operator deploys are unvalidated
  * (#2315 step 2). What this guarantees is ORDER: where validation runs, a rejected candidate never goes live.
+ *
+ * Also a no-op on a worker that `freeze-after-load` froze after its boot load; see components/DESIGN.md.
  */
 // `componentLoader.setErrorReporter` is ONE process-global callback, so two components validating
 // concurrently on the same worker cross-attribute their failures: B installs its reporter while A is
@@ -460,6 +463,12 @@ async function packageComponent(req) {
 let validationChain = Promise.resolve();
 
 async function validateComponentLoads(candidateDirPath, emit) {
+	if (!isMainThread && require('../security/jsLoader.ts').laterLoadsMeetFrozenIntrinsics()) {
+		log.trace(
+			`Not load-validating ${path.basename(candidateDirPath)}: this worker's intrinsics are frozen after its boot load, so the load would not match the one a restarted worker performs`
+		);
+		return;
+	}
 	const run = validationChain.then(
 		() => validateComponentLoadsExclusive(candidateDirPath, emit),
 		() => validateComponentLoadsExclusive(candidateDirPath, emit)
@@ -573,6 +582,32 @@ function logRestartOutcome(restart, what) {
 			`The restart after ${what} was still running after ${RESTART_WAIT_CEILING_MS}ms; worker threads may still be running the previous code`
 		);
 }
+
+// A peer's validation load and swap, which have no allowance of their own to sum.
+const PEER_DEPLOY_VALIDATION_MARGIN_MS = 10 * 60 * 1000;
+
+/**
+ * How long the origin waits for each peer to answer a replicated deploy: every wait and command the peer is
+ * allowed for this request, each at its full allowance. That includes the preparation lock's wait: a peer already
+ * preparing the same component for another deploy holds this one there for a preparation budget before the lock
+ * re-checks the holder. The lock keeps waiting while that holder is alive, so a peer queued behind a longer
+ * preparation, or behind several, can take longer; so can one validating plugins whose configured timeouts outlast
+ * the margin. Such a peer is reported as not answering, which says nothing of its outcome.
+ */
+function peerDeployAnswerTimeoutMs(req) {
+	const { RESTART_WAIT_CEILING_MS } = require('./awaitRestart.ts');
+	const payloadWaitMs = coerceTimeoutMs(req.deployment_timeout, DEFAULT_AWAIT_ROW_TIMEOUT_MS);
+	const installTimeoutMs = coerceTimeoutMs(req.install_timeout, undefined);
+	return Math.min(
+		payloadWaitMs * (req.credentials?.length ? 2 : 1) +
+			// the lock's wait on another deploy's preparation, then this deploy's own
+			2 * componentPreparationBudgetMs(installTimeoutMs) +
+			PEER_DEPLOY_VALIDATION_MARGIN_MS +
+			(req.restart === true ? RESTART_WAIT_CEILING_MS : 0),
+		hdbTerms.MAX_SET_TIMEOUT_MS
+	);
+}
+
 /**
  * Can deploy a component in multiple ways. If a 'package' is provided all it will do is write that package to
  * harperdb-config, when HDB is restarted the package will be installed in hdb/nodeModules. If a base64 encoded string is passed it
@@ -890,7 +925,10 @@ async function deployComponent(req) {
 		// finish()'s single write; live SSE 'peer' events still fire below.
 		recorder?.seal();
 		emit('phase', { phase: 'replicate', status: 'start' });
-		let response = await server.replication.replicateOperation(req, { onPeerResult });
+		let response = await server.replication.replicateOperation(req, {
+			onPeerResult,
+			timeoutMs: peerDeployAnswerTimeoutMs(req),
+		});
 		emit('phase', { phase: 'replicate', status: 'done' });
 		if (recorder && response?.replicated) {
 			// Fallback path for replicators that don't honor onPeerResult: re-record the
@@ -1589,6 +1627,7 @@ exports.dropCustomFunctionProject = dropCustomFunctionProject;
 exports.packageComponent = packageComponent;
 exports.deployComponent = deployComponent;
 exports.unconfirmedStagingPeers = unconfirmedStagingPeers;
+exports.peerDeployAnswerTimeoutMs = peerDeployAnswerTimeoutMs;
 exports.getComponents = getComponents;
 exports.getComponentFile = getComponentFile;
 exports.setComponentFile = setComponentFile;

package/config/configUtils.ts

@@ -32,6 +32,9 @@ import * as env from '../utility/environment/environmentManager.ts';
 import { prepareRuntimeEnvConfig, hasPersistedEnvConfigState, discardConfigState } from './harperConfigEnvVars.ts';
 import { warnComponentEnvConfigVars, resolveConfiguredPath } from './componentEnvPrepass.ts';
 import { isStartableThreadHeapMemory } from '../server/threads/threadHeapMemory.ts';
+import { fsyncTolerantSync, isUnsupportedSyncError } from '../utility/fsync.ts';
+
+export { isUnsupportedSyncError } from '../utility/fsync.ts';
 
 const { DATABASES_PARAM_CONFIG, CONFIG_PARAMS, CONFIG_PARAM_MAP } = hdbTerms;
 const UNINIT_GET_CONFIG_ERR = 'Unable to get config value because config is uninitialized';
@@ -133,25 +136,6 @@ type AtomicWriteOptions = RenameRetryOptions & {
 	durable?: boolean;
 };
 
-// Codes that mean "this platform or filesystem will not fsync this handle", as opposed to "the write did
-// not reach storage". Windows raises EPERM fsyncing perfectly healthy files and cannot open a directory
-// for fsync at all; network and overlay mounts return EINVAL or ENOTSUP. None of those say anything about
-// durability, and treating them as failures would fail every durable write on those platforms. EIO and
-// ENOSPC are not in the set on purpose.
-const UNSUPPORTED_SYNC_CODES = new Set(['EPERM', 'EINVAL', 'ENOTSUP', 'EOPNOTSUPP', 'EBADF', 'EISDIR']);
-
-export function isUnsupportedSyncError(error: unknown): boolean {
-	return UNSUPPORTED_SYNC_CODES.has((error as NodeJS.ErrnoException)?.code ?? '');
-}
-
-function fsyncTolerantSync(fd: number) {
-	try {
-		fs.fsyncSync(fd);
-	} catch (error) {
-		if (!isUnsupportedSyncError(error)) throw error;
-	}
-}
-
 // `flags` matters on Windows, which only flushes a handle opened for writing; a directory cannot be opened
 // for writing anywhere, and Windows cannot open one for fsync at all.
 function syncPathSync(targetPath: string, flags: 'r' | 'r+') {

package/dataLayer/DESIGN.md

@@ -40,6 +40,20 @@ System tables replicate by default. To opt out, add the name to `NON_REPLICATING
 
 If the table needs `audit: true`, set it both in the schema (for fresh installs) **and** on the `CreateTableObject` instance in the directive (for upgrades) — otherwise the two paths diverge.
 
+**A replicated system table whose owner declares more than a name list** — a table-level `expiration`, an index — needs that declaration on every node, every boot. `hdb_oidc_token_use` is the case: `security/authn/oidc/tokenUseTable.ts` holds its one definition, and `bin/run.ts initialize()` applies it after the upgrade step and before worker threads start (skipped in read-only mode; a failure is logged and retried on the next start), as do the 5.3.0 directive and the exchange path. Each replay row carries its own expiry as record metadata, which replication applies on the receiving node; the table's stored `expiration` is what arms the cleanup scan that removes those rows once a node loads the table. Metadata rather than an `@expiresAt` attribute, because the attribute needs an index and a sweep of its own on every node, while reads and the cleanup scan already honor the metadata. `systemSchema.json` keeps only its primary key; the declaration completes the table on the first boot after install. Every other route to a copy leaves it partial, which is why the declaration cannot live only on the exchange path or in the directive:
+
+- A node that has not upgraded yet gets any system table it lacks from a peer's replication handshake (`DB_SCHEMA`), created from a snapshot that carries each attribute's name, type and primary-key bit — never `indexed`, `expiresAt` or the table's `expiration` — and schema-defined like the peer's copy. From then on the handshake keeps that local definition and logs `Schema for '<db>.<table>' is defined locally, but attribute '<name>: <type>' from '<node>' does not match local attribute which does not exist` for every attribute a peer has and it lacks, so no peer repairs it; the directive that runs when that node upgrades finds the table already there.
+- A node that never exercises the owner's code path (a passive cluster member for the OIDC exchange) never declares the rest, and the rows replicated to it are never removed.
+- An install already on a pre-release of the directive's version never runs the directive again (the `compareVersions` caveat above): from `5.3.0-beta.2`, `getVersionsForUpgrade` selects nothing for `5.3.0`, `5.3.0-beta.3` or `5.3.1`.
+
+A local `table()` declaration is authoritative, so repeating it repairs any of those shapes in place, keeping every row and its expiry, and is a no-op once the shape matches (measured 2026-09-25: under 1 ms; repairing a copy holding 100,000 replay rows, under 10 ms on either engine, since no row is rewritten and no index is built). Limits that come from the storage layer rather than this table:
+
+- A thread that only loads the table scans daily (the load path keeps the default interval; only a declaring thread uses a quarter of the `expiration`), so a spent row can stay on disk up to a day past its expiry. Reads skip it meanwhile.
+- Under `threads: 0` the main thread loads and declares the table before `startHTTPThreads` makes it worker 0. `scheduleCleanup` records an interval only once its thread has a worker index, so the first expiring write or declaration after that arms the scan; a single-threaded node that writes nothing after a restart does not scan until it does.
+- An attribute a declaration drops keeps its index store on disk: `table()` looks that store up by table name rather than attribute name, so it never drops it. A node that served exchanges on 5.3.0-beta.2 keeps its old `expiresAt` index; after a restart nothing opens it.
+
+The certificate verification tables (`hdb_certificate_cache`, `hdb_crl_cache`, `hdb_revoked_certificates`) follow the same pattern through `security/certificateVerification/verificationTables.ts`, applied in `initialize()` right after the OIDC table and by the verification path's getters. `ensureCertificateVerificationTables` also waits for the revoked table's index backfill and reads its outcome back from the catalog, since a failed backfill still settles. What their rows hold is in security/DESIGN.md.
+
 ## RocksDB backup/restore: the restore lock + marker protocol (`dataLayer/restoreMarker.ts`, `dataLayer/rocksdbBackup.ts`)
 
 The `restore_backup` operation restores a user database on a live server by closing it across all
@@ -109,11 +123,21 @@ Three non-obvious mechanics keep that safe:
   exited job worker leaves no residual handle to be mistaken for a live holder.
 - **`dropDatabase` and `restore_backup` serialize on the same lock, not a check-then-act probe.**
   A drop's `destroy()` interleaving with a restore's purge-and-copy on the same directory would gut
-  a "successful" restore (or vice versa). `dropDatabase` therefore _acquires_ the restore lock
-  (`acquireRestoreLock`, marker-less) for each RocksDB root store and holds it across the whole drop,
-  releasing in a `finally`; a restore in progress makes the acquire fail with 409, and a leftover
-  incomplete-restore marker (lock free, detected via `restoreMarkerPresent`, which — unlike
-  `checkRestoreState` — is safe while this thread holds the lock) is refused rather than dropped over.
+  a "successful" restore (or vice versa). `dropDatabase` takes the restore lock for every RocksDB or
+  LMDB root and publishes a positional `.dropping` marker beside each root before deleting any of
+  them. A restore in progress makes the acquire fail with 409; `beginRestore` likewise refuses a
+  surviving drop marker. Each marker records the root store's blob identity and exact pinned blob
+  paths, so a retry removes what an already-marked handle owned even if `storage.blobPaths` changes.
+  An integrity digest makes damaged identity content fail closed instead of redirecting deletion.
+  Markers are removed only after every root and blob path has been removed.
+  Startup scans and cold opens reject the marked root and the rest of its logical database graph, so
+  a crash while publishing or canceling several markers cannot expose a partial database. Retrying
+  `drop_database` re-enumerates that graph and completes marker publication and deletion even when
+  the in-memory catalog is gone. A configured `databases.<name>.path` is the exclusive root of that
+  logical graph, not a container for independent databases: every physical database directly beneath
+  it is an alias-owned member, and dropping any alias drops the graph. A strict worker-close failure
+  similarly keeps that worker's physical roots fenced and remembered; a same-name drop retry
+  re-attempts those closes before marker publication or deletion.
   `database()`'s on-demand open still uses the read-only `throwIfBlockedByRestore` (a
   `create_table`/`create_schema` must not resurrect a half-purged directory as a fresh empty DB), but
   the destructive drop path now uses the exclusive lock so the race is closed, not merely narrowed.

package/dataLayer/harperBridge/ResourceBridge.ts

@@ -1,6 +1,19 @@
 import searchValidator from '../../validation/searchValidator.ts';
+import { randomUUID } from 'node:crypto';
+import { threadId } from 'node:worker_threads';
 import { handleHDBError, ClientError, hdbErrors } from '../../utility/errors/hdbError.ts';
-import { table, getDatabases, database, dropDatabase, type Table } from '../../resources/databases.ts';
+import {
+	table,
+	getDatabases,
+	database,
+	databaseDropPreparationTargets,
+	dropDatabase,
+	type Table,
+} from '../../resources/databases.ts';
+import {
+	claimDatabaseDropPreparations,
+	releaseDatabaseDropPreparations,
+} from '../../resources/databaseDropPreparation.ts';
 import insertUpdateValidate from './bridgeUtility/insertUpdateValidate.js';
 import SearchObject from '../SearchObject.ts';
 import {
@@ -186,8 +199,31 @@ export class ResourceBridge extends BridgeMethods {
 	}
 
 	async dropSchema(dropSchemaObj) {
-		await dropDatabase(dropSchemaObj.schema);
-		signalling.signalSchemaChange(new SchemaEventMsg(process.pid, OPERATIONS_ENUM.DROP_SCHEMA, dropSchemaObj.schema));
+		const preparationId = randomUUID();
+		const { rootPaths } = databaseDropPreparationTargets(dropSchemaObj.schema);
+		const completion = () => {
+			const message: any = new SchemaEventMsg(process.pid, OPERATIONS_ENUM.DROP_SCHEMA, dropSchemaObj.schema);
+			message.dropPreparationId = preparationId;
+			message.dropPreparationOwnerThreadId = threadId;
+			message.dropPreparationRootPaths = rootPaths;
+			return message;
+		};
+		const preparation: any = completion();
+		preparation.prepareDrop = true;
+		claimDatabaseDropPreparations(rootPaths, preparationId, threadId, dropSchemaObj.schema);
+		try {
+			await signalling.signalSchemaChangeToPeers(preparation);
+			await dropDatabase(dropSchemaObj.schema, rootPaths);
+		} finally {
+			// The first round reaches current peers before local release; the second catches a worker whose
+			// startup snapshot inherited the fence. Cleanup-proven job workers are already exiting.
+			await signalling.signalSchemaChange(completion(), {
+				peersFirst: true,
+				includeJobWorkers: 'active',
+				peerRounds: 2,
+			});
+			releaseDatabaseDropPreparations(rootPaths, preparationId);
+		}
 	}
 
 	async updateRecords(updateObj) {