@harperfast/harper 5.3.0 → 5.3.1

package/resources/indexes/fullTextQueryIndex.ts

@@ -36,10 +36,24 @@ import {
 	type DerivedNativeIndexHost,
 } from './hnswDerivedIndex.ts';
 import { loggerWithTag } from '../../utility/logging/logger.ts';
+import type { CandidateKeyPlan } from '../search.ts';
+import type { Id } from '../ResourceInterface.ts';
 
 const RAW_PAGE_SIZE = 256;
+const MAX_FILTERED_RAW_PAGE_SIZE = 4_096;
 const MIN_RAW_PAGE_SIZE = 32;
 const RAW_PAGE_OVERFETCH_FACTOR = 2;
+const RAW_PAGE_ZERO_YIELD_GROWTH_FACTOR = 4;
+const RAW_PAGE_YIELD_INTERVAL = 256;
+const CANDIDATE_KEYS_PER_SOURCE_READ = 8;
+const MAX_CANDIDATE_KEYS = 4_096;
+// Four key-only reads remain cheaper than loading and decoding the authoritative record they replace.
+const MAX_POINT_PROBE_READS_PER_CANDIDATE = 4;
+const REJECTED_GATE_SAMPLE_SIZE = 8;
+const CANDIDATE_GATE_FAILURE_RETRY_MILLISECONDS = 1_000;
+// Fulltext 0.5.0 Query API v3 uses a 13-byte response header and 13 bytes per versioned hit before string data.
+const NATIVE_SEARCH_RESPONSE_HEADER_BYTES = 13;
+const NATIVE_SEARCH_HIT_OVERHEAD_BYTES = 13;
 const MAX_RELOAD_FAILURES_BEFORE_REOPEN = 3;
 const HIGHLIGHT_BLOB_READ_TIMEOUT_MILLISECONDS = 5_000;
 const READER_DRAIN_GRACE_MILLISECONDS = 1_000;
@@ -72,6 +86,12 @@ type ReaderSlot = {
 	reloadFailures: number;
 };
 
+type CandidateGate = {
+	complete: boolean;
+	maxReadsPerCandidate: number;
+	has(primaryKey: Id): boolean;
+};
+
 type FullTextQueryHost = Pick<DerivedNativeIndexHost, 'readiness' | 'waitForCoverage' | 'requestRebuild'>;
 
 class FullTextReaderPublicationError extends Error {}
@@ -97,6 +117,8 @@ export type FullTextQueryIndexOptions = {
 
 export class FullTextQueryIndex {
 	readonly filteredSearch = true;
+	readonly candidateKeyFilter = true;
+	readonly candidateKeyProbe = true;
 	readonly filePrimary = true;
 	readonly #options: FullTextQueryIndexOptions;
 	readonly #nativeOptions: NativeFullTextIndexConfiguration & {
@@ -112,6 +134,7 @@ export class FullTextQueryIndex {
 	#readerOperation?: Promise<ReaderSlot>;
 	#retiredReaderSlots = new Set<ReaderSlot>();
 	#maxSearchWindow?: number;
+	#maxFilteredRawPageSize?: number;
 	#maxAutocompleteResults?: number;
 	#maxSearchBudgetMilliseconds?: number;
 	#maxTraceRecords?: number;
@@ -125,6 +148,13 @@ export class FullTextQueryIndex {
 	#publicationRebuildRequested = false;
 	#refreshFailureWarned = false;
 	#unexpectedSearchFailureWarned = false;
+	#incompletePageFailureLogged = false;
+	#candidateCollectionFailureWarned = false;
+	#candidateProbeFailureWarned = false;
+	#candidateCollectionRetryAfter = 0;
+	#candidateProbeRetryAfter = 0;
+	#candidateCollectionFailureEpoch = 0;
+	#candidateProbeFailureEpoch = 0;
 
 	constructor(options: FullTextQueryIndexOptions) {
 		this.#options = options;
@@ -184,6 +214,7 @@ export class FullTextQueryIndex {
 			minResults?: number;
 			resultOffset?: number;
 			assertTransactionActive?: () => void;
+			candidateKeys?: CandidateKeyPlan;
 		} = {}
 	): Promise<Array<{ key: unknown; $score: number; $highlights?: Record<string, unknown>; loadedEntry: any }>> {
 		const maxIndexLagMilliseconds = condition.maxIndexLagMilliseconds ?? DEFAULT_MAX_INDEX_LAG_MILLISECONDS;
@@ -225,11 +256,13 @@ export class FullTextQueryIndex {
 				context?.signal?.throwIfAborted();
 				const result = await this.#search(condition, context, options);
 				this.#unexpectedSearchFailureWarned = false;
+				this.#incompletePageFailureLogged = false;
 				return result;
 			}
 			coverage = this.#queryCoverage(maxIndexLagMilliseconds);
 			const result = await this.#search(condition, context, options);
 			this.#unexpectedSearchFailureWarned = false;
+			this.#incompletePageFailureLogged = false;
 			return result;
 		};
 		const operation = execute().catch((error) => {
@@ -333,6 +366,7 @@ export class FullTextQueryIndex {
 			minResults?: number;
 			resultOffset?: number;
 			assertTransactionActive?: () => void;
+			candidateKeys?: CandidateKeyPlan;
 		}
 	): Promise<Array<{ key: unknown; $score: number; $highlights?: Record<string, unknown>; loadedEntry: any }>> {
 		const readiness = readDerivedIndexReadiness(this.#options.auditStore, this.#options.readinessId);
@@ -395,13 +429,25 @@ export class FullTextQueryIndex {
 			let moreMayExist = false;
 			let staleVersionHits = 0;
 			let expiredHits = 0;
+			let rejectedGateSampleCount = 0;
+			let rejectedGateKeys: Id[] | undefined;
+			let rejectedGateVersions: string[] | undefined;
+			let rawPageSize = bounded
+				? Math.min(
+						this.#maxFilteredRawPageSize!,
+						RAW_PAGE_SIZE,
+						Math.max(MIN_RAW_PAGE_SIZE, target * RAW_PAGE_OVERFETCH_FACTOR)
+					)
+				: Math.min(this.#maxFilteredRawPageSize!, RAW_PAGE_SIZE);
 			const transaction = context && this.#options.Table._readTxnForContext(context);
+			const candidatePlan = options.filter ? options.candidateKeys : undefined;
+			let candidateGate: CandidateGate | undefined;
+			let candidateGatePlanned = false;
+			let candidateProbeFailureEpoch = 0;
 			while (accepted.length < target && offset < searchWindow) {
 				if (context?.signal?.aborted) throw context.signal.reason ?? new Error('Full-text search aborted');
-				const desiredPageSize = bounded
-					? Math.max(MIN_RAW_PAGE_SIZE, (target - accepted.length) * RAW_PAGE_OVERFETCH_FACTOR)
-					: RAW_PAGE_SIZE;
-				const limit = autocomplete ? searchWindow : Math.min(RAW_PAGE_SIZE, desiredPageSize, searchWindow - offset);
+				const limit = autocomplete ? searchWindow : Math.min(rawPageSize, searchWindow - offset);
+				const acceptedBeforePage = accepted.length;
 				const result = await reader.search(
 					{
 						query,
@@ -413,15 +459,111 @@ export class FullTextQueryIndex {
 				);
 				if (!bounded && !autocomplete && result.totalRelation === 'exact' && result.total > searchWindow)
 					throw new ClientError(`Full-text query exceeds the ${searchWindow}-result search window; add a limit`, 400);
+				// Fulltext 0.5.0 Query API v3 only reports lower-bound while a full requested page remains retrievable.
+				if (
+					result.totalRelation === 'lower-bound' &&
+					result.hits.length < limit &&
+					offset + result.hits.length < result.total
+				) {
+					if (!this.#incompletePageFailureLogged) {
+						this.#incompletePageFailureLogged = true;
+						logger.error?.(
+							`Full-text index '${this.#definition.name}' returned ${result.hits.length} of ${limit} hits at offset ${offset} with lower-bound total ${result.total}`
+						);
+					}
+					throw new ServerError('Full-text index returned an incomplete result page', 500);
+				}
+				// An exact total can safely continue from the returned offset; only a short lower-bound page breaks the v3 contract.
 				moreMayExist =
 					result.totalRelation === 'exact' ? offset + result.hits.length < result.total : result.hits.length === limit;
 				if (result.hits.length === 0) break;
-				for (const hit of result.hits) {
-					const key = decodeNativeId(hit.id, this.#options.Table.tableId);
-					options.assertTransactionActive?.();
-					const entry = this.#options.Table.primaryStore.getEntry(key, { transaction });
+				if (!candidateGatePlanned) {
+					candidateGatePlanned = true;
+					if (candidatePlan) {
+						const assertSearchActive = () => {
+							if (context?.signal?.aborted) throw context.signal.reason ?? new Error('Full-text search aborted');
+							options.assertTransactionActive?.();
+							remainingSearchBudget(deadline);
+						};
+						assertSearchActive();
+						const nativeHitCount = Math.min(searchWindow, Math.max(result.hits.length, result.total));
+						candidateGate = await this.#candidateGate(
+							candidatePlan,
+							target,
+							nativeHitCount,
+							async () => {
+								await new Promise((resolve) => setImmediate(resolve));
+								assertSearchActive();
+							},
+							assertSearchActive
+						);
+						candidateProbeFailureEpoch = this.#candidateProbeFailureEpoch;
+					}
+				}
+				const pointReadsPerCandidate = candidateGate?.maxReadsPerCandidate ?? 0;
+				const workYieldInterval = Math.min(
+					limit,
+					pointReadsPerCandidate === 0
+						? RAW_PAGE_YIELD_INTERVAL
+						: Math.max(1, Math.floor(RAW_PAGE_YIELD_INTERVAL / (pointReadsPerCandidate + 1)))
+				);
+				let nextYieldIndex = workYieldInterval;
+				for (let hitIndex = 0; hitIndex < result.hits.length; hitIndex++) {
+					if (hitIndex === nextYieldIndex) {
+						await new Promise((resolve) => setImmediate(resolve));
+						if (context?.signal?.aborted) throw context.signal.reason ?? new Error('Full-text search aborted');
+						remainingSearchBudget(deadline);
+						nextYieldIndex += workYieldInterval;
+					}
+					const hit = result.hits[hitIndex];
+					const key = decodeNativeId(hit.id, this.#options.Table.tableId) as Id;
 					if (typeof hit.version !== 'string')
 						throw new ServerError('Full-text index returned a hit without a source version', 500);
+					options.assertTransactionActive?.();
+					let candidateDecides = false;
+					if (candidateGate) {
+						try {
+							const admitted = candidateGate.has(key);
+							if (
+								candidateGate.maxReadsPerCandidate > 0 &&
+								candidateProbeFailureEpoch === this.#candidateProbeFailureEpoch
+							) {
+								this.#candidateProbeFailureWarned = false;
+								this.#candidateProbeRetryAfter = 0;
+							}
+							if (!admitted) {
+								if (rejectedGateSampleCount < REJECTED_GATE_SAMPLE_SIZE) {
+									(rejectedGateKeys ??= [])[rejectedGateSampleCount] = key;
+									(rejectedGateVersions ??= [])[rejectedGateSampleCount] = hit.version;
+									rejectedGateSampleCount++;
+								}
+								continue;
+							}
+							candidateDecides = candidateGate.complete;
+						} catch (error) {
+							const usedPointProbe = candidateGate.maxReadsPerCandidate > 0;
+							candidateGate = undefined;
+							if (usedPointProbe) {
+								this.#candidateProbeFailureEpoch++;
+								this.#candidateProbeRetryAfter = Date.now() + CANDIDATE_GATE_FAILURE_RETRY_MILLISECONDS;
+								if (!this.#candidateProbeFailureWarned) {
+									this.#candidateProbeFailureWarned = true;
+									logger.warn?.('could not probe the full-text companion index; using the record predicate', error);
+								}
+							} else {
+								this.#candidateCollectionFailureEpoch++;
+								this.#candidateCollectionRetryAfter = Date.now() + CANDIDATE_GATE_FAILURE_RETRY_MILLISECONDS;
+								if (!this.#candidateCollectionFailureWarned) {
+									this.#candidateCollectionFailureWarned = true;
+									logger.warn?.(
+										'could not apply the full-text candidate-key set; using the record predicate; point probes may resume on later requests',
+										error
+									);
+								}
+							}
+						}
+					}
+					const entry = this.#options.Table.primaryStore.getEntry(key, { transaction });
 					if (!entry?.value || hit.version !== String(entry.version)) {
 						staleVersionHits++;
 						continue;
@@ -434,17 +576,50 @@ export class FullTextQueryIndex {
 						expiredHits++;
 						continue;
 					}
-					if (options.filter && !options.filter(key, entry)) continue;
+					if (options.filter && !candidateDecides && !options.filter(key, entry)) continue;
 					accepted.push({ key, $score: hit.score, nativeId: hit.id, record: entry.value, recordEntry: entry });
 					if (accepted.length >= target) break;
 				}
 				offset += result.hits.length;
 				if (autocomplete) break;
 				if (!moreMayExist) break;
-				if (accepted.length < target) await new Promise((resolve) => setImmediate(resolve));
+				if (accepted.length < target) {
+					if (bounded) {
+						const acceptedThisPage = accepted.length - acceptedBeforePage;
+						rawPageSize = nextFilteredRawPageSize(
+							limit,
+							result.hits.length,
+							acceptedThisPage,
+							target - accepted.length,
+							this.#maxFilteredRawPageSize!
+						);
+					}
+					await new Promise((resolve) => setImmediate(resolve));
+				}
 			}
 			if (!bounded && moreMayExist)
 				throw new ClientError(`Full-text query exceeds the ${searchWindow}-result search window; add a limit`, 400);
+			if (
+				bounded &&
+				accepted.length < target &&
+				moreMayExist &&
+				staleVersionHits === 0 &&
+				rejectedGateSampleCount > 0
+			) {
+				options.assertTransactionActive?.();
+				for (let sampleIndex = 0; sampleIndex < rejectedGateSampleCount; sampleIndex++) {
+					const entry = this.#options.Table.primaryStore.getEntry(rejectedGateKeys![sampleIndex], { transaction });
+					if (
+						!entry?.value ||
+						rejectedGateVersions![sampleIndex] !== String(entry.version) ||
+						this.#options.Table.isFullTextSearchEntryCurrent?.(entry) === false
+					) {
+						staleVersionHits++;
+						break;
+					}
+					if (entry.expiresAt !== undefined && entry.expiresAt < Date.now()) expiredHits++;
+				}
+			}
 			if (bounded && accepted.length < target && moreMayExist && staleVersionHits > 0)
 				throw new DerivedIndexLagError(
 					`Full-text index '${this.#definition.name}' changed while searching; retry this query`
@@ -475,6 +650,64 @@ export class FullTextQueryIndex {
 		}
 	}
 
+	async #candidateGate(
+		plan: CandidateKeyPlan,
+		target: number,
+		maxSourceReads: number,
+		cooperate: () => Promise<void>,
+		assertActive: () => void
+	): Promise<CandidateGate | undefined> {
+		const store = this.#options.Table.primaryStore;
+		const estimatedRecords =
+			typeof store.getEstimatedKeyCount === 'function' ? store.getEstimatedKeyCount() : store.getStats?.().entryCount;
+		const recordCount = Math.max(1, Number.isFinite(estimatedRecords) ? estimatedRecords : target);
+		const selectivity = Math.min(1, Math.max(1, plan.estimatedCount) / recordCount);
+		const expectedSourceReads = Math.min(maxSourceReads, Math.ceil(target / selectivity));
+		const expectedRejectedReads = Math.ceil(expectedSourceReads * (1 - selectivity));
+		const maxKeys = Math.min(MAX_CANDIDATE_KEYS, Math.ceil(expectedRejectedReads * CANDIDATE_KEYS_PER_SOURCE_READ));
+		const now = Date.now();
+		if (plan.estimatedCount <= maxKeys && now >= this.#candidateCollectionRetryAfter) {
+			const collectionFailureEpoch = this.#candidateCollectionFailureEpoch;
+			try {
+				const collected = await plan.collect(maxKeys, cooperate);
+				assertActive();
+				if (collectionFailureEpoch === this.#candidateCollectionFailureEpoch) {
+					this.#candidateCollectionFailureWarned = false;
+					this.#candidateCollectionRetryAfter = 0;
+				}
+				if (collected)
+					return {
+						complete: collected.complete,
+						maxReadsPerCandidate: 0,
+						has: (primaryKey) => collected.keys.has(primaryKey),
+					};
+			} catch (error) {
+				assertActive();
+				this.#candidateCollectionFailureEpoch++;
+				this.#candidateCollectionRetryAfter = Date.now() + CANDIDATE_GATE_FAILURE_RETRY_MILLISECONDS;
+				if (!this.#candidateCollectionFailureWarned) {
+					this.#candidateCollectionFailureWarned = true;
+					logger.warn?.(
+						'could not build the full-text candidate-key set; using point probes or the record predicate',
+						error
+					);
+				}
+			}
+		}
+		const probe = plan.probe;
+		if (
+			!probe ||
+			now < this.#candidateProbeRetryAfter ||
+			probe.maxReadsPerCandidate > MAX_POINT_PROBE_READS_PER_CANDIDATE
+		)
+			return;
+		const probeSelectivity = Math.min(1, Math.max(1, probe.estimatedCount) / recordCount);
+		const expectedAvoidedSourceReadsPerCandidate = 1 - probeSelectivity;
+		return probe.maxReadsPerCandidate <= expectedAvoidedSourceReadsPerCandidate * CANDIDATE_KEYS_PER_SOURCE_READ
+			? probe
+			: undefined;
+	}
+
 	async #addHighlights(
 		reader: NativeFullTextReader,
 		definition: FullTextDefinition,
@@ -827,7 +1060,22 @@ export class FullTextQueryIndex {
 			this.#nativeOptions.limits.searchThreads
 		);
 		this.#maxSearchWindow = info.limits.maxSearchWindow;
-		this.#maxAutocompleteResults = info.limits.maxAutocompleteResults;
+		const responseHitCapacity = Math.floor(
+			(info.limits.maxSearchResponseBytes - NATIVE_SEARCH_RESPONSE_HEADER_BYTES) /
+				(NATIVE_SEARCH_HIT_OVERHEAD_BYTES + info.limits.maxRecordIdBytes + info.limits.maxRecordVersionBytes)
+		);
+		if (responseHitCapacity < 1)
+			throw new TypeError('@harperfast/fulltext/native search response limit cannot hold one maximum-size hit');
+		if (responseHitCapacity < info.limits.maxAutocompleteResults)
+			logger.warn?.(
+				`Full-text index '${this.#definition.name}' limits autocomplete to ${responseHitCapacity} results because the native response envelope cannot hold its advertised maximum`
+			);
+		this.#maxFilteredRawPageSize = Math.min(
+			MAX_FILTERED_RAW_PAGE_SIZE,
+			info.limits.maxSearchWindow,
+			responseHitCapacity
+		);
+		this.#maxAutocompleteResults = Math.min(info.limits.maxAutocompleteResults, responseHitCapacity);
 		this.#maxSearchBudgetMilliseconds = info.limits.maxSearchBudgetMilliseconds;
 		this.#maxTraceRecords = info.limits.maxTraceRecords;
 		this.#maxTraceSourceBytes = info.limits.maxTraceSourceBytes;
@@ -858,6 +1106,21 @@ export class FullTextQueryIndex {
 	}
 }
 
+function nextFilteredRawPageSize(
+	currentPageSize: number,
+	rawHits: number,
+	acceptedHits: number,
+	remainingResults: number,
+	maxPageSize: number
+): number {
+	const projected =
+		acceptedHits === 0
+			? // With no selectivity signal, grow independently of the remaining target to minimize native round trips.
+				currentPageSize * RAW_PAGE_ZERO_YIELD_GROWTH_FACTOR
+			: Math.ceil((remainingResults * rawHits * RAW_PAGE_OVERFETCH_FACTOR) / acceptedHits);
+	return Math.min(maxPageSize, Math.max(MIN_RAW_PAGE_SIZE, projected));
+}
+
 export async function pauseNativeFullTextQueryReaders(
 	path: string,
 	readinessId: string,

package/resources/models/DESIGN.md

@@ -12,15 +12,15 @@ Every sample the adapter (`generativeDecision.ts`) takes is a `models.generate`
 
 ## Required capabilities are checked on the candidate about to be invoked
 
-The router's candidate list is advisory: the default router filters by `requires`, but a custom router installed with `registerRouter` may return any backend. `assertCapabilities` (`Models.ts`) therefore re-checks every required capability against `capabilities()` immediately before each candidate is invoked, in every facade method, fallbacks included, and for a stream both before the iterable is returned and again on its first `next()`; a miss is a `ModelCapabilityError` recorded as `capability_unsupported`, and the next candidate is tried, except by a stream, which only ever uses its first candidate. Candidates are backend instances captured when the call was resolved, so the check sees a backend's own `capabilities()` at invocation time, not a registry entry a reload has since replaced; the built-in providers return one frozen object, so the check allocates nothing. The generative decision backend relies on it: by default it passes `requires: ['structuredOutput']` on every sample, so a generative candidate whose integration does not send the response schema as a decoding constraint (Anthropic and Bedrock today) is refused before a completion is billed, whatever router produced it, unless the entry sets `requireStructuredOutput: false`. Enforced by the custom-router cases in `unitTests/resources/models/Models.test.js`; the bootstrap cases in `generativeDecision.test.js` cover the default router's filtering of the same requirement.
+The router's candidate list is advisory: the default router filters by `requires`, but a custom router installed with `registerRouter` may return any backend. `assertCapabilities` (`Models.ts`) therefore re-checks every required capability against `capabilities()` immediately before each candidate is invoked, in every facade method, fallbacks included, and for a stream both before the iterable is returned and again on its first `next()`; a miss is a `ModelCapabilityError` recorded as `capability_unsupported`, and the next candidate is tried, except by a stream, which only ever uses its first candidate. Candidates are backend instances captured when the call was resolved, so the check sees a backend's own `capabilities()` at invocation time, not a registry entry a reload has since replaced; the built-in providers return frozen constants, so the check allocates nothing. The generative decision backend relies on it: by default it passes `requires: ['structuredOutput']` on every sample, so a generative candidate whose integration does not send the response schema as a decoding constraint (Anthropic and Bedrock today) is refused before a completion is billed, whatever router produced it, unless the entry sets `requireStructuredOutput: false`. Enforced by the custom-router cases in `unitTests/resources/models/Models.test.js`; the bootstrap cases in `generativeDecision.test.js` cover the default router's filtering of the same requirement.
 
 ## Likelihood scoring is a leaf-level hook with voting as the fallback
 
-A generative backend may implement `scoreChoices` (#2838): one finite, unnormalized log-likelihood per choice, in order, for a whole call, or a `ChoiceScoringUnsupportedError` for a call it cannot score. A decline is weaker news than a failure, and the two are kept apart at every layer. The facade's internal `models.scoreChoices` validates the vector's shape before it records a success, so a malformed result is one failed attempt against that backend and the next candidate is tried; when every candidate fails it surfaces the first failure, and the primary's decline only when every candidate declined, so a decline never hides a broken fallback. A declined call is recorded as `scoring_unsupported` with the usage the error carries, because a completion that came back without log-probabilities was still billed: those tokens land on that attempt's failure row and in the token metric while the call-count metric stays a count of successes. They are billed once: no other error's `usage` is read, no other method's failure row carries usage (the `decide` row stays usage-free when a decline propagates under `score`), and the adapter aborts sibling calls with a fresh reason rather than the decline itself, so a call cut short is recorded as `aborted` and not as another copy of the decline. The adapter scores one leaf per call (each property of an object schema separately), normalizes by softmax, and never mixes paths: in `auto` mode a decline, or a `ModelCapabilityError`, discards any leaf already scored and votes the whole decision under the same budget, so a distribution is wholly scored or wholly voted; any other error fails the decision. `auto` asks the router synchronously before deciding, so a generative backend whose capabilities say no costs no attempt, no exception and no analytics row. The same probe reads each candidate's `maxScoredChoices`: when no scoring candidate can take the schema's largest leaf, `auto` votes without a scoring attempt, and the facade treats a call above a candidate's limit as a decline it records without invoking that backend, so sibling leaves are never scored and then discarded. A candidate that advertises scoring without implementing it still counts as able, so its contract failure is reported rather than hidden by a vote. `maxScoredChoices` is a limit, not a `Capability`: it cannot be required. The OpenAI backend labels the choices `A`..`T` on one line each (a value that cannot sit on one line as itself is JSON-quoted, so it cannot pose as a further option and values that differ only in whitespace stay distinct), asks for one token with `top_logprobs: 20`, combines spellings of a label by log-sum-exp, and places a label with no listed spelling only when the mass the list leaves over is smaller than the leading observed label's, so a wholly unlisted label can never be the chosen value; otherwise the call is declined rather than guessed. An unlisted label then stands at a floor no higher than the smallest listed alternative or the leftover, finite even when the list leaves nothing over. Every placed score is still an estimate: an observed label's own unlisted spellings can add up to the leftover to it, so two observed labels closer than that may be ordered wrongly, and a response with no label at all is declined rather than reported as uniform. A malformed alternative is a failure, not a decline. A model that refuses `logprobs`, by a 400 (by `param`, or by a message that names it and says it is unsupported) or by an answer with no log-probability field at all, has scoring switched off until reload and `capabilities()` says so for the configured model, so later decisions vote without an attempt; a completion with no content token or no alternatives, and an answer that is simply not a label, are declined for that call only, and any other HTTP failure stays an error so an outage does not silently turn into voting. Probabilities from scoring are still `calibrated: false`. Enforced by `unitTests/resources/models/generativeDecision.test.js`, `scoreChoices.test.js` and `unitTests/components/openai/index.test.js`.
+A generative backend may implement `scoreChoices` (#2838): one finite, unnormalized log-likelihood per choice, in order, for a whole call, or a `ChoiceScoringUnsupportedError` for a call it cannot score. A decline is weaker news than a failure, and the two are kept apart at every layer. The facade's internal `models.scoreChoices` validates the vector's shape before it records a success, so a malformed result is one failed attempt against that backend and the next candidate is tried; when every candidate fails it surfaces the first failure, and the primary's decline only when every candidate declined, so a decline never hides a broken fallback. A declined call is recorded as `scoring_unsupported` with the usage the error carries, because a completion that came back without log-probabilities was still billed: those tokens land on that attempt's failure row and in the token metric while the call-count metric stays a count of successes. They are billed once: no other error's `usage` is read, no other method's failure row carries usage (the `decide` row stays usage-free when a decline propagates under `score`), and the adapter aborts sibling calls with a fresh reason rather than the decline itself, so a call cut short is recorded as `aborted` and not as another copy of the decline. The adapter scores one leaf per call (each property of an object schema separately), normalizes by softmax, and never mixes paths: in `auto` mode a decline, or a `ModelCapabilityError`, discards any leaf already scored and votes the whole decision under the same budget, so a distribution is wholly scored or wholly voted; any other error fails the decision. `auto` asks the router synchronously before deciding, so a generative backend whose capabilities say no costs no attempt, no exception and no analytics row. The same probe reads each candidate's `maxScoredChoices`: when no scoring candidate can take the schema's largest leaf, `auto` votes without a scoring attempt, and the facade treats a call above a candidate's limit as a decline it records without invoking that backend, so sibling leaves are never scored and then discarded. A candidate that advertises scoring without implementing it still counts as able, so its contract failure is reported rather than hidden by a vote. `maxScoredChoices` is a limit, not a `Capability`: it cannot be required. The OpenAI backend labels the choices `A`..`T` on one line each (a value that cannot sit on one line as itself is JSON-quoted, so it cannot pose as a further option and values that differ only in whitespace stay distinct), asks for one token with `top_logprobs: 20`, combines spellings of a label by log-sum-exp, and places a label with no listed spelling only when the mass the list leaves over is smaller than the leading observed label's, so a wholly unlisted label can never be the chosen value; otherwise the call is declined rather than guessed. Each unlisted label then takes an equal share of that leftover, so together they never exceed it and one can still sit above a listed tail label; when the list leaves nothing over they sit a fixed margin (`UNLISTED_MARGIN`) below its smallest entry, so the vector stays finite. Every placed score is still an estimate: an observed label's own unlisted spellings can add up to the leftover to it, so two observed labels closer than that may be ordered wrongly, and a response with no label at all is declined rather than reported as uniform. A malformed alternative is a failure, not a decline. A model that refuses `logprobs`, by a 400 (by `param`, or by a message that names it and says it is unsupported) or by an answer with no log-probability field at all, has scoring switched off until reload and `capabilities()` says so for the configured model, so later decisions vote without an attempt; a completion with no content token or no alternatives, and an answer that is simply not a label, are declined for that call only, and any other HTTP failure stays an error so an outage does not silently turn into voting. Probabilities from scoring are still `calibrated: false`. Enforced by `unitTests/resources/models/generativeDecision.test.js`, `scoreChoices.test.js`, `scoringLimit.test.js` and `unitTests/components/openai/index.test.js`.
 
 ## `@decide` and `@embed` share one trigger rule
 
-Both directives derive an attribute on the originating node before the record is staged, in `_writeUpdate` and in the cache-from-source write. `writeHookApplies` and `sourceState` (`embedHook.ts`) are the only trigger logic: a replication receive, an `x-replicate-from: none` write or an audit replay never re-derives, a payload without the source leaves the derived attributes alone, a null source clears them, and a CRDT op is skipped. `@decide` writes the chosen value and, when named, its probability from one `models.decide` call, and a failure fails the write, so the two are never from different decisions and never half-written; a missing decider or an override without the promised probability fails the write too. `runWriteJobs` aborts the siblings of a failed hook and settles them before the failure is reported, so a rejected write neither waits on a hook that honors the signal nor keeps being mutated by one; the built-in hooks forward the signal to their model calls. `assertDerivedFieldOwnership` gives every derived field one writer, refuses a directive whose source is another directive's output, because the hooks run concurrently within a write, holds a decided field and its confidence to what the loader accepts, and refuses an `Object.prototype` key as any of those fields, because `sourceState` tests presence with `in`. The loader runs it, and `declareTable` runs it on the complete list (a peer's fields merged, an omitted primary key inherited) before that list replaces the live one or reaches the catalog, so a programmatic `table({ attributes })` declaration is held to the same rules and a refused declaration changes nothing. `Table.updatedAttributes` runs it too but only logs a violation, because it also runs when a descriptor an earlier build accepted is loaded from the catalog, and a table must keep loading. The hooks receive the request's signal, so a client that disconnects cancels the model calls as it did before the hooks were combined. A `decide` descriptor change persists without a reindex: a stored decision does not depend on the model, unlike an embedding, whose `version` carries the model. Enforced by `unitTests/resources/models/decideHook.test.js`, `decideDirective.test.js`, `decideWrite.test.js` and `integrationTests/server/decide-directive.test.ts`.
+Both directives derive an attribute on the originating node before the record is staged, in `_writeUpdate` and in the cache-from-source write. `writeHookApplies` and `sourceState` (`embedHook.ts`) are the only trigger logic: a replication receive, an `x-replicate-from: none` write or an audit replay never re-derives, a payload without the source leaves the derived attributes alone, a null source clears them, and a CRDT op is skipped. `@decide` writes the chosen value and, when named, its probability from one `models.decide` call, and a failure fails the write, so the two are never from different decisions and never half-written; a missing decider or an override without the promised probability fails the write too. `runWriteJobs` aborts the siblings of a failed hook and settles them before the failure is reported, so a rejected write neither waits on a hook that honors the signal nor keeps being mutated by one; the built-in hooks forward the signal to their model calls. `assertDerivedFieldOwnership` gives every derived field one writer, refuses a directive whose source is another directive's output, because the hooks run concurrently within a write, holds a decided field and its confidence to what the loader accepts, and refuses an `Object.prototype` key as any of those fields, because `sourceState` tests presence with `in`. The loader runs it, and `declareTable` runs it on the complete list (a peer's fields merged, an omitted primary key inherited) before that list replaces the live one or reaches the catalog, so a programmatic `table({ attributes })` declaration is held to the same rules and a refused declaration changes nothing. `Table.updatedAttributes` runs it too but only logs a violation, because it also runs when a descriptor an earlier build accepted is loaded from the catalog, and a table must keep loading. The hooks receive the request's signal, so a client that disconnects cancels the model calls as it did before the hooks were combined. A `decide` descriptor change persists without a reindex: a stored decision does not depend on the model, unlike an embedding, whose `version` carries the model. Enforced by `unitTests/resources/models/decideHook.test.js`, `decideDirective.test.js`, `decideRegistry.test.js`, `decideWrite.test.js` and `integrationTests/server/decide-directive.test.ts`.
 
 ## Built-ins serve one kind
 
@@ -37,3 +37,11 @@ A leaf opts in with `noMatch: true` (#2846, decided on #2843). The backend then
 ## A `@decide` decision is recorded only when the record can reach it
 
 The built-in decider passes `persist: true` only when the directive names a `decision` attribute, so a directive nobody can report outcomes for writes no `hdb_model_decisions` row (#2852). With one, the decider records the decision and the hook stores `Decision.id` beside the value and confidence; a null source clears all three. The id is provenance an outcome report trusts, so `buildDecideBefore` rejects a write that carries it unless that same write derives or clears it, or arrives by replication or replay; `x-replicate-from: none` and a CRDT operation on the source do not qualify, since any request can send them. The association is bounded, not atomic: the decision commits in its own transaction before the record, so an aborted write, a later source write, or a delete leaves an unreferenced row that expires with the retention. On a read-only node no write can commit, so `Table` skips the model hooks and serves a source-cache fill without staging it (`noCacheStore`), for every table. Enforced by `decideHook.test.js`, `decideWrite.test.js`, `decideOutcome.test.js` and `integrationTests/server/decide-directive.test.ts`.
+
+## A fitted calibration never crosses a score source, and never waits on storage or fails a call
+
+`calibrated: true` from a fit (#2841) means an eligible, unexpired version, fitted under the active policy, was applied to every field of the decision. A population is `populationKey` (`calibration.ts`): tenant, logical model, the decision entry's fingerprint, the signature, the instructions hash and the schema hash. The fingerprint is `sourceFingerprint` (`decisionStore.ts`): the entry's logical name, because a module-backed backend may choose its scorer by name, and its expanded configuration without its named credential fields (`apiKey`, `accessKeyId`, `secretAccessKey`, `sessionToken`, `authorization`, `password`, `token`, `credentials` and the like, matched by exact name) or `fallback`, published per backend by `bootstrap.ts` into `backendRegistry.ts`. An operator-set `revision` covers a deployment change the configuration cannot show, such as a floating model tag. The accounting context's `app` is left out on purpose, because it is the calling route, not an application, and would split one question into a population per endpoint. The generative adapter signs a decision with `;source=<fingerprint>` only when every inner call reported the same identified source through the `SERVED_SOURCE` hook. The facade calls that hook after a successful attempt, outside its `try`, and swallows its faults. A fallback inside a vote, or a backend registered from code, leaves the decision unsigned, and an unsigned decision is never calibrated or learned from. A custom decision backend registered from code identifies its score source only through the `signature` it returns, which it must change whenever that source changes; Harper cannot see a replaced implementation behind the same name and signature. Only decisions recorded while calibration is enabled carry a `population`, so a fit never reads scores whose source was not collected. `getCalibrations` returns that same key on every summary, because the fields a summary otherwise shows (signature and hashes) do not include the entry fingerprint: a caller matching a decision to its report on those alone would read another population's risk after an entry setting changed.
+
+`applyFits` (`calibrationStore.ts`) reads the in-memory cache only. A key never loaded returns the decision raw and starts one load; an entry older than the freshness window keeps serving while it refreshes, so a population is uncorrected only before its first load or after a load times out or fails, at most `maxLoads` in flight, each counted until its read settles, so a store whose reads never settle holds the slots and leaves that node's decisions uncorrected rather than piling up reads. Faults are logged through `safeErrorMessage` and return the decision raw. Reliability scores the value a decision returned, which on tied scores need not be the schema-first one, so a fit is judged on the decisions it will actually serve. A reload that leaves the calibration block unchanged keeps the cache and the scheduled job, so frequent unrelated config changes never postpone the first run. Calibration runs after the analytics row, outside the candidate loop, so it never bills a second call. `requires: ['calibrated']` still routes on the backend's own claim. A schema with a no-match leaf is never fit-calibrated, because its no-match score would stay raw under the flag; its population gets reports whose reason is `no-match-schema`, never an eligible version.
+
+Versions in `hdb_model_calibrations` (`replicate: true`) are immutable. `fitId` covers the input digest, the policy and the result, so a repeated run writes nothing. A row is applied only when its `key`, its `rank` prefix and the key recomputed from its `population` and `field` all match the key it was read under, so a malformed replicated row cannot cross a population. The reader takes the greatest `rank` (`key | evidenceAt | fittedAt | fitId`) in one descending prefix read. `evidenceAt` is the newest decision or truth fact a version read, so the version that saw newer evidence wins whatever node wrote it and in whatever order rows arrive; two runs on different nodes that both step past the same previous version tie on `evidenceAt`, and the later `fittedAt` wins until a run that reads both writes a successor. A new version's `evidenceAt` is also at least one above the version it replaces, so a correction stamped by a node with a slower clock still supersedes it. A version is written whenever a population's input digest changes, even with no labels left, so a correction revokes an older eligible version. Rows expire after their decisions do. `calibrate()` can only lower a run's budgets, never raise them or change its policy, and a run invalidates only the cache entries of the keys it wrote. A population's decisions are read newest first through the indexed `populationRank` (`population | expiresAt | id`) with a limit, so a large population is never loaded in full. The limit is the newest decisions whose examples, at their cost across every field, fit in the configured `maxBytes` beside the population's own head: a size fixed by the configuration and the schema, so a population too large for the byte budget is still refit, on the same sample each run. A run that cannot afford that sample, because `calibrate()` lowered `maxBytes`, stops on the byte budget and writes nothing, because a smaller sample could revoke a fit its evidence still supports. Heads and a population's sample are budgeted separately, each against `maxBytes`, so other populations in a run never crowd a sample out, and a run holds at most about twice `maxBytes`. A head is renewed only by a run that read decisions for it, so a population whose decisions have all expired lapses instead of staying in the table. Discovery uses at most half of `maxDecisions`, split between the newest decisions and a saved cursor that keeps the ids on its boundary, and splits its population budget the same way. A row that hits a cap is left for the next run, a newly discovered population gets a head at once, and ties at one timestamp never stall the cursor. No discovery page asks for more rows than the scan may still take. Decisions read for fitting come out of the rest of the same budget, so one run reads at most `maxDecisions`, plus one decision per deferred population. A population is read up to one decision past what is left of the budget, and deferred to a later run when that extra decision exists, so a truncated read is never fitted and a population that exactly fits is not deferred forever; a population a run could not finish, deferred or stopped by the byte or time budget, moves to the back of the queue so it cannot hold up the ones behind it, and configuration requires `maxDecisions` of at least twice `maxExamplesPerKey` plus 2, so the first population fitted in a run always has room for a full sample, because a partial sample would write a version that revokes a fit its evidence still supports. A failed run fails its scheduled job. A run walks known populations least recently fitted first, then those discovered newest first, under global decision, population, byte and time budgets, and writes a population only after its report is complete. Its job is registered with `registerInternalJobs` (`resources/scheduler/engine.ts`), under a reserved prefix no component can register or unregister. Enforced by `unitTests/resources/models/calibration.test.js`, `calibrationStore.test.js`, `sourceIdentity.test.js` and `unitTests/resources/scheduler/engine.test.js`.