@harperfast/harper 5.3.0-beta.4 → 5.3.0

package/resources/fullTextSchemaLifecycle.ts

@@ -2,7 +2,9 @@ import { ClientError } from '../utility/errors/hdbError.ts';
 import {
 	compileFullTextDefinition,
 	compileFullTextDefinitions,
+	compileFullTextFields,
 	compileValidFullTextDefinitions,
+	persistedFullTextIndexNames,
 	sortFullTextDefinitions,
 	type FullTextDefinition,
 	type FullTextSchemaAttribute,
@@ -22,6 +24,49 @@ export function definitionsEqual(left: unknown, right: unknown): boolean {
 	return serializeFullTextState(left ?? []) === serializeFullTextState(right ?? []);
 }
 
+export function readPersistedFullTextFields(
+	values: unknown,
+	definitions: unknown,
+	attributes: readonly FullTextSchemaAttribute[],
+	warn: FullTextWarning
+): string[] {
+	const persistedNames = persistedFullTextIndexNames(definitions);
+	if (values === undefined) return persistedNames;
+	const names = persistedNames.map((name) => ({ name }));
+	try {
+		return compileFullTextFields(values, names, attributes, true);
+	} catch (error) {
+		if (!(error instanceof ClientError)) throw error;
+		warn(`Invalid persisted full-text fields; reserving every persisted index name: ${error.message}`);
+		return persistedNames;
+	}
+}
+
+export function mergePeerFullTextFields(
+	persistedValues: unknown,
+	incomingValues: unknown,
+	persistedDefinitions: unknown,
+	finalDefinitions: unknown,
+	attributes: readonly FullTextSchemaAttribute[],
+	warn: FullTextWarning
+): string[] {
+	const retained = readPersistedFullTextFields(persistedValues, persistedDefinitions, attributes, warn);
+	const finalNames = persistedFullTextIndexNames(finalDefinitions);
+	if (incomingValues === undefined) incomingValues = finalNames;
+	try {
+		compileFullTextFields(
+			incomingValues,
+			finalNames.map((name) => ({ name })),
+			attributes
+		);
+	} catch (error) {
+		if (!(error instanceof ClientError)) throw error;
+		warn(`Ignoring invalid peer full-text fields: ${error.message}`);
+	}
+	const existingNames = new Set(persistedFullTextIndexNames(persistedDefinitions));
+	return [...new Set([...retained, ...finalNames.filter((name) => !existingNames.has(name))])].sort();
+}
+
 export function readPersistedFullTextDefinitions(
 	values: unknown,
 	attributes: readonly FullTextSchemaAttribute[],

package/resources/graphql.ts

@@ -8,7 +8,7 @@ import type { DirectiveNode, NamedTypeNode, StringValueNode, ValueNode } from 'g
 import { ClientError } from '../utility/errors/hdbError.ts';
 import { attributeToFragment, type JsonSchemaFragment } from './jsonSchemaTypes.ts';
 import harperLogger from '../utility/logging/harper_logger.ts';
-import { compileFullTextDefinitions } from './fullTextSchema.ts';
+import { compileFullTextDefinitions, compileFullTextFields } from './fullTextSchema.ts';
 import { validateDecisionSchema } from './models/decision.ts';
 import { assertDerivedFieldOwnership } from './models/embedHook.ts';
 import type { DecideConfig } from './models/decideHook.ts';
@@ -253,6 +253,7 @@ async function processGraphQLSchema(
 					attributes,
 					properties: typeProperties,
 					fullTextIndexes: [],
+					fullTextFields: [],
 				};
 				if (definition.description?.value) typeDef.description = definition.description.value;
 				types.set(typeName, typeDef);
@@ -287,15 +288,8 @@ async function processGraphQLSchema(
 					if (directive.name.value === 'splitSegments') typeDef.splitSegments = true;
 					if (directive.name.value === 'replicate') typeDef.replicate = true;
 					if (directive.name.value === 'hidden') typeDef.hidden = true;
-					if (directive.name.value === 'fullText') {
-						const definition: Record<string, unknown> = Object.create(null);
-						for (const arg of directive.arguments || []) {
-							if (Object.hasOwn(definition, arg.name.value))
-								throw new ClientError(`@fullText declares "${arg.name.value}" more than once`, 400);
-							definition[arg.name.value] = coerceDirectiveValue(arg.value);
-						}
-						typeDef.fullTextIndexes.push(definition);
-					}
+					if (directive.name.value === 'fullText')
+						throw new ClientError('@fullText must be declared on a FullText field', 400);
 					if (directive.name.value === 'export') {
 						typeDef.export = true;
 						for (const arg of directive.arguments) {
@@ -318,12 +312,40 @@ async function processGraphQLSchema(
 						};
 					}
 					const typeName = (type as NamedTypeNode).name?.value;
+					if (typeName === 'FullText') throw new ClientError('FullText fields require @fullText', 400);
 					const property = { type: typeName };
 					Object.defineProperty(property, 'location', { value: type.loc.startToken });
 					return property;
 				}
 				const attributesObject = {};
 				for (const field of definition.fields) {
+					const name = field.name.value;
+					const fullTextDirectives = field.directives.filter(({ name }) => name.value === 'fullText');
+					if (fullTextDirectives.length > 0) {
+						if (fullTextDirectives.length !== 1)
+							throw new ClientError(`Full-text field "${name}" requires exactly one @fullText directive`, 400);
+						if (field.type.kind !== 'NamedType' || field.type.name.value !== 'FullText')
+							throw new ClientError(`@fullText field "${name}" requires the nullable FullText type`, 400);
+						for (const directive of field.directives) {
+							if (directive.name.value !== 'fullText')
+								throw new ClientError(
+									`Full-text field "${name}" cannot use @${directive.name.value}; configure permissions and record values on its source fields`,
+									400
+								);
+						}
+						const index: Record<string, unknown> = Object.create(null);
+						index.name = name;
+						for (const arg of fullTextDirectives[0].arguments || []) {
+							if (arg.name.value === 'name')
+								throw new ClientError(`@fullText uses the field name "${name}"; remove its name argument`, 400);
+							if (Object.hasOwn(index, arg.name.value))
+								throw new ClientError(`@fullText declares "${arg.name.value}" more than once`, 400);
+							index[arg.name.value] = coerceDirectiveValue(arg.value);
+						}
+						typeDef.fullTextIndexes.push(index);
+						typeDef.fullTextFields.push(name);
+						continue;
+					}
 					const property = getProperty(field.type);
 					property.name = field.name.value;
 					if (field.description?.value) property.description = field.description.value;
@@ -392,8 +414,6 @@ async function processGraphQLSchema(
 							// Resolved after the field's other directives so @primaryKey / @computed in any
 							// order are seen; the ownership checks below need every field of the type.
 							property.decide = directive;
-						} else if (directiveName === 'fullText') {
-							throw new ClientError('@fullText must be declared on a @table type, not on a field', 400);
 						} else if (directiveName === 'relationship') {
 							const relationshipDefinition = {};
 							for (const arg of directive.arguments) {
@@ -447,6 +467,7 @@ async function processGraphQLSchema(
 				if (typeDef.fullTextIndexes.length > 0 && !typeDef.table)
 					throw new ClientError('@fullText is only supported on a @table type', 400);
 				typeDef.fullTextIndexes = compileFullTextDefinitions(typeDef.fullTextIndexes, attributes);
+				typeDef.fullTextFields = compileFullTextFields(typeDef.fullTextFields, typeDef.fullTextIndexes, attributes);
 				// Project the array form into the canonical `properties` Record (JSON-Schema-shaped,
 				// keyed by attribute name). Both shapes are co-populated in this single pass;
 				// downstream consumers (MCP, OpenAPI) read whichever form they prefer.

package/resources/indexes/DESIGN.md

@@ -178,11 +178,38 @@ unit switch does not shift the estimate on its own. The ef term remains the conf
 not the runtime auto-scaled value, so the planner increasingly underestimates vector traversal cost
 as an automatically scaled graph grows.
 
+## A filtered vector search may admit from an index key set only if the set holds every predicate match
+
+`search.ts → planCandidateKeys` turns companion AND conditions a secondary index answers (equality,
+`gt`/`ge`/`lt`/`le`, the `between` family, nested AND/OR over those) into key-only range scans, and the
+HNSW index admits from the keys instead of loading a record per visited node (#2688). A set covering
+every pushed-down condition, with no `rowFilter`/`vectorFilter`, replaces the predicate; otherwise it
+gates the residual predicate. The post-filter re-checks every row, so the set may over-admit but must
+never omit: an omission is unrecoverable and makes results depend on whether the cost model built it.
+
+`planCandidateKeyScan` enforces that by running the condition's own `filterByType` predicate on the
+records an index cannot supply: one lacking the attribute, a null one when the index holds no nulls,
+and a `0`, because a number written as `-0` is indexed under an encoding that sorts past every number
+and decodes to a wrong primary key (ordered-binary; index-led queries miss that row too). A condition
+admitting any of them stays on the predicate path. `compareKeys` ranks undefined below every scalar, so
+this declines `lt`/`le` over a number, string or boolean bound, `ge null` and any range covering 0; the
+`lt`/`le` that remain (a BigInt bound) start at `null`, not `searchByIndex`'s `true`, so indexed nulls
+are read. Normalizing `-0` in the encoder or index writer would let zero-covering ranges plan again.
+
+Building the set is a cost decision: one index-entry read per match against a record load per visited
+node, where filling `ef` matches at selectivity `s` takes about `ef / s` visits. `KEYS_PER_PREDICATE_VISIT`
+is that ratio at the warm, regression-safe end of the measured range; `MAX_CANDIDATE_KEYS` and half the
+node count cap the synchronous scan. A complete set derives the visit budget from its selectivity,
+widening the `filterExpansion` budget by at most `ALLOW_SET_BUDGET_MAX_WIDENING` and never narrowing it;
+a configured `filterExpansion` stays authoritative. `CandidateKeySet` is the one identity rule for both
+the scan's intersection and admission. A node-id bitset for the native plane is not built: the
+pk → node-id lookup costs more than the record load it saves.
+
 ## Derived-index runtime: committed-log delivery to native index backends (`resources/derivedIndexRuntime.ts`)
 
-A derived index (the native HNSW plane, a future Tantivy full-text index) is a materialized view
-that lives outside the record transaction: its apply is native, costs 0.2–1.4 ms per mutation, and
-its durability barrier is an `msync` or a segment publish, none of which belong on the commit path.
+A derived index (the native HNSW or Tantivy full-text plane) is a materialized view
+that lives outside the record transaction: its apply is native and its durability barrier is an
+`msync` or a segment publish, neither of which belongs on the commit path.
 The runtime is the one Harper-side implementation of the delivery protocol in harper#2489: **the
 transaction log is the durable fact, a commit only wakes a runner, and every backend resumes from an
 exact cursor into that log.** There is deliberately no second delivery fact — no transactional
@@ -327,16 +354,29 @@ cursor is backend-owned and validation is Harper's.
 `@harperfast/fulltext/native`; it does not implement another replay or ownership protocol. Harper
 turns each resolved mutation into one stable document id from `tableId` and the record id's
 ordered-binary storage-key bytes, and
-passes only the schema-selected string and array fields to the wrapper. Harper does not rescan array
-contents; the wrapper owns value validation, Tantivy schema, exact frame partitioning, its exclusive writer, segment publication, and file
-lifecycle. Harper keeps accepted runtime batches in a 64 MiB bounded queue and submits at most 256
+passes only the schema-selected string and array fields to the wrapper. Harper validates each selected
+value, including every array element, and resolves text Blobs before submission. The wrapper owns
+native document validation, Tantivy schema, exact frame partitioning, its exclusive writer, segment
+publication, and file lifecycle. Harper keeps accepted runtime batches in a 64 MiB bounded queue and submits at most 256
 records or 5 ms of conversion work per turn, so the runtime's 4096-record chunk cannot become one
-long event-loop task. A rebuild chunk without a source-size estimate consumes the adapter's entire
+long event-loop task. Awaited Blob reads yield the event loop and therefore do not consume that
+conversion budget; otherwise ordinary storage latency would fragment a rebuild into small native
+mutation submissions. Blob-bearing slices also have a wall-time ceiling derived from the writer shutdown
+budget, so a run of slow reads cannot keep one drain turn alive through hundreds of per-record read
+timeouts. A rebuild chunk without a source-size estimate consumes the adapter's entire
 queue-byte allowance, ensuring that only one unknown-size chunk is retained at a time. Wrapper
 rejections remove the previous document and count it as unindexable; they do not leave stale search
 content. A projector returning null or no string-valued fields deletes the prior document rather
 than indexing an empty replacement.
 
+The elected runner opens one writer per index with two Tantivy indexing threads. The wrapper passes
+the 64 MiB total writer budget to Tantivy's multi-threaded writer, which divides it into a 32 MiB
+arena per indexing thread. Harper does not search through that writer handle, so it keeps the one
+search worker required by the wrapper. Query readers can open on every Harper worker, so each uses
+only the wrapper's minimum search pool for ordinary-versus-expensive query isolation rather than
+scaling with host CPU. Read-only handles reserve search threads and queue bytes, but the wrapper
+does not reserve their validation-only indexing-thread or writer-memory values.
+
 Every runtime flush is a native publication barrier. Full-text activation chooses and benchmarks the
 runtime flush thresholds; the adapter does not reinterpret a flush because the runtime uses durable
 cursor progress to bound replay work and transaction-log retention.
@@ -356,7 +396,7 @@ not prove the index files should be reset. Persistent writer unavailability emit
 or record content. Ownership handoff does not finish until drain and close prove quiescence. A
 writer-open failure that lands after shutdown begins discards queued work before finalization so it
 cannot enter another retry drain. Harper
-gives native close its own 35-second bound and bounds the complete handoff at 70 seconds. If that proof fails, shutdown rejects and the runtime keeps its
+gives native close its own 35-second bound and bounds the complete handoff at 105 seconds. If that proof fails, shutdown rejects and the runtime keeps its
 runner lock, preventing a second writer. The underlying native operation continues and a later operator
 retry attaches to the same shutdown rather than starting a competing close. A cursor that cannot fit
 the native commit-payload limit is terminal for that backend instance: accepted work is rollback-closed
@@ -373,6 +413,8 @@ Inspection is synchronous and writer-free. The first durable-cursor read in each
 refreshes native state, while later reads in the same acquisition use the cache; every shutdown path
 invalidates it, including an owner that received no batch. A refresh failure never falls back to a
 cached checkpoint because the runtime can publish `ready` before lazy writer-open reconciliation.
+Query readers never use inspection: their committed payload belongs to the native snapshot they
+actually search, while inspection is an ownership-acquisition and recovery primitive.
 The synchronous backend contract has no retryable acquisition result, so an inspection exception
 deliberately fails closed: the runtime condemns the generation and rebuilds from authoritative
 records rather than trusting an unverified cursor. A transient filesystem error can therefore cost
@@ -381,7 +423,10 @@ Missing, cursorless, incompatible, or malformed native state has no usable curso
 the runtime's ordinary local rebuild from records.
 Reset first asks the wrapper to retire the live generation atomically, then reclaims only wrapper-
 validated retired paths. The reset operation is tracked and bounded; shutdown attaches to the same
-operation and keeps the runner lock if it cannot prove settlement. Retired-path reclamation is
+operation and keeps the runner lock if it cannot prove settlement. If that operation outlives its
+owner's bound, a successor waits for the same operation to settle and then issues its own epoch's
+reset; it does not spend the remaining rebuild attempts on immediate owner-epoch mismatches.
+Retired-path reclamation is
 best-effort, serialized per lifecycle, and never extends that reset handoff. The native directory is node-local derived state: restarts reuse it and
 replay after its payload; replicas independently derive it from their own applied transaction log;
 backup and restore need only authoritative records and schema. A new or unusable directory serves no
@@ -396,16 +441,120 @@ the catalog first would turn a crash in that interval into an untraceable native
 
 Tantivy files are not an opaque encrypted cache. They contain the document-id term dictionary,
 analyzed term dictionaries, postings, frequencies and optionally positions; `surfaceTerms: true`
-also stores the original projected strings needed for surface-term features. Operators must protect
+adds an unstemmed companion term field for surface-term features, but does not store the original
+source values. Operators must protect
 the full-text directory with the same filesystem controls as Harper data. Removing source records
 does not erase old segment bytes immediately; normal Tantivy merge/reclamation governs physical
 removal, and destroying an index uses the wrapper's retirement protocol.
 
-The binding remains unloaded until a full-text declaration is activated. Before activation can
-construct this backend, Harper must exact-pin the Fulltext package, document the dependency in
-`dependencies.md`, and prove that its native prebuild loads on Linux, macOS, and Windows CI. A
-missing or incompatible binding is an activation error; Harper must not silently omit the declared
-index.
+The binding remains unloaded until a full-text declaration is activated. Harper exact-pins the
+Fulltext package, documents it in `dependencies.md`, and loads its native prebuild in dedicated
+Linux, macOS, and Windows CI jobs. A missing or incompatible binding is an activation error; Harper
+does not silently omit the declared index.
+
+### Full-text query plane
+
+Each declared full-text index can lazily open a read-only native handle beside its derived-index writer. Readers
+are keyed by native path, readiness id and owner epoch. A process-wide atomic publication generation
+invalidates stale handles and notifies peer workers. A warm reader reloads in the background after a
+notification, while every query still samples the generation and reloads on mismatch; notification
+delivery is an optimization, not a correctness dependency. Workers that never query an index open no
+reader or publication subscription. The shared derived-index runtime allocates this publication state
+only for backends with external reader snapshots; HNSW does not pay for it. A waiting query samples the
+generation after its coverage wait, so the data-publication bump that precedes the covering watermark
+is included in reader acquisition. The wrapper refreshes the reader's committed payload with the
+searcher, and Harper keeps that decoded checkpoint on the reader slot so coverage always describes the
+snapshot being searched.
+Coverage-only publications do not change the document set: the runtime permits them only after every
+accepted index mutation through that capture is durable. The shared live watermark may therefore wake a
+waiting query before Tantivy commits the newer restart checkpoint; the existing reader is data-current,
+and the later publication notification reloads its checkpoint. Catalog-scale qualification measures this
+metadata-only reload cadence so unrelated write traffic cannot become a hidden query-plane cost.
+Missing, cursorless or malformed payloads fail closed and request one rebuild. A strict, non-waiting query on a
+cold worker can return retryable lag while it opens the reader; retrying, or using
+`waitForIndexMilliseconds`, allows the aligned reader to finish opening without synchronous filesystem
+inspection. An epoch or query-configuration change requires a replacement handle. A publication reload
+uses the existing handle only while it has no active lease; otherwise Harper opens a replacement and
+retires the old handle after its search finishes. One logical query therefore uses one native snapshot
+across all result pages and highlight tracing. The wrapper's reported query budget bounds that whole
+query, including page fetches, source Blob reads and tracing, rather than restarting for each native
+call. Highlight source materialization is bounded by both the wrapper's trace-record count and source-byte
+limit before Blob reads start. Query leaves with the same field set reuse that materialization; distinct
+field sets remain independently bounded. Requesting highlights from an index that did not enable them is rejected.
+A retired handle accepts no new leases but remains open until its active searches finish.
+Rejected closes remain tracked and are retried during the next pause or close; a fulfilled native
+cleanup warning is logged. Neither replaces an otherwise successful query response.
+One failed reload leaves the last aligned snapshot installed and returns retryable lag. Three
+consecutive reload failures retire that handle so the next query reopens from native storage.
+Table close, drop and schema replacement never force-close an active native handle. They stop waiting
+after the wrapper's query budget plus a small settlement grace and report a retryable teardown failure;
+the retired handle remains registered for reset fencing and closes when the query eventually releases
+its lease. It unregisters only after that drain settles.
+
+Reset is destructive, so the elected writer first pauses readers in every worker and waits for their
+active leases to drain. Harper does not force-close a handle that native code may still be using; a
+stuck native request therefore delays reset rather than risking use-after-close. The reset handoff
+allows two reader-coordination bounds plus the native reset bound. Resume carries the
+exact readiness-id and owner-epoch token that paused the path, preventing an old reset from reopening
+readers after a newer owner has paused them. Epochs from different readiness generations are never
+compared because each generation owns an independent counter.
+The runtime publishes `rebuilding` before invoking reset, so a worker created after the pause
+broadcast cannot admit a new reader. A worker that misses resume clears a token after that
+token's own shared readiness reaches ready at the same or a newer owner epoch. A predecessor generation
+that crashes during reset may never publish ready again. A successor that observed the foreign token
+before reaching ready can clear it when that same successor becomes ready: reaching ready requires the
+stable backend runner lock, so the predecessor's reset can no longer be active. A generation that was
+already ready when it observed the token must acquire the runner lock before clearing it. Recovery waits
+if a writer still owns that lock.
+The pause covers every reader on the physical path, including a superseded generation. Each active reset
+retains its own token; stale resumes cannot clear newer tokens. A worker with no query index on the path
+retains no token because it has no reader to fence, and any later attachment remains gated by its
+generation's shared readiness.
+After the last local attachment for a superseded readiness id drains and unregisters, Harper clears
+that id's exact-epoch token from surviving readers on the path and remembers that the id retired. A
+delayed pause for that retired id cannot reinstall its fence on a successor. A readiness id never seen
+on this worker is still accepted, so a successor reset fences a predecessor reader on a worker whose
+schema refresh is late. Closing attachments remain registered until their readers drain, and a duplicate
+attachment for the same readiness id keeps the fence. This closes both handoff directions without
+assuming an ordering between generation-local epochs. Retirement uses an explicit synthetic token that
+still pauses every reader because it is not tied to a generation attachment. If the native
+reader violates its contract by rejecting close, Harper logs the failure and proceeds with reset
+rather than wedging the path indefinitely; the reset therefore assumes that rejected handle is dead.
+
+Weights and highlighting are query configuration. Changing either refreshes readers without rotating
+the persisted generation. Analyzer behavior, stop words, positions, surface terms, synonyms, source
+media types, source-field membership and source order are storage identity and require a new generation. The
+wrapper validates that identity on inspection and open, so an older analyzer generation fails closed
+and rebuilds instead of serving mixed tokenization semantics.
+
+Highlighting is declaration-controlled and off unless configured. Snippet fragments contain source
+plaintext, so Harper authorizes every highlighted source field before search; selecting
+`$highlights` is also selecting that configured plaintext egress. Harper bounds each native trace call
+by both record count and encoded source bytes and sends only the fields used by that query leaf. A
+single record larger than the native trace limit is rejected before crossing the binding.
+
+Search hits carry the source record version. Harper loads the authoritative record through its read
+transaction and omits a hit when that version no longer matches; it never attaches an old score or
+highlight to new content. Bounded searches over-fetch at least 32 and at most 256 native hits per
+page, growing through the native result window only when version checks or structured filters reject
+candidates. Exhausting that window because native versions are stale is retryable index lag; filter-
+only exhaustion remains a client error asking for a narrower query. During bounded lag a smaller
+result can still temporarily omit a recently changed record when the native result set ends before
+the search window is exhausted.
+REST exposes the index coverage header, and callers that require current coverage use
+`maxIndexLagMilliseconds: 0` or `waitForIndexMilliseconds`.
+Full-text score descending is the only supported ordering in this release. Count requests return
+`recordCount: null` and `recordCountExact: false`; the native candidate total cannot become an exact
+Harper count after authorization, structured filtering and source-version checks.
+
+A declared Blob source is part of one index document. An oversized, invalid UTF-8 or otherwise
+permanently unusable Blob makes that whole document unindexable rather than publishing a partial
+document with different match semantics. A transient Blob read failure rolls back the accepted
+native batch and replays it. Each source read is time-bounded and replay uses the writer's bounded
+backoff, but an unreadable authoritative record never becomes an index deletion. Publication remains
+behind that record until its source is readable, preserving cursor and publication atomicity at the
+cost of delaying unrelated records in the batch. The normal derived-index lag policy makes prolonged
+failure visible and eventually applies write backpressure rather than silently diverging replicas.
 
 ### Bounded delivery
 
@@ -570,6 +719,9 @@ health transitions. An owner refreshes idle coverage at its flush cadence withou
 release deadline. A query within the certified age bound reads only shared memory and the monotonic
 clock; strict or older queries compare the persisted vector with current physical positions. This also
 certifies an unchanged index after owner release or process restart, when no usable time proof remains.
+An old owner can race with a non-ready transition and restore an earlier coverage time, but consumers
+gate the time on stable ready readiness; after rebuild, the rebuilt anchor still covers that earlier true
+prefix.
 Concurrent waiters on one worker/index share a single 25 ms poll timer, removed when all resolve,
 time out, or abort. A first waiter nudges its local runner without changing writer-lag accounting or
 retrying a deferred batch; an active peer owner already refreshes at flush cadence. No cross-worker

package/resources/indexes/HierarchicalNavigableSmallWorld.ts

@@ -4,6 +4,7 @@ import { FLOAT32_OPTIONS, pack, unpack } from 'msgpackr';
 import { loggerWithTag } from '../../utility/logging/logger.ts';
 import { ClientError, ServerError, DerivedIndexLagError } from '../../utility/errors/hdbError.ts';
 import type { Id } from '../../resources/ResourceInterface.ts';
+import type { CandidateKeyPlan, CandidateKeySet } from '../search.ts';
 import {
 	DEFAULT_MAX_INDEX_LAG_MILLISECONDS,
 	MAX_WAIT_FOR_INDEX_MILLISECONDS,
@@ -269,6 +270,20 @@ const MAX_LEVEL = 10; // should give good high-level skip list performance up to
 // islands are small (bounded by the deleted node's neighborhood), so a probe that visits this many
 // nodes without reaching the entry point is treated as connected-but-far and repaired conservatively.
 const PROBE_VISIT_LIMIT = 256;
+// Candidate keys one predicate visit pays for: a key-only index-entry read against a record load
+// and decode. Measured warm at 0.37 us against 2.1 us, rounded up from that 5.7; the same load is
+// ~130 us cold, so the warm end is the regression-safe calibration — over-stating the ratio would
+// build allow-sets for broad filters the predicate already answers cheaply. See DESIGN.md.
+const KEYS_PER_PREDICATE_VISIT = 8;
+// Absolute ceiling on index entries one query may read building an allow-set, whatever the cost
+// model allows. The scan is synchronous, so this bounds both the event-loop stall and the retained
+// key set; past it the predicate path — which is incremental and budget-bounded — is the safer cost.
+const MAX_CANDIDATE_KEYS = 50_000;
+// Filling ef matches at selectivity s costs ~ef/s visits; this is the slack over that.
+const ALLOW_SET_BUDGET_SLACK = 2;
+// How far an allow-set may widen the filterExpansion visit budget. A visit got cheaper, not free:
+// the JS graph still loads a node and computes a distance for each one, synchronously.
+const ALLOW_SET_BUDGET_MAX_WIDENING = 4;
 type Connection = {
 	id: number;
 	distance: number;
@@ -437,6 +452,10 @@ export class HierarchicalNavigableSmallWorld {
 	// during traversal (predicate-aware / ACORN-style filtering), so companion conditions and RBAC can
 	// be pushed down instead of post-filtering an under-filled candidate set (#1241).
 	filteredSearch = true;
+	// Signals that search() also accepts a CandidateKeyPlan and admits from it directly, so an indexed
+	// companion condition costs a key-only index scan per matching record instead of a record load and
+	// decode per visited node. See DESIGN.md.
+	candidateKeyFilter = true;
 	indexStore: any;
 	M: number = 16; // max number of connections per layer
 	efConstruction: number = 100; // size of dynamic candidate list
@@ -457,6 +476,10 @@ export class HierarchicalNavigableSmallWorld {
 	// latency for recall on selective function predicates; 24 fills to roughly 4% selectivity before
 	// the budget binds. Per-query override via the search options.
 	filterExpansion = 24;
+	// Whether the schema pinned filterExpansion. An explicit pin is an authoritative cost ceiling and
+	// keeps its exact meaning on every path; only an index that never set it gets the
+	// selectivity-derived budget an allow-set makes measurable.
+	filterExpansionConfigured = false;
 
 	idIncrementer: BigInt64Array | undefined;
 	distance: (a: number[], b: number[]) => number;
@@ -534,7 +557,10 @@ export class HierarchicalNavigableSmallWorld {
 			if (configuredEfConstructionSearch !== undefined) this.efConstructionSearch = configuredEfConstructionSearch;
 			if (configuredML !== undefined) this.mL = configuredML;
 			if (configuredOptimizeRouting !== undefined) this.optimizeRouting = configuredOptimizeRouting;
-			if (configuredFilterExpansion !== undefined) this.filterExpansion = configuredFilterExpansion;
+			if (configuredFilterExpansion !== undefined) {
+				this.filterExpansion = configuredFilterExpansion;
+				this.filterExpansionConfigured = true;
+			}
 		}
 		if (options?.nativePlane) {
 			const { nativeM, nativeEfConstruction, nativeML, nativeOptimizeRouting, maxNodes, keyCap, layer0Cap } =
@@ -847,6 +873,52 @@ export class HierarchicalNavigableSmallWorld {
 		});
 	}
 
+	/**
+	 * Run a candidate-key plan when its scans are cheaper than the predicate visits they replace, and
+	 * return the matching primary keys with whether they alone decide admission. Undefined means
+	 * "take the predicate path": the plan was too broad, its narrowest term overran the scan budget,
+	 * or the scan failed — none of which is a fault of the graph.
+	 */
+	private collectCandidateKeys(
+		plan: CandidateKeyPlan,
+		ef: number,
+		predicateMaxVisits: number
+	): { keys: CandidateKeySet; complete: boolean } | undefined {
+		const nodeCount = this.approximateNodeCount();
+		const estimated = Math.max(1, plan.estimatedCount);
+		// Filling ef matches at selectivity s costs ~ef/s visits, and s is estimated/nodeCount, so a
+		// broad filter needs few predicate visits however cheap the test that would replace them.
+		const predicateVisits = Math.min(predicateMaxVisits, (ef * nodeCount) / estimated);
+		let maxKeys = Math.min(Math.ceil(predicateVisits * KEYS_PER_PREDICATE_VISIT), MAX_CANDIDATE_KEYS);
+		// An allow-set over a large fraction of the graph is not a filter worth building.
+		if (nodeCount > 0) maxKeys = Math.min(maxKeys, Math.max(1, nodeCount >> 1));
+		if (estimated > maxKeys) return undefined;
+		try {
+			return plan.collect(maxKeys) ?? undefined;
+		} catch (error) {
+			// A sibling index that started rebuilding or closed between planning and the scan is a
+			// reason to filter the other way, not to fail — and not to reach the caller's handler,
+			// which reads an unclassified search error as plane corruption and requests a rebuild.
+			logger.warn?.('could not build the HNSW candidate-key allow-set; using the predicate path', error);
+			return undefined;
+		}
+	}
+
+	/**
+	 * Visit budget for a traversal an allow-set decides, derived from the selectivity the set makes
+	 * measurable rather than from filterExpansion — which exists to bound an admission test expensive
+	 * enough to be worth bounding, and this one is a set lookup. It only ever widens the
+	 * filterExpansion budget, never narrows it: a cheaper visit makes a tighter budget buy nothing
+	 * and cost recall. An empty set keeps that budget, so a zero-match filter returns empty after a
+	 * bounded descent rather than crawling the graph.
+	 */
+	private allowSetVisitBudget(allowed: number, predicateMaxVisits: number, budgetEf: number): number {
+		if (allowed <= 0) return predicateMaxVisits;
+		const selectivity = Math.min(1, allowed / Math.max(1, this.approximateNodeCount()));
+		const derived = budgetEf * Math.ceil(ALLOW_SET_BUDGET_SLACK / selectivity);
+		return Math.min(Math.max(predicateMaxVisits, derived), predicateMaxVisits * ALLOW_SET_BUDGET_MAX_WIDENING);
+	}
+
 	attachDerivedHost(host: DerivedNativeIndexHost): void {
 		this.derivedHost = host;
 	}
@@ -1796,7 +1868,9 @@ export class HierarchicalNavigableSmallWorld {
 	 * cheap even when the predicate loads a record.
 	 */
 	private admit(filter: (primaryKey: Id) => boolean, filterState: FilterState | undefined, primaryKey: Id): boolean {
-		if (filterState) filterState.filterEvaluations++;
+		// A filter fronted by an allow-set counts its own evaluations: the counter reports how much of
+		// the graph a filtered query had to LOAD, and a membership test loads nothing.
+		if (filterState && !filterState.countsOwnEvaluations) filterState.filterEvaluations++;
 		return filter(primaryKey);
 	}
 
@@ -1839,6 +1913,7 @@ export class HierarchicalNavigableSmallWorld {
 		{
 			filter,
 			minResults,
+			candidateKeys,
 		}: {
 			// Predicate-aware traversal (#1241). When provided, only nodes for which `filter(primaryKey)`
 			// returns true are admitted to the result list at layer 0; routing is unaffected. Composed by
@@ -1850,6 +1925,8 @@ export class HierarchicalNavigableSmallWorld {
 			// (AUTO_EF_MAX) however large the limit was. Raising ef to cover the request keeps `limit`
 			// meaningful; the caller pays for what it asked for.
 			minResults?: number;
+			// Always optional: the predicate path is the fallback for every plan this declines.
+			candidateKeys?: CandidateKeyPlan;
 		} = {}
 	) {
 		const waiting = this.filePrimary && waitForIndexMilliseconds > 0;
@@ -1930,15 +2007,53 @@ export class HierarchicalNavigableSmallWorld {
 		// second-regime search ef (up to AUTO_EF_CEILING) must not silently quadruple the filtered
 		// worst case — the recall decision and the filtered-scan budget are separate decisions. Callers
 		// wanting a deeper filtered search set an explicit ef or filterExpansion, and own the cost.
+		const budgetEf = explicitEf || this.efSearchConfigured ? resolvedEf : Math.min(resolvedEf, AUTO_EF_MAX);
+		const predicateMaxVisits =
+			budgetEf * (filterExpansion && filterExpansion > 0 ? filterExpansion : this.filterExpansion);
+		const allowedKeys =
+			filter && candidateKeys ? this.collectCandidateKeys(candidateKeys, resolvedEf, predicateMaxVisits) : undefined;
+		const allowDecides = Boolean(allowedKeys?.complete);
 		const filterState: FilterState | undefined = filter
 			? {
-					maxVisits:
-						(explicitEf || this.efSearchConfigured ? resolvedEf : Math.min(resolvedEf, AUTO_EF_MAX)) *
-						(filterExpansion && filterExpansion > 0 ? filterExpansion : this.filterExpansion),
+					maxVisits: predicateMaxVisits,
 					nodesVisited: 0,
 					filterEvaluations: 0,
+					candidateKeys: allowedKeys?.keys.size,
 				}
 			: undefined;
+		if (allowedKeys && filter && filterState) {
+			// Both traversals admit by primary key — the plane hands its predicate the key of every
+			// candidate — so the set is a membership test in front of the record load, and a complete
+			// plan removes the load entirely.
+			const allowed = allowedKeys.keys;
+			const admits = (primaryKey: Id) => allowed.has(primaryKey);
+			const residual = filter;
+			const state = filterState;
+			state.countsOwnEvaluations = true;
+			filter = allowDecides
+				? admits
+				: (primaryKey: Id) => {
+						if (!admits(primaryKey)) return false;
+						state.filterEvaluations++;
+						return residual(primaryKey);
+					};
+			// Only a filter the set DECIDES gets the derived budget: an admitted visit on the residual
+			// path still costs a record decode, so that path keeps today's ceiling. A positive per-query
+			// or any schema-configured filterExpansion is an authoritative cost ceiling and is used
+			// exactly as given; a per-query 0 still means unset.
+			if (allowDecides) {
+				const configured =
+					filterExpansion && filterExpansion > 0
+						? filterExpansion
+						: this.filterExpansionConfigured
+							? this.filterExpansion
+							: undefined;
+				state.maxVisits =
+					configured === undefined
+						? this.allowSetVisitBudget(allowedKeys.keys.size, predicateMaxVisits, budgetEf)
+						: budgetEf * configured;
+			}
+		}
 		const rerankK =
 			comparator === 'sort' && minResults !== undefined
 				? Math.min(effectiveEf, Math.max(RERANK_MIN, minResults * RERANK_FACTOR))
@@ -2328,6 +2443,9 @@ type FilterState = {
 	maxVisits: number;
 	nodesVisited: number;
 	filterEvaluations: number;
+	candidateKeys?: number;
+	// The filter counts its own record-loading evaluations, so admit() must not count again.
+	countsOwnEvaluations?: boolean;
 };
 /**
  * Attach filtered-traversal counters to the returned entries array so callers (search.ts / explain) can
@@ -2338,6 +2456,8 @@ function withStats<T extends any[]>(entries: T, filterState: FilterState | undef
 	if (filterState) {
 		Object.defineProperty(entries, 'nodesVisited', { value: filterState.nodesVisited, enumerable: false });
 		Object.defineProperty(entries, 'filterEvaluations', { value: filterState.filterEvaluations, enumerable: false });
+		if (filterState.candidateKeys !== undefined)
+			Object.defineProperty(entries, 'candidateKeys', { value: filterState.candidateKeys, enumerable: false });
 	}
 	return entries;
 }