@harperfast/harper 5.3.0-beta.4 → 5.3.0

package/resources/search.ts

@@ -10,6 +10,10 @@ import { writeKeyId, getReadTransactionGuard } from './DatabaseTransaction.ts';
 import { recordAction } from './analytics/write';
 import { RocksDatabase } from '@harperfast/rocksdb-js';
 import { appendHeader } from '../server/serverHelpers/Headers.ts';
+import type { FullTextCondition } from './indexes/fullTextQueryIndex.ts';
+import { FULL_TEXT_POSITIVE_COMPARATORS, fullTextComparatorMode } from './indexes/fullTextQueryProtocol.ts';
+
+export { fullTextComparatorMode };
 
 // these are ratios/percentages of overall table size
 const OPEN_RANGE_ESTIMATE = 0.3;
@@ -72,7 +76,11 @@ export function executeConditions(
 	filtered,
 	recordAccess?
 ) {
+	conditions = combineFullTextConditions(conditions, operator, table);
 	const firstSearch = conditions[0];
+	// A same-index full-text OR is already one native expression. Route that single traversal through
+	// the AND path so row and vector guards are pushed down before the requested page is filled.
+	if (operator === 'or' && conditions.length === 1 && firstSearch.fullTextQuery) operator = 'and';
 	// Record-level guards (caller-supplied vectorFilter + rowFilter) apply to every record
 	// the query returns, independent of which condition leads (#1241). `recordAccess` is supplied only on
 	// the top-level executeConditions call (Table.search) and deliberately NOT threaded into the recursive
@@ -105,19 +113,29 @@ export function executeConditions(
 		return recordGuards ? transformToEntries(deduped, recordGuards) : deduped;
 	} else {
 		// AND: use the indexed query for the first condition and filter by all subsequent conditions.
-		if (isFilterablePushdown(firstSearch, table)) {
+		const pushdownIndex = filterablePushdownIndex(firstSearch, table);
+		if (pushdownIndex) {
 			// The lead is a filterable custom index (HNSW). It doesn't populate the join map (`filtered`)
 			// that sibling filters read, so — unlike the general case below — we can build the sibling
 			// filters BEFORE executing the lead and push them into the traversal, so filtering happens
 			// during the graph walk instead of post-filtering an under-filled candidate set (#1241). They
 			// also stay in the post-filter chain as a deterministic, harmless re-check.
-			const filters = mapConditionsToFilters(conditions.slice(1), true, firstSearch.estimated_count);
+			const siblings = conditions.slice(1);
+			const filters = mapConditionsToFilters(siblings, true, firstSearch.estimated_count);
 			const recordFilters = recordGuards ? filters.concat(recordGuards) : filters;
+			// The sibling conditions a secondary-index range scan answers exactly can seed an allow-set
+			// the index admits from, instead of loading and decoding a record at every visited node.
+			// The post-filter chain below stays whole, so the set only has to be a SUPERSET of the true
+			// matches — the index decides whether building it beats evaluating the predicate.
+			const candidateKeys =
+				pushdownIndex.candidateKeyFilter && siblings.length > 0
+					? planCandidateKeys(siblings, table, txn, !recordGuards)
+					: undefined;
 			// Execute a COPY carrying the pushed-down predicate, never mutating the caller's condition
 			// object (which may be reused across requests with different users / guards).
 			const lead =
 				recordFilters.length > 0
-					? { ...firstSearch, recordFilter: composeRecordFilter(recordFilters, table, context) }
+					? { ...firstSearch, recordFilter: composeRecordFilter(recordFilters, table, context), candidateKeys }
 					: firstSearch;
 			const results = executeCondition(lead);
 			return recordFilters.length > 0 ? transformToEntries(results, recordFilters) : results;
@@ -148,6 +166,7 @@ export function executeConditions(
 			context,
 			minResults:
 				request.limit === 0 ? 0 : request.limit !== undefined ? (request.offset || 0) + request.limit : undefined,
+			resultOffset: request.offset || 0,
 		});
 	}
 	function mapConditionsToFilters(conditions, intersection, estimatedIncomingCount) {
@@ -193,12 +212,164 @@ function buildRecordGuards(recordAccess): ((record: any) => boolean)[] | undefin
 	return guards.length > 0 ? guards : undefined;
 }
 
-/** True when a condition's index is a custom index that participates in predicate-aware traversal (HNSW). */
-function isFilterablePushdown(condition, table): boolean {
+function conditionIndex(table: any, attributeName: any, preferFullText = false): any {
+	if (Array.isArray(attributeName) && attributeName.length === 1) attributeName = attributeName[0];
+	if (typeof attributeName !== 'string') return;
+	return preferFullText ? table.fullTextQueryIndexes?.[attributeName] : table.indices?.[attributeName];
+}
+
+function combineFullTextConditions(conditions: any[], operator: string | undefined, table: any): any[] {
+	if (table.hasFullTextQueryIndexes === false || !containsFullTextCondition(conditions, table)) return conditions;
+	let indexName: string | undefined;
+	let positive = false;
+	const expressions: any[] = [];
+	const leaves: FullTextCondition['fullTextLeaves'] = [];
+	const remaining: any[] = [];
+	let includeHighlights = false;
+	let maxIndexLagMilliseconds: number | undefined;
+	let waitForIndexMilliseconds: number | undefined;
+	for (const condition of conditions) {
+		const compiled = compileFullTextExpression(condition, table);
+		if (!compiled) {
+			remaining.push(condition);
+			continue;
+		}
+		if (operator === 'or' && compiled.remaining?.length)
+			throw new ClientError('A full-text OR group cannot mix full-text and record conditions', 400);
+		if (indexName && indexName !== compiled.indexName)
+			throw new ClientError('One query cannot combine conditions from different full-text indexes', 400);
+		indexName = compiled.indexName;
+		expressions.push(compiled.expression);
+		leaves.push(...compiled.leaves);
+		positive ||= compiled.positive;
+		includeHighlights ||= compiled.includeHighlights;
+		maxIndexLagMilliseconds = mergeFullTextOption(
+			'maxIndexLagMilliseconds',
+			maxIndexLagMilliseconds,
+			compiled.maxIndexLagMilliseconds
+		);
+		waitForIndexMilliseconds = mergeFullTextOption(
+			'waitForIndexMilliseconds',
+			waitForIndexMilliseconds,
+			compiled.waitForIndexMilliseconds
+		);
+		if (compiled.remaining) remaining.push(...compiled.remaining);
+	}
+	if (!indexName) return conditions;
+	if (operator === 'or' && remaining.length > 0)
+		throw new ClientError('A full-text OR group can only contain conditions on the same full-text index', 400);
+	if (!positive) throw new ClientError('A full-text query requires at least one non-negated condition', 400);
+	const expression =
+		expressions.length === 1 ? expressions[0] : { operator: operator === 'or' ? 'or' : 'and', clauses: expressions };
+	const combined = {
+		attribute: indexName,
+		comparator: 'matches',
+		value: '',
+		fullTextQuery: expression,
+		fullTextLeaves: leaves,
+		includeHighlights,
+		...(maxIndexLagMilliseconds === undefined ? null : { maxIndexLagMilliseconds }),
+		...(waitForIndexMilliseconds === undefined ? null : { waitForIndexMilliseconds }),
+	};
+	return operator === 'or' ? [combined] : [combined, ...remaining];
+}
+
+function containsFullTextCondition(conditions: any[], table: any): boolean {
+	for (const condition of conditions) {
+		if (condition.conditions) {
+			if (containsFullTextCondition(condition.conditions, table)) return true;
+			continue;
+		}
+		const indexName = condition[0] ?? condition.attribute;
+		if (
+			typeof indexName === 'string' &&
+			table.fullTextQueryIndexes?.[indexName] &&
+			fullTextComparatorMode(condition.comparator)
+		)
+			return true;
+	}
+	return false;
+}
+
+function compileFullTextExpression(
+	condition: any,
+	table: any
+):
+	| {
+			indexName: string;
+			expression: any;
+			leaves: NonNullable<FullTextCondition['fullTextLeaves']>;
+			positive: boolean;
+			includeHighlights: boolean;
+			maxIndexLagMilliseconds?: number;
+			waitForIndexMilliseconds?: number;
+			remaining?: any[];
+	  }
+	| undefined {
+	if (condition.conditions) {
+		const children = condition.conditions.map((child) => compileFullTextExpression(child, table));
+		const fullTextChildren = children.filter(Boolean) as Array<NonNullable<(typeof children)[number]>>;
+		if (fullTextChildren.length === 0) return;
+		const groupOperator = condition.operator === 'or' ? 'or' : 'and';
+		const remaining = condition.conditions.filter((_child, index) => !children[index]);
+		for (const child of fullTextChildren) if (child.remaining) remaining.push(...child.remaining);
+		if (groupOperator === 'or' && remaining.length > 0)
+			throw new ClientError('A full-text OR group cannot mix full-text and record conditions', 400);
+		const indexName = fullTextChildren[0].indexName;
+		if (fullTextChildren.some((child) => child.indexName !== indexName))
+			throw new ClientError('One query cannot combine conditions from different full-text indexes', 400);
+		return {
+			indexName,
+			expression: { operator: groupOperator, clauses: fullTextChildren.map((child) => child.expression) },
+			leaves: fullTextChildren.flatMap((child) => child.leaves),
+			positive: fullTextChildren.some((child) => child.positive),
+			includeHighlights: fullTextChildren.some((child) => child.includeHighlights),
+			maxIndexLagMilliseconds: fullTextChildren.reduce(
+				(value, child) => mergeFullTextOption('maxIndexLagMilliseconds', value, child.maxIndexLagMilliseconds),
+				undefined as number | undefined
+			),
+			waitForIndexMilliseconds: fullTextChildren.reduce(
+				(value, child) => mergeFullTextOption('waitForIndexMilliseconds', value, child.waitForIndexMilliseconds),
+				undefined as number | undefined
+			),
+			...(remaining.length > 0 ? { remaining } : null),
+		};
+	}
+	const indexName = condition[0] ?? condition.attribute;
+	if (typeof indexName !== 'string' || !table.fullTextQueryIndexes?.[indexName]) return;
+	const mode = fullTextComparatorMode(condition.comparator);
+	if (!mode) return;
+	if (typeof (condition[1] ?? condition.value) !== 'string')
+		throw new ClientError(`Full-text index '${indexName}' requires a full-text comparator and string value`, 400);
+	const text = condition[1] ?? condition.value;
+	const leaf = { text, mode, ...(condition.fields ? { fields: condition.fields } : null) };
+	return {
+		indexName,
+		expression: condition.negated ? { operator: 'not', clause: leaf } : leaf,
+		leaves: condition.negated ? [] : [leaf],
+		positive: !condition.negated,
+		includeHighlights: condition.includeHighlights === true,
+		maxIndexLagMilliseconds: condition.maxIndexLagMilliseconds,
+		waitForIndexMilliseconds: condition.waitForIndexMilliseconds,
+	};
+}
+
+function mergeFullTextOption(name: string, current: number | undefined, next: number | undefined): number | undefined {
+	if (next === undefined) return current;
+	if (current !== undefined && current !== next)
+		throw new ClientError(`Combined full-text conditions must use the same ${name}`, 400);
+	return next;
+}
+
+function filterablePushdownIndex(condition, table): any {
 	const attributeName = condition?.attribute ?? condition?.[0];
-	if (attributeName == null || table == null) return false;
-	const index = attributeName === table.primaryKey ? table.primaryStore : table.indices?.[attributeName];
-	return Boolean(index?.customIndex?.filteredSearch);
+	if (attributeName == null || table == null) return undefined;
+	const fullTextMode = Boolean(fullTextComparatorMode(condition?.comparator));
+	const index =
+		attributeName === table.primaryKey && !fullTextMode
+			? table.primaryStore
+			: conditionIndex(table, attributeName, fullTextMode);
+	return index?.customIndex?.filteredSearch ? index.customIndex : undefined;
 }
 
 /**
@@ -207,14 +378,14 @@ function isFilterablePushdown(condition, table): boolean {
  * `primaryStore.getEntry` and frozen before the predicates see it. Verdicts are memoized per query since
  * the graph can reach a node from multiple neighbors. A missing/deleted record fails the predicate.
  */
-function composeRecordFilter(recordFilters, table, context): (primaryKey: Id) => boolean {
+function composeRecordFilter(recordFilters, table, context): (primaryKey: Id, suppliedEntry?: any) => boolean {
 	const memo = new Map<Id, boolean>();
 	const transaction = context && table._readTxnForContext ? table._readTxnForContext(context) : undefined;
-	return (primaryKey: Id) => {
+	return (primaryKey: Id, suppliedEntry?: any) => {
 		const cached = memo.get(primaryKey);
 		if (cached !== undefined) return cached;
 		let verdict = false;
-		const entry = table.primaryStore.getEntry(primaryKey, { transaction });
+		const entry = suppliedEntry ?? table.primaryStore.getEntry(primaryKey, { transaction });
 		const record = entry?.value;
 		if (record != null) {
 			freezeRecord(record);
@@ -231,6 +402,279 @@ function composeRecordFilter(recordFilters, table, context): (primaryKey: Id) =>
 	};
 }
 
+/**
+ * The candidate-key plan a filterable custom index receives alongside its pushed-down predicate:
+ * the sibling AND conditions this query can answer from secondary-index range scans alone.
+ * The index admits from the keys instead of re-deriving the condition per visited node, so a
+ * condition on an indexed attribute costs a key-only index-entry read per matching record rather
+ * than a record load and decode per node the traversal reaches.
+ */
+export interface CandidateKeyPlan {
+	/** Planner estimate of the matching record count; the index sizes its work budget from it. */
+	estimatedCount: number;
+	/**
+	 * The matching primary keys, with `complete` true when the set alone decides admission, so the
+	 * index may drop the traversal predicate. That needs every pushed-down condition covered, no
+	 * opaque record guard (`rowFilter` / `vectorFilter`), and every term actually scanned — a term
+	 * too wide for `maxKeys` is skipped rather than abandoning the whole set, leaving a narrower
+	 * superset that gates the predicate instead of replacing it. Null when even the narrowest term
+	 * overruns, which means the whole plan was not worth what it read.
+	 */
+	collect(maxKeys: number): { keys: CandidateKeySet; complete: boolean } | null;
+}
+
+/**
+ * Primary keys by storage identity. Strings and non-zero numbers are their own identity, which spares
+ * encoding the common key shapes, and the stores encode -0 and 0 apart where Set membership does not.
+ * The rest go by writeKeyId, the encoder the stores index by — flattenKey joins composite keys with
+ * NUL and collapses ['a', 'b\0c'] and ['a\0b', 'c'] — kept apart from the values because an encoded
+ * identity is a string that can equal a string key: writeKeyId(['a', 'b']) is 'a\0b'. An index scan
+ * decodes an int64 as the number it encodes as when that is exact, while a traversal may report the
+ * BigInt, so the two are the same key.
+ */
+export class CandidateKeySet {
+	#byValue = new Set<Id>();
+	#encoded = new Set<unknown>();
+	#zero = false;
+	#negativeZero = false;
+	#holdsBigInt = false;
+	add(primaryKey: Id): void {
+		if (typeof primaryKey === 'string' || (typeof primaryKey === 'number' && primaryKey !== 0))
+			this.#byValue.add(primaryKey);
+		else if (primaryKey === 0) {
+			if (Object.is(primaryKey, -0)) this.#negativeZero = true;
+			else this.#zero = true;
+		} else {
+			if (typeof primaryKey === 'bigint') this.#holdsBigInt = true;
+			this.#encoded.add(writeKeyId(primaryKey));
+		}
+	}
+	has(primaryKey: Id): boolean {
+		if (typeof primaryKey === 'string') return this.#byValue.has(primaryKey);
+		if (typeof primaryKey === 'number') {
+			if (primaryKey === 0) return Object.is(primaryKey, -0) ? this.#negativeZero : this.#zero;
+			return this.#byValue.has(primaryKey) || (this.#holdsBigInt && this.#encoded.has(writeKeyId(primaryKey)));
+		}
+		if (typeof primaryKey === 'bigint') {
+			const number = Number(primaryKey);
+			if (BigInt(number) === primaryKey && (number === 0 ? this.#zero : this.#byValue.has(number))) return true;
+		}
+		return this.#encoded.has(writeKeyId(primaryKey));
+	}
+	get size(): number {
+		return this.#byValue.size + this.#encoded.size + (this.#zero ? 1 : 0) + (this.#negativeZero ? 1 : 0);
+	}
+}
+
+interface CandidateKeyScan {
+	index: any;
+	range: any;
+	estimatedCount: number;
+}
+/** Scans whose UNION is one AND term; a single-element `scans` is a leaf condition. */
+interface CandidateKeyTerm {
+	scans: CandidateKeyScan[];
+	estimatedCount: number;
+}
+
+// Comparators whose secondary-index range is EXACTLY the condition over the values an index holds,
+// so scanning that range yields every indexed match and nothing else. `ne`, `in`, `contains`,
+// `ends_with` and any negated condition are excluded because searchByIndex answers them with a scan
+// plus a record filter.
+const CANDIDATE_KEY_COMPARATORS = new Set([
+	'equals',
+	'lt',
+	'le',
+	'gt',
+	'ge',
+	'between',
+	'gele',
+	'gelt',
+	'gtlt',
+	'gtle',
+]);
+
+/**
+ * Plan one leaf condition as an index range scan, or undefined when its matches cannot be read off
+ * an index exactly, or when the scan's size cannot be estimated.
+ */
+function planCandidateKeyScan(condition, table): CandidateKeyScan | undefined {
+	if (condition.negated) return undefined;
+	const attributeName = condition.attribute ?? condition[0];
+	if (typeof attributeName !== 'string' || attributeName === table.primaryKey) return undefined;
+	const index = usableIndex(table, attributeName);
+	if (!index || index.customIndex) return undefined;
+	const comparator = ALTERNATE_COMPARATOR_NAMES[condition.comparator] ?? condition.comparator ?? 'equals';
+	if (!CANDIDATE_KEY_COMPARATORS.has(comparator)) return undefined;
+	// The scan has to hold every key the record predicate admits, or results would depend on whether
+	// the plan was built. The index cannot supply a record lacking the attribute, a null one unless it
+	// indexes nulls, or one written as -0: ordered-binary encodes -0 past every number, and that entry's
+	// primary key decodes wrong. compareKeys ranks undefined below every scalar, so `rank < 100` admits
+	// a record with no rank at all; a condition admitting any of these is left to the predicate.
+	const predicate = filterByType(condition, table, undefined, null);
+	if (predicate({}) || predicate({ [attributeName]: 0 }) || (!index.indexNulls && predicate({ [attributeName]: null })))
+		return undefined;
+	let value = condition[1] ?? condition.value;
+	if (value instanceof Date) value = value.getTime();
+	let start;
+	let end;
+	let inclusiveEnd = false;
+	let exclusiveStart = false;
+	switch (comparator) {
+		case 'lt':
+			// null, not searchByIndex's `true`: `true` sorts ABOVE null and false, so starting there
+			// would drop indexed rows the record predicate's compareKeys admits — an omission the
+			// post-filter cannot undo.
+			start = null;
+			end = value;
+			break;
+		case 'le':
+			start = null;
+			end = value;
+			inclusiveEnd = true;
+			break;
+		case 'gt':
+			start = value;
+			exclusiveStart = true;
+			break;
+		case 'ge':
+			start = value;
+			break;
+		case 'between':
+		case 'gele':
+		case 'gelt':
+		case 'gtlt':
+		case 'gtle':
+			if (!Array.isArray(value)) return undefined;
+			start = value[0] instanceof Date ? value[0].getTime() : value[0];
+			end = value[1] instanceof Date ? value[1].getTime() : value[1];
+			inclusiveEnd = comparator === 'gele' || comparator === 'gtle' || comparator === 'between';
+			exclusiveStart = comparator === 'gtlt' || comparator === 'gtle';
+			break;
+		default:
+			start = value;
+			end = value;
+			inclusiveEnd = true;
+	}
+	// ordered-binary encodes -0 above every number, where the predicate's compareKeys calls it 0.
+	if (start === 0) start = 0;
+	if (end === 0) end = 0;
+	// An over-length string key is stored truncated behind an overflow marker, which makes the range
+	// wider than the condition and needs a record filter to narrow again.
+	if (
+		(typeof start === 'string' && start.length > MAX_SEARCH_KEY_LENGTH) ||
+		(typeof end === 'string' && end.length > MAX_SEARCH_KEY_LENGTH)
+	)
+		return undefined;
+	if (condition.estimated_count === undefined) estimateCondition(table)(condition);
+	let estimatedCount = condition.estimated_count;
+	if (!(estimatedCount >= 0) || !Number.isFinite(estimatedCount)) return undefined;
+	// estimateRangeCondition measures the range searchByIndex would run, which starts at `true`. Only
+	// lt/le widen below that, and the block they add is read FIRST and can dwarf the rest of the
+	// range, so it has to be counted or the scan overruns a budget sized without it. A statistical
+	// estimate, never getValuesCount: that one steps the whole block, which is the unbounded
+	// per-query cost this is here to avoid. The other comparators' own estimates already cover
+	// whatever they start at.
+	if (comparator === 'lt' || comparator === 'le') {
+		let belowTrue: number | undefined;
+		try {
+			belowTrue = index.estimateCount?.({ start: null, end: true })?.count;
+		} catch {
+			return undefined;
+		}
+		if (!(belowTrue >= 0)) return undefined;
+		estimatedCount += belowTrue;
+	}
+	return { index, range: { start, end, inclusiveEnd, exclusiveStart }, estimatedCount };
+}
+
+function planCandidateKeys(conditions, table, transaction, guardFree: boolean): CandidateKeyPlan | undefined {
+	const terms: CandidateKeyTerm[] = [];
+	const planned = collectCandidateKeyTerms(conditions, table, terms) && guardFree;
+	if (terms.length === 0) return undefined;
+	const estimatedCount = terms.reduce((narrowest, term) => Math.min(narrowest, term.estimatedCount), Infinity);
+	return {
+		estimatedCount,
+		collect: (maxKeys) => collectCandidateKeys(terms, transaction, maxKeys, planned),
+	};
+}
+
+/** Returns whether every condition was covered. */
+function collectCandidateKeyTerms(conditions, table, terms: CandidateKeyTerm[]): boolean {
+	let complete = true;
+	for (const condition of conditions) {
+		if (condition.conditions) {
+			if (condition.operator === 'or') {
+				// A union is only sound in full: dropping one branch would exclude the records only
+				// that branch matches, and admission may over-admit but never omit.
+				const scans: CandidateKeyScan[] = [];
+				for (const child of condition.conditions) {
+					const scan = child.conditions ? undefined : planCandidateKeyScan(child, table);
+					if (!scan) {
+						complete = false;
+						break;
+					}
+					scans.push(scan);
+				}
+				if (scans.length === condition.conditions.length) {
+					terms.push({ scans, estimatedCount: scans.reduce((total, scan) => total + scan.estimatedCount, 0) });
+				}
+			} else if (!collectCandidateKeyTerms(condition.conditions, table, terms)) complete = false;
+			continue;
+		}
+		const scan = planCandidateKeyScan(condition, table);
+		if (scan) terms.push({ scans: [scan], estimatedCount: scan.estimatedCount });
+		else complete = false;
+	}
+	return complete;
+}
+
+/**
+ * Run the planned scans narrowest-estimate first, intersecting on storage key identity. `maxKeys`
+ * bounds index entries READ, not keys retained — a wide later scan whose intersection is tiny still
+ * reads its whole range, and that is the cost the caller budgeted against. A term whose estimate
+ * exceeds what is left of that budget is dropped instead: fewer AND terms is a wider superset, which
+ * the predicate then narrows, and spending the budget on it only to abandon the set is pure loss.
+ */
+function collectCandidateKeys(
+	terms: CandidateKeyTerm[],
+	transaction,
+	maxKeys: number,
+	planned: boolean
+): { keys: CandidateKeySet; complete: boolean } | null {
+	const ordered = terms.length > 1 ? terms.slice().sort((a, b) => a.estimatedCount - b.estimatedCount) : terms;
+	let matched: CandidateKeySet | undefined;
+	let complete = planned;
+	let scanned = 0;
+	for (const term of ordered) {
+		if (matched && term.estimatedCount > maxKeys - scanned) {
+			complete = false;
+			continue;
+		}
+		const next = new CandidateKeySet();
+		let overran = false;
+		for (const scan of term.scans) {
+			for (const { value: primaryKey } of scan.index.getRange({ ...scan.range, values: true, transaction })) {
+				if (++scanned > maxKeys) {
+					overran = true;
+					break;
+				}
+				if (!matched || matched.has(primaryKey)) next.add(primaryKey);
+			}
+			if (overran) break;
+		}
+		// A half-scanned term is not a superset of its own matches, so it is dropped whole.
+		if (overran) {
+			if (!matched) return null;
+			complete = false;
+			break;
+		}
+		matched = next;
+		if (matched.size === 0) break;
+	}
+	return { keys: matched!, complete };
+}
+
 /**
  * A record's index entries are not adjacent — the composite `[indexedValue, primaryKey]` key sorts
  * on the indexed value first — so comparing neighbours cannot collapse them and the scan has to
@@ -289,16 +733,22 @@ export function searchByIndex(
 		// approximate index returns a fixed-size candidate list, so without this a query asking for more
 		// rows than that list holds silently gets a short result set. Only custom indexes read it.
 		minResults?: number;
+		// Rows before this offset are needed for correct paging but do not need highlight metadata.
+		resultOffset?: number;
 	} = {}
 ): AsyncIterable<Id | { key: Id; value: any }> {
 	// A stale positional caller passes `allowFullScan` here. Type checking only covers .ts callers, so
 	// fail loud rather than silently reading every option as undefined.
 	if (typeof options !== 'object' || options === null)
 		throw new TypeError('searchByIndex: the 5th argument is an options object (#2165), not a positional value');
-	const { allowFullScan, filtered, context, minResults } = options;
+	const { allowFullScan, filtered, context, minResults, resultOffset } = options;
 	let attribute_name = searchCondition[0] ?? searchCondition.attribute;
 	let value = searchCondition[1] ?? searchCondition.value;
 	const comparator = searchCondition.comparator;
+	const fullTextMode = fullTextComparatorMode(comparator);
+	if (fullTextMode && typeof attribute_name !== 'string')
+		throw new ClientError('Full-text comparator requires an index directly on the queried table', 400);
+	if (!fullTextMode) Table.assertFullTextRecordField?.(attribute_name);
 	if (value === undefined && (comparator as any) !== 'sort') {
 		throw new ClientError(`Search condition for ${attribute_name} must have a value`);
 	}
@@ -368,8 +818,14 @@ export function searchByIndex(
 			needFullScan = true;
 		}
 	}
-	const isPrimaryKey = attribute_name === Table.primaryKey || attribute_name == null;
-	const index = isPrimaryKey ? Table.primaryStore : Table.indices[attribute_name];
+	if (fullTextMode) {
+		if (typeof attribute_name !== 'string' || !Table.fullTextIndexes?.some(({ name }) => name === attribute_name))
+			throw new ClientError('Full-text comparator requires a declared @fullText index', 400);
+		if (!Table.fullTextQueryIndexes?.[attribute_name]?.customIndex)
+			throw new IndexRebuildingError(`Full-text index '${attribute_name}' is not ready`);
+	}
+	const isPrimaryKey = !fullTextMode && (attribute_name === Table.primaryKey || attribute_name == null);
+	const index = isPrimaryKey ? Table.primaryStore : conditionIndex(Table, attribute_name, Boolean(fullTextMode));
 	let start;
 	let end, inclusiveEnd, exclusiveStart, stringPrefix;
 	if (value instanceof Date) value = value.getTime();
@@ -379,6 +835,8 @@ export function searchByIndex(
 		// the bounded index iteration would only visit *included* rows.
 		start = true;
 		needFullScan = true;
+	} else if (fullTextMode) {
+		start = true;
 	} else
 		switch (ALTERNATE_COMPARATOR_NAMES[comparator] || comparator) {
 			case 'lt':
@@ -554,6 +1012,7 @@ export function searchByIndex(
 			// exploring until it has enough MATCHING results, rather than post-filtering an under-filled
 			// candidate set. Only indexes that opt in (filteredSearch) receive it; others post-filter as before.
 			const recordFilter = index.customIndex.filteredSearch ? searchCondition.recordFilter : undefined;
+			const candidateKeys = index.customIndex.candidateKeyFilter ? (searchCondition as any).candidateKeys : undefined;
 			const waiting = index.customIndex.filePrimary && searchCondition.waitForIndexMilliseconds > 0;
 			const start = waiting ? Promise.withResolvers<void>() : undefined;
 			const controller = waiting ? new AbortController() : undefined;
@@ -563,22 +1022,27 @@ export function searchByIndex(
 					? AbortSignal.any([context.signal, controller.signal])
 					: controller.signal
 				: undefined;
+			const indexSearchStart = waiting ? start.promise.then(() => checkActive?.()) : undefined;
+			indexSearchStart?.catch(() => {});
 			const searchContext = waiting
 				? Object.create(context, {
 						signal: { value: signal },
-						indexSearchStart: { value: start.promise.then(() => checkActive?.()) },
+						indexSearchStart: { value: indexSearchStart },
 					})
 				: context;
 			const searched = index.customIndex.search(searchCondition, searchContext, {
 				filter:
 					waiting && recordFilter
-						? (id) => {
+						? (id, entry) => {
 								if (signal.aborted) return false;
 								checkActive?.();
-								return recordFilter(id);
+								return recordFilter(id, entry);
 							}
 						: recordFilter,
 				minResults,
+				resultOffset,
+				assertTransactionActive: checkActive,
+				candidateKeys,
 			});
 			const coverage = (searched as any).indexCoverage;
 			if (!waiting && coverage && context?.responseHeaders) {
@@ -595,11 +1059,13 @@ export function searchByIndex(
 					.map((entry) => {
 						// if the custom index returns an entry with metadata, merge it with the loaded entry
 						if (typeof entry === 'object' && entry) {
-							const { key, ...otherProps } = entry;
+							const { key, loadedEntry: suppliedEntry, ...otherProps } = entry;
 							if (key == null) return SKIP; // primaryKey missing from HNSW node — skip rather than crash
-							const loadedEntry = Table.primaryStore.getEntry(key, {
-								transaction: waiting ? transaction : context && Table._readTxnForContext(context),
-							});
+							const loadedEntry =
+								suppliedEntry ??
+								Table.primaryStore.getEntry(key, {
+									transaction: waiting ? transaction : context && Table._readTxnForContext(context),
+								});
 							if (!loadedEntry) return SKIP; // record was deleted/expired or not yet visible
 							freezeRecord(loadedEntry?.value);
 							recordRead(loadedEntry);
@@ -653,7 +1119,10 @@ export function searchByIndex(
 						},
 						return(value?: any) {
 							closed = true;
-							if (!settled) controller?.abort();
+							if (!settled) {
+								start?.resolve();
+								controller?.abort();
+							}
 							iteratorPromise.then(
 								(inner) => (inner as any).return?.(value),
 								() => {}
@@ -919,7 +1388,15 @@ const LIST_VALUE_COMPARATORS = new Set(['in', 'between']);
 
 // Base comparators that accept the `not_` prefix to produce a negated form.
 // `not_equal` is an existing alias for `ne` and keeps its existing semantics.
-const NEGATABLE_BASE_COMPARATORS = new Set(['in', 'between', 'starts_with', 'ends_with', 'contains', 'equals']);
+const NEGATABLE_BASE_COMPARATORS = new Set([
+	'in',
+	'between',
+	'starts_with',
+	'ends_with',
+	'contains',
+	'equals',
+	...FULL_TEXT_POSITIVE_COMPARATORS,
+]);
 
 /**
  * Resolve a comparator name to a (possibly stripped) base comparator and a
@@ -1342,7 +1819,7 @@ function estimateRangeCondition(table, condition, searchType, fraction) {
 
 /** The index a condition can be driven by; searchByIndex refuses a rebuilding one, so it reads as absent. */
 function usableIndex(table, attributeName): any {
-	const index = attributeName == null ? undefined : table.indices[attributeName];
+	const index = attributeName == null ? undefined : conditionIndex(table, attributeName);
 	return index?.isIndexing ? undefined : index;
 }
 
@@ -1352,7 +1829,9 @@ function usableIndex(table, attributeName): any {
  */
 function drivesRebuildingIndex(table, attributeName, relationshipOffset = 0): boolean {
 	if (!Array.isArray(attributeName))
-		return attributeName != null && attributeName !== table.primaryKey && !!table.indices[attributeName]?.isIndexing;
+		return (
+			attributeName != null && attributeName !== table.primaryKey && !!conditionIndex(table, attributeName)?.isIndexing
+		);
 	if (relationshipOffset >= attributeName.length - 1)
 		return drivesRebuildingIndex(table, attributeName[relationshipOffset]);
 	const attribute = findAttribute(table.attributes, attributeName[relationshipOffset]);
@@ -1430,6 +1909,12 @@ export function estimateCondition(table) {
 					const index = usableIndex(table, attribute_name);
 					condition.estimated_count = index ? index.getValuesCount(condition[1] ?? condition.value) : Infinity;
 				}
+			} else if (fullTextComparatorMode(searchType)) {
+				const attributeName = condition[0] ?? condition.attribute;
+				const index = conditionIndex(table, attributeName, true);
+				condition.estimated_count = index?.customIndex?.estimateCount
+					? index.customIndex.estimateCount(condition.value)
+					: Infinity;
 			} else if (searchType === 'contains' || searchType === 'ends_with' || searchType === 'ne') {
 				const attribute_name = condition[0] ?? condition.attribute;
 				const index = usableIndex(table, attribute_name);
@@ -1557,7 +2042,9 @@ function parseBlock(query, expectedEnd) {
 					// caught at execution time with a clearer error.
 					if (FIQL_OPERATOR_NAME.test(value)) comparator = value;
 					else recordError(`invalid FIQL operator ${value}`);
-					valueDecoder = typedDecoding; // use typed/auto-cast decoding for FIQL operators
+					valueDecoder = fullTextComparatorMode(resolveComparator(comparator).comparator)
+						? decodeURIComponent
+						: typedDecoding;
 				} else {
 					// standard equal comparison
 					valueDecoder = decodeURIComponent; // use strict decoding