@harperfast/harper 5.3.0-beta.4 → 5.3.0

package/resources/derivedIndexRuntime.ts

@@ -120,6 +120,8 @@ export interface DerivedIndexBackendHost {
 	/** True while `epoch` is the most recently minted owner epoch for this backend. */
 	isOwnerEpoch(epoch: bigint): boolean;
 	getReadiness(): DerivedIndexReadiness;
+	/** Publish a durable backend-only revision that peer workers must observe. */
+	publicationChanged(): void;
 }
 
 /**
@@ -131,6 +133,8 @@ export interface DerivedIndexBackendHost {
  */
 export interface DerivedIndexBackend {
 	readonly id: string;
+	/** Allocate and publish cross-worker query revisions for backends with external reader state. */
+	readonly publishesQueryRevisions?: boolean;
 	/** Receives the epoch fence and readiness reader before any delivery. */
 	attach(host: DerivedIndexBackendHost): void;
 	getDurableCursor(): DerivedIndexCursor | undefined;
@@ -345,8 +349,7 @@ export class DerivedIndexRuntime {
 				this.#runners.delete(registration.backend.id);
 				const waiting = this.#coverageWaits.get(registration.backend.id);
 				if (waiting) {
-					for (const waiter of waiting.waiters)
-						waiter.finish(new ServerError('The native HNSW index is unavailable', 503));
+					for (const waiter of waiting.waiters) waiter.finish(new ServerError('The derived index is unavailable', 503));
 				}
 				this.#stopListeningIfIdle();
 			}
@@ -387,7 +390,7 @@ export class DerivedIndexRuntime {
 
 	waitForCoverage(backendId: string, since: bigint, timeout: number, signal?: AbortSignal): Promise<void> {
 		if (this.#stopped || !this.#runners.has(backendId))
-			return Promise.reject(new ServerError('The native HNSW index is unavailable', 503));
+			return Promise.reject(new ServerError('The derived index is unavailable', 503));
 		let group = this.#coverageWaits.get(backendId);
 		const first = !group;
 		if (!group) this.#coverageWaits.set(backendId, (group = { waiters: new Set() }));
@@ -432,7 +435,7 @@ export class DerivedIndexRuntime {
 			const after = readReadiness(views);
 			if (after.state !== 'ready')
 				throw new ServerError(
-					`The native HNSW index is ${after.state === 'unavailable' ? 'unavailable' : 'rebuilding'}`,
+					`The derived index is ${after.state === 'unavailable' ? 'unavailable' : 'rebuilding'}`,
 					503
 				);
 			const now = derivedIndexTime(this.#logStore.rootStore);
@@ -440,7 +443,7 @@ export class DerivedIndexRuntime {
 			for (const waiter of group.waiters) {
 				if (before.state === 'ready' && before.ownerEpoch === after.ownerEpoch && time >= waiter.since) waiter.finish();
 				else if (now >= waiter.deadline)
-					waiter.finish(new DerivedIndexLagError('Timed out waiting for native HNSW index coverage; retry this query'));
+					waiter.finish(new DerivedIndexLagError('Timed out waiting for derived index coverage; retry this query'));
 				else delay = Math.min(delay, Math.max(1, Number(waiter.deadline - now) / 1e6));
 			}
 			if (group.waiters.size) group.timer = setTimeout(() => this.#pollCoverage(backendId, group), delay);
@@ -482,7 +485,7 @@ export class DerivedIndexRuntime {
 		if (this.#stopping) return this.#stopping;
 		this.#stopped = true;
 		for (const group of this.#coverageWaits.values()) {
-			for (const waiter of group.waiters) waiter.finish(new ServerError('The native HNSW index is unavailable', 503));
+			for (const waiter of group.waiters) waiter.finish(new ServerError('The derived index is unavailable', 503));
 		}
 		for (const runner of this.#runners.values()) this.#track(runner, runner.stop());
 		this.#runners.clear();
@@ -649,7 +652,9 @@ class DerivedIndexRunner {
 	#ownerEpoch?: bigint;
 	#readinessBuffer: SharedReadinessBuffer;
 	#wakeBuffer: SharedReadinessBuffer;
+	#publicationBuffer?: SharedReadinessBuffer;
 	#sharedViews: SharedViews;
+	#publicationRevision?: BigInt64Array;
 	#resetting?: Promise<void>;
 	status: DerivedIndexRunnerStatus = { state: 'idle' };
 
@@ -703,10 +708,15 @@ class DerivedIndexRunner {
 		);
 		this.#wakeBuffer = this.readinessId === this.id ? this.#readinessBuffer : wakeBuffer(logStore, this.id, notified);
 		this.#sharedViews = sharedViewsOf(this.#readinessBuffer);
+		if (registration.backend.publishesQueryRevisions) {
+			this.#publicationBuffer = publicationBuffer(logStore, this.readinessId);
+			this.#publicationRevision = new BigInt64Array(this.#publicationBuffer, 0, 1);
+		}
 		try {
 			registration.backend.attach({
 				isOwnerEpoch: (epoch) => Atomics.load(this.#sharedViews.epoch, 0) === epoch,
 				getReadiness: () => this.getReadiness(),
+				publicationChanged: () => this.#publicationChanged(),
 			});
 			this.#unsubscribeBackend = registration.backend.onStateChange((change = 'changed') =>
 				this.#backendStateChanged(change)
@@ -714,6 +724,7 @@ class DerivedIndexRunner {
 		} catch (error) {
 			this.#readinessBuffer.cancel?.();
 			if (this.#wakeBuffer !== this.#readinessBuffer) this.#wakeBuffer.cancel?.();
+			this.#publicationBuffer?.cancel?.();
 			throw error;
 		}
 		this.#unregisterTables = registerDerivedIndexTables(
@@ -775,6 +786,7 @@ class DerivedIndexRunner {
 			this.#unsubscribeBackend?.();
 			this.#readinessBuffer.cancel?.();
 			if (this.#wakeBuffer !== this.#readinessBuffer) this.#wakeBuffer.cancel?.();
+			this.#publicationBuffer?.cancel?.();
 		} catch (error) {
 			logger.warn?.(`Derived index '${this.id}' cleanup hook threw`, error);
 		}
@@ -1639,6 +1651,8 @@ class DerivedIndexRunner {
 			}
 		}
 		this.#boundaryPending = false;
+		// A waiter that observes the covering clock must also observe this durable publication revision.
+		if (offeredIndex > 0) this.#publicationChanged();
 		for (let i = offeredIndex; i >= 0; i--) {
 			const coverage = this.#offeredCursors[i].coverage;
 			if (coverage) {
@@ -1654,6 +1668,16 @@ class DerivedIndexRunner {
 		return true;
 	}
 
+	#publicationChanged(): void {
+		if (!this.#publicationRevision || !this.#publicationBuffer) return;
+		Atomics.add(this.#publicationRevision, 0, 1n);
+		try {
+			this.#publicationBuffer.notify?.();
+		} catch (error) {
+			logger.warn?.(`Derived index '${this.id}' could not notify peers of its publication`, error);
+		}
+	}
+
 	#publishCoverage(capture: CoverageCapture) {
 		if (!this.#owned || this.#rebuilding || Atomics.load(this.#shared().epoch, 0) !== this.#ownerEpoch) return;
 		if (capture.time <= Atomics.load(this.#shared().coverage, 0)) return;
@@ -2192,6 +2216,18 @@ function wakeBuffer(
 	}) as SharedReadinessBuffer;
 }
 
+function publicationBuffer(
+	logStore: RocksTransactionLogStore,
+	backendId: string,
+	callback?: () => void
+): SharedReadinessBuffer {
+	return logStore.getUserSharedBuffer(
+		`derived-index:${backendId}:publication`,
+		new ArrayBuffer(8),
+		callback ? { callback } : undefined
+	) as SharedReadinessBuffer;
+}
+
 type SharedViews = {
 	words: Int32Array;
 	epoch: BigInt64Array;
@@ -2207,6 +2243,7 @@ function sharedViewsOf(buffer: ArrayBufferLike): SharedViews {
 }
 
 const readinessViews = new WeakMap<object, Map<string, SharedViews>>();
+const publicationViews = new WeakMap<object, Map<string, BigInt64Array>>();
 
 function readReadiness({ words, epoch }: SharedViews): DerivedIndexReadiness {
 	const stateIndex = Atomics.load(words, READINESS_STATE);
@@ -2235,6 +2272,42 @@ export function readDerivedIndexReadiness(
 	return readReadiness(getReadinessViews(logStore, backendId));
 }
 
+export function readDerivedIndexPublicationRevision(
+	logStore: RocksTransactionLogStore,
+	backendId: string
+): { ownerEpoch: bigint; revision: bigint } {
+	const readiness = getReadinessViews(logStore, backendId);
+	let byBackend = publicationViews.get(logStore);
+	if (!byBackend) publicationViews.set(logStore, (byBackend = new Map()));
+	let revision = byBackend.get(backendId);
+	if (!revision) {
+		const buffer = publicationBuffer(logStore, backendId);
+		byBackend.set(backendId, (revision = new BigInt64Array(buffer, 0, 1)));
+	}
+	return {
+		ownerEpoch: Atomics.load(readiness.epoch, 0),
+		revision: Atomics.load(revision, 0),
+	};
+}
+
+export type DerivedIndexPublicationSubscription = {
+	revision(): bigint;
+	close(): void;
+};
+
+export function subscribeDerivedIndexPublications(
+	logStore: RocksTransactionLogStore,
+	backendId: string,
+	callback: () => void
+): DerivedIndexPublicationSubscription {
+	const buffer = publicationBuffer(logStore, backendId, callback);
+	const revision = new BigInt64Array(buffer, 0, 1);
+	return {
+		revision: () => Atomics.load(revision, 0),
+		close: () => buffer.cancel?.(),
+	};
+}
+
 /** Publish setup-time state before a backend runner exists. */
 export function publishDerivedIndexReadiness(
 	logStore: RocksTransactionLogStore,

package/resources/derivedIndexes.ts

@@ -1,5 +1,9 @@
+import { randomUUID } from 'node:crypto';
 import { ClientError } from '../utility/errors/hdbError.ts';
+import { settleBeforeDeadline } from '../utility/when.ts';
+import { ITC_EVENT_TYPES } from '../utility/hdbTerms.ts';
 import { loggerWithTag } from '../utility/logging/logger.ts';
+import { sendItcEventStrict } from '../server/threads/itc.js';
 import { RocksDatabase } from '@harperfast/rocksdb-js';
 import type { RocksTransactionLogStore } from './RocksTransactionLogStore.ts';
 import {
@@ -23,9 +27,17 @@ import {
 } from './indexes/hnswDerivedIndex.ts';
 import {
 	createNativeFullTextDerivedIndexBackend,
+	nativeFullTextIndexPath,
 	retireNativeFullTextDerivedIndexStorage,
 } from './indexes/nativeFullTextDerivedIndexLifecycle.ts';
 import type { NativeFullTextModule } from './indexes/fullTextNativeBinding.ts';
+import {
+	FULL_TEXT_QUERY_PAUSE_OPERATION,
+	FULL_TEXT_QUERY_RESUME_OPERATION,
+	FullTextQueryIndex,
+	pauseNativeFullTextQueryReaders,
+	resumeNativeFullTextQueryReaders,
+} from './indexes/fullTextQueryIndex.ts';
 import { fullTextStorageDefinition, type FullTextDefinition } from './fullTextSchema.ts';
 
 const logger = loggerWithTag('HNSW');
@@ -33,14 +45,17 @@ const fullTextLogger = loggerWithTag('fulltext-derived-index');
 const derivedIndexLogger = loggerWithTag('derived-index');
 const DEFAULT_MAX_LAG_MILLISECONDS = 30_000;
 const DEFAULT_FULL_TEXT_RETIREMENT_RETRY_MILLISECONDS = 70_000;
-const FULL_TEXT_LIMITS = Object.freeze({
-	indexingThreads: 1,
+const FULL_TEXT_READER_COORDINATION_TIMEOUT_MILLISECONDS = 35_000;
+const FULL_TEXT_RESET_TIMEOUT_MILLISECONDS = 105_000;
+const FULL_TEXT_WRITER_LIMITS = Object.freeze({
+	indexingThreads: 2,
 	searchThreads: 1,
-	writerMemoryBytes: 32 * 1024 * 1024,
+	writerMemoryBytes: 64 * 1024 * 1024,
 	maxQueuedCommands: 16,
 	maxQueuedBytes: 64 * 1024 * 1024,
 	maxBatchBytes: 8 * 1024 * 1024,
 });
+const FULL_TEXT_READER_LIMITS = Object.freeze({ ...FULL_TEXT_WRITER_LIMITS, searchThreads: 2 });
 
 type RegisteredTable = { current: { Table: any }; owners: Set<{ Table: any }> };
 type RegisteredBackend = {
@@ -71,6 +86,7 @@ type FullTextTestConfiguration = {
 	binding: NativeFullTextModule;
 	closeTimeoutMilliseconds: number;
 	shutdownTimeoutMilliseconds: number;
+	readerCoordinationTimeoutMilliseconds?: number;
 	runnerOptions: {
 		rebuildBackoffMilliseconds: number;
 		maxRebuildBackoffMilliseconds: number;
@@ -291,6 +307,7 @@ export function attachDerivedIndexes(
 	const hnswRetryBackendIds: string[] = [];
 	const registeredBackends = new Map<string, RegisteredBackend>();
 	const fullTextSetups = new Map<string, Promise<void>>();
+	const fullTextQueryIndexes = new Map<string, FullTextQueryIndex>();
 	let closing = false;
 	let registrationReleased = false;
 	let closeOperation: Promise<void> | undefined;
@@ -393,6 +410,23 @@ export function attachDerivedIndexes(
 			);
 			return;
 		}
+		if (!fullTextQueryIndexes.has(definition.name)) {
+			const queryIndex = new FullTextQueryIndex({
+				Table,
+				definition,
+				auditStore,
+				readinessId,
+				indexId: id,
+				storePath: Table.primaryStore.rootStore.path,
+				storeName: `${Table.tableName}/${definition.name}`,
+				sourceGeneration: `${Table.tableId}:${generation}`,
+				limits: { ...FULL_TEXT_READER_LIMITS },
+				...(fullTextTest ? { binding: fullTextTest.binding } : null),
+			});
+			fullTextQueryIndexes.set(definition.name, queryIndex);
+			Table.fullTextQueryIndexes[definition.name] = { customIndex: queryIndex };
+			Table.hasFullTextQueryIndexes = true;
+		}
 
 		const { settlePredecessor, predecessorSettled } = beginBackendHandoff(registered, id);
 		const quiescePredecessor = async () => {
@@ -431,6 +465,7 @@ export function attachDerivedIndexes(
 			if (retryUnavailableReadiness) retryDerivedIndexUnavailable(auditStore, readinessId);
 			const storage = fullTextStorageDefinition(definition);
 			const storeName = `${Table.tableName}/${definition.name}`;
+			const nativePath = nativeFullTextIndexPath(Table.primaryStore.rootStore.path, storeName);
 			const warningKey = `${Table.primaryStore.rootStore.path}:${storeName}`;
 			if (!warnedAuditIndexes.has(warningKey)) {
 				warnedAuditIndexes.add(warningKey);
@@ -443,12 +478,17 @@ export function attachDerivedIndexes(
 				storePath: Table.primaryStore.rootStore.path,
 				storeName,
 				sourceGeneration: `${Table.tableId}:${generation}`,
-				fields: storage.fields,
+				fields: definition.fields.map(({ name, weight }) => ({ name, weight })),
 				analyzer: storage.analyzer,
 				stopWords: storage.stopWords,
 				positions: storage.positions,
 				surfaceTerms: storage.surfaceTerms,
-				limits: { ...FULL_TEXT_LIMITS },
+				synonyms: storage.synonyms,
+				limits: { ...FULL_TEXT_WRITER_LIMITS },
+				hasBlobSources: definition.fields.some(({ mediaType }) => mediaType === 'text/plain'),
+				beforeReset: (ownerEpoch) => coordinateFullTextQueryReaders(nativePath, readinessId, ownerEpoch, true),
+				afterReset: (ownerEpoch) => coordinateFullTextQueryReaders(nativePath, readinessId, ownerEpoch, false),
+				shutdownTimeoutMilliseconds: FULL_TEXT_RESET_TIMEOUT_MILLISECONDS,
 				...(fullTextTest
 					? {
 							binding: fullTextTest.binding,
@@ -468,6 +508,11 @@ export function attachDerivedIndexes(
 					...fullTextTest?.runnerOptions,
 				},
 			});
+			fullTextQueryIndexes.get(definition.name)?.attachDerivedHost({
+				readiness: () => registered.runtime.getReadiness(id),
+				requestRebuild: () => registered.runtime.requestRebuild(id),
+				waitForCoverage: (since, timeout, signal) => registered.runtime.waitForCoverage(id, since, timeout, signal),
+			});
 		})().catch((error) => {
 			if (!isCurrent()) return;
 			// Setup runs on every worker, but readiness is shared. A local setup failure must not
@@ -494,6 +539,11 @@ export function attachDerivedIndexes(
 			const settlements = dropping
 				? [...(registered.tableBackends.get(Table.tableId) ?? [])].map((backend) => backend.settle())
 				: releases.map((release) => release());
+			for (const queryIndex of fullTextQueryIndexes.values()) settlements.push(queryIndex.close());
+			for (const [name, queryIndex] of fullTextQueryIndexes) {
+				if (Table.fullTextQueryIndexes[name]?.customIndex === queryIndex) delete Table.fullTextQueryIndexes[name];
+			}
+			Table.hasFullTextQueryIndexes = Object.keys(Table.fullTextQueryIndexes).length > 0;
 			const retained = new Set(
 				((Table.fullTextIndexes ?? []) as FullTextDefinition[]).map((definition) => definition.name)
 			);
@@ -559,10 +609,14 @@ export function attachDerivedIndexes(
 		matchesCurrent() {
 			if (closing || fullTextRetirementRecoveryFailed || !matchesCurrentHnsw()) return false;
 			const currentDefinitions = (Table.fullTextIndexes ?? []) as FullTextDefinition[];
-			return (
-				fullTextActivationSnapshot(Table, currentDefinitions) === fullTextSnapshot &&
-				JSON.stringify(Table.fullTextIndexRetirements ?? []) === fullTextRetirementSnapshot
-			);
+			if (
+				fullTextActivationSnapshot(Table, currentDefinitions) !== fullTextSnapshot ||
+				JSON.stringify(Table.fullTextIndexRetirements ?? []) !== fullTextRetirementSnapshot
+			)
+				return false;
+			for (const definition of currentDefinitions)
+				fullTextQueryIndexes.get(definition.name)?.updateDefinition(definition);
+			return true;
 		},
 		retryUnavailableFullText() {
 			if (closing) return;
@@ -629,6 +683,40 @@ export function attachDerivedIndexes(
 	}
 }
 
+async function coordinateFullTextQueryReaders(
+	path: string,
+	readinessId: string,
+	ownerEpoch: bigint,
+	pause: boolean,
+	timeout = FULL_TEXT_READER_COORDINATION_TIMEOUT_MILLISECONDS,
+	allowUnregisteredReadiness = false
+): Promise<void> {
+	if (pause) {
+		const draining = pauseNativeFullTextQueryReaders(path, readinessId, ownerEpoch, allowUnregisteredReadiness);
+		await settleBeforeDeadline([draining], Date.now() + timeout, () =>
+			Object.assign(new Error('Full-text reader drain timed out'), {
+				code: 'E_FULL_TEXT_READER_COORDINATION_TIMEOUT',
+				retryable: true,
+			})
+		);
+		await draining;
+	} else resumeNativeFullTextQueryReaders(path, readinessId, ownerEpoch);
+	await sendItcEventStrict(
+		{
+			type: ITC_EVENT_TYPES.SCHEMA,
+			message: {
+				operation: pause ? FULL_TEXT_QUERY_PAUSE_OPERATION : FULL_TEXT_QUERY_RESUME_OPERATION,
+				path,
+				readinessId,
+				ownerEpoch: ownerEpoch.toString(),
+				allowUnregisteredReadiness,
+			},
+		},
+		timeout,
+		true
+	);
+}
+
 /** Retire durable native indexes even when their in-memory attachment could not be restored. */
 export async function retireFullTextIndexes(
 	Table: any,
@@ -638,20 +726,48 @@ export async function retireFullTextIndexes(
 	const fullTextTest = fullTextTestConfiguration;
 	const retryMilliseconds =
 		fullTextTest?.shutdownTimeoutMilliseconds ?? DEFAULT_FULL_TEXT_RETIREMENT_RETRY_MILLISECONDS;
+	const coordinationTimeoutMilliseconds =
+		fullTextTest?.readerCoordinationTimeoutMilliseconds ?? FULL_TEXT_READER_COORDINATION_TIMEOUT_MILLISECONDS;
 	const deadline = Date.now() + retryMilliseconds;
 	let retired = true;
+	const rootStore = Table.primaryStore.rootStore;
 	for (const definition of definitions) {
 		if (!shouldContinue()) return false;
+		const indexId = fullTextDerivedIndexId(Table, definition.name);
+		const storeName = `${Table.tableName}/${definition.name}`;
+		const nativePath = nativeFullTextIndexPath(rootStore.path, storeName);
+		const lockKey = `derived-index:${indexId}:runner`;
+		const readinessId = `${indexId}:retirement:${randomUUID()}`;
 		let retryDelayMilliseconds = 10;
 		let retrying = false;
 		for (;;) {
 			try {
-				await retireNativeFullTextDerivedIndexStorage({
-					storePath: Table.primaryStore.rootStore.path,
-					storeName: `${Table.tableName}/${definition.name}`,
-					indexId: fullTextDerivedIndexId(Table, definition.name),
-					...(fullTextTest ? { binding: fullTextTest.binding } : {}),
-				});
+				if (!rootStore.tryLock(lockKey))
+					throw Object.assign(new Error('Full-text runner still owns the index'), { code: 'E_LOCK_BUSY' });
+				try {
+					// Keep orphan-pause recovery and every writer fenced until peer readers have drained.
+					try {
+						await coordinateFullTextQueryReaders(
+							nativePath,
+							readinessId,
+							0n,
+							true,
+							Math.max(1, Math.min(coordinationTimeoutMilliseconds, deadline - Date.now())),
+							true
+						);
+						if (!shouldContinue()) return false;
+						await retireNativeFullTextDerivedIndexStorage({
+							storePath: rootStore.path,
+							storeName,
+							indexId,
+							...(fullTextTest ? { binding: fullTextTest.binding } : {}),
+						});
+					} finally {
+						await coordinateFullTextQueryReaders(nativePath, readinessId, 0n, false, coordinationTimeoutMilliseconds);
+					}
+				} finally {
+					rootStore.unlock(lockKey);
+				}
 				break;
 			} catch (error) {
 				if (!shouldContinue()) return false;
@@ -659,8 +775,9 @@ export async function retireFullTextIndexes(
 					error && typeof error === 'object' && 'code' in error && typeof error.code === 'string'
 						? error.code
 						: undefined;
+				const retryable = error && typeof error === 'object' && 'retryable' in error && error.retryable === true;
 				const remaining = deadline - Date.now();
-				if (code === 'E_LOCK_BUSY' && remaining > 0) {
+				if ((code === 'E_LOCK_BUSY' || retryable) && remaining > 0) {
 					if (!retrying) {
 						retrying = true;
 						fullTextLogger.warn?.(
@@ -802,8 +919,8 @@ export function assertFullTextActivationSupported(
 
 function fullTextProjection(definition: FullTextDefinition) {
 	return (record: Record<string, unknown>) => {
-		const projection: Record<string, string | string[]> = Object.create(null);
-		for (const { name } of definition.fields) {
+		const projection: Record<string, string | string[] | Blob> = Object.create(null);
+		for (const { name, mediaType } of definition.fields) {
 			const value = record[name];
 			if (typeof value === 'string') {
 				projection[name] = value;
@@ -819,6 +936,13 @@ function fullTextProjection(definition: FullTextDefinition) {
 				projection[name] = copied;
 				continue;
 			}
+			if (mediaType === 'text/plain' && value instanceof Blob) {
+				const actualMediaType = value.type.split(';', 1)[0].trim().toLowerCase();
+				if (actualMediaType && actualMediaType !== mediaType)
+					throw new ClientError(`Full-text Blob source '${name}' must contain ${mediaType} data`, 400);
+				projection[name] = value;
+				continue;
+			}
 			if (value != null) throw new ClientError(`Full-text source '${name}' contains a non-text value`, 400);
 		}
 		return projection;

package/resources/fullTextSchema.ts

@@ -10,11 +10,11 @@ const FULL_TEXT_ARGUMENTS = new Set([
 	'synonyms',
 	'highlighting',
 ]);
-const SOURCE_ARGUMENTS = new Set(['name', 'weight', 'highlight']);
+const SOURCE_ARGUMENTS = new Set(['name', 'weight', 'highlight', 'mediaType']);
 const SYNONYM_ARGUMENTS = new Set(['source', 'replacements']);
 const HIGHLIGHTING_ARGUMENTS = new Set(['maxFragments', 'fragmentLength']);
 
-const DEFAULT_ANALYZER = 'english@1';
+const DEFAULT_ANALYZER = 'english@2';
 const DEFAULT_MAX_FRAGMENTS = 3;
 const DEFAULT_FRAGMENT_LENGTH = 160;
 
@@ -22,6 +22,7 @@ export type FullTextSource = {
 	name: string;
 	weight: number;
 	highlight?: boolean;
+	mediaType?: 'text/plain';
 };
 
 export type FullTextSynonymRule = {
@@ -37,7 +38,7 @@ export type FullTextHighlighting = {
 export type FullTextDefinition = {
 	name: string;
 	fields: FullTextSource[];
-	analyzer: 'english@1';
+	analyzer: 'english@2';
 	stopWords: boolean;
 	positions: boolean;
 	surfaceTerms: boolean;
@@ -47,9 +48,9 @@ export type FullTextDefinition = {
 
 export type FullTextStorageDefinition = Pick<
 	FullTextDefinition,
-	'analyzer' | 'stopWords' | 'positions' | 'surfaceTerms'
+	'analyzer' | 'stopWords' | 'positions' | 'surfaceTerms' | 'synonyms'
 > & {
-	fields: Array<Pick<FullTextSource, 'name' | 'weight'>>;
+	fields: Array<Pick<FullTextSource, 'name' | 'mediaType'>>;
 };
 
 export type FullTextIndexGenerations = Record<string, string>;
@@ -68,11 +69,18 @@ export function persistedFullTextIndexNames(values: unknown): string[] {
 
 export function fullTextStorageDefinition(definition: FullTextDefinition): FullTextStorageDefinition {
 	return {
-		fields: definition.fields.map(({ name, weight }) => ({ name, weight })),
+		fields: definition.fields.map(({ name, mediaType }) => ({
+			name,
+			...(mediaType ? { mediaType } : null),
+		})),
 		analyzer: definition.analyzer,
 		stopWords: definition.stopWords,
 		positions: definition.positions,
 		surfaceTerms: definition.surfaceTerms,
+		synonyms: (definition.synonyms ?? []).map(({ source, replacements }) => ({
+			source,
+			replacements: [...replacements],
+		})),
 	};
 }
 
@@ -109,6 +117,29 @@ export type FullTextSchemaAttribute = {
 	relationship?: unknown;
 };
 
+export function compileFullTextFields(
+	values: unknown,
+	definitions: readonly Pick<FullTextDefinition, 'name'>[],
+	attributes: readonly FullTextSchemaAttribute[],
+	requireAll = false
+): string[] {
+	if (!Array.isArray(values)) throw schemaError('fullTextFields must be a list of index names');
+	const indexes = new Set(definitions.map(({ name }) => name));
+	const stored = new Set(attributes.map(({ name }) => name));
+	const names = new Set<string>();
+	for (const name of values) {
+		if (typeof name !== 'string' || name.length === 0)
+			throw schemaError('fullTextFields must contain non-empty index names');
+		if (names.has(name)) throw schemaError(`Full-text field "${name}" is declared more than once`);
+		if (!indexes.has(name)) throw schemaError(`Full-text field "${name}" requires a declared @fullText index`);
+		if (stored.has(name)) throw schemaError(`Full-text field "${name}" conflicts with a stored attribute`);
+		names.add(name);
+	}
+	if (requireAll && names.size !== indexes.size)
+		throw schemaError('fullTextFields must include every declared @fullText index');
+	return [...names].sort();
+}
+
 export function compileFullTextDefinitions(
 	values: readonly unknown[],
 	attributes: readonly FullTextSchemaAttribute[]
@@ -163,10 +194,11 @@ export function compileFullTextDefinition(
 	if (typeof definition.name !== 'string' || definition.name.length === 0)
 		throw schemaError('@fullText requires a non-empty string "name"');
 	const indexName = definition.name;
-
 	if (!Array.isArray(definition.fields) || definition.fields.length === 0)
 		throw schemaError(`@fullText index "${indexName}" requires a non-empty "fields" list`);
 	const attributesByName = new Map(attributes.map((attribute) => [attribute.name, attribute]));
+	if (attributesByName.has(indexName))
+		throw schemaError(`Full-text field "${indexName}" conflicts with a stored attribute`);
 	const sourceNames = new Set<string>();
 	const fields = definition.fields.map((entry, index) => {
 		const source = requireObject(entry, `@fullText fields[${index}] on index "${indexName}"`);
@@ -184,8 +216,12 @@ export function compileFullTextDefinition(
 			);
 		if (!isSupportedSource(attribute))
 			throw schemaError(
-				`@fullText source field "${source.name}" must be String or [String]; got "${displayType(attribute)}"`
+				`@fullText source field "${source.name}" must be String, [String], or a declared text/plain Blob; got "${displayType(attribute)}"`
 			);
+		if (attribute.type === 'Blob' && source.mediaType !== 'text/plain')
+			throw schemaError(`@fullText Blob source field "${source.name}" requires mediaType: "text/plain"`);
+		if (attribute.type !== 'Blob' && source.mediaType !== undefined)
+			throw schemaError(`@fullText source field "${source.name}" only accepts mediaType for Blob attributes`);
 		const weight = source.weight === undefined ? 1 : source.weight;
 		if (typeof weight !== 'number' || !Number.isFinite(weight) || weight <= 0)
 			throw schemaError(`@fullText source field "${source.name}" requires a finite weight greater than zero`);
@@ -196,6 +232,7 @@ export function compileFullTextDefinition(
 			name: source.name,
 			weight,
 			...(typeof source.highlight === 'boolean' ? { highlight: source.highlight } : {}),
+			...(source.mediaType === 'text/plain' ? { mediaType: 'text/plain' as const } : {}),
 		};
 	});
 
@@ -207,6 +244,11 @@ export function compileFullTextDefinition(
 	const surfaceTerms = booleanOption(definition, 'surfaceTerms', true, indexName);
 	const synonyms = compileSynonyms(definition.synonyms, indexName);
 	const highlighting = compileHighlighting(definition.highlighting, indexName);
+	const highlightedFields = fields.filter(({ highlight }) => highlight === true);
+	if (highlighting && highlightedFields.length === 0)
+		throw schemaError(`@fullText index "${indexName}" requires at least one source field with highlight: true`);
+	if (highlighting && !surfaceTerms)
+		throw schemaError(`@fullText index "${indexName}" highlighting requires surfaceTerms: true`);
 
 	return {
 		name: indexName,
@@ -274,7 +316,11 @@ function booleanOption(
 }
 
 function isSupportedSource(attribute: FullTextSchemaAttribute): boolean {
-	return attribute.type === 'String' || (attribute.type === 'array' && attribute.elements?.type === 'String');
+	return (
+		attribute.type === 'String' ||
+		attribute.type === 'Blob' ||
+		(attribute.type === 'array' && attribute.elements?.type === 'String')
+	);
 }
 
 function displayType(attribute: FullTextSchemaAttribute): string {