@harperfast/harper 5.3.0-beta.3 → 5.3.0-beta.4

package/resources/fullTextSchema.ts

@@ -45,6 +45,61 @@ export type FullTextDefinition = {
 	highlighting?: FullTextHighlighting;
 };
 
+export type FullTextStorageDefinition = Pick<
+	FullTextDefinition,
+	'analyzer' | 'stopWords' | 'positions' | 'surfaceTerms'
+> & {
+	fields: Array<Pick<FullTextSource, 'name' | 'weight'>>;
+};
+
+export type FullTextIndexGenerations = Record<string, string>;
+
+/** Names are enough to locate native storage even when source attributes are missing or stale. */
+export function persistedFullTextIndexNames(values: unknown): string[] {
+	if (!Array.isArray(values)) return [];
+	const names = new Set<string>();
+	for (const value of values) {
+		if (!value || typeof value !== 'object' || Array.isArray(value)) continue;
+		const name = (value as { name?: unknown }).name;
+		if (typeof name === 'string' && name.length > 0) names.add(name);
+	}
+	return [...names].sort();
+}
+
+export function fullTextStorageDefinition(definition: FullTextDefinition): FullTextStorageDefinition {
+	return {
+		fields: definition.fields.map(({ name, weight }) => ({ name, weight })),
+		analyzer: definition.analyzer,
+		stopWords: definition.stopWords,
+		positions: definition.positions,
+		surfaceTerms: definition.surfaceTerms,
+	};
+}
+
+export function reconcileFullTextIndexGenerations(
+	previousDefinitions: readonly FullTextDefinition[],
+	previousGenerations: unknown,
+	nextDefinitions: readonly FullTextDefinition[],
+	createGeneration: () => string
+): FullTextIndexGenerations {
+	const previousByName = new Map(previousDefinitions.map((definition) => [definition.name, definition]));
+	const durable =
+		previousGenerations && typeof previousGenerations === 'object' && !Array.isArray(previousGenerations)
+			? (previousGenerations as Record<string, unknown>)
+			: Object.create(null);
+	const generations: FullTextIndexGenerations = Object.create(null);
+	for (const definition of nextDefinitions) {
+		const previous = previousByName.get(definition.name);
+		const generation = durable[definition.name];
+		const storageUnchanged =
+			previous &&
+			JSON.stringify(fullTextStorageDefinition(previous)) === JSON.stringify(fullTextStorageDefinition(definition));
+		generations[definition.name] =
+			storageUnchanged && typeof generation === 'string' && generation.length > 0 ? generation : createGeneration();
+	}
+	return generations;
+}
+
 export type FullTextSchemaAttribute = {
 	name: string;
 	type?: string;

package/resources/fullTextSchemaLifecycle.ts

@@ -34,21 +34,6 @@ export function readPersistedFullTextDefinitions(
 	});
 }
 
-/**
- * Produces the catalog state that is safe while source descriptors transition. The caller publishes
- * the requested definitions only after their source descriptors are durable.
- */
-export function safeInterimFullTextDefinitions(
-	values: unknown,
-	attributes: readonly FullTextSchemaAttribute[],
-	requested: readonly FullTextDefinition[]
-): FullTextDefinition[] {
-	const requestedByName = new Map(requested.map((definition) => [definition.name, definition]));
-	return compileValidFullTextDefinitions(values ?? [], attributes, () => {}).filter((definition) =>
-		definitionsEqual(definition, requestedByName.get(definition.name))
-	);
-}
-
 export function retainFullTextDefinitions(
 	values: unknown,
 	durableAttributes: readonly FullTextSchemaAttribute[],

package/resources/graphql.ts

@@ -4,11 +4,15 @@ import { scopedTableFactory, table } from './databases.ts';
 import { getWorkerIndex } from '../server/threads/manageThreads.js';
 import { thisThreadOwnsApplication } from '../server/threads/isolatedApplications.ts';
 import { Resources } from './Resources.ts';
-import type { NamedTypeNode, StringValueNode, ValueNode } from 'graphql';
+import type { DirectiveNode, NamedTypeNode, StringValueNode, ValueNode } from 'graphql';
 import { ClientError } from '../utility/errors/hdbError.ts';
 import { attributeToFragment, type JsonSchemaFragment } from './jsonSchemaTypes.ts';
 import harperLogger from '../utility/logging/harper_logger.ts';
 import { compileFullTextDefinitions } from './fullTextSchema.ts';
+import { validateDecisionSchema } from './models/decision.ts';
+import { assertDerivedFieldOwnership } from './models/embedHook.ts';
+import type { DecideConfig } from './models/decideHook.ts';
+import type { DecisionLeaf } from './models/types.ts';
 
 const PRIMITIVE_TYPES = ['ID', 'Int', 'Float', 'Long', 'String', 'Boolean', 'Date', 'Bytes', 'Any', 'BigInt', 'Blob'];
 
@@ -41,6 +45,101 @@ function coerceDirectiveValue(node: ValueNode): any {
 	}
 }
 
+const DECIDE_STRING_ARGS = new Set(['source', 'model', 'confidence', 'decision', 'instructions']);
+const DECIDE_INT_ARGS = new Set(['minimum', 'maximum']);
+
+/**
+ * `@decide(source, values | minimum/maximum, model?, confidence?, decision?, instructions?)`: the attribute
+ * type selects the decision leaf (String → the `values` enum, Boolean, Int → a bounded range),
+ * and the leaf is validated here so a bad closed set fails the schema load, not the first write.
+ * Field references (source, confidence, decision) are checked once every field of the type is known.
+ */
+function parseDecideDirective(directive: DirectiveNode, property: any): DecideConfig {
+	const target = `@decide on "${property.name}"`;
+	const args: {
+		source?: string;
+		model?: string;
+		confidence?: string;
+		decision?: string;
+		instructions?: string;
+		values?: string[];
+		minimum?: number;
+		maximum?: number;
+	} = {};
+	for (const arg of directive.arguments || []) {
+		const name = arg.name.value;
+		if (Object.hasOwn(args, name)) throw new ClientError(`${target} declares "${name}" more than once`, 400);
+		const value = arg.value;
+		if (DECIDE_STRING_ARGS.has(name)) {
+			if (value.kind !== 'StringValue')
+				throw new ClientError(`@decide(${name}: ...) on "${property.name}" expects a string literal`, 400);
+			args[name] = value.value;
+		} else if (DECIDE_INT_ARGS.has(name)) {
+			if (value.kind !== 'IntValue')
+				throw new ClientError(`@decide(${name}: ...) on "${property.name}" expects an integer literal`, 400);
+			args[name] = Number(value.value);
+		} else if (name === 'values') {
+			if (value.kind !== 'ListValue' || value.values.some((item) => item.kind !== 'StringValue'))
+				throw new ClientError(`@decide(values: ...) on "${property.name}" expects a list of string literals`, 400);
+			args.values = value.values.map((item) => (item as StringValueNode).value);
+		} else throw new ClientError(`${target} has an unknown argument "${name}"`, 400);
+	}
+	if (!args.source) throw new ClientError(`${target} requires a "source" argument`, 400);
+	for (const name of ['confidence', 'decision'] as const)
+		if (args[name] === '') throw new ClientError(`${target}: "${name}" must name a field`, 400);
+	const hasRange = args.minimum !== undefined || args.maximum !== undefined;
+	let schema: DecisionLeaf;
+	switch (property.type) {
+		case 'String':
+			if (!args.values)
+				throw new ClientError(`${target} requires "values" (the allowed strings) on a String attribute`, 400);
+			if (hasRange) throw new ClientError(`${target} takes "minimum" and "maximum" only on an Int attribute`, 400);
+			schema = { enum: args.values };
+			break;
+		case 'Boolean':
+			if (args.values || hasRange)
+				throw new ClientError(`${target} takes no "values", "minimum" or "maximum" on a Boolean attribute`, 400);
+			schema = { type: 'boolean' };
+			break;
+		case 'Int':
+			if (args.values)
+				throw new ClientError(`${target} takes "minimum" and "maximum", not "values", on an Int attribute`, 400);
+			if (args.minimum === undefined || args.maximum === undefined)
+				throw new ClientError(`${target} requires both "minimum" and "maximum" on an Int attribute`, 400);
+			// Table validation holds an Int to 32 bits; a wider bound would load and then fail every write.
+			for (const [name, bound] of Object.entries({ minimum: args.minimum, maximum: args.maximum }))
+				if (bound >> 0 !== bound)
+					throw new ClientError(`${target}: "${name}" must be an integer from -2147483648 to 2147483647`, 400);
+			schema = { type: 'integer', minimum: args.minimum, maximum: args.maximum };
+			break;
+		default:
+			throw new ClientError(
+				`${target} requires a String, Boolean or Int attribute type; got "${property.type === 'array' ? '[...]' : property.type}"`,
+				400
+			);
+	}
+	try {
+		validateDecisionSchema(schema);
+	} catch (err) {
+		throw new ClientError(`${target}: ${(err as Error).message}`, 400);
+	}
+	if (property.nullable === false)
+		throw new ClientError(`${target} cannot be declared non-null: a null source clears the attribute`, 400);
+	if (property.isPrimaryKey || property.computed)
+		throw new ClientError(`${target} cannot combine with @primaryKey or @computed`, 400);
+	const fields = new Set([property.name, args.source, args.confidence, args.decision].filter(Boolean));
+	if (fields.size !== 2 + (args.confidence ? 1 : 0) + (args.decision ? 1 : 0))
+		throw new ClientError(
+			`${target}: the attribute, "source", "confidence" and "decision" must be different fields`,
+			400
+		);
+	const config: DecideConfig = { source: args.source, model: args.model ?? 'default', schema };
+	if (args.confidence) config.confidence = args.confidence;
+	if (args.decision) config.decision = args.decision;
+	if (args.instructions) config.instructions = args.instructions;
+	return config;
+}
+
 if (!server.knownGraphQLDirectives) {
 	server.knownGraphQLDirectives = [];
 }
@@ -52,6 +151,7 @@ server.knownGraphQLDirectives.push(
 	'indexed',
 	'computed',
 	'embed',
+	'decide',
 	'fullText',
 	'relationship',
 	'createdTime',
@@ -288,6 +388,10 @@ async function processGraphQLSchema(
 									property.version = `embed:${embedDefinition.model}`;
 								}
 							}
+						} else if (directiveName === 'decide') {
+							// Resolved after the field's other directives so @primaryKey / @computed in any
+							// order are seen; the ownership checks below need every field of the type.
+							property.decide = directive;
 						} else if (directiveName === 'fullText') {
 							throw new ClientError('@fullText must be declared on a @table type, not on a field', 400);
 						} else if (directiveName === 'relationship') {
@@ -337,18 +441,9 @@ async function processGraphQLSchema(
 								400
 							);
 					}
+					if (property.decide) property.decide = parseDecideDirective(property.decide, property);
 				}
-				// @embed source must reference a declared field; a typo would silently leave
-				// the vector column unpopulated (the source key never appears in write payloads).
-				for (const prop of attributes as any[]) {
-					// Object.hasOwn (not `in`): `attributesObject` is a plain object, so `in` would
-					// match inherited prototype keys (toString, constructor) and pass a bad source.
-					if (prop.embed && !Object.hasOwn(attributesObject, prop.embed.source))
-						throw new ClientError(
-							`@embed on "${prop.name}" references unknown source field "${prop.embed.source}"`,
-							400
-						);
-				}
+				assertDerivedFieldOwnership(attributes as any[]);
 				if (typeDef.fullTextIndexes.length > 0 && !typeDef.table)
 					throw new ClientError('@fullText is only supported on a @table type', 400);
 				typeDef.fullTextIndexes = compileFullTextDefinitions(typeDef.fullTextIndexes, attributes);

package/resources/indexes/DESIGN.md

@@ -387,6 +387,13 @@ replay after its payload; replicas independently derive it from their own applie
 backup and restore need only authoritative records and schema. A new or unusable directory serves no
 full-text queries until rebuild and catch-up publish `ready`.
 
+A table drop keeps the tombstoned catalog descriptor, including its native index names, until
+retirement completes. The drop broadcast first unloads peer attachments and awaits their writer
+shutdown; only then may the wrapper retire native storage, followed by RocksDB column families and
+catalog rows. If native retirement cannot prove success, the table remains logically dropped and
+unloaded but its tombstone stays durable so restart or same-name creation retries cleanup. Removing
+the catalog first would turn a crash in that interval into an untraceable native-directory leak.
+
 Tantivy files are not an opaque encrypted cache. They contain the document-id term dictionary,
 analyzed term dictionaries, postings, frequencies and optionally positions; `surfaceTerms: true`
 also stores the original projected strings needed for surface-term features. Operators must protect

package/resources/indexes/hnswDerivedIndex.ts

@@ -5,9 +5,7 @@ import type { RocksTransactionLogStore } from '../RocksTransactionLogStore.ts';
 import {
 	DERIVED_INDEX_ACCEPTED,
 	DERIVED_INDEX_DEFERRED,
-	DerivedIndexRuntime,
 	readDerivedIndexReadiness,
-	readDerivedIndexCoverage,
 	sameDerivedIndexPositions,
 	type DerivedIndexBackend,
 	type DerivedIndexBackendHost,
@@ -42,9 +40,6 @@ const QUEUE_CAPACITY_BYTES = 64 * 1024 * 1024;
 const APPLY_SLICE_MILLIS = 5;
 const BARRIER_IDLE_MULTIPLE = 3;
 const BARRIER_IDLE_CEILING_MILLISECONDS = 7_500;
-// Writes to an index this far behind fail with a retryable 503 (see the runtime's lag policy).
-const DEFAULT_MAX_LAG_MILLISECONDS = 30_000;
-
 /**
  * The native index methods the backend drives. `applyDerivedValue` inserts, replaces or removes
  * one primary key's vector and stages its mapping as pending; `flushDerived` is the durability
@@ -277,261 +272,6 @@ export class HnswDerivedIndexBackend implements DerivedIndexBackend {
 	}
 }
 
-// One generation owns each physical backend; destructive cleanup also awaits superseded generations.
-type RegisteredTable = { current: { Table: any }; owners: Set<{ Table: any }> };
-type RegisteredBackend = {
-	tableId: number;
-	settle: () => Promise<void>;
-};
-type Registered = {
-	runtime: DerivedIndexRuntime;
-	tables: Map<number, RegisteredTable>;
-	backends: Map<string, RegisteredBackend>;
-	tableBackends: Map<number, Set<RegisteredBackend>>;
-	droppingTables: Set<number>;
-	retryUnavailable: Set<string>;
-};
-const runtimes = new WeakMap<object, Registered>();
-const retryUnavailableByStore = new Set<string>();
-
-function retryUnavailableKey(auditStore: RocksTransactionLogStore, backendId: string): string {
-	return `${auditStore.rootStore.path}\0${backendId}`;
-}
-
-function markUnavailableRetry(registered: Registered, auditStore: RocksTransactionLogStore, backendId: string): void {
-	registered.retryUnavailable.add(backendId);
-	retryUnavailableByStore.add(retryUnavailableKey(auditStore, backendId));
-}
-
-function consumeUnavailableRetry(
-	registered: Registered,
-	auditStore: RocksTransactionLogStore,
-	backendId: string
-): boolean {
-	const local = registered.retryUnavailable.delete(backendId);
-	const storeScoped = retryUnavailableByStore.delete(retryUnavailableKey(auditStore, backendId));
-	return local || storeScoped;
-}
-
-function runtimeFor(auditStore: RocksTransactionLogStore): Registered {
-	let registered = runtimes.get(auditStore);
-	if (registered) return registered;
-	const tables = new Map<number, RegisteredTable>();
-	const runtime = new DerivedIndexRuntime(
-		auditStore,
-		(tableId, recordId) => {
-			const entry = tables.get(tableId)?.current.Table.primaryStore.getEntry(recordId);
-			return entry?.value == null ? undefined : { version: entry.version, value: entry.value };
-		},
-		{
-			scanRecords: (tableId) =>
-				tables
-					.get(tableId)!
-					.current.Table.primaryStore.getRange({ versions: true, snapshot: false })
-					.map(({ key, value, version }) => ({ recordId: key, version, value })),
-		}
-	);
-	registered = {
-		runtime,
-		tables,
-		backends: new Map(),
-		tableBackends: new Map(),
-		droppingTables: new Set(),
-		retryUnavailable: new Set(),
-	};
-	runtimes.set(auditStore, registered);
-	return registered;
-}
-
-const warnedAuditIndexes = new Set<string>();
-
-/**
- * Register every post-commit custom index of a table with the shared derived-index runtime of its
- * database. Returns the release for the table's registrations, or undefined when it has none. Runs
- * on every worker; the runtime elects one owner per index.
- */
-export function attachDerivedIndexes(Table: any):
-	| {
-			close(dropping?: boolean): Promise<void>;
-			restoreAfterFailedDrop(): ReturnType<typeof attachDerivedIndexes>;
-			completeDrop(dropped?: boolean): void;
-	  }
-	| undefined {
-	const hnswAttributes = Table.attributes.filter(
-		(attribute: any) => attribute.indexed?.type === 'HNSW' && Table.indices[attribute.name]?.customIndex
-	);
-	const attributes = hnswAttributes.filter((attribute: any) => Table.indices[attribute.name]?.customIndex?.postCommit);
-	if (Table.audit !== true && attributes.length > 0) {
-		throw new ClientError(
-			`Table '${Table.databaseName}.${Table.tableName}' must enable audit logging before using a post-commit derived index`
-		);
-	}
-	if (!Table.auditStore) return;
-	const auditStore = Table.auditStore as RocksTransactionLogStore;
-	const registered = runtimeFor(auditStore);
-	if (registered.droppingTables.has(Table.tableId)) return;
-	for (const attribute of hnswAttributes) {
-		const indexStore = Table.indices[attribute.name];
-		const id = `hnsw:${indexStore.name}`;
-		// A transient JS-backed declaration can precede the first native registration while a
-		// schema is being installed. Only an explicit transition away from an already-registered
-		// native backend should make a later native registration eligible to rearm an exhausted
-		// rebuild budget.
-		if (attribute.indexed.nativePlane === false && !indexStore.customIndex.postCommit && registered.backends.has(id))
-			markUnavailableRetry(registered, auditStore, id);
-	}
-	if (attributes.length === 0) return;
-	const installed = { Table };
-	let registeredTable = registered.tables.get(Table.tableId);
-	if (registeredTable) {
-		registeredTable.owners.add(installed);
-		registeredTable.current = installed;
-	} else {
-		registeredTable = { current: installed, owners: new Set([installed]) };
-		registered.tables.set(Table.tableId, registeredTable);
-	}
-	const releases: Array<() => Promise<void>> = [];
-	const backendIds: string[] = [];
-	const registeredBackends = new Map<string, RegisteredBackend>();
-	for (const attribute of attributes) {
-		const indexStore = Table.indices[attribute.name];
-		const index = indexStore.customIndex as DerivedNativeIndex & { postCommit: true };
-		const id = `hnsw:${indexStore.name}`;
-		const warningKey = `${Table.databaseName}.${Table.tableName}.${indexStore.name}`;
-		if (!warnedAuditIndexes.has(warningKey)) {
-			warnedAuditIndexes.add(warningKey);
-			logger.warn?.(
-				`Derived index ${indexStore.name} requires auditing; the audit API retains full record history for the configured retention window`
-			);
-		}
-		const resolver = Table.propertyResolvers?.[attribute.name];
-		const label = `Vector for attribute "${attribute.name}"`;
-		index.attachDerivedHost({
-			readiness: () => registered.runtime.getReadiness(id),
-			coverage: (maxLagMilliseconds) =>
-				readDerivedIndexCoverage(
-					auditStore,
-					id,
-					() => indexStore.getSync(DERIVED_INDEX_CURSOR_KEY),
-					maxLagMilliseconds
-				),
-			requestRebuild: () => registered.runtime.requestRebuild(id),
-			waitForCoverage: (since, timeout, signal) => registered.runtime.waitForCoverage(id, since, timeout, signal),
-		});
-		let predecessor = registered.backends.get(id);
-		const inherited = predecessor;
-		const settlePredecessor = () => {
-			const current = predecessor;
-			if (!current) return Promise.resolve();
-			const settled = current.settle();
-			settled.then(
-				() => {
-					if (predecessor === current) predecessor = undefined;
-				},
-				() => {}
-			);
-			return settled;
-		};
-		const predecessorSettled = settlePredecessor();
-		predecessorSettled.catch(() => {});
-		if (registered.backends.get(id) === inherited) registered.backends.delete(id);
-		const release = registered.runtime.register({
-			backend: new HnswDerivedIndexBackend(id, index),
-			projections: new Map([
-				[
-					Table.tableId,
-					(record: any) => {
-						const vector = resolver ? resolver(record) : record[attribute.name];
-						if (vector == null) return undefined;
-						index.assertDerivedValue(vector, label);
-						return vector;
-					},
-				],
-			]),
-			options: { maxLagMilliseconds: attribute.indexed?.maxLagMilliseconds ?? DEFAULT_MAX_LAG_MILLISECONDS },
-		});
-		let settling: Promise<void> | undefined;
-		const registeredBackend: RegisteredBackend = {
-			tableId: Table.tableId,
-			settle: () => {
-				if (settling) return settling;
-				const attempt = Promise.allSettled([settlePredecessor(), release()]).then((results) => {
-					if (results[1].status === 'fulfilled') {
-						if (registered.backends.get(id) === registeredBackend) registered.backends.delete(id);
-						const tableBackends = registered.tableBackends.get(Table.tableId);
-						tableBackends?.delete(registeredBackend);
-						if (tableBackends?.size === 0) registered.tableBackends.delete(Table.tableId);
-					}
-					const failures = results
-						.filter((result): result is PromiseRejectedResult => result.status === 'rejected')
-						.map((result) => result.reason);
-					if (failures.length === 1) throw failures[0];
-					if (failures.length) throw new AggregateError(failures, `Could not settle HNSW backend '${id}'`);
-				});
-				settling = attempt;
-				attempt.catch(() => {
-					if (settling === attempt) settling = undefined;
-				});
-				return attempt;
-			},
-		};
-		let tableBackends = registered.tableBackends.get(Table.tableId);
-		if (!tableBackends) registered.tableBackends.set(Table.tableId, (tableBackends = new Set()));
-		tableBackends.add(registeredBackend);
-		registered.backends.set(id, registeredBackend);
-		registeredBackends.set(id, registeredBackend);
-		const retryUnavailable = consumeUnavailableRetry(registered, auditStore, id);
-		if (retryUnavailable && registered.runtime.getReadiness(id).state === 'unavailable')
-			registered.runtime.requestRebuild(id);
-		backendIds.push(id);
-		releases.push(() => registeredBackend.settle());
-	}
-	return {
-		async close(dropping = false) {
-			if (dropping) registered.droppingTables.add(Table.tableId);
-			const settlements = dropping
-				? [...(registered.tableBackends.get(Table.tableId) ?? [])].map((backend) => backend.settle())
-				: releases.map((release) => release());
-			const tableRegistration = registered.tables.get(Table.tableId);
-			if (tableRegistration) {
-				tableRegistration.owners.delete(installed);
-				if (tableRegistration.owners.size === 0) {
-					registered.tables.delete(Table.tableId);
-				} else if (tableRegistration.current === installed) {
-					tableRegistration.current = tableRegistration.owners.values().next().value;
-				}
-			}
-			if (dropping) {
-				await Promise.all(settlements);
-			} else {
-				for (const result of await Promise.allSettled(settlements)) {
-					if (result.status === 'rejected')
-						logger.error(`Could not settle a superseded HNSW backend for table ${Table.tableId}`, result.reason);
-				}
-			}
-		},
-		restoreAfterFailedDrop() {
-			registered.droppingTables.delete(Table.tableId);
-			const currentTable = registered.tables.get(Table.tableId)?.current.Table;
-			if (currentTable && currentTable !== Table) return;
-			for (const registration of registered.tables.values()) {
-				const current = registration.current.Table;
-				if (current !== Table && current.databasePath === Table.databasePath && current.tableName === Table.tableName)
-					return;
-			}
-			for (const [id, registeredBackend] of registeredBackends) {
-				const current = registered.backends.get(id);
-				if (current && current !== registeredBackend) return;
-			}
-			return attachDerivedIndexes(Table);
-		},
-		completeDrop(dropped = true) {
-			registered.droppingTables.delete(Table.tableId);
-			if (dropped) for (const id of backendIds) markUnavailableRetry(registered, auditStore, id);
-		},
-	};
-}
-
 /** Shared readiness of an index on any worker, registered or not. */
 export function derivedIndexReadiness(auditStore: RocksTransactionLogStore, indexStoreName: string) {
 	return readDerivedIndexReadiness(auditStore, `hnsw:${indexStoreName}`);

package/resources/indexes/nativeFullTextDerivedIndexLifecycle.ts

@@ -1,5 +1,6 @@
 import { createHash } from 'node:crypto';
-import { isAbsolute, join, resolve } from 'node:path';
+import { existsSync } from 'node:fs';
+import { dirname, isAbsolute, join, resolve } from 'node:path';
 import { loggerWithTag } from '../../utility/logging/logger.ts';
 import {
 	FullTextDerivedIndexBackend,
@@ -29,6 +30,11 @@ export type NativeFullTextDerivedIndexLifecycleOptions = NativeFullTextIndexConf
 export type NativeFullTextDerivedIndexBackendOptions = Omit<FullTextDerivedIndexBackendOptions, 'lifecycle'> &
 	Omit<NativeFullTextDerivedIndexLifecycleOptions, 'indexId'>;
 
+export type RetireNativeFullTextDerivedIndexStorageOptions = Pick<
+	NativeFullTextDerivedIndexLifecycleOptions,
+	'storePath' | 'storeName' | 'indexId' | 'binding'
+>;
+
 export class NativeFullTextDerivedIndexLifecycle {
 	readonly #options: NativeFullTextDerivedIndexLifecycleOptions;
 	readonly #path: string;
@@ -48,7 +54,7 @@ export class NativeFullTextDerivedIndexLifecycle {
 			limits: { ...options.limits },
 		};
 		// Hashing bounds filenames for arbitrary store names; the index id remains in native metadata and logs.
-		this.#path = join(resolve(options.storePath), `${digest(options.storeName)}.fulltext`);
+		this.#path = nativeFullTextIndexPath(options.storePath, options.storeName);
 		this.#generation = digest(options.sourceGeneration);
 	}
 
@@ -135,10 +141,7 @@ export class NativeFullTextDerivedIndexLifecycle {
 
 	async #getBinding(): Promise<NativeFullTextModule> {
 		if (this.#binding) return this.#binding;
-		const configured = this.#options.binding;
-		this.#binding = configured
-			? await validateFullTextNativeBinding(typeof configured === 'function' ? await configured() : configured)
-			: await loadFullTextNativeBinding();
+		this.#binding = await resolveNativeFullTextBinding(this.#options.binding);
 		return this.#binding;
 	}
 
@@ -148,16 +151,64 @@ export class NativeFullTextDerivedIndexLifecycle {
 	}
 }
 
+/** Retire one quiesced native directory after its Harper declaration or table is removed. */
+export async function retireNativeFullTextDerivedIndexStorage(
+	options: RetireNativeFullTextDerivedIndexStorageOptions
+): Promise<void> {
+	if (!isAbsolute(options.storePath)) throw new TypeError('Full-text storePath must be absolute');
+	if (!options.storeName) throw new TypeError('Full-text storeName is required');
+	if (!options.indexId) throw new TypeError('Full-text indexId is required');
+	const path = nativeFullTextIndexPath(options.storePath, options.storeName);
+	if (!existsSync(path)) {
+		if (existsSync(join(dirname(path), '.fulltext-retired'))) {
+			try {
+				const binding = await resolveNativeFullTextBinding(options.binding);
+				const reclaimed = await binding.reclaimRetiredNativeFullTextIndexes({ path });
+				if (reclaimed.failed > 0)
+					logWarning(`Could not remove ${reclaimed.failed} retired paths for full-text index '${options.indexId}'`);
+			} catch (error) {
+				logWarning(`Could not reclaim retired storage for full-text index '${options.indexId}'`, error);
+			}
+		}
+		return;
+	}
+	const binding = await resolveNativeFullTextBinding(options.binding);
+	// The wrapper owns the physical lifecycle fence: reset refuses an active/unproven writer and
+	// takes Tantivy's writer lock before renaming. Harper treats that refusal as best-effort cleanup,
+	// never as permission to remove a live directory itself.
+	const result = await binding.resetNativeFullTextIndex({ path, indexId: options.indexId });
+	void binding
+		.reclaimRetiredNativeFullTextIndexes({
+			path,
+			retiredPath: result.state === 'reset' ? result.retiredPath : undefined,
+		})
+		.then((reclaimed) => {
+			if (reclaimed.failed > 0)
+				logWarning(`Could not remove ${reclaimed.failed} retired paths for full-text index '${options.indexId}'`);
+		})
+		.catch((error) => logWarning(`Could not reclaim retired storage for full-text index '${options.indexId}'`, error));
+}
+
 export async function createNativeFullTextDerivedIndexBackend(
 	options: NativeFullTextDerivedIndexBackendOptions
 ): Promise<FullTextDerivedIndexBackend> {
-	const maxCursorPayloadBytes = options.maxCursorPayloadBytes ?? HARPER_FULLTEXT_MAX_CURSOR_PAYLOAD_BYTES;
-	if (maxCursorPayloadBytes > HARPER_FULLTEXT_MAX_CURSOR_PAYLOAD_BYTES)
+	const configuredMaxCursorPayloadBytes = options.maxCursorPayloadBytes;
+	if (
+		configuredMaxCursorPayloadBytes !== undefined &&
+		configuredMaxCursorPayloadBytes > HARPER_FULLTEXT_MAX_CURSOR_PAYLOAD_BYTES
+	)
 		throw new RangeError(`Full-text maxCursorPayloadBytes must not exceed ${HARPER_FULLTEXT_MAX_CURSOR_PAYLOAD_BYTES}`);
 	const lifecycle = new NativeFullTextDerivedIndexLifecycle({ ...options, indexId: options.id });
 	await lifecycle.initialize({ reclaimRetired: false });
-	if (maxCursorPayloadBytes > lifecycle.maxCommitPayloadBytes)
+	if (
+		configuredMaxCursorPayloadBytes !== undefined &&
+		configuredMaxCursorPayloadBytes > lifecycle.maxCommitPayloadBytes
+	)
 		throw new RangeError('Full-text maxCursorPayloadBytes exceeds the native commit payload capacity');
+	const maxCursorPayloadBytes = Math.min(
+		configuredMaxCursorPayloadBytes ?? HARPER_FULLTEXT_MAX_CURSOR_PAYLOAD_BYTES,
+		lifecycle.maxCommitPayloadBytes
+	);
 	const backend = new FullTextDerivedIndexBackend({
 		...options,
 		id: options.id,
@@ -168,6 +219,18 @@ export async function createNativeFullTextDerivedIndexBackend(
 	return backend;
 }
 
+function nativeFullTextIndexPath(storePath: string, storeName: string): string {
+	return join(resolve(storePath), `${digest(storeName)}.fulltext`);
+}
+
+async function resolveNativeFullTextBinding(
+	configured?: NativeFullTextModule | (() => Promise<NativeFullTextModule>)
+): Promise<NativeFullTextModule> {
+	return configured
+		? validateFullTextNativeBinding(typeof configured === 'function' ? await configured() : configured)
+		: loadFullTextNativeBinding();
+}
+
 function digest(value: string): string {
 	return createHash('sha256').update(value).digest('hex');
 }