@harperfast/harper 5.3.0-beta.4 → 5.3.0

package/dist/resources/indexes/fullTextQueryIndex.js

@@ -0,0 +1,943 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.FullTextQueryIndex = exports.fullTextComparatorMode = exports.FULL_TEXT_QUERY_RESUME_OPERATION = exports.FULL_TEXT_QUERY_PAUSE_OPERATION = void 0;
+exports.pauseNativeFullTextQueryReaders = pauseNativeFullTextQueryReaders;
+exports.resumeNativeFullTextQueryReaders = resumeNativeFullTextQueryReaders;
+const node_crypto_1 = require("node:crypto");
+const ordered_binary_1 = require("ordered-binary");
+const hdbError_ts_1 = require("../../utility/errors/hdbError.js");
+const derivedIndexRuntime_ts_1 = require("../derivedIndexRuntime.js");
+const fullTextNativeBinding_ts_1 = require("./fullTextNativeBinding.js");
+const nativeFullTextDerivedIndexLifecycle_ts_1 = require("./nativeFullTextDerivedIndexLifecycle.js");
+const fullTextDerivedIndex_ts_1 = require("./fullTextDerivedIndex.js");
+const fullTextQueryProtocol_ts_1 = require("./fullTextQueryProtocol.js");
+var fullTextQueryProtocol_ts_2 = require("./fullTextQueryProtocol.js");
+Object.defineProperty(exports, "FULL_TEXT_QUERY_PAUSE_OPERATION", { enumerable: true, get: function () { return fullTextQueryProtocol_ts_2.FULL_TEXT_QUERY_PAUSE_OPERATION; } });
+Object.defineProperty(exports, "FULL_TEXT_QUERY_RESUME_OPERATION", { enumerable: true, get: function () { return fullTextQueryProtocol_ts_2.FULL_TEXT_QUERY_RESUME_OPERATION; } });
+Object.defineProperty(exports, "fullTextComparatorMode", { enumerable: true, get: function () { return fullTextQueryProtocol_ts_2.fullTextComparatorMode; } });
+const hnswDerivedIndex_ts_1 = require("./hnswDerivedIndex.js");
+const logger_ts_1 = require("../../utility/logging/logger.js");
+const RAW_PAGE_SIZE = 256;
+const MIN_RAW_PAGE_SIZE = 32;
+const RAW_PAGE_OVERFETCH_FACTOR = 2;
+const MAX_RELOAD_FAILURES_BEFORE_REOPEN = 3;
+const HIGHLIGHT_BLOB_READ_TIMEOUT_MILLISECONDS = 5_000;
+const READER_DRAIN_GRACE_MILLISECONDS = 1_000;
+const UTF8_DECODER = new TextDecoder('utf-8', { fatal: true });
+const logger = (0, logger_ts_1.loggerWithTag)('fulltext-query-index');
+class FullTextReaderPublicationError extends Error {
+}
+const queryIndexesByPath = new Map();
+const pausedQueryPaths = new Map();
+const retiredReadinessIdsByPath = new Map();
+const MAX_RETIRED_READINESS_IDS_PER_PATH = 64;
+class FullTextQueryIndex {
+    filteredSearch = true;
+    filePrimary = true;
+    #options;
+    #nativeOptions;
+    #definition;
+    #definitionSnapshot;
+    #configurationRevision = 0;
+    #binding;
+    #readerSlot;
+    #readerOperation;
+    #retiredReaderSlots = new Set();
+    #maxSearchWindow;
+    #maxAutocompleteResults;
+    #maxSearchBudgetMilliseconds;
+    #maxTraceRecords;
+    #maxTraceSourceBytes;
+    #closed = false;
+    #pauses = new Map();
+    #foreignPausesObservedBeforeReady = new Map();
+    #derivedHost;
+    #derivedHostReady = Promise.withResolvers();
+    #publicationSubscription;
+    #publicationRebuildRequested = false;
+    #refreshFailureWarned = false;
+    #unexpectedSearchFailureWarned = false;
+    constructor(options) {
+        this.#options = options;
+        this.#definition = options.definition;
+        this.#definitionSnapshot = queryDefinitionSnapshot(options.definition);
+        this.#nativeOptions = {
+            path: (0, nativeFullTextDerivedIndexLifecycle_ts_1.nativeFullTextIndexPath)(options.storePath, options.storeName),
+            indexId: options.indexId,
+            generation: digestGeneration(options.sourceGeneration),
+            fields: options.definition.fields.map(({ name, weight }) => ({ name, weight })),
+            analyzer: options.definition.analyzer,
+            stopWords: options.definition.stopWords,
+            positions: options.definition.positions,
+            surfaceTerms: options.definition.surfaceTerms,
+            synonyms: options.definition.synonyms.map(({ source, replacements }) => ({
+                source,
+                replacements: [...replacements],
+            })),
+            limits: { ...options.limits },
+        };
+        let indexes = queryIndexesByPath.get(this.#nativeOptions.path);
+        if (!indexes)
+            queryIndexesByPath.set(this.#nativeOptions.path, (indexes = new Set()));
+        indexes.add(this);
+        const retiredReadinessIds = retiredReadinessIdsByPath.get(this.#nativeOptions.path);
+        if (retiredReadinessIds?.delete(options.readinessId) && retiredReadinessIds.size === 0)
+            retiredReadinessIdsByPath.delete(this.#nativeOptions.path);
+        const pauses = pausedQueryPaths.get(this.#nativeOptions.path);
+        if (pauses) {
+            this.#pauses = new Map(pauses);
+            if ((0, derivedIndexRuntime_ts_1.readDerivedIndexReadiness)(options.auditStore, options.readinessId).state !== 'ready')
+                for (const [readinessId, ownerEpoch] of pauses)
+                    if (readinessId !== options.readinessId)
+                        this.#foreignPausesObservedBeforeReady.set(readinessId, ownerEpoch);
+        }
+    }
+    updateDefinition(definition) {
+        const snapshot = queryDefinitionSnapshot(definition);
+        if (snapshot === this.#definitionSnapshot)
+            return;
+        this.#definition = definition;
+        this.#definitionSnapshot = snapshot;
+        this.#nativeOptions.fields = definition.fields.map(({ name, weight }) => ({ name, weight }));
+        this.#configurationRevision++;
+    }
+    estimateCount() {
+        const store = this.#options.Table.primaryStore;
+        const records = typeof store.getEstimatedKeyCount === 'function' ? store.getEstimatedKeyCount() : store.getStats?.().entryCount;
+        return Math.max(1, Math.round((Number.isFinite(records) ? records : 20) * 0.05));
+    }
+    search(condition, context, options = {}) {
+        const maxIndexLagMilliseconds = condition.maxIndexLagMilliseconds ?? hnswDerivedIndex_ts_1.DEFAULT_MAX_INDEX_LAG_MILLISECONDS;
+        const waitForIndexMilliseconds = condition.waitForIndexMilliseconds ?? 0;
+        assertFreshnessOptions(maxIndexLagMilliseconds, waitForIndexMilliseconds);
+        const waiting = waitForIndexMilliseconds > 0;
+        let coverage;
+        const execute = async () => {
+            assertReady((0, derivedIndexRuntime_ts_1.readDerivedIndexReadiness)(this.#options.auditStore, this.#options.readinessId), this.#definition.name);
+            if (waiting) {
+                await context?.indexSearchStart;
+                context?.signal?.throwIfAborted();
+                const waitDeadline = performance.now() + waitForIndexMilliseconds;
+                const started = (0, derivedIndexRuntime_ts_1.derivedIndexTime)(this.#options.Table.primaryStore.rootStore);
+                let host = this.#derivedHost;
+                if (!host) {
+                    const remaining = waitDeadline - performance.now();
+                    if (remaining <= 0)
+                        throw new hdbError_ts_1.DerivedIndexLagError('Timed out waiting for the local full-text query host; retry this query');
+                    host = await this.#waitForDerivedHost(remaining, context?.signal);
+                }
+                const state = host?.readiness().state;
+                if (state !== 'ready')
+                    throw new hdbError_ts_1.ServerError(`Full-text index '${this.#definition.name}' is ${state === 'unavailable' ? 'unavailable' : 'rebuilding'}`, 503);
+                context?.signal?.throwIfAborted();
+                if (options.minResults === 0) {
+                    this.#unexpectedSearchFailureWarned = false;
+                    return [];
+                }
+                if (this.#readCoverage(0).state !== 'current') {
+                    await host.waitForCoverage(started, Math.max(0, waitDeadline - performance.now()), context?.signal);
+                }
+                context?.signal?.throwIfAborted();
+                const result = await this.#search(condition, context, options);
+                this.#unexpectedSearchFailureWarned = false;
+                return result;
+            }
+            coverage = this.#queryCoverage(maxIndexLagMilliseconds);
+            const result = await this.#search(condition, context, options);
+            this.#unexpectedSearchFailureWarned = false;
+            return result;
+        };
+        const operation = execute().catch((error) => {
+            if (error === context?.signal?.reason)
+                throw error;
+            if (nativeErrorNeedsRebuild(error)) {
+                this.#requestRebuild();
+                void this.#retireAllReaders().catch(() => undefined);
+            }
+            const publicError = publicSearchError(error, this.#definition.name);
+            if (publicError !== error &&
+                publicError instanceof hdbError_ts_1.ServerError &&
+                publicError.statusCode === 500 &&
+                !this.#unexpectedSearchFailureWarned) {
+                this.#unexpectedSearchFailureWarned = true;
+                logger.error?.(`Unexpected full-text search failure on '${this.#definition.name}'`, error);
+            }
+            throw publicError;
+        });
+        if (!waiting && coverage)
+            Object.defineProperty(operation, 'indexCoverage', { value: coverage });
+        operation.catch(() => { });
+        return operation;
+    }
+    attachDerivedHost(host) {
+        this.#derivedHost = host;
+        this.#derivedHostReady.resolve(host);
+    }
+    async #waitForDerivedHost(timeout, signal) {
+        signal?.throwIfAborted();
+        let timer;
+        let abort;
+        const unavailable = new Promise((_resolve, reject) => {
+            timer = setTimeout(() => reject(new hdbError_ts_1.DerivedIndexLagError('Timed out waiting for the local full-text query host; retry this query')), timeout);
+            if (signal) {
+                abort = () => reject(signal.reason ?? new Error('Index wait aborted'));
+                signal.addEventListener('abort', abort, { once: true });
+            }
+        });
+        try {
+            return await Promise.race([this.#derivedHostReady.promise, unavailable]);
+        }
+        finally {
+            clearTimeout(timer);
+            if (abort)
+                signal.removeEventListener('abort', abort);
+        }
+    }
+    async close() {
+        this.#closed = true;
+        const publicationSubscription = this.#publicationSubscription;
+        this.#publicationSubscription = undefined;
+        publicationSubscription?.close();
+        const retirement = this.#retireAllReaders();
+        void retirement.then(() => this.#unregister(), () => this.#unregister());
+        await withTimeout(retirement, (this.#maxSearchBudgetMilliseconds ?? 30_000) + READER_DRAIN_GRACE_MILLISECONDS, () => new hdbError_ts_1.ServerError('Full-text reader drain did not settle before timeout', 503));
+    }
+    async pause(readinessId, ownerEpoch) {
+        const current = this.#pauses.get(readinessId);
+        if (current !== undefined && current > ownerEpoch)
+            return;
+        if (current !== ownerEpoch) {
+            logger.info?.(`Pausing full-text readers for '${this.#definition.name}' while '${readinessId}' resets at owner epoch ${ownerEpoch}`);
+            if (readinessId !== this.#options.readinessId &&
+                (0, derivedIndexRuntime_ts_1.readDerivedIndexReadiness)(this.#options.auditStore, this.#options.readinessId).state !== 'ready')
+                this.#foreignPausesObservedBeforeReady.set(readinessId, ownerEpoch);
+            else
+                this.#foreignPausesObservedBeforeReady.delete(readinessId);
+        }
+        this.#pauses.set(readinessId, ownerEpoch);
+        await this.#retireAllReaders();
+    }
+    resume(readinessId, ownerEpoch) {
+        if (!this.#closed && this.#pauses.get(readinessId) === ownerEpoch) {
+            this.#pauses.delete(readinessId);
+            this.#foreignPausesObservedBeforeReady.delete(readinessId);
+        }
+    }
+    async #search(condition, context, options) {
+        const readiness = (0, derivedIndexRuntime_ts_1.readDerivedIndexReadiness)(this.#options.auditStore, this.#options.readinessId);
+        assertReady(readiness, this.#definition.name);
+        if (!this.#derivedHost)
+            throw new hdbError_ts_1.IndexRebuildingError(`Full-text index '${this.#definition.name}' is not ready`);
+        if (options.minResults === 0)
+            return [];
+        const publicationGeneration = this.#publicationRevision();
+        const acquisitionTimeout = Math.min(hnswDerivedIndex_ts_1.MAX_WAIT_FOR_INDEX_MILLISECONDS, Math.max(1, condition.waitForIndexMilliseconds ?? hnswDerivedIndex_ts_1.MAX_WAIT_FOR_INDEX_MILLISECONDS));
+        const acquisition = this.#acquireReader(readiness.ownerEpoch, publicationGeneration);
+        let abortListener;
+        const abort = context?.signal
+            ? new Promise((_resolve, reject) => {
+                abortListener = () => reject(context.signal.reason ?? new Error('Full-text search aborted'));
+                if (context.signal.aborted)
+                    abortListener();
+                else
+                    context.signal.addEventListener('abort', abortListener, { once: true });
+            })
+            : undefined;
+        let lease;
+        try {
+            lease = await withTimeout(abort ? Promise.race([acquisition, abort]) : acquisition, acquisitionTimeout, () => new hdbError_ts_1.ServerError('Full-text reader acquisition exceeded the index-wait budget', 503));
+        }
+        catch (error) {
+            void acquisition.then((orphaned) => orphaned.release(), () => { });
+            throw error;
+        }
+        finally {
+            if (abortListener)
+                context.signal.removeEventListener('abort', abortListener);
+        }
+        const reader = lease.reader;
+        try {
+            const deadline = performance.now() + this.#maxSearchBudgetMilliseconds;
+            const maxSearchWindow = this.#maxSearchWindow;
+            const query = condition.fullTextQuery ?? leafExpression(condition);
+            const autocomplete = expressionUsesAutocomplete(query);
+            const searchWindow = autocomplete ? this.#maxAutocompleteResults : maxSearchWindow;
+            const bounded = options.minResults !== undefined;
+            const target = bounded ? Math.max(1, options.minResults) : searchWindow;
+            if (target > searchWindow)
+                throw new hdbError_ts_1.ClientError(`Full-text query exceeds the ${searchWindow}-result ${autocomplete ? 'autocomplete ' : ''}search window; reduce offset or limit`, 400);
+            const accepted = [];
+            let offset = 0;
+            let moreMayExist = false;
+            let staleVersionHits = 0;
+            let expiredHits = 0;
+            const transaction = context && this.#options.Table._readTxnForContext(context);
+            while (accepted.length < target && offset < searchWindow) {
+                if (context?.signal?.aborted)
+                    throw context.signal.reason ?? new Error('Full-text search aborted');
+                const desiredPageSize = bounded
+                    ? Math.max(MIN_RAW_PAGE_SIZE, (target - accepted.length) * RAW_PAGE_OVERFETCH_FACTOR)
+                    : RAW_PAGE_SIZE;
+                const limit = autocomplete ? searchWindow : Math.min(RAW_PAGE_SIZE, desiredPageSize, searchWindow - offset);
+                const result = await reader.search({
+                    query,
+                    offset,
+                    limit,
+                    ...(!bounded && offset === 0 ? { exactTotal: true } : null),
+                }, { remainingBudgetMilliseconds: remainingSearchBudget(deadline) });
+                if (!bounded && !autocomplete && result.totalRelation === 'exact' && result.total > searchWindow)
+                    throw new hdbError_ts_1.ClientError(`Full-text query exceeds the ${searchWindow}-result search window; add a limit`, 400);
+                moreMayExist =
+                    result.totalRelation === 'exact' ? offset + result.hits.length < result.total : result.hits.length === limit;
+                if (result.hits.length === 0)
+                    break;
+                for (const hit of result.hits) {
+                    const key = decodeNativeId(hit.id, this.#options.Table.tableId);
+                    options.assertTransactionActive?.();
+                    const entry = this.#options.Table.primaryStore.getEntry(key, { transaction });
+                    if (typeof hit.version !== 'string')
+                        throw new hdbError_ts_1.ServerError('Full-text index returned a hit without a source version', 500);
+                    if (!entry?.value || hit.version !== String(entry.version)) {
+                        staleVersionHits++;
+                        continue;
+                    }
+                    if (this.#options.Table.isFullTextSearchEntryCurrent?.(entry) === false) {
+                        staleVersionHits++;
+                        continue;
+                    }
+                    if (entry.expiresAt !== undefined && entry.expiresAt < Date.now()) {
+                        expiredHits++;
+                        continue;
+                    }
+                    if (options.filter && !options.filter(key, entry))
+                        continue;
+                    accepted.push({ key, $score: hit.score, nativeId: hit.id, record: entry.value, recordEntry: entry });
+                    if (accepted.length >= target)
+                        break;
+                }
+                offset += result.hits.length;
+                if (autocomplete)
+                    break;
+                if (!moreMayExist)
+                    break;
+                if (accepted.length < target)
+                    await new Promise((resolve) => setImmediate(resolve));
+            }
+            if (!bounded && moreMayExist)
+                throw new hdbError_ts_1.ClientError(`Full-text query exceeds the ${searchWindow}-result search window; add a limit`, 400);
+            if (bounded && accepted.length < target && moreMayExist && staleVersionHits > 0)
+                throw new hdbError_ts_1.DerivedIndexLagError(`Full-text index '${this.#definition.name}' changed while searching; retry this query`);
+            if (bounded && accepted.length < target && moreMayExist)
+                throw new hdbError_ts_1.ClientError(expiredHits > 0
+                    ? `Expired full-text matches exhausted the ${searchWindow}-result search window; retry after index cleanup or narrow the query`
+                    : `Full-text filters exhausted the ${searchWindow}-result search window; narrow the query or reduce offset`, 400);
+            if (condition.includeHighlights && accepted.length > 0)
+                await this.#addHighlights(reader, lease.definition, condition, accepted.slice(options.resultOffset ?? 0), deadline);
+            return accepted.map((entry) => ({
+                key: entry.key,
+                $score: entry.$score,
+                loadedEntry: entry.recordEntry,
+                ...(entry.$highlights ? { $highlights: entry.$highlights } : null),
+            }));
+        }
+        finally {
+            await lease.release();
+        }
+    }
+    async #addHighlights(reader, definition, condition, entries, deadline) {
+        const highlighting = definition.highlighting;
+        if (!highlighting)
+            return;
+        const highlightFields = new Set(definition.fields.filter((field) => field.highlight === true).map((field) => field.name));
+        if (highlightFields.size === 0)
+            return;
+        const leaves = (condition.fullTextLeaves ?? [leafDescriptor(condition)]).map((leaf) => ({
+            ...leaf,
+            fields: leaf.fields?.filter((field) => highlightFields.has(field)) ?? [...highlightFields],
+        }));
+        const byId = new Map(entries.map((entry) => [entry.nativeId, entry]));
+        const leafGroups = new Map();
+        for (const leaf of leaves) {
+            if (leaf.fields.length === 0)
+                continue;
+            const key = JSON.stringify([...leaf.fields].sort());
+            let group = leafGroups.get(key);
+            if (!group) {
+                const selected = new Set(leaf.fields);
+                const fields = definition.fields.map(({ name }) => name).filter((name) => selected.has(name));
+                leafGroups.set(key, (group = { fields, leaves: [] }));
+            }
+            group.leaves.push(leaf);
+        }
+        for (const { fields, leaves: fieldLeaves } of leafGroups.values()) {
+            const leafFields = new Set(fields);
+            for (const sourceEntries of traceEntryBatches(entries, definition, leafFields, this.#maxTraceRecords, this.#maxTraceSourceBytes)) {
+                const records = await Promise.all(sourceEntries.map(async ({ nativeId, record }) => ({
+                    id: nativeId,
+                    fields: await sourceFields(record, definition, leafFields, deadline),
+                })));
+                for (const leaf of fieldLeaves) {
+                    for (const traceRecords of traceSourceBatches(records, this.#maxTraceSourceBytes)) {
+                        const traced = await reader.traceMatches({ text: leaf.text, mode: leaf.mode, fields }, traceRecords, {
+                            remainingBudgetMilliseconds: remainingSearchBudget(deadline),
+                            snippets: true,
+                            fragmentLength: highlighting.fragmentLength,
+                            maxFragmentsPerValue: highlighting.maxFragments,
+                        });
+                        if (!traced.complete)
+                            throw new hdbError_ts_1.ServerError('Full-text index returned incomplete highlights', 500);
+                        for (const record of traced.records) {
+                            const entry = byId.get(record.id);
+                            if (!entry)
+                                continue;
+                            const highlights = (entry.$highlights ??= Object.create(null));
+                            for (const value of record.values) {
+                                if (!leafFields.has(value.field))
+                                    continue;
+                                const values = (highlights[value.field] ??= []);
+                                values.push({
+                                    valueIndex: value.valueIndex,
+                                    spans: value.spans,
+                                    ...(value.fragments ? { fragments: value.fragments } : null),
+                                });
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+    async #acquireReader(ownerEpoch, publicationGeneration) {
+        const slot = await this.#readerFor(ownerEpoch, publicationGeneration, true);
+        let released = false;
+        return {
+            reader: slot.reader,
+            definition: slot.definition,
+            release: () => {
+                if (released)
+                    return Promise.resolve();
+                released = true;
+                slot.active--;
+                if (slot.active === 0) {
+                    for (const resolve of slot.idleWaiters?.splice(0) ?? [])
+                        resolve();
+                    if (slot.retired) {
+                        try {
+                            void this.#closeReaderSlot(slot).catch((error) => this.#warnReaderClose(error));
+                        }
+                        catch { }
+                    }
+                }
+                return Promise.resolve();
+            },
+        };
+    }
+    #readerFor(ownerEpoch, publicationGeneration, reserve = false) {
+        if (this.#closed)
+            return Promise.reject(new hdbError_ts_1.ServerError('Full-text index is closed', 503));
+        let ownReadiness;
+        let hasForeignPause = false;
+        for (const [readinessId, pausedEpoch] of this.#pauses) {
+            const readiness = (0, derivedIndexRuntime_ts_1.readDerivedIndexReadiness)(this.#options.auditStore, readinessId);
+            const completed = readiness.state === 'ready' && readiness.ownerEpoch >= pausedEpoch;
+            const successorCompleted = readinessId !== this.#options.readinessId &&
+                this.#foreignPausesObservedBeforeReady.get(readinessId) === pausedEpoch &&
+                (ownReadiness ??= (0, derivedIndexRuntime_ts_1.readDerivedIndexReadiness)(this.#options.auditStore, this.#options.readinessId)).state ===
+                    'ready';
+            if (completed || successorCompleted) {
+                resumeNativeFullTextQueryReaders(this.#nativeOptions.path, readinessId, pausedEpoch);
+            }
+            else if (readinessId !== this.#options.readinessId)
+                hasForeignPause = true;
+        }
+        if (hasForeignPause) {
+            const { auditStore, indexId, readinessId } = this.#options;
+            const lockKey = `derived-index:${indexId}:runner`;
+            if (auditStore.tryLock(lockKey)) {
+                try {
+                    // Every generation holds this lock through reset and shutdown, so acquisition proves reset quiescence.
+                    if ((0, derivedIndexRuntime_ts_1.readDerivedIndexReadiness)(auditStore, readinessId).state === 'ready')
+                        for (const [pausedId, pausedEpoch] of [...this.#pauses])
+                            if (pausedId !== readinessId)
+                                resumeNativeFullTextQueryReaders(this.#nativeOptions.path, pausedId, pausedEpoch);
+                }
+                finally {
+                    auditStore.unlock(lockKey);
+                }
+            }
+        }
+        const blocked = this.#pauses.entries().next();
+        if (!blocked.done) {
+            const [readinessId, ownerEpoch] = blocked.value;
+            return Promise.reject(new hdbError_ts_1.IndexRebuildingError(`Full-text index '${this.#definition.name}' is waiting for reader fence '${readinessId}' at owner epoch ${ownerEpoch}`));
+        }
+        const current = this.#readerSlot;
+        if (!this.#readerOperation &&
+            current &&
+            !current.retired &&
+            current.ownerEpoch === ownerEpoch &&
+            current.publicationGeneration >= publicationGeneration &&
+            current.configurationRevision === this.#configurationRevision) {
+            if (reserve)
+                current.active++;
+            return Promise.resolve(current);
+        }
+        if (this.#readerOperation)
+            return this.#readerOperation.then(() => this.#readerFor(ownerEpoch, publicationGeneration, reserve));
+        const configurationRevision = this.#configurationRevision;
+        this.#readerOperation = (async () => {
+            const existing = this.#readerSlot;
+            const existingMatchesIdentity = existing?.ownerEpoch === ownerEpoch && existing.configurationRevision === configurationRevision;
+            // Installing this operation synchronously prevents leases after observing a newer publication.
+            const canReload = existing?.active === 0 && existingMatchesIdentity;
+            try {
+                let reader;
+                let publication;
+                if (canReload) {
+                    const reloadedPublicationGeneration = this.#publicationRevision();
+                    await existing.reader.reload();
+                    existing.publication = this.#decodeReaderPublication(existing.reader);
+                    existing.reloadFailures = 0;
+                    existing.publicationGeneration = reloadedPublicationGeneration;
+                    return existing;
+                }
+                else {
+                    const binding = await this.#getBinding();
+                    reader = await binding.openNativeFullTextReader(this.#nativeOptions);
+                    try {
+                        publication = this.#decodeReaderPublication(reader);
+                    }
+                    catch (error) {
+                        await reader.close().catch((closeError) => this.#warnReaderClose(closeError));
+                        throw error;
+                    }
+                    if (existing)
+                        this.#retireReaderSlot(existing);
+                }
+                const slot = {
+                    reader,
+                    definition: this.#definition,
+                    ownerEpoch,
+                    publicationGeneration,
+                    publication,
+                    configurationRevision,
+                    active: 0,
+                    retired: false,
+                    reloadFailures: 0,
+                };
+                this.#readerSlot = slot;
+                return slot;
+            }
+            catch (error) {
+                if (canReload && error instanceof FullTextReaderPublicationError) {
+                    this.#retireReaderSlot(existing);
+                }
+                else if (canReload && ++existing.reloadFailures >= MAX_RELOAD_FAILURES_BEFORE_REOPEN) {
+                    this.#retireReaderSlot(existing);
+                }
+                else if (!canReload && existing && existing.active === 0 && !existingMatchesIdentity) {
+                    this.#readerSlot = undefined;
+                    this.#retireReaderSlot(existing);
+                }
+                throw error;
+            }
+            finally {
+                this.#readerOperation = undefined;
+            }
+        })();
+        return this.#readerOperation.then(() => this.#readerFor(ownerEpoch, publicationGeneration, reserve));
+    }
+    #decodeReaderPublication(reader) {
+        try {
+            if (reader.committedPayload === undefined)
+                throw new Error('missing publication');
+            const publication = (0, fullTextDerivedIndex_ts_1.decodeFullTextPublication)(reader.committedPayload);
+            if (!publication.cursor)
+                throw new Error('missing durable cursor');
+            this.#publicationRebuildRequested = false;
+            return publication;
+        }
+        catch (cause) {
+            this.#requestRebuild();
+            throw new FullTextReaderPublicationError(`Full-text index '${this.#definition.name}' has no valid search checkpoint`, { cause });
+        }
+    }
+    #refreshReader(openCold) {
+        try {
+            if (this.#closed || (!openCold && !this.#readerSlot && !this.#readerOperation))
+                return;
+            const readiness = (0, derivedIndexRuntime_ts_1.readDerivedIndexReadiness)(this.#options.auditStore, this.#options.readinessId);
+            if (readiness.state !== 'ready')
+                return;
+            void this.#readerFor(readiness.ownerEpoch, this.#publicationRevision()).catch((error) => {
+                if (nativeErrorNeedsRebuild(error))
+                    this.#requestRebuild();
+            });
+            this.#refreshFailureWarned = false;
+        }
+        catch (error) {
+            if (this.#refreshFailureWarned)
+                return;
+            this.#refreshFailureWarned = true;
+            try {
+                logger.warn?.('Could not refresh a full-text reader after native publication', error);
+            }
+            catch { }
+        }
+    }
+    #publicationRevision() {
+        if (this.#closed)
+            throw new hdbError_ts_1.ServerError('Full-text index is closed', 503);
+        if (!this.#publicationSubscription)
+            this.#publicationSubscription = (0, derivedIndexRuntime_ts_1.subscribeDerivedIndexPublications)(this.#options.auditStore, this.#options.readinessId, () => this.#refreshReader(false));
+        return this.#publicationSubscription.revision();
+    }
+    #requestRebuild() {
+        const host = this.#derivedHost;
+        if (!host || this.#publicationRebuildRequested)
+            return;
+        try {
+            if (host.requestRebuild())
+                this.#publicationRebuildRequested = true;
+        }
+        catch (error) {
+            try {
+                logger.warn?.(`Could not request a rebuild for full-text index '${this.#definition.name}'`, error);
+            }
+            catch { }
+        }
+    }
+    #retireReaderSlot(slot) {
+        if (slot.retired)
+            return;
+        slot.retired = true;
+        this.#retiredReaderSlots.add(slot);
+        if (this.#readerSlot === slot)
+            this.#readerSlot = undefined;
+        if (slot.active === 0)
+            this.#closeReaderSlot(slot).catch((error) => this.#warnReaderClose(error));
+    }
+    #closeReaderSlot(slot) {
+        if (!slot.closeOperation) {
+            const operation = Promise.resolve()
+                .then(() => slot.reader.close())
+                .then((result) => {
+                this.#retiredReaderSlots.delete(slot);
+                if (result?.cleanupError)
+                    this.#warnReaderClose(result.cleanupError);
+            });
+            slot.closeOperation = operation;
+            void operation.catch(() => {
+                if (slot.closeOperation === operation)
+                    slot.closeOperation = undefined;
+            });
+        }
+        return slot.closeOperation;
+    }
+    async #retireAllReaders() {
+        await this.#readerOperation?.catch(() => undefined);
+        if (this.#readerSlot)
+            this.#retireReaderSlot(this.#readerSlot);
+        await Promise.all([...this.#retiredReaderSlots].map((slot) => this.#closeWhenIdle(slot)));
+    }
+    #unregister() {
+        const path = this.#nativeOptions.path;
+        const indexes = queryIndexesByPath.get(path);
+        indexes?.delete(this);
+        const sameReadinessRemains = [...(indexes ?? [])].some((index) => index.#options.readinessId === this.#options.readinessId);
+        if (!sameReadinessRemains)
+            rememberRetiredReadinessId(path, this.#options.readinessId);
+        if (!indexes || indexes.size === 0) {
+            queryIndexesByPath.delete(path);
+            pausedQueryPaths.delete(path);
+            retiredReadinessIdsByPath.delete(path);
+            return;
+        }
+        if (sameReadinessRemains)
+            return;
+        const ownerEpoch = pausedQueryPaths.get(path)?.get(this.#options.readinessId);
+        if (ownerEpoch === undefined)
+            return;
+        clearPathPause(path, this.#options.readinessId, ownerEpoch);
+        for (const index of indexes)
+            index.resume(this.#options.readinessId, ownerEpoch);
+    }
+    async #closeWhenIdle(slot) {
+        if (slot.active > 0)
+            await new Promise((resolve) => (slot.idleWaiters ??= []).push(resolve));
+        await this.#closeReaderSlot(slot).catch((error) => this.#warnReaderClose(error));
+    }
+    #warnReaderClose(error) {
+        try {
+            logger.warn?.('Could not close a retired full-text reader; the reader remains retired', error);
+        }
+        catch { }
+    }
+    async #getBinding() {
+        if (this.#binding)
+            return this.#binding;
+        const configured = this.#options.binding;
+        const binding = configured
+            ? typeof configured === 'function'
+                ? await configured()
+                : configured
+            : await (0, fullTextNativeBinding_ts_1.loadFullTextNativeBinding)();
+        const info = (0, fullTextNativeBinding_ts_1.validateFullTextQueryRuntimeInfo)(await binding.runtimeInfo());
+        this.#nativeOptions.limits.searchThreads = (0, fullTextNativeBinding_ts_1.nativeFullTextSearchThreads)(info, this.#nativeOptions.limits.searchThreads);
+        this.#maxSearchWindow = info.limits.maxSearchWindow;
+        this.#maxAutocompleteResults = info.limits.maxAutocompleteResults;
+        this.#maxSearchBudgetMilliseconds = info.limits.maxSearchBudgetMilliseconds;
+        this.#maxTraceRecords = info.limits.maxTraceRecords;
+        this.#maxTraceSourceBytes = info.limits.maxTraceSourceBytes;
+        return (this.#binding = binding);
+    }
+    #queryCoverage(maxLagMilliseconds) {
+        const coverage = this.#readCoverage(maxLagMilliseconds);
+        if (!coverage || coverage.state === 'unknown') {
+            this.#refreshReader(true);
+            const age = coverage?.lagUpperBoundMilliseconds;
+            throw new hdbError_ts_1.DerivedIndexLagError(`Cannot certify full-text index coverage within ${maxLagMilliseconds} ms` +
+                (age === undefined ? '; freshness is unknown' : `; last certified ${Math.ceil(age)} ms ago`) +
+                '; retry this query');
+        }
+        return coverage;
+    }
+    #readCoverage(maxLagMilliseconds) {
+        return (0, derivedIndexRuntime_ts_1.readDerivedIndexCoverage)(this.#options.auditStore, this.#options.readinessId, () => this.#readerSlot?.publication.cursor, maxLagMilliseconds);
+    }
+}
+exports.FullTextQueryIndex = FullTextQueryIndex;
+async function pauseNativeFullTextQueryReaders(path, readinessId, ownerEpoch, allowUnregisteredReadiness = false) {
+    const indexes = queryIndexesByPath.get(path);
+    if (!indexes || indexes.size === 0)
+        return;
+    if (!allowUnregisteredReadiness && retiredReadinessIdsByPath.get(path)?.has(readinessId))
+        return;
+    let pauses = pausedQueryPaths.get(path);
+    if (!pauses)
+        pausedQueryPaths.set(path, (pauses = new Map()));
+    const current = pauses.get(readinessId);
+    if (current === undefined || current <= ownerEpoch)
+        pauses.set(readinessId, ownerEpoch);
+    await Promise.all([...indexes].map((index) => index.pause(readinessId, ownerEpoch)));
+}
+function resumeNativeFullTextQueryReaders(path, readinessId, ownerEpoch) {
+    clearPathPause(path, readinessId, ownerEpoch);
+    for (const index of queryIndexesByPath.get(path) ?? [])
+        index.resume(readinessId, ownerEpoch);
+}
+function clearPathPause(path, readinessId, ownerEpoch) {
+    const pauses = pausedQueryPaths.get(path);
+    if (pauses?.get(readinessId) !== ownerEpoch)
+        return;
+    pauses.delete(readinessId);
+    if (pauses.size === 0)
+        pausedQueryPaths.delete(path);
+}
+function rememberRetiredReadinessId(path, readinessId) {
+    let readinessIds = retiredReadinessIdsByPath.get(path);
+    if (!readinessIds)
+        retiredReadinessIdsByPath.set(path, (readinessIds = new Set()));
+    readinessIds.delete(readinessId);
+    readinessIds.add(readinessId);
+    if (readinessIds.size > MAX_RETIRED_READINESS_IDS_PER_PATH)
+        readinessIds.delete(readinessIds.values().next().value);
+}
+function assertFreshnessOptions(maxIndexLagMilliseconds, waitForIndexMilliseconds) {
+    if (typeof maxIndexLagMilliseconds !== 'number' ||
+        !Number.isFinite(maxIndexLagMilliseconds) ||
+        maxIndexLagMilliseconds < 0)
+        throw new hdbError_ts_1.ClientError('maxIndexLagMilliseconds must be a finite nonnegative number', 400);
+    if (typeof waitForIndexMilliseconds !== 'number' ||
+        !Number.isFinite(waitForIndexMilliseconds) ||
+        waitForIndexMilliseconds < 0 ||
+        waitForIndexMilliseconds > hnswDerivedIndex_ts_1.MAX_WAIT_FOR_INDEX_MILLISECONDS)
+        throw new hdbError_ts_1.ClientError(`waitForIndexMilliseconds must be a finite number between 0 and ${hnswDerivedIndex_ts_1.MAX_WAIT_FOR_INDEX_MILLISECONDS}`, 400);
+}
+function leafExpression(condition) {
+    const leaf = leafDescriptor(condition);
+    return { text: leaf.text, mode: leaf.mode, ...(leaf.fields ? { fields: leaf.fields } : null) };
+}
+function expressionUsesAutocomplete(expression) {
+    if ('text' in expression)
+        return expression.mode === 'prefix' || expression.mode === 'fuzzy-prefix';
+    if (expression.operator === 'not')
+        return expressionUsesAutocomplete(expression.clause);
+    return expression.clauses.some(expressionUsesAutocomplete);
+}
+function leafDescriptor(condition) {
+    const mode = (0, fullTextQueryProtocol_ts_1.fullTextComparatorMode)(condition.comparator);
+    if (!mode || typeof condition.value !== 'string' || condition.value.length === 0)
+        throw new hdbError_ts_1.ClientError('Full-text conditions require a supported comparator and non-empty string value', 400);
+    return { text: condition.value, mode, ...(condition.fields ? { fields: condition.fields } : null) };
+}
+async function sourceFields(record, definition, selected, queryDeadline) {
+    const fields = Object.create(null);
+    const deadline = Math.min(queryDeadline, performance.now() + HIGHLIGHT_BLOB_READ_TIMEOUT_MILLISECONDS);
+    for (const source of definition.fields) {
+        if (!selected.has(source.name))
+            continue;
+        const value = record[source.name];
+        if (typeof value === 'string')
+            fields[source.name] = value;
+        else if (Array.isArray(value))
+            fields[source.name] = value.filter((entry) => typeof entry === 'string');
+        else if (source.mediaType === 'text/plain' && value instanceof Blob) {
+            const remaining = deadline - performance.now();
+            if (remaining <= 0)
+                throw new hdbError_ts_1.DerivedIndexLagError('Full-text highlight source read timed out; retry the query');
+            let bytes;
+            try {
+                bytes = await withTimeout(value.arrayBuffer(), remaining);
+            }
+            catch {
+                throw new hdbError_ts_1.DerivedIndexLagError('Full-text highlight source is temporarily unavailable; retry the query');
+            }
+            fields[source.name] = UTF8_DECODER.decode(bytes);
+        }
+    }
+    return fields;
+}
+function traceSourceBytes(fields) {
+    let bytes = 0;
+    for (const value of Object.values(fields)) {
+        if (typeof value === 'string')
+            bytes += Buffer.byteLength(value);
+        else
+            for (const item of value)
+                bytes += Buffer.byteLength(item);
+    }
+    return bytes;
+}
+function traceEntryBytes(record, definition, selected) {
+    let bytes = 0;
+    for (const source of definition.fields) {
+        if (!selected.has(source.name))
+            continue;
+        const value = record[source.name];
+        if (typeof value === 'string')
+            bytes += Buffer.byteLength(value);
+        else if (Array.isArray(value)) {
+            for (const item of value)
+                if (typeof item === 'string')
+                    bytes += Buffer.byteLength(item);
+        }
+        else if (source.mediaType === 'text/plain' && value instanceof Blob)
+            bytes += value.size;
+    }
+    return bytes;
+}
+function traceEntryBatches(entries, definition, selected, maxRecords, maxBytes) {
+    const batches = [];
+    let batch = [];
+    let batchBytes = 0;
+    for (const entry of entries) {
+        const bytes = traceEntryBytes(entry.record, definition, selected);
+        if (bytes > maxBytes)
+            throw new hdbError_ts_1.ClientError(`Full-text highlight source exceeds the ${maxBytes}-byte native trace limit`, 400);
+        if (batch.length > 0 && (batch.length >= maxRecords || batchBytes + bytes > maxBytes)) {
+            batches.push(batch);
+            batch = [];
+            batchBytes = 0;
+        }
+        batch.push(entry);
+        batchBytes += bytes;
+    }
+    if (batch.length > 0)
+        batches.push(batch);
+    return batches;
+}
+function traceSourceBatches(records, maxBytes) {
+    const batches = [];
+    let batch = [];
+    let batchBytes = 0;
+    for (const record of records) {
+        const bytes = traceSourceBytes(record.fields);
+        if (bytes > maxBytes)
+            throw new hdbError_ts_1.ClientError(`Full-text highlight source exceeds the ${maxBytes}-byte native trace limit`, 400);
+        if (batch.length > 0 && batchBytes + bytes > maxBytes) {
+            batches.push(batch);
+            batch = [];
+            batchBytes = 0;
+        }
+        batch.push(record);
+        batchBytes += bytes;
+    }
+    if (batch.length > 0)
+        batches.push(batch);
+    return batches;
+}
+function decodeNativeId(id, tableId) {
+    const prefix = `${tableId}.`;
+    if (!id.startsWith(prefix))
+        throw new hdbError_ts_1.ServerError('Full-text index returned an invalid record identity', 500);
+    try {
+        return (0, ordered_binary_1.fromBufferKey)(Buffer.from(id.slice(prefix.length), 'base64url'));
+    }
+    catch {
+        throw new hdbError_ts_1.ServerError('Full-text index returned an invalid record identity', 500);
+    }
+}
+function assertReady(readiness, name) {
+    if (readiness.state === 'ready')
+        return;
+    if (readiness.state === 'unavailable')
+        throw new hdbError_ts_1.ServerError(`Full-text index '${name}' is unavailable`, 503);
+    throw new hdbError_ts_1.IndexRebuildingError(`Full-text index '${name}' is not ready`);
+}
+function publicSearchError(error, name) {
+    if (error instanceof FullTextReaderPublicationError)
+        return new hdbError_ts_1.IndexRebuildingError(`Full-text index '${name}' is not ready`);
+    if (error instanceof hdbError_ts_1.ClientError || error instanceof hdbError_ts_1.ServerError)
+        return error;
+    const code = error && typeof error === 'object' && 'code' in error ? error.code : undefined;
+    if (code === 'E_RELOAD_FAILED')
+        return new hdbError_ts_1.DerivedIndexLagError(`Full-text index '${name}' could not refresh its search snapshot; retry this query`);
+    if (typeof code === 'string' && REBUILD_REQUIRED_CODES.has(code))
+        return new hdbError_ts_1.IndexRebuildingError(`Full-text index '${name}' is not ready`);
+    if (code === 'E_QUEUE_FULL' || code === 'E_TIMEOUT' || code === 'E_RESOURCE_LIMIT')
+        return new hdbError_ts_1.ServerError(`Full-text index '${name}' is busy`, 503);
+    if (code === 'E_INVALID_ARGUMENT' || code === 'E_PREFIX_TOO_BROAD')
+        return new hdbError_ts_1.ClientError(`Full-text query on '${name}' is invalid`, 400);
+    if (code === 'E_RESULT_TOO_LARGE')
+        return new hdbError_ts_1.ClientError(`Full-text query on '${name}' is too large`, 413);
+    return new hdbError_ts_1.ServerError(`Full-text search on '${name}' failed`, 500);
+}
+function nativeErrorNeedsRebuild(error) {
+    const code = error && typeof error === 'object' && 'code' in error ? error.code : undefined;
+    return typeof code === 'string' && REBUILD_REQUIRED_CODES.has(code);
+}
+const REBUILD_REQUIRED_CODES = new Set([
+    'E_INDEX_NOT_READY',
+    'E_INDEX_CORRUPT',
+    'E_IDENTITY_MISMATCH',
+    'E_INCOMPLETE_CREATE',
+    'E_SCHEMA_MISMATCH',
+    'E_INDEX_FORMAT_INCOMPATIBLE',
+]);
+function remainingSearchBudget(deadline) {
+    const remaining = Math.floor(deadline - performance.now());
+    if (remaining <= 0)
+        throw new hdbError_ts_1.ServerError('Full-text search exceeded its execution budget', 503);
+    return remaining;
+}
+function digestGeneration(value) {
+    return (0, node_crypto_1.createHash)('sha256').update(value).digest('hex');
+}
+function queryDefinitionSnapshot(definition) {
+    return JSON.stringify({
+        fields: definition.fields.map(({ name, weight, highlight }) => ({ name, weight, highlight })),
+        highlighting: definition.highlighting,
+    });
+}
+function withTimeout(promise, milliseconds, timeoutError = () => new Error('Full-text highlight source read timed out')) {
+    let timer;
+    return Promise.race([
+        promise,
+        new Promise((_resolve, reject) => {
+            timer = setTimeout(() => reject(timeoutError()), milliseconds);
+            timer.unref?.();
+        }),
+    ]).finally(() => clearTimeout(timer));
+}
+//# sourceMappingURL=fullTextQueryIndex.js.map