@plebeiantech/lat.md 0.12.3-fork.2 → 0.12.3-fork.21

package/dist/src/search/embedder.js

@@ -1,7 +1,8 @@
 import { createEmbedder, EmbeddingAuthError, } from '@lat.md/embed';
 import minilm from '@lat.md/embed-minilm-fp16';
-import { getLlmKey, getRepoEmbedding } from '../config.js';
+import { getLlmKey, getRepoEmbedding } from '@plebeiantech/lat.md-core/config';
 export { EmbeddingAuthError };
+const defaultCreateSearchEngine = (key) => key ? createEmbedder({ key }) : createEmbedder({ model: minilm });
 /** `meta.embedding_model` value for an embedder, e.g. `local:minilm-l6-v2:384`. */
 export function modelKey(embedder) {
     return `${embedder.name}:${embedder.dimensions}`;
@@ -33,15 +34,15 @@ export function localEmbedder() {
  * regenerable `.cache` was wiped) the durable per-repo preference wins over the
  * env, so a repo switched to local stays local across cache loss.
  */
-export async function embedderForIndex(storedModel, latDir) {
+export async function embedderForIndex(storedModel, latDir, createSearchEngine = defaultCreateSearchEngine) {
     if (storedModel === null) {
         if (getRepoEmbedding(latDir) === 'local')
-            return localEmbedder();
+            return createSearchEngine();
         // No durable preference — decide from the environment, record it later.
-        return embedderFromEnv();
+        return createSearchEngine(getLlmKey());
     }
     if (storedModel.startsWith('local:')) {
-        return localEmbedder(); // ignores LAT_LLM_KEY entirely
+        return createSearchEngine(); // ignores LAT_LLM_KEY entirely
     }
     // Remote-pinned index: needs a key that resolves to the same model.
     const key = getLlmKey();
@@ -49,7 +50,7 @@ export async function embedderForIndex(storedModel, latDir) {
         throw new ReindexRequiredError(`This index was built with '${storedModel}', but no LAT_LLM_KEY is set. ` +
             `Run 'lat reindex' to switch to the local model or restore the key.`);
     }
-    const embedder = await createEmbedder({ key });
+    const embedder = await createSearchEngine(key);
     if (modelKey(embedder) !== storedModel) {
         throw new ReindexRequiredError(`This index was built with '${storedModel}', but the current key resolves ` +
             `to '${modelKey(embedder)}'. Run 'lat reindex' to rebuild.`);

package/dist/src/search/index.d.ts

@@ -1,9 +1,13 @@
-import type { Client } from '@libsql/client';
+import { type SearchDb } from './db.js';
+import { type MarkdownProjectAnalysis } from '@plebeiantech/lat.md-core/project-analysis';
 import type { Embedder } from './embedder.js';
+export declare function projectFingerprint(project: MarkdownProjectAnalysis): string;
 export type IndexStats = {
     added: number;
     updated: number;
     removed: number;
     unchanged: number;
 };
-export declare function indexSections(latDir: string, db: Client, embedder: Embedder, onProgress?: (done: number, total: number) => void): Promise<IndexStats>;
+/** Exact identifier tokens complement FTS's punctuation and long-token handling. */
+export declare function identifierTokens(text: string): string[];
+export declare function indexSections(latDir: string, db: SearchDb, embedder: Embedder, onProgress?: (done: number, total: number) => void, analyzedProject?: MarkdownProjectAnalysis): Promise<IndexStats>;

package/dist/src/search/index.js

@@ -1,66 +1,189 @@
-import { createHash } from 'node:crypto';
-import { readFile } from 'node:fs/promises';
-import { dirname, join } from 'node:path';
-import { loadAllSections, flattenSections } from '../lattice.js';
-function hashContent(text) {
-    return createHash('sha256').update(text).digest('hex');
+import { PARSER_CACHE_VERSION } from '@plebeiantech/lat.md-core/parser-cache';
+import { synchronizeLexical } from './lexical.js';
+import { dirname } from 'node:path';
+import { CREATE_PASSAGE_FTS } from './db.js';
+import { analyzeMarkdownProject, } from '@plebeiantech/lat.md-core/project-analysis';
+import { chunkFile, digest, embeddingFingerprint } from './chunks.js';
+export function projectFingerprint(project) {
+    return digest(`parser:${PARSER_CACHE_VERSION}\n` +
+        [...project.files.values()]
+            .map((f) => `${f.projectPath}\0${digest(f.content)}`)
+            .sort()
+            .join('\n'));
 }
-async function sectionContent(section, projectRoot) {
-    const filePath = join(projectRoot, section.filePath);
-    const content = await readFile(filePath, 'utf-8');
-    const lines = content.split('\n');
-    return lines.slice(section.startLine - 1, section.endLine).join('\n');
+/** Exact identifier tokens complement FTS's punctuation and long-token handling. */
+export function identifierTokens(text) {
+    return [
+        ...new Set((text.match(/[\p{L}\p{N}_][\p{L}\p{N}_./:#@-]*/gu) ?? [])
+            .map((t) => t.toLowerCase())
+            .filter((t) => /[_./:#@-]/.test(t) || t.length >= 40)),
+    ];
 }
-export async function indexSections(latDir, db, embedder, onProgress) {
-    const projectRoot = dirname(latDir);
-    const allSections = await loadAllSections(latDir);
-    const flat = flattenSections(allSections);
-    // Build current state: id -> { section, content, hash }
-    const current = new Map();
-    for (const s of flat) {
-        const text = await sectionContent(s, projectRoot);
-        current.set(s.id, { section: s, content: text, hash: hashContent(text) });
+export async function indexSections(latDir, db, embedder, onProgress, analyzedProject) {
+    const project = analyzedProject ??
+        (await analyzeMarkdownProject(latDir, dirname(latDir), {
+            executor: 'auto',
+        }));
+    const fingerprint = embeddingFingerprint(embedder);
+    const projectHash = projectFingerprint(project);
+    const oldFingerprint = (await db.execute("SELECT value FROM meta WHERE key='fingerprint'")).rows[0]?.value;
+    if (oldFingerprint && oldFingerprint !== fingerprint)
+        throw new Error('Search chunking or embedding model changed; run lat reindex.');
+    const byFile = new Map();
+    for (const s of project.sections) {
+        const list = byFile.get(s.filePath) ?? [];
+        list.push(s);
+        byFile.set(s.filePath, list);
     }
-    // Get existing hashes from DB
-    const existing = new Map();
-    const rows = await db.execute('SELECT id, content_hash FROM sections');
-    for (const row of rows.rows) {
-        existing.set(row.id, row.content_hash);
+    const files = new Map([...project.files.values()].map((f) => [f.projectPath, f]));
+    const passages = [...byFile].flatMap(([path, sections]) => {
+        const file = files.get(path);
+        if (!file)
+            throw new Error(`Missing analyzed file: ${path}`);
+        return chunkFile(file, sections, embedder);
+    });
+    const existing = new Map((await db.execute('SELECT id,content_hash FROM sections')).rows.map((r) => [
+        r.id,
+        r.content_hash,
+    ]));
+    const sectionHashes = new Map();
+    const owned = new Map();
+    for (const p of passages) {
+        const list = owned.get(p.sectionId) ?? [];
+        list.push(p);
+        owned.set(p.sectionId, list);
     }
-    // Partition into new, changed, unchanged, deleted
-    const toEmbed = [];
-    let unchanged = 0;
-    for (const [id, entry] of current) {
-        const existingHash = existing.get(id);
-        if (existingHash === entry.hash) {
-            unchanged++;
+    for (const s of project.sections) {
+        sectionHashes.set(s.id, digest(JSON.stringify([
+            { ...s, children: undefined },
+            (owned.get(s.id) ?? []).map((p) => [p.inputHash, p.spans]),
+        ])));
+    }
+    const changed = new Set([...sectionHashes]
+        .filter(([id, hash]) => existing.get(id) !== hash)
+        .map(([id]) => id));
+    const removed = [...existing.keys()].filter((id) => !sectionHashes.has(id));
+    const stats = {
+        added: [...changed].filter((id) => !existing.has(id)).length,
+        updated: [...changed].filter((id) => existing.has(id)).length,
+        removed: removed.length,
+        unchanged: project.sections.length - changed.size,
+    };
+    if (!changed.size && !removed.length && oldFingerprint === fingerprint) {
+        // Lexical maintenance can repair old generations without embedding work.
+        await db.execute('BEGIN');
+        try {
+            await synchronizeLexical(db);
+            await db.execute({
+                sql: "UPDATE meta SET value=? WHERE key='project_hash' AND value<>?",
+                args: [projectHash, projectHash],
+            });
+            await db.execute('COMMIT');
         }
-        else {
-            toEmbed.push({ id, content: entry.content, section: entry.section });
+        catch (error) {
+            await db.execute('ROLLBACK');
+            throw error;
         }
+        return stats;
     }
-    const toDelete = [...existing.keys()].filter((id) => !current.has(id));
-    // Embed new/changed sections
-    if (toEmbed.length > 0) {
-        const texts = toEmbed.map((e) => e.content);
-        const vectors = await embedder.embed(texts, onProgress);
-        const now = Date.now();
-        for (let i = 0; i < toEmbed.length; i++) {
-            const { id, content, section } = toEmbed[i];
-            const hash = current.get(id).hash;
-            const vecJson = JSON.stringify(vectors[i]);
+    const storedHashes = new Set((await db.execute('SELECT hash FROM embeddings')).rows.map((r) => r.hash));
+    const missing = new Map(passages
+        .filter((p) => !storedHashes.has(p.inputHash))
+        .map((p) => [p.inputHash, p.input]));
+    const entries = [...missing];
+    const vectors = entries.length
+        ? await embedder.embed(entries.map(([, input]) => input), onProgress)
+        : [];
+    if (vectors.length !== entries.length ||
+        vectors.some((v) => v.length !== embedder.dimensions || v.some((n) => !Number.isFinite(n))))
+        throw new Error('Embedding backend returned invalid vectors');
+    const rebuildFts = !existing.size ||
+        // Tantivy retains deleted versions in BM25 statistics until rebuilt.
+        stats.updated > 0 ||
+        removed.length > 0 ||
+        passages.filter((p) => changed.has(p.sectionId)).length > 512;
+    await db.execute('BEGIN');
+    try {
+        if (rebuildFts)
+            await db.execute('DROP INDEX IF EXISTS chunks_fts');
+        for (let i = 0; i < entries.length; i++)
+            await db.execute({
+                sql: 'INSERT INTO embeddings VALUES (?,vector32(?))',
+                args: [entries[i][0], JSON.stringify(vectors[i])],
+            });
+        for (const id of [...changed, ...removed]) {
+            await db.execute({
+                sql: 'DELETE FROM lexical_chunks WHERE id IN (SELECT id FROM chunks WHERE section_id=?)',
+                args: [id],
+            });
+            await db.execute({
+                sql: 'DELETE FROM identifiers WHERE chunk_id IN (SELECT id FROM chunks WHERE section_id=?)',
+                args: [id],
+            });
             await db.execute({
-                sql: `INSERT OR REPLACE INTO sections (id, file, heading, content, content_hash, embedding, updated_at)
-              VALUES (?, ?, ?, ?, ?, vector(?), ?)`,
-                args: [id, section.file, section.heading, content, hash, vecJson, now],
+                sql: 'DELETE FROM chunks WHERE section_id=?',
+                args: [id],
             });
+            await db.execute({ sql: 'DELETE FROM sections WHERE id=?', args: [id] });
         }
+        const parents = new Map();
+        for (const s of project.sections)
+            for (const child of s.children)
+                parents.set(child.id, s.id);
+        for (const s of project.sections)
+            if (changed.has(s.id))
+                await db.execute({
+                    sql: 'INSERT INTO sections VALUES (?,?,?,?,?,?,?,?)',
+                    args: [
+                        s.id,
+                        s.file,
+                        s.heading,
+                        s.firstParagraph,
+                        sectionHashes.get(s.id),
+                        parents.get(s.id) ?? null,
+                        s.startLine,
+                        s.endLine,
+                    ],
+                });
+        for (const p of passages)
+            if (changed.has(p.sectionId)) {
+                const row = (await db.execute({
+                    sql: 'INSERT INTO chunks(source_id,section_id,ordinal,type,spans,body,heading,path,input_hash) VALUES (?,?,?,?,?,?,?,?,?) RETURNING id',
+                    args: [
+                        p.id,
+                        p.sectionId,
+                        p.ordinal,
+                        p.type,
+                        JSON.stringify(p.spans),
+                        p.text,
+                        p.heading,
+                        p.path,
+                        p.inputHash,
+                    ],
+                })).rows[0];
+                for (const token of identifierTokens(`${p.sectionId}\n${project.sectionById.get(p.sectionId.toLowerCase())?.filePath ?? ''}\n${p.heading}\n${p.path}\n${p.text}`))
+                    await db.execute({
+                        sql: 'INSERT INTO identifiers VALUES (?,?)',
+                        args: [token, row.id],
+                    });
+            }
+        await db.execute('DELETE FROM embeddings WHERE hash NOT IN (SELECT input_hash FROM chunks)');
+        await db.execute({
+            sql: 'INSERT OR REPLACE INTO meta VALUES (?,?)',
+            args: ['fingerprint', fingerprint],
+        });
+        await db.execute({
+            sql: 'INSERT OR REPLACE INTO meta VALUES (?,?)',
+            args: ['project_hash', projectHash],
+        });
+        await synchronizeLexical(db);
+        if (rebuildFts)
+            await db.execute(CREATE_PASSAGE_FTS);
+        await db.execute('COMMIT');
     }
-    // Delete removed sections
-    for (const id of toDelete) {
-        await db.execute({ sql: 'DELETE FROM sections WHERE id = ?', args: [id] });
+    catch (error) {
+        await db.execute('ROLLBACK');
+        throw error;
     }
-    const added = toEmbed.filter((e) => !existing.has(e.id)).length;
-    const updated = toEmbed.filter((e) => existing.has(e.id)).length;
-    return { added, updated, removed: toDelete.length, unchanged };
+    return stats;
 }

package/dist/src/search/lexical.d.ts

@@ -0,0 +1,6 @@
+import { type SearchDb } from './db.js';
+export declare const LEXICAL_VERSION = "snowball-english-rust-stemmers-1.2.0-v1:unicode-words-v1:live-statistics-v1";
+/** Same analysis for indexed fields and queries; exact identifiers bypass it. */
+export declare function lexicalTokens(text: string): string[];
+/** Populate derived lexical rows, reusing original passages and vectors. */
+export declare function synchronizeLexical(db: SearchDb): Promise<void>;

package/dist/src/search/lexical.js

@@ -0,0 +1,36 @@
+import { stemWords, STEMMER_VERSION } from '@lat.md/stemmer';
+import { CREATE_PASSAGE_FTS } from './db.js';
+export const LEXICAL_VERSION = `${STEMMER_VERSION}:unicode-words-v1:live-statistics-v1`;
+/** Same analysis for indexed fields and queries; exact identifiers bypass it. */
+export function lexicalTokens(text) {
+    const tokens = text.toLowerCase().match(/[\p{L}\p{N}_]+/gu) ?? [];
+    const english = tokens.filter((token) => /^[a-z]+$/.test(token));
+    const stems = stemWords(english);
+    let i = 0;
+    return tokens.map((token) => (/^[a-z]+$/.test(token) ? stems[i++] : token));
+}
+/** Populate derived lexical rows, reusing original passages and vectors. */
+export async function synchronizeLexical(db) {
+    const version = (await db.execute("SELECT value FROM meta WHERE key='lexical_version'")).rows[0]?.value;
+    if (version !== LEXICAL_VERSION) {
+        await db.execute('DROP INDEX IF EXISTS chunks_fts');
+        await db.execute('DELETE FROM lexical_chunks');
+    }
+    await db.execute('DELETE FROM lexical_chunks WHERE id NOT IN (SELECT id FROM chunks)');
+    const rows = (await db.execute('SELECT id,body,heading,path FROM chunks WHERE id NOT IN (SELECT id FROM lexical_chunks)')).rows;
+    for (const row of rows)
+        await db.execute({
+            sql: 'INSERT INTO lexical_chunks VALUES (?,?,?,?)',
+            args: [
+                row.id,
+                ...[row.body, row.heading, row.path].map((text) => lexicalTokens(text).join(' ')),
+            ],
+        });
+    if (version !== LEXICAL_VERSION) {
+        await db.execute(CREATE_PASSAGE_FTS);
+        await db.execute({
+            sql: 'INSERT OR REPLACE INTO meta VALUES (?,?)',
+            args: ['lexical_version', LEXICAL_VERSION],
+        });
+    }
+}

package/dist/src/search/lock.d.ts

@@ -0,0 +1,5 @@
+export type LockMode = 'shared' | 'exclusive';
+export declare function acquireSearchLock(cacheDir: string, timeoutMs?: number): Promise<() => Promise<void>>;
+/** A turnstile prevents new readers bypassing a writer waiting for readers to drain. */
+export declare function acquireSearchAccess(cacheDir: string, mode: LockMode, timeoutMs?: number): Promise<() => Promise<void>>;
+export declare function withSearchAccess<T>(cacheDir: string, mode: LockMode, work: () => Promise<T>): Promise<T>;

package/dist/src/search/lock.js

@@ -0,0 +1,45 @@
+import { open } from 'node:fs/promises';
+import { join } from 'node:path';
+import { setTimeout } from 'node:timers/promises';
+/** Lock files stay in place so every process locks the same inode. */
+async function acquireFileLock(path, mode, deadline, message) {
+    const { tryLock } = await import('fs-native-extensions');
+    const handle = await open(path, 'a+');
+    try {
+        while (!tryLock(handle.fd, { shared: mode === 'shared' })) {
+            const remaining = deadline - performance.now();
+            if (remaining <= 0)
+                throw new Error(message);
+            await setTimeout(Math.min(10, remaining));
+        }
+        return () => handle.close();
+    }
+    catch (error) {
+        await handle.close();
+        throw error;
+    }
+}
+export function acquireSearchLock(cacheDir, timeoutMs = 120_000) {
+    return acquireFileLock(join(cacheDir, 'search-write.lock'), 'exclusive', performance.now() + timeoutMs, 'Search index writer is busy; retry shortly.');
+}
+/** A turnstile prevents new readers bypassing a writer waiting for readers to drain. */
+export async function acquireSearchAccess(cacheDir, mode, timeoutMs = 120_000) {
+    const deadline = performance.now() + timeoutMs;
+    const message = 'Search database is busy; retry shortly.';
+    const releaseGate = await acquireFileLock(join(cacheDir, 'search-access-gate.lock'), 'exclusive', deadline, message);
+    try {
+        return await acquireFileLock(join(cacheDir, 'search-access.lock'), mode, deadline, message);
+    }
+    finally {
+        await releaseGate();
+    }
+}
+export async function withSearchAccess(cacheDir, mode, work) {
+    const release = await acquireSearchAccess(cacheDir, mode);
+    try {
+        return await work();
+    }
+    finally {
+        await release();
+    }
+}

package/dist/src/search/query.d.ts

@@ -0,0 +1,26 @@
+import type { Section, SectionMatch } from '@plebeiantech/lat.md-core/lattice-model';
+import { type SearchDb } from './db.js';
+import { type CreateSearchEngine } from './embedder.js';
+import { type SearchResult } from './search.js';
+export type IndexedSearchResult = {
+    query: string;
+    matches: SectionMatch[];
+};
+export type IndexedSearchSession = {
+    search: (query: string, limit: number, minSimilarity?: number) => Promise<SearchResult[]>;
+    close: () => Promise<void>;
+};
+/** Resolve scored index rows against one already-analyzed project snapshot. */
+export declare function resolveSearchMatches(results: readonly SearchResult[], sectionById: ReadonlyMap<string, Section>): SectionMatch[];
+/** Reuse an embedder with scoped database access for a sequence of index queries. */
+export declare function openIndexedSearchSession(latDir: string, options?: {
+    cacheDir?: string;
+    createSearchEngine?: CreateSearchEngine;
+}): Promise<IndexedSearchSession>;
+/** Query a finished index and resolve its ids from precomputed section data. */
+export declare function searchIndexedSections(latDir: string, query: string, limit: number, sectionById: ReadonlyMap<string, Section>, options?: {
+    cacheDir?: string;
+    minSimilarity?: number;
+}): Promise<IndexedSearchResult>;
+export declare function indexMetadata(db: SearchDb): Promise<Map<string, string>>;
+export declare function sameIndexMetadata(a: Map<string, string>, b: Map<string, string>): boolean;

package/dist/src/search/query.js

@@ -0,0 +1,103 @@
+import { LEXICAL_VERSION } from './lexical.js';
+import { ReindexRequiredError } from './embedder.js';
+import { existsSync } from 'node:fs';
+import { join } from 'node:path';
+import { hasIndex } from './db.js';
+import { closeDb, openDb } from './db.js';
+import { embedderForIndex } from './embedder.js';
+import { searchSections, prepareSearchQuery, } from './search.js';
+/** Resolve scored index rows against one already-analyzed project snapshot. */
+export function resolveSearchMatches(results, sectionById) {
+    return results.flatMap((result) => {
+        const section = sectionById.get(result.id.toLowerCase());
+        return section
+            ? [
+                {
+                    section,
+                    reason: 'hybrid match',
+                    rankScore: result.rankScore,
+                    semanticSimilarity: result.semanticSimilarity,
+                    lexicalScore: result.lexicalScore,
+                    semanticRank: result.semanticRank,
+                    lexicalRank: result.lexicalRank,
+                    evidence: result.evidence,
+                    diagnostics: result.diagnostics,
+                },
+            ]
+            : [];
+    });
+}
+/** Reuse an embedder with scoped database access for a sequence of index queries. */
+export async function openIndexedSearchSession(latDir, options = {}) {
+    const cacheDir = options.cacheDir ?? join(latDir, '.cache');
+    if (!existsSync(cacheDir) || !hasIndex(cacheDir))
+        return { search: async () => [], close: async () => { } };
+    const readMetadata = async () => {
+        const db = openDb(latDir, options.cacheDir, true);
+        try {
+            return await indexMetadata(db);
+        }
+        finally {
+            await closeDb(db);
+        }
+    };
+    const initial = await readMetadata();
+    const stored = initial.get('embedding_model') ?? null;
+    if (stored && initial.get('lexical_version') !== LEXICAL_VERSION)
+        throw new ReindexRequiredError('Search lexical index changed; run lat search to update it.');
+    const embedder = stored
+        ? await embedderForIndex(stored, latDir, options.createSearchEngine)
+        : null;
+    let closed = false;
+    const active = new Set();
+    return {
+        search(query, limit, minSimilarity) {
+            if (closed)
+                return Promise.reject(new Error('Search session is closed'));
+            const operation = (async () => {
+                if (!embedder)
+                    return [];
+                const vector = await prepareSearchQuery(query, embedder, limit, minSimilarity);
+                const db = openDb(latDir, options.cacheDir, true);
+                try {
+                    if (!sameIndexMetadata(initial, await indexMetadata(db)))
+                        throw new ReindexRequiredError('Search index changed; reopen the search session to refresh its documents and model.');
+                    return await searchSections(db, query, embedder, limit, minSimilarity, vector);
+                }
+                finally {
+                    await closeDb(db);
+                }
+            })();
+            active.add(operation);
+            void operation.finally(() => active.delete(operation)).catch(() => { });
+            return operation;
+        },
+        async close() {
+            closed = true;
+            await Promise.allSettled([...active]);
+        },
+    };
+}
+/** Query a finished index and resolve its ids from precomputed section data. */
+export async function searchIndexedSections(latDir, query, limit, sectionById, options = {}) {
+    const session = await openIndexedSearchSession(latDir, options);
+    try {
+        const results = await session.search(query, limit, options.minSimilarity);
+        return {
+            query,
+            matches: resolveSearchMatches(results, sectionById),
+        };
+    }
+    finally {
+        await session.close();
+    }
+}
+export async function indexMetadata(db) {
+    return new Map((await db.execute('SELECT key,value FROM meta')).rows.map((row) => [
+        row.key,
+        row.value,
+    ]));
+}
+export function sameIndexMetadata(a, b) {
+    return (a.size === b.size && [...a].every(([key, value]) => b.get(key) === value));
+}

package/dist/src/search/search.d.ts

@@ -1,9 +1,19 @@
-import type { Client } from '@libsql/client';
+import type { SearchDb } from './db.js';
 import type { Embedder } from './embedder.js';
-export type SearchResult = {
-    id: string;
-    file: string;
-    heading: string;
-    content: string;
+import type { SearchResult } from './types.js';
+export type { SearchResult, SearchEvidence, SearchDiagnostics, } from './types.js';
+export declare const DEFAULT_SEARCH_LIMIT = 5;
+export declare const DEFAULT_MIN_SIMILARITY = 0.2;
+export type Candidate = {
+    id: number;
+    sectionId: string;
+    score: number;
+    exact?: boolean;
 };
-export declare function searchSections(db: Client, query: string, embedder: Embedder, limit?: number): Promise<SearchResult[]>;
+export declare function collapse(candidates: Candidate[]): Map<string, Candidate & {
+    rank: number;
+}>;
+export declare function literalFtsQuery(query: string): string;
+export declare function searchSections(db: SearchDb, query: string, embedder: Embedder, limit?: number, minSimilarity?: number, preparedVector?: number[]): Promise<SearchResult[]>;
+/** Prepare embeddings before acquiring a database access lock. */
+export declare function prepareSearchQuery(query: string, embedder: Embedder, limit?: number, minSimilarity?: number): Promise<number[]>;