@plebeiantech/lat.md 0.12.3-fork.16 → 0.12.3-fork.17

package/dist/src/search/index.js

@@ -1,85 +1,189 @@
-import { createHash } from 'node:crypto';
-import { dirname, join } from 'node:path';
+import { PARSER_CACHE_VERSION } from '../parser-cache.js';
+import { synchronizeLexical } from './lexical.js';
+import { dirname } from 'node:path';
+import { CREATE_PASSAGE_FTS } from './db.js';
 import { analyzeMarkdownProject, } from '../project-analysis.js';
-function hashContent(text) {
-    return createHash('sha256').update(text).digest('hex');
+import { chunkFile, digest, embeddingFingerprint } from './chunks.js';
+export function projectFingerprint(project) {
+    return digest(`parser:${PARSER_CACHE_VERSION}\n` +
+        [...project.files.values()]
+            .map((f) => `${f.projectPath}\0${digest(f.content)}`)
+            .sort()
+            .join('\n'));
 }
-/**
- * Read each lat.md file once, keyed by project-relative path. Section text is
- * sliced from these lines by range: reading the file per section instead is
- * O(sections × file size) — a 3.5 MB file holding 12k sections was re-read 12k
- * times (~42 GB of string work) on every search.
- */
-function loadFileLines(sections, projectRoot, project) {
-    const lines = new Map();
-    for (const { filePath } of sections) {
-        if (lines.has(filePath))
-            continue;
-        const content = project.filesByAbsolutePath.get(join(projectRoot, filePath))?.content;
-        if (content === undefined)
-            throw new Error(`Analyzed Markdown file not found: ${filePath}`);
-        lines.set(filePath, content.split('\n'));
-    }
-    return lines;
-}
-function sectionContent(section, lines) {
-    return lines
-        .get(section.filePath)
-        .slice(section.startLine - 1, section.endLine)
-        .join('\n');
+/** Exact identifier tokens complement FTS's punctuation and long-token handling. */
+export function identifierTokens(text) {
+    return [
+        ...new Set((text.match(/[\p{L}\p{N}_][\p{L}\p{N}_./:#@-]*/gu) ?? [])
+            .map((t) => t.toLowerCase())
+            .filter((t) => /[_./:#@-]/.test(t) || t.length >= 40)),
+    ];
 }
 export async function indexSections(latDir, db, embedder, onProgress, analyzedProject) {
-    const projectRoot = dirname(latDir);
     const project = analyzedProject ??
-        (await analyzeMarkdownProject(latDir, projectRoot, { executor: 'auto' }));
-    const flat = project.sections;
-    // Build current state: id -> { section, content, hash }
-    const current = new Map();
-    const lines = loadFileLines(flat, projectRoot, project);
-    for (const s of flat) {
-        const text = sectionContent(s, lines);
-        current.set(s.id, { section: s, content: text, hash: hashContent(text) });
+        (await analyzeMarkdownProject(latDir, dirname(latDir), {
+            executor: 'auto',
+        }));
+    const fingerprint = embeddingFingerprint(embedder);
+    const projectHash = projectFingerprint(project);
+    const oldFingerprint = (await db.execute("SELECT value FROM meta WHERE key='fingerprint'")).rows[0]?.value;
+    if (oldFingerprint && oldFingerprint !== fingerprint)
+        throw new Error('Search chunking or embedding model changed; run lat reindex.');
+    const byFile = new Map();
+    for (const s of project.sections) {
+        const list = byFile.get(s.filePath) ?? [];
+        list.push(s);
+        byFile.set(s.filePath, list);
+    }
+    const files = new Map([...project.files.values()].map((f) => [f.projectPath, f]));
+    const passages = [...byFile].flatMap(([path, sections]) => {
+        const file = files.get(path);
+        if (!file)
+            throw new Error(`Missing analyzed file: ${path}`);
+        return chunkFile(file, sections, embedder);
+    });
+    const existing = new Map((await db.execute('SELECT id,content_hash FROM sections')).rows.map((r) => [
+        r.id,
+        r.content_hash,
+    ]));
+    const sectionHashes = new Map();
+    const owned = new Map();
+    for (const p of passages) {
+        const list = owned.get(p.sectionId) ?? [];
+        list.push(p);
+        owned.set(p.sectionId, list);
     }
-    // Get existing hashes from DB
-    const existing = new Map();
-    const rows = await db.execute('SELECT id, content_hash FROM sections');
-    for (const row of rows.rows) {
-        existing.set(row.id, row.content_hash);
+    for (const s of project.sections) {
+        sectionHashes.set(s.id, digest(JSON.stringify([
+            { ...s, children: undefined },
+            (owned.get(s.id) ?? []).map((p) => [p.inputHash, p.spans]),
+        ])));
     }
-    // Partition into new, changed, unchanged, deleted
-    const toEmbed = [];
-    let unchanged = 0;
-    for (const [id, entry] of current) {
-        const existingHash = existing.get(id);
-        if (existingHash === entry.hash) {
-            unchanged++;
+    const changed = new Set([...sectionHashes]
+        .filter(([id, hash]) => existing.get(id) !== hash)
+        .map(([id]) => id));
+    const removed = [...existing.keys()].filter((id) => !sectionHashes.has(id));
+    const stats = {
+        added: [...changed].filter((id) => !existing.has(id)).length,
+        updated: [...changed].filter((id) => existing.has(id)).length,
+        removed: removed.length,
+        unchanged: project.sections.length - changed.size,
+    };
+    if (!changed.size && !removed.length && oldFingerprint === fingerprint) {
+        // Lexical maintenance can repair old generations without embedding work.
+        await db.execute('BEGIN');
+        try {
+            await synchronizeLexical(db);
+            await db.execute({
+                sql: "UPDATE meta SET value=? WHERE key='project_hash' AND value<>?",
+                args: [projectHash, projectHash],
+            });
+            await db.execute('COMMIT');
         }
-        else {
-            toEmbed.push({ id, content: entry.content, section: entry.section });
+        catch (error) {
+            await db.execute('ROLLBACK');
+            throw error;
         }
+        return stats;
     }
-    const toDelete = [...existing.keys()].filter((id) => !current.has(id));
-    // Embed new/changed sections
-    if (toEmbed.length > 0) {
-        const texts = toEmbed.map((e) => e.content);
-        const vectors = await embedder.embed(texts, onProgress);
-        const now = Date.now();
-        for (let i = 0; i < toEmbed.length; i++) {
-            const { id, content, section } = toEmbed[i];
-            const hash = current.get(id).hash;
-            const vecJson = JSON.stringify(vectors[i]);
+    const storedHashes = new Set((await db.execute('SELECT hash FROM embeddings')).rows.map((r) => r.hash));
+    const missing = new Map(passages
+        .filter((p) => !storedHashes.has(p.inputHash))
+        .map((p) => [p.inputHash, p.input]));
+    const entries = [...missing];
+    const vectors = entries.length
+        ? await embedder.embed(entries.map(([, input]) => input), onProgress)
+        : [];
+    if (vectors.length !== entries.length ||
+        vectors.some((v) => v.length !== embedder.dimensions || v.some((n) => !Number.isFinite(n))))
+        throw new Error('Embedding backend returned invalid vectors');
+    const rebuildFts = !existing.size ||
+        // Tantivy retains deleted versions in BM25 statistics until rebuilt.
+        stats.updated > 0 ||
+        removed.length > 0 ||
+        passages.filter((p) => changed.has(p.sectionId)).length > 512;
+    await db.execute('BEGIN');
+    try {
+        if (rebuildFts)
+            await db.execute('DROP INDEX IF EXISTS chunks_fts');
+        for (let i = 0; i < entries.length; i++)
+            await db.execute({
+                sql: 'INSERT INTO embeddings VALUES (?,vector32(?))',
+                args: [entries[i][0], JSON.stringify(vectors[i])],
+            });
+        for (const id of [...changed, ...removed]) {
+            await db.execute({
+                sql: 'DELETE FROM lexical_chunks WHERE id IN (SELECT id FROM chunks WHERE section_id=?)',
+                args: [id],
+            });
+            await db.execute({
+                sql: 'DELETE FROM identifiers WHERE chunk_id IN (SELECT id FROM chunks WHERE section_id=?)',
+                args: [id],
+            });
             await db.execute({
-                sql: `INSERT OR REPLACE INTO sections (id, file, heading, content, content_hash, embedding, updated_at)
-              VALUES (?, ?, ?, ?, ?, vector(?), ?)`,
-                args: [id, section.file, section.heading, content, hash, vecJson, now],
+                sql: 'DELETE FROM chunks WHERE section_id=?',
+                args: [id],
             });
+            await db.execute({ sql: 'DELETE FROM sections WHERE id=?', args: [id] });
         }
+        const parents = new Map();
+        for (const s of project.sections)
+            for (const child of s.children)
+                parents.set(child.id, s.id);
+        for (const s of project.sections)
+            if (changed.has(s.id))
+                await db.execute({
+                    sql: 'INSERT INTO sections VALUES (?,?,?,?,?,?,?,?)',
+                    args: [
+                        s.id,
+                        s.file,
+                        s.heading,
+                        s.firstParagraph,
+                        sectionHashes.get(s.id),
+                        parents.get(s.id) ?? null,
+                        s.startLine,
+                        s.endLine,
+                    ],
+                });
+        for (const p of passages)
+            if (changed.has(p.sectionId)) {
+                const row = (await db.execute({
+                    sql: 'INSERT INTO chunks(source_id,section_id,ordinal,type,spans,body,heading,path,input_hash) VALUES (?,?,?,?,?,?,?,?,?) RETURNING id',
+                    args: [
+                        p.id,
+                        p.sectionId,
+                        p.ordinal,
+                        p.type,
+                        JSON.stringify(p.spans),
+                        p.text,
+                        p.heading,
+                        p.path,
+                        p.inputHash,
+                    ],
+                })).rows[0];
+                for (const token of identifierTokens(`${p.sectionId}\n${project.sectionById.get(p.sectionId.toLowerCase())?.filePath ?? ''}\n${p.heading}\n${p.path}\n${p.text}`))
+                    await db.execute({
+                        sql: 'INSERT INTO identifiers VALUES (?,?)',
+                        args: [token, row.id],
+                    });
+            }
+        await db.execute('DELETE FROM embeddings WHERE hash NOT IN (SELECT input_hash FROM chunks)');
+        await db.execute({
+            sql: 'INSERT OR REPLACE INTO meta VALUES (?,?)',
+            args: ['fingerprint', fingerprint],
+        });
+        await db.execute({
+            sql: 'INSERT OR REPLACE INTO meta VALUES (?,?)',
+            args: ['project_hash', projectHash],
+        });
+        await synchronizeLexical(db);
+        if (rebuildFts)
+            await db.execute(CREATE_PASSAGE_FTS);
+        await db.execute('COMMIT');
     }
-    // Delete removed sections
-    for (const id of toDelete) {
-        await db.execute({ sql: 'DELETE FROM sections WHERE id = ?', args: [id] });
+    catch (error) {
+        await db.execute('ROLLBACK');
+        throw error;
     }
-    const added = toEmbed.filter((e) => !existing.has(e.id)).length;
-    const updated = toEmbed.filter((e) => existing.has(e.id)).length;
-    return { added, updated, removed: toDelete.length, unchanged };
+    return stats;
 }

package/dist/src/search/lexical.d.ts

@@ -0,0 +1,6 @@
+import { type SearchDb } from './db.js';
+export declare const LEXICAL_VERSION = "snowball-english-rust-stemmers-1.2.0-v1:unicode-words-v1:live-statistics-v1";
+/** Same analysis for indexed fields and queries; exact identifiers bypass it. */
+export declare function lexicalTokens(text: string): string[];
+/** Populate derived lexical rows, reusing original passages and vectors. */
+export declare function synchronizeLexical(db: SearchDb): Promise<void>;

package/dist/src/search/lexical.js

@@ -0,0 +1,36 @@
+import { stemWords, STEMMER_VERSION } from '@lat.md/stemmer';
+import { CREATE_PASSAGE_FTS } from './db.js';
+export const LEXICAL_VERSION = `${STEMMER_VERSION}:unicode-words-v1:live-statistics-v1`;
+/** Same analysis for indexed fields and queries; exact identifiers bypass it. */
+export function lexicalTokens(text) {
+    const tokens = text.toLowerCase().match(/[\p{L}\p{N}_]+/gu) ?? [];
+    const english = tokens.filter((token) => /^[a-z]+$/.test(token));
+    const stems = stemWords(english);
+    let i = 0;
+    return tokens.map((token) => (/^[a-z]+$/.test(token) ? stems[i++] : token));
+}
+/** Populate derived lexical rows, reusing original passages and vectors. */
+export async function synchronizeLexical(db) {
+    const version = (await db.execute("SELECT value FROM meta WHERE key='lexical_version'")).rows[0]?.value;
+    if (version !== LEXICAL_VERSION) {
+        await db.execute('DROP INDEX IF EXISTS chunks_fts');
+        await db.execute('DELETE FROM lexical_chunks');
+    }
+    await db.execute('DELETE FROM lexical_chunks WHERE id NOT IN (SELECT id FROM chunks)');
+    const rows = (await db.execute('SELECT id,body,heading,path FROM chunks WHERE id NOT IN (SELECT id FROM lexical_chunks)')).rows;
+    for (const row of rows)
+        await db.execute({
+            sql: 'INSERT INTO lexical_chunks VALUES (?,?,?,?)',
+            args: [
+                row.id,
+                ...[row.body, row.heading, row.path].map((text) => lexicalTokens(text).join(' ')),
+            ],
+        });
+    if (version !== LEXICAL_VERSION) {
+        await db.execute(CREATE_PASSAGE_FTS);
+        await db.execute({
+            sql: 'INSERT OR REPLACE INTO meta VALUES (?,?)',
+            args: ['lexical_version', LEXICAL_VERSION],
+        });
+    }
+}

package/dist/src/search/lock.d.ts

@@ -0,0 +1,2 @@
+/** The persistent file must never be unlinked: all writers lock the same inode. */
+export declare function acquireSearchLock(cacheDir: string, timeoutMs?: number): Promise<() => Promise<void>>;

package/dist/src/search/lock.js

@@ -0,0 +1,23 @@
+import { open } from 'node:fs/promises';
+import { join } from 'node:path';
+import { setTimeout } from 'node:timers/promises';
+/** The persistent file must never be unlinked: all writers lock the same inode. */
+export async function acquireSearchLock(cacheDir, timeoutMs = 120_000) {
+    const { tryLock } = await import('fs-native-extensions');
+    const handle = await open(join(cacheDir, 'search-write.lock'), 'a+');
+    const deadline = performance.now() + timeoutMs;
+    try {
+        while (!tryLock(handle.fd)) {
+            const remaining = deadline - performance.now();
+            if (remaining <= 0)
+                throw new Error('Search index writer is busy; retry shortly.');
+            await setTimeout(Math.min(100, remaining));
+        }
+        // Closing releases the kernel lock, including on abnormal process exit.
+        return () => handle.close();
+    }
+    catch (error) {
+        await handle.close();
+        throw error;
+    }
+}

package/dist/src/search/query.d.ts

@@ -6,7 +6,7 @@ export type IndexedSearchResult = {
     matches: SectionMatch[];
 };
 export type IndexedSearchSession = {
-    search: (query: string, limit: number, threshold?: number) => Promise<SearchResult[]>;
+    search: (query: string, limit: number, minSimilarity?: number) => Promise<SearchResult[]>;
     close: () => Promise<void>;
 };
 /** Resolve scored index rows against one already-analyzed project snapshot. */
@@ -19,5 +19,5 @@ export declare function openIndexedSearchSession(latDir: string, options?: {
 /** Query a finished index and resolve its ids from precomputed section data. */
 export declare function searchIndexedSections(latDir: string, query: string, limit: number, sectionById: ReadonlyMap<string, Section>, options?: {
     cacheDir?: string;
-    threshold?: number;
+    minSimilarity?: number;
 }): Promise<IndexedSearchResult>;

package/dist/src/search/query.js

@@ -1,4 +1,9 @@
-import { closeDb, ensureMeta, ensureSectionsSchema, getStoredModel, openDb, } from './db.js';
+import { LEXICAL_VERSION } from './lexical.js';
+import { ReindexRequiredError } from './embedder.js';
+import { existsSync } from 'node:fs';
+import { join } from 'node:path';
+import { hasIndex } from './db.js';
+import { closeDb, getStoredModel, openDb } from './db.js';
 import { embedderForIndex } from './embedder.js';
 import { searchSections } from './search.js';
 /** Resolve scored index rows against one already-analyzed project snapshot. */
@@ -9,8 +14,14 @@ export function resolveSearchMatches(results, sectionById) {
             ? [
                 {
                     section,
-                    reason: 'semantic match',
-                    score: result.score,
+                    reason: 'hybrid match',
+                    rankScore: result.rankScore,
+                    semanticSimilarity: result.semanticSimilarity,
+                    lexicalScore: result.lexicalScore,
+                    semanticRank: result.semanticRank,
+                    lexicalRank: result.lexicalRank,
+                    evidence: result.evidence,
+                    diagnostics: result.diagnostics,
                 },
             ]
             : [];
@@ -18,7 +29,10 @@ export function resolveSearchMatches(results, sectionById) {
 }
 /** Open one reusable database and embedder for a sequence of index queries. */
 export async function openIndexedSearchSession(latDir, options = {}) {
-    const db = openDb(latDir, options.cacheDir);
+    const cacheDir = options.cacheDir ?? join(latDir, '.cache');
+    if (!existsSync(cacheDir) || !hasIndex(cacheDir))
+        return { search: async () => [], close: async () => { } };
+    const db = openDb(latDir, options.cacheDir, true);
     let closed = false;
     const close = async () => {
         if (closed)
@@ -27,7 +41,6 @@ export async function openIndexedSearchSession(latDir, options = {}) {
         await closeDb(db);
     };
     try {
-        await ensureMeta(db);
         const stored = await getStoredModel(db);
         if (stored === null) {
             return {
@@ -39,13 +52,15 @@ export async function openIndexedSearchSession(latDir, options = {}) {
                 close,
             };
         }
+        const lexicalVersion = (await db.execute("SELECT value FROM meta WHERE key='lexical_version'")).rows[0]?.value;
+        if (lexicalVersion !== LEXICAL_VERSION)
+            throw new ReindexRequiredError('Search lexical index changed; run lat search to update it.');
         const embedder = await embedderForIndex(stored, latDir, options.createSearchEngine);
-        await ensureSectionsSchema(db, embedder.dimensions);
         return {
-            async search(query, limit, threshold) {
+            async search(query, limit, minSimilarity) {
                 if (closed)
                     throw new Error('Search session is closed');
-                return searchSections(db, query, embedder, limit, threshold);
+                return searchSections(db, query, embedder, limit, minSimilarity);
             },
             close,
         };
@@ -59,7 +74,7 @@ export async function openIndexedSearchSession(latDir, options = {}) {
 export async function searchIndexedSections(latDir, query, limit, sectionById, options = {}) {
     const session = await openIndexedSearchSession(latDir, options);
     try {
-        const results = await session.search(query, limit, options.threshold);
+        const results = await session.search(query, limit, options.minSimilarity);
         return {
             query,
             matches: resolveSearchMatches(results, sectionById),

package/dist/src/search/search.d.ts

@@ -1,12 +1,17 @@
-import type { Client } from '@libsql/client';
+import type { SearchDb } from './db.js';
 import type { Embedder } from './embedder.js';
+import type { SearchResult } from './types.js';
+export type { SearchResult, SearchEvidence, SearchDiagnostics, } from './types.js';
 export declare const DEFAULT_SEARCH_LIMIT = 5;
-export declare const DEFAULT_SEARCH_THRESHOLD = 0.35;
-export type SearchResult = {
-    id: string;
-    file: string;
-    heading: string;
-    content: string;
+export declare const DEFAULT_MIN_SIMILARITY = 0.2;
+export type Candidate = {
+    id: number;
+    sectionId: string;
     score: number;
+    exact?: boolean;
 };
-export declare function searchSections(db: Client, query: string, embedder: Embedder, limit?: number, threshold?: number): Promise<SearchResult[]>;
+export declare function collapse(candidates: Candidate[]): Map<string, Candidate & {
+    rank: number;
+}>;
+export declare function literalFtsQuery(query: string): string;
+export declare function searchSections(db: SearchDb, query: string, embedder: Embedder, limit?: number, minSimilarity?: number): Promise<SearchResult[]>;

package/dist/src/search/search.js

@@ -1,25 +1,157 @@
+import { lexicalTokens } from './lexical.js';
 export const DEFAULT_SEARCH_LIMIT = 5;
-export const DEFAULT_SEARCH_THRESHOLD = 0.35;
-export async function searchSections(db, query, embedder, limit = DEFAULT_SEARCH_LIMIT, threshold = DEFAULT_SEARCH_THRESHOLD) {
-    const [queryVec] = await embedder.embed([query]);
-    const vecJson = JSON.stringify(queryVec);
-    const rows = await db.execute({
-        sql: `SELECT s.id, s.file, s.heading, s.content,
-                 1.0 - vector_distance_cos(s.embedding, vector(?)) AS score
-          FROM vector_top_k('sections_vec_idx', vector(?), ?) AS v
-          JOIN sections AS s ON s.rowid = v.id
-          ORDER BY score DESC`,
-        args: [vecJson, vecJson, limit],
-    });
-    const results = rows.rows.map((row) => {
-        const score = Number(row.score);
-        return {
-            id: row.id,
-            file: row.file,
-            heading: row.heading,
-            content: row.content,
-            score: Number.isFinite(score) ? score : 0,
-        };
-    });
-    return results.filter((result) => result.score >= threshold);
+export const DEFAULT_MIN_SIMILARITY = 0.2;
+export function collapse(candidates) {
+    const unique = new Map();
+    for (const c of candidates)
+        if (!unique.has(c.sectionId))
+            unique.set(c.sectionId, c);
+    let rank = 0, previous;
+    return new Map([...unique].map(([id, c]) => {
+        if (!previous ||
+            c.score !== previous.score ||
+            !!c.exact !== !!previous.exact)
+            rank++;
+        previous = c;
+        return [id, { ...c, rank }];
+    }));
+}
+export function literalFtsQuery(query) {
+    const stopwords = new Set('a an and are as at be by do does for from how i in is it of on or our that the their this to we what when where which who why with you your use uses used have has had can could would should'.split(' '));
+    return (query.match(/[\p{L}\p{N}_]+/gu) ?? [])
+        .filter((t) => !stopwords.has(t.toLowerCase()))
+        .flatMap((t) => lexicalTokens(t))
+        .map((t) => `"${t.replaceAll('"', '\\"')}"`)
+        .join(' OR ');
+}
+export async function searchSections(db, query, embedder, limit = DEFAULT_SEARCH_LIMIT, minSimilarity = DEFAULT_MIN_SIMILARITY) {
+    query = query.trim();
+    if (!query)
+        return [];
+    if (!Number.isInteger(limit) || limit < 1)
+        throw new Error('limit must be a positive integer');
+    if (!Number.isFinite(minSimilarity) || minSimilarity < 0 || minSimilarity > 1)
+        throw new Error('min-similarity must be a number from 0 to 1');
+    if (embedder.countTokens(query) > embedder.maxInputTokens)
+        throw new Error('Search query exceeds the embedding model token limit; shorten the query.');
+    const [vector] = await embedder.embed([query]);
+    if (!vector ||
+        vector.length !== embedder.dimensions ||
+        vector.some((n) => !Number.isFinite(n)))
+        throw new Error('Embedding backend returned an invalid query vector');
+    const vectorJson = JSON.stringify(vector), fts = literalFtsQuery(query);
+    const target = Math.max(50, limit), cap = 10 * target;
+    const owners = new Map((await db.execute('SELECT id,section_id FROM chunks')).rows.map((r) => [
+        r.id,
+        r.section_id,
+    ]));
+    const exactRows = (await db.execute({
+        sql: 'SELECT chunk_id FROM identifiers WHERE token=? LIMIT ?',
+        args: [query.toLowerCase(), cap],
+    })).rows;
+    const exact = exactRows.map((r) => ({
+        id: r.chunk_id,
+        sectionId: owners.get(r.chunk_id),
+        score: 0,
+        exact: true,
+    }));
+    async function retrieve(channel) {
+        let count = 2 * target, candidates = [], capped = false;
+        while (true) {
+            const rows = channel === 'lexical'
+                ? fts
+                    ? (await db.execute({
+                        sql: 'SELECT id,fts_score(body,heading,path,?) AS score FROM lexical_chunks ORDER BY score DESC LIMIT ?',
+                        args: [fts, count],
+                    })).rows
+                    : []
+                : (await db.execute({
+                    sql: 'SELECT c.id,1-vector_distance_cos(e.embedding,vector32(?)) AS score FROM chunks c JOIN embeddings e ON c.input_hash=e.hash ORDER BY score DESC LIMIT ?',
+                    args: [vectorJson, count],
+                })).rows;
+            const hits = rows
+                .filter((r) => channel === 'lexical' ? r.score > 0 : r.score >= minSimilarity)
+                .map((r) => ({
+                id: r.id,
+                sectionId: owners.get(r.id),
+                score: Number(r.score),
+            }));
+            if (channel === 'lexical') {
+                const exactIds = new Set(exact.map((c) => c.id));
+                candidates = [...exact, ...hits.filter((c) => !exactIds.has(c.id))];
+            }
+            else
+                candidates = hits;
+            candidates.sort((a, b) => Number(!!b.exact) - Number(!!a.exact) ||
+                b.score - a.score ||
+                a.sectionId.localeCompare(b.sectionId) ||
+                a.id - b.id);
+            if (new Set(candidates.map((c) => c.sectionId)).size >= target ||
+                rows.length < count ||
+                hits.length < rows.length ||
+                !rows.length)
+                break;
+            if (count >= cap) {
+                capped = true;
+                break;
+            }
+            count = Math.min(cap, count * 2);
+        }
+        return { list: collapse(candidates), capped, count: candidates.length };
+    }
+    const lexical = await retrieve('lexical'), semantic = await retrieve('semantic');
+    const all = new Set([...lexical.list.keys(), ...semantic.list.keys()]);
+    const ranked = [...all]
+        .map((id) => ({ id, l: lexical.list.get(id), v: semantic.list.get(id) }))
+        .map((r) => ({
+        ...r,
+        rankScore: (r.l ? 1 / (60 + r.l.rank) : 0) + (r.v ? 1 / (60 + r.v.rank) : 0),
+    }))
+        .sort((a, b) => b.rankScore - a.rankScore || a.id.localeCompare(b.id))
+        .slice(0, limit);
+    const results = [];
+    for (const r of ranked) {
+        const s = (await db.execute({
+            sql: 'SELECT * FROM sections WHERE id=?',
+            args: [r.id],
+        })).rows[0];
+        if (!s)
+            continue;
+        const channels = [
+            ...(r.l ? [{ candidate: r.l, channel: 'lexical' }] : []),
+            ...(r.v ? [{ candidate: r.v, channel: 'semantic' }] : []),
+        ].sort((a, b) => a.candidate.rank - b.candidate.rank);
+        const seen = new Set(), evidence = [];
+        for (const { candidate, channel } of channels)
+            if (!seen.has(candidate.id)) {
+                seen.add(candidate.id);
+                const p = (await db.execute({
+                    sql: 'SELECT source_id,body,spans FROM chunks WHERE id=?',
+                    args: [candidate.id],
+                })).rows[0];
+                evidence.push({
+                    chunkId: p.source_id,
+                    text: p.body,
+                    spans: JSON.parse(p.spans),
+                    channel,
+                });
+            }
+        results.push({
+            id: s.id,
+            file: s.file,
+            heading: s.heading,
+            content: s.content,
+            rankScore: r.rankScore,
+            ...(r.v ? { semanticSimilarity: r.v.score, semanticRank: r.v.rank } : {}),
+            ...(r.l ? { lexicalScore: r.l.score, lexicalRank: r.l.rank } : {}),
+            evidence,
+            diagnostics: {
+                lexicalCapped: lexical.capped,
+                semanticCapped: semantic.capped,
+                lexicalCandidates: lexical.count,
+                semanticCandidates: semantic.count,
+            },
+        });
+    }
+    return results;
 }