@plebeiantech/lat.md 0.12.3-fork.15 → 0.12.3-fork.17

package/dist/src/search/db.js

@@ -1,54 +1,124 @@
-import { createClient } from '@libsql/client';
-import { mkdirSync } from 'node:fs';
+import { connect } from '@tursodatabase/database';
+import { existsSync, mkdirSync, mkdtempSync, copyFileSync } from 'node:fs';
+import { rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
 import { join } from 'node:path';
-export function openDb(latDir, requestedCacheDir) {
+export const INDEX_FILE = 'search.db';
+export const CREATE_PASSAGE_FTS = "CREATE INDEX IF NOT EXISTS chunks_fts ON lexical_chunks USING fts(body,heading,path) WITH (tokenizer='whitespace',weights='body=1.0,heading=2.0,path=0.5')";
+export function hasIndex(cacheDir) {
+    return existsSync(join(cacheDir, INDEX_FILE));
+}
+/** Small SQL adapter; callers never depend on a libSQL connection. */
+export class SearchDb {
+    path;
+    snapshot;
+    connection;
+    constructor(path, snapshot = false) {
+        this.path = path;
+        this.snapshot = snapshot;
+    }
+    snapshotDir;
+    get() {
+        if (this.connection)
+            return this.connection;
+        let path = this.path;
+        if (this.snapshot) {
+            this.snapshotDir = mkdtempSync(join(tmpdir(), 'lat-search-reader-'));
+            path = join(this.snapshotDir, 'index.db');
+            copyFileSync(this.path, path);
+        }
+        return (this.connection = connect(path, {
+            experimental: ['index_method'],
+            timeout: 10000,
+        }));
+    }
+    async execute(statement) {
+        const { sql, args = [] } = typeof statement === 'string' ? { sql: statement } : statement;
+        const db = await this.get();
+        const prepared = await db.prepare(sql);
+        try {
+            return { rows: await prepared.all(...args) };
+        }
+        finally {
+            prepared.close();
+        }
+    }
+    async close() {
+        try {
+            if (this.connection) {
+                const db = await this.connection;
+                await db.close();
+                this.connection = undefined;
+            }
+        }
+        finally {
+            if (this.snapshotDir) {
+                try {
+                    await rm(this.snapshotDir, {
+                        recursive: true,
+                        force: true,
+                        maxRetries: 3,
+                        retryDelay: 20,
+                    });
+                }
+                catch (error) {
+                    const code = error.code;
+                    if (process.platform !== 'win32' ||
+                        !['EBUSY', 'EPERM', 'ENOTEMPTY'].includes(code ?? ''))
+                        throw error;
+                    // Native handles can outlive close on Windows; this is only an owned temp copy.
+                }
+                this.snapshotDir = undefined;
+            }
+        }
+    }
+    async checkpoint() {
+        await this.execute('PRAGMA wal_checkpoint(TRUNCATE)');
+    }
+}
+export function openDb(latDir, requestedCacheDir, readOnly = false) {
     const cacheDir = requestedCacheDir ?? join(latDir, '.cache');
     mkdirSync(cacheDir, { recursive: true });
-    const client = createClient({
-        url: `file:${join(cacheDir, 'vectors.db')}`,
-    });
-    return client;
+    return new SearchDb(join(cacheDir, INDEX_FILE), readOnly);
 }
-/** Create the `meta` key/value table if absent. */
 export async function ensureMeta(db) {
-    await db.execute(`CREATE TABLE IF NOT EXISTS meta (
-      key TEXT PRIMARY KEY,
-      value TEXT NOT NULL
-    )`);
-}
-/**
- * The embedding backend this index was built with, e.g. `local:minilm-l6-v2:384`
- * or `openai:1536`. This record is *authoritative*: it governs which backend
- * `lat search` uses. Null when the index has never been built.
- */
+    await db.execute('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)');
+}
 export async function getStoredModel(db) {
-    const rows = await db.execute("SELECT value FROM meta WHERE key = 'embedding_model'");
+    const rows = await db.execute("SELECT value FROM meta WHERE key='embedding_model'");
     return rows.rows[0]?.value ?? null;
 }
 export async function setStoredModel(db, value) {
     await db.execute({
-        sql: "INSERT OR REPLACE INTO meta (key, value) VALUES ('embedding_model', ?)",
-        args: [value],
+        sql: 'INSERT OR REPLACE INTO meta VALUES (?,?)',
+        args: ['embedding_model', value],
     });
 }
-/** Create the `sections` table (fixed-width vector column) + index if absent. */
-export async function ensureSectionsSchema(db, dimensions) {
-    await db.execute(`CREATE TABLE IF NOT EXISTS sections (
-      id TEXT PRIMARY KEY,
-      file TEXT NOT NULL,
-      heading TEXT NOT NULL,
-      content TEXT NOT NULL,
-      content_hash TEXT NOT NULL,
-      embedding F32_BLOB(${dimensions}),
-      updated_at INTEGER NOT NULL
-    )`);
-    await db.execute(`CREATE INDEX IF NOT EXISTS sections_vec_idx
-     ON sections (libsql_vector_idx(embedding))`);
-}
-/** Drop the vector table (used by `lat reindex` before a full rebuild). */
+export async function ensureSectionsSchema(db, _dimensions) {
+    for (const sql of [
+        'CREATE TABLE IF NOT EXISTS sections (id TEXT PRIMARY KEY, file TEXT NOT NULL, heading TEXT NOT NULL, content TEXT NOT NULL, content_hash TEXT NOT NULL, parent_id TEXT, start_line INTEGER, end_line INTEGER)',
+        'CREATE TABLE IF NOT EXISTS embeddings (hash TEXT PRIMARY KEY, embedding BLOB NOT NULL)',
+        'CREATE TABLE IF NOT EXISTS chunks (id INTEGER PRIMARY KEY, source_id TEXT UNIQUE NOT NULL, section_id TEXT NOT NULL, ordinal INTEGER NOT NULL, type TEXT NOT NULL, spans TEXT NOT NULL, body TEXT NOT NULL, heading TEXT NOT NULL, path TEXT NOT NULL, input_hash TEXT NOT NULL)',
+        'CREATE INDEX IF NOT EXISTS chunks_section ON chunks(section_id)',
+        'CREATE TABLE IF NOT EXISTS lexical_chunks (id INTEGER PRIMARY KEY, body TEXT NOT NULL, heading TEXT NOT NULL, path TEXT NOT NULL)',
+        'CREATE TABLE IF NOT EXISTS identifiers (token TEXT NOT NULL, chunk_id INTEGER NOT NULL, PRIMARY KEY(token,chunk_id))',
+    ])
+        await db.execute(sql);
+    const oldIndex = (await db.execute("SELECT tbl_name FROM sqlite_master WHERE name='chunks_fts'")).rows[0];
+    if (oldIndex?.tbl_name === 'chunks')
+        await db.execute('DROP INDEX chunks_fts');
+    await db.execute(CREATE_PASSAGE_FTS);
+}
 export async function dropSections(db) {
-    await db.execute('DROP TABLE IF EXISTS sections');
+    for (const name of [
+        'identifiers',
+        'lexical_chunks',
+        'chunks',
+        'embeddings',
+        'sections',
+    ])
+        await db.execute(`DROP TABLE IF EXISTS ${name}`);
 }
 export async function closeDb(db) {
-    db.close();
+    await db.close();
 }

package/dist/src/search/index.d.ts

@@ -1,10 +1,13 @@
-import type { Client } from '@libsql/client';
+import { type SearchDb } from './db.js';
 import { type MarkdownProjectAnalysis } from '../project-analysis.js';
 import type { Embedder } from './embedder.js';
+export declare function projectFingerprint(project: MarkdownProjectAnalysis): string;
 export type IndexStats = {
     added: number;
     updated: number;
     removed: number;
     unchanged: number;
 };
-export declare function indexSections(latDir: string, db: Client, embedder: Embedder, onProgress?: (done: number, total: number) => void, analyzedProject?: MarkdownProjectAnalysis): Promise<IndexStats>;
+/** Exact identifier tokens complement FTS's punctuation and long-token handling. */
+export declare function identifierTokens(text: string): string[];
+export declare function indexSections(latDir: string, db: SearchDb, embedder: Embedder, onProgress?: (done: number, total: number) => void, analyzedProject?: MarkdownProjectAnalysis): Promise<IndexStats>;

package/dist/src/search/index.js

@@ -1,85 +1,189 @@
-import { createHash } from 'node:crypto';
-import { dirname, join } from 'node:path';
+import { PARSER_CACHE_VERSION } from '../parser-cache.js';
+import { synchronizeLexical } from './lexical.js';
+import { dirname } from 'node:path';
+import { CREATE_PASSAGE_FTS } from './db.js';
 import { analyzeMarkdownProject, } from '../project-analysis.js';
-function hashContent(text) {
-    return createHash('sha256').update(text).digest('hex');
+import { chunkFile, digest, embeddingFingerprint } from './chunks.js';
+export function projectFingerprint(project) {
+    return digest(`parser:${PARSER_CACHE_VERSION}\n` +
+        [...project.files.values()]
+            .map((f) => `${f.projectPath}\0${digest(f.content)}`)
+            .sort()
+            .join('\n'));
 }
-/**
- * Read each lat.md file once, keyed by project-relative path. Section text is
- * sliced from these lines by range: reading the file per section instead is
- * O(sections × file size) — a 3.5 MB file holding 12k sections was re-read 12k
- * times (~42 GB of string work) on every search.
- */
-function loadFileLines(sections, projectRoot, project) {
-    const lines = new Map();
-    for (const { filePath } of sections) {
-        if (lines.has(filePath))
-            continue;
-        const content = project.filesByAbsolutePath.get(join(projectRoot, filePath))?.content;
-        if (content === undefined)
-            throw new Error(`Analyzed Markdown file not found: ${filePath}`);
-        lines.set(filePath, content.split('\n'));
-    }
-    return lines;
-}
-function sectionContent(section, lines) {
-    return lines
-        .get(section.filePath)
-        .slice(section.startLine - 1, section.endLine)
-        .join('\n');
+/** Exact identifier tokens complement FTS's punctuation and long-token handling. */
+export function identifierTokens(text) {
+    return [
+        ...new Set((text.match(/[\p{L}\p{N}_][\p{L}\p{N}_./:#@-]*/gu) ?? [])
+            .map((t) => t.toLowerCase())
+            .filter((t) => /[_./:#@-]/.test(t) || t.length >= 40)),
+    ];
 }
 export async function indexSections(latDir, db, embedder, onProgress, analyzedProject) {
-    const projectRoot = dirname(latDir);
     const project = analyzedProject ??
-        (await analyzeMarkdownProject(latDir, projectRoot, { executor: 'auto' }));
-    const flat = project.sections;
-    // Build current state: id -> { section, content, hash }
-    const current = new Map();
-    const lines = loadFileLines(flat, projectRoot, project);
-    for (const s of flat) {
-        const text = sectionContent(s, lines);
-        current.set(s.id, { section: s, content: text, hash: hashContent(text) });
+        (await analyzeMarkdownProject(latDir, dirname(latDir), {
+            executor: 'auto',
+        }));
+    const fingerprint = embeddingFingerprint(embedder);
+    const projectHash = projectFingerprint(project);
+    const oldFingerprint = (await db.execute("SELECT value FROM meta WHERE key='fingerprint'")).rows[0]?.value;
+    if (oldFingerprint && oldFingerprint !== fingerprint)
+        throw new Error('Search chunking or embedding model changed; run lat reindex.');
+    const byFile = new Map();
+    for (const s of project.sections) {
+        const list = byFile.get(s.filePath) ?? [];
+        list.push(s);
+        byFile.set(s.filePath, list);
+    }
+    const files = new Map([...project.files.values()].map((f) => [f.projectPath, f]));
+    const passages = [...byFile].flatMap(([path, sections]) => {
+        const file = files.get(path);
+        if (!file)
+            throw new Error(`Missing analyzed file: ${path}`);
+        return chunkFile(file, sections, embedder);
+    });
+    const existing = new Map((await db.execute('SELECT id,content_hash FROM sections')).rows.map((r) => [
+        r.id,
+        r.content_hash,
+    ]));
+    const sectionHashes = new Map();
+    const owned = new Map();
+    for (const p of passages) {
+        const list = owned.get(p.sectionId) ?? [];
+        list.push(p);
+        owned.set(p.sectionId, list);
     }
-    // Get existing hashes from DB
-    const existing = new Map();
-    const rows = await db.execute('SELECT id, content_hash FROM sections');
-    for (const row of rows.rows) {
-        existing.set(row.id, row.content_hash);
+    for (const s of project.sections) {
+        sectionHashes.set(s.id, digest(JSON.stringify([
+            { ...s, children: undefined },
+            (owned.get(s.id) ?? []).map((p) => [p.inputHash, p.spans]),
+        ])));
     }
-    // Partition into new, changed, unchanged, deleted
-    const toEmbed = [];
-    let unchanged = 0;
-    for (const [id, entry] of current) {
-        const existingHash = existing.get(id);
-        if (existingHash === entry.hash) {
-            unchanged++;
+    const changed = new Set([...sectionHashes]
+        .filter(([id, hash]) => existing.get(id) !== hash)
+        .map(([id]) => id));
+    const removed = [...existing.keys()].filter((id) => !sectionHashes.has(id));
+    const stats = {
+        added: [...changed].filter((id) => !existing.has(id)).length,
+        updated: [...changed].filter((id) => existing.has(id)).length,
+        removed: removed.length,
+        unchanged: project.sections.length - changed.size,
+    };
+    if (!changed.size && !removed.length && oldFingerprint === fingerprint) {
+        // Lexical maintenance can repair old generations without embedding work.
+        await db.execute('BEGIN');
+        try {
+            await synchronizeLexical(db);
+            await db.execute({
+                sql: "UPDATE meta SET value=? WHERE key='project_hash' AND value<>?",
+                args: [projectHash, projectHash],
+            });
+            await db.execute('COMMIT');
         }
-        else {
-            toEmbed.push({ id, content: entry.content, section: entry.section });
+        catch (error) {
+            await db.execute('ROLLBACK');
+            throw error;
         }
+        return stats;
     }
-    const toDelete = [...existing.keys()].filter((id) => !current.has(id));
-    // Embed new/changed sections
-    if (toEmbed.length > 0) {
-        const texts = toEmbed.map((e) => e.content);
-        const vectors = await embedder.embed(texts, onProgress);
-        const now = Date.now();
-        for (let i = 0; i < toEmbed.length; i++) {
-            const { id, content, section } = toEmbed[i];
-            const hash = current.get(id).hash;
-            const vecJson = JSON.stringify(vectors[i]);
+    const storedHashes = new Set((await db.execute('SELECT hash FROM embeddings')).rows.map((r) => r.hash));
+    const missing = new Map(passages
+        .filter((p) => !storedHashes.has(p.inputHash))
+        .map((p) => [p.inputHash, p.input]));
+    const entries = [...missing];
+    const vectors = entries.length
+        ? await embedder.embed(entries.map(([, input]) => input), onProgress)
+        : [];
+    if (vectors.length !== entries.length ||
+        vectors.some((v) => v.length !== embedder.dimensions || v.some((n) => !Number.isFinite(n))))
+        throw new Error('Embedding backend returned invalid vectors');
+    const rebuildFts = !existing.size ||
+        // Tantivy retains deleted versions in BM25 statistics until rebuilt.
+        stats.updated > 0 ||
+        removed.length > 0 ||
+        passages.filter((p) => changed.has(p.sectionId)).length > 512;
+    await db.execute('BEGIN');
+    try {
+        if (rebuildFts)
+            await db.execute('DROP INDEX IF EXISTS chunks_fts');
+        for (let i = 0; i < entries.length; i++)
+            await db.execute({
+                sql: 'INSERT INTO embeddings VALUES (?,vector32(?))',
+                args: [entries[i][0], JSON.stringify(vectors[i])],
+            });
+        for (const id of [...changed, ...removed]) {
+            await db.execute({
+                sql: 'DELETE FROM lexical_chunks WHERE id IN (SELECT id FROM chunks WHERE section_id=?)',
+                args: [id],
+            });
+            await db.execute({
+                sql: 'DELETE FROM identifiers WHERE chunk_id IN (SELECT id FROM chunks WHERE section_id=?)',
+                args: [id],
+            });
             await db.execute({
-                sql: `INSERT OR REPLACE INTO sections (id, file, heading, content, content_hash, embedding, updated_at)
-              VALUES (?, ?, ?, ?, ?, vector(?), ?)`,
-                args: [id, section.file, section.heading, content, hash, vecJson, now],
+                sql: 'DELETE FROM chunks WHERE section_id=?',
+                args: [id],
             });
+            await db.execute({ sql: 'DELETE FROM sections WHERE id=?', args: [id] });
         }
+        const parents = new Map();
+        for (const s of project.sections)
+            for (const child of s.children)
+                parents.set(child.id, s.id);
+        for (const s of project.sections)
+            if (changed.has(s.id))
+                await db.execute({
+                    sql: 'INSERT INTO sections VALUES (?,?,?,?,?,?,?,?)',
+                    args: [
+                        s.id,
+                        s.file,
+                        s.heading,
+                        s.firstParagraph,
+                        sectionHashes.get(s.id),
+                        parents.get(s.id) ?? null,
+                        s.startLine,
+                        s.endLine,
+                    ],
+                });
+        for (const p of passages)
+            if (changed.has(p.sectionId)) {
+                const row = (await db.execute({
+                    sql: 'INSERT INTO chunks(source_id,section_id,ordinal,type,spans,body,heading,path,input_hash) VALUES (?,?,?,?,?,?,?,?,?) RETURNING id',
+                    args: [
+                        p.id,
+                        p.sectionId,
+                        p.ordinal,
+                        p.type,
+                        JSON.stringify(p.spans),
+                        p.text,
+                        p.heading,
+                        p.path,
+                        p.inputHash,
+                    ],
+                })).rows[0];
+                for (const token of identifierTokens(`${p.sectionId}\n${project.sectionById.get(p.sectionId.toLowerCase())?.filePath ?? ''}\n${p.heading}\n${p.path}\n${p.text}`))
+                    await db.execute({
+                        sql: 'INSERT INTO identifiers VALUES (?,?)',
+                        args: [token, row.id],
+                    });
+            }
+        await db.execute('DELETE FROM embeddings WHERE hash NOT IN (SELECT input_hash FROM chunks)');
+        await db.execute({
+            sql: 'INSERT OR REPLACE INTO meta VALUES (?,?)',
+            args: ['fingerprint', fingerprint],
+        });
+        await db.execute({
+            sql: 'INSERT OR REPLACE INTO meta VALUES (?,?)',
+            args: ['project_hash', projectHash],
+        });
+        await synchronizeLexical(db);
+        if (rebuildFts)
+            await db.execute(CREATE_PASSAGE_FTS);
+        await db.execute('COMMIT');
     }
-    // Delete removed sections
-    for (const id of toDelete) {
-        await db.execute({ sql: 'DELETE FROM sections WHERE id = ?', args: [id] });
+    catch (error) {
+        await db.execute('ROLLBACK');
+        throw error;
     }
-    const added = toEmbed.filter((e) => !existing.has(e.id)).length;
-    const updated = toEmbed.filter((e) => existing.has(e.id)).length;
-    return { added, updated, removed: toDelete.length, unchanged };
+    return stats;
 }

package/dist/src/search/lexical.d.ts

@@ -0,0 +1,6 @@
+import { type SearchDb } from './db.js';
+export declare const LEXICAL_VERSION = "snowball-english-rust-stemmers-1.2.0-v1:unicode-words-v1:live-statistics-v1";
+/** Same analysis for indexed fields and queries; exact identifiers bypass it. */
+export declare function lexicalTokens(text: string): string[];
+/** Populate derived lexical rows, reusing original passages and vectors. */
+export declare function synchronizeLexical(db: SearchDb): Promise<void>;

package/dist/src/search/lexical.js

@@ -0,0 +1,36 @@
+import { stemWords, STEMMER_VERSION } from '@lat.md/stemmer';
+import { CREATE_PASSAGE_FTS } from './db.js';
+export const LEXICAL_VERSION = `${STEMMER_VERSION}:unicode-words-v1:live-statistics-v1`;
+/** Same analysis for indexed fields and queries; exact identifiers bypass it. */
+export function lexicalTokens(text) {
+    const tokens = text.toLowerCase().match(/[\p{L}\p{N}_]+/gu) ?? [];
+    const english = tokens.filter((token) => /^[a-z]+$/.test(token));
+    const stems = stemWords(english);
+    let i = 0;
+    return tokens.map((token) => (/^[a-z]+$/.test(token) ? stems[i++] : token));
+}
+/** Populate derived lexical rows, reusing original passages and vectors. */
+export async function synchronizeLexical(db) {
+    const version = (await db.execute("SELECT value FROM meta WHERE key='lexical_version'")).rows[0]?.value;
+    if (version !== LEXICAL_VERSION) {
+        await db.execute('DROP INDEX IF EXISTS chunks_fts');
+        await db.execute('DELETE FROM lexical_chunks');
+    }
+    await db.execute('DELETE FROM lexical_chunks WHERE id NOT IN (SELECT id FROM chunks)');
+    const rows = (await db.execute('SELECT id,body,heading,path FROM chunks WHERE id NOT IN (SELECT id FROM lexical_chunks)')).rows;
+    for (const row of rows)
+        await db.execute({
+            sql: 'INSERT INTO lexical_chunks VALUES (?,?,?,?)',
+            args: [
+                row.id,
+                ...[row.body, row.heading, row.path].map((text) => lexicalTokens(text).join(' ')),
+            ],
+        });
+    if (version !== LEXICAL_VERSION) {
+        await db.execute(CREATE_PASSAGE_FTS);
+        await db.execute({
+            sql: 'INSERT OR REPLACE INTO meta VALUES (?,?)',
+            args: ['lexical_version', LEXICAL_VERSION],
+        });
+    }
+}

package/dist/src/search/lock.d.ts

@@ -0,0 +1,2 @@
+/** The persistent file must never be unlinked: all writers lock the same inode. */
+export declare function acquireSearchLock(cacheDir: string, timeoutMs?: number): Promise<() => Promise<void>>;

package/dist/src/search/lock.js

@@ -0,0 +1,23 @@
+import { open } from 'node:fs/promises';
+import { join } from 'node:path';
+import { setTimeout } from 'node:timers/promises';
+/** The persistent file must never be unlinked: all writers lock the same inode. */
+export async function acquireSearchLock(cacheDir, timeoutMs = 120_000) {
+    const { tryLock } = await import('fs-native-extensions');
+    const handle = await open(join(cacheDir, 'search-write.lock'), 'a+');
+    const deadline = performance.now() + timeoutMs;
+    try {
+        while (!tryLock(handle.fd)) {
+            const remaining = deadline - performance.now();
+            if (remaining <= 0)
+                throw new Error('Search index writer is busy; retry shortly.');
+            await setTimeout(Math.min(100, remaining));
+        }
+        // Closing releases the kernel lock, including on abnormal process exit.
+        return () => handle.close();
+    }
+    catch (error) {
+        await handle.close();
+        throw error;
+    }
+}

package/dist/src/search/query.d.ts

@@ -1,20 +1,23 @@
 import type { Section, SectionMatch } from '../lattice-model.js';
 import { type CreateSearchEngine } from './embedder.js';
+import { type SearchResult } from './search.js';
 export type IndexedSearchResult = {
     query: string;
     matches: SectionMatch[];
 };
 export type IndexedSearchSession = {
-    search: (query: string, limit: number, threshold?: number) => Promise<IndexedSearchResult>;
+    search: (query: string, limit: number, minSimilarity?: number) => Promise<SearchResult[]>;
     close: () => Promise<void>;
 };
+/** Resolve scored index rows against one already-analyzed project snapshot. */
+export declare function resolveSearchMatches(results: readonly SearchResult[], sectionById: ReadonlyMap<string, Section>): SectionMatch[];
 /** Open one reusable database and embedder for a sequence of index queries. */
-export declare function openIndexedSearchSession(latDir: string, sectionById: ReadonlyMap<string, Section>, options?: {
+export declare function openIndexedSearchSession(latDir: string, options?: {
     cacheDir?: string;
     createSearchEngine?: CreateSearchEngine;
 }): Promise<IndexedSearchSession>;
 /** Query a finished index and resolve its ids from precomputed section data. */
 export declare function searchIndexedSections(latDir: string, query: string, limit: number, sectionById: ReadonlyMap<string, Section>, options?: {
     cacheDir?: string;
-    threshold?: number;
+    minSimilarity?: number;
 }): Promise<IndexedSearchResult>;

package/dist/src/search/query.js

@@ -1,9 +1,38 @@
-import { closeDb, ensureMeta, ensureSectionsSchema, getStoredModel, openDb, } from './db.js';
+import { LEXICAL_VERSION } from './lexical.js';
+import { ReindexRequiredError } from './embedder.js';
+import { existsSync } from 'node:fs';
+import { join } from 'node:path';
+import { hasIndex } from './db.js';
+import { closeDb, getStoredModel, openDb } from './db.js';
 import { embedderForIndex } from './embedder.js';
 import { searchSections } from './search.js';
+/** Resolve scored index rows against one already-analyzed project snapshot. */
+export function resolveSearchMatches(results, sectionById) {
+    return results.flatMap((result) => {
+        const section = sectionById.get(result.id.toLowerCase());
+        return section
+            ? [
+                {
+                    section,
+                    reason: 'hybrid match',
+                    rankScore: result.rankScore,
+                    semanticSimilarity: result.semanticSimilarity,
+                    lexicalScore: result.lexicalScore,
+                    semanticRank: result.semanticRank,
+                    lexicalRank: result.lexicalRank,
+                    evidence: result.evidence,
+                    diagnostics: result.diagnostics,
+                },
+            ]
+            : [];
+    });
+}
 /** Open one reusable database and embedder for a sequence of index queries. */
-export async function openIndexedSearchSession(latDir, sectionById, options = {}) {
-    const db = openDb(latDir, options.cacheDir);
+export async function openIndexedSearchSession(latDir, options = {}) {
+    const cacheDir = options.cacheDir ?? join(latDir, '.cache');
+    if (!existsSync(cacheDir) || !hasIndex(cacheDir))
+        return { search: async () => [], close: async () => { } };
+    const db = openDb(latDir, options.cacheDir, true);
     let closed = false;
     const close = async () => {
         if (closed)
@@ -12,40 +41,26 @@ export async function openIndexedSearchSession(latDir, sectionById, options = {}
         await closeDb(db);
     };
     try {
-        await ensureMeta(db);
         const stored = await getStoredModel(db);
         if (stored === null) {
             return {
-                search: async (query) => {
+                search: async () => {
                     if (closed)
                         throw new Error('Search session is closed');
-                    return { query, matches: [] };
+                    return [];
                 },
                 close,
             };
         }
+        const lexicalVersion = (await db.execute("SELECT value FROM meta WHERE key='lexical_version'")).rows[0]?.value;
+        if (lexicalVersion !== LEXICAL_VERSION)
+            throw new ReindexRequiredError('Search lexical index changed; run lat search to update it.');
         const embedder = await embedderForIndex(stored, latDir, options.createSearchEngine);
-        await ensureSectionsSchema(db, embedder.dimensions);
         return {
-            async search(query, limit, threshold) {
+            async search(query, limit, minSimilarity) {
                 if (closed)
                     throw new Error('Search session is closed');
-                const results = await searchSections(db, query, embedder, limit, threshold);
-                return {
-                    query,
-                    matches: results.flatMap((result) => {
-                        const section = sectionById.get(result.id.toLowerCase());
-                        return section
-                            ? [
-                                {
-                                    section,
-                                    reason: 'semantic match',
-                                    score: result.score,
-                                },
-                            ]
-                            : [];
-                    }),
-                };
+                return searchSections(db, query, embedder, limit, minSimilarity);
             },
             close,
         };
@@ -57,9 +72,13 @@ export async function openIndexedSearchSession(latDir, sectionById, options = {}
 }
 /** Query a finished index and resolve its ids from precomputed section data. */
 export async function searchIndexedSections(latDir, query, limit, sectionById, options = {}) {
-    const session = await openIndexedSearchSession(latDir, sectionById, options);
+    const session = await openIndexedSearchSession(latDir, options);
     try {
-        return await session.search(query, limit, options.threshold);
+        const results = await session.search(query, limit, options.minSimilarity);
+        return {
+            query,
+            matches: resolveSearchMatches(results, sectionById),
+        };
     }
     finally {
         await session.close();