@plebeiantech/lat.md 0.12.3-fork.2 → 0.12.3-fork.20

package/dist/src/mcp/server.js

@@ -2,14 +2,16 @@ import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
 import { z } from 'zod';
 import { dirname } from 'node:path';
-import { findLatticeDir } from '../lattice.js';
-import { plainStyler } from '../context.js';
-import { locateCommand } from '../cli/locate.js';
-import { sectionCommand } from '../cli/section.js';
+import { findLatticeDir } from '@plebeiantech/lat.md-core/project-discovery';
+import { plainStyler, } from '@plebeiantech/lat.md-core/context';
+import { locateCommand } from '@plebeiantech/lat.md-core/cli/locate';
+import { sectionCommand } from '@plebeiantech/lat.md-core/cli/section';
 import { searchCommand } from '../cli/search.js';
-import { expandCommand } from '../cli/expand.js';
-import { checkAllCommand } from '../cli/check.js';
-import { refsCommand } from '../cli/refs.js';
+import { DEFAULT_SEARCH_LIMIT, DEFAULT_MIN_SIMILARITY, } from '../search/search.js';
+import { expandCommand } from '@plebeiantech/lat.md-core/cli/expand';
+import { checkAllCommand } from '@plebeiantech/lat.md-core/cli/check';
+import { refsCommand } from '@plebeiantech/lat.md-core/cli/refs';
+import { externalListCommand, externalShowCommand, } from '@plebeiantech/lat.md-core/cli/external';
 function toMcp(result) {
     const content = [{ type: 'text', text: result.output }];
     return result.isError ? { content, isError: true } : { content };
@@ -27,24 +29,34 @@ export async function startMcpServer() {
         styler: plainStyler,
         mode: 'mcp',
     };
+    const requestContext = () => ({ ...ctx, analysis: undefined });
     const server = new McpServer({
         name: 'lat',
         version: '1.0.0',
     });
-    server.tool('lat_locate', 'Find sections by name (exact, fuzzy, subsequence matching)', { query: z.string().describe('Section name or id to search for') }, async ({ query }) => toMcp(await locateCommand(ctx, query)));
-    server.tool('lat_section', 'Show a section with its content, outgoing wiki link targets, and incoming references', {
-        query: z.string().describe('Section id to look up (short or full form)'),
-    }, async ({ query }) => toMcp(await sectionCommand(ctx, query)));
-    server.tool('lat_search', 'Semantic search across lat.md sections using embeddings', {
+    server.tool('lat_locate', 'Find sections by name (exact, fuzzy, subsequence matching)', { query: z.string().describe('Section name or id to search for') }, async ({ query }) => toMcp(await locateCommand(requestContext(), query)));
+    server.tool('lat_section', 'Show a local section or exact external target with content and references', {
+        query: z
+            .string()
+            .describe('Local section id or exact handle:path#fragment target'),
+    }, async ({ query }) => toMcp(await sectionCommand(requestContext(), query)));
+    server.tool('lat_search', 'Hybrid lexical and semantic search across lat.md sections', {
         query: z.string().describe('Search query in natural language'),
         limit: z
             .number()
             .optional()
-            .default(5)
-            .describe('Max results (default 5)'),
-    }, async ({ query, limit }) => toMcp(await searchCommand(ctx, query, { limit })));
-    server.tool('lat_expand', 'Expand [[refs]] in text to resolved lat.md section paths with context', { text: z.string().describe('Text containing [[refs]] to expand') }, async ({ text: input }) => toMcp(await expandCommand(ctx, input)));
-    server.tool('lat_check', 'Run full lat.md validation: links, code references, indexes, and section structure', {}, async () => toMcp(await checkAllCommand(ctx)));
+            .default(DEFAULT_SEARCH_LIMIT)
+            .describe(`Max results (default ${DEFAULT_SEARCH_LIMIT})`),
+        minSimilarity: z
+            .number()
+            .min(0)
+            .max(1)
+            .optional()
+            .default(DEFAULT_MIN_SIMILARITY)
+            .describe('Minimum cosine similarity score (default 0.20)'),
+    }, async ({ query, limit, minSimilarity }) => toMcp(await searchCommand(requestContext(), query, { limit, minSimilarity })));
+    server.tool('lat_expand', 'Expand [[refs]] in text to resolved lat.md section paths with context', { text: z.string().describe('Text containing [[refs]] to expand') }, async ({ text: input }) => toMcp(await expandCommand(requestContext(), input)));
+    server.tool('lat_check', 'Run full lat.md validation: links, code references, indexes, and section structure', {}, async () => toMcp(await checkAllCommand(requestContext())));
     server.tool('lat_refs', 'Find sections that reference a given section via wiki links or @lat code comments', {
         query: z.string().describe('Section id to find references for'),
         scope: z
@@ -52,7 +64,11 @@ export async function startMcpServer() {
             .optional()
             .default('md+code')
             .describe('Where to search: md, code, or md+code'),
-    }, async ({ query, scope }) => toMcp(await refsCommand(ctx, query, scope)));
+    }, async ({ query, scope }) => toMcp(await refsCommand(requestContext(), query, scope)));
+    server.tool('lat_external_list', 'List configured external sources without fetching or changing caches', {}, async () => toMcp(await externalListCommand(ctx, true)));
+    server.tool('lat_external_show', 'Show one configured external source or exact external target without fetching it', {
+        source: z.string().describe('External source handle or exact target'),
+    }, async ({ source }) => toMcp(await externalShowCommand(ctx, source, true)));
     const transport = new StdioServerTransport();
     await server.connect(transport);
 }

package/dist/src/search/cache.d.ts

@@ -0,0 +1,3 @@
+import { SearchDb } from './db.js';
+/** Build a private staging database; access locks protect copies and publication of search.db. */
+export declare function writeIndex<T>(latDir: string, cacheDir: string | undefined, rebuild: boolean, work: (db: SearchDb, storedModel: string | null) => Promise<T>): Promise<T>;

package/dist/src/search/cache.js

@@ -0,0 +1,104 @@
+import { copyFile, mkdir, rename, rm, stat } from 'node:fs/promises';
+import { join } from 'node:path';
+import { acquireSearchLock, withSearchAccess } from './lock.js';
+import { SearchDb, hasIndex, INDEX_FILE, ensureMeta, getStoredModel, } from './db.js';
+/** Retry transient Windows handles without deleting the usable index first. */
+async function withFileRetry(work) {
+    for (let attempt = 0;; attempt++) {
+        try {
+            await work();
+            return;
+        }
+        catch (error) {
+            const code = error.code;
+            // Windows may briefly retain a staging handle or a reader handle.
+            if (process.platform !== 'win32' ||
+                attempt >= 9 ||
+                !['EBUSY', 'EPERM', 'EACCES'].includes(code ?? ''))
+                throw error;
+            await new Promise((resolve) => setTimeout(resolve, 50 * (attempt + 1)));
+        }
+    }
+}
+async function removeStaging(path) {
+    for (const suffix of ['', '-wal', '-shm', '-journal'])
+        await withFileRetry(() => rm(path + suffix, { force: true }));
+}
+/** Build a private staging database; access locks protect copies and publication of search.db. */
+export async function writeIndex(latDir, cacheDir, rebuild, work) {
+    const dir = cacheDir ?? join(latDir, '.cache');
+    await mkdir(dir, { recursive: true });
+    const path = join(dir, '_search.db');
+    const activePath = join(dir, INDEX_FILE);
+    let db;
+    const release = await acquireSearchLock(dir);
+    try {
+        // A crashed writer may have left a partial database and journal behind.
+        await removeStaging(path);
+        const existing = hasIndex(dir);
+        if (existing && !rebuild) {
+            await withSearchAccess(dir, 'exclusive', async () => {
+                // Recover/checkpoint a WAL left by a process that died during a query.
+                const source = new SearchDb(activePath);
+                try {
+                    await source.checkpoint();
+                }
+                finally {
+                    await source.close();
+                }
+                await copyFile(activePath, path);
+            });
+        }
+        db = new SearchDb(path);
+        const model = existing && !rebuild ? await getStoredModel(db) : null;
+        await ensureMeta(db);
+        const changesBefore = (await db.execute('SELECT total_changes() AS n'))
+            .rows[0].n;
+        const result = await work(db, model);
+        const unchanged = existing &&
+            !rebuild &&
+            (await db.execute('SELECT total_changes() AS n')).rows[0].n ===
+                changesBefore;
+        await db.checkpoint();
+        await db.close();
+        db = undefined;
+        if (!unchanged)
+            await withSearchAccess(dir, 'exclusive', async () => {
+                // Recover old sidecars before replacing their database, including after a crash.
+                if (existing) {
+                    const wal = await stat(activePath + '-wal').catch((error) => {
+                        if (error.code !== 'ENOENT')
+                            throw error;
+                        return undefined;
+                    });
+                    // A fresh rebuild must also repair an unreadable database with no WAL.
+                    if (wal && wal.size > 0) {
+                        const source = new SearchDb(activePath);
+                        try {
+                            await source.checkpoint();
+                        }
+                        finally {
+                            await source.close();
+                        }
+                    }
+                    for (const suffix of ['-wal', '-shm', '-journal'])
+                        await withFileRetry(() => rm(activePath + suffix, { force: true }));
+                }
+                await withFileRetry(() => rename(path, activePath));
+            });
+        return result;
+    }
+    finally {
+        try {
+            await db?.close();
+        }
+        finally {
+            try {
+                await removeStaging(path);
+            }
+            finally {
+                await release();
+            }
+        }
+    }
+}

package/dist/src/search/chunks.d.ts

@@ -0,0 +1,23 @@
+import type { SourceSpan } from './types.js';
+import type { Embedder } from './embedder.js';
+import type { MarkdownFileAnalysis } from '@plebeiantech/lat.md-core/markdown-analysis';
+import type { Section } from '@plebeiantech/lat.md-core/lattice-model';
+export declare const CHUNK_POLICY = "owned-blocks-v1";
+export type Passage = {
+    id: string;
+    sectionId: string;
+    ordinal: number;
+    type: string;
+    spans: SourceSpan[];
+    text: string;
+    input: string;
+    inputHash: string;
+    heading: string;
+    path: string;
+};
+export declare const digest: (text: string) => string;
+export declare function embeddingFingerprint(embedder: Embedder): string;
+/** Find a fitting Unicode-safe prefix without assuming token-count monotonicity. */
+export declare function fittingPrefix(text: string, fits: (text: string) => boolean): number;
+/** Every body block is owned once; headings provide context instead of copied subtrees. */
+export declare function chunkFile(file: MarkdownFileAnalysis, sections: readonly Section[], embedder: Embedder): Passage[];

package/dist/src/search/chunks.js

@@ -0,0 +1,197 @@
+import { createHash } from 'node:crypto';
+export const CHUNK_POLICY = 'owned-blocks-v1';
+export const digest = (text) => createHash('sha256').update(text).digest('hex');
+export function embeddingFingerprint(embedder) {
+    return `${CHUNK_POLICY}:${embedder.name}:${embedder.dimensions}:${embedder.maxInputTokens}:${embedder.tokenizerFingerprint}`;
+}
+/** Find a fitting Unicode-safe prefix without assuming token-count monotonicity. */
+export function fittingPrefix(text, fits) {
+    if (fits(text))
+        return text.length;
+    let size = text.length;
+    while (size > 0) {
+        size = Math.floor(size / 2);
+        if (size && /[\uD800-\uDBFF]/.test(text[size - 1]))
+            size--;
+        if (size && fits(text.slice(0, size)))
+            return size;
+    }
+    return 0;
+}
+/** Every body block is owned once; headings provide context instead of copied subtrees. */
+export function chunkFile(file, sections, embedder) {
+    const local = embedder.name.startsWith('local:');
+    const target = local ? 192 : 512;
+    const contextBudget = local ? 48 : 96;
+    const source = file.content;
+    const lineStarts = [0];
+    for (let i = 0; i < source.length; i++)
+        if (source[i] === '\n')
+            lineStarts.push(i + 1);
+    const lineAt = (offset) => {
+        let lo = 0, hi = lineStarts.length;
+        while (lo + 1 < hi) {
+            const mid = (lo + hi) >>> 1;
+            if (lineStarts[mid] <= offset)
+                lo = mid;
+            else
+                hi = mid;
+        }
+        return lo + 1;
+    };
+    const own = new Map();
+    const ordered = [...sections].sort((a, b) => a.startLine - b.startLine || a.depth - b.depth);
+    let sectionIndex = 0;
+    let ancestors = [];
+    for (const block of file.blocks) {
+        if (block.type === 'heading' || block.type === 'yaml')
+            continue;
+        while (sectionIndex < ordered.length &&
+            ordered[sectionIndex].startLine <= block.startLine) {
+            const next = ordered[sectionIndex++];
+            ancestors = ancestors.filter((s) => s.endLine >= next.startLine);
+            ancestors.push(next);
+        }
+        ancestors = ancestors.filter((s) => s.endLine >= block.endLine);
+        const owner = ancestors.at(-1);
+        if (!owner)
+            throw new Error(`No section owns ${file.path}:${block.startLine}`);
+        const blocks = own.get(owner.id) ?? [];
+        blocks.push(block);
+        own.set(owner.id, blocks);
+    }
+    const result = [];
+    for (const section of sections) {
+        const headings = section.id.split('#').slice(1);
+        const rawContext = `Section: ${section.heading}\nPage: ${file.headingTitles[0] ?? file.path}\nPath: ${headings.slice(0, -1).join(' > ')}`;
+        const contextLength = fittingPrefix(rawContext, (t) => embedder.countTokens(t) <= contextBudget);
+        const context = rawContext.slice(0, contextLength);
+        const path = headings.slice(0, -1).join(' > ');
+        let ordinal = 0;
+        const inputFor = (text, extra) => {
+            const extraSize = fittingPrefix(extra, (t) => embedder.countTokens(t) <= contextBudget / 2);
+            const fullContext = [extra.slice(0, extraSize), context]
+                .filter(Boolean)
+                .join('\n');
+            const n = fittingPrefix(fullContext, (c) => embedder.countTokens(c) <= contextBudget &&
+                embedder.countTokens(`${c}\n\n${text}`) <= embedder.maxInputTokens);
+            return n ? `${fullContext.slice(0, n)}\n\n${text}` : text;
+        };
+        const fits = (text, extra = '') => embedder.countTokens(text) <= target &&
+            embedder.countTokens(inputFor(text, extra)) <= embedder.maxInputTokens;
+        const split = (block, extra = '') => {
+            const text = source.slice(block.start, block.end);
+            if (fits(text, extra))
+                return [
+                    { start: block.start, end: block.end, type: block.type, extra },
+                ];
+            if (block.children.length) {
+                const pieces = [];
+                let start = block.start;
+                for (let i = 0; i < block.children.length; i++) {
+                    const child = block.children[i];
+                    const end = i + 1 < block.children.length
+                        ? block.children[i + 1].start
+                        : block.end;
+                    let label = extra;
+                    if (block.type === 'table')
+                        label = `Row: ${i}\nTable headers: ${source.slice(block.children[0].start, block.children[0].end)}\n${extra}`;
+                    if (block.type === 'tableRow')
+                        label = `Column: ${i + 1}\n${extra}`;
+                    if (block.type === 'listItem')
+                        label += `\nList item: ${text.split('\n')[0]}`;
+                    pieces.push(...split({ ...child, start, end }, label));
+                    start = end;
+                }
+                return pieces;
+            }
+            const pieces = [];
+            let start = block.start;
+            const label = block.type === 'code'
+                ? `${extra}\nCode: ${block.language ?? ''}`
+                : extra;
+            while (start < block.end) {
+                const rest = source.slice(start, block.end);
+                let n = fittingPrefix(rest, (t) => fits(t, label));
+                if (!n)
+                    throw new Error(`Cannot fit source text at ${file.path}:${lineAt(start)}`);
+                if (n < rest.length) {
+                    const prefix = rest.slice(0, n);
+                    const boundary = block.type === 'code'
+                        ? prefix.lastIndexOf('\n') + 1
+                        : [...prefix.matchAll(/[.!?](?:\s+|$)/g)].at(-1)?.index;
+                    const preferred = typeof boundary === 'number' && boundary > n / 2
+                        ? boundary
+                        : prefix.search(/\s+\S*$/);
+                    if (preferred > n / 2 && fits(rest.slice(0, preferred), label))
+                        n = preferred;
+                }
+                pieces.push({ start, end: start + n, type: block.type, extra: label });
+                start += n;
+            }
+            return pieces;
+        };
+        const pieces = (own.get(section.id) ?? []).flatMap((block) => split(block));
+        if (!pieces.length) {
+            const start = lineStarts[section.startLine - 1] ?? 0;
+            const end = source.indexOf('\n', start);
+            pieces.push(...split({
+                type: 'heading',
+                start,
+                end: end < 0 ? source.length : end,
+                startLine: section.startLine,
+                endLine: section.startLine,
+                children: [],
+            }));
+        }
+        let pending;
+        const emit = (piece) => {
+            const text = source.slice(piece.start, piece.end);
+            if (!text.trim())
+                return;
+            const input = inputFor(text, piece.extra);
+            if (embedder.countTokens(input) > embedder.maxInputTokens)
+                throw new Error('Chunk exceeds embedding limit');
+            const inputHash = digest(`${embeddingFingerprint(embedder)}\0${input}`);
+            result.push({
+                id: `${section.id}:${ordinal}`,
+                sectionId: section.id,
+                ordinal: ordinal++,
+                type: piece.type,
+                spans: [
+                    {
+                        start: piece.start,
+                        end: piece.end,
+                        startLine: lineAt(piece.start),
+                        endLine: lineAt(Math.max(piece.start, piece.end - 1)),
+                    },
+                ],
+                text,
+                input,
+                inputHash,
+                heading: section.heading,
+                path,
+            });
+        };
+        for (const piece of pieces) {
+            if (pending &&
+                !pending.extra &&
+                !piece.extra &&
+                !source.slice(pending.end, piece.start).trim() &&
+                fits(source.slice(pending.start, piece.end)))
+                pending = {
+                    ...pending,
+                    end: piece.end,
+                    type: pending.type === piece.type ? piece.type : 'mixed',
+                };
+            else {
+                if (pending)
+                    emit(pending);
+                pending = piece;
+            }
+        }
+        if (pending)
+            emit(pending);
+    }
+    return result;
+}

package/dist/src/search/db.d.ts

@@ -1,16 +1,28 @@
-import { type Client } from '@libsql/client';
-export declare function openDb(latDir: string): Client;
-/** Create the `meta` key/value table if absent. */
-export declare function ensureMeta(db: Client): Promise<void>;
-/**
- * The embedding backend this index was built with, e.g. `local:minilm-l6-v2:384`
- * or `openai:1536`. This record is *authoritative*: it governs which backend
- * `lat search` uses. Null when the index has never been built.
- */
-export declare function getStoredModel(db: Client): Promise<string | null>;
-export declare function setStoredModel(db: Client, value: string): Promise<void>;
-/** Create the `sections` table (fixed-width vector column) + index if absent. */
-export declare function ensureSectionsSchema(db: Client, dimensions: number): Promise<void>;
-/** Drop the vector table (used by `lat reindex` before a full rebuild). */
-export declare function dropSections(db: Client): Promise<void>;
-export declare function closeDb(db: Client): Promise<void>;
+export declare const INDEX_FILE = "search.db";
+export declare const CREATE_PASSAGE_FTS = "CREATE INDEX IF NOT EXISTS chunks_fts ON lexical_chunks USING fts(body,heading,path) WITH (tokenizer='whitespace',weights='body=1.0,heading=2.0,path=0.5')";
+export declare function hasIndex(cacheDir: string): boolean;
+/** Small SQL adapter; callers never depend on a libSQL connection. */
+export declare class SearchDb {
+    readonly path: string;
+    private readonly published;
+    private connection;
+    private release;
+    constructor(path: string, published?: boolean);
+    private get;
+    private connect;
+    execute(statement: string | {
+        sql: string;
+        args?: any[];
+    }): Promise<{
+        rows: any[];
+    }>;
+    close(): Promise<void>;
+    checkpoint(): Promise<void>;
+}
+export declare function openDb(latDir: string, requestedCacheDir?: string, published?: boolean): SearchDb;
+export declare function ensureMeta(db: SearchDb): Promise<void>;
+export declare function getStoredModel(db: SearchDb): Promise<string | null>;
+export declare function setStoredModel(db: SearchDb, value: string): Promise<void>;
+export declare function ensureSectionsSchema(db: SearchDb, _dimensions: number): Promise<void>;
+export declare function dropSections(db: SearchDb): Promise<void>;
+export declare function closeDb(db: SearchDb): Promise<void>;

package/dist/src/search/db.js

@@ -1,54 +1,120 @@
-import { createClient } from '@libsql/client';
-import { mkdirSync } from 'node:fs';
-import { join } from 'node:path';
-export function openDb(latDir) {
-    const cacheDir = join(latDir, '.cache');
+import { connect } from '@tursodatabase/database';
+import { existsSync, mkdirSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { acquireSearchAccess } from './lock.js';
+export const INDEX_FILE = 'search.db';
+export const CREATE_PASSAGE_FTS = "CREATE INDEX IF NOT EXISTS chunks_fts ON lexical_chunks USING fts(body,heading,path) WITH (tokenizer='whitespace',weights='body=1.0,heading=2.0,path=0.5')";
+export function hasIndex(cacheDir) {
+    return existsSync(join(cacheDir, INDEX_FILE));
+}
+/** Small SQL adapter; callers never depend on a libSQL connection. */
+export class SearchDb {
+    path;
+    published;
+    connection;
+    release;
+    constructor(path, published = false) {
+        this.path = path;
+        this.published = published;
+    }
+    get() {
+        return (this.connection ??= this.connect());
+    }
+    async connect() {
+        // Turso 0.7.2 FTS needs writable connections and exclusive process access.
+        if (this.published)
+            this.release = await acquireSearchAccess(dirname(this.path), 'exclusive');
+        try {
+            return await connect(this.path, {
+                experimental: ['index_method'],
+                timeout: 10000,
+            });
+        }
+        catch (error) {
+            await this.release?.();
+            this.release = undefined;
+            throw error;
+        }
+    }
+    async execute(statement) {
+        const { sql, args = [] } = typeof statement === 'string' ? { sql: statement } : statement;
+        const db = await this.get();
+        const prepared = await db.prepare(sql);
+        try {
+            return { rows: await prepared.all(...args) };
+        }
+        finally {
+            prepared.close();
+        }
+    }
+    async close() {
+        try {
+            if (this.connection) {
+                const db = await this.connection;
+                try {
+                    // Leave a self-contained file for incremental copies and publication.
+                    if (this.published)
+                        await this.checkpoint();
+                }
+                finally {
+                    await db.close();
+                }
+            }
+        }
+        finally {
+            this.connection = undefined;
+            const release = this.release;
+            this.release = undefined;
+            await release?.();
+        }
+    }
+    async checkpoint() {
+        await this.execute('PRAGMA wal_checkpoint(TRUNCATE)');
+    }
+}
+export function openDb(latDir, requestedCacheDir, published = false) {
+    const cacheDir = requestedCacheDir ?? join(latDir, '.cache');
     mkdirSync(cacheDir, { recursive: true });
-    const client = createClient({
-        url: `file:${join(cacheDir, 'vectors.db')}`,
-    });
-    return client;
+    return new SearchDb(join(cacheDir, INDEX_FILE), published);
 }
-/** Create the `meta` key/value table if absent. */
 export async function ensureMeta(db) {
-    await db.execute(`CREATE TABLE IF NOT EXISTS meta (
-      key TEXT PRIMARY KEY,
-      value TEXT NOT NULL
-    )`);
-}
-/**
- * The embedding backend this index was built with, e.g. `local:minilm-l6-v2:384`
- * or `openai:1536`. This record is *authoritative*: it governs which backend
- * `lat search` uses. Null when the index has never been built.
- */
+    await db.execute('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)');
+}
 export async function getStoredModel(db) {
-    const rows = await db.execute("SELECT value FROM meta WHERE key = 'embedding_model'");
+    const rows = await db.execute("SELECT value FROM meta WHERE key='embedding_model'");
     return rows.rows[0]?.value ?? null;
 }
 export async function setStoredModel(db, value) {
     await db.execute({
-        sql: "INSERT OR REPLACE INTO meta (key, value) VALUES ('embedding_model', ?)",
-        args: [value],
+        sql: 'INSERT OR REPLACE INTO meta VALUES (?,?)',
+        args: ['embedding_model', value],
     });
 }
-/** Create the `sections` table (fixed-width vector column) + index if absent. */
-export async function ensureSectionsSchema(db, dimensions) {
-    await db.execute(`CREATE TABLE IF NOT EXISTS sections (
-      id TEXT PRIMARY KEY,
-      file TEXT NOT NULL,
-      heading TEXT NOT NULL,
-      content TEXT NOT NULL,
-      content_hash TEXT NOT NULL,
-      embedding F32_BLOB(${dimensions}),
-      updated_at INTEGER NOT NULL
-    )`);
-    await db.execute(`CREATE INDEX IF NOT EXISTS sections_vec_idx
-     ON sections (libsql_vector_idx(embedding))`);
-}
-/** Drop the vector table (used by `lat reindex` before a full rebuild). */
+export async function ensureSectionsSchema(db, _dimensions) {
+    for (const sql of [
+        'CREATE TABLE IF NOT EXISTS sections (id TEXT PRIMARY KEY, file TEXT NOT NULL, heading TEXT NOT NULL, content TEXT NOT NULL, content_hash TEXT NOT NULL, parent_id TEXT, start_line INTEGER, end_line INTEGER)',
+        'CREATE TABLE IF NOT EXISTS embeddings (hash TEXT PRIMARY KEY, embedding BLOB NOT NULL)',
+        'CREATE TABLE IF NOT EXISTS chunks (id INTEGER PRIMARY KEY, source_id TEXT UNIQUE NOT NULL, section_id TEXT NOT NULL, ordinal INTEGER NOT NULL, type TEXT NOT NULL, spans TEXT NOT NULL, body TEXT NOT NULL, heading TEXT NOT NULL, path TEXT NOT NULL, input_hash TEXT NOT NULL)',
+        'CREATE INDEX IF NOT EXISTS chunks_section ON chunks(section_id)',
+        'CREATE TABLE IF NOT EXISTS lexical_chunks (id INTEGER PRIMARY KEY, body TEXT NOT NULL, heading TEXT NOT NULL, path TEXT NOT NULL)',
+        'CREATE TABLE IF NOT EXISTS identifiers (token TEXT NOT NULL, chunk_id INTEGER NOT NULL, PRIMARY KEY(token,chunk_id))',
+    ])
+        await db.execute(sql);
+    const oldIndex = (await db.execute("SELECT tbl_name FROM sqlite_master WHERE name='chunks_fts'")).rows[0];
+    if (oldIndex?.tbl_name === 'chunks')
+        await db.execute('DROP INDEX chunks_fts');
+    await db.execute(CREATE_PASSAGE_FTS);
+}
 export async function dropSections(db) {
-    await db.execute('DROP TABLE IF EXISTS sections');
+    for (const name of [
+        'identifiers',
+        'lexical_chunks',
+        'chunks',
+        'embeddings',
+        'sections',
+    ])
+        await db.execute(`DROP TABLE IF EXISTS ${name}`);
 }
 export async function closeDb(db) {
-    db.close();
+    await db.close();
 }

package/dist/src/search/embedder.d.ts

@@ -1,6 +1,7 @@
 import { EmbeddingAuthError, type Embedder } from '@lat.md/embed';
 export type { Embedder };
 export { EmbeddingAuthError };
+export type CreateSearchEngine = (key?: string) => Promise<Embedder>;
 /** `meta.embedding_model` value for an embedder, e.g. `local:minilm-l6-v2:384`. */
 export declare function modelKey(embedder: Embedder): string;
 /** Thrown when the stored index can't be served by the current environment and
@@ -22,4 +23,4 @@ export declare function localEmbedder(): Promise<Embedder>;
  * regenerable `.cache` was wiped) the durable per-repo preference wins over the
  * env, so a repo switched to local stays local across cache loss.
  */
-export declare function embedderForIndex(storedModel: string | null, latDir: string): Promise<Embedder>;
+export declare function embedderForIndex(storedModel: string | null, latDir: string, createSearchEngine?: CreateSearchEngine): Promise<Embedder>;