@plebeiantech/lat.md 0.12.3-fork.16 → 0.12.3-fork.17

package/dist/src/parser-cache.d.ts

@@ -1,5 +1,5 @@
 /** Version of persistent parser outputs and their shared on-disk contract. */
-export declare const PARSER_CACHE_VERSION = 1;
+export declare const PARSER_CACHE_VERSION = 3;
 export type ParsedCacheEntry = {
     version: number;
     contentHash: string;

package/dist/src/parser-cache.js

@@ -3,7 +3,7 @@ import { mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises';
 import { basename, dirname, extname, join, relative } from 'node:path';
 import { toPosix } from './path.js';
 /** Version of persistent parser outputs and their shared on-disk contract. */
-export const PARSER_CACHE_VERSION = 1;
+export const PARSER_CACHE_VERSION = 3;
 /** Hash complete parser input bytes for cache invalidation. */
 export function hashParserContent(content) {
     return createHash('sha1').update(content).digest('hex');

package/dist/src/search/cache.d.ts

@@ -0,0 +1,3 @@
+import { SearchDb } from './db.js';
+/** Build a private staging database; readers only open snapshots of search.db. */
+export declare function writeIndex<T>(latDir: string, cacheDir: string | undefined, rebuild: boolean, work: (db: SearchDb, storedModel: string | null) => Promise<T>): Promise<T>;

package/dist/src/search/cache.js

@@ -0,0 +1,71 @@
+import { copyFile, mkdir, rename, rm } from 'node:fs/promises';
+import { join } from 'node:path';
+import { acquireSearchLock } from './lock.js';
+import { SearchDb, hasIndex, INDEX_FILE, ensureMeta, getStoredModel, } from './db.js';
+/** Retry transient Windows handles without deleting the usable index first. */
+async function withFileRetry(work) {
+    for (let attempt = 0;; attempt++) {
+        try {
+            await work();
+            return;
+        }
+        catch (error) {
+            const code = error.code;
+            // Windows may briefly retain a staging handle or a reader's copy handle.
+            if (process.platform !== 'win32' ||
+                attempt >= 9 ||
+                !['EBUSY', 'EPERM', 'EACCES'].includes(code ?? ''))
+                throw error;
+            await new Promise((resolve) => setTimeout(resolve, 50 * (attempt + 1)));
+        }
+    }
+}
+async function removeStaging(path) {
+    for (const suffix of ['', '-wal', '-shm', '-journal'])
+        await withFileRetry(() => rm(path + suffix, { force: true }));
+}
+/** Build a private staging database; readers only open snapshots of search.db. */
+export async function writeIndex(latDir, cacheDir, rebuild, work) {
+    const dir = cacheDir ?? join(latDir, '.cache');
+    await mkdir(dir, { recursive: true });
+    const path = join(dir, '_search.db');
+    const activePath = join(dir, INDEX_FILE);
+    let db;
+    const release = await acquireSearchLock(dir);
+    try {
+        // A crashed writer may have left a partial database and journal behind.
+        await removeStaging(path);
+        const existing = hasIndex(dir);
+        if (existing && !rebuild)
+            await copyFile(activePath, path);
+        db = new SearchDb(path);
+        const model = existing && !rebuild ? await getStoredModel(db) : null;
+        await ensureMeta(db);
+        const changesBefore = (await db.execute('SELECT total_changes() AS n'))
+            .rows[0].n;
+        const result = await work(db, model);
+        const unchanged = existing &&
+            !rebuild &&
+            (await db.execute('SELECT total_changes() AS n')).rows[0].n ===
+                changesBefore;
+        await db.checkpoint();
+        await db.close();
+        db = undefined;
+        if (!unchanged)
+            await withFileRetry(() => rename(path, activePath));
+        return result;
+    }
+    finally {
+        try {
+            await db?.close();
+        }
+        finally {
+            try {
+                await removeStaging(path);
+            }
+            finally {
+                await release();
+            }
+        }
+    }
+}

package/dist/src/search/chunks.d.ts

@@ -0,0 +1,23 @@
+import type { SourceSpan } from './types.js';
+import type { Embedder } from './embedder.js';
+import type { MarkdownFileAnalysis } from '../markdown-analysis.js';
+import type { Section } from '../lattice-model.js';
+export declare const CHUNK_POLICY = "owned-blocks-v1";
+export type Passage = {
+    id: string;
+    sectionId: string;
+    ordinal: number;
+    type: string;
+    spans: SourceSpan[];
+    text: string;
+    input: string;
+    inputHash: string;
+    heading: string;
+    path: string;
+};
+export declare const digest: (text: string) => string;
+export declare function embeddingFingerprint(embedder: Embedder): string;
+/** Find a fitting Unicode-safe prefix without assuming token-count monotonicity. */
+export declare function fittingPrefix(text: string, fits: (text: string) => boolean): number;
+/** Every body block is owned once; headings provide context instead of copied subtrees. */
+export declare function chunkFile(file: MarkdownFileAnalysis, sections: readonly Section[], embedder: Embedder): Passage[];

package/dist/src/search/chunks.js

@@ -0,0 +1,197 @@
+import { createHash } from 'node:crypto';
+export const CHUNK_POLICY = 'owned-blocks-v1';
+export const digest = (text) => createHash('sha256').update(text).digest('hex');
+export function embeddingFingerprint(embedder) {
+    return `${CHUNK_POLICY}:${embedder.name}:${embedder.dimensions}:${embedder.maxInputTokens}:${embedder.tokenizerFingerprint}`;
+}
+/** Find a fitting Unicode-safe prefix without assuming token-count monotonicity. */
+export function fittingPrefix(text, fits) {
+    if (fits(text))
+        return text.length;
+    let size = text.length;
+    while (size > 0) {
+        size = Math.floor(size / 2);
+        if (size && /[\uD800-\uDBFF]/.test(text[size - 1]))
+            size--;
+        if (size && fits(text.slice(0, size)))
+            return size;
+    }
+    return 0;
+}
+/** Every body block is owned once; headings provide context instead of copied subtrees. */
+export function chunkFile(file, sections, embedder) {
+    const local = embedder.name.startsWith('local:');
+    const target = local ? 192 : 512;
+    const contextBudget = local ? 48 : 96;
+    const source = file.content;
+    const lineStarts = [0];
+    for (let i = 0; i < source.length; i++)
+        if (source[i] === '\n')
+            lineStarts.push(i + 1);
+    const lineAt = (offset) => {
+        let lo = 0, hi = lineStarts.length;
+        while (lo + 1 < hi) {
+            const mid = (lo + hi) >>> 1;
+            if (lineStarts[mid] <= offset)
+                lo = mid;
+            else
+                hi = mid;
+        }
+        return lo + 1;
+    };
+    const own = new Map();
+    const ordered = [...sections].sort((a, b) => a.startLine - b.startLine || a.depth - b.depth);
+    let sectionIndex = 0;
+    let ancestors = [];
+    for (const block of file.blocks) {
+        if (block.type === 'heading' || block.type === 'yaml')
+            continue;
+        while (sectionIndex < ordered.length &&
+            ordered[sectionIndex].startLine <= block.startLine) {
+            const next = ordered[sectionIndex++];
+            ancestors = ancestors.filter((s) => s.endLine >= next.startLine);
+            ancestors.push(next);
+        }
+        ancestors = ancestors.filter((s) => s.endLine >= block.endLine);
+        const owner = ancestors.at(-1);
+        if (!owner)
+            throw new Error(`No section owns ${file.path}:${block.startLine}`);
+        const blocks = own.get(owner.id) ?? [];
+        blocks.push(block);
+        own.set(owner.id, blocks);
+    }
+    const result = [];
+    for (const section of sections) {
+        const headings = section.id.split('#').slice(1);
+        const rawContext = `Section: ${section.heading}\nPage: ${file.headingTitles[0] ?? file.path}\nPath: ${headings.slice(0, -1).join(' > ')}`;
+        const contextLength = fittingPrefix(rawContext, (t) => embedder.countTokens(t) <= contextBudget);
+        const context = rawContext.slice(0, contextLength);
+        const path = headings.slice(0, -1).join(' > ');
+        let ordinal = 0;
+        const inputFor = (text, extra) => {
+            const extraSize = fittingPrefix(extra, (t) => embedder.countTokens(t) <= contextBudget / 2);
+            const fullContext = [extra.slice(0, extraSize), context]
+                .filter(Boolean)
+                .join('\n');
+            const n = fittingPrefix(fullContext, (c) => embedder.countTokens(c) <= contextBudget &&
+                embedder.countTokens(`${c}\n\n${text}`) <= embedder.maxInputTokens);
+            return n ? `${fullContext.slice(0, n)}\n\n${text}` : text;
+        };
+        const fits = (text, extra = '') => embedder.countTokens(text) <= target &&
+            embedder.countTokens(inputFor(text, extra)) <= embedder.maxInputTokens;
+        const split = (block, extra = '') => {
+            const text = source.slice(block.start, block.end);
+            if (fits(text, extra))
+                return [
+                    { start: block.start, end: block.end, type: block.type, extra },
+                ];
+            if (block.children.length) {
+                const pieces = [];
+                let start = block.start;
+                for (let i = 0; i < block.children.length; i++) {
+                    const child = block.children[i];
+                    const end = i + 1 < block.children.length
+                        ? block.children[i + 1].start
+                        : block.end;
+                    let label = extra;
+                    if (block.type === 'table')
+                        label = `Row: ${i}\nTable headers: ${source.slice(block.children[0].start, block.children[0].end)}\n${extra}`;
+                    if (block.type === 'tableRow')
+                        label = `Column: ${i + 1}\n${extra}`;
+                    if (block.type === 'listItem')
+                        label += `\nList item: ${text.split('\n')[0]}`;
+                    pieces.push(...split({ ...child, start, end }, label));
+                    start = end;
+                }
+                return pieces;
+            }
+            const pieces = [];
+            let start = block.start;
+            const label = block.type === 'code'
+                ? `${extra}\nCode: ${block.language ?? ''}`
+                : extra;
+            while (start < block.end) {
+                const rest = source.slice(start, block.end);
+                let n = fittingPrefix(rest, (t) => fits(t, label));
+                if (!n)
+                    throw new Error(`Cannot fit source text at ${file.path}:${lineAt(start)}`);
+                if (n < rest.length) {
+                    const prefix = rest.slice(0, n);
+                    const boundary = block.type === 'code'
+                        ? prefix.lastIndexOf('\n') + 1
+                        : [...prefix.matchAll(/[.!?](?:\s+|$)/g)].at(-1)?.index;
+                    const preferred = typeof boundary === 'number' && boundary > n / 2
+                        ? boundary
+                        : prefix.search(/\s+\S*$/);
+                    if (preferred > n / 2 && fits(rest.slice(0, preferred), label))
+                        n = preferred;
+                }
+                pieces.push({ start, end: start + n, type: block.type, extra: label });
+                start += n;
+            }
+            return pieces;
+        };
+        const pieces = (own.get(section.id) ?? []).flatMap((block) => split(block));
+        if (!pieces.length) {
+            const start = lineStarts[section.startLine - 1] ?? 0;
+            const end = source.indexOf('\n', start);
+            pieces.push(...split({
+                type: 'heading',
+                start,
+                end: end < 0 ? source.length : end,
+                startLine: section.startLine,
+                endLine: section.startLine,
+                children: [],
+            }));
+        }
+        let pending;
+        const emit = (piece) => {
+            const text = source.slice(piece.start, piece.end);
+            if (!text.trim())
+                return;
+            const input = inputFor(text, piece.extra);
+            if (embedder.countTokens(input) > embedder.maxInputTokens)
+                throw new Error('Chunk exceeds embedding limit');
+            const inputHash = digest(`${embeddingFingerprint(embedder)}\0${input}`);
+            result.push({
+                id: `${section.id}:${ordinal}`,
+                sectionId: section.id,
+                ordinal: ordinal++,
+                type: piece.type,
+                spans: [
+                    {
+                        start: piece.start,
+                        end: piece.end,
+                        startLine: lineAt(piece.start),
+                        endLine: lineAt(Math.max(piece.start, piece.end - 1)),
+                    },
+                ],
+                text,
+                input,
+                inputHash,
+                heading: section.heading,
+                path,
+            });
+        };
+        for (const piece of pieces) {
+            if (pending &&
+                !pending.extra &&
+                !piece.extra &&
+                !source.slice(pending.end, piece.start).trim() &&
+                fits(source.slice(pending.start, piece.end)))
+                pending = {
+                    ...pending,
+                    end: piece.end,
+                    type: pending.type === piece.type ? piece.type : 'mixed',
+                };
+            else {
+                if (pending)
+                    emit(pending);
+                pending = piece;
+            }
+        }
+        if (pending)
+            emit(pending);
+    }
+    return result;
+}

package/dist/src/search/db.d.ts

@@ -1,16 +1,27 @@
-import { type Client } from '@libsql/client';
-export declare function openDb(latDir: string, requestedCacheDir?: string): Client;
-/** Create the `meta` key/value table if absent. */
-export declare function ensureMeta(db: Client): Promise<void>;
-/**
- * The embedding backend this index was built with, e.g. `local:minilm-l6-v2:384`
- * or `openai:1536`. This record is *authoritative*: it governs which backend
- * `lat search` uses. Null when the index has never been built.
- */
-export declare function getStoredModel(db: Client): Promise<string | null>;
-export declare function setStoredModel(db: Client, value: string): Promise<void>;
-/** Create the `sections` table (fixed-width vector column) + index if absent. */
-export declare function ensureSectionsSchema(db: Client, dimensions: number): Promise<void>;
-/** Drop the vector table (used by `lat reindex` before a full rebuild). */
-export declare function dropSections(db: Client): Promise<void>;
-export declare function closeDb(db: Client): Promise<void>;
+export declare const INDEX_FILE = "search.db";
+export declare const CREATE_PASSAGE_FTS = "CREATE INDEX IF NOT EXISTS chunks_fts ON lexical_chunks USING fts(body,heading,path) WITH (tokenizer='whitespace',weights='body=1.0,heading=2.0,path=0.5')";
+export declare function hasIndex(cacheDir: string): boolean;
+/** Small SQL adapter; callers never depend on a libSQL connection. */
+export declare class SearchDb {
+    readonly path: string;
+    private readonly snapshot;
+    private connection;
+    constructor(path: string, snapshot?: boolean);
+    private snapshotDir;
+    private get;
+    execute(statement: string | {
+        sql: string;
+        args?: any[];
+    }): Promise<{
+        rows: any[];
+    }>;
+    close(): Promise<void>;
+    checkpoint(): Promise<void>;
+}
+export declare function openDb(latDir: string, requestedCacheDir?: string, readOnly?: boolean): SearchDb;
+export declare function ensureMeta(db: SearchDb): Promise<void>;
+export declare function getStoredModel(db: SearchDb): Promise<string | null>;
+export declare function setStoredModel(db: SearchDb, value: string): Promise<void>;
+export declare function ensureSectionsSchema(db: SearchDb, _dimensions: number): Promise<void>;
+export declare function dropSections(db: SearchDb): Promise<void>;
+export declare function closeDb(db: SearchDb): Promise<void>;

package/dist/src/search/db.js

@@ -1,54 +1,124 @@
-import { createClient } from '@libsql/client';
-import { mkdirSync } from 'node:fs';
+import { connect } from '@tursodatabase/database';
+import { existsSync, mkdirSync, mkdtempSync, copyFileSync } from 'node:fs';
+import { rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
 import { join } from 'node:path';
-export function openDb(latDir, requestedCacheDir) {
+export const INDEX_FILE = 'search.db';
+export const CREATE_PASSAGE_FTS = "CREATE INDEX IF NOT EXISTS chunks_fts ON lexical_chunks USING fts(body,heading,path) WITH (tokenizer='whitespace',weights='body=1.0,heading=2.0,path=0.5')";
+export function hasIndex(cacheDir) {
+    return existsSync(join(cacheDir, INDEX_FILE));
+}
+/** Small SQL adapter; callers never depend on a libSQL connection. */
+export class SearchDb {
+    path;
+    snapshot;
+    connection;
+    constructor(path, snapshot = false) {
+        this.path = path;
+        this.snapshot = snapshot;
+    }
+    snapshotDir;
+    get() {
+        if (this.connection)
+            return this.connection;
+        let path = this.path;
+        if (this.snapshot) {
+            this.snapshotDir = mkdtempSync(join(tmpdir(), 'lat-search-reader-'));
+            path = join(this.snapshotDir, 'index.db');
+            copyFileSync(this.path, path);
+        }
+        return (this.connection = connect(path, {
+            experimental: ['index_method'],
+            timeout: 10000,
+        }));
+    }
+    async execute(statement) {
+        const { sql, args = [] } = typeof statement === 'string' ? { sql: statement } : statement;
+        const db = await this.get();
+        const prepared = await db.prepare(sql);
+        try {
+            return { rows: await prepared.all(...args) };
+        }
+        finally {
+            prepared.close();
+        }
+    }
+    async close() {
+        try {
+            if (this.connection) {
+                const db = await this.connection;
+                await db.close();
+                this.connection = undefined;
+            }
+        }
+        finally {
+            if (this.snapshotDir) {
+                try {
+                    await rm(this.snapshotDir, {
+                        recursive: true,
+                        force: true,
+                        maxRetries: 3,
+                        retryDelay: 20,
+                    });
+                }
+                catch (error) {
+                    const code = error.code;
+                    if (process.platform !== 'win32' ||
+                        !['EBUSY', 'EPERM', 'ENOTEMPTY'].includes(code ?? ''))
+                        throw error;
+                    // Native handles can outlive close on Windows; this is only an owned temp copy.
+                }
+                this.snapshotDir = undefined;
+            }
+        }
+    }
+    async checkpoint() {
+        await this.execute('PRAGMA wal_checkpoint(TRUNCATE)');
+    }
+}
+export function openDb(latDir, requestedCacheDir, readOnly = false) {
     const cacheDir = requestedCacheDir ?? join(latDir, '.cache');
     mkdirSync(cacheDir, { recursive: true });
-    const client = createClient({
-        url: `file:${join(cacheDir, 'vectors.db')}`,
-    });
-    return client;
+    return new SearchDb(join(cacheDir, INDEX_FILE), readOnly);
 }
-/** Create the `meta` key/value table if absent. */
 export async function ensureMeta(db) {
-    await db.execute(`CREATE TABLE IF NOT EXISTS meta (
-      key TEXT PRIMARY KEY,
-      value TEXT NOT NULL
-    )`);
-}
-/**
- * The embedding backend this index was built with, e.g. `local:minilm-l6-v2:384`
- * or `openai:1536`. This record is *authoritative*: it governs which backend
- * `lat search` uses. Null when the index has never been built.
- */
+    await db.execute('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)');
+}
 export async function getStoredModel(db) {
-    const rows = await db.execute("SELECT value FROM meta WHERE key = 'embedding_model'");
+    const rows = await db.execute("SELECT value FROM meta WHERE key='embedding_model'");
     return rows.rows[0]?.value ?? null;
 }
 export async function setStoredModel(db, value) {
     await db.execute({
-        sql: "INSERT OR REPLACE INTO meta (key, value) VALUES ('embedding_model', ?)",
-        args: [value],
+        sql: 'INSERT OR REPLACE INTO meta VALUES (?,?)',
+        args: ['embedding_model', value],
     });
 }
-/** Create the `sections` table (fixed-width vector column) + index if absent. */
-export async function ensureSectionsSchema(db, dimensions) {
-    await db.execute(`CREATE TABLE IF NOT EXISTS sections (
-      id TEXT PRIMARY KEY,
-      file TEXT NOT NULL,
-      heading TEXT NOT NULL,
-      content TEXT NOT NULL,
-      content_hash TEXT NOT NULL,
-      embedding F32_BLOB(${dimensions}),
-      updated_at INTEGER NOT NULL
-    )`);
-    await db.execute(`CREATE INDEX IF NOT EXISTS sections_vec_idx
-     ON sections (libsql_vector_idx(embedding))`);
-}
-/** Drop the vector table (used by `lat reindex` before a full rebuild). */
+export async function ensureSectionsSchema(db, _dimensions) {
+    for (const sql of [
+        'CREATE TABLE IF NOT EXISTS sections (id TEXT PRIMARY KEY, file TEXT NOT NULL, heading TEXT NOT NULL, content TEXT NOT NULL, content_hash TEXT NOT NULL, parent_id TEXT, start_line INTEGER, end_line INTEGER)',
+        'CREATE TABLE IF NOT EXISTS embeddings (hash TEXT PRIMARY KEY, embedding BLOB NOT NULL)',
+        'CREATE TABLE IF NOT EXISTS chunks (id INTEGER PRIMARY KEY, source_id TEXT UNIQUE NOT NULL, section_id TEXT NOT NULL, ordinal INTEGER NOT NULL, type TEXT NOT NULL, spans TEXT NOT NULL, body TEXT NOT NULL, heading TEXT NOT NULL, path TEXT NOT NULL, input_hash TEXT NOT NULL)',
+        'CREATE INDEX IF NOT EXISTS chunks_section ON chunks(section_id)',
+        'CREATE TABLE IF NOT EXISTS lexical_chunks (id INTEGER PRIMARY KEY, body TEXT NOT NULL, heading TEXT NOT NULL, path TEXT NOT NULL)',
+        'CREATE TABLE IF NOT EXISTS identifiers (token TEXT NOT NULL, chunk_id INTEGER NOT NULL, PRIMARY KEY(token,chunk_id))',
+    ])
+        await db.execute(sql);
+    const oldIndex = (await db.execute("SELECT tbl_name FROM sqlite_master WHERE name='chunks_fts'")).rows[0];
+    if (oldIndex?.tbl_name === 'chunks')
+        await db.execute('DROP INDEX chunks_fts');
+    await db.execute(CREATE_PASSAGE_FTS);
+}
 export async function dropSections(db) {
-    await db.execute('DROP TABLE IF EXISTS sections');
+    for (const name of [
+        'identifiers',
+        'lexical_chunks',
+        'chunks',
+        'embeddings',
+        'sections',
+    ])
+        await db.execute(`DROP TABLE IF EXISTS ${name}`);
 }
 export async function closeDb(db) {
-    db.close();
+    await db.close();
 }

package/dist/src/search/index.d.ts

@@ -1,10 +1,13 @@
-import type { Client } from '@libsql/client';
+import { type SearchDb } from './db.js';
 import { type MarkdownProjectAnalysis } from '../project-analysis.js';
 import type { Embedder } from './embedder.js';
+export declare function projectFingerprint(project: MarkdownProjectAnalysis): string;
 export type IndexStats = {
     added: number;
     updated: number;
     removed: number;
     unchanged: number;
 };
-export declare function indexSections(latDir: string, db: Client, embedder: Embedder, onProgress?: (done: number, total: number) => void, analyzedProject?: MarkdownProjectAnalysis): Promise<IndexStats>;
+/** Exact identifier tokens complement FTS's punctuation and long-token handling. */
+export declare function identifierTokens(text: string): string[];
+export declare function indexSections(latDir: string, db: SearchDb, embedder: Embedder, onProgress?: (done: number, total: number) => void, analyzedProject?: MarkdownProjectAnalysis): Promise<IndexStats>;