@plebeiantech/lat.md 0.12.3-fork.17 → 0.12.3-fork.20

package/dist/src/search/cache.js

@@ -1,6 +1,6 @@
-import { copyFile, mkdir, rename, rm } from 'node:fs/promises';
+import { copyFile, mkdir, rename, rm, stat } from 'node:fs/promises';
 import { join } from 'node:path';
-import { acquireSearchLock } from './lock.js';
+import { acquireSearchLock, withSearchAccess } from './lock.js';
 import { SearchDb, hasIndex, INDEX_FILE, ensureMeta, getStoredModel, } from './db.js';
 /** Retry transient Windows handles without deleting the usable index first. */
 async function withFileRetry(work) {
@@ -11,7 +11,7 @@ async function withFileRetry(work) {
         }
         catch (error) {
             const code = error.code;
-            // Windows may briefly retain a staging handle or a reader's copy handle.
+            // Windows may briefly retain a staging handle or a reader handle.
             if (process.platform !== 'win32' ||
                 attempt >= 9 ||
                 !['EBUSY', 'EPERM', 'EACCES'].includes(code ?? ''))
@@ -24,7 +24,7 @@ async function removeStaging(path) {
     for (const suffix of ['', '-wal', '-shm', '-journal'])
         await withFileRetry(() => rm(path + suffix, { force: true }));
 }
-/** Build a private staging database; readers only open snapshots of search.db. */
+/** Build a private staging database; access locks protect copies and publication of search.db. */
 export async function writeIndex(latDir, cacheDir, rebuild, work) {
     const dir = cacheDir ?? join(latDir, '.cache');
     await mkdir(dir, { recursive: true });
@@ -36,8 +36,19 @@ export async function writeIndex(latDir, cacheDir, rebuild, work) {
         // A crashed writer may have left a partial database and journal behind.
         await removeStaging(path);
         const existing = hasIndex(dir);
-        if (existing && !rebuild)
-            await copyFile(activePath, path);
+        if (existing && !rebuild) {
+            await withSearchAccess(dir, 'exclusive', async () => {
+                // Recover/checkpoint a WAL left by a process that died during a query.
+                const source = new SearchDb(activePath);
+                try {
+                    await source.checkpoint();
+                }
+                finally {
+                    await source.close();
+                }
+                await copyFile(activePath, path);
+            });
+        }
         db = new SearchDb(path);
         const model = existing && !rebuild ? await getStoredModel(db) : null;
         await ensureMeta(db);
@@ -52,7 +63,29 @@ export async function writeIndex(latDir, cacheDir, rebuild, work) {
         await db.close();
         db = undefined;
         if (!unchanged)
-            await withFileRetry(() => rename(path, activePath));
+            await withSearchAccess(dir, 'exclusive', async () => {
+                // Recover old sidecars before replacing their database, including after a crash.
+                if (existing) {
+                    const wal = await stat(activePath + '-wal').catch((error) => {
+                        if (error.code !== 'ENOENT')
+                            throw error;
+                        return undefined;
+                    });
+                    // A fresh rebuild must also repair an unreadable database with no WAL.
+                    if (wal && wal.size > 0) {
+                        const source = new SearchDb(activePath);
+                        try {
+                            await source.checkpoint();
+                        }
+                        finally {
+                            await source.close();
+                        }
+                    }
+                    for (const suffix of ['-wal', '-shm', '-journal'])
+                        await withFileRetry(() => rm(activePath + suffix, { force: true }));
+                }
+                await withFileRetry(() => rename(path, activePath));
+            });
         return result;
     }
     finally {

package/dist/src/search/chunks.d.ts

@@ -1,7 +1,7 @@
 import type { SourceSpan } from './types.js';
 import type { Embedder } from './embedder.js';
-import type { MarkdownFileAnalysis } from '../markdown-analysis.js';
-import type { Section } from '../lattice-model.js';
+import type { MarkdownFileAnalysis } from '@plebeiantech/lat.md-core/markdown-analysis';
+import type { Section } from '@plebeiantech/lat.md-core/lattice-model';
 export declare const CHUNK_POLICY = "owned-blocks-v1";
 export type Passage = {
     id: string;

package/dist/src/search/db.d.ts

@@ -4,11 +4,12 @@ export declare function hasIndex(cacheDir: string): boolean;
 /** Small SQL adapter; callers never depend on a libSQL connection. */
 export declare class SearchDb {
     readonly path: string;
-    private readonly snapshot;
+    private readonly published;
     private connection;
-    constructor(path: string, snapshot?: boolean);
-    private snapshotDir;
+    private release;
+    constructor(path: string, published?: boolean);
     private get;
+    private connect;
     execute(statement: string | {
         sql: string;
         args?: any[];
@@ -18,7 +19,7 @@ export declare class SearchDb {
     close(): Promise<void>;
     checkpoint(): Promise<void>;
 }
-export declare function openDb(latDir: string, requestedCacheDir?: string, readOnly?: boolean): SearchDb;
+export declare function openDb(latDir: string, requestedCacheDir?: string, published?: boolean): SearchDb;
 export declare function ensureMeta(db: SearchDb): Promise<void>;
 export declare function getStoredModel(db: SearchDb): Promise<string | null>;
 export declare function setStoredModel(db: SearchDb, value: string): Promise<void>;

package/dist/src/search/db.js

@@ -1,8 +1,7 @@
 import { connect } from '@tursodatabase/database';
-import { existsSync, mkdirSync, mkdtempSync, copyFileSync } from 'node:fs';
-import { rm } from 'node:fs/promises';
-import { tmpdir } from 'node:os';
-import { join } from 'node:path';
+import { existsSync, mkdirSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { acquireSearchAccess } from './lock.js';
 export const INDEX_FILE = 'search.db';
 export const CREATE_PASSAGE_FTS = "CREATE INDEX IF NOT EXISTS chunks_fts ON lexical_chunks USING fts(body,heading,path) WITH (tokenizer='whitespace',weights='body=1.0,heading=2.0,path=0.5')";
 export function hasIndex(cacheDir) {
@@ -11,26 +10,31 @@ export function hasIndex(cacheDir) {
 /** Small SQL adapter; callers never depend on a libSQL connection. */
 export class SearchDb {
     path;
-    snapshot;
+    published;
     connection;
-    constructor(path, snapshot = false) {
+    release;
+    constructor(path, published = false) {
         this.path = path;
-        this.snapshot = snapshot;
+        this.published = published;
     }
-    snapshotDir;
     get() {
-        if (this.connection)
-            return this.connection;
-        let path = this.path;
-        if (this.snapshot) {
-            this.snapshotDir = mkdtempSync(join(tmpdir(), 'lat-search-reader-'));
-            path = join(this.snapshotDir, 'index.db');
-            copyFileSync(this.path, path);
+        return (this.connection ??= this.connect());
+    }
+    async connect() {
+        // Turso 0.7.2 FTS needs writable connections and exclusive process access.
+        if (this.published)
+            this.release = await acquireSearchAccess(dirname(this.path), 'exclusive');
+        try {
+            return await connect(this.path, {
+                experimental: ['index_method'],
+                timeout: 10000,
+            });
+        }
+        catch (error) {
+            await this.release?.();
+            this.release = undefined;
+            throw error;
         }
-        return (this.connection = connect(path, {
-            experimental: ['index_method'],
-            timeout: 10000,
-        }));
     }
     async execute(statement) {
         const { sql, args = [] } = typeof statement === 'string' ? { sql: statement } : statement;
@@ -47,39 +51,31 @@ export class SearchDb {
         try {
             if (this.connection) {
                 const db = await this.connection;
-                await db.close();
-                this.connection = undefined;
-            }
-        }
-        finally {
-            if (this.snapshotDir) {
                 try {
-                    await rm(this.snapshotDir, {
-                        recursive: true,
-                        force: true,
-                        maxRetries: 3,
-                        retryDelay: 20,
-                    });
+                    // Leave a self-contained file for incremental copies and publication.
+                    if (this.published)
+                        await this.checkpoint();
                 }
-                catch (error) {
-                    const code = error.code;
-                    if (process.platform !== 'win32' ||
-                        !['EBUSY', 'EPERM', 'ENOTEMPTY'].includes(code ?? ''))
-                        throw error;
-                    // Native handles can outlive close on Windows; this is only an owned temp copy.
+                finally {
+                    await db.close();
                 }
-                this.snapshotDir = undefined;
             }
         }
+        finally {
+            this.connection = undefined;
+            const release = this.release;
+            this.release = undefined;
+            await release?.();
+        }
     }
     async checkpoint() {
         await this.execute('PRAGMA wal_checkpoint(TRUNCATE)');
     }
 }
-export function openDb(latDir, requestedCacheDir, readOnly = false) {
+export function openDb(latDir, requestedCacheDir, published = false) {
     const cacheDir = requestedCacheDir ?? join(latDir, '.cache');
     mkdirSync(cacheDir, { recursive: true });
-    return new SearchDb(join(cacheDir, INDEX_FILE), readOnly);
+    return new SearchDb(join(cacheDir, INDEX_FILE), published);
 }
 export async function ensureMeta(db) {
     await db.execute('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)');

package/dist/src/search/embedder.js

@@ -1,6 +1,6 @@
 import { createEmbedder, EmbeddingAuthError, } from '@lat.md/embed';
 import minilm from '@lat.md/embed-minilm-fp16';
-import { getLlmKey, getRepoEmbedding } from '../config.js';
+import { getLlmKey, getRepoEmbedding } from '@plebeiantech/lat.md-core/config';
 export { EmbeddingAuthError };
 const defaultCreateSearchEngine = (key) => key ? createEmbedder({ key }) : createEmbedder({ model: minilm });
 /** `meta.embedding_model` value for an embedder, e.g. `local:minilm-l6-v2:384`. */

package/dist/src/search/index.d.ts

@@ -1,5 +1,5 @@
 import { type SearchDb } from './db.js';
-import { type MarkdownProjectAnalysis } from '../project-analysis.js';
+import { type MarkdownProjectAnalysis } from '@plebeiantech/lat.md-core/project-analysis';
 import type { Embedder } from './embedder.js';
 export declare function projectFingerprint(project: MarkdownProjectAnalysis): string;
 export type IndexStats = {

package/dist/src/search/index.js

@@ -1,8 +1,8 @@
-import { PARSER_CACHE_VERSION } from '../parser-cache.js';
+import { PARSER_CACHE_VERSION } from '@plebeiantech/lat.md-core/parser-cache';
 import { synchronizeLexical } from './lexical.js';
 import { dirname } from 'node:path';
 import { CREATE_PASSAGE_FTS } from './db.js';
-import { analyzeMarkdownProject, } from '../project-analysis.js';
+import { analyzeMarkdownProject, } from '@plebeiantech/lat.md-core/project-analysis';
 import { chunkFile, digest, embeddingFingerprint } from './chunks.js';
 export function projectFingerprint(project) {
     return digest(`parser:${PARSER_CACHE_VERSION}\n` +

package/dist/src/search/lock.d.ts

@@ -1,2 +1,5 @@
-/** The persistent file must never be unlinked: all writers lock the same inode. */
+export type LockMode = 'shared' | 'exclusive';
 export declare function acquireSearchLock(cacheDir: string, timeoutMs?: number): Promise<() => Promise<void>>;
+/** A turnstile prevents new readers bypassing a writer waiting for readers to drain. */
+export declare function acquireSearchAccess(cacheDir: string, mode: LockMode, timeoutMs?: number): Promise<() => Promise<void>>;
+export declare function withSearchAccess<T>(cacheDir: string, mode: LockMode, work: () => Promise<T>): Promise<T>;

package/dist/src/search/lock.js

@@ -1,19 +1,17 @@
 import { open } from 'node:fs/promises';
 import { join } from 'node:path';
 import { setTimeout } from 'node:timers/promises';
-/** The persistent file must never be unlinked: all writers lock the same inode. */
-export async function acquireSearchLock(cacheDir, timeoutMs = 120_000) {
+/** Lock files stay in place so every process locks the same inode. */
+async function acquireFileLock(path, mode, deadline, message) {
     const { tryLock } = await import('fs-native-extensions');
-    const handle = await open(join(cacheDir, 'search-write.lock'), 'a+');
-    const deadline = performance.now() + timeoutMs;
+    const handle = await open(path, 'a+');
     try {
-        while (!tryLock(handle.fd)) {
+        while (!tryLock(handle.fd, { shared: mode === 'shared' })) {
             const remaining = deadline - performance.now();
             if (remaining <= 0)
-                throw new Error('Search index writer is busy; retry shortly.');
-            await setTimeout(Math.min(100, remaining));
+                throw new Error(message);
+            await setTimeout(Math.min(10, remaining));
         }
-        // Closing releases the kernel lock, including on abnormal process exit.
         return () => handle.close();
     }
     catch (error) {
@@ -21,3 +19,27 @@ export async function acquireSearchLock(cacheDir, timeoutMs = 120_000) {
         throw error;
     }
 }
+export function acquireSearchLock(cacheDir, timeoutMs = 120_000) {
+    return acquireFileLock(join(cacheDir, 'search-write.lock'), 'exclusive', performance.now() + timeoutMs, 'Search index writer is busy; retry shortly.');
+}
+/** A turnstile prevents new readers bypassing a writer waiting for readers to drain. */
+export async function acquireSearchAccess(cacheDir, mode, timeoutMs = 120_000) {
+    const deadline = performance.now() + timeoutMs;
+    const message = 'Search database is busy; retry shortly.';
+    const releaseGate = await acquireFileLock(join(cacheDir, 'search-access-gate.lock'), 'exclusive', deadline, message);
+    try {
+        return await acquireFileLock(join(cacheDir, 'search-access.lock'), mode, deadline, message);
+    }
+    finally {
+        await releaseGate();
+    }
+}
+export async function withSearchAccess(cacheDir, mode, work) {
+    const release = await acquireSearchAccess(cacheDir, mode);
+    try {
+        return await work();
+    }
+    finally {
+        await release();
+    }
+}

package/dist/src/search/query.d.ts

@@ -1,4 +1,5 @@
-import type { Section, SectionMatch } from '../lattice-model.js';
+import type { Section, SectionMatch } from '@plebeiantech/lat.md-core/lattice-model';
+import { type SearchDb } from './db.js';
 import { type CreateSearchEngine } from './embedder.js';
 import { type SearchResult } from './search.js';
 export type IndexedSearchResult = {
@@ -11,7 +12,7 @@ export type IndexedSearchSession = {
 };
 /** Resolve scored index rows against one already-analyzed project snapshot. */
 export declare function resolveSearchMatches(results: readonly SearchResult[], sectionById: ReadonlyMap<string, Section>): SectionMatch[];
-/** Open one reusable database and embedder for a sequence of index queries. */
+/** Reuse an embedder with scoped database access for a sequence of index queries. */
 export declare function openIndexedSearchSession(latDir: string, options?: {
     cacheDir?: string;
     createSearchEngine?: CreateSearchEngine;
@@ -21,3 +22,5 @@ export declare function searchIndexedSections(latDir: string, query: string, lim
     cacheDir?: string;
     minSimilarity?: number;
 }): Promise<IndexedSearchResult>;
+export declare function indexMetadata(db: SearchDb): Promise<Map<string, string>>;
+export declare function sameIndexMetadata(a: Map<string, string>, b: Map<string, string>): boolean;

package/dist/src/search/query.js

@@ -3,9 +3,9 @@ import { ReindexRequiredError } from './embedder.js';
 import { existsSync } from 'node:fs';
 import { join } from 'node:path';
 import { hasIndex } from './db.js';
-import { closeDb, getStoredModel, openDb } from './db.js';
+import { closeDb, openDb } from './db.js';
 import { embedderForIndex } from './embedder.js';
-import { searchSections } from './search.js';
+import { searchSections, prepareSearchQuery, } from './search.js';
 /** Resolve scored index rows against one already-analyzed project snapshot. */
 export function resolveSearchMatches(results, sectionById) {
     return results.flatMap((result) => {
@@ -27,48 +27,56 @@ export function resolveSearchMatches(results, sectionById) {
             : [];
     });
 }
-/** Open one reusable database and embedder for a sequence of index queries. */
+/** Reuse an embedder with scoped database access for a sequence of index queries. */
 export async function openIndexedSearchSession(latDir, options = {}) {
     const cacheDir = options.cacheDir ?? join(latDir, '.cache');
     if (!existsSync(cacheDir) || !hasIndex(cacheDir))
         return { search: async () => [], close: async () => { } };
-    const db = openDb(latDir, options.cacheDir, true);
-    let closed = false;
-    const close = async () => {
-        if (closed)
-            return;
-        closed = true;
-        await closeDb(db);
+    const readMetadata = async () => {
+        const db = openDb(latDir, options.cacheDir, true);
+        try {
+            return await indexMetadata(db);
+        }
+        finally {
+            await closeDb(db);
+        }
     };
-    try {
-        const stored = await getStoredModel(db);
-        if (stored === null) {
-            return {
-                search: async () => {
-                    if (closed)
-                        throw new Error('Search session is closed');
+    const initial = await readMetadata();
+    const stored = initial.get('embedding_model') ?? null;
+    if (stored && initial.get('lexical_version') !== LEXICAL_VERSION)
+        throw new ReindexRequiredError('Search lexical index changed; run lat search to update it.');
+    const embedder = stored
+        ? await embedderForIndex(stored, latDir, options.createSearchEngine)
+        : null;
+    let closed = false;
+    const active = new Set();
+    return {
+        search(query, limit, minSimilarity) {
+            if (closed)
+                return Promise.reject(new Error('Search session is closed'));
+            const operation = (async () => {
+                if (!embedder)
                     return [];
-                },
-                close,
-            };
-        }
-        const lexicalVersion = (await db.execute("SELECT value FROM meta WHERE key='lexical_version'")).rows[0]?.value;
-        if (lexicalVersion !== LEXICAL_VERSION)
-            throw new ReindexRequiredError('Search lexical index changed; run lat search to update it.');
-        const embedder = await embedderForIndex(stored, latDir, options.createSearchEngine);
-        return {
-            async search(query, limit, minSimilarity) {
-                if (closed)
-                    throw new Error('Search session is closed');
-                return searchSections(db, query, embedder, limit, minSimilarity);
-            },
-            close,
-        };
-    }
-    catch (error) {
-        await close();
-        throw error;
-    }
+                const vector = await prepareSearchQuery(query, embedder, limit, minSimilarity);
+                const db = openDb(latDir, options.cacheDir, true);
+                try {
+                    if (!sameIndexMetadata(initial, await indexMetadata(db)))
+                        throw new ReindexRequiredError('Search index changed; reopen the search session to refresh its documents and model.');
+                    return await searchSections(db, query, embedder, limit, minSimilarity, vector);
+                }
+                finally {
+                    await closeDb(db);
+                }
+            })();
+            active.add(operation);
+            void operation.finally(() => active.delete(operation)).catch(() => { });
+            return operation;
+        },
+        async close() {
+            closed = true;
+            await Promise.allSettled([...active]);
+        },
+    };
 }
 /** Query a finished index and resolve its ids from precomputed section data. */
 export async function searchIndexedSections(latDir, query, limit, sectionById, options = {}) {
@@ -84,3 +92,12 @@ export async function searchIndexedSections(latDir, query, limit, sectionById, o
         await session.close();
     }
 }
+export async function indexMetadata(db) {
+    return new Map((await db.execute('SELECT key,value FROM meta')).rows.map((row) => [
+        row.key,
+        row.value,
+    ]));
+}
+export function sameIndexMetadata(a, b) {
+    return (a.size === b.size && [...a].every(([key, value]) => b.get(key) === value));
+}

package/dist/src/search/search.d.ts

@@ -14,4 +14,6 @@ export declare function collapse(candidates: Candidate[]): Map<string, Candidate
     rank: number;
 }>;
 export declare function literalFtsQuery(query: string): string;
-export declare function searchSections(db: SearchDb, query: string, embedder: Embedder, limit?: number, minSimilarity?: number): Promise<SearchResult[]>;
+export declare function searchSections(db: SearchDb, query: string, embedder: Embedder, limit?: number, minSimilarity?: number, preparedVector?: number[]): Promise<SearchResult[]>;
+/** Prepare embeddings before acquiring a database access lock. */
+export declare function prepareSearchQuery(query: string, embedder: Embedder, limit?: number, minSimilarity?: number): Promise<number[]>;

package/dist/src/search/search.js

@@ -24,21 +24,12 @@ export function literalFtsQuery(query) {
         .map((t) => `"${t.replaceAll('"', '\\"')}"`)
         .join(' OR ');
 }
-export async function searchSections(db, query, embedder, limit = DEFAULT_SEARCH_LIMIT, minSimilarity = DEFAULT_MIN_SIMILARITY) {
+export async function searchSections(db, query, embedder, limit = DEFAULT_SEARCH_LIMIT, minSimilarity = DEFAULT_MIN_SIMILARITY, preparedVector) {
     query = query.trim();
     if (!query)
         return [];
-    if (!Number.isInteger(limit) || limit < 1)
-        throw new Error('limit must be a positive integer');
-    if (!Number.isFinite(minSimilarity) || minSimilarity < 0 || minSimilarity > 1)
-        throw new Error('min-similarity must be a number from 0 to 1');
-    if (embedder.countTokens(query) > embedder.maxInputTokens)
-        throw new Error('Search query exceeds the embedding model token limit; shorten the query.');
-    const [vector] = await embedder.embed([query]);
-    if (!vector ||
-        vector.length !== embedder.dimensions ||
-        vector.some((n) => !Number.isFinite(n)))
-        throw new Error('Embedding backend returned an invalid query vector');
+    const vector = preparedVector ??
+        (await prepareSearchQuery(query, embedder, limit, minSimilarity));
     const vectorJson = JSON.stringify(vector), fts = literalFtsQuery(query);
     const target = Math.max(50, limit), cap = 10 * target;
     const owners = new Map((await db.execute('SELECT id,section_id FROM chunks')).rows.map((r) => [
@@ -155,3 +146,21 @@ export async function searchSections(db, query, embedder, limit = DEFAULT_SEARCH
     }
     return results;
 }
+/** Prepare embeddings before acquiring a database access lock. */
+export async function prepareSearchQuery(query, embedder, limit = DEFAULT_SEARCH_LIMIT, minSimilarity = DEFAULT_MIN_SIMILARITY) {
+    query = query.trim();
+    if (!query)
+        return [];
+    if (!Number.isInteger(limit) || limit < 1)
+        throw new Error('limit must be a positive integer');
+    if (!Number.isFinite(minSimilarity) || minSimilarity < 0 || minSimilarity > 1)
+        throw new Error('min-similarity must be a number from 0 to 1');
+    if (embedder.countTokens(query) > embedder.maxInputTokens)
+        throw new Error('Search query exceeds the embedding model token limit; shorten the query.');
+    const [vector] = await embedder.embed([query]);
+    if (!vector ||
+        vector.length !== embedder.dimensions ||
+        vector.some((n) => !Number.isFinite(n)))
+        throw new Error('Embedding backend returned an invalid query vector');
+    return vector;
+}

package/dist/src/search/types.d.ts

@@ -1,21 +1,5 @@
-export type SourceSpan = {
-    start: number;
-    end: number;
-    startLine: number;
-    endLine: number;
-};
-export type SearchEvidence = {
-    chunkId: string;
-    text: string;
-    spans: SourceSpan[];
-    channel: 'lexical' | 'semantic';
-};
-export type SearchDiagnostics = {
-    lexicalCapped: boolean;
-    semanticCapped: boolean;
-    lexicalCandidates: number;
-    semanticCandidates: number;
-};
+import type { SearchEvidence, SearchDiagnostics } from '@plebeiantech/lat.md-core/search-metadata';
+export type { SourceSpan, SearchEvidence, SearchDiagnostics, } from '@plebeiantech/lat.md-core/search-metadata';
 export type SearchResult = {
     id: string;
     file: string;