@plebeiantech/lat.md 0.12.3-fork.13 → 0.12.3-fork.14

package/dist/src/search/db.d.ts

@@ -1,5 +1,5 @@
 import { type Client } from '@libsql/client';
-export declare function openDb(latDir: string): Client;
+export declare function openDb(latDir: string, requestedCacheDir?: string): Client;
 /** Create the `meta` key/value table if absent. */
 export declare function ensureMeta(db: Client): Promise<void>;
 /**

package/dist/src/search/db.js

@@ -1,8 +1,8 @@
 import { createClient } from '@libsql/client';
 import { mkdirSync } from 'node:fs';
 import { join } from 'node:path';
-export function openDb(latDir) {
-    const cacheDir = join(latDir, '.cache');
+export function openDb(latDir, requestedCacheDir) {
+    const cacheDir = requestedCacheDir ?? join(latDir, '.cache');
     mkdirSync(cacheDir, { recursive: true });
     const client = createClient({
         url: `file:${join(cacheDir, 'vectors.db')}`,

package/dist/src/search/embedder.d.ts

@@ -1,6 +1,7 @@
 import { EmbeddingAuthError, type Embedder } from '@lat.md/embed';
 export type { Embedder };
 export { EmbeddingAuthError };
+export type CreateSearchEngine = (key?: string) => Promise<Embedder>;
 /** `meta.embedding_model` value for an embedder, e.g. `local:minilm-l6-v2:384`. */
 export declare function modelKey(embedder: Embedder): string;
 /** Thrown when the stored index can't be served by the current environment and
@@ -22,4 +23,4 @@ export declare function localEmbedder(): Promise<Embedder>;
  * regenerable `.cache` was wiped) the durable per-repo preference wins over the
  * env, so a repo switched to local stays local across cache loss.
  */
-export declare function embedderForIndex(storedModel: string | null, latDir: string): Promise<Embedder>;
+export declare function embedderForIndex(storedModel: string | null, latDir: string, createSearchEngine?: CreateSearchEngine): Promise<Embedder>;

package/dist/src/search/embedder.js

@@ -2,6 +2,7 @@ import { createEmbedder, EmbeddingAuthError, } from '@lat.md/embed';
 import minilm from '@lat.md/embed-minilm-fp16';
 import { getLlmKey, getRepoEmbedding } from '../config.js';
 export { EmbeddingAuthError };
+const defaultCreateSearchEngine = (key) => key ? createEmbedder({ key }) : createEmbedder({ model: minilm });
 /** `meta.embedding_model` value for an embedder, e.g. `local:minilm-l6-v2:384`. */
 export function modelKey(embedder) {
     return `${embedder.name}:${embedder.dimensions}`;
@@ -33,15 +34,15 @@ export function localEmbedder() {
  * regenerable `.cache` was wiped) the durable per-repo preference wins over the
  * env, so a repo switched to local stays local across cache loss.
  */
-export async function embedderForIndex(storedModel, latDir) {
+export async function embedderForIndex(storedModel, latDir, createSearchEngine = defaultCreateSearchEngine) {
     if (storedModel === null) {
         if (getRepoEmbedding(latDir) === 'local')
-            return localEmbedder();
+            return createSearchEngine();
         // No durable preference — decide from the environment, record it later.
-        return embedderFromEnv();
+        return createSearchEngine(getLlmKey());
     }
     if (storedModel.startsWith('local:')) {
-        return localEmbedder(); // ignores LAT_LLM_KEY entirely
+        return createSearchEngine(); // ignores LAT_LLM_KEY entirely
     }
     // Remote-pinned index: needs a key that resolves to the same model.
     const key = getLlmKey();
@@ -49,7 +50,7 @@ export async function embedderForIndex(storedModel, latDir) {
         throw new ReindexRequiredError(`This index was built with '${storedModel}', but no LAT_LLM_KEY is set. ` +
             `Run 'lat reindex' to switch to the local model or restore the key.`);
     }
-    const embedder = await createEmbedder({ key });
+    const embedder = await createSearchEngine(key);
     if (modelKey(embedder) !== storedModel) {
         throw new ReindexRequiredError(`This index was built with '${storedModel}', but the current key resolves ` +
             `to '${modelKey(embedder)}'. Run 'lat reindex' to rebuild.`);

package/dist/src/search/index.d.ts

@@ -1,4 +1,5 @@
 import type { Client } from '@libsql/client';
+import { type MarkdownProjectAnalysis } from '../project-analysis.js';
 import type { Embedder } from './embedder.js';
 export type IndexStats = {
     added: number;
@@ -6,4 +7,4 @@ export type IndexStats = {
     removed: number;
     unchanged: number;
 };
-export declare function indexSections(latDir: string, db: Client, embedder: Embedder, onProgress?: (done: number, total: number) => void): Promise<IndexStats>;
+export declare function indexSections(latDir: string, db: Client, embedder: Embedder, onProgress?: (done: number, total: number) => void, analyzedProject?: MarkdownProjectAnalysis): Promise<IndexStats>;

package/dist/src/search/index.js

@@ -1,7 +1,6 @@
 import { createHash } from 'node:crypto';
-import { readFile } from 'node:fs/promises';
 import { dirname, join } from 'node:path';
-import { loadAllSections, flattenSections } from '../lattice.js';
+import { analyzeMarkdownProject, } from '../project-analysis.js';
 function hashContent(text) {
     return createHash('sha256').update(text).digest('hex');
 }
@@ -11,12 +10,14 @@ function hashContent(text) {
  * O(sections × file size) — a 3.5 MB file holding 12k sections was re-read 12k
  * times (~42 GB of string work) on every search.
  */
-async function loadFileLines(sections, projectRoot) {
+function loadFileLines(sections, projectRoot, project) {
     const lines = new Map();
     for (const { filePath } of sections) {
         if (lines.has(filePath))
             continue;
-        const content = await readFile(join(projectRoot, filePath), 'utf-8');
+        const content = project.filesByAbsolutePath.get(join(projectRoot, filePath))?.content;
+        if (content === undefined)
+            throw new Error(`Analyzed Markdown file not found: ${filePath}`);
         lines.set(filePath, content.split('\n'));
     }
     return lines;
@@ -27,13 +28,14 @@ function sectionContent(section, lines) {
         .slice(section.startLine - 1, section.endLine)
         .join('\n');
 }
-export async function indexSections(latDir, db, embedder, onProgress) {
+export async function indexSections(latDir, db, embedder, onProgress, analyzedProject) {
     const projectRoot = dirname(latDir);
-    const allSections = await loadAllSections(latDir);
-    const flat = flattenSections(allSections);
+    const project = analyzedProject ??
+        (await analyzeMarkdownProject(latDir, projectRoot, { executor: 'auto' }));
+    const flat = project.sections;
     // Build current state: id -> { section, content, hash }
     const current = new Map();
-    const lines = await loadFileLines(flat, projectRoot);
+    const lines = loadFileLines(flat, projectRoot, project);
     for (const s of flat) {
         const text = sectionContent(s, lines);
         current.set(s.id, { section: s, content: text, hash: hashContent(text) });

package/dist/src/search/query.d.ts

@@ -0,0 +1,20 @@
+import type { Section, SectionMatch } from '../lattice-model.js';
+import { type CreateSearchEngine } from './embedder.js';
+export type IndexedSearchResult = {
+    query: string;
+    matches: SectionMatch[];
+};
+export type IndexedSearchSession = {
+    search: (query: string, limit: number, threshold?: number) => Promise<IndexedSearchResult>;
+    close: () => Promise<void>;
+};
+/** Open one reusable database and embedder for a sequence of index queries. */
+export declare function openIndexedSearchSession(latDir: string, sectionById: ReadonlyMap<string, Section>, options?: {
+    cacheDir?: string;
+    createSearchEngine?: CreateSearchEngine;
+}): Promise<IndexedSearchSession>;
+/** Query a finished index and resolve its ids from precomputed section data. */
+export declare function searchIndexedSections(latDir: string, query: string, limit: number, sectionById: ReadonlyMap<string, Section>, options?: {
+    cacheDir?: string;
+    threshold?: number;
+}): Promise<IndexedSearchResult>;

package/dist/src/search/query.js

@@ -0,0 +1,67 @@
+import { closeDb, ensureMeta, ensureSectionsSchema, getStoredModel, openDb, } from './db.js';
+import { embedderForIndex } from './embedder.js';
+import { searchSections } from './search.js';
+/** Open one reusable database and embedder for a sequence of index queries. */
+export async function openIndexedSearchSession(latDir, sectionById, options = {}) {
+    const db = openDb(latDir, options.cacheDir);
+    let closed = false;
+    const close = async () => {
+        if (closed)
+            return;
+        closed = true;
+        await closeDb(db);
+    };
+    try {
+        await ensureMeta(db);
+        const stored = await getStoredModel(db);
+        if (stored === null) {
+            return {
+                search: async (query) => {
+                    if (closed)
+                        throw new Error('Search session is closed');
+                    return { query, matches: [] };
+                },
+                close,
+            };
+        }
+        const embedder = await embedderForIndex(stored, latDir, options.createSearchEngine);
+        await ensureSectionsSchema(db, embedder.dimensions);
+        return {
+            async search(query, limit, threshold) {
+                if (closed)
+                    throw new Error('Search session is closed');
+                const results = await searchSections(db, query, embedder, limit, threshold);
+                return {
+                    query,
+                    matches: results.flatMap((result) => {
+                        const section = sectionById.get(result.id.toLowerCase());
+                        return section
+                            ? [
+                                {
+                                    section,
+                                    reason: 'semantic match',
+                                    score: result.score,
+                                },
+                            ]
+                            : [];
+                    }),
+                };
+            },
+            close,
+        };
+    }
+    catch (error) {
+        await close();
+        throw error;
+    }
+}
+/** Query a finished index and resolve its ids from precomputed section data. */
+export async function searchIndexedSections(latDir, query, limit, sectionById, options = {}) {
+    const session = await openIndexedSearchSession(latDir, sectionById, options);
+    try {
+        return await session.search(query, limit, options.threshold);
+    }
+    finally {
+        await session.close();
+    }
+}

package/dist/src/search/search.d.ts

@@ -1,5 +1,7 @@
 import type { Client } from '@libsql/client';
 import type { Embedder } from './embedder.js';
+export declare const DEFAULT_SEARCH_LIMIT = 5;
+export declare const DEFAULT_SEARCH_THRESHOLD = 0.35;
 export type SearchResult = {
     id: string;
     file: string;
@@ -7,4 +9,4 @@ export type SearchResult = {
     content: string;
     score: number;
 };
-export declare function searchSections(db: Client, query: string, embedder: Embedder, limit?: number): Promise<SearchResult[]>;
+export declare function searchSections(db: Client, query: string, embedder: Embedder, limit?: number, threshold?: number): Promise<SearchResult[]>;

package/dist/src/search/search.js

@@ -1,4 +1,6 @@
-export async function searchSections(db, query, embedder, limit = 5) {
+export const DEFAULT_SEARCH_LIMIT = 5;
+export const DEFAULT_SEARCH_THRESHOLD = 0.35;
+export async function searchSections(db, query, embedder, limit = DEFAULT_SEARCH_LIMIT, threshold = DEFAULT_SEARCH_THRESHOLD) {
     const [queryVec] = await embedder.embed([query]);
     const vecJson = JSON.stringify(queryVec);
     const rows = await db.execute({
@@ -9,7 +11,7 @@ export async function searchSections(db, query, embedder, limit = 5) {
           ORDER BY score DESC`,
         args: [vecJson, vecJson, limit],
     });
-    return rows.rows.map((row) => {
+    const results = rows.rows.map((row) => {
         const score = Number(row.score);
         return {
             id: row.id,
@@ -19,4 +21,5 @@ export async function searchSections(db, query, embedder, limit = 5) {
             score: Number.isFinite(score) ? score : 0,
         };
     });
+    return results.filter((result) => result.score >= threshold);
 }

package/dist/src/source-formats.d.ts

@@ -0,0 +1,7 @@
+/** Source file extensions supported by Lat's parser and code-reference scan. */
+export declare const SOURCE_FILE_EXTENSIONS: readonly [".c", ".dart", ".go", ".h", ".java", ".js", ".jsx", ".py", ".rs", ".ts", ".tsx"];
+/** A source extension supported everywhere Lat accepts source files. */
+export type SourceFileExtension = (typeof SOURCE_FILE_EXTENSIONS)[number];
+export declare function isSourceFileExtension(extension: string): extension is SourceFileExtension;
+export declare function sourceFileExtension(path: string): SourceFileExtension | null;
+export declare function isSourceFilePath(path: string): boolean;

package/dist/src/source-formats.js

@@ -0,0 +1,26 @@
+import { extname } from 'node:path';
+/** Source file extensions supported by Lat's parser and code-reference scan. */
+export const SOURCE_FILE_EXTENSIONS = [
+    '.c',
+    '.dart',
+    '.go',
+    '.h',
+    '.java',
+    '.js',
+    '.jsx',
+    '.py',
+    '.rs',
+    '.ts',
+    '.tsx',
+];
+const sourceFileExtensionSet = new Set(SOURCE_FILE_EXTENSIONS);
+export function isSourceFileExtension(extension) {
+    return sourceFileExtensionSet.has(extension);
+}
+export function sourceFileExtension(path) {
+    const extension = extname(path);
+    return isSourceFileExtension(extension) ? extension : null;
+}
+export function isSourceFilePath(path) {
+    return sourceFileExtension(path) !== null;
+}

package/dist/src/source-parser.d.ts

@@ -1,4 +1,4 @@
-import type { Section } from './lattice.js';
+import type { Section } from './lattice-model.js';
 export type SourceSymbol = {
     name: string;
     kind: 'function' | 'class' | 'const' | 'type' | 'interface' | 'method' | 'variable';
@@ -7,16 +7,55 @@ export type SourceSymbol = {
     endLine: number;
     signature: string;
 };
-/** All source file extensions that lat can parse (derived from grammarMap). */
-export declare const SOURCE_EXTENSIONS: ReadonlySet<string>;
+export type SourceAnalysisCacheStatus = 'disabled' | 'hit' | 'miss';
+export type SourceAnalysisTimings = {
+    readMs: number;
+    hashMs: number;
+    cacheReadMs: number;
+    cacheWriteMs: number;
+    parseMs: number;
+    cacheStatus: SourceAnalysisCacheStatus;
+};
+export type SourceFileAnalysis = {
+    path: string;
+    symbols: SourceSymbol[];
+    timings: SourceAnalysisTimings;
+};
+export type AnalyzeSourceSymbolsOptions = {
+    identity?: string;
+    cache?: boolean;
+    readMs?: number;
+    runtime?: SourceParserRuntime;
+};
+export type ResolveSourceSymbolOptions = {
+    latDir?: string;
+    cache?: boolean;
+    onFileAnalyzed?: (analysis: SourceFileAnalysis) => void;
+    runtime?: SourceParserRuntime;
+};
 export declare function parseSourceSymbols(filePath: string, content: string): Promise<SourceSymbol[]>;
+type ResolvedSourceCacheEntry = {
+    symbols: SourceSymbol[];
+    analysis?: SourceFileAnalysis;
+    error?: string;
+};
+/** Request-scoped owner for in-flight and completed source parser work. */
+export declare class SourceParserRuntime {
+    readonly analyses: Map<string, Promise<SourceFileAnalysis>>;
+    readonly symbols: Map<string, Promise<ResolvedSourceCacheEntry>>;
+    clear(): void;
+}
+/** Return the parser cache path for one project source file. */
+export declare function sourceAnalysisCachePath(latDir: string, projectRoot: string, absolutePath: string): string;
+/** Analyze source content through the shared versioned persistent cache. */
+export declare function analyzeSourceSymbols(filePath: string, content: string, latDir: string, options?: AnalyzeSourceSymbolsOptions): Promise<SourceFileAnalysis>;
 /** Clear the symbol cache. Call between top-level operations. */
 export declare function clearSymbolCache(): void;
 /**
  * Check whether a source file path (relative to projectRoot) has a given symbol.
  * Used by lat check to validate source code wiki links lazily.
  */
-export declare function resolveSourceSymbol(filePath: string, symbolPath: string, projectRoot: string): Promise<{
+export declare function resolveSourceSymbol(filePath: string, symbolPath: string, projectRoot: string, options?: ResolveSourceSymbolOptions): Promise<{
     found: boolean;
     symbols: SourceSymbol[];
     error?: string;
@@ -25,3 +64,4 @@ export declare function resolveSourceSymbol(filePath: string, symbolPath: string
  * Convert source symbols to Section objects for uniform handling.
  */
 export declare function sourceSymbolsToSections(symbols: SourceSymbol[], filePath: string): Section[];
+export {};