@plebeiantech/lat.md 0.12.3-fork.16 → 0.12.3-fork.17

package/README.md

@@ -103,7 +103,7 @@ Alternatively, install `@plebeiantech/lat.md` via npm/mise, run `lat init` in yo
 
 ## How it works
 
-Run `lat init` to scaffold a `lat.md/` directory, then write markdown files describing your architecture, business logic, test specs — whatever matters. Link between sections using `[[file#Section#Subsection]]` syntax. Link to source code symbols with `[[src/auth.ts#validateToken]]`. Annotate source code with `// @lat: [[section-id]]` (or `# @lat: [[section-id]]` in Python) comments to tie implementation back to concepts.
+Run `lat init` to scaffold a `lat.md/` directory, then write markdown files describing your architecture, business logic, test specs — whatever matters. Link between sections using `[[file#Section#Subsection]]` syntax. Link to source code symbols with `[[src/auth.ts#validateToken]]`. Annotate source code with `// @lat: [[section-id]]` (or `# @lat: [[section-id]]` in Python and PHP) comments to tie implementation back to concepts.
 
 ```
 my-project/

package/dist/src/cli/index.js

@@ -8,7 +8,7 @@ import { dirname, join } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { Command, InvalidArgumentError } from 'commander';
 import { resolveCheckContext, resolveContext } from './context.js';
-import { DEFAULT_SEARCH_LIMIT, DEFAULT_SEARCH_THRESHOLD, } from '../search/search.js';
+import { DEFAULT_SEARCH_LIMIT, DEFAULT_MIN_SIMILARITY, } from '../search/search.js';
 function parsePort(value) {
     if (!/^\d+$/.test(value)) {
         throw new InvalidArgumentError('port must be an integer from 1 to 65535');
@@ -34,7 +34,7 @@ function configureUiRun(command) {
 function parseSimilarityThreshold(value) {
     const threshold = Number(value);
     if (!Number.isFinite(threshold) || threshold < 0 || threshold > 1) {
-        throw new InvalidArgumentError('threshold must be a number from 0 to 1');
+        throw new InvalidArgumentError('min-similarity must be a number from 0 to 1');
     }
     return threshold;
 }
@@ -339,11 +339,16 @@ program
 });
 program
     .command('search')
-    .description('Semantic search across lat.md sections')
+    .description('Hybrid search across lat.md sections')
     .argument('[query]', 'search query in plain English')
     .option('--limit <n>', `max results (default: ${DEFAULT_SEARCH_LIMIT})`, String(DEFAULT_SEARCH_LIMIT))
-    .option('--debug', 'show result similarity scores')
-    .option('--threshold <score>', `minimum cosine similarity score (default: ${DEFAULT_SEARCH_THRESHOLD})`, parseSimilarityThreshold)
+    .option('--preview <variant>', 'preview passage, intro, or both', (value) => {
+    if (!['passage', 'intro', 'both'].includes(value))
+        throw new InvalidArgumentError('preview must be passage, intro, or both');
+    return value;
+}, 'passage')
+    .option('--debug', 'show retrieval scores and candidate diagnostics')
+    .option('--min-similarity <score>', `minimum cosine similarity score (default: ${DEFAULT_MIN_SIMILARITY})`, parseSimilarityThreshold)
     .action(async (query, opts) => {
     const ctx = resolveContext(program.opts());
     const { searchCommand, cliProgress } = await import('./search.js');
@@ -351,7 +356,8 @@ program
     const result = await searchCommand(ctx, query, {
         limit: parseInt(opts.limit),
         debug: opts.debug,
-        threshold: opts.threshold,
+        minSimilarity: opts.minSimilarity,
+        preview: opts.preview,
     }, progress);
     handleResult(result);
 });

package/dist/src/cli/init.js

@@ -8,7 +8,7 @@ import { findTemplatesDir } from './templates.js';
 import { readAgentsTemplate, readCursorRulesTemplate, readPiExtensionTemplate, readOpenCodePluginTemplate, readSkillTemplate, } from './gen.js';
 import { getLlmKey, getRepoEmbedding, setRepoEmbedding } from '../config.js';
 import { makeStyler } from './context.js';
-import { closeDb, ensureMeta, getStoredModel, openDb } from '../search/db.js';
+import { closeDb, getStoredModel, openDb } from '../search/db.js';
 import { modelKey } from '../search/embedder.js';
 import { reindexCommand } from './reindex.js';
 import { INIT_VERSION, writeInitMeta, readInitVersion, readFileHash, contentHash, } from '../init-version.js';
@@ -668,11 +668,10 @@ async function setupCodex(root, latDir, hashes, ask, style) {
         ' Run /hooks in Codex to review and trust the project hooks.');
 }
 async function readStoredEmbeddingModel(latDir) {
-    if (!existsSync(join(latDir, '.cache', 'vectors.db')))
+    if (!existsSync(join(latDir, '.cache', 'search.db')))
         return null;
-    const db = openDb(latDir);
+    const db = openDb(latDir, undefined, true);
     try {
-        await ensureMeta(db);
         return await getStoredModel(db);
     }
     finally {

package/dist/src/cli/reindex.js

@@ -1,5 +1,6 @@
+import { writeIndex } from '../search/cache.js';
 import readline from 'node:readline/promises';
-import { openDb, ensureMeta, setStoredModel, ensureSectionsSchema, dropSections, closeDb, } from '../search/db.js';
+import { ensureMeta, setStoredModel, ensureSectionsSchema, dropSections, } from '../search/db.js';
 import { embedderFromEnv, localEmbedder, modelKey, EmbeddingAuthError, } from '../search/embedder.js';
 import { getLlmKey, getRepoEmbedding, setRepoEmbedding } from '../config.js';
 import { indexSections } from '../search/index.js';
@@ -83,11 +84,9 @@ export async function reindexCommand(ctx, opts) {
             embedder = await localEmbedder();
         }
     }
-    const db = openDb(ctx.latDir);
     const interactive = ctx.mode === 'cli' && !!process.stderr.isTTY;
-    try {
+    return writeIndex(ctx.latDir, undefined, true, async (db) => {
         await ensureMeta(db);
-        await dropSections(db);
         await ensureSectionsSchema(db, embedder.dimensions);
         const label = `Reindexing with ${embedder.name}`;
         // Interactive terminals get a live progress line driven per embed-chunk
@@ -130,8 +129,5 @@ export async function reindexCommand(ctx, opts) {
             output: s.green(`Reindexed ${stats.added} sections`) +
                 ` using ${s.cyan(embedder.name)}.`,
         };
-    }
-    finally {
-        await closeDb(db);
-    }
+    });
 }

package/dist/src/cli/search.d.ts

@@ -27,7 +27,7 @@ export declare function runSearch(latDir: string, query: string, limit: number,
     buildIndex?: boolean;
     project?: MarkdownProjectAnalysis;
     sectionById?: ReadonlyMap<string, Section>;
-    threshold?: number;
+    minSimilarity?: number;
     cacheDir?: string;
 }): Promise<SearchResult>;
 /**
@@ -41,5 +41,6 @@ export declare function cliProgress(s: Styler): IndexProgress;
 export declare function searchCommand(ctx: CmdContext, query: string | undefined, opts: {
     limit: number;
     debug?: boolean;
-    threshold?: number;
+    minSimilarity?: number;
+    preview?: 'passage' | 'intro' | 'both';
 }, progress?: IndexProgress): Promise<CmdResult>;

package/dist/src/cli/search.js

@@ -1,71 +1,55 @@
 import { readFile } from 'node:fs/promises';
+import { LEXICAL_VERSION } from '../search/lexical.js';
+import { hasIndex } from '../search/db.js';
+import { embeddingFingerprint } from '../search/chunks.js';
+import { writeIndex } from '../search/cache.js';
 import { dirname, join } from 'node:path';
-import { openDb, ensureMeta, getStoredModel, setStoredModel, ensureSectionsSchema, dropSections, closeDb, } from '../search/db.js';
-import { getLlmKey } from '../config.js';
+import { openDb, getStoredModel, setStoredModel, ensureSectionsSchema, closeDb, } from '../search/db.js';
 import { embedderForIndex, modelKey, ReindexRequiredError, EmbeddingAuthError, } from '../search/embedder.js';
-import { indexSections } from '../search/index.js';
+import { indexSections, projectFingerprint, } from '../search/index.js';
 import { searchSections } from '../search/search.js';
 import { resolveSearchMatches, searchIndexedSections, } from '../search/query.js';
 import { analyzeMarkdownProject, commandProjectAnalysis, } from '../project-analysis.js';
 import { formatSectionPreview, formatNavHints } from '../format.js';
 import { provenanceNote, formatProvenanceNote } from './check-status.js';
 async function withDb(latDir, progress, project, cacheDir, fn) {
-    const db = openDb(latDir, cacheDir);
-    try {
-        await ensureMeta(db);
-        const stored = await getStoredModel(db);
-        // The stored model is authoritative — no silent backend switch. Throws
-        // ReindexRequiredError if the environment can't serve the stored index.
-        // Rebuilding / switching backends is `lat reindex`, never `lat search`.
-        const embedder = await embedderForIndex(stored, latDir);
-        await ensureSectionsSchema(db, embedder.dimensions);
-        const countResult = await db.execute('SELECT COUNT(*) as n FROM sections');
-        let isEmpty = countResult.rows[0].n === 0;
-        // Legacy cache: a version before the model was recorded left rows behind
-        // with no `meta.embedding_model`. Those vectors may be a different backend
-        // (and dimension) than the resolved embedder, and `CREATE TABLE IF NOT
-        // EXISTS` won't migrate the column width — so drop and rebuild from scratch
-        // under the resolved backend rather than querying a mismatched table.
-        if (!stored && !isEmpty) {
-            await dropSections(db);
-            await ensureSectionsSchema(db, embedder.dimensions);
-            isEmpty = true;
-        }
-        // If the repo is pinned to local but a key is set, say so — otherwise it
-        // looks like the key is being silently ignored.
-        if (isEmpty && embedder.name.startsWith('local:') && process.stderr.isTTY) {
-            let hasKey = false;
-            try {
-                hasKey = !!getLlmKey();
-            }
-            catch {
-                /* key source misconfigured — irrelevant, we're local */
-            }
-            if (hasKey) {
-                process.stderr.write('This repo is configured for local embeddings; ignoring LAT_LLM_KEY.\n');
-            }
-        }
-        progress?.beforeIndex?.(isEmpty);
+    if (hasIndex(cacheDir ?? join(latDir, '.cache'))) {
+        const db = openDb(latDir, cacheDir, true);
         try {
-            const stats = await indexSections(latDir, db, embedder, undefined, project);
-            // Pin the backend only after a successful index, so a failed build never
-            // leaves the repo wrongly pinned to an empty index.
-            if (!stored)
-                await setStoredModel(db, modelKey(embedder));
-            progress?.afterIndex?.(stats, isEmpty);
+            const stored = await getStoredModel(db);
+            const embedder = await embedderForIndex(stored, latDir);
+            const metadata = new Map((await db.execute('SELECT key,value FROM meta')).rows.map((row) => [
+                row.key,
+                row.value,
+            ]));
+            if (metadata.get('fingerprint') !== embeddingFingerprint(embedder))
+                throw new ReindexRequiredError('Search chunking or embedding model changed; run lat reindex.');
+            if (stored &&
+                metadata.get('lexical_version') === LEXICAL_VERSION &&
+                metadata.get('project_hash') === projectFingerprint(project)) {
+                progress?.afterIndex?.({
+                    added: 0,
+                    updated: 0,
+                    removed: 0,
+                    unchanged: project.sections.length,
+                }, false);
+                return await fn(db, embedder, project);
+            }
         }
-        catch (err) {
-            // Failed fresh build → drop the half-created table so the next run is
-            // truly fresh (re-resolves the backend cleanly) rather than stuck.
-            if (!stored)
-                await dropSections(db);
-            throw err;
+        finally {
+            await closeDb(db);
         }
-        return await fn(db, embedder, project);
-    }
-    finally {
-        await closeDb(db);
     }
+    return writeIndex(latDir, cacheDir, false, async (db, stored) => {
+        const embedder = await embedderForIndex(stored, latDir);
+        await ensureSectionsSchema(db, embedder.dimensions);
+        const isEmpty = (await db.execute('SELECT COUNT(*) AS n FROM sections')).rows[0].n === 0;
+        progress?.beforeIndex?.(isEmpty);
+        const stats = await indexSections(latDir, db, embedder, undefined, project);
+        await setStoredModel(db, modelKey(embedder));
+        progress?.afterIndex?.(stats, isEmpty);
+        return fn(db, embedder, project);
+    });
 }
 /**
  * Run a semantic search across lat.md sections.
@@ -86,7 +70,7 @@ export async function runSearch(latDir, query, limit, progress, opts) {
             })).sectionById;
         return searchIndexedSections(latDir, query, limit, sectionById, {
             cacheDir: opts.cacheDir,
-            threshold: opts.threshold,
+            minSimilarity: opts.minSimilarity,
         });
     }
     const project = opts?.project ??
@@ -94,7 +78,7 @@ export async function runSearch(latDir, query, limit, progress, opts) {
             executor: 'auto',
         }));
     return withDb(latDir, progress, project, opts?.cacheDir, async (db, embedder, analyzed) => {
-        const results = await searchSections(db, query, embedder, limit, opts?.threshold);
+        const results = await searchSections(db, query, embedder, limit, opts?.minSimilarity);
         return {
             query,
             matches: resolveSearchMatches(results, analyzed.sectionById),
@@ -158,7 +142,32 @@ async function formatSearchMatches(ctx, query, matches, opts) {
             lines.push('');
         lines.push(formatSectionPreview(ctx, matches[i].section, {
             reason: matches[i].reason,
-            score: opts?.showScores ? matches[i].score : undefined,
+            spans: opts?.preview === 'intro'
+                ? undefined
+                : matches[i].evidence?.[0]?.spans,
+            score: opts?.showScores ? matches[i].rankScore : undefined,
+            text: opts?.preview === 'intro'
+                ? undefined
+                : matches[i].evidence?.length
+                    ? (opts?.preview === 'both'
+                        ? matches[i].section.firstParagraph + '\n\n'
+                        : '') + matches[i].evidence[0].text
+                    : undefined,
+            debug: opts?.showScores && matches[i].rankScore !== undefined
+                ? JSON.stringify({
+                    lexicalRank: matches[i].lexicalRank,
+                    semanticRank: matches[i].semanticRank,
+                    lexicalScore: matches[i].lexicalScore,
+                    semanticSimilarity: matches[i].semanticSimilarity,
+                    lexicalContribution: matches[i].lexicalRank
+                        ? 1 / (60 + matches[i].lexicalRank)
+                        : 0,
+                    semanticContribution: matches[i].semanticRank
+                        ? 1 / (60 + matches[i].semanticRank)
+                        : 0,
+                    ...matches[i].diagnostics,
+                })
+                : undefined,
         }));
         const note = notes.get(matches[i].section.filePath);
         if (note)
@@ -177,7 +186,7 @@ export async function searchCommand(ctx, query, opts, progress) {
         const project = await commandProjectAnalysis(ctx);
         const result = await runSearch(ctx.latDir, query, opts.limit, progress, {
             project,
-            threshold: opts.threshold,
+            minSimilarity: opts.minSimilarity,
         });
         if (result.matches.length === 0) {
             return { output: 'No results found.' };
@@ -185,6 +194,7 @@ export async function searchCommand(ctx, query, opts, progress) {
         return {
             output: (await formatSearchMatches(ctx, query, result.matches, {
                 showScores: opts.debug,
+                preview: opts.preview ?? 'passage',
             })) + formatNavHints(ctx),
         };
     }

package/dist/src/format.d.ts

@@ -1,11 +1,16 @@
+import type { SourceSpan } from './search/types.js';
 import type { Section, SectionMatch } from './lattice-model.js';
 import type { CmdContext, Styler } from './context.js';
 export declare function formatSectionId(id: string, s: Styler): string;
 export declare function formatSectionPreview(ctx: CmdContext, section: Section, opts?: {
     reason?: string;
     score?: number;
+    text?: string;
+    debug?: string;
+    spans?: SourceSpan[];
 }): string;
 export declare function formatResultList(ctx: CmdContext, header: string, matches: SectionMatch[], opts?: {
     showScores?: boolean;
+    preview?: 'passage' | 'intro' | 'both';
 }): string;
 export declare function formatNavHints(ctx: CmdContext): string;

package/dist/src/format.js

@@ -12,6 +12,7 @@ export function formatSectionPreview(ctx, section, opts) {
     const kind = section.id.includes('#') ? 'Section' : 'File';
     const details = [
         opts?.reason,
+        opts?.debug,
         opts?.score === undefined ? undefined : `score: ${opts.score.toFixed(6)}`,
     ].filter((detail) => detail !== undefined);
     const detailsSuffix = details.length > 0 ? ' ' + s.dim(`(${details.join(', ')})`) : '';
@@ -19,9 +20,11 @@ export function formatSectionPreview(ctx, section, opts) {
         `${s.dim('*')} ${s.dim(kind + ':')} [[${formatSectionId(section.id, s)}]]${detailsSuffix}`,
         `  ${s.dim('Defined in')} ${s.cyan(relPath)}${s.dim(`:${section.startLine}-${section.endLine}`)}`,
     ];
-    if (section.firstParagraph) {
-        lines.push('', `  ${s.dim('>')} ${section.firstParagraph}`);
-    }
+    if (opts?.spans?.length)
+        lines.push(`  ${s.dim('Matched lines:')} ${opts.spans.map((span) => `${span.startLine}-${span.endLine}`).join(', ')}`);
+    const text = opts?.text ?? section.firstParagraph;
+    if (text)
+        lines.push('', ...text.split('\n').map((line) => `  ${s.dim('>')} ${line}`));
     return lines.join('\n');
 }
 export function formatResultList(ctx, header, matches, opts) {
@@ -31,7 +34,32 @@ export function formatResultList(ctx, header, matches, opts) {
             lines.push('');
         lines.push(formatSectionPreview(ctx, matches[i].section, {
             reason: matches[i].reason,
-            score: opts?.showScores ? matches[i].score : undefined,
+            spans: opts?.preview === 'intro'
+                ? undefined
+                : matches[i].evidence?.[0]?.spans,
+            score: opts?.showScores ? matches[i].rankScore : undefined,
+            text: opts?.preview === 'intro'
+                ? undefined
+                : matches[i].evidence?.length
+                    ? (opts?.preview === 'both'
+                        ? matches[i].section.firstParagraph + '\n\n'
+                        : '') + matches[i].evidence[0].text
+                    : undefined,
+            debug: opts?.showScores && matches[i].rankScore !== undefined
+                ? JSON.stringify({
+                    lexicalRank: matches[i].lexicalRank,
+                    semanticRank: matches[i].semanticRank,
+                    lexicalScore: matches[i].lexicalScore,
+                    semanticSimilarity: matches[i].semanticSimilarity,
+                    lexicalContribution: matches[i].lexicalRank
+                        ? 1 / (60 + matches[i].lexicalRank)
+                        : 0,
+                    semanticContribution: matches[i].semanticRank
+                        ? 1 / (60 + matches[i].semanticRank)
+                        : 0,
+                    ...matches[i].diagnostics,
+                })
+                : undefined,
         }));
     }
     lines.push('');

package/dist/src/lattice-model.d.ts

@@ -1,3 +1,4 @@
+import type { SearchEvidence, SearchDiagnostics } from './search/types.js';
 import type { ForkFrontmatter } from './fork/frontmatter-fields.js';
 export type Section = {
     id: string;
@@ -72,6 +73,12 @@ export declare function resolveRef(target: string, sectionIds: Set<string>, file
 export type SectionMatch = {
     section: Section;
     reason: string;
-    score?: number;
+    rankScore?: number;
+    semanticSimilarity?: number;
+    lexicalScore?: number;
+    semanticRank?: number;
+    lexicalRank?: number;
+    evidence?: SearchEvidence[];
+    diagnostics?: SearchDiagnostics;
 };
 export declare function findSections(sections: Section[], query: string): SectionMatch[];

package/dist/src/lattice-model.js

@@ -94,8 +94,11 @@ export function buildSectionSlugIndex(sections) {
         const levels = [
             ...stack,
             {
+                id: section.id,
                 depth: section.depth,
-                literal: section.heading.toLowerCase(),
+                literal: section.id
+                    .slice((stack.at(-1)?.id ?? section.file).length + 1)
+                    .toLowerCase(),
                 slug: headingSlug.toLowerCase(),
             },
         ];

package/dist/src/lattice.d.ts

@@ -1,4 +1,4 @@
-import type { LatFrontmatter, MdLink, Ref, Section } from './lattice-model.js';
+import { type LatFrontmatter, type MdLink, type Ref, type Section } from './lattice-model.js';
 import type { Root } from 'mdast';
 import type { Profiler } from './profiler.js';
 export { buildFileIndex, buildSectionSlugIndex, findSections, flattenSections, resolveRef, } from './lattice-model.js';
@@ -9,6 +9,6 @@ export type { ForkFrontmatter, FrontmatterProblem, } from './fork/frontmatter-fi
 export declare function parseFrontmatter(content: string): LatFrontmatter;
 export declare function parseSections(filePath: string, content: string, projectRoot?: string, tree?: Root): Section[];
 export declare function loadAllSections(latticeDir: string, projectRoot?: string, profile?: Profiler): Promise<Section[]>;
-export declare function extractRefs(filePath: string, content: string, projectRoot?: string, tree?: Root): Ref[];
+export declare function extractRefs(filePath: string, content: string, projectRoot?: string, tree?: Root, sections?: Section[]): Ref[];
 /** Extract link destinations and undefined reference syntax. */
 export declare function extractLinks(content: string, tree?: Root): MdLink[];

package/dist/src/lattice.js

@@ -6,6 +6,7 @@ import { parse } from './parser.js';
 import { toPosix } from './path.js';
 import { visit } from 'unist-util-visit';
 import { listLatticeFiles } from './project-discovery.js';
+import { flattenSections, } from './lattice-model.js';
 import { LAT_FIELDS, } from './fork/frontmatter-fields.js';
 export { buildFileIndex, buildSectionSlugIndex, findSections, flattenSections, resolveRef, } from './lattice-model.js';
 export { findLatticeDir, findProjectRoot, listLatticeFiles, } from './project-discovery.js';
@@ -56,14 +57,8 @@ export function parseFrontmatter(content) {
         result.problems = problems;
     return result;
 }
-function headingText(node) {
-    return node.children
-        .filter((c) => c.type === 'text')
-        .map((c) => c.value)
-        .join('');
-}
 /** Extract the rendered text GitHub uses as input to its heading slugger. */
-function githubHeadingText(node) {
+function headingText(node) {
     if (!node || typeof node !== 'object')
         return '';
     const value = node;
@@ -77,7 +72,7 @@ function githubHeadingText(node) {
         return value.data?.alias ?? value.value ?? '';
     }
     if (value.children) {
-        return value.children.map(githubHeadingText).join('');
+        return value.children.map(headingText).join('');
     }
     return '';
 }
@@ -112,6 +107,8 @@ export function parseSections(filePath, content, projectRoot, tree = parse(conte
     const stack = [];
     const flat = [];
     const slugger = new GithubSlugger();
+    const usedIds = new Set();
+    const nextSuffix = new Map();
     visit(tree, 'heading', (node) => {
         const heading = headingText(node);
         const depth = node.depth;
@@ -121,7 +118,14 @@ export function parseSections(filePath, content, projectRoot, tree = parse(conte
             stack.pop();
         }
         const parent = stack.length > 0 ? stack[stack.length - 1] : null;
-        const id = parent ? `${parent.id}#${heading}` : `${file}#${heading}`;
+        const baseId = parent ? `${parent.id}#${heading}` : `${file}#${heading}`;
+        let id = baseId;
+        const key = baseId.toLowerCase();
+        let suffix = nextSuffix.get(key) ?? 1;
+        while (usedIds.has(id.toLowerCase()))
+            id = `${baseId}-${suffix++}`;
+        nextSuffix.set(key, suffix);
+        usedIds.add(id.toLowerCase());
         const section = {
             id,
             heading,
@@ -132,7 +136,7 @@ export function parseSections(filePath, content, projectRoot, tree = parse(conte
             startLine,
             endLine: 0,
             firstParagraph: '',
-            githubSlug: slugger.slug(githubHeadingText(node)),
+            githubSlug: slugger.slug(headingText(node)),
         };
         if (parent) {
             parent.children.push(section);
@@ -190,34 +194,13 @@ export async function loadAllSections(latticeDir, projectRoot = dirname(latticeD
     }
     return all;
 }
-export function extractRefs(filePath, content, projectRoot, tree = parse(content)) {
+export function extractRefs(filePath, content, projectRoot, tree = parse(content), sections = parseSections(filePath, content, projectRoot, tree)) {
     const file = projectRoot
         ? toPosix(relative(projectRoot, filePath)).replace(/\.md$/, '')
         : basename(filePath, '.md');
     const refs = [];
-    // Build a flat list of sections to determine enclosing section for each wiki link
-    const flat = [];
-    visit(tree, 'heading', (node) => {
-        flat.push({
-            id: '', // filled below
-            startLine: node.position.start.line,
-        });
-    });
-    // Re-derive ids using the same logic as parseSections
-    const stack = [];
-    let idx = 0;
-    visit(tree, 'heading', (node) => {
-        const heading = headingText(node);
-        const depth = node.depth;
-        while (stack.length > 0 && stack[stack.length - 1].depth >= depth) {
-            stack.pop();
-        }
-        const parent = stack.length > 0 ? stack[stack.length - 1] : null;
-        const id = parent ? `${parent.id}#${heading}` : `${file}#${heading}`;
-        flat[idx].id = id;
-        stack.push({ id, depth });
-        idx++;
-    });
+    // Reuse canonical identities, including duplicate suffixes and formatted text.
+    const flat = flattenSections(sections);
     visit(tree, 'wikiLink', (node) => {
         const line = node.position.start.line;
         // Find enclosing section: last heading with startLine <= link line

package/dist/src/markdown-analysis.d.ts

@@ -6,6 +6,16 @@ export type MarkdownParagraph = {
     endLine: number;
     text: string;
 };
+/** Source-positioned block facts; never retains the parser AST. */
+export type MarkdownBlock = {
+    type: string;
+    start: number;
+    end: number;
+    startLine: number;
+    endLine: number;
+    language?: string;
+    children: MarkdownBlock[];
+};
 export type MarkdownDestinationLink = {
     kind: 'image' | 'link';
     line: number;
@@ -37,6 +47,7 @@ export type MarkdownFileAnalysis = {
     headingTitles: string[];
     wikiRefs: Ref[];
     paragraphs: MarkdownParagraph[];
+    blocks: MarkdownBlock[];
     markdownLinks: MarkdownDestinationLink[];
     validationLinks: MdLink[];
     indexEntries: string[];

package/dist/src/markdown-analysis.js

@@ -5,6 +5,35 @@ import { extractLinks, extractRefs, parseFrontmatter, parseSections, } from './l
 import { parse } from './parser.js';
 import { toPosix } from './path.js';
 import { analyzeLocalMarkdownDiagnostics, } from './markdown-validation.js';
+function extractBlocks(tree) {
+    const block = (node) => {
+        if (node.position?.start.offset === undefined ||
+            node.position?.end.offset === undefined)
+            return;
+        return {
+            type: node.type,
+            start: node.position.start.offset,
+            end: node.position.end.offset,
+            startLine: node.position.start.line,
+            endLine: node.position.end.line,
+            ...('lang' in node && node.lang ? { language: node.lang } : {}),
+            children: [
+                'list',
+                'listItem',
+                'blockquote',
+                'table',
+                'tableRow',
+            ].includes(node.type)
+                ? ('children' in node ? node.children : [])
+                    .map(block)
+                    .filter((item) => !!item)
+                : [],
+        };
+    };
+    return tree.children
+        .map(block)
+        .filter((item) => !!item);
+}
 function elapsed(work) {
     const start = performance.now();
     return [work(), performance.now() - start];
@@ -96,7 +125,7 @@ function extractIndexEntries(content) {
 export function analyzeMarkdownFile(absolutePath, content, latDir, projectRoot) {
     const [tree, parseMs] = elapsed(() => parse(content));
     const [sections, sectionsMs] = elapsed(() => parseSections(absolutePath, content, projectRoot, tree));
-    const [wikiRefs, refsMs] = elapsed(() => extractRefs(absolutePath, content, projectRoot, tree));
+    const [wikiRefs, refsMs] = elapsed(() => extractRefs(absolutePath, content, projectRoot, tree, sections));
     const [validationLinks, linksMs] = elapsed(() => extractLinks(content, tree));
     const [[paragraphs, markdownLinks, headingTitles], paragraphsMs] = elapsed(() => [
         extractParagraphs(content, tree),
@@ -116,6 +145,7 @@ export function analyzeMarkdownFile(absolutePath, content, latDir, projectRoot)
         headingTitles,
         wikiRefs,
         paragraphs,
+        blocks: extractBlocks(tree),
         markdownLinks,
         validationLinks,
         indexEntries,

package/dist/src/mcp/server.js

@@ -7,7 +7,7 @@ import { plainStyler } from '../context.js';
 import { locateCommand } from '../cli/locate.js';
 import { sectionCommand } from '../cli/section.js';
 import { searchCommand } from '../cli/search.js';
-import { DEFAULT_SEARCH_LIMIT, DEFAULT_SEARCH_THRESHOLD, } from '../search/search.js';
+import { DEFAULT_SEARCH_LIMIT, DEFAULT_MIN_SIMILARITY, } from '../search/search.js';
 import { expandCommand } from '../cli/expand.js';
 import { checkAllCommand } from '../cli/check.js';
 import { refsCommand } from '../cli/refs.js';
@@ -40,21 +40,21 @@ export async function startMcpServer() {
             .string()
             .describe('Local section id or exact handle:path#fragment target'),
     }, async ({ query }) => toMcp(await sectionCommand(requestContext(), query)));
-    server.tool('lat_search', 'Semantic search across lat.md sections using embeddings', {
+    server.tool('lat_search', 'Hybrid lexical and semantic search across lat.md sections', {
         query: z.string().describe('Search query in natural language'),
         limit: z
             .number()
             .optional()
             .default(DEFAULT_SEARCH_LIMIT)
             .describe(`Max results (default ${DEFAULT_SEARCH_LIMIT})`),
-        threshold: z
+        minSimilarity: z
             .number()
             .min(0)
             .max(1)
             .optional()
-            .default(DEFAULT_SEARCH_THRESHOLD)
-            .describe('Minimum cosine similarity score (default 0.35)'),
-    }, async ({ query, limit, threshold }) => toMcp(await searchCommand(requestContext(), query, { limit, threshold })));
+            .default(DEFAULT_MIN_SIMILARITY)
+            .describe('Minimum cosine similarity score (default 0.20)'),
+    }, async ({ query, limit, minSimilarity }) => toMcp(await searchCommand(requestContext(), query, { limit, minSimilarity })));
     server.tool('lat_expand', 'Expand [[refs]] in text to resolved lat.md section paths with context', { text: z.string().describe('Text containing [[refs]] to expand') }, async ({ text: input }) => toMcp(await expandCommand(requestContext(), input)));
     server.tool('lat_check', 'Run full lat.md validation: links, code references, indexes, and section structure', {}, async () => toMcp(await checkAllCommand(requestContext())));
     server.tool('lat_refs', 'Find sections that reference a given section via wiki links or @lat code comments', {