lat.md 0.1.0 → 0.1.2

package/dist/src/search/index.js

@@ -0,0 +1,66 @@
+import { createHash } from 'node:crypto';
+import { readFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { loadAllSections, flattenSections } from '../lattice.js';
+import { embed } from './embeddings.js';
+function hashContent(text) {
+    return createHash('sha256').update(text).digest('hex');
+}
+async function sectionContent(section, latDir) {
+    const filePath = join(latDir, section.file + '.md');
+    const content = await readFile(filePath, 'utf-8');
+    const lines = content.split('\n');
+    return lines.slice(section.startLine - 1, section.endLine).join('\n');
+}
+export async function indexSections(latDir, db, provider, key) {
+    const allSections = await loadAllSections(latDir);
+    const flat = flattenSections(allSections);
+    // Build current state: id -> { section, content, hash }
+    const current = new Map();
+    for (const s of flat) {
+        const text = await sectionContent(s, latDir);
+        current.set(s.id, { section: s, content: text, hash: hashContent(text) });
+    }
+    // Get existing hashes from DB
+    const existing = new Map();
+    const rows = await db.execute('SELECT id, content_hash FROM sections');
+    for (const row of rows.rows) {
+        existing.set(row.id, row.content_hash);
+    }
+    // Partition into new, changed, unchanged, deleted
+    const toEmbed = [];
+    let unchanged = 0;
+    for (const [id, entry] of current) {
+        const existingHash = existing.get(id);
+        if (existingHash === entry.hash) {
+            unchanged++;
+        }
+        else {
+            toEmbed.push({ id, content: entry.content, section: entry.section });
+        }
+    }
+    const toDelete = [...existing.keys()].filter((id) => !current.has(id));
+    // Embed new/changed sections
+    if (toEmbed.length > 0) {
+        const texts = toEmbed.map((e) => e.content);
+        const vectors = await embed(texts, provider, key);
+        const now = Date.now();
+        for (let i = 0; i < toEmbed.length; i++) {
+            const { id, content, section } = toEmbed[i];
+            const hash = current.get(id).hash;
+            const vecJson = JSON.stringify(vectors[i]);
+            await db.execute({
+                sql: `INSERT OR REPLACE INTO sections (id, file, heading, content, content_hash, embedding, updated_at)
+              VALUES (?, ?, ?, ?, ?, vector(?), ?)`,
+                args: [id, section.file, section.heading, content, hash, vecJson, now],
+            });
+        }
+    }
+    // Delete removed sections
+    for (const id of toDelete) {
+        await db.execute({ sql: 'DELETE FROM sections WHERE id = ?', args: [id] });
+    }
+    const added = toEmbed.filter((e) => !existing.has(e.id)).length;
+    const updated = toEmbed.filter((e) => existing.has(e.id)).length;
+    return { added, updated, removed: toDelete.length, unchanged };
+}

package/dist/src/search/provider.d.ts

@@ -0,0 +1,8 @@
+export type EmbeddingProvider = {
+    name: string;
+    apiBase: string;
+    model: string;
+    dimensions: number;
+    headers: (key: string) => Record<string, string>;
+};
+export declare function detectProvider(key: string): EmbeddingProvider;

package/dist/src/search/provider.js

@@ -0,0 +1,40 @@
+const openai = {
+    name: 'openai',
+    apiBase: 'https://api.openai.com/v1',
+    model: 'text-embedding-3-small',
+    dimensions: 1536,
+    headers: (key) => ({
+        Authorization: `Bearer ${key}`,
+        'Content-Type': 'application/json',
+    }),
+};
+const vercel = {
+    name: 'vercel',
+    apiBase: 'https://ai-gateway.vercel.sh/v1',
+    model: 'openai/text-embedding-3-small',
+    dimensions: 1536,
+    headers: (key) => ({
+        Authorization: `Bearer ${key}`,
+        'Content-Type': 'application/json',
+    }),
+};
+export function detectProvider(key) {
+    if (key.startsWith('REPLAY_LAT_LLM_KEY::')) {
+        const replayUrl = key.slice('REPLAY_LAT_LLM_KEY::'.length);
+        return {
+            name: 'replay',
+            apiBase: replayUrl,
+            model: 'replay',
+            dimensions: 1536,
+            headers: () => ({ 'Content-Type': 'application/json' }),
+        };
+    }
+    if (key.startsWith('sk-ant-')) {
+        throw new Error("Anthropic doesn't offer an embedding model. Set LAT_LLM_KEY to an OpenAI (sk-...) or Vercel AI (vck_...) key.");
+    }
+    if (key.startsWith('vck_'))
+        return vercel;
+    if (key.startsWith('sk-'))
+        return openai;
+    throw new Error(`Unrecognized LAT_LLM_KEY prefix. Supported: OpenAI (sk-...), Vercel AI (vck_...).`);
+}

package/dist/src/search/search.d.ts

@@ -0,0 +1,9 @@
+import type { Client } from '@libsql/client';
+import type { EmbeddingProvider } from './provider.js';
+export type SearchResult = {
+    id: string;
+    file: string;
+    heading: string;
+    content: string;
+};
+export declare function searchSections(db: Client, query: string, provider: EmbeddingProvider, key: string, limit?: number): Promise<SearchResult[]>;

package/dist/src/search/search.js

@@ -0,0 +1,17 @@
+import { embed } from './embeddings.js';
+export async function searchSections(db, query, provider, key, limit = 5) {
+    const [queryVec] = await embed([query], provider, key);
+    const vecJson = JSON.stringify(queryVec);
+    const rows = await db.execute({
+        sql: `SELECT s.id, s.file, s.heading, s.content
+          FROM vector_top_k('sections_vec_idx', vector(?), ?) AS v
+          JOIN sections AS s ON s.rowid = v.id`,
+        args: [vecJson, limit],
+    });
+    return rows.rows.map((row) => ({
+        id: row.id,
+        file: row.file,
+        heading: row.heading,
+        content: row.content,
+    }));
+}

package/package.json

@@ -1,13 +1,31 @@
 {
   "name": "lat.md",
-  "version": "0.1.0",
-  "description": "Anchor source code to high-level concepts defined in markdown",
+  "version": "0.1.2",
+  "description": "A knowledge graph for your codebase, written in markdown",
   "type": "module",
+  "packageManager": "pnpm@10.30.2",
+  "license": "MIT",
+  "author": "Yury Selivanov",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/1st1/lat.md.git"
+  },
+  "homepage": "https://github.com/1st1/lat.md",
+  "keywords": [
+    "markdown",
+    "knowledge-graph",
+    "documentation",
+    "agents",
+    "cli",
+    "wiki-links",
+    "codebase"
+  ],
   "bin": {
     "lat": "./dist/src/cli/index.js"
   },
   "files": [
-    "dist/src"
+    "dist/src",
+    "templates"
   ],
   "scripts": {
     "build": "tsc",
@@ -15,7 +33,8 @@
     "test:watch": "vitest",
     "typecheck": "tsc --noEmit",
     "format": "prettier --write 'src/**/*.ts'",
-    "format:check": "prettier --check 'src/**/*.ts'"
+    "format:check": "prettier --check 'src/**/*.ts'",
+    "cook-test-rag": "tsx scripts/cook-test-rag.ts"
   },
   "devDependencies": {
     "@types/mdast": "^4.0.4",
@@ -23,10 +42,15 @@
     "mdast-util-from-markdown": "^2.0.3",
     "micromark-util-types": "^2.0.2",
     "prettier": "^3.8.1",
+    "tsx": "^4.21.0",
     "typescript": "^5.7.0",
     "vitest": "^3.0.0"
   },
   "dependencies": {
+    "@libsql/client": "^0.17.0",
+    "chalk": "^5.6.2",
+    "commander": "^14.0.3",
+    "ignore-walk": "^8.0.0",
     "mdast-util-to-markdown": "^2.1.0",
     "remark-parse": "^11.0.0",
     "remark-stringify": "^11.0.0",

package/templates/AGENTS.md

@@ -0,0 +1,56 @@
+# Before starting work
+
+- Run `lat search` to find sections relevant to your task. Read them to understand the design intent before writing code.
+- Run `lat prompt` on user prompts to expand any `[[refs]]` — this resolves section names to file locations and provides context.
+
+# Post-task checklist (REQUIRED — do not skip)
+
+After EVERY task, before responding to the user:
+
+- [ ] Update `lat.md/` if you added or changed any functionality, architecture, tests, or behavior
+- [ ] Run `lat check` — all wiki links and code refs must pass
+- [ ] Do not skip these steps. Do not consider your task done until both are complete.
+
+---
+
+# What is lat.md?
+
+This project uses [lat.md](https://www.npmjs.com/package/lat.md) to maintain a structured knowledge graph of its architecture, design decisions, and test specs in the `lat.md/` directory. It is a set of cross-linked markdown files that describe **what** this project does and **why** — the domain concepts, key design decisions, business logic, and test specifications. Use it to ground your work in the actual architecture rather than guessing.
+
+# Commands
+
+```bash
+lat locate "Section Name"      # find a section by name (exact, fuzzy)
+lat refs "file#Section"        # find what references a section
+lat search "natural language"  # semantic search across all sections
+lat prompt "user prompt text"  # expand [[refs]] to resolved locations
+lat check                      # validate all links and code refs
+```
+
+Run `lat --help` when in doubt about available commands or options.
+
+If `lat search` fails because `LAT_LLM_KEY` is not set, explain to the user that semantic search requires an API key (`export LAT_LLM_KEY=sk-...` for OpenAI or `export LAT_LLM_KEY=vck_...` for Vercel). If the user doesn't want to set it up, use `lat locate` for direct lookups instead.
+
+# Syntax primer
+
+- **Section ids**: `file-stem#Heading#SubHeading` (e.g. `cli#search#Indexing`)
+- **Wiki links**: `[[target]]` or `[[target|alias]]` — cross-references between sections
+- **Code refs**: `// @lat: [[section-id]]` (JS/TS) or `# @lat: [[section-id]]` (Python) — ties source code to concepts
+
+# Test specs
+
+Key tests can be described as sections in `lat.md/` files (e.g. `tests.md`). Add frontmatter to require that every leaf section is referenced by a `// @lat:` or `# @lat:` comment in test code:
+
+```markdown
+---
+lat:
+  require-code-mention: true
+---
+# Tests
+
+## User login
+### Rejects expired tokens
+### Handles missing password
+```
+
+Each test in code should reference its spec: `// @lat: [[tests#User login#Rejects expired tokens]]`. Running `lat check` will flag any spec section not covered by a code reference, and any code reference pointing to a nonexistent section.

package/templates/README

@@ -0,0 +1 @@
+This directory is used as source of templates for commands like `lat init`.

package/templates/init/README.md

@@ -0,0 +1,5 @@
+This directory defines the high-level concepts, business logic, and architecture of this project using markdown.
+
+It is managed by [lat.md](https://www.npmjs.com/package/lat.md) — a tool that anchors source code to these definitions.
+
+Install the `lat` command with `npm i -g lat.md` and run `lat --help`.