lattice-graph 0.1.0 → 0.3.0

package/README.md

@@ -1,13 +1,11 @@
 <table align="center"><tr><td>
 <pre>
-
 ██╗      █████╗ ████████╗████████╗██╗ ██████╗███████╗
 ██║     ██╔══██╗╚══██╔══╝╚══██╔══╝██║██╔════╝██╔════╝
 ██║     ███████║   ██║      ██║   ██║██║     █████╗
 ██║     ██╔══██║   ██║      ██║   ██║██║     ██╔══╝
 ███████╗██║  ██║   ██║      ██║   ██║╚██████╗███████╗
 ╚══════╝╚═╝  ╚═╝   ╚═╝      ╚═╝   ╚═╝ ╚═════╝╚══════╝
-
 </pre>
 </td></tr></table>
 
@@ -16,6 +14,7 @@
 </p>
 
 <p align="center">
+  <a href="https://www.npmjs.com/package/lattice-graph"><img src="https://img.shields.io/npm/v/lattice-graph" alt="npm"></a>
   <a href="https://bun.sh"><img src="https://img.shields.io/badge/runtime-Bun-f472b6" alt="Bun"></a>
   <a href="LICENSE"><img src="https://img.shields.io/badge/license-MIT-blue" alt="License"></a>
 </p>
@@ -71,7 +70,7 @@ bunx lattice-graph build
 ### From source
 
 ```bash
-git clone https://github.com/your-org/lattice.git
+git clone https://github.com/kilupskalvis/lattice.git
 cd lattice
 bun install
 bun run build          # compiles to ./lattice binary
@@ -80,18 +79,26 @@ bun run build          # compiles to ./lattice binary
 ## Quick Start
 
 ```bash
-# Initialize Lattice in your project
+# 1. Initialize Lattice in your project
 cd your-project
 lattice init                        # creates .lattice/ and lattice.toml
+# Edit lattice.toml to set the correct root, frameworks, etc.
 
-# Build the knowledge graph
-lattice build                       # parses all files, builds SQLite graph
+# 2. Build the structural graph
+lattice build                       # parses all files into a knowledge graph
 
-# Query the graph
+# 3. Tag your codebase (give this output to your coding agent)
+lattice populate                    # outputs tag spec, examples, and workflow
+# The agent reads the instructions, adds @lattice: tags to your source code,
+# then validates with:
+lattice build && lattice lint       # rebuild and check for errors
+
+# 4. Now the agent can navigate the codebase through the graph
 lattice overview                    # project landscape: flows, boundaries, events
 lattice flow checkout               # full call tree from entry point to boundaries
 lattice context charge              # callers, callees, flows, boundary info
 lattice impact charge               # what breaks if you change this function
+lattice code charge                 # read just the function source to edit
 ```
 
 ## Tags
@@ -132,6 +139,48 @@ FLOW: checkout
 
 `create_order`, `charge`, `save_order` — none of these need tags. Their flow membership is derived.
 
+### Async job dispatch patterns
+
+For systems that use message queues (SQS, RabbitMQ), Lambda invocations, or Celery tasks, use `emits`/`handles` to connect the submission side to the processing side:
+
+```python
+# @lattice:flow ingest-vacancies
+@router.post("/vacancies")
+def create_vacancies(request, service):
+    job = service.submit_ingest(request.data)
+    return {"job_id": job.id}
+
+# @lattice:emits job.ingest-vacancies
+def submit_ingest(self, data):
+    self.queue.submit(data)            # submits to SQS
+
+# @lattice:handles job.ingest-vacancies
+def process_ingest_vacancies(data, services):
+    normalized = services.normalize(data)    # calls OpenAI
+    services.writer.index(normalized)        # writes to Weaviate
+```
+
+With these tags, `lattice flow ingest-vacancies` shows the complete chain from the API route through the queue to the worker, including normalization and indexing. Without the `emits`/`handles` tags, the flow tree stops at the queue submission.
+
+**Key rule:** Worker handlers, Lambda consumers, and background job processors are NOT separate flows — they are the receiving side of an async dispatch. Tag them with `@lattice:handles`, not `@lattice:flow`.
+
+### Tags and protocols/interfaces
+
+When using dependency injection or protocol-based dispatch, place `emits` tags on the function that the flow actually passes through — not on a concrete implementation behind an interface:
+
+```python
+# Good: tag is on the function the flow reaches
+# @lattice:emits job.sync
+def submit_sync(self):
+    self._invoker.invoke_sync()
+
+# Bad: tag on concrete class that flow doesn't reach directly
+class LambdaInvoker:
+    # @lattice:emits job.sync  ← flow resolves to protocol, not this
+    def invoke_sync(self):
+        lambda_client.invoke(...)
+```
+
 ## Commands
 
 ### Build Commands
@@ -262,13 +311,12 @@ The linter detects:
 
 ## How It Works
 
-1. **Tree-sitter** parses source files into ASTs (Python and TypeScript supported)
-2. **Extractors** walk the AST to extract symbols (functions, classes, methods), call edges, imports, and framework patterns
-3. **Tag parser** reads `@lattice:` comments and associates them with the function below
+1. **LSP servers** (`typescript-language-server` for TypeScript, `zubanls` for Python) provide type-checked symbol and call information
+2. **Combined edge strategies** — both `outgoingCalls` and `references` are used to maximize edge coverage across files
+3. **Tag scanner** reads `@lattice:` comments and associates them with the function below
 4. **Graph builder** inserts everything into a SQLite database with nodes, edges, and tags
-5. **Event synthesis** creates invisible edges from `@lattice:emits` to `@lattice:handles` nodes
-6. **Cross-file resolution** matches callee names to known symbols across the codebase
-7. **CLI queries** traverse the graph and return compact, scoped results
+5. **Event synthesis** creates invisible edges from `@lattice:emits` to `@lattice:handles` nodes, connecting async flows
+6. **CLI queries** traverse the graph and return compact, scoped results
 
 The graph is stored at `.lattice/graph.db` — a single SQLite file.
 
@@ -285,29 +333,24 @@ exclude = ["node_modules", "venv", ".git", "dist", "__pycache__"]
 [python]
 source_roots = ["src"]
 test_paths = ["tests"]
-frameworks = ["fastapi"]
 
 [typescript]
 source_roots = ["src"]
 test_paths = ["__tests__"]
-frameworks = ["express"]
 
 [lint]
 strict = false
 ignore = ["scripts/**"]
-
-[lint.boundaries]
-packages = ["stripe", "boto3", "psycopg2", "requests", "sendgrid"]
 ```
 
 ## Supported Languages
 
-| Language | Status | Frameworks |
-|----------|--------|------------|
-| Python | Supported | FastAPI, Flask, Django, Celery |
-| TypeScript | Supported | Express, NestJS, Next.js |
+| Language | LSP Server | Status |
+|----------|-----------|--------|
+| Python | [zuban](https://github.com/zubanls/zuban) | Supported |
+| TypeScript | typescript-language-server | Supported |
 
-Adding a new language requires implementing one extractor. The graph schema, CLI commands, linter, and output formatting are all language-agnostic.
+Adding a new language requires adding its LSP server to the defaults. The graph schema, CLI commands, linter, and output formatting are all language-agnostic.
 
 ## Agent Integration (Claude Code)
 
@@ -370,14 +413,15 @@ The hooks remind the agent to try Lattice before falling back to traditional too
 ## Requirements
 
 - [Bun](https://bun.sh) >= 1.0 (for development and compilation)
-- No runtime dependencies for the compiled binary (except WASM grammars in `node_modules/`)
+- For TypeScript: `npm install -g typescript-language-server typescript`
+- For Python: `pip install zuban`
 
 ## Development
 
 ```bash
 bun install                         # install dependencies + git hooks
 bun run dev                         # run CLI in development mode
-bun run test                        # run 182 tests
+bun run test                        # run 187 tests
 bun run lint                        # biome check
 bun run typecheck                   # tsc --noEmit
 bun run check                       # lint + typecheck + tests

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "lattice-graph",
-	"version": "0.1.0",
+	"version": "0.3.0",
 	"description": "Knowledge graph CLI for coding agents — navigate code through flows, not grep.",
 	"module": "src/main.ts",
 	"type": "module",
@@ -9,10 +9,12 @@
 	},
 	"files": [
 		"src",
+		"scripts",
 		"README.md",
 		"LICENSE"
 	],
 	"scripts": {
+		"postinstall": "bun scripts/postinstall.ts",
 		"dev": "bun src/main.ts",
 		"test": "bun test",
 		"lint": "bunx biome check src/ tests/",
@@ -27,14 +29,14 @@
 		"knowledge-graph",
 		"coding-agent",
 		"code-navigation",
-		"tree-sitter",
-		"ast",
+		"lsp",
+		"language-server",
 		"static-analysis"
 	],
 	"license": "MIT",
 	"repository": {
 		"type": "git",
-		"url": "https://github.com/your-org/lattice"
+		"url": "https://github.com/kilupskalvis/lattice"
 	},
 	"engines": {
 		"bun": ">=1.0.0"
@@ -44,13 +46,10 @@
 		"@types/bun": "latest",
 		"lefthook": "^2.1.4"
 	},
-	"peerDependencies": {
-		"typescript": "^5"
-	},
 	"dependencies": {
 		"commander": "^14.0.3",
 		"smol-toml": "^1.6.0",
-		"tree-sitter-wasms": "0.1.11",
-		"web-tree-sitter": "0.24.7"
+		"typescript": "^5",
+		"typescript-language-server": "^4"
 	}
 }

package/scripts/postinstall.ts

@@ -0,0 +1,69 @@
+#!/usr/bin/env bun
+/**
+ * Postinstall script — creates a minimal Python venv and installs zuban into it.
+ * This gives zuban a proper Python environment with typeshed stubs.
+ */
+import { existsSync } from "node:fs";
+import { join } from "node:path";
+
+const VENDOR_DIR = join(import.meta.dir, "..", "vendor");
+const VENV_DIR = join(VENDOR_DIR, "venv");
+
+function findPython(): string | undefined {
+	for (const cmd of ["python3", "python"]) {
+		if (Bun.which(cmd)) return cmd;
+	}
+	return undefined;
+}
+
+async function main() {
+	const isWindows = process.platform === "win32";
+	const zubanBin = join(VENV_DIR, isWindows ? "Scripts" : "bin", isWindows ? "zubanls.exe" : "zubanls");
+
+	if (existsSync(zubanBin)) {
+		console.log("zuban already installed");
+		return;
+	}
+
+	const python = findPython();
+	if (!python) {
+		console.warn("Warning: Python not found. Python support requires python3 in PATH.");
+		return;
+	}
+
+	console.log("Installing zuban for Python support...");
+
+	try {
+		// Create venv
+		const venvResult = Bun.spawnSync([python, "-m", "venv", VENV_DIR], {
+			stdout: "ignore",
+			stderr: "pipe",
+		});
+		if (venvResult.exitCode !== 0) {
+			throw new Error(`Failed to create venv: ${venvResult.stderr.toString()}`);
+		}
+
+		// Install zuban via pip
+		const pip = join(VENV_DIR, isWindows ? "Scripts" : "bin", "pip");
+		const pipResult = Bun.spawnSync([pip, "install", "zuban", "--quiet"], {
+			stdout: "ignore",
+			stderr: "pipe",
+		});
+		if (pipResult.exitCode !== 0) {
+			throw new Error(`Failed to install zuban: ${pipResult.stderr.toString()}`);
+		}
+
+		if (!existsSync(zubanBin)) {
+			throw new Error("zubanls binary not found after installation");
+		}
+
+		console.log("zuban installed successfully");
+	} catch (error) {
+		console.warn(
+			`Warning: Failed to install zuban: ${error instanceof Error ? error.message : error}. ` +
+				"Python support requires: pip install zuban",
+		);
+	}
+}
+
+main();

package/src/commands/build.ts

@@ -1,39 +1,22 @@
-import type { Database } from "bun:sqlite";
 import { mkdirSync } from "node:fs";
-import { join, relative } from "node:path";
-import type { Extractor } from "../extract/extractor.ts";
-import { initTreeSitter } from "../extract/parser.ts";
-import { createPythonExtractor } from "../extract/python/extractor.ts";
-import { createTypeScriptExtractor } from "../extract/typescript/extractor.ts";
+import { join } from "node:path";
 import { createDatabase } from "../graph/database.ts";
 import {
-	insertEdges,
-	insertNodes,
-	insertTags,
-	insertUnresolved,
-	synthesizeEventEdges,
-} from "../graph/writer.ts";
+	type BuildStats,
+	buildGraph,
+	buildLanguageConfig,
+	type LanguageConfig,
+} from "../lsp/builder.ts";
 import type { LatticeConfig } from "../types/config.ts";
-import type { ExtractionResult } from "../types/graph.ts";
 import { err, ok, type Result } from "../types/result.ts";
 
-/** Statistics from a completed build. */
-type BuildStats = {
-	readonly fileCount: number;
-	readonly nodeCount: number;
-	readonly edgeCount: number;
-	readonly tagCount: number;
-	readonly eventEdgeCount: number;
-	readonly durationMs: number;
-};
-
 /**
- * Executes a full build of the knowledge graph.
- * Walks the source tree, runs extractors on all matching files,
- * inserts results into SQLite, and synthesizes event edges.
+ * Performs a full graph build using LSP extraction.
+ * Clears existing data, spawns a language server per configured language,
+ * extracts symbols and call hierarchy, scans for tags, and writes everything to SQLite.
  *
- * @param projectRoot - Absolute or relative path to the project root
- * @param config - Parsed Lattice configuration
+ * @param projectRoot - Absolute path to the project root
+ * @param config - Parsed lattice.toml configuration
  * @returns Build statistics or an error message
  */
 // @lattice:flow build
@@ -41,168 +24,56 @@ async function executeBuild(
 	projectRoot: string,
 	config: LatticeConfig,
 ): Promise<Result<BuildStats, string>> {
-	const startTime = Date.now();
-
 	try {
-		// Initialize tree-sitter and create extractors
-		await initTreeSitter();
-		const extractors = await createExtractors(config);
-		if (extractors.length === 0) {
-			return err("No extractors available for configured languages");
-		}
-
-		// Build extension → extractor mapping
-		const extByExt = new Map<string, Extractor>();
-		for (const ext of extractors) {
-			for (const fileExt of ext.fileExtensions) {
-				extByExt.set(fileExt, ext);
-			}
-		}
-
-		// Create .lattice directory and database
 		const latticeDir = join(projectRoot, ".lattice");
 		mkdirSync(latticeDir, { recursive: true });
 		const dbPath = join(latticeDir, "graph.db");
 		const db = createDatabase(dbPath);
 
-		// Walk the source tree
-		const sourceRoot = join(projectRoot, config.root);
-		const glob = new Bun.Glob("**/*");
-		const files: string[] = [];
-
-		for await (const path of glob.scan({ cwd: sourceRoot, dot: false })) {
-			const ext = `.${path.split(".").pop()}`;
-			if (!extByExt.has(ext)) continue;
-			if (isExcluded(path, config.exclude)) continue;
-			files.push(path);
-		}
-
-		// Extract all files
-		let totalNodes = 0;
-		let totalEdges = 0;
-		let totalTags = 0;
+		db.run("DELETE FROM external_calls");
+		db.run("DELETE FROM tags");
+		db.run("DELETE FROM edges");
+		db.run("DELETE FROM nodes");
 
-		for (const file of files) {
-			const ext = `.${file.split(".").pop()}`;
-			const extractor = extByExt.get(ext);
-			if (!extractor) continue;
+		const languageConfigs = buildLanguageConfigs(config);
 
-			const fullPath = join(sourceRoot, file);
-			const source = await Bun.file(fullPath).text();
-			const relativePath = relative(projectRoot, fullPath);
-
-			const result: ExtractionResult = await extractor.extract(relativePath, source);
-
-			insertNodes(db, result.nodes);
-			insertEdges(db, result.edges);
-			insertTags(db, result.tags);
-			insertUnresolved(db, result.unresolved);
-
-			totalNodes += result.nodes.length;
-			totalEdges += result.edges.length;
-			totalTags += result.tags.length;
-		}
-
-		// Cross-file resolution: resolve uncertain edges by matching callee names to known nodes
-		resolveCrossFileEdges(db);
-
-		// Synthesize event edges
-		synthesizeEventEdges(db);
-		const eventEdgeRow = db.query("SELECT COUNT(*) as c FROM edges WHERE kind = 'event'").get() as {
-			c: number;
-		};
+		const stats = await buildGraph({
+			projectRoot,
+			db,
+			languageConfigs,
+			exclude: config.exclude,
+		});
 
-		// Write build metadata
-		const now = new Date().toISOString();
-		db.run("INSERT OR REPLACE INTO meta (key, value) VALUES ('last_build', ?)", [now]);
-		db.run("INSERT OR REPLACE INTO meta (key, value) VALUES ('lattice_version', '0.1.0')");
+		db.run("INSERT OR REPLACE INTO meta (key, value) VALUES ('last_build', ?)", [
+			String(Date.now()),
+		]);
 
 		db.close();
-
-		return ok({
-			fileCount: files.length,
-			nodeCount: totalNodes,
-			edgeCount: totalEdges + eventEdgeRow.c,
-			tagCount: totalTags,
-			eventEdgeCount: eventEdgeRow.c,
-			durationMs: Date.now() - startTime,
-		});
-	} catch (error) {
-		return err(`Build failed: ${error instanceof Error ? error.message : String(error)}`);
-	}
-}
-
-/** Creates extractors for all configured languages. */
-async function createExtractors(config: LatticeConfig): Promise<readonly Extractor[]> {
-	const extractors: Extractor[] = [];
-	for (const lang of config.languages) {
-		if (lang === "python") {
-			extractors.push(await createPythonExtractor());
-		}
-		if (lang === "typescript") {
-			extractors.push(await createTypeScriptExtractor());
-		}
+		return ok(stats);
+	} catch (e) {
+		return err(e instanceof Error ? e.message : String(e));
 	}
-	return extractors;
 }
 
-/**
- * Resolves uncertain cross-file edges by matching callee names to known nodes.
- * If a callee name (e.g., "create_order") matches exactly one node name in the graph,
- * the edge target is updated to the full node ID.
- */
-function resolveCrossFileEdges(db: Database): void {
-	// Find all uncertain edges where target_id is not a known node
-	const uncertainEdges = db
-		.query(
-			`SELECT e.rowid, e.source_id, e.target_id FROM edges e
-			WHERE e.certainty = 'uncertain'
-			AND NOT EXISTS (SELECT 1 FROM nodes WHERE id = e.target_id)`,
-		)
-		.all() as { rowid: number; source_id: string; target_id: string }[];
+/** Builds LanguageConfig entries from LatticeConfig for each configured language. */
+function buildLanguageConfigs(config: LatticeConfig): readonly LanguageConfig[] {
+	const configs: LanguageConfig[] = [];
 
-	// Build a name→id map for all nodes (only keep unambiguous names)
-	const allNodes = db.query("SELECT id, name FROM nodes").all() as { id: string; name: string }[];
-	const nameToIds = new Map<string, string[]>();
-	for (const node of allNodes) {
-		const existing = nameToIds.get(node.name);
-		if (existing) {
-			existing.push(node.id);
-		} else {
-			nameToIds.set(node.name, [node.id]);
-		}
+	if (config.languages.includes("typescript") && config.typescript) {
+		configs.push(
+			buildLanguageConfig("typescript", config.typescript.sourceRoots, config.typescript.testPaths),
+		);
+	} else if (config.languages.includes("typescript")) {
+		configs.push(buildLanguageConfig("typescript", [config.root], []));
 	}
 
-	const deleteStmt = db.prepare("DELETE FROM edges WHERE rowid = ?");
-	const insertStmt = db.prepare(
-		"INSERT OR IGNORE INTO edges (source_id, target_id, kind, certainty) VALUES (?, ?, 'calls', 'certain')",
-	);
-
-	const tx = db.transaction(() => {
-		for (const edge of uncertainEdges) {
-			// Try to resolve the callee name
-			const calleeName = edge.target_id.split(".").pop() ?? edge.target_id;
-			const candidates = nameToIds.get(calleeName);
-
-			if (candidates && candidates.length === 1 && candidates[0]) {
-				// Unambiguous match — replace the uncertain edge
-				deleteStmt.run(edge.rowid);
-				insertStmt.run(edge.source_id, candidates[0]);
-			}
-			// If ambiguous or not found, leave the uncertain edge as-is
-		}
-	});
-	tx();
-}
-
-/** Checks if a file path matches any exclude pattern. */
-function isExcluded(filePath: string, excludePatterns: readonly string[]): boolean {
-	for (const pattern of excludePatterns) {
-		if (filePath.includes(pattern.replace("**", "").replace("*", ""))) {
-			return true;
-		}
+	if (config.languages.includes("python") && config.python) {
+		configs.push(buildLanguageConfig("python", config.python.sourceRoots, config.python.testPaths));
+	} else if (config.languages.includes("python")) {
+		configs.push(buildLanguageConfig("python", [config.root], ["tests"]));
 	}
-	return false;
+
+	return configs;
 }
 
-export { type BuildStats, executeBuild };
+export { executeBuild };

package/src/commands/init.ts

@@ -1,5 +1,5 @@
 import { existsSync, mkdirSync, writeFileSync } from "node:fs";
-import { join } from "node:path";
+import { join, resolve } from "node:path";
 import { err, ok, type Result } from "../types/result.ts";
 
 /**
@@ -16,16 +16,22 @@ function executeInit(projectRoot: string): Result<string, string> {
 		const latticeDir = join(projectRoot, ".lattice");
 		mkdirSync(latticeDir, { recursive: true });
 
+		// Detect languages
+		const languages = detectLanguages(projectRoot);
+
 		// Generate lattice.toml if it doesn't exist
 		const tomlPath = join(projectRoot, "lattice.toml");
 		if (!existsSync(tomlPath)) {
-			const languages = detectLanguages(projectRoot);
 			const root = detectRoot(projectRoot);
 			const toml = generateToml(languages, root);
 			writeFileSync(tomlPath, toml);
 		}
 
-		return ok("Initialized Lattice project");
+		// Check LSP server availability
+		const warnings = checkLspAvailability(languages);
+		const message = ["Initialized Lattice project", ...warnings].join("\n");
+
+		return ok(message);
 	} catch (error) {
 		return err(`Init failed: ${error instanceof Error ? error.message : String(error)}`);
 	}
@@ -84,23 +90,11 @@ function generateToml(languages: readonly string[], root: string): string {
 	];
 
 	if (languages.includes("python")) {
-		lines.push(
-			"[python]",
-			`source_roots = ["${root}"]`,
-			'test_paths = ["tests"]',
-			"frameworks = []",
-			"",
-		);
+		lines.push("[python]", `source_roots = ["${root}"]`, 'test_paths = ["tests"]', "");
 	}
 
 	if (languages.includes("typescript")) {
-		lines.push(
-			"[typescript]",
-			`source_roots = ["${root}"]`,
-			'test_paths = ["tests"]',
-			"frameworks = []",
-			"",
-		);
+		lines.push("[typescript]", `source_roots = ["${root}"]`, 'test_paths = ["tests"]', "");
 	}
 
 	lines.push("[lint]", "strict = false", "ignore = []", "");
@@ -108,4 +102,32 @@ function generateToml(languages: readonly string[], root: string): string {
 	return lines.join("\n");
 }
 
+/** Checks if language server binaries are available (bundled or in PATH). */
+function checkLspAvailability(languages: readonly string[]): readonly string[] {
+	const warnings: string[] = [];
+	const latticeRoot = resolve(import.meta.dir, "..", "..");
+
+	const checks: Record<string, { bundled: string; system: string; install: string }> = {
+		typescript: {
+			bundled: join(latticeRoot, "node_modules", ".bin", "typescript-language-server"),
+			system: "typescript-language-server",
+			install: "reinstall lattice-graph",
+		},
+		python: {
+			bundled: join(latticeRoot, "vendor", "venv", "bin", "zubanls"),
+			system: "zubanls",
+			install: "reinstall lattice-graph (or pip install zuban)",
+		},
+	};
+
+	for (const lang of languages) {
+		const check = checks[lang];
+		if (!check) continue;
+		if (!existsSync(check.bundled) && !Bun.which(check.system)) {
+			warnings.push(`Warning: ${check.system} not found. ${check.install}`);
+		}
+	}
+	return warnings;
+}
+
 export { executeInit };