lattice-graph 0.1.0 → 0.3.0

package/src/files.ts

@@ -0,0 +1,56 @@
+import { readdirSync, statSync } from "node:fs";
+import { join } from "node:path";
+
+/**
+ * Checks if a file or directory name matches any exclusion pattern.
+ *
+ * @param name - File or directory name to check
+ * @param excludePatterns - Patterns to exclude
+ * @returns True if the name matches any pattern
+ */
+function isExcluded(name: string, excludePatterns: readonly string[]): boolean {
+	return excludePatterns.some((pattern) => name.includes(pattern));
+}
+
+/**
+ * Recursively discovers files matching given extensions, excluding directories by pattern.
+ *
+ * @param root - Root directory to search
+ * @param extensions - File extensions to include (e.g., [".ts", ".tsx"])
+ * @param exclude - Directory name patterns to skip
+ * @returns Array of absolute file paths
+ */
+function discoverFiles(
+	root: string,
+	extensions: readonly string[],
+	exclude: readonly string[],
+): readonly string[] {
+	const files: string[] = [];
+
+	function walk(dir: string): void {
+		let names: string[];
+		try {
+			names = readdirSync(dir) as string[];
+		} catch {
+			return;
+		}
+		for (const name of names) {
+			const fullPath = join(dir, name);
+			try {
+				const stat = statSync(fullPath);
+				if (stat.isDirectory()) {
+					if (!isExcluded(name, exclude)) walk(fullPath);
+				} else if (extensions.some((ext) => name.endsWith(ext))) {
+					files.push(fullPath);
+				}
+			} catch {
+				// skip inaccessible files
+			}
+		}
+	}
+
+	walk(root);
+	return files;
+}
+
+export { discoverFiles, isExcluded };

package/src/graph/database.ts

@@ -1,7 +1,7 @@
 import { Database } from "bun:sqlite";
 import { err, ok, type Result } from "../types/result.ts";
 
-const SCHEMA_VERSION = "1";
+const SCHEMA_VERSION = "2";
 
 const SCHEMA_SQL = `
 CREATE TABLE IF NOT EXISTS nodes (
@@ -21,7 +21,6 @@ CREATE TABLE IF NOT EXISTS edges (
 	source_id   TEXT NOT NULL,
 	target_id   TEXT NOT NULL,
 	kind        TEXT NOT NULL,
-	certainty   TEXT DEFAULT 'certain',
 	PRIMARY KEY (source_id, target_id, kind)
 );
 
@@ -32,12 +31,11 @@ CREATE TABLE IF NOT EXISTS tags (
 	PRIMARY KEY (node_id, kind, value)
 );
 
-CREATE TABLE IF NOT EXISTS unresolved (
-	file        TEXT NOT NULL,
-	line        INTEGER NOT NULL,
-	expression  TEXT NOT NULL,
-	reason      TEXT NOT NULL,
-	PRIMARY KEY (file, line, expression)
+CREATE TABLE IF NOT EXISTS external_calls (
+	node_id     TEXT NOT NULL,
+	package     TEXT NOT NULL,
+	symbol      TEXT NOT NULL,
+	PRIMARY KEY (node_id, package, symbol)
 );
 
 CREATE TABLE IF NOT EXISTS meta (

package/src/graph/writer.ts

@@ -1,5 +1,5 @@
 import type { Database } from "bun:sqlite";
-import type { Edge, Node, Tag, UnresolvedReference } from "../types/graph.ts";
+import type { Edge, ExternalCall, Node, Tag } from "../types/graph.ts";
 
 /**
  * Inserts nodes into the graph database.
@@ -40,11 +40,11 @@ function insertNodes(db: Database, nodes: readonly Node[]): void {
  */
 function insertEdges(db: Database, edges: readonly Edge[]): void {
 	const stmt = db.prepare(
-		"INSERT OR IGNORE INTO edges (source_id, target_id, kind, certainty) VALUES (?, ?, ?, ?)",
+		"INSERT OR IGNORE INTO edges (source_id, target_id, kind) VALUES (?, ?, ?)",
 	);
 	const tx = db.transaction(() => {
 		for (const edge of edges) {
-			stmt.run(edge.sourceId, edge.targetId, edge.kind, edge.certainty);
+			stmt.run(edge.sourceId, edge.targetId, edge.kind);
 		}
 	});
 	tx();
@@ -68,26 +68,26 @@ function insertTags(db: Database, tags: readonly Tag[]): void {
 }
 
 /**
- * Inserts unresolved references into the database.
+ * Inserts external call records for lint boundary detection.
  * Uses INSERT OR IGNORE to skip duplicates.
  *
  * @param db - An open Database handle
- * @param refs - Unresolved references to insert
+ * @param calls - External calls to insert
  */
-function insertUnresolved(db: Database, refs: readonly UnresolvedReference[]): void {
+function insertExternalCalls(db: Database, calls: readonly ExternalCall[]): void {
 	const stmt = db.prepare(
-		"INSERT OR IGNORE INTO unresolved (file, line, expression, reason) VALUES (?, ?, ?, ?)",
+		"INSERT OR IGNORE INTO external_calls (node_id, package, symbol) VALUES (?, ?, ?)",
 	);
 	const tx = db.transaction(() => {
-		for (const ref of refs) {
-			stmt.run(ref.file, ref.line, ref.expression, ref.reason);
+		for (const call of calls) {
+			stmt.run(call.nodeId, call.package, call.symbol);
 		}
 	});
 	tx();
 }
 
 /**
- * Deletes all nodes, edges, tags, and unresolved references for a given file.
+ * Deletes all nodes, edges, tags, and external calls for a given file.
  * Edges where the file's nodes are either source or target are removed.
  *
  * @param db - An open Database handle
@@ -95,23 +95,19 @@ function insertUnresolved(db: Database, refs: readonly UnresolvedReference[]): v
  */
 function deleteFileData(db: Database, file: string): void {
 	const tx = db.transaction(() => {
-		// Get node IDs for this file
-		const nodeIds = db.query("SELECT id FROM nodes WHERE file = ?").all(file) as { id: string }[];
+		const nodeIds = db.query("SELECT id FROM nodes WHERE file = ?").all(file) as {
+			id: string;
+		}[];
 		const ids = nodeIds.map((n) => n.id);
 
 		if (ids.length > 0) {
 			const placeholders = ids.map(() => "?").join(",");
-			// Delete tags for these nodes
 			db.run(`DELETE FROM tags WHERE node_id IN (${placeholders})`, ids);
-			// Delete edges where these nodes are source or target
 			db.run(`DELETE FROM edges WHERE source_id IN (${placeholders})`, ids);
 			db.run(`DELETE FROM edges WHERE target_id IN (${placeholders})`, ids);
-			// Delete the nodes themselves
+			db.run(`DELETE FROM external_calls WHERE node_id IN (${placeholders})`, ids);
 			db.run(`DELETE FROM nodes WHERE id IN (${placeholders})`, ids);
 		}
-
-		// Delete unresolved references for this file
-		db.run("DELETE FROM unresolved WHERE file = ?", [file]);
 	});
 	tx();
 }
@@ -127,8 +123,8 @@ function synthesizeEventEdges(db: Database): void {
 	const tx = db.transaction(() => {
 		db.run("DELETE FROM edges WHERE kind = 'event'");
 		db.run(`
-			INSERT OR IGNORE INTO edges (source_id, target_id, kind, certainty)
-			SELECT e.node_id, h.node_id, 'event', 'certain'
+			INSERT OR IGNORE INTO edges (source_id, target_id, kind)
+			SELECT e.node_id, h.node_id, 'event'
 			FROM tags e
 			JOIN tags h ON e.value = h.value
 			WHERE e.kind = 'emits' AND h.kind = 'handles'
@@ -140,8 +136,8 @@ function synthesizeEventEdges(db: Database): void {
 export {
 	deleteFileData,
 	insertEdges,
+	insertExternalCalls,
 	insertNodes,
 	insertTags,
-	insertUnresolved,
 	synthesizeEventEdges,
 };

package/src/lsp/builder.ts

@@ -0,0 +1,248 @@
+import type { Database } from "bun:sqlite";
+import { existsSync } from "node:fs";
+import { join, relative, resolve } from "node:path";
+import { scanTags } from "../extract/tag-scanner.ts";
+import { discoverFiles } from "../files.ts";
+import {
+	insertEdges,
+	insertExternalCalls,
+	insertNodes,
+	insertTags,
+	synthesizeEventEdges,
+} from "../graph/writer.ts";
+import type { Edge, ExternalCall, Node, Tag } from "../types/graph.ts";
+import { outgoingCallsToEdges } from "./calls.ts";
+import { createLspClient } from "./client.ts";
+import { documentSymbolsToNodesWithPositions, type NodeWithPosition } from "./symbols.ts";
+
+/** Per-language extraction configuration. */
+type LanguageConfig = {
+	readonly language: string;
+	readonly extensions: readonly string[];
+	readonly sourceRoots: readonly string[];
+	readonly testPaths: readonly string[];
+};
+
+/** Options for building the graph. */
+type BuildGraphOptions = {
+	readonly projectRoot: string;
+	readonly db: Database;
+	readonly languageConfigs: readonly LanguageConfig[];
+	readonly exclude: readonly string[];
+};
+
+/** Statistics from a graph build. */
+type BuildStats = {
+	readonly fileCount: number;
+	readonly nodeCount: number;
+	readonly edgeCount: number;
+	readonly tagCount: number;
+	readonly durationMs: number;
+};
+
+/** Resolves the LSP server binary for a language, checking bundled paths first. */
+function resolveLspServer(
+	language: string,
+): { command: string; args: readonly string[]; languageId: string } | undefined {
+	if (language === "typescript") {
+		// Check node_modules/.bin/ first (bundled with lattice-graph)
+		const bundled = join(
+			import.meta.dir,
+			"..",
+			"..",
+			"node_modules",
+			".bin",
+			"typescript-language-server",
+		);
+		const command = existsSync(bundled) ? bundled : "typescript-language-server";
+		return { command, args: ["--stdio"], languageId: "typescript" };
+	}
+	if (language === "python") {
+		const bundled = join(import.meta.dir, "..", "..", "vendor", "venv", "bin", "zubanls");
+		const command = existsSync(bundled) ? bundled : "zubanls";
+		return { command, args: [], languageId: "python" };
+	}
+	return undefined;
+}
+
+/**
+ * Builds the knowledge graph by querying LSP servers for symbols and call hierarchy,
+ * scanning for @lattice: tags, and writing everything to SQLite.
+ * Spawns one LSP server per language. Uses both outgoingCalls and references
+ * strategies to maximize edge coverage.
+ *
+ * @param opts - Build configuration
+ * @returns Build statistics
+ */
+async function buildGraph(opts: BuildGraphOptions): Promise<BuildStats> {
+	const start = performance.now();
+	const { projectRoot, db } = opts;
+
+	let totalFiles = 0;
+	const allNodes: Node[] = [];
+	const allEdges: Edge[] = [];
+	const allTags: Tag[] = [];
+	const allExternalCalls: ExternalCall[] = [];
+
+	for (const langConfig of opts.languageConfigs) {
+		const files: string[] = [];
+		for (const srcRoot of langConfig.sourceRoots) {
+			const absRoot = resolve(projectRoot, srcRoot);
+			files.push(...discoverFiles(absRoot, langConfig.extensions, opts.exclude));
+		}
+
+		if (files.length === 0) continue;
+		totalFiles += files.length;
+
+		const lsp = resolveLspServer(langConfig.language);
+		if (!lsp) continue;
+
+		const client = await createLspClient({
+			command: lsp.command,
+			args: [...lsp.args],
+			rootUri: `file://${projectRoot}`,
+			languageId: lsp.languageId,
+		});
+
+		try {
+			await client.waitForReady(files[0] as string);
+
+			// Phase 1: extract symbols and tags from all files
+			type FileData = {
+				filePath: string;
+				relativePath: string;
+				nodesWithPos: readonly NodeWithPosition[];
+			};
+			const fileDataList: FileData[] = [];
+
+			for (const filePath of files) {
+				const relativePath = relative(projectRoot, filePath);
+				const isTest = langConfig.testPaths.some((tp) => relativePath.startsWith(tp));
+				const source = await Bun.file(filePath).text();
+
+				const symbols = await client.documentSymbol(filePath);
+				const nodesWithPos = documentSymbolsToNodesWithPositions(
+					symbols,
+					relativePath,
+					langConfig.language,
+					isTest,
+				);
+				const nodes = nodesWithPos.map((nwp) => nwp.node);
+				allNodes.push(...nodes);
+
+				const { tags } = scanTags(source, nodes, langConfig.language);
+				allTags.push(...tags);
+
+				fileDataList.push({ filePath, relativePath, nodesWithPos });
+			}
+
+			// Phase 2a: outgoingCalls — "what does each function call?"
+			for (const fd of fileDataList) {
+				for (const nwp of fd.nodesWithPos) {
+					if (nwp.node.kind !== "function" && nwp.node.kind !== "method") continue;
+
+					try {
+						const items = await client.prepareCallHierarchy(
+							fd.filePath,
+							nwp.selectionLine,
+							nwp.selectionCharacter,
+						);
+						if (items.length === 0) continue;
+						const item = items[0];
+						if (!item) continue;
+
+						const calls = await client.outgoingCalls(item);
+						const { edges, externalCalls } = outgoingCallsToEdges(nwp.node.id, calls, projectRoot);
+						allEdges.push(...edges);
+						allExternalCalls.push(...externalCalls);
+					} catch {
+						// outgoingCalls not supported by this server — skip silently
+					}
+				}
+			}
+
+			// Phase 2b: references — "who references each function?"
+			const nodesByFile = new Map<string, readonly Node[]>();
+			for (const fd of fileDataList) {
+				nodesByFile.set(
+					fd.relativePath,
+					fd.nodesWithPos
+						.filter((nwp) => nwp.node.kind === "function" || nwp.node.kind === "method")
+						.map((nwp) => nwp.node),
+				);
+			}
+
+			for (const fd of fileDataList) {
+				for (const nwp of fd.nodesWithPos) {
+					if (nwp.node.kind !== "function" && nwp.node.kind !== "method") continue;
+
+					try {
+						const refs = await client.references(
+							fd.filePath,
+							nwp.selectionLine,
+							nwp.selectionCharacter,
+						);
+
+						for (const ref of refs) {
+							const refFile = ref.uri.startsWith(`file://${projectRoot}/`)
+								? ref.uri.slice(`file://${projectRoot}/`.length)
+								: undefined;
+							if (!refFile) continue;
+
+							const fileFunctions = nodesByFile.get(refFile);
+							if (!fileFunctions) continue;
+
+							const refLine = ref.range.start.line + 1;
+							const caller = fileFunctions.find(
+								(n) => refLine >= n.lineStart && refLine <= n.lineEnd,
+							);
+							if (!caller || caller.id === nwp.node.id) continue;
+
+							allEdges.push({ sourceId: caller.id, targetId: nwp.node.id, kind: "calls" });
+						}
+					} catch {
+						// references not supported by this server — skip silently
+					}
+				}
+			}
+		} finally {
+			await client.shutdown();
+		}
+	}
+
+	insertNodes(db, allNodes);
+	insertEdges(db, allEdges);
+	insertTags(db, allTags);
+	insertExternalCalls(db, allExternalCalls);
+	synthesizeEventEdges(db);
+
+	const durationMs = Math.round(performance.now() - start);
+	return {
+		fileCount: totalFiles,
+		nodeCount: allNodes.length,
+		edgeCount: allEdges.length,
+		tagCount: allTags.length,
+		durationMs,
+	};
+}
+
+/**
+ * Builds a LanguageConfig from a language name and config sections.
+ * Uses sensible defaults for LSP commands.
+ */
+function buildLanguageConfig(
+	language: string,
+	sourceRoots: readonly string[],
+	testPaths: readonly string[],
+): LanguageConfig {
+	const extensions = language === "python" ? [".py"] : [".ts", ".tsx"];
+	return { language, extensions, sourceRoots, testPaths };
+}
+
+export {
+	type BuildGraphOptions,
+	type BuildStats,
+	buildGraph,
+	buildLanguageConfig,
+	type LanguageConfig,
+};

package/src/lsp/calls.ts

@@ -0,0 +1,84 @@
+import type { Edge, ExternalCall } from "../types/graph.ts";
+import type { CallHierarchyOutgoingCall } from "./types.ts";
+
+/** Result of converting outgoing calls to edges. */
+type CallConversionResult = {
+	readonly edges: readonly Edge[];
+	readonly externalCalls: readonly ExternalCall[];
+};
+
+/**
+ * Converts LSP CallHierarchyOutgoingCall responses into Lattice Edges.
+ * Calls to symbols outside the project root are collected as external calls.
+ *
+ * @param sourceId - The Lattice node ID of the calling function
+ * @param calls - Outgoing calls from LSP
+ * @param projectRoot - Absolute path to the project root
+ * @returns Internal edges and external call records
+ */
+function outgoingCallsToEdges(
+	sourceId: string,
+	calls: readonly CallHierarchyOutgoingCall[],
+	projectRoot: string,
+): CallConversionResult {
+	const edges: Edge[] = [];
+	const externalCalls: ExternalCall[] = [];
+	const projectFilePrefix = `file://${projectRoot}/`;
+
+	for (const call of calls) {
+		const uri = call.to.uri;
+
+		if (!uri.startsWith(projectFilePrefix) || uri.includes("/node_modules/")) {
+			// Skip type declarations — not runtime calls
+			if (isTypeDeclaration(uri)) continue;
+
+			const pkg = extractPackageName(uri);
+			if (pkg) {
+				externalCalls.push({ nodeId: sourceId, package: pkg, symbol: call.to.name });
+			}
+			continue;
+		}
+
+		const relativePath = uri.slice(projectFilePrefix.length);
+		const targetId = `${relativePath}::${call.to.name}`;
+		edges.push({ sourceId, targetId, kind: "calls" });
+	}
+
+	return { edges, externalCalls };
+}
+
+/**
+ * Extracts the package name from a node_modules URI.
+ * Handles scoped packages (@scope/package).
+ */
+function extractPackageName(uri: string): string | undefined {
+	const decoded = decodeURIComponent(uri);
+	const nodeModulesIdx = decoded.indexOf("/node_modules/");
+	if (nodeModulesIdx === -1) return undefined;
+	const afterNm = decoded.slice(nodeModulesIdx + "/node_modules/".length);
+	if (afterNm.startsWith("@")) {
+		const parts = afterNm.split("/");
+		return `${parts[0]}/${parts[1]}`;
+	}
+	return afterNm.split("/")[0];
+}
+
+/**
+ * Checks if a URI points to a type-only package (not a runtime dependency).
+ * Type definition packages (@types/*, typescript, bun-types) are filtered out.
+ * Actual library .d.ts stubs (e.g., stripe/index.d.ts) are kept — they represent runtime deps.
+ */
+function isTypeDeclaration(uri: string): boolean {
+	// TypeScript type-only packages
+	if (uri.includes("/node_modules/@types/")) return true;
+	if (uri.includes("/node_modules/%40types/")) return true;
+	if (uri.includes("/node_modules/typescript/")) return true;
+	if (uri.includes("/node_modules/bun-types/")) return true;
+	// Python type stubs
+	if (uri.includes("/typeshed/")) return true;
+	if (uri.includes("/typestubs/")) return true;
+	if (uri.includes("-stubs/")) return true;
+	return false;
+}
+
+export { type CallConversionResult, extractPackageName, isTypeDeclaration, outgoingCallsToEdges };