lattice-graph 0.1.0

package/src/commands/update.ts

@@ -0,0 +1,175 @@
+import { Database } from "bun:sqlite";
+import { existsSync, statSync } from "node:fs";
+import { join, relative } from "node:path";
+import type { Extractor } from "../extract/extractor.ts";
+import { initTreeSitter } from "../extract/parser.ts";
+import { createPythonExtractor } from "../extract/python/extractor.ts";
+import { createTypeScriptExtractor } from "../extract/typescript/extractor.ts";
+import { checkSchemaVersion } from "../graph/database.ts";
+import {
+	deleteFileData,
+	insertEdges,
+	insertNodes,
+	insertTags,
+	insertUnresolved,
+	synthesizeEventEdges,
+} from "../graph/writer.ts";
+import type { LatticeConfig } from "../types/config.ts";
+import type { ExtractionResult } from "../types/graph.ts";
+import { err, isOk, ok, type Result } from "../types/result.ts";
+
+/** Statistics from an incremental update. */
+type UpdateStats = {
+	readonly totalFiles: number;
+	readonly filesReindexed: number;
+	readonly durationMs: number;
+};
+
+/**
+ * Performs an incremental update of the knowledge graph.
+ * Only re-indexes files that have changed since the last build.
+ * Falls back to a full rebuild if >30% of files are dirty.
+ *
+ * @param projectRoot - Path to the project root
+ * @param config - Lattice configuration
+ * @returns Update statistics or an error message
+ */
+// @lattice:flow update
+async function executeUpdate(
+	projectRoot: string,
+	config: LatticeConfig,
+): Promise<Result<UpdateStats, string>> {
+	const startTime = Date.now();
+	const dbPath = join(projectRoot, ".lattice", "graph.db");
+
+	if (!existsSync(dbPath)) {
+		return err("No existing graph found. Run 'lattice build' first.");
+	}
+
+	try {
+		const db = new Database(dbPath);
+		const schemaCheck = checkSchemaVersion(db);
+		if (!isOk(schemaCheck)) {
+			db.close();
+			return err(schemaCheck.error);
+		}
+
+		// Get last build time
+		const lastBuildRow = db.query("SELECT value FROM meta WHERE key = 'last_build'").get() as {
+			value: string;
+		} | null;
+		if (!lastBuildRow) {
+			db.close();
+			return err("No last_build timestamp found. Run 'lattice build' first.");
+		}
+		const lastBuild = new Date(lastBuildRow.value);
+
+		// Initialize extractors
+		await initTreeSitter();
+		const extractors = await createExtractors(config);
+		const extByExt = new Map<string, Extractor>();
+		for (const ext of extractors) {
+			for (const fileExt of ext.fileExtensions) {
+				extByExt.set(fileExt, ext);
+			}
+		}
+
+		// Scan all files, find changed ones
+		const sourceRoot = join(projectRoot, config.root);
+		const glob = new Bun.Glob("**/*");
+		const allFiles: string[] = [];
+		const changedFiles: string[] = [];
+
+		for await (const path of glob.scan({ cwd: sourceRoot, dot: false })) {
+			const ext = `.${path.split(".").pop()}`;
+			if (!extByExt.has(ext)) continue;
+			if (isExcluded(path, config.exclude)) continue;
+			allFiles.push(path);
+
+			const fullPath = join(sourceRoot, path);
+			const stat = statSync(fullPath);
+			if (stat.mtime > lastBuild) {
+				changedFiles.push(path);
+			}
+		}
+
+		// Fall back to full rebuild if >30% changed
+		if (changedFiles.length > allFiles.length * 0.3) {
+			db.close();
+			const { executeBuild } = await import("./build.ts");
+			return executeBuild(projectRoot, config).then((result) => {
+				if (isOk(result)) {
+					return ok({
+						totalFiles: allFiles.length,
+						filesReindexed: allFiles.length,
+						durationMs: Date.now() - startTime,
+					});
+				}
+				return err(result.error);
+			});
+		}
+
+		// Re-extract changed files
+		for (const file of changedFiles) {
+			const ext = `.${file.split(".").pop()}`;
+			const extractor = extByExt.get(ext);
+			if (!extractor) continue;
+
+			const fullPath = join(sourceRoot, file);
+			const source = await Bun.file(fullPath).text();
+			const relativePath = relative(projectRoot, fullPath);
+
+			// Delete old data for this file
+			deleteFileData(db, relativePath);
+
+			// Re-extract
+			const result: ExtractionResult = await extractor.extract(relativePath, source);
+			insertNodes(db, result.nodes);
+			insertEdges(db, result.edges);
+			insertTags(db, result.tags);
+			insertUnresolved(db, result.unresolved);
+		}
+
+		// Rebuild event edges
+		synthesizeEventEdges(db);
+
+		// Update timestamp
+		db.run("INSERT OR REPLACE INTO meta (key, value) VALUES ('last_build', ?)", [
+			new Date().toISOString(),
+		]);
+
+		db.close();
+
+		return ok({
+			totalFiles: allFiles.length,
+			filesReindexed: changedFiles.length,
+			durationMs: Date.now() - startTime,
+		});
+	} catch (error) {
+		return err(`Update failed: ${error instanceof Error ? error.message : String(error)}`);
+	}
+}
+
+/** Creates extractors for configured languages. */
+async function createExtractors(config: LatticeConfig): Promise<readonly Extractor[]> {
+	const extractors: Extractor[] = [];
+	for (const lang of config.languages) {
+		if (lang === "python") {
+			extractors.push(await createPythonExtractor());
+		}
+		if (lang === "typescript") {
+			extractors.push(await createTypeScriptExtractor());
+		}
+	}
+	return extractors;
+}
+
+/** Checks if a file path matches any exclude pattern. */
+function isExcluded(filePath: string, excludePatterns: readonly string[]): boolean {
+	for (const pattern of excludePatterns) {
+		if (filePath.includes(pattern.replace("**", "").replace("*", ""))) return true;
+	}
+	return false;
+}
+
+export { executeUpdate, type UpdateStats };

package/src/config.ts

@@ -0,0 +1,93 @@
+import { parse as parseToml } from "smol-toml";
+import type { LatticeConfig, LintConfig, PythonConfig, TypeScriptConfig } from "./types/config.ts";
+import { err, ok, type Result } from "./types/result.ts";
+
+const DEFAULT_EXCLUDE = ["node_modules", "venv", ".git", "dist", "__pycache__", ".lattice"];
+
+/**
+ * Parses a TOML string into a validated LatticeConfig.
+ * Returns an error for invalid TOML, missing required fields, or validation failures.
+ *
+ * @param tomlString - Raw contents of a lattice.toml file
+ * @returns Parsed and validated config, or an error message
+ */
+function parseConfig(tomlString: string): Result<LatticeConfig, string> {
+	let raw: Record<string, unknown>;
+	try {
+		raw = parseToml(tomlString) as Record<string, unknown>;
+	} catch {
+		return err("Invalid TOML syntax");
+	}
+
+	const project = raw.project;
+	if (!isRecord(project)) {
+		return err("Missing [project] section");
+	}
+
+	const languages = project.languages;
+	if (!isStringArray(languages) || languages.length === 0) {
+		return err("project.languages must be a non-empty array of strings");
+	}
+
+	const root = typeof project.root === "string" ? project.root : ".";
+	const exclude = isStringArray(project.exclude) ? project.exclude : DEFAULT_EXCLUDE;
+
+	const pythonConfig = languages.includes("python") ? parsePythonSection(raw.python) : undefined;
+
+	const typescriptConfig = languages.includes("typescript")
+		? parseTypeScriptSection(raw.typescript)
+		: undefined;
+
+	const lintConfig = parseLintSection(raw.lint);
+
+	return ok({
+		languages,
+		root,
+		exclude,
+		python: pythonConfig,
+		typescript: typescriptConfig,
+		lint: lintConfig,
+	});
+}
+
+/** Parses the [python] section with defaults for missing fields. */
+function parsePythonSection(raw: unknown): PythonConfig {
+	const section = isRecord(raw) ? raw : {};
+	return {
+		sourceRoots: isStringArray(section.source_roots) ? section.source_roots : ["."],
+		testPaths: isStringArray(section.test_paths) ? section.test_paths : ["tests"],
+		frameworks: isStringArray(section.frameworks) ? section.frameworks : [],
+	};
+}
+
+/** Parses the [typescript] section with defaults for missing fields. */
+function parseTypeScriptSection(raw: unknown): TypeScriptConfig {
+	const section = isRecord(raw) ? raw : {};
+	return {
+		sourceRoots: isStringArray(section.source_roots) ? section.source_roots : ["."],
+		testPaths: isStringArray(section.test_paths) ? section.test_paths : ["__tests__"],
+		tsconfig: typeof section.tsconfig === "string" ? section.tsconfig : undefined,
+		frameworks: isStringArray(section.frameworks) ? section.frameworks : [],
+	};
+}
+
+/** Parses the [lint] section with defaults for missing fields. */
+function parseLintSection(raw: unknown): LintConfig {
+	const section = isRecord(raw) ? raw : {};
+	return {
+		strict: section.strict === true,
+		ignore: isStringArray(section.ignore) ? section.ignore : [],
+	};
+}
+
+/** Type guard for plain objects. */
+function isRecord(value: unknown): value is Record<string, unknown> {
+	return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+/** Type guard for string arrays. */
+function isStringArray(value: unknown): value is string[] {
+	return Array.isArray(value) && value.every((item) => typeof item === "string");
+}
+
+export { parseConfig };

package/src/extract/extractor.ts

@@ -0,0 +1,13 @@
+import type { ExtractionResult } from "../types/graph.ts";
+
+/**
+ * Language-specific extractor that produces nodes, edges, and tags from source files.
+ * Each supported language implements this type.
+ */
+type Extractor = {
+	readonly language: string;
+	readonly fileExtensions: readonly string[];
+	readonly extract: (filePath: string, source: string) => Promise<ExtractionResult>;
+};
+
+export type { Extractor };

package/src/extract/parser.ts

@@ -0,0 +1,117 @@
+/**
+ * Shared tree-sitter parser initialization.
+ * Uses web-tree-sitter with WASM grammars from tree-sitter-wasms.
+ */
+
+// web-tree-sitter 0.24.x uses CJS exports
+// biome-ignore lint/suspicious/noExplicitAny: web-tree-sitter 0.24 has no ESM types
+const TreeSitter = require("web-tree-sitter") as any;
+
+type TreeSitterParser = {
+	setLanguage(lang: TreeSitterLanguage): void;
+	parse(input: string): TreeSitterTree;
+};
+
+type TreeSitterLanguage = {
+	readonly version: number;
+};
+
+type TreeSitterTree = {
+	readonly rootNode: TreeSitterNode;
+};
+
+type TreeSitterNode = {
+	readonly type: string;
+	readonly text: string;
+	readonly startPosition: { row: number; column: number };
+	readonly endPosition: { row: number; column: number };
+	readonly startIndex: number;
+	readonly endIndex: number;
+	readonly childCount: number;
+	readonly children: readonly TreeSitterNode[];
+	readonly parent: TreeSitterNode | null;
+	readonly firstChild: TreeSitterNode | null;
+	readonly lastChild: TreeSitterNode | null;
+	readonly nextSibling: TreeSitterNode | null;
+	readonly previousSibling: TreeSitterNode | null;
+	childForFieldName(fieldName: string): TreeSitterNode | null;
+	childrenForFieldName(fieldName: string): readonly TreeSitterNode[];
+	descendantsOfType(type: string | readonly string[]): readonly TreeSitterNode[];
+};
+
+let initialized = false;
+const languageCache = new Map<string, TreeSitterLanguage>();
+
+/**
+ * Resolves the WASM file path for a language grammar.
+ * Tries multiple locations to support both dev and compiled binary modes:
+ * 1. Relative to this source file (dev: bun src/main.ts)
+ * 2. Relative to the compiled binary (binary: ./lattice)
+ * 3. Relative to CWD (when node_modules exists in working dir)
+ */
+function grammarPath(language: string): string {
+	const { existsSync } = require("node:fs");
+	const { dirname, join } = require("node:path");
+	const filename = `tree-sitter-${language}.wasm`;
+	const subpath = `node_modules/tree-sitter-wasms/out/${filename}`;
+
+	// Try relative to this source file (dev mode)
+	const packageRoot = new URL("../../", import.meta.url).pathname;
+	const devPath = join(packageRoot, subpath);
+	if (existsSync(devPath)) return devPath;
+
+	// Try relative to the binary's location (compiled mode)
+	const binDir = dirname(process.execPath);
+	const binPath = join(binDir, subpath);
+	if (existsSync(binPath)) return binPath;
+
+	// Try relative to CWD
+	const cwdPath = join(process.cwd(), subpath);
+	if (existsSync(cwdPath)) return cwdPath;
+
+	// Fall back — will produce a clear error
+	return devPath;
+}
+
+/**
+ * Initializes tree-sitter WASM runtime. Must be called once before parsing.
+ * Safe to call multiple times — subsequent calls are no-ops.
+ */
+async function initTreeSitter(): Promise<void> {
+	if (initialized) return;
+	await TreeSitter.init();
+	initialized = true;
+}
+
+/**
+ * Creates a parser configured for the given language.
+ *
+ * @param language - Language name matching the WASM grammar file (e.g., "python", "typescript")
+ * @returns A configured parser ready to parse source code
+ */
+async function createParser(language: string): Promise<TreeSitterParser> {
+	await initTreeSitter();
+
+	const cached = languageCache.get(language);
+	const lang = cached ?? (await loadAndCacheLanguage(language));
+
+	const parser = new TreeSitter();
+	parser.setLanguage(lang);
+	return parser as TreeSitterParser;
+}
+
+/** Loads a language grammar from WASM and caches it. */
+async function loadAndCacheLanguage(language: string): Promise<TreeSitterLanguage> {
+	const lang = (await TreeSitter.Language.load(grammarPath(language))) as TreeSitterLanguage;
+	languageCache.set(language, lang);
+	return lang;
+}
+
+export {
+	createParser,
+	initTreeSitter,
+	type TreeSitterLanguage,
+	type TreeSitterNode,
+	type TreeSitterParser,
+	type TreeSitterTree,
+};

package/src/extract/python/calls.ts

@@ -0,0 +1,121 @@
+import type { TreeSitterNode, TreeSitterTree } from "../parser.ts";
+
+/** A raw call detected in the AST before resolution to graph edges. */
+type RawCall = {
+	readonly sourceId: string;
+	readonly callee: string;
+	readonly line: number;
+};
+
+/**
+ * Extracts function calls from a Python AST.
+ * Each call is scoped to its enclosing function or method.
+ *
+ * @param tree - Parsed tree-sitter tree
+ * @param filePath - Relative file path for source ID construction
+ * @returns Raw calls with caller ID and callee expression
+ */
+function extractPythonCalls(tree: TreeSitterTree, filePath: string): readonly RawCall[] {
+	const calls: RawCall[] = [];
+	visitForCalls(tree.rootNode, filePath, [], calls);
+	return calls;
+}
+
+/** Scope entry for tracking the enclosing function/class context. */
+type ScopeEntry = { readonly name: string };
+
+/**
+ * Recursively walks the AST to find call expressions inside function bodies.
+ * Tracks the enclosing function scope to produce correct source IDs.
+ */
+function visitForCalls(
+	node: TreeSitterNode,
+	filePath: string,
+	scopeStack: readonly ScopeEntry[],
+	results: RawCall[],
+): void {
+	// Enter a new scope for function/class definitions
+	if (node.type === "function_definition" || node.type === "class_definition") {
+		const nameNode = node.childForFieldName("name");
+		if (nameNode) {
+			const newScope = [...scopeStack, { name: nameNode.text }];
+			for (const child of node.children) {
+				visitForCalls(child, filePath, newScope, results);
+			}
+			return;
+		}
+	}
+
+	// Handle decorated definitions — unwrap to the inner definition
+	if (node.type === "decorated_definition") {
+		for (const child of node.children) {
+			if (child.type === "function_definition" || child.type === "class_definition") {
+				visitForCalls(child, filePath, scopeStack, results);
+			}
+		}
+		return;
+	}
+
+	// Detect call expressions inside a function scope
+	if (node.type === "call" && scopeStack.length > 0) {
+		const callee = extractCalleeName(node);
+		if (callee) {
+			const sourceId = `${filePath}::${scopeStack.map((s) => s.name).join(".")}`;
+			results.push({
+				sourceId,
+				callee,
+				line: node.startPosition.row + 1,
+			});
+		}
+	}
+
+	// Recurse into children
+	for (const child of node.children) {
+		visitForCalls(child, filePath, scopeStack, results);
+	}
+}
+
+/**
+ * Extracts the callee name from a call node.
+ * Handles: simple calls (foo()), attribute calls (obj.method()), chained calls (a.b.c()).
+ */
+function extractCalleeName(callNode: TreeSitterNode): string | undefined {
+	const funcNode = callNode.children[0];
+	if (!funcNode) return undefined;
+
+	if (funcNode.type === "identifier") {
+		return funcNode.text;
+	}
+
+	if (funcNode.type === "attribute") {
+		return flattenAttribute(funcNode);
+	}
+
+	return undefined;
+}
+
+/**
+ * Flattens a nested attribute access into a dotted string.
+ * e.g., attribute(attribute(identifier("stripe"), "charges"), "create") → "stripe.charges.create"
+ */
+function flattenAttribute(node: TreeSitterNode): string {
+	const parts: string[] = [];
+	let current: TreeSitterNode | null = node;
+
+	while (current?.type === "attribute") {
+		const attrName = current.children.at(-1);
+		if (attrName?.type === "identifier") {
+			parts.unshift(attrName.text);
+		}
+		current = current.children[0] ?? null;
+	}
+
+	// The leftmost part is an identifier
+	if (current?.type === "identifier") {
+		parts.unshift(current.text);
+	}
+
+	return parts.join(".");
+}
+
+export { extractPythonCalls, type RawCall };

package/src/extract/python/extractor.ts

@@ -0,0 +1,171 @@
+import type { Edge, ExtractionResult, Tag } from "../../types/graph.ts";
+import { isOk, unwrap } from "../../types/result.ts";
+import type { Extractor } from "../extractor.ts";
+import { createParser, type TreeSitterParser } from "../parser.ts";
+import { parseTags } from "../tags.ts";
+import { extractPythonCalls } from "./calls.ts";
+import { detectPythonFrameworks } from "./frameworks.ts";
+import { extractPythonSymbols } from "./symbols.ts";
+
+/**
+ * Creates a Python extractor with an initialized tree-sitter parser.
+ * Must be called after initTreeSitter().
+ *
+ * @returns An Extractor configured for Python source files
+ */
+async function createPythonExtractor(): Promise<Extractor> {
+	const parser = await createParser("python");
+
+	return {
+		language: "python",
+		fileExtensions: [".py"],
+		extract: (filePath: string, source: string): Promise<ExtractionResult> =>
+			extractPython(parser, filePath, source),
+	};
+}
+
+/**
+ * Extracts symbols, calls, imports, tags, and framework metadata from a Python file.
+ *
+ * @param parser - Initialized tree-sitter parser for Python
+ * @param filePath - Relative file path for node ID construction
+ * @param source - Raw source code
+ * @returns Complete extraction result with nodes, edges, tags, and unresolved references
+ */
+async function extractPython(
+	parser: TreeSitterParser,
+	filePath: string,
+	source: string,
+): Promise<ExtractionResult> {
+	if (!source.trim()) {
+		return { nodes: [], edges: [], tags: [], unresolved: [] };
+	}
+
+	const tree = parser.parse(source);
+
+	// 1. Extract symbols (functions, classes, methods)
+	const nodes = [...extractPythonSymbols(tree, filePath, source)];
+
+	// 2. Extract calls and convert to edges
+	const rawCalls = extractPythonCalls(tree, filePath);
+	const edges: Edge[] = [];
+	const nodeIds = new Set(nodes.map((n) => n.id));
+
+	for (const call of rawCalls) {
+		// Try to resolve the callee to a known node in this file
+		const targetId = resolveCalleeInFile(call.callee, filePath, nodeIds);
+		if (targetId) {
+			edges.push({
+				sourceId: call.sourceId,
+				targetId,
+				kind: "calls",
+				certainty: "certain",
+			});
+		} else {
+			// The callee is external or unresolvable within this file.
+			// Store as an edge with the raw callee expression as target.
+			// Cross-file resolution happens in the build command.
+			edges.push({
+				sourceId: call.sourceId,
+				targetId: call.callee,
+				kind: "calls",
+				certainty: "uncertain",
+			});
+		}
+	}
+
+	// 3. Parse lattice tags from comments above functions
+	const tags = extractTagsFromSource(source, filePath, nodes);
+
+	// 4. Detect framework patterns and attach route metadata to nodes
+	const frameworks = detectPythonFrameworks(tree, filePath);
+	for (const detection of frameworks) {
+		if (detection.route) {
+			const node = nodes.find((n) => n.name === detection.functionName);
+			if (node) {
+				const idx = nodes.indexOf(node);
+				nodes[idx] = {
+					...node,
+					metadata: { ...node.metadata, route: detection.route },
+				};
+			}
+		}
+	}
+
+	return { nodes, edges, tags, unresolved: [] };
+}
+
+/**
+ * Resolves a callee name to a node ID within the same file.
+ * Handles simple names (bar → filePath::bar) and self.method references.
+ */
+function resolveCalleeInFile(
+	callee: string,
+	filePath: string,
+	nodeIds: Set<string>,
+): string | undefined {
+	// Direct match: callee is a function name in this file
+	const directId = `${filePath}::${callee}`;
+	if (nodeIds.has(directId)) return directId;
+
+	// self.method → try ClassName.method for each class in this file
+	if (callee.startsWith("self.")) {
+		const methodName = callee.slice(5);
+		for (const id of nodeIds) {
+			if (id.endsWith(`.${methodName}`) && id.startsWith(`${filePath}::`)) {
+				return id;
+			}
+		}
+	}
+
+	return undefined;
+}
+
+/**
+ * Extracts lattice tags by finding comment blocks directly above function definitions.
+ * Associates each parsed tag with the node ID of the function below it.
+ */
+function extractTagsFromSource(
+	source: string,
+	_filePath: string,
+	nodes: readonly { readonly id: string; readonly lineStart: number }[],
+): readonly Tag[] {
+	const lines = source.split("\n");
+	const tags: Tag[] = [];
+
+	for (const node of nodes) {
+		// Collect comment lines directly above the function (no blank lines between)
+		const commentLines: string[] = [];
+		let lineIdx = node.lineStart - 2; // lineStart is 1-based, array is 0-based
+		while (lineIdx >= 0) {
+			const line = lines[lineIdx]?.trim();
+			if (!line) break;
+			if (line.startsWith("#") || line.startsWith("//") || line.startsWith("/*")) {
+				commentLines.unshift(line);
+				lineIdx--;
+			} else if (line.startsWith("@")) {
+				// Skip decorators — they're between the tags and the function
+				lineIdx--;
+			} else {
+				break;
+			}
+		}
+
+		if (commentLines.length === 0) continue;
+
+		const parseResult = parseTags(commentLines.join("\n"));
+		if (isOk(parseResult)) {
+			for (const parsed of unwrap(parseResult)) {
+				tags.push({
+					nodeId: node.id,
+					kind: parsed.kind,
+					value: parsed.value,
+				});
+			}
+		}
+	}
+
+	return tags;
+}
+
+export { createPythonExtractor };