@shanepadgett/tau-agent 0.26.0 → 0.28.0

package/extensions/explore/ast/adapter.ts

@@ -0,0 +1,97 @@
+import type { Tree } from "web-tree-sitter";
+import type { Decl, ImportRef } from "./ir.ts";
+import type { PackageSurfaceResolver } from "./package-surface.ts";
+
+export type LanguageCapabilities = {
+	shape: boolean;
+	search: boolean;
+	fileDeps: boolean;
+	callEdges: boolean;
+	packageSurface: boolean;
+};
+
+/**
+ * Contextual pattern wrap for languages where a bare snippet parses as the wrong kind
+ * (e.g. Go `fmt.Println($A)` → type_conversion). Shared search substitutes the user
+ * pattern for `$PATTERN` and tries each wrap after the bare pattern.
+ */
+export type AstSearchPatternContext = {
+	/** Source template containing the literal placeholder `$PATTERN`. */
+	readonly context: string;
+	/** tree-sitter kind name selected from the parsed context. */
+	readonly selector: string;
+};
+
+export type ExtractResult = {
+	decls: Decl[];
+	imports: ImportRef[];
+};
+
+/** Result of adapter-owned import specifier → file resolution. */
+export type FileDepResolution =
+	| { kind: "internal"; paths: string[] }
+	| { kind: "external"; id: string }
+	| { kind: "unresolved" };
+
+/**
+ * Host services for file-dep resolution.
+ * Language-agnostic: resolvers never touch the engine type.
+ */
+export type FileDepHost = {
+	readonly cwd: string;
+	/** Scope root for this query. Resolvers must not walk above it for internal hits. */
+	readonly scopeRoot: string;
+	pathExists(path: string): Promise<boolean>;
+	isFile(path: string): Promise<boolean>;
+	/** Basenames; empty if missing or not a directory. */
+	readDir(path: string): Promise<string[]>;
+	/** True when path is a registered source file this graph should consider. */
+	ownsPath(path: string): boolean;
+	/** Session-scoped memo bag; cleared with graph invalidation. */
+	readonly memo: Map<string, unknown>;
+};
+
+export type FileDepResolver = (
+	fromPath: string,
+	specifier: string,
+	host: FileDepHost,
+	signal: AbortSignal,
+) => Promise<FileDepResolution>;
+
+/**
+ * Grammar-backed language. `id` must match a pin in `ast/grammars/manifest`.
+ * Engine owns wasm load/parse; adapter only walks the tree.
+ * Language-owned hooks (`importNoiseIdentifiers`, `resolveFileDep`, optional `resolvePackageSurface`, callEdges)
+ * stay on the adapter — tools/queries never branch on language id.
+ */
+export type GrammarAdapter = {
+	readonly mode: "grammar";
+	readonly id: string;
+	readonly extensions: readonly string[];
+	readonly capabilities: LanguageCapabilities;
+	/** Keywords/types ignored when intersecting import text with a declaration slice (`show`). */
+	readonly importNoiseIdentifiers: ReadonlySet<string>;
+	extract(tree: Tree, source: string): ExtractResult;
+	/** Required when `capabilities.fileDeps` is true. */
+	readonly resolveFileDep?: FileDepResolver;
+	readonly resolvePackageSurface?: PackageSurfaceResolver;
+	/** Optional ast_search contextual wraps; shared matcher stays language-blind. */
+	readonly patternContexts?: readonly AstSearchPatternContext[];
+};
+
+/**
+ * Source-only language (Markdown). No tree-sitter grammar.
+ * Discriminant keeps engine free of `language ===` branches.
+ */
+export type SourceAdapter = {
+	readonly mode: "source";
+	readonly id: string;
+	readonly extensions: readonly string[];
+	readonly capabilities: LanguageCapabilities;
+	readonly importNoiseIdentifiers: ReadonlySet<string>;
+	extract(source: string): ExtractResult;
+	readonly resolveFileDep?: FileDepResolver;
+	readonly resolvePackageSurface?: PackageSurfaceResolver;
+};
+
+export type LanguageAdapter = GrammarAdapter | SourceAdapter;

package/extensions/explore/ast/engine.ts

@@ -0,0 +1,470 @@
+import { createHash } from "node:crypto";
+import { readFile } from "node:fs/promises";
+import { resolve } from "node:path";
+import {
+	initializeTreeSitter,
+	parse as astGrepParse,
+	registerDynamicLanguage,
+	type SgNode,
+	type SgRoot,
+} from "@ast-grep/wasm";
+import { Language, Parser } from "web-tree-sitter";
+import type { AdapterRegistry } from "./registry.ts";
+import { createDefaultRegistry } from "./registry.ts";
+import type { FileIr } from "./ir.ts";
+import { grammarWasmPath, loadGrammarManifest, runtimeWasmPath, type GrammarPin } from "./grammars/manifest.ts";
+import { formatPathForDisplay, pathResolutionError, resolveExplorePath } from "../traverse.ts";
+
+export type FileSource = {
+	ir: FileIr;
+	/** Decoded source text this IR's offsets index into (UTF-16 code units). */
+	source: string;
+};
+
+export type AstSearchBinding = {
+	name: string;
+	/** Exact matched text (single node or multi-node join of exact pieces). */
+	text: string;
+};
+
+export type AstSearchHit = {
+	/** 1-indexed inclusive. */
+	startLine: number;
+	endLine: number;
+	/** Exact match preview (edit-grade). */
+	text: string;
+	bindings: AstSearchBinding[];
+};
+
+export type ExploreEngine = {
+	/** Absolute session cwd — path resolution and directory scans use this. */
+	readonly cwd: string;
+	readonly registry: AdapterRegistry;
+	/** Resolve path, read bytes, return cached or freshly extracted FileIr. */
+	irForFile(path: string): Promise<FileIr>;
+	/** One read: IR (cached when hash matches) plus the decoded source for slicing. */
+	sourceForFile(path: string): Promise<FileSource>;
+	/**
+	 * Structural pattern search over one source string.
+	 * Engine owns @ast-grep/wasm init + grammar registration from pinned wasm paths.
+	 * Does not use the IR cache; does not cache match results.
+	 */
+	searchInSource(languageId: string, source: string, pattern: string): Promise<AstSearchHit[]>;
+	invalidate(paths: readonly string[]): void;
+	clear(): void;
+	/**
+	 * Drop IR cache and delete the retained Parser.
+	 * Loaded Language modules are dropped from JS maps; web-tree-sitter 0.26 has no Language.delete().
+	 */
+	shutdown(): void;
+};
+
+export type ExploreEngineOptions = {
+	cwd: string;
+	registry?: AdapterRegistry;
+};
+
+type CacheEntry = {
+	contentHash: string;
+	ir: FileIr;
+};
+
+function contentHashOf(bytes: Uint8Array): string {
+	return createHash("sha256").update(bytes).digest("hex");
+}
+
+function lineCountOf(source: string): number {
+	if (source.length === 0) return 0;
+	let count = 1;
+	for (let i = 0; i < source.length; i += 1) {
+		const code = source.charCodeAt(i);
+		if (code === 10) count += 1;
+		else if (code === 13) {
+			count += 1;
+			if (source.charCodeAt(i + 1) === 10) i += 1;
+		}
+	}
+	return count;
+}
+
+function pinById(id: string): GrammarPin {
+	const pin = loadGrammarManifest().grammars.find((entry) => entry.id === id);
+	if (pin === undefined) {
+		throw new Error(`No grammar artifact for language: ${id}`);
+	}
+	return pin;
+}
+
+/** ast-grep metavars: `$NAME` one node, `$$$NAME` sibling sequence. Skip `$_`. */
+function metaVarNames(pattern: string): { singles: string[]; multis: string[] } {
+	const singles: string[] = [];
+	const multis: string[] = [];
+	let i = 0;
+	while (i < pattern.length) {
+		if (pattern[i] !== "$") {
+			i += 1;
+			continue;
+		}
+		let dollars = 0;
+		while (i < pattern.length && pattern[i] === "$") {
+			dollars += 1;
+			i += 1;
+		}
+		let name = "";
+		while (i < pattern.length) {
+			const c = pattern[i];
+			if (c === undefined) break;
+			const isAlpha = (c >= "A" && c <= "Z") || c === "_";
+			const isAlnum = isAlpha || (c >= "0" && c <= "9");
+			if ((name.length === 0 && isAlpha) || (name.length > 0 && isAlnum)) {
+				name += c;
+				i += 1;
+				continue;
+			}
+			break;
+		}
+		if (name.length === 0 || name === "_") continue;
+		if (dollars >= 3) {
+			if (!multis.includes(name)) multis.push(name);
+		} else if (dollars === 1) {
+			if (!singles.includes(name)) singles.push(name);
+		}
+	}
+	return { singles, multis };
+}
+
+function hitKey(hit: AstSearchHit): string {
+	return `${hit.startLine}:${hit.endLine}:${hit.text}`;
+}
+
+/** @ast-grep/wasm Pos.index is Unicode scalar offset, not UTF-16. */
+function sliceByCharOffset(source: string, startChar: number, endChar: number): string {
+	let start = 0;
+	let end = source.length;
+	let charIndex = 0;
+	for (let i = 0; i < source.length;) {
+		if (charIndex === startChar) start = i;
+		const code = source.charCodeAt(i);
+		const step = code >= 0xd800 && code <= 0xdbff ? 2 : 1;
+		i += step;
+		charIndex += 1;
+		if (charIndex === endChar) {
+			end = i;
+			break;
+		}
+	}
+	if (charIndex < endChar) end = source.length;
+	return source.slice(start, end);
+}
+
+function bindingsFromNode(node: SgNode, pattern: string, source: string): AstSearchBinding[] {
+	const { singles, multis } = metaVarNames(pattern);
+	const out: AstSearchBinding[] = [];
+	for (const name of singles) {
+		const match = node.getMatch(name);
+		if (match !== undefined) out.push({ name, text: match.text() });
+	}
+	for (const name of multis) {
+		const parts = node.getMultipleMatches(name);
+		if (parts.length === 0) continue;
+		const first = parts[0];
+		const last = parts[parts.length - 1];
+		if (first === undefined || last === undefined) continue;
+		const start = first.range().start.index;
+		const end = last.range().end.index;
+		out.push({ name, text: sliceByCharOffset(source, start, end) });
+	}
+	return out;
+}
+
+function nodeToHit(node: SgNode, pattern: string, source: string): AstSearchHit {
+	const range = node.range();
+	return {
+		startLine: range.start.line + 1,
+		endLine: range.end.line + 1,
+		text: node.text(),
+		bindings: bindingsFromNode(node, pattern, source),
+	};
+}
+
+function collectHits(
+	root: SgRoot,
+	matcher: unknown,
+	pattern: string,
+	source: string,
+	into: Map<string, AstSearchHit>,
+): void {
+	const nodes = root.root().findAll(matcher);
+	for (const node of nodes) {
+		const hit = nodeToHit(node, pattern, source);
+		const key = hitKey(hit);
+		if (!into.has(key)) into.set(key, hit);
+	}
+}
+
+/**
+ * In-process web-tree-sitter host.
+ * Parse → extract plain FileIr → tree.delete() immediately. Cache IR only.
+ * All IR offsets are UTF-16 code units into the decoded source string.
+ * Only this module resolves wasm paths via the grammar manifest.
+ * ast_search uses @ast-grep/wasm on the same pinned grammar paths (decision 11-A).
+ */
+export function createExploreEngine(options: ExploreEngineOptions): ExploreEngine {
+	const cwd = resolve(options.cwd);
+	const registry = options.registry ?? createDefaultRegistry();
+	const irCache = new Map<string, CacheEntry>();
+	const languages = new Map<string, Language>();
+	const languageLoads = new Map<string, Promise<Language>>();
+	const searchLangReady = new Set<string>();
+
+	let initPromise: Promise<void> | undefined;
+	let searchInitPromise: Promise<void> | undefined;
+	let parser: Parser | undefined;
+	let shutDown = false;
+
+	const assertLive = (): void => {
+		if (shutDown) {
+			throw new Error("Explore engine has been shut down");
+		}
+	};
+
+	const ensureReady = async (): Promise<Parser> => {
+		assertLive();
+		if (parser !== undefined) return parser;
+		if (initPromise === undefined) {
+			initPromise = Parser.init({ locateFile: () => runtimeWasmPath() }).then(() => {
+				if (shutDown) return;
+				parser = new Parser();
+			});
+		}
+		await initPromise;
+		assertLive();
+		if (parser === undefined) throw new Error("Explore engine failed to initialize parser");
+		return parser;
+	};
+
+	const ensureSearchReady = async (): Promise<void> => {
+		assertLive();
+		// IR path init first so locateFile pins web-tree-sitter.wasm for the process.
+		await ensureReady();
+		if (searchInitPromise === undefined) {
+			searchInitPromise = initializeTreeSitter();
+		}
+		await searchInitPromise;
+		assertLive();
+	};
+
+	const ensureSearchLanguage = async (languageId: string): Promise<void> => {
+		assertLive();
+		if (searchLangReady.has(languageId)) return;
+		const adapter = registry.adapterForId(languageId);
+		if (adapter === undefined) {
+			throw new Error(`Unregistered language: ${languageId}`);
+		}
+		if (adapter.mode !== "grammar" || !adapter.capabilities.search) {
+			throw new Error(`Language does not support structural search: ${languageId}`);
+		}
+		await ensureSearchReady();
+		assertLive();
+		await registerDynamicLanguage({
+			[languageId]: {
+				libraryPath: grammarWasmPath(pinById(languageId)),
+				expandoChar: "µ",
+			},
+		});
+		assertLive();
+		searchLangReady.add(languageId);
+	};
+
+	const loadLanguage = async (languageId: string): Promise<Language> => {
+		assertLive();
+		const cached = languages.get(languageId);
+		if (cached !== undefined) return cached;
+
+		let pending = languageLoads.get(languageId);
+		if (pending === undefined) {
+			pending = Language.load(grammarWasmPath(pinById(languageId))).then(
+				(language) => {
+					languageLoads.delete(languageId);
+					if (!shutDown) {
+						languages.set(languageId, language);
+					}
+					return language;
+				},
+				(error: unknown) => {
+					languageLoads.delete(languageId);
+					throw error;
+				},
+			);
+			languageLoads.set(languageId, pending);
+		}
+
+		const language = await pending;
+		assertLive();
+		return language;
+	};
+
+	const buildIr = async (absolutePath: string, source: string, hash: string): Promise<FileIr> => {
+		const adapter = registry.adapterForPath(absolutePath);
+		if (adapter === undefined) {
+			throw new Error(`Unsupported language for path: ${formatPathForDisplay(absolutePath, cwd)}`);
+		}
+		const lineCount = lineCountOf(source);
+
+		if (adapter.mode === "source") {
+			const extracted = adapter.extract(source);
+			return {
+				path: absolutePath,
+				contentHash: hash,
+				languageId: adapter.id,
+				lineCount,
+				decls: extracted.decls,
+				imports: extracted.imports,
+				parseDegraded: false,
+			};
+		}
+
+		const activeParser = await ensureReady();
+		const language = await loadLanguage(adapter.id);
+		assertLive();
+		activeParser.setLanguage(language);
+		activeParser.reset();
+		const tree = activeParser.parse(source);
+		if (tree === null) {
+			throw new Error(`Parse failed for ${formatPathForDisplay(absolutePath, cwd)}`);
+		}
+		try {
+			const extracted = adapter.extract(tree, source);
+			return {
+				path: absolutePath,
+				contentHash: hash,
+				languageId: adapter.id,
+				lineCount,
+				decls: extracted.decls,
+				imports: extracted.imports,
+				parseDegraded: tree.rootNode.hasError,
+			};
+		} finally {
+			tree.delete();
+		}
+	};
+
+	const loadSource = async (path: string): Promise<FileSource> => {
+		assertLive();
+		const absolutePath = resolveExplorePath(cwd, path);
+		let bytes: Buffer;
+		try {
+			bytes = await readFile(absolutePath);
+		} catch (error) {
+			throw pathResolutionError(error, path);
+		}
+		assertLive();
+		const hash = contentHashOf(bytes);
+		const source = bytes.toString("utf8");
+		const cached = irCache.get(absolutePath);
+		if (cached !== undefined && cached.contentHash === hash) {
+			return { ir: cached.ir, source };
+		}
+		const ir = await buildIr(absolutePath, source, hash);
+		assertLive();
+		irCache.set(absolutePath, { contentHash: hash, ir });
+		return { ir, source };
+	};
+
+	return {
+		cwd,
+		registry,
+
+		async irForFile(path: string): Promise<FileIr> {
+			const source = await loadSource(path);
+			return source.ir;
+		},
+
+		async sourceForFile(path: string): Promise<FileSource> {
+			return loadSource(path);
+		},
+
+		async searchInSource(languageId: string, source: string, pattern: string): Promise<AstSearchHit[]> {
+			assertLive();
+			const adapter = registry.adapterForId(languageId);
+			if (adapter === undefined) {
+				throw new Error(`Unregistered language: ${languageId}`);
+			}
+			if (adapter.mode !== "grammar" || !adapter.capabilities.search) {
+				throw new Error(`Language does not support structural search: ${languageId}`);
+			}
+			await ensureSearchLanguage(languageId);
+			assertLive();
+
+			let root: SgRoot;
+			try {
+				root = astGrepParse(languageId, source);
+			} catch (error) {
+				const message = error instanceof Error ? error.message : String(error);
+				throw new Error(`Parse failed for structural search (${languageId}): ${message}`);
+			}
+
+			try {
+				const byKey = new Map<string, AstSearchHit>();
+				const matchers: unknown[] = [pattern];
+				const contexts = adapter.patternContexts ?? [];
+				for (const wrap of contexts) {
+					if (!wrap.context.includes("$PATTERN")) continue;
+					matchers.push({
+						rule: {
+							pattern: {
+								context: wrap.context.split("$PATTERN").join(pattern),
+								selector: wrap.selector,
+							},
+						},
+					});
+				}
+
+				let anyMatcherOk = false;
+				let lastError: string | undefined;
+				for (const matcher of matchers) {
+					try {
+						collectHits(root, matcher, pattern, source, byKey);
+						anyMatcherOk = true;
+					} catch (error) {
+						lastError = error instanceof Error ? error.message : String(error);
+					}
+				}
+				if (!anyMatcherOk) {
+					throw new Error(`Invalid pattern: ${lastError ?? "unknown pattern error"}`);
+				}
+
+				return [...byKey.values()].sort((a, b) => {
+					if (a.startLine !== b.startLine) return a.startLine - b.startLine;
+					if (a.endLine !== b.endLine) return a.endLine - b.endLine;
+					return a.text.localeCompare(b.text);
+				});
+			} finally {
+				root.free();
+			}
+		},
+
+		invalidate(paths: readonly string[]): void {
+			for (const path of paths) {
+				irCache.delete(resolveExplorePath(cwd, path));
+			}
+		},
+
+		clear(): void {
+			irCache.clear();
+		},
+
+		shutdown(): void {
+			if (shutDown) return;
+			shutDown = true;
+			irCache.clear();
+			languages.clear();
+			languageLoads.clear();
+			searchLangReady.clear();
+			parser?.delete();
+			parser = undefined;
+			initPromise = undefined;
+			searchInitPromise = undefined;
+		},
+	};
+}

package/extensions/explore/ast/format/ast-search.ts

@@ -0,0 +1,97 @@
+import type { AstSearchMatch, AstSearchResult } from "../queries/ast-search.ts";
+import type { ScanOutcome } from "../scan.ts";
+import type { TraversalLimit } from "../../traverse.ts";
+import { formatPathForDisplay } from "../../traverse.ts";
+
+function lineRange(startLine: number, endLine: number): string {
+	return startLine === endLine ? `L${startLine}` : `L${startLine}-${endLine}`;
+}
+
+function formatMatchRows(match: AstSearchMatch): string[] {
+	const lines: string[] = [];
+	const preview = match.text.length > 0 ? match.text : "(empty)";
+	const previewLines = preview.split("\n");
+	const first = previewLines[0] ?? "(empty)";
+	lines.push(`${lineRange(match.startLine, match.endLine)} ${first}`);
+	for (let i = 1; i < previewLines.length; i += 1) {
+		lines.push(`  ${previewLines[i] ?? ""}`);
+	}
+	for (const binding of match.bindings) {
+		const bindLines = binding.text.split("\n");
+		lines.push(`  $${binding.name} = ${bindLines[0] ?? ""}`);
+		for (let i = 1; i < bindLines.length; i += 1) {
+			lines.push(`    ${bindLines[i] ?? ""}`);
+		}
+	}
+	if (match.enclosing !== undefined) {
+		lines.push(`  in ${match.enclosing.kind} ${match.enclosing.name} L${match.enclosing.startLine}`);
+	}
+	if (match.parseDegraded) {
+		lines.push("  warning: parser recovered with errors");
+	}
+	return lines;
+}
+
+/** One complete file group (path header + matches). */
+export function formatAstSearchFile(path: string, matches: readonly AstSearchMatch[], cwd: string): string {
+	const lines = [formatPathForDisplay(path, cwd)];
+	for (const match of matches) {
+		lines.push(...formatMatchRows(match));
+	}
+	return lines.join("\n");
+}
+
+export function formatAstSearchEmpty(): string {
+	return "No matches";
+}
+
+function limitLabel(limit: TraversalLimit): string {
+	switch (limit) {
+		case "maxFiles":
+			return "file count";
+		case "maxSourceBytes":
+			return "source bytes";
+		case "maxDepth":
+			return "depth";
+		case "maxElapsedMs":
+			return "elapsed time";
+		case "cancelled":
+			return "cancelled";
+	}
+}
+
+/** Footers only when something was cut, a budget tripped, or file diagnostics exist. */
+export function formatAstSearchFooters(result: AstSearchResult, cwd: string): string | undefined {
+	const limits: string[] = [];
+	if (result.resultLimitReached) {
+		limits.push(`matches omitted (result limit ${result.resultLimit})`);
+	}
+	const scan: ScanOutcome | undefined = result.scan;
+	if (scan?.limit !== undefined && scan.limit !== "cancelled") {
+		limits.push(limitLabel(scan.limit));
+	}
+	const diagnostics = result.fileErrors.map((err) => `${formatPathForDisplay(err.path, cwd)}: ${err.message}`);
+	const parts: string[] = [];
+	if (limits.length > 0) parts.push(`limits reached: ${limits.join(", ")}`);
+	if (diagnostics.length > 0) parts.push(`file errors: ${diagnostics.join("; ")}`);
+	if (parts.length === 0) return undefined;
+	return parts.join("\n");
+}
+
+/** Group matches by file (path order of first appearance). */
+export function groupAstSearchByFile(
+	matches: readonly AstSearchMatch[],
+): { path: string; matches: AstSearchMatch[] }[] {
+	const groups: { path: string; matches: AstSearchMatch[] }[] = [];
+	const indexByPath = new Map<string, number>();
+	for (const match of matches) {
+		const existing = indexByPath.get(match.path);
+		if (existing === undefined) {
+			indexByPath.set(match.path, groups.length);
+			groups.push({ path: match.path, matches: [match] });
+		} else {
+			groups[existing]?.matches.push(match);
+		}
+	}
+	return groups;
+}

package/extensions/explore/ast/format/composite.ts

@@ -0,0 +1,26 @@
+export type CompositeBlock = {
+	id: string;
+	label: string;
+	text: string;
+};
+
+/** Shared section packing for impact/context complete-block emission. */
+export function compositeSectionBlocks(args: {
+	header: string;
+	emptyLabel: string;
+	sections: readonly { id: string; label: string; text: string }[];
+	footers: readonly string[];
+}): CompositeBlock[] {
+	const blocks: CompositeBlock[] = [{ id: "header", label: "header", text: args.header }];
+	if (args.sections.length === 0) {
+		blocks.push({ id: "empty", label: "empty", text: args.emptyLabel });
+	} else {
+		for (const section of args.sections) {
+			blocks.push({ id: section.id, label: section.label, text: section.text });
+		}
+	}
+	if (args.footers.length > 0) {
+		blocks.push({ id: "footer", label: "footer", text: args.footers.join("\n") });
+	}
+	return blocks;
+}