@shanepadgett/tau-agent 0.27.0 → 0.28.0

package/extensions/explore/ast/markdown.ts

@@ -0,0 +1,145 @@
+import type { ExtractResult } from "./adapter.ts";
+import type { Decl } from "./ir.ts";
+
+const ATX_HEADING = /^(#{1,6})[ \t]+(.*?)[ \t]*#*[ \t]*$/;
+const OPEN_FENCE = /^( {0,3})(`{3,}|~{3,})(.*)$/;
+
+type LineInfo = {
+	/** 1-indexed. */
+	line: number;
+	text: string;
+	/** Offset of the first code unit of this line. */
+	startOffset: number;
+	/** Offset just past the line content (before newline, or EOF). */
+	endOffset: number;
+	/** Offset of the first code unit of the following line (or EOF). */
+	nextOffset: number;
+};
+
+type OpenHeading = {
+	decl: Decl;
+	depth: number;
+};
+
+function splitLines(source: string): LineInfo[] {
+	if (source.length === 0) {
+		return [{ line: 1, text: "", startOffset: 0, endOffset: 0, nextOffset: 0 }];
+	}
+
+	const lines: LineInfo[] = [];
+	let line = 1;
+	let i = 0;
+
+	while (i < source.length) {
+		const start = i;
+		while (i < source.length) {
+			const ch = source.charCodeAt(i);
+			if (ch === 10 || ch === 13) break;
+			i += 1;
+		}
+		const end = i;
+		let next = i;
+		if (next < source.length && source.charCodeAt(next) === 13) {
+			next += 1;
+		}
+		if (next < source.length && source.charCodeAt(next) === 10) {
+			next += 1;
+		}
+		lines.push({ line, text: source.slice(start, end), startOffset: start, endOffset: end, nextOffset: next });
+		line += 1;
+		i = next;
+	}
+
+	return lines;
+}
+
+function closeOpen(stack: OpenHeading[], depth: number, endLine: number, endOffset: number): void {
+	while (stack.length > 0) {
+		const top = stack[stack.length - 1];
+		if (top === undefined || top.depth < depth) break;
+		stack.pop();
+		const decl = top.decl;
+		decl.endLine = endLine;
+		decl.endOffset = endOffset;
+		if (decl.bodyStartOffset !== undefined) {
+			decl.bodyEndOffset = endOffset;
+			if (decl.bodyStartOffset >= endOffset) {
+				delete decl.bodyStartOffset;
+				delete decl.bodyEndOffset;
+			}
+		}
+	}
+}
+
+/**
+ * ATX heading scanner. Fenced code (``` / ~~~, length ≥ 3, matching closer) is not heading material.
+ * Section range runs to the line before the next same-or-shallower heading, else EOF.
+ */
+export function extractMarkdown(source: string): ExtractResult {
+	const lines = splitLines(source);
+	const roots: Decl[] = [];
+	const stack: OpenHeading[] = [];
+	let inFence: { char: "`" | "~"; length: number } | undefined;
+
+	for (const info of lines) {
+		if (inFence !== undefined) {
+			const close = info.text.match(/^ {0,3}(`{3,}|~{3,})[ \t]*$/);
+			const marker = close?.[1];
+			if (marker !== undefined && marker[0] === inFence.char && marker.length >= inFence.length) {
+				inFence = undefined;
+			}
+			continue;
+		}
+
+		const fence = info.text.match(OPEN_FENCE);
+		const fenceMarker = fence?.[2];
+		if (fenceMarker !== undefined) {
+			const char = fenceMarker[0] === "~" ? "~" : "`";
+			inFence = { char, length: fenceMarker.length };
+			continue;
+		}
+
+		const heading = info.text.match(ATX_HEADING);
+		if (heading === null) continue;
+		const hashes = heading[1];
+		if (hashes === undefined) continue;
+		const depth = hashes.length;
+		const name = (heading[2] ?? "").trim();
+		if (name.length === 0) continue;
+
+		const prev = info.line > 1 ? lines[info.line - 2] : undefined;
+		const closeEndLine = prev === undefined ? info.line : prev.line;
+		closeOpen(stack, depth, closeEndLine, info.startOffset);
+
+		const parent = stack[stack.length - 1];
+		const qualifiedName = parent === undefined ? name : `${parent.decl.qualifiedName}.${name}`;
+		const decl: Decl = {
+			kind: "heading",
+			name,
+			qualifiedName,
+			startLine: info.line,
+			endLine: info.line,
+			startOffset: info.startOffset,
+			endOffset: info.endOffset,
+			signatureEndOffset: info.endOffset,
+			bodyStartOffset: info.nextOffset,
+			bodyEndOffset: info.nextOffset,
+			visibility: "public",
+			exported: true,
+			children: [],
+			calls: [],
+			bases: [],
+		};
+
+		if (parent === undefined) roots.push(decl);
+		else parent.decl.children.push(decl);
+		stack.push({ decl, depth });
+	}
+
+	const last = lines[lines.length - 1];
+	const eofLine = last === undefined ? 1 : last.line;
+	const eofOffset = last === undefined ? 0 : last.nextOffset;
+	closeOpen(stack, 1, eofLine, eofOffset);
+
+	return { decls: roots, imports: [] };
+}

package/extensions/explore/ast/package-surface.ts

@@ -0,0 +1,42 @@
+import type { Decl } from "./ir.ts";
+import type { FileSource } from "./engine.ts";
+
+/** Stable key for a declaration on a package surface (path + identity spans). */
+export function packageDeclKey(path: string, decl: Decl): string {
+	return `${path}\0${decl.qualifiedName}\0${decl.startLine}\0${decl.startOffset}`;
+}
+
+/**
+ * Host services for a package-surface resolver.
+ * Language-agnostic: resolvers never touch the engine type.
+ */
+export type PackageSurfaceHost = {
+	readonly cwd: string;
+	sourceForFile(path: string): Promise<FileSource>;
+	/** True when path belongs to a language that participates in this surface capability. */
+	ownsPath(path: string): boolean;
+};
+
+/**
+ * Resolved public package surface for one language family.
+ * Discover matches ordinary IR decls against `declKeys` — no language branches.
+ */
+export type PackageSurfaceGraph = {
+	declKeys: ReadonlySet<string>;
+	/** packageDeclKey → caller import/access line when known. */
+	accessByDecl: ReadonlyMap<string, string>;
+	/** Absolute defining files that hold surface decls. */
+	paths: readonly string[];
+	filesVisited: number;
+};
+
+/**
+ * Language-owned package surface resolution.
+ * `undefined` = this language has no package root at the scope (try next resolver).
+ * Throw only for hard failures (broken manifest, etc.).
+ */
+export type PackageSurfaceResolver = (
+	scopeRoot: string,
+	host: PackageSurfaceHost,
+	signal: AbortSignal,
+) => Promise<PackageSurfaceGraph | undefined>;

package/extensions/explore/ast/queries/ast-search.ts

@@ -0,0 +1,306 @@
+import { lstat, readFile } from "node:fs/promises";
+import type { AstSearchHit, ExploreEngine } from "../engine.ts";
+import type { Decl, DeclKind, FileIr } from "../ir.ts";
+import { walkDecls } from "../query.ts";
+import { scanSources, type ScanOutcome } from "../scan.ts";
+import { formatPathForDisplay, pathResolutionError, resolveExplorePath } from "../../traverse.ts";
+
+const PATTERN_MAX_BYTES = 16 * 1024;
+
+export type AstSearchMatch = AstSearchHit & {
+	path: string;
+	/** Innermost enclosing decl when it helps disambiguate multi-hit files. */
+	enclosing: { name: string; kind: DeclKind; startLine: number } | undefined;
+	parseDegraded: boolean;
+};
+
+export type AstSearchResult = {
+	matches: AstSearchMatch[];
+	resultLimit: number;
+	resultLimitReached: boolean;
+	languageId: string;
+	scan: ScanOutcome | undefined;
+	/** Files that failed to search (path + reason). */
+	fileErrors: { path: string; message: string }[];
+};
+
+function assertPattern(pattern: string): void {
+	if (pattern.length === 0) throw new Error("pattern must be non-empty");
+	const bytes = Buffer.byteLength(pattern, "utf8");
+	if (bytes > PATTERN_MAX_BYTES) {
+		throw new Error(`pattern exceeds ${PATTERN_MAX_BYTES} bytes (${bytes})`);
+	}
+}
+
+function resolveLanguageId(
+	engine: ExploreEngine,
+	language: string | undefined,
+	absolutePath: string | undefined,
+	isDirectory: boolean,
+): string {
+	if (isDirectory) {
+		if (language === undefined || language.length === 0) {
+			throw new Error("language is required for directory targets");
+		}
+	}
+	if (language !== undefined && language.length > 0) {
+		const adapter = engine.registry.adapterForId(language);
+		if (adapter === undefined) {
+			throw new Error(`Unregistered language: ${language}`);
+		}
+		if (!adapter.capabilities.search) {
+			throw new Error(`Language does not support structural search: ${language}`);
+		}
+		if (absolutePath !== undefined && !isDirectory) {
+			const byPath = engine.registry.adapterForPath(absolutePath);
+			if (byPath === undefined) {
+				throw new Error(`Unsupported language for path: ${formatPathForDisplay(absolutePath, engine.cwd)}`);
+			}
+			if (byPath.id !== language) {
+				throw new Error(`language mismatch: path is ${byPath.id}, requested ${language}`);
+			}
+		}
+		return language;
+	}
+	if (absolutePath === undefined) {
+		throw new Error("language is required");
+	}
+	const byPath = engine.registry.adapterForPath(absolutePath);
+	if (byPath === undefined) {
+		throw new Error(`Unsupported language for path: ${formatPathForDisplay(absolutePath, engine.cwd)}`);
+	}
+	if (!byPath.capabilities.search) {
+		throw new Error(`Language does not support structural search: ${byPath.id}`);
+	}
+	return byPath.id;
+}
+
+function innermostEnclosing(
+	ir: FileIr,
+	startLine: number,
+	endLine: number,
+): { name: string; kind: DeclKind; startLine: number } | undefined {
+	let best: Decl | undefined;
+	walkDecls(ir.decls, (decl) => {
+		if (decl.startLine <= startLine && decl.endLine >= endLine) {
+			// Prefer strictly containing named scopes over the match's own span when equal.
+			if (
+				best === undefined ||
+				decl.startLine > best.startLine ||
+				(decl.startLine === best.startLine && decl.endLine < best.endLine)
+			) {
+				best = decl;
+			}
+		}
+	});
+	if (best === undefined) return undefined;
+	// Skip file-level module/package wrappers that don't disambiguate.
+	if (best.kind === "module" || best.kind === "package") return undefined;
+	// If the match is the whole decl body span, scope adds little.
+	if (best.startLine === startLine && best.endLine === endLine) return undefined;
+	return { name: best.name, kind: best.kind, startLine: best.startLine };
+}
+
+function attachEnclosing(
+	path: string,
+	hits: readonly AstSearchHit[],
+	ir: FileIr | undefined,
+	parseDegraded: boolean,
+	disambiguate: boolean,
+): AstSearchMatch[] {
+	return hits.map((hit) => ({
+		...hit,
+		path,
+		parseDegraded,
+		enclosing: disambiguate && ir !== undefined ? innermostEnclosing(ir, hit.startLine, hit.endLine) : undefined,
+	}));
+}
+
+async function searchOneFile(
+	engine: ExploreEngine,
+	absolutePath: string,
+	languageId: string,
+	pattern: string,
+	signal: AbortSignal,
+): Promise<{ matches: AstSearchMatch[]; error: string | undefined }> {
+	signal.throwIfAborted();
+	let source: string;
+	try {
+		const bytes = await readFile(absolutePath);
+		source = bytes.toString("utf8");
+	} catch (error) {
+		return { matches: [], error: pathResolutionError(error, absolutePath).message };
+	}
+	signal.throwIfAborted();
+
+	let hits: AstSearchHit[];
+	try {
+		hits = await engine.searchInSource(languageId, source, pattern);
+	} catch (error) {
+		const message = error instanceof Error ? error.message : String(error);
+		// Invalid pattern is global — rethrow.
+		if (message.startsWith("Invalid pattern:")) throw error;
+		return { matches: [], error: message };
+	}
+
+	let ir: FileIr | undefined;
+	let parseDegraded = false;
+	if (hits.length > 1) {
+		try {
+			const file = await engine.sourceForFile(absolutePath);
+			ir = file.ir;
+			parseDegraded = file.ir.parseDegraded;
+		} catch {
+			// Enclosing scope is best-effort.
+		}
+	} else if (hits.length === 1) {
+		// Still surface parse uncertainty when IR is cheap via cache after sourceForFile.
+		try {
+			const file = await engine.sourceForFile(absolutePath);
+			parseDegraded = file.ir.parseDegraded;
+			// Single hit: only attach enclosing when degraded parse needs trust signal? Spec:
+			// enclosing when disambiguating — single hit does not need it.
+			ir = undefined;
+		} catch {
+			// ignore
+		}
+	}
+
+	return {
+		matches: attachEnclosing(absolutePath, hits, ir, parseDegraded, hits.length > 1),
+		error: undefined,
+	};
+}
+
+/**
+ * Structural pattern search on one file or a language-filtered directory walk.
+ * Matcher and grammar lifecycle live on the engine; this query only walks and shapes.
+ */
+export async function astSearch(
+	engine: ExploreEngine,
+	pathInput: string,
+	pattern: string,
+	language: string | undefined,
+	resultLimit: number,
+	signal: AbortSignal,
+): Promise<AstSearchResult> {
+	signal.throwIfAborted();
+	assertPattern(pattern);
+
+	const absolutePath = resolveExplorePath(engine.cwd, pathInput);
+	let stats;
+	try {
+		stats = await lstat(absolutePath);
+	} catch (error) {
+		throw pathResolutionError(error, pathInput);
+	}
+
+	const isDirectory = stats.isDirectory();
+	const languageId = resolveLanguageId(engine, language, isDirectory ? undefined : absolutePath, isDirectory);
+
+	const matches: AstSearchMatch[] = [];
+	const fileErrors: { path: string; message: string }[] = [];
+	let resultLimitReached = false;
+	let scan: ScanOutcome | undefined;
+
+	if (!isDirectory) {
+		const one = await searchOneFile(engine, absolutePath, languageId, pattern, signal);
+		if (one.error !== undefined) {
+			throw new Error(one.error);
+		}
+		if (one.matches.length > resultLimit) {
+			resultLimitReached = true;
+			matches.push(...one.matches.slice(0, resultLimit));
+		} else {
+			matches.push(...one.matches);
+		}
+		return {
+			matches,
+			resultLimit,
+			resultLimitReached,
+			languageId,
+			scan: undefined,
+			fileErrors,
+		};
+	}
+
+	// Directory: walk supported files; keep only the requested language.
+	const generator = scanSources({
+		engine,
+		cwd: engine.cwd,
+		root: absolutePath,
+		signal,
+		// scanSources already filters to registered languages; further filter below.
+	});
+
+	let step = await generator.next();
+	while (!step.done) {
+		signal.throwIfAborted();
+		const file = step.value;
+		if (file.ir.languageId !== languageId) {
+			step = await generator.next();
+			continue;
+		}
+
+		// Prefer source already loaded by scan (avoids second read for IR).
+		let hits: AstSearchHit[];
+		try {
+			hits = await engine.searchInSource(languageId, file.source, pattern);
+		} catch (error) {
+			const message = error instanceof Error ? error.message : String(error);
+			if (message.startsWith("Invalid pattern:")) throw error;
+			fileErrors.push({ path: file.ir.path, message });
+			step = await generator.next();
+			continue;
+		}
+
+		const room = resultLimit - matches.length;
+		if (room <= 0) {
+			if (hits.length > 0) resultLimitReached = true;
+			// Still drain walk for budget outcome? Stop early for latency.
+			break;
+		}
+
+		const limited = hits.length > room ? hits.slice(0, room) : hits;
+		if (hits.length > room) resultLimitReached = true;
+
+		const withScope = attachEnclosing(
+			file.ir.path,
+			limited,
+			hits.length > 1 ? file.ir : undefined,
+			file.ir.parseDegraded,
+			hits.length > 1,
+		);
+		matches.push(...withScope);
+
+		if (matches.length >= resultLimit) {
+			resultLimitReached = true;
+			break;
+		}
+		step = await generator.next();
+	}
+
+	if (signal.aborted) throw new Error("ast_search cancelled");
+
+	if (step.done) {
+		scan = step.value;
+	} else {
+		scan = {
+			limit: undefined,
+			filesVisited: 0,
+			sourceBytes: 0,
+			elapsedMs: 0,
+			filesEmitted: matches.length > 0 ? 1 : 0,
+		};
+	}
+	if (scan.limit === "cancelled") throw new Error("ast_search cancelled");
+
+	return {
+		matches,
+		resultLimit,
+		resultLimitReached,
+		languageId,
+		scan,
+		fileErrors,
+	};
+}

package/extensions/explore/ast/queries/composite-target.ts

@@ -0,0 +1,77 @@
+import { pathResolutionError, resolveExplorePath } from "../../traverse.ts";
+import type { ExploreEngine } from "../engine.ts";
+import type { Candidate } from "../identity.ts";
+import { resolveTarget } from "../identity.ts";
+import { isCallableLike, isTypeLike, type Decl, type DeclKind, type FileIr } from "../ir.ts";
+
+export type CompositeTarget = {
+	path: string;
+	name: string;
+	qualifiedName: string;
+	kind: DeclKind;
+	startLine: number;
+};
+
+export type CompositeResolved = {
+	scopeDir: string;
+	target: CompositeTarget;
+	decl: Decl;
+	ir: FileIr;
+	source: string;
+	parseDegraded: boolean;
+};
+
+export type CompositeResolveResult =
+	| { kind: "resolved"; value: CompositeResolved }
+	| { kind: "candidates"; candidates: Candidate[] }
+	| { kind: "notFound" }
+	| { kind: "error"; message: string };
+
+/** Resolve one callable/type target under a directory scope for impact/context. */
+export async function resolveCompositeTarget(
+	engine: ExploreEngine,
+	scopePath: string,
+	targetPath: string | undefined,
+	name: string,
+	line: number | undefined,
+	toolName: "impact" | "context",
+	signal: AbortSignal,
+): Promise<CompositeResolveResult> {
+	let scopeDir: string;
+	try {
+		scopeDir = resolveExplorePath(engine.cwd, scopePath);
+	} catch (error) {
+		return { kind: "error", message: pathResolutionError(error, scopePath).message };
+	}
+
+	const resolution = await resolveTarget(engine, scopeDir, { path: targetPath, name, line }, signal);
+	if (resolution.kind === "candidates") return { kind: "candidates", candidates: resolution.candidates };
+	if (resolution.kind === "notFound") return { kind: "notFound" };
+
+	const kind = resolution.decl.kind;
+	if (!isCallableLike(kind) && !isTypeLike(kind)) {
+		const alt = toolName === "impact" ? "deps/reverse_deps/references" : "show/references";
+		return {
+			kind: "error",
+			message: `${toolName} supports callable and type targets, not ${kind}. Use ${alt}.`,
+		};
+	}
+
+	return {
+		kind: "resolved",
+		value: {
+			scopeDir,
+			target: {
+				path: resolution.path,
+				name: resolution.decl.name,
+				qualifiedName: resolution.decl.qualifiedName,
+				kind,
+				startLine: resolution.decl.startLine,
+			},
+			decl: resolution.decl,
+			ir: resolution.ir,
+			source: resolution.source,
+			parseDegraded: resolution.ir.parseDegraded,
+		},
+	};
+}