@xynogen/pix-runtime 0.13.2 → 0.14.0

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@xynogen/pix-runtime",
-	"version": "0.13.2",
+	"version": "0.14.0",
 	"description": "Pix shared runtime — versioned pix.json config, atomic persistence, typed change events",
 	"type": "module",
 	"main": "src/index.ts",
@@ -38,7 +38,8 @@
 		"./os": "./src/os.ts",
 		"./audio": "./src/audio.ts",
 		"./safe-path": "./src/safe-path.ts",
-		"./which": "./src/which.ts"
+		"./which": "./src/which.ts",
+		"./ast": "./src/ast/index.ts"
 	},
 	"keywords": [
 		"pi",
@@ -63,9 +64,18 @@
 	},
 	"peerDependencies": {
 		"@earendil-works/pi-coding-agent": "*",
-		"@earendil-works/pi-tui": "*"
+		"@earendil-works/pi-tui": "*",
+		"@ast-grep/napi": "^0.45.0"
 	},
 	"dependencies": {
 		"xxhash-wasm": "1.1.0"
+	},
+	"peerDependenciesMeta": {
+		"@ast-grep/napi": {
+			"optional": true
+		}
+	},
+	"devDependencies": {
+		"@ast-grep/napi": "^0.45.0"
 	}
 }

package/src/ast/engine.ts

@@ -0,0 +1,219 @@
+/**
+ * engine.ts — the lazy @ast-grep/napi loader and language map.
+ *
+ * The native addon loads on first tool call, never at extension registration.
+ * The addon bundles six grammars: ts, tsx, js, jsx, css, html. We map file
+ * extensions to language aliases. Non-bundled languages use grammars.ts, which
+ * loads cached grammar packages or installs them only after user consent.
+ *
+ * Loader adapted from pi-lens `clients/deps/ast-grep-napi.ts` (MIT). See
+ * packages/pix-diagnostics/LICENSE.pi-lens.
+ *
+ * `@ast-grep/napi` is an optional peer: pix-runtime never forces the native addon on
+ * every install. Packages that parse (pix-astgrep, pix-graph) depend on it; without it
+ * `loadEngine()` returns undefined and callers degrade.
+ * ponytail: resolution assumes npm hoists napi next to pix-runtime (the flat Pi install).
+ */
+
+import { createRequire } from "node:module";
+import { extname } from "node:path";
+import { pathToFileURL } from "node:url";
+
+// The typed shape of the addon accessor we use (parse → root → findAll).
+export interface SgEdit {
+	startPos: number;
+	endPos: number;
+	insertedText: string;
+}
+export interface SgNode {
+	text(): string;
+	kind(): string;
+	range(): { start: SgPos; end: SgPos };
+	findAll(matcher: string | { rule: Record<string, unknown> }): SgNode[];
+	children(): SgNode[];
+	isNamed(): boolean;
+	/** Child stored under a tree-sitter field name (`name`, `body`, `function`, …). */
+	field(name: string): SgNode | null;
+	replace(text: string): SgEdit;
+	commitEdits(edits: SgEdit[]): string;
+	getMatch(name: string): SgNode | null;
+	getMultipleMatches(name: string): SgNode[];
+}
+interface SgPos {
+	line: number;
+	column: number;
+	index: number;
+}
+interface SgRoot {
+	root(): SgNode;
+}
+interface LangApi {
+	parse(source: string): SgRoot;
+}
+interface AstGrepNapi {
+	ts: LangApi;
+	tsx: LangApi;
+	js: LangApi;
+	jsx: LangApi;
+	css: LangApi;
+	html: LangApi;
+	/** Top-level parse for a dynamically registered language, keyed by alias. */
+	parse(lang: string, source: string): SgRoot;
+	registerDynamicLanguage(langs: Record<string, unknown>): void;
+}
+
+const _require = createRequire(import.meta.url);
+let cached: AstGrepNapi | undefined;
+let loadPromise: Promise<AstGrepNapi | undefined> | undefined;
+
+/** Load the native addon once. Returns `undefined` when it cannot load. */
+export async function loadEngine(): Promise<AstGrepNapi | undefined> {
+	if (cached) return cached;
+	if (loadPromise) return loadPromise;
+	loadPromise = (async () => {
+		try {
+			const entry = _require.resolve("@ast-grep/napi");
+			// SAFETY: the addon has no bundled ESM types; the accessors we use
+			// (ts/tsx/js/jsx/css/html .parse) are verified at runtime by tests.
+			const mod = (await import(pathToFileURL(entry).href)) as unknown as AstGrepNapi;
+			cached = mod;
+			return mod;
+		} catch {
+			return undefined;
+		} finally {
+			loadPromise = undefined;
+		}
+	})();
+	return loadPromise;
+}
+
+/** The six grammars the napi addon parses in-process without any package. */
+export const LANGUAGES = ["typescript", "tsx", "javascript", "jsx", "css", "html"] as const;
+export type Language = (typeof LANGUAGES)[number];
+const BUNDLED = new Set<string>(LANGUAGES);
+
+/** True when napi parses this alias in-process (no `@ast-grep/lang-*` needed). */
+export function isBundled(lang: string): lang is Language {
+	return BUNDLED.has(lang);
+}
+
+/**
+ * Supported ast-grep language aliases. The six bundled ones parse in-process;
+ * the rest have published `@ast-grep/lang-*` packages (see grammars.ts).
+ * This intentionally excludes languages without an installable grammar.
+ */
+export const ALL_LANGUAGES = [
+	"typescript",
+	"tsx",
+	"javascript",
+	"jsx",
+	"css",
+	"html",
+	"python",
+	"go",
+	"rust",
+	"java",
+	"c",
+	"cpp",
+	"csharp",
+	"ruby",
+	"php",
+	"swift",
+	"kotlin",
+	"scala",
+	"lua",
+	"elixir",
+	"json",
+	"yaml",
+	"bash",
+	"haskell",
+] as const;
+export type AnyLanguage = (typeof ALL_LANGUAGES)[number];
+
+// File extension → language alias, from the ast-grep language table.
+const EXT_TO_LANG: Record<string, AnyLanguage> = {
+	".ts": "typescript",
+	".mts": "typescript",
+	".cts": "typescript",
+	".tsx": "tsx",
+	".js": "javascript",
+	".mjs": "javascript",
+	".cjs": "javascript",
+	".jsx": "jsx",
+	".css": "css",
+	".html": "html",
+	".htm": "html",
+	".xhtml": "html",
+	".py": "python",
+	".pyi": "python",
+	".go": "go",
+	".rs": "rust",
+	".java": "java",
+	".c": "c",
+	".h": "c",
+	".cc": "cpp",
+	".cpp": "cpp",
+	".cxx": "cpp",
+	".hpp": "cpp",
+	".hh": "cpp",
+	".cu": "cpp",
+	".cs": "csharp",
+	".rb": "ruby",
+	".gemspec": "ruby",
+	".php": "php",
+	".swift": "swift",
+	".kt": "kotlin",
+	".kts": "kotlin",
+	".scala": "scala",
+	".sc": "scala",
+	".sbt": "scala",
+	".lua": "lua",
+	".ex": "elixir",
+	".exs": "elixir",
+	".json": "json",
+	".yml": "yaml",
+	".yaml": "yaml",
+	".sh": "bash",
+	".bash": "bash",
+	".zsh": "bash",
+	".ksh": "bash",
+	".hs": "haskell",
+};
+
+/** Map a file path to any ast-grep language alias, or `undefined`. */
+export function languageForFile(path: string): AnyLanguage | undefined {
+	return EXT_TO_LANG[extname(path).toLowerCase()];
+}
+
+/** Get the addon accessor for a language, or `undefined` when unsupported. */
+export function accessorFor(engine: AstGrepNapi, lang: Language): LangApi {
+	switch (lang) {
+		case "typescript":
+			return engine.ts;
+		case "tsx":
+			return engine.tsx;
+		case "javascript":
+			return engine.js;
+		case "jsx":
+			return engine.jsx;
+		case "css":
+			return engine.css;
+		case "html":
+			return engine.html;
+		default:
+			throw new Error(`unsupported language: ${lang as string}`);
+	}
+}
+
+/** Parse source into a root node for one bundled language. */
+export function parse(engine: AstGrepNapi, lang: Language, source: string): SgNode {
+	return accessorFor(engine, lang).parse(source).root();
+}
+
+/**
+ * Parse source for any language alias, including a dynamically registered one.
+ * The caller must have run `ensureLanguage(lang)` first for non-bundled ones.
+ */
+export function parseDynamic(engine: AstGrepNapi, lang: string, source: string): SgNode {
+	return engine.parse(lang, source).root();
+}

package/src/ast/grammars.ts

@@ -0,0 +1,221 @@
+/**
+ * grammars.ts — lazy, package-owned multi-language grammar loading.
+ *
+ * The napi addon bundles six grammars (see engine.ts). Every other ast-grep
+ * language ships as its own `@ast-grep/lang-<x>` npm package. This module loads
+ * one on demand:
+ *
+ *   1. If napi already parses it (the six), no grammar is needed.
+ *   2. Else try to import `@ast-grep/lang-<x>` from the pix cache, then from
+ *      the resolvable node_modules.
+ *   3. If absent, report `needs-install` with the exact package name. The tool
+ *      asks the user once, then this module installs it INTO the pix cache
+ *      (never the agent, never a global write). A later call finds it cached.
+ *
+ * The cache lives at `<agentDir>/pix-astgrep/grammars`. We install with the
+ * user's package manager into that private prefix, so a read-only global pix
+ * install still works.
+ */
+
+import { existsSync } from "node:fs";
+import { mkdir } from "node:fs/promises";
+import { createRequire } from "node:module";
+import { join } from "node:path";
+import { pathToFileURL } from "node:url";
+import { runTool } from "../exec.ts";
+import { agentDir } from "../paths.ts";
+import { isBundled, loadEngine, parse, parseDynamic, type SgNode } from "./engine.ts";
+
+const require = createRequire(import.meta.url);
+const errorText = (err: unknown): string => (err instanceof Error ? err.message : String(err));
+
+/** Canonical language alias → its `@ast-grep/lang-*` package name. */
+const LANG_PACKAGE: Record<string, string> = {
+	python: "@ast-grep/lang-python",
+	go: "@ast-grep/lang-go",
+	rust: "@ast-grep/lang-rust",
+	java: "@ast-grep/lang-java",
+	c: "@ast-grep/lang-c",
+	cpp: "@ast-grep/lang-cpp",
+	csharp: "@ast-grep/lang-csharp",
+	ruby: "@ast-grep/lang-ruby",
+	php: "@ast-grep/lang-php",
+	swift: "@ast-grep/lang-swift",
+	kotlin: "@ast-grep/lang-kotlin",
+	scala: "@ast-grep/lang-scala",
+	lua: "@ast-grep/lang-lua",
+	elixir: "@ast-grep/lang-elixir",
+	json: "@ast-grep/lang-json",
+	yaml: "@ast-grep/lang-yaml",
+	bash: "@ast-grep/lang-bash",
+	haskell: "@ast-grep/lang-haskell",
+};
+
+/** Languages that need a `@ast-grep/lang-*` package (not the napi six). */
+export function packageForLanguage(lang: string): string | undefined {
+	return LANG_PACKAGE[lang];
+}
+
+/**
+ * Cache root for installed grammars: `<agentDir>/pix-astgrep/grammars`. The path predates the
+ * move into pix-runtime; it stays so grammars users already installed keep working.
+ */
+export function grammarCacheDir(): string {
+	return join(agentDir(), "pix-astgrep", "grammars");
+}
+
+// ponytail: napi registers once per process. Load all cached grammars together;
+// adding a language afterward needs an agent restart, not a silently ignored registration.
+const stateKey = Symbol.for("pix-astgrep.grammars");
+const globals = globalThis as typeof globalThis & {
+	[stateKey]?: { registered: Set<string>; declined: Set<string>; registration?: Promise<void> };
+};
+globals[stateKey] ??= { registered: new Set(), declined: new Set() };
+const state = globals[stateKey];
+const { registered, declined } = state;
+
+export type GrammarResult =
+	| { kind: "ready" }
+	| { kind: "napi" }
+	| { kind: "needs-install"; pkg: string }
+	| { kind: "declined"; pkg: string }
+	| { kind: "unsupported" }
+	| { kind: "error"; message: string };
+
+/** Import a grammar module from the cache prefix, then this package. */
+async function importGrammar(pkg: string): Promise<unknown | undefined> {
+	const cached = join(grammarCacheDir(), "node_modules", pkg);
+	const candidates = existsSync(cached) ? [cached, pkg] : [pkg];
+	for (const spec of candidates) {
+		let entry: string;
+		try {
+			entry = require.resolve(spec);
+		} catch (err) {
+			if ((err as NodeJS.ErrnoException).code === "MODULE_NOT_FOUND") continue;
+			throw err;
+		}
+		const mod = (await import(pathToFileURL(entry).href)) as { default?: unknown };
+		return mod.default ?? mod;
+	}
+	return undefined;
+}
+
+/**
+ * Make a language usable. Returns `ready` when registered, `napi` when the
+ * addon already covers it, `needs-install` with the package to install, or
+ * `unsupported` for an unknown alias.
+ */
+export async function ensureLanguage(lang: string): Promise<GrammarResult> {
+	const engine = await loadEngine();
+	if (!engine) return { kind: "error", message: "ast-grep engine unavailable" };
+
+	// The six bundled grammars never need a package.
+	if (isBundled(lang)) {
+		return { kind: "napi" };
+	}
+	const pkg = packageForLanguage(lang);
+	if (!pkg) return { kind: "unsupported" };
+	if (registered.has(lang)) return { kind: "ready" };
+
+	try {
+		const grammar = await importGrammar(pkg);
+		if (!grammar) return { kind: "needs-install", pkg };
+		state.registration ??= (async () => {
+			const grammars: Record<string, unknown> = {};
+			for (const [alias, name] of Object.entries(LANG_PACKAGE)) {
+				const loaded = await importGrammar(name);
+				if (loaded) grammars[alias] = loaded;
+			}
+			engine.registerDynamicLanguage(grammars);
+			for (const alias of Object.keys(grammars)) registered.add(alias);
+		})();
+		await state.registration;
+		return registered.has(lang)
+			? { kind: "ready" }
+			: {
+					kind: "error",
+					message: `${pkg} is installed, but napi's language registry is already initialized. Restart the agent to load newly installed grammars.`,
+				};
+	} catch (err) {
+		return { kind: "error", message: errorText(err) };
+	}
+}
+
+/**
+ * Install a `@ast-grep/lang-*` package into the pix cache prefix. This runs a
+ * network install — the caller must get user consent first. It never touches a
+ * global path or this package's own node_modules.
+ */
+export async function installGrammar(pkg: string): Promise<{ ok: boolean; message?: string }> {
+	const dir = grammarCacheDir();
+	try {
+		await mkdir(dir, { recursive: true });
+		// A minimal package.json so npm installs into this prefix cleanly.
+		const pkgJson = join(dir, "package.json");
+		if (!existsSync(pkgJson)) {
+			const { writeFile } = await import("node:fs/promises");
+			await writeFile(pkgJson, JSON.stringify({ name: "pix-astgrep-grammars", private: true }));
+		}
+		const result = await runTool("npm", ["install", "--save", "--prefix", dir, pkg], {
+			cwd: dir,
+			timeoutMs: 120_000,
+		});
+		if (result.code !== 0) {
+			return {
+				ok: false,
+				message: result.timedOut
+					? "grammar installation timed out"
+					: result.stderr || `npm exited ${result.code}`,
+			};
+		}
+		return { ok: true };
+	} catch (err) {
+		return { ok: false, message: errorText(err) };
+	}
+}
+
+export type ParseOutcome = { kind: "ok"; root: SgNode } | GrammarResult;
+
+/** A one-time consent callback: return true to install `pkg`. */
+export type ConsentFn = (pkg: string) => Promise<boolean>;
+
+/**
+ * Parse one file's source for any language alias, installing a missing grammar
+ * with consent. The bundled six always parse. A non-bundled language is loaded
+ * from the pix cache. When absent, `consent(pkg)` decides whether to install it
+ * INTO the cache (once per language). A declined language is remembered so the
+ * caller is not asked again this session.
+ */
+export async function parseWithGrammar(
+	lang: string,
+	source: string,
+	consent?: ConsentFn,
+): Promise<ParseOutcome> {
+	const engine = await loadEngine();
+	if (!engine) return { kind: "error", message: "ast-grep engine unavailable" };
+
+	try {
+		if (isBundled(lang)) return { kind: "ok", root: parse(engine, lang, source) };
+		if (declined.has(lang)) {
+			return { kind: "declined", pkg: packageForLanguage(lang) ?? lang };
+		}
+
+		let ready = await ensureLanguage(lang);
+		if (ready.kind === "needs-install") {
+			if (!consent) return ready;
+			if (!(await consent(ready.pkg))) {
+				declined.add(lang);
+				return { kind: "declined", pkg: ready.pkg };
+			}
+			const installed = await installGrammar(ready.pkg);
+			if (!installed.ok) {
+				return { kind: "error", message: installed.message ?? `failed to install ${ready.pkg}` };
+			}
+			ready = await ensureLanguage(lang);
+		}
+		if (ready.kind !== "ready") return ready;
+		return { kind: "ok", root: parseDynamic(engine, lang, source) };
+	} catch (err) {
+		return { kind: "error", message: errorText(err) };
+	}
+}

package/src/ast/index.ts

@@ -0,0 +1,28 @@
+/**
+ * pix-runtime/ast — shared ast-grep parsing: the lazy napi loader, the file → language map,
+ * and consent-gated `@ast-grep/lang-*` grammar loading. UI-free; the install prompt lives in
+ * `@xynogen/pix-pretty/grammar-consent`.
+ */
+export {
+	ALL_LANGUAGES,
+	type AnyLanguage,
+	isBundled,
+	LANGUAGES,
+	type Language,
+	languageForFile,
+	loadEngine,
+	parse,
+	parseDynamic,
+	type SgEdit,
+	type SgNode,
+} from "./engine.ts";
+export {
+	type ConsentFn,
+	ensureLanguage,
+	type GrammarResult,
+	grammarCacheDir,
+	installGrammar,
+	type ParseOutcome,
+	packageForLanguage,
+	parseWithGrammar,
+} from "./grammars.ts";