@code-yeongyu/senpi-codemode 2026.9.23 → 2026.9.24-2

package/src/tool/display-js.ts

@@ -0,0 +1,100 @@
+import { isRecord, parseFile, type SourceNode } from "./display-js-ast.ts";
+import { finishLayout } from "./display-js-layout.ts";
+import { maskSource, restoreComments, restoreValues } from "./display-js-mask.ts";
+
+// Bun.Transpiler is a transpiler, not a formatter: printed directly it rewrites literals ("a\nb"
+// becomes a multi-line template, 0xff becomes 255, emoji become escapes), folds `typeof
+// undefined`, and drops comments and directives. It only ever lays out a masked cell here, and the
+// original text is substituted back; a cell that cannot be restored exactly is shown as sent.
+const INDENT = "  ";
+
+export type JsPrinter = (code: string) => string;
+
+let bunPrinter: JsPrinter | null | undefined;
+
+function resolveBunPrinter(): JsPrinter | null {
+	if (typeof process.versions.bun !== "string") return null;
+	const bun: unknown = Reflect.get(globalThis, "Bun");
+	if (!isRecord(bun) || typeof bun.Transpiler !== "function") return null;
+	const transpiler: unknown = Reflect.construct(bun.Transpiler, [
+		{ loader: "js", trimUnusedImports: false, deadCodeElimination: false, inline: false },
+	]);
+	if (!isRecord(transpiler) || typeof transpiler.transformSync !== "function") return null;
+	const transformSync = transpiler.transformSync;
+	return (code) => {
+		const output: unknown = Reflect.apply(transformSync, transpiler, [code]);
+		if (typeof output !== "string") throw new TypeError("Bun.Transpiler.transformSync returned a non-string");
+		return output;
+	};
+}
+
+/** The built-in Bun printer when this renderer runs on Bun; null on Node. */
+export function bunJsPrinter(): JsPrinter | null {
+	if (bunPrinter === undefined) bunPrinter = resolveBunPrinter();
+	return bunPrinter;
+}
+
+function layout(masked: string, prefix: string, print: JsPrinter): string | undefined {
+	try {
+		return print(masked).trim();
+	} catch {
+		// Module syntax only parses unwrapped; the kernel's top-level `return` only inside a function.
+	}
+	const header = `async function ${prefix}cell() {`;
+	let lines: string[];
+	try {
+		lines = print(`${header}\n${masked}\n}`).trim().split("\n");
+	} catch {
+		return undefined;
+	}
+	if (lines[0] !== header || lines.at(-1) !== "}") return undefined;
+	return lines
+		.slice(1, -1)
+		.map((line) => (line.startsWith(INDENT) ? line.slice(INDENT.length) : line))
+		.join("\n");
+}
+
+function sameTokens(left: string, right: string): boolean {
+	const strip = (text: string) => text.replace(/[\s;,()]/gu, "");
+	return strip(left) === strip(right);
+}
+
+const POSITION_KEYS = new Set([
+	"start",
+	"end",
+	"loc",
+	"range",
+	"extra",
+	"leadingComments",
+	"trailingComments",
+	"innerComments",
+]);
+
+// Positions, raw spellings, and parenthesization flags aside, the preview must be the same program:
+// the character guard alone cannot see a dropped `(`, which can change how an expression groups.
+function programSignature(file: SourceNode): string {
+	return JSON.stringify({ program: file.record.program, comments: file.record.comments }, (key, value: unknown) => {
+		if (POSITION_KEYS.has(key)) return undefined;
+		if (typeof value === "bigint") return `${value}n`;
+		// A stray `;` in a statement list is a no-op the printer drops.
+		return Array.isArray(value) ? value.filter((item) => !isRecord(item) || item.type !== "EmptyStatement") : value;
+	});
+}
+
+function sameProgram(original: SourceNode, preview: string): boolean {
+	const reparsed = parseFile(preview);
+	return reparsed !== undefined && programSignature(reparsed) === programSignature(original);
+}
+
+/** The printer's layout of `code` with its literal, name, and comment text intact, or undefined. */
+export function prettifyJs(code: string, print: JsPrinter): string | undefined {
+	const file = parseFile(code);
+	if (file === undefined) return undefined;
+	const mask = maskSource(code, file);
+	if (mask === undefined) return undefined;
+	const printed = layout(mask.masked, mask.prefix, print);
+	const withComments = printed === undefined ? undefined : restoreComments(printed, mask.comments);
+	const restored = withComments === undefined ? undefined : restoreValues(withComments, mask);
+	const finished = restored === undefined ? undefined : finishLayout(restored);
+	return finished !== undefined && sameTokens(finished, code) && sameProgram(file, finished) ? finished : undefined;
+}

package/src/tool/display-python-script.ts

@@ -0,0 +1,127 @@
+// Runs in the user's own Python interpreter: `python -c <script> <ruff timeout s> <strategies...>` with the cell on
+// stdin, printing {"via", "code"} as JSON. A strategy's output is kept only when it parses to the same
+// AST as the source and keeps every string, number, and comment token's text (senpi#2076).
+export const PYTHON_FORMATTER_SCRIPT = [
+	"import ast, io, json, re, shutil, subprocess, sys, tokenize",
+	"",
+	"def emit(via, code):",
+	"    sys.stdout.write(json.dumps({'via': via, 'code': code}))",
+	"    sys.exit(0)",
+	"",
+	"RUFF_TIMEOUT = float(sys.argv[1])",
+	"FLAGS = ast.PyCF_ONLY_AST | getattr(ast, 'PyCF_ALLOW_TOP_LEVEL_AWAIT', 0)",
+	"",
+	"def parse(text):",
+	"    return compile(text, '<cell>', 'exec', flags=FLAGS, dont_inherit=True)",
+	"",
+	"def literal_texts(text):",
+	"    kinds = {tokenize.STRING, tokenize.NUMBER, tokenize.COMMENT} | token_kinds('FSTRING_MIDDLE', 'TSTRING_MIDDLE')",
+	"    return [t.string for t in tokenize.generate_tokens(io.StringIO(text).readline) if t.type in kinds]",
+	"",
+	"def equivalent(source, formatted):",
+	"    try:",
+	"        same_tree = ast.dump(parse(source)) == ast.dump(parse(formatted))",
+	"        return same_tree and literal_texts(source) == literal_texts(formatted)",
+	"    except (SyntaxError, ValueError, tokenize.TokenError):",
+	"        return False",
+	"",
+	"def ruff_binary():",
+	"    found = shutil.which('ruff')",
+	"    if found:",
+	"        return found",
+	"    try:",
+	"        from ruff.__main__ import find_ruff_bin",
+	"        return find_ruff_bin()",
+	"    except Exception:",
+	"        return None",
+	"",
+	"def with_ruff(source):",
+	"    binary = ruff_binary()",
+	"    if not binary:",
+	"        return None",
+	"    args = [binary, 'format', '--quiet', '--no-cache', '--config', \"format.quote-style = 'preserve'\",",
+	"            '--stdin-filename', 'cell.py', '-']",
+	"    try:",
+	"        done = subprocess.run(args, input=source, capture_output=True, text=True, timeout=RUFF_TIMEOUT)",
+	"    except Exception:",
+	"        return None",
+	"    return done.stdout if done.returncode == 0 else None",
+	"",
+	"def with_black(source):",
+	"    try:",
+	"        import black",
+	"        return black.format_str(source, mode=black.Mode(string_normalization=False))",
+	"    except Exception:",
+	"        return None",
+	"",
+	"def token_kinds(*names):",
+	"    return {getattr(tokenize, name) for name in names if hasattr(tokenize, name)}",
+	"",
+	"def literal_spans(source):",
+	"    tokens = list(tokenize.generate_tokens(io.StringIO(source).readline))",
+	"    starts = [0]",
+	"    for line in source.splitlines(keepends=True):",
+	"        starts.append(starts[-1] + len(line))",
+	"    offset = lambda position: starts[position[0] - 1] + position[1]",
+	"    opening, closing = token_kinds('FSTRING_START', 'TSTRING_START'), token_kinds('FSTRING_END', 'TSTRING_END')",
+	"    spans, depth = [], 0",
+	"    for token in tokens:",
+	"        if token.type == tokenize.COMMENT:",
+	"            return None",
+	"        if depth:",
+	"            depth += 1 if token.type in opening else -1 if token.type in closing else 0",
+	"            if depth == 0:",
+	"                spans[-1][1] = offset(token.end)",
+	"            continue",
+	"        is_string = token.type == tokenize.STRING or token.type in opening",
+	"        if not is_string and token.type != tokenize.NUMBER:",
+	"            continue",
+	"        start, end = offset(token.start), offset(token.end)",
+	"        joined = spans and spans[-1][2] and is_string and not re.sub(r'[\\s\\\\]', '', source[spans[-1][1]:start])",
+	"        if joined:",
+	"            spans[-1][1] = end",
+	"        else:",
+	"            spans.append([start, end, is_string])",
+	"        if token.type in opening:",
+	"            depth = 1",
+	"    return spans",
+	"",
+	"def with_ast(source):",
+	"    if not hasattr(ast, 'unparse'):",
+	"        return None",
+	"    try:",
+	"        spans = literal_spans(source)",
+	"    except (tokenize.TokenError, SyntaxError):",
+	"        return None",
+	"    if spans is None:",
+	"        return None",
+	"    prefix = '_display'",
+	"    while prefix in source:",
+	"        prefix += '_'",
+	"    raws, masked, cursor = {}, [], 0",
+	"    for index, (start, end, _) in enumerate(spans):",
+	"        name = prefix + str(index)",
+	"        raws[name] = source[start:end]",
+	"        masked.append(source[cursor:start] + name)",
+	"        cursor = end",
+	"    masked.append(source[cursor:])",
+	"    try:",
+	"        tree = parse(''.join(masked))",
+	"    except (SyntaxError, ValueError):",
+	"        return None",
+	"    seen = []",
+	"    def restore(match):",
+	"        seen.append(match.group(0))",
+	"        return raws.get(match.group(0), match.group(0))",
+	"    text = re.sub(r'\\b' + re.escape(prefix) + r'\\d+\\b', restore, ast.unparse(tree))",
+	"    return text if sorted(seen) == sorted(raws) else None",
+	"",
+	"STRATEGIES = {'ruff': with_ruff, 'black': with_black, 'ast': with_ast}",
+	"source = sys.stdin.read()",
+	"for name in sys.argv[2:]:",
+	"    strategy = STRATEGIES.get(name)",
+	"    formatted = strategy(source) if strategy else None",
+	"    if formatted is not None and equivalent(source, formatted):",
+	"        emit(name, formatted.rstrip('\\n'))",
+	"emit('none', source)",
+].join("\n");

package/src/tool/display-python.ts

@@ -0,0 +1,150 @@
+import { spawn } from "node:child_process";
+import { createInterpreterDetector } from "../interpreters/detect.ts";
+import { PYTHON_FORMATTER_SCRIPT } from "./display-python-script.ts";
+
+export type PythonStrategy = "ruff" | "black" | "ast";
+export type PythonInvocation = { readonly command: string; readonly args: readonly string[] };
+
+export interface PythonFormatOptions {
+	readonly strategies: readonly PythonStrategy[];
+	readonly timeoutMs: number;
+	readonly env?: NodeJS.ProcessEnv;
+}
+
+export interface PythonDisplayOptions extends PythonFormatOptions {
+	readonly interpreter: () => Promise<PythonInvocation | null>;
+}
+
+export interface PythonDisplay {
+	/**
+	 * The formatted cell once it is known, otherwise the cell as sent. With `onFormatted`, a missing
+	 * result is formatted in the background and `onFormatted` runs when a formatted cell is ready.
+	 */
+	display(code: string, onFormatted?: () => void): string;
+}
+
+const DEFAULT_STRATEGIES: readonly PythonStrategy[] = ["ruff", "black", "ast"];
+const FORMAT_TIMEOUT_MS = 5_000;
+const MAX_CONCURRENT_FORMATTERS = 2;
+const CACHE_LIMIT = 64;
+
+function formattedCode(stdout: string): string | undefined {
+	try {
+		const parsed: unknown = JSON.parse(stdout);
+		if (parsed === null || typeof parsed !== "object" || !("code" in parsed)) return undefined;
+		return typeof parsed.code === "string" ? parsed.code : undefined;
+	} catch (error) {
+		if (error instanceof SyntaxError) return undefined;
+		throw error;
+	}
+}
+
+// ruff runs inside the formatter process, so it gets most of the caller's budget and never outlives it.
+function ruffTimeoutSeconds(timeoutMs: number): string {
+	return String(Math.max(1, timeoutMs - 1_000) / 1_000);
+}
+
+export function formatPythonCell(
+	code: string,
+	invocation: PythonInvocation,
+	options: PythonFormatOptions,
+): Promise<string | undefined> {
+	return new Promise((resolve) => {
+		const child = spawn(
+			invocation.command,
+			[
+				...invocation.args,
+				"-c",
+				PYTHON_FORMATTER_SCRIPT,
+				ruffTimeoutSeconds(options.timeoutMs),
+				...options.strategies,
+			],
+			{
+				stdio: ["pipe", "pipe", "ignore"],
+				timeout: options.timeoutMs,
+				windowsHide: true,
+				...(options.env === undefined ? {} : { env: options.env }),
+			},
+		);
+		let stdout = "";
+		child.stdout.setEncoding("utf8");
+		child.stdout.on("data", (chunk: string) => {
+			stdout += chunk;
+		});
+		child.on("error", () => resolve(undefined));
+		child.on("close", (exitCode) => resolve(exitCode === 0 ? formattedCode(stdout) : undefined));
+		child.stdin.on("error", () => resolve(undefined));
+		child.stdin.end(code);
+	});
+}
+
+export function createPythonDisplay(options: PythonDisplayOptions): PythonDisplay {
+	const settled = new Map<string, string>();
+	const listeners = new Map<string, Set<() => void>>();
+	const queue: string[] = [];
+	let running = 0;
+
+	const remember = (code: string, formatted: string) => {
+		if (settled.size >= CACHE_LIMIT) settled.delete(settled.keys().next().value ?? "");
+		settled.set(code, formatted);
+	};
+
+	const format = async (code: string): Promise<string | undefined> => {
+		const invocation = await options.interpreter();
+		return invocation === null ? undefined : formatPythonCell(code, invocation, options);
+	};
+
+	const pump = () => {
+		while (running < MAX_CONCURRENT_FORMATTERS) {
+			const code = queue.shift();
+			if (code === undefined) return;
+			running += 1;
+			void format(code)
+				.catch(() => undefined)
+				.then((formatted) => {
+					remember(code, formatted ?? code);
+					const waiting = listeners.get(code);
+					listeners.delete(code);
+					if (formatted !== undefined && formatted !== code) for (const listener of waiting ?? []) listener();
+				})
+				.finally(() => {
+					running -= 1;
+					pump();
+				});
+		}
+	};
+
+	return {
+		display(code, onFormatted) {
+			const known = settled.get(code);
+			if (known !== undefined || onFormatted === undefined) return known ?? code;
+			const waiting = listeners.get(code);
+			if (waiting !== undefined) {
+				waiting.add(onFormatted);
+				return code;
+			}
+			listeners.set(code, new Set([onFormatted]));
+			queue.push(code);
+			pump();
+			return code;
+		},
+	};
+}
+
+async function detectedPython(): Promise<PythonInvocation | null> {
+	const detected = await createInterpreterDetector().detect("py");
+	if (!detected.ok) return null;
+	const [command, ...args] = detected.path.split(" ");
+	return command === undefined ? null : { command, args };
+}
+
+let userPython: Promise<PythonInvocation | null> | undefined;
+
+export const pythonDisplay: PythonDisplay = createPythonDisplay({
+	interpreter: () => {
+		userPython ??= detectedPython();
+		return userPython;
+	},
+	strategies: DEFAULT_STRATEGIES,
+	timeoutMs: FORMAT_TIMEOUT_MS,
+});

package/src/tool/eval-request.ts

@@ -1,22 +1,26 @@
 import type { ExtensionContext } from "@code-yeongyu/senpi";
-import { EVAL_SUMMARY_MAX_LENGTH, type EvalControlInput, type EvalToolInput, type EvalToolRequest } from "./types.ts";
+import {
+	type EvalControlInput,
+	type EvalLanguage,
+	type EvalToolInput,
+	type EvalToolRequest,
+	evalLanguageOrder,
+} from "./types.ts";
 
 const NON_INTERACTIVE_MODES = new Set(["print", "json"]);
 
-const ELLIPSIS = "...";
-
-// Harness-side enforcement of the schema maxLength: the tool advertises the limit, but an
-// over-limit value is force-truncated here (prepareArguments runs before schema validation)
-// instead of failing the call.
-export function clampEvalSummary(value: unknown): string | undefined {
+// A summary is one line of any length: whitespace (including newlines) collapses to single
+// spaces, and a blank value counts as absent. The TUI bounds how much of it a collapsed block shows.
+export function normalizeEvalSummary(value: unknown): string | undefined {
 	if (typeof value !== "string") return undefined;
 	const normalized = value.trim().replace(/\s+/gu, " ");
-	if (normalized.length === 0) return undefined;
-	if (normalized.length <= EVAL_SUMMARY_MAX_LENGTH) return normalized;
-	return `${normalized.slice(0, EVAL_SUMMARY_MAX_LENGTH - ELLIPSIS.length)}${ELLIPSIS}`;
+	return normalized.length === 0 ? undefined : normalized;
 }
 
-export function parseEvalRequest(params: unknown): EvalToolRequest {
+export function parseEvalRequest(
+	params: unknown,
+	enabledLanguages: readonly EvalLanguage[] = evalLanguageOrder,
+): EvalToolRequest {
 	if (!isRecord(params)) throw new TypeError("eval parameters must be an object");
 	if (params.action === "list") return { action: "list" };
 	if (params.action === "peek" || params.action === "stop") {
@@ -26,12 +30,13 @@ export function parseEvalRequest(params: unknown): EvalToolRequest {
 	}
 	if (params.action !== undefined && params.action !== "run")
 		throw new TypeError(`Unknown eval action "${String(params.action)}"`);
-	if (!isEvalLanguage(params.language)) throw new TypeError("eval run requires language");
-	if (typeof params.code !== "string") throw new TypeError("eval run requires code");
-	const summary = clampEvalSummary(params.summary);
+	if (!isEvalLanguage(params.language)) throw new TypeError(evalRunRequiresLanguageMessage(enabledLanguages));
+	if (typeof params.code !== "string")
+		throw new TypeError("eval run requires code — the cell body to execute, verbatim");
+	const summary = normalizeEvalSummary(params.summary);
 	if (summary === undefined)
 		throw new TypeError(
-			"eval run requires summary — one line in the user's language: what this cell does and for what purpose",
+			"eval run requires summary — one line in the user's language: what you are working on and for what purpose",
 		);
 	if (params.on_timeout !== undefined && params.on_timeout !== "detach" && params.on_timeout !== "error")
 		throw new TypeError(`Unknown eval on_timeout value "${String(params.on_timeout)}"`);
@@ -62,3 +67,7 @@ function isRecord(value: unknown): value is Record<string, unknown> {
 function isEvalLanguage(value: unknown): value is EvalToolInput["language"] {
 	return value === "py" || value === "js" || value === "rb" || value === "jl";
 }
+
+function evalRunRequiresLanguageMessage(languages: readonly EvalLanguage[]): string {
+	return `eval run requires language — one of ${languages.map((language) => `"${language}"`).join(", ")}`;
+}

package/src/tool/eval-tool.ts

@@ -8,7 +8,7 @@ import {
 import { buildEvalPrompt } from "../prompt/eval-prompt.ts";
 import { EvalDetachedCellManager } from "./detached-cell-manager.ts";
 import { executeEvalControl } from "./detached-eval-result.ts";
-import { clampEvalSummary, isEvalControlRequest, parseEvalRequest } from "./eval-request.ts";
+import { isEvalControlRequest, normalizeEvalSummary, parseEvalRequest } from "./eval-request.ts";
 import type { CreateEvalToolOptions } from "./eval-tool-options.ts";
 import { runEvalCell } from "./run-eval-cell.ts";
 import {
@@ -73,7 +73,7 @@ export function createEvalTool(options: CreateEvalToolOptions) {
 	async function execute(
 		...[toolCallId, params, signal, onUpdate, ctx]: EvalExecuteArgs<EvalToolRequest>
 	): Promise<AgentToolResult<EvalResultDetails>> {
-		const request = parseEvalRequest(params);
+		const request = parseEvalRequest(params, languages);
 		if (isEvalControlRequest(request)) return await executeEvalControl(cellManager, request);
 		if (options.proxyExecutor) return await options.proxyExecutor(request, signal);
 		if (!languages.includes(request.language))
@@ -109,7 +109,7 @@ export function createEvalTool(options: CreateEvalToolOptions) {
 			const record = args as Record<string, unknown>;
 			if (record.action === "peek" || record.action === "stop" || record.action === "list")
 				return args as EvalToolRequest;
-			const summary = clampEvalSummary(record.summary);
+			const summary = normalizeEvalSummary(record.summary);
 			if (summary === undefined) delete record.summary;
 			else record.summary = summary;
 			return args as EvalToolRequest;