@code-yeongyu/senpi-codemode 2026.10.6 → 2026.10.8

package/src/environments/editable-check.ts

@@ -0,0 +1,116 @@
+import { lstat, readdir, readFile, realpath } from "node:fs/promises";
+import { join, relative, resolve, sep } from "node:path";
+import { EnvironmentError } from "./py-installer.ts";
+
+/**
+ * Refuses a staged revision that links to code outside itself: a PEP 660 editable install
+ * (`direct_url.json` with `dir_info.editable`), a legacy `setup.py develop` (`.egg-link`), a `.pth` path
+ * line that resolves outside the revision, or a symlink anywhere in it that does. `.pth` lines starting
+ * with `import` are code that runs at startup, like any installed package's code, and are not inspected.
+ * A refused revision is never published.
+ */
+export async function assertNoEditableInstalls(staging: string): Promise<void> {
+	const root = await realpath(staging).catch(() => resolve(staging));
+	let entries: string[];
+	try {
+		entries = await readdir(staging);
+	} catch {
+		return;
+	}
+	for (const entry of entries) {
+		const path = join(staging, entry);
+		if (entry.endsWith(".egg-link"))
+			refuse(entry.replace(/\.egg-link$/u, ""), "a legacy editable install (.egg-link)");
+		if (
+			entry.endsWith(".dist-info") &&
+			isEditable(await readFile(join(path, "direct_url.json"), "utf8").catch(() => ""))
+		) {
+			refuse(entry.replace(/\.dist-info$/u, ""), "an editable install");
+		}
+		if (entry.endsWith(".pth")) {
+			for (const line of (await readFile(path, "utf8").catch(() => "")).split(/\r?\n/u)) {
+				const text = line.trim();
+				if (text === "" || text.startsWith("#") || text.startsWith("import ")) continue;
+				if (!(await inside(root, resolve(staging, text))))
+					refuse(entry, `a path outside the environment (${text})`);
+			}
+		}
+	}
+	await refuseOutsideLinks(root, staging);
+}
+
+/** Every symlink in the revision, at any depth, must resolve inside it; links are checked, not followed. */
+async function refuseOutsideLinks(root: string, dir: string): Promise<void> {
+	for (const entry of await readdir(dir)) {
+		const path = join(dir, entry);
+		const info = await lstat(path);
+		if (info.isSymbolicLink()) {
+			if (!(await inside(root, path))) refuse(relative(root, path), "a link that points outside the environment");
+		} else if (info.isDirectory()) {
+			await refuseOutsideLinks(root, path);
+		}
+	}
+}
+
+/**
+ * pip reports what it installed; each of those distributions must be in the revision. A configuration
+ * that redirects pip elsewhere (a target, root or prefix from any config file or variable, on any pip
+ * version) leaves them missing here, so the install fails instead of publishing an empty revision.
+ */
+export async function assertInstalledInRevision(stdout: string, staging: string): Promise<void> {
+	const line = stdout.split("\n").find((entry) => entry.startsWith("Successfully installed "));
+	if (line === undefined) return;
+	const present = new Set((await readdir(staging).catch(() => [] as string[])).flatMap(installedName));
+	for (const token of line.slice("Successfully installed ".length).trim().split(/\s+/u)) {
+		const name = token.slice(0, token.lastIndexOf("-"));
+		if (!present.has(canonicalName(name))) {
+			throw new EnvironmentError(
+				"environment_install_failed",
+				`${name} was installed outside the session's environment (pip's configuration redirected it); nothing was published`,
+			);
+		}
+	}
+}
+
+/**
+ * The project a metadata entry records: `name-version.dist-info` (wheels), or `name-version[-pyX.Y].egg-info`
+ * (a setup.py project pip <= 23.0 installs without the wheel package). Both escape `-` in the name to `_`.
+ */
+function installedName(entry: string): string[] {
+	if (entry.endsWith(".dist-info")) {
+		const stem = entry.slice(0, -".dist-info".length);
+		return [canonicalName(stem.slice(0, stem.lastIndexOf("-")))];
+	}
+	if (entry.endsWith(".egg-info")) return [canonicalName(entry.split("-")[0] ?? "")];
+	return [];
+}
+
+function canonicalName(name: string): string {
+	return name.toLowerCase().replace(/[-_.]+/gu, "_");
+}
+
+async function inside(root: string, path: string): Promise<boolean> {
+	const real = await realpath(path).catch(() => resolve(path));
+	const rel = relative(root, real);
+	return (
+		rel === "" || (!rel.startsWith(`..${sep}`) && rel !== ".." && !rel.startsWith(sep) && !/^[A-Za-z]:/u.test(rel))
+	);
+}
+
+function refuse(name: string, what: string): never {
+	throw new EnvironmentError(
+		"environment_install_failed",
+		`${name} was installed as ${what}; packages must live in the session's environment root, so editable and linked installs are not allowed`,
+	);
+}
+
+function isEditable(text: string): boolean {
+	try {
+		const value: unknown = JSON.parse(text);
+		if (typeof value !== "object" || value === null || !("dir_info" in value)) return false;
+		const dirInfo = value.dir_info;
+		return typeof dirInfo === "object" && dirInfo !== null && "editable" in dirInfo && dirInfo.editable === true;
+	} catch {
+		return false;
+	}
+}

package/src/environments/install-lock.ts

@@ -0,0 +1,223 @@
+import { randomUUID } from "node:crypto";
+import { watch } from "node:fs";
+import { link, readdir, readFile, rm, stat, writeFile } from "node:fs/promises";
+import { hostname } from "node:os";
+import { join } from "node:path";
+
+const LOCK_FILE = ".install.lock";
+const REAP_PREFIX = ".install.lock.reap.";
+const RECHECK_MS = 500;
+// An unparseable lock can only come from a crash in an older version's create-then-write; it is stale once old.
+const UNREADABLE_STALE_MS = 5_000;
+const WAIT_NOTICE_MS = 30_000;
+
+type Holder = { readonly pid: number; readonly host: string; readonly nonce: string };
+
+export type LockWaitNotice = {
+	readonly holder: { readonly pid: number; readonly host: string } | undefined;
+	readonly waitedMs: number;
+};
+
+/**
+ * Serialises installs into one environment root across sessions and processes.
+ *
+ * The lock file is published complete (written to a unique temp file, then `link`ed into place, which fails
+ * if a lock exists), so it is never seen empty. A lock whose holder is a dead process on this host, or
+ * that is unreadable and old, is stale. Each lock incarnation has its own key (its holder's nonce), and
+ * only the waiter that exclusively creates the reap claim for that key may remove it, after re-reading the
+ * lock under the claim and finding the same incarnation. A stale incarnation's holder is dead and nobody
+ * else may remove it, so it can't change between that re-read and the removal: a live lock is never
+ * deleted. A claimer that dies passes the claim on to the next claim in the chain.
+ */
+export async function withRootLock<T>(
+	base: string,
+	fn: () => Promise<T>,
+	signal?: AbortSignal,
+	onWait?: (notice: LockWaitNotice) => void,
+): Promise<T> {
+	const path = join(base, LOCK_FILE);
+	const mine: Holder = { pid: process.pid, host: hostname(), nonce: randomUUID() };
+	const started = Date.now();
+	let noticed = false;
+	for (;;) {
+		signal?.throwIfAborted();
+		if (await tryCreate(base, path, mine)) break;
+		if (await takeOverIfStale(base, path)) continue;
+		if (!noticed && Date.now() - started >= WAIT_NOTICE_MS) {
+			noticed = true;
+			const holder = await readHolder(path);
+			onWait?.({ holder: holder && { pid: holder.pid, host: holder.host }, waitedMs: Date.now() - started });
+		}
+		await waitForRelease(base, path, signal);
+	}
+	try {
+		return await fn();
+	} finally {
+		const holder = await readHolder(path);
+		if (holder?.nonce === mine.nonce) await rm(path, { force: true });
+	}
+}
+
+async function tryCreate(base: string, path: string, holder: Holder): Promise<boolean> {
+	const temp = join(base, `.install.lock.${holder.nonce}`);
+	await writeFile(temp, JSON.stringify(holder), { mode: 0o600 });
+	try {
+		await link(temp, path);
+		return true;
+	} catch (error) {
+		if (errorCode(error) === "EEXIST") return false;
+		throw error;
+	} finally {
+		await rm(temp, { force: true });
+	}
+}
+
+async function takeOverIfStale(base: string, path: string): Promise<boolean> {
+	const key = await staleKey(path);
+	if (key === undefined) return false;
+	const claim = await claimReap(base, key);
+	if (claim === undefined) return false;
+	try {
+		// Under the claim nobody else may remove this incarnation, and its holder is dead, so it is still the
+		// lock judged stale exactly when it still has the same key.
+		if ((await staleKey(path)) !== key) return false;
+		await rm(path, { force: true });
+		await removeDeadClaims(base, key);
+		return true;
+	} finally {
+		await rm(claim, { force: true });
+	}
+}
+
+/** Once the incarnation is gone, its claims only matter to waiters that will find a different lock. */
+async function removeDeadClaims(base: string, key: string): Promise<void> {
+	const prefix = `${REAP_PREFIX}${key}.`;
+	for (const name of (await readdir(base)).filter((entry) => entry.startsWith(prefix))) {
+		const holder = await readHolder(join(base, name));
+		if (holder !== undefined && holder.host === hostname() && !isAlive(holder.pid))
+			await rm(join(base, name), { force: true });
+	}
+}
+
+/**
+ * Creates the next free reap claim for one lock incarnation: `<key>.0`, or the claim after the last one
+ * whose holder died. Returns undefined when a live waiter already holds it.
+ */
+async function claimReap(base: string, key: string): Promise<string | undefined> {
+	for (let depth = 0; ; depth++) {
+		const claim = join(base, `${REAP_PREFIX}${key}.${depth}`);
+		if (await tryCreate(base, claim, { pid: process.pid, host: hostname(), nonce: randomUUID() })) return claim;
+		const holder = await readHolder(claim);
+		// Gone again (its holder finished): the caller re-reads the lock and retries.
+		if (holder === undefined) return undefined;
+		if (holder.host !== hostname() || isAlive(holder.pid)) return undefined;
+	}
+}
+
+async function reapInProgress(base: string, key: string): Promise<boolean> {
+	const prefix = `${REAP_PREFIX}${key}.`;
+	let names: string[];
+	try {
+		names = (await readdir(base)).filter((name) => name.startsWith(prefix));
+	} catch {
+		return false;
+	}
+	for (const name of names) {
+		const holder = await readHolder(join(base, name));
+		if (holder !== undefined && holder.host === hostname() && isAlive(holder.pid)) return true;
+	}
+	return false;
+}
+
+/** The incarnation key of a stale lock (its holder's nonce, or its inode when unreadable), else undefined. */
+async function staleKey(path: string): Promise<string | undefined> {
+	let content: string;
+	try {
+		content = await readFile(path, "utf8");
+	} catch {
+		return undefined;
+	}
+	const holder = parseHolder(content);
+	if (holder !== undefined && (holder.host !== hostname() || isAlive(holder.pid))) return undefined;
+	if (holder !== undefined && /^[\w-]+$/u.test(holder.nonce)) return holder.nonce;
+	try {
+		const info = await stat(path);
+		// A dead holder's lock without a usable nonce, or an unreadable old one, is keyed by its inode.
+		return holder !== undefined || Date.now() - info.mtimeMs > UNREADABLE_STALE_MS ? `ino${info.ino}` : undefined;
+	} catch {
+		return undefined;
+	}
+}
+
+async function readHolder(path: string): Promise<Holder | undefined> {
+	try {
+		return parseHolder(await readFile(path, "utf8"));
+	} catch {
+		return undefined;
+	}
+}
+
+function parseHolder(content: string): Holder | undefined {
+	let value: unknown;
+	try {
+		value = JSON.parse(content);
+	} catch {
+		return undefined;
+	}
+	if (typeof value !== "object" || value === null) return undefined;
+	const pid = "pid" in value ? value.pid : undefined;
+	const host = "host" in value ? value.host : undefined;
+	const nonce = "nonce" in value ? value.nonce : undefined;
+	return typeof pid === "number" && typeof host === "string"
+		? { pid, host, nonce: typeof nonce === "string" ? nonce : "" }
+		: undefined;
+}
+
+function isAlive(pid: number): boolean {
+	try {
+		process.kill(pid, 0);
+		return true;
+	} catch (error) {
+		return errorCode(error) !== "ESRCH";
+	}
+}
+
+function errorCode(error: unknown): string | undefined {
+	return error instanceof Error && "code" in error && typeof error.code === "string" ? error.code : undefined;
+}
+
+/** Wakes when the lock file goes away; the periodic recheck covers dropped file-watch events and staleness. */
+function waitForRelease(base: string, path: string, signal?: AbortSignal): Promise<void> {
+	return new Promise((resolve, reject) => {
+		const watcher = watch(base, () => check());
+		const timer = setInterval(() => check(), RECHECK_MS);
+		const onAbort = () => done(() => reject(signal?.reason));
+		signal?.addEventListener("abort", onAbort, { once: true });
+		let settled = false;
+		function done(settle: () => void): void {
+			if (settled) return;
+			settled = true;
+			watcher.close();
+			clearInterval(timer);
+			signal?.removeEventListener("abort", onAbort);
+			settle();
+		}
+		function check(): void {
+			void stat(path).then(
+				// A stale lock is worth retrying only when no live waiter is already reaping it; otherwise wait.
+				() =>
+					staleKey(path).then(
+						async (key) => key !== undefined && !(await reapInProgress(base, key)) && done(resolve),
+					),
+				() => done(resolve),
+			);
+		}
+		// An abort during the claim inspection that led here fired before this listener existed.
+		if (signal?.aborted) {
+			onAbort();
+			return;
+		}
+		// The holder may have released between the failed create and the watcher starting; no event would follow.
+		check();
+	});
+}

package/src/environments/py-environment.ts

@@ -0,0 +1,98 @@
+import { execFile } from "node:child_process";
+import { createHash } from "node:crypto";
+import { join } from "node:path";
+import { promisify } from "node:util";
+import { assertInstalledInRevision, assertNoEditableInstalls } from "./editable-check.ts";
+import { parsePipRequirements, runPipInstall } from "./py-installer.ts";
+import { publishNextRevision, type Revision } from "./revision-store.ts";
+
+const execFileAsync = promisify(execFile);
+
+export type EnvironmentMode = "managed" | "project";
+
+export interface InstallReceipt {
+	readonly manager: "pip";
+	readonly mode: EnvironmentMode;
+	readonly root: string;
+	readonly revision: number;
+	readonly requested: readonly string[];
+	readonly resolved: readonly string[];
+	readonly changed: boolean;
+}
+
+export function managedEnvironmentBase(artifactsDir: string, language: "py", abiTag: string): string {
+	return join(artifactsDir, "environments", language, abiTag);
+}
+
+export function projectPythonBase(cwd: string): string {
+	return join(cwd, ".senpi", "python-packages");
+}
+
+export async function pythonAbiTag(interpreter: string): Promise<string> {
+	const { stdout } = await execFileAsync(interpreter, [
+		"-c",
+		"import sys, sysconfig; print(sys.implementation.cache_tag, sysconfig.get_platform(), sys.executable)",
+	]);
+	return createHash("sha256").update(stdout.trim()).digest("hex").slice(0, 12);
+}
+
+export async function installPythonPackages(input: {
+	readonly base: string;
+	readonly mode: EnvironmentMode;
+	readonly interpreter: string;
+	readonly requirements: string;
+	readonly cwd: string;
+	readonly signal: AbortSignal;
+	readonly onOutput?: (stream: "stdout" | "stderr", data: string) => void;
+}): Promise<InstallReceipt> {
+	const requested = parsePipRequirements(input.requirements);
+	let stdout = "";
+	const { revision } = await publishNextRevision(
+		input.base,
+		async (staging) => {
+			await runPipInstall({
+				interpreter: input.interpreter,
+				root: staging,
+				args: requested,
+				cwd: input.cwd,
+				signal: input.signal,
+				onOutput: (stream, data) => {
+					if (stream === "stdout") stdout += data;
+					input.onOutput?.(stream, data);
+				},
+			});
+			await assertInstalledInRevision(stdout, staging);
+			await assertNoEditableInstalls(staging);
+		},
+		input.signal,
+		({ holder, waitedMs }) =>
+			input.onOutput?.(
+				"stderr",
+				`[senpi] waiting ${Math.round(waitedMs / 1000)}s for another install into this environment${holder === undefined ? "" : ` (pid ${holder.pid} on ${holder.host})`}\n`,
+			),
+	);
+	const resolved = installedDistributions(stdout);
+	return receipt(input.mode, revision, requested, resolved);
+}
+
+function receipt(
+	mode: EnvironmentMode,
+	revision: Revision,
+	requested: readonly string[],
+	resolved: readonly string[],
+): InstallReceipt {
+	return {
+		manager: "pip",
+		mode,
+		root: revision.dir,
+		revision: revision.number,
+		requested,
+		resolved,
+		changed: resolved.length > 0,
+	};
+}
+
+function installedDistributions(stdout: string): string[] {
+	const line = stdout.split("\n").find((entry) => entry.startsWith("Successfully installed "));
+	return line === undefined ? [] : line.slice("Successfully installed ".length).trim().split(/\s+/);
+}

package/src/environments/py-installer.ts

@@ -0,0 +1,187 @@
+import { spawn } from "node:child_process";
+import { devNull } from "node:os";
+import { terminateProcessTrees } from "../kernels/js/process-tree-host.ts";
+
+export type EnvironmentErrorCode =
+	| "environment_install_failed"
+	| "environment_install_timeout"
+	| "environment_install_cancelled"
+	| "environment_installer_unavailable"
+	| "environment_resolution_conflict"
+	| "environment_language_mismatch";
+
+export class EnvironmentError extends Error {
+	readonly name = "EnvironmentError";
+	readonly code: EnvironmentErrorCode;
+
+	constructor(code: EnvironmentErrorCode, message: string) {
+		super(`${code}: ${message}`);
+		this.code = code;
+	}
+}
+
+// pip accepts any unambiguous prefix of a long option and grouped short flags, so a denylist of destination
+// options can't be complete. Only these options are accepted, each spelled out in full.
+const ALLOWED_FLAGS = new Set([
+	"--upgrade",
+	"--no-deps",
+	"--pre",
+	"--force-reinstall",
+	"--no-index",
+	"--no-cache-dir",
+	"--only-binary",
+	"--no-binary",
+	"--prefer-binary",
+	"--index-url",
+	"--extra-index-url",
+	"--find-links",
+	"--constraint",
+	"--requirement",
+	"--quiet",
+	"--verbose",
+]);
+const FLAGS_WITH_VALUE = new Set([
+	"--only-binary",
+	"--no-binary",
+	"--index-url",
+	"--extra-index-url",
+	"--find-links",
+	"--constraint",
+	"--requirement",
+]);
+const SHORT_ALIASES: Readonly<Record<string, string>> = {
+	"-U": "--upgrade",
+	"-i": "--index-url",
+	"-f": "--find-links",
+	"-c": "--constraint",
+	"-r": "--requirement",
+	"-q": "--quiet",
+	"-v": "--verbose",
+};
+const STDERR_TAIL_BYTES = 4_096;
+const PIP_TREE_GRACE_MS = 2_000;
+
+export function parsePipRequirements(text: string): string[] {
+	const args = text
+		.trim()
+		.split(/\s+/)
+		.filter((arg) => arg !== "");
+	const command = args[0] === "install" ? args.slice(1) : undefined;
+	if (command === undefined) {
+		throw new EnvironmentError("environment_install_failed", "only `%pip install <requirements>` is supported");
+	}
+	if (command.length === 0) throw new EnvironmentError("environment_install_failed", "name at least one requirement");
+	const normalized: string[] = [];
+	for (let index = 0; index < command.length; index++) {
+		const arg = command[index] ?? "";
+		if (!arg.startsWith("-")) {
+			normalized.push(arg);
+			continue;
+		}
+		const [spelled = arg, attached] = arg.split(/=(.*)/su, 2);
+		const flag = SHORT_ALIASES[spelled] ?? spelled;
+		if (!ALLOWED_FLAGS.has(flag)) {
+			throw new EnvironmentError(
+				"environment_install_failed",
+				`${spelled} is not allowed: %pip accepts ${[...ALLOWED_FLAGS].join(", ")} (each spelled out in full); packages always install into the session's environment root`,
+			);
+		}
+		if (!FLAGS_WITH_VALUE.has(flag)) {
+			if (attached !== undefined)
+				throw new EnvironmentError("environment_install_failed", `${spelled} takes no value`);
+			normalized.push(flag);
+			continue;
+		}
+		const value = attached ?? command[++index];
+		if (value === undefined || value === "" || value.startsWith("-")) {
+			throw new EnvironmentError("environment_install_failed", `${spelled} needs a value`);
+		}
+		normalized.push(`${flag}=${value}`);
+	}
+	const named = normalized.some((arg) => !arg.startsWith("-") || arg.startsWith("--requirement="));
+	if (!named) throw new EnvironmentError("environment_install_failed", "name at least one requirement");
+	return normalized;
+}
+
+/**
+ * pip's environment with every PIP_* variable removed and its config file pointed at nothing: `--isolated`
+ * alone still honours PIP_CONFIG_FILE, and a configured target, root or prefix would install outside the revision.
+ */
+export function isolatedPipEnv(): NodeJS.ProcessEnv {
+	const env: NodeJS.ProcessEnv = {};
+	for (const [key, value] of Object.entries(process.env)) {
+		if (!key.toUpperCase().startsWith("PIP_")) env[key] = value;
+	}
+	// pip skips every config file when PIP_CONFIG_FILE equals Python's os.devnull: "nul" on Windows, not Node's "\\\\.\\nul".
+	return { ...env, PIP_CONFIG_FILE: process.platform === "win32" ? "nul" : devNull, PYTHONNOUSERSITE: "1" };
+}
+
+export function runPipInstall(input: {
+	readonly interpreter: string;
+	readonly root: string;
+	readonly args: readonly string[];
+	readonly cwd: string;
+	readonly signal: AbortSignal;
+	readonly onOutput?: (stream: "stdout" | "stderr", data: string) => void;
+}): Promise<void> {
+	const argv = [
+		"-m",
+		"pip",
+		"install",
+		// Ignore pip's config files and PIP_* variables: a configured `root` or `prefix` would write outside the revision.
+		"--isolated",
+		"--disable-pip-version-check",
+		"--no-input",
+		"--target",
+		input.root,
+		...input.args,
+	];
+	return new Promise((resolve, reject) => {
+		if (input.signal.aborted) {
+			reject(new EnvironmentError("environment_install_cancelled", "the install was cancelled before it started"));
+			return;
+		}
+		const child = spawn(input.interpreter, argv, {
+			cwd: input.cwd,
+			stdio: ["ignore", "pipe", "pipe"],
+			env: isolatedPipEnv(),
+		});
+		let stderrTail = "";
+		child.stdout.setEncoding("utf8").on("data", (data: string) => input.onOutput?.("stdout", data));
+		child.stderr.setEncoding("utf8").on("data", (data: string) => {
+			stderrTail = (stderrTail + data).slice(-STDERR_TAIL_BYTES);
+			input.onOutput?.("stderr", data);
+		});
+		// pip's build backends run in their own subprocesses; stopping only pip would leave them writing.
+		const onAbort = () => {
+			const pid = child.pid;
+			if (pid === undefined) child.kill("SIGKILL");
+			else void terminateProcessTrees([pid], { graceMs: PIP_TREE_GRACE_MS }).catch(() => child.kill("SIGKILL"));
+		};
+		input.signal.addEventListener("abort", onAbort, { once: true });
+		child.once("error", (error) => {
+			input.signal.removeEventListener("abort", onAbort);
+			reject(
+				new EnvironmentError("environment_installer_unavailable", `${input.interpreter} -m pip: ${error.message}`),
+			);
+		});
+		child.once("close", (code, signal) => {
+			input.signal.removeEventListener("abort", onAbort);
+			if (input.signal.aborted) {
+				reject(new EnvironmentError("environment_install_cancelled", "the install was cancelled; pip was stopped"));
+			} else if (code === 0) resolve();
+			else if (/No module named pip/.test(stderrTail)) {
+				reject(new EnvironmentError("environment_installer_unavailable", `${input.interpreter} has no pip module`));
+			} else if (/ResolutionImpossible|conflicting dependencies/.test(stderrTail)) {
+				reject(new EnvironmentError("environment_resolution_conflict", stderrTail.trim()));
+			} else {
+				reject(
+					new EnvironmentError(
+						"environment_install_failed",
+						stderrTail.trim() || `pip exited with ${code ?? signal}`,
+					),
+				);
+			}
+		});
+	});
+}