@alisio/core 0.5.2 → 0.6.0

package/dist/forge/python/pyodide-backend.js

@@ -0,0 +1,289 @@
+/**
+ * `PyodideBackend` (Python spec §4.3, §8): Pyodide in a child `node` started with the Permission
+ * Model. `isolation: "wasm-process"`: the file system is limited to the Pyodide folder and the job (the scratchpad as a link-free copy),
+ * there are no child processes, workers or addons; the network is NOT restricted on Node 22/24
+ * (the self-probe says what this machine does) and Python reaches the runner's JavaScript through
+ * the FFI. Memory is bounded by an RSS watchdog (soft), time by the kernel's deadline plus a startup
+ * margin, and termination is SIGKILL of the child, confirmed by its exit (PD13).
+ */
+import { createHash } from "node:crypto";
+import { mkdir, readFile, realpath, rename, rm, stat, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { killProcessTree, runProcess } from "../../runtime/process.js";
+import { classifyModules, scanImports } from "./import-scan.js";
+import { inspectPyodide, pyodideDir, verifyPyodideFiles, } from "./pyodide-install.js";
+import { PYODIDE_MANIFEST, PYODIDE_PROFILE_ORDER, } from "./pyodide-manifest.js";
+import { PYODIDE_RUNNER_SOURCE } from "./pyodide-runner-source.js";
+import { readRssMb } from "./rss.js";
+import { copyScratchNoFollow } from "./scratch-copy.js";
+import { settle } from "./settle.js";
+import { childEnv, currentNodeEnvironment, permissionArgs, probeNetwork, resolveNode, } from "./wasm-node.js";
+/** How often the watchdog samples the child (measured in P0: up to one tick of overshoot). */
+const SAMPLE_MS = 250;
+/** How often the parent looks for the runner's ready file. */
+const READY_POLL_MS = 50;
+const PROFILE_FIX = (profile) => `alisio analysis setup --wasm${profile === "science" ? " --profile science" : ""}`;
+/** Keys needed to load `wanted` (and what they depend on), in the manifest's load order. */
+export function loadPlan(manifest, profile, wanted) {
+    const needed = new Set();
+    const add = (key) => {
+        if (needed.has(key))
+            return;
+        const pkg = manifest.packages[key];
+        if (!pkg)
+            return;
+        needed.add(key);
+        for (const dep of pkg.depends)
+            add(dep);
+    };
+    for (const key of wanted)
+        add(key);
+    const ordered = manifest.profiles[profile].filter((key) => needed.has(key));
+    const packages = [];
+    const wheels = [];
+    const files = [];
+    for (const key of ordered) {
+        const pkg = manifest.packages[key];
+        files.push(pkg);
+        if (pkg.source === "pyodide")
+            packages.push(key);
+        else
+            wheels.push(pkg.name);
+    }
+    return { packages, wheels, files };
+}
+export class PyodideBackend {
+    id = "wasm";
+    #deps;
+    #manifest;
+    /** The profile found installed by the last `check()`; `supports` is synchronous and uses it. */
+    #installed;
+    constructor(deps) {
+        this.#deps = deps;
+        this.#manifest = deps.manifest ?? PYODIDE_MANIFEST;
+    }
+    get #dir() {
+        return pyodideDir(this.#deps.stateRoot, this.#manifest);
+    }
+    #node() {
+        return resolveNode(this.#deps.settings().nodePath, (this.#deps.environment ?? currentNodeEnvironment)());
+    }
+    async check(_signal) {
+        const node = await this.#node();
+        if (!node.ok) {
+            this.#installed = undefined;
+            return { ready: false, reason: node.reason, fix: "" };
+        }
+        const installation = await inspectPyodide(this.#dir, this.#manifest);
+        this.#installed = installation.ok ? installation.profile : undefined;
+        return installation.ok ? { ready: true } : { ready: false, reason: installation.reason };
+    }
+    /** Static facts plus the observed network answer; `ready` is the live state of `check()`. */
+    async describe() {
+        const state = await this.check();
+        let networkDeniedByOS = false;
+        if (state.ready) {
+            const node = await this.#node();
+            if (node.ok) {
+                const dir = this.#dir;
+                networkDeniedByOS = (await probeNetwork(node, permissionArgs({ read: [dir], write: [] })))
+                    .denied;
+            }
+        }
+        return {
+            id: "python-wasm",
+            language: "python",
+            version: `pyodide-${this.#manifest.version} (python ${this.#manifest.python})`,
+            isolation: "wasm-process",
+            // Registered once: readiness is checked on every call (like the other backends).
+            ready: true,
+            ...(state.ready ? {} : { reason: state.reason }),
+            capabilities: {
+                cooperativeInterrupt: false,
+                hardTermination: true,
+                hardMemoryLimit: false,
+                networkDeniedByOS,
+                toolCalls: false,
+            },
+            profiles: this.#installed
+                ? PYODIDE_PROFILE_ORDER.slice(0, PYODIDE_PROFILE_ORDER.indexOf(this.#installed) + 1).map(String)
+                : [],
+        };
+    }
+    supports(requirements) {
+        const { maxInputBytes } = this.#deps.settings();
+        if (requirements.inputBytes > maxInputBytes)
+            return {
+                ok: false,
+                code: "INPUT_TOO_LARGE",
+                reason: `the inputs are ${Math.ceil(requirements.inputBytes / (1024 * 1024))} MB, over analysis.wasm.maxInputBytes (${Math.floor(maxInputBytes / (1024 * 1024))} MB)`,
+            };
+        const found = classifyModules(requirements.importedModules);
+        if (found.unknown.length)
+            return {
+                ok: false,
+                code: "PACKAGE_UNAVAILABLE",
+                reason: `module${found.unknown.length === 1 ? "" : "s"} ${found.unknown.join(", ")} ${found.unknown.length === 1 ? "is" : "are"} not in the wasm profiles`,
+            };
+        const wanted = [found.profile, ...requirements.extras].reduce((top, p) => (PYODIDE_PROFILE_ORDER.indexOf(p) > PYODIDE_PROFILE_ORDER.indexOf(top) ? p : top), "base");
+        if (this.#installed === undefined)
+            return {
+                ok: false,
+                code: "NOT_READY",
+                reason: "Pyodide is not installed",
+                fix: PROFILE_FIX("analysis"),
+            };
+        if (PYODIDE_PROFILE_ORDER.indexOf(wanted) > PYODIDE_PROFILE_ORDER.indexOf(this.#installed))
+            return {
+                ok: false,
+                code: "PROFILE_UNSUPPORTED",
+                reason: `this script needs the ${wanted} profile and ${this.#installed} is installed`,
+                fix: PROFILE_FIX(wanted),
+            };
+        return { ok: true };
+    }
+    async launch(job, sink) {
+        const settings = this.#deps.settings();
+        const node = await this.#node();
+        if (!node.ok)
+            throw new Error(`runtime_unavailable: ${node.reason}`);
+        const installation = await inspectPyodide(this.#dir, this.#manifest);
+        if (!installation.ok)
+            throw new Error(`runtime_unavailable: ${installation.reason}`);
+        // What this script loads: its imports, nothing of the profile it does not use.
+        const source = await readFile(job.paths.main, "utf8");
+        const wantedPackages = classifyModules(scanImports(source)).packages;
+        const plan = loadPlan(this.#manifest, installation.profile, wantedPackages);
+        const mismatch = await verifyPyodideFiles(installation.dir, [
+            ...this.#manifest.core,
+            ...plan.files,
+        ]);
+        if (mismatch)
+            throw new Error(`runtime_unavailable: Pyodide file ${mismatch} does not match its pinned SHA-256; run "alisio analysis setup --wasm" again`);
+        // The runner is written next to the distribution so the restricted child may read it.
+        const runnerName = `runner-${createHash("sha256").update(PYODIDE_RUNNER_SOURCE).digest("hex").slice(0, 12)}.mjs`;
+        const runner = join(installation.dir, runnerName);
+        if ((await readFile(runner, "utf8").catch(() => undefined)) !== PYODIDE_RUNNER_SOURCE) {
+            const temporary = `${runner}.${process.pid}.tmp`;
+            await writeFile(temporary, PYODIDE_RUNNER_SOURCE);
+            await rename(temporary, runner);
+        }
+        const real = (path) => realpath(path);
+        // The Permission Model follows links that already exist, so the real scratchpad is never
+        // granted: the child gets a copy made here without following any link (M1).
+        const scratchCopy = join(job.paths.root, "scratch-copy");
+        let scratch;
+        if (job.scratch) {
+            const source = await real(job.scratch).catch(() => undefined);
+            if (source) {
+                await rm(scratchCopy, { recursive: true, force: true });
+                await copyScratchNoFollow(source, scratchCopy);
+                scratch = await real(scratchCopy);
+            }
+        }
+        const [dist, script, input, work, staging] = await Promise.all([
+            real(installation.dir),
+            real(job.paths.script),
+            real(job.paths.input),
+            real(job.paths.work),
+            real(job.paths.staging),
+        ]);
+        await mkdir(staging, { recursive: true });
+        const spec = {
+            dist,
+            packages: plan.packages,
+            wheels: plan.wheels,
+            job: { script, input, work, out: staging, ...(scratch ? { scratch } : {}) },
+        };
+        const args = [
+            ...permissionArgs({
+                read: [dist, script, input, work, staging, ...(scratch ? [scratch] : [])],
+                write: [work, staging],
+            }),
+            join(dist, runnerName),
+            JSON.stringify(spec),
+        ];
+        const stop = new AbortController();
+        const memory = {
+            peakMb: 0,
+        };
+        const runtime = {
+            mode: "wasm",
+            pyodide: this.#manifest.version,
+            python: this.#manifest.python,
+            node: node.version,
+            profile: installation.profile,
+            extras: [],
+            packages: [...plan.packages, ...plan.wheels],
+            memoryMb: settings.memoryMb,
+        };
+        const spawnedAt = Date.now();
+        // The runner leaves a file when Pyodide and the packages are loaded: its modification time
+        // ends the startup, whenever the parent happens to look.
+        const readyFile = join(work, ".forge", "wasm-ready");
+        const readStartup = async () => {
+            try {
+                const info = await stat(readyFile);
+                runtime.startupMs ??= Math.max(0, Math.round(info.mtimeMs - spawnedAt));
+                return true;
+            }
+            catch {
+                return false;
+            }
+        };
+        const watchReady = setInterval(() => {
+            void readStartup().then((found) => found && clearInterval(watchReady));
+        }, READY_POLL_MS);
+        watchReady.unref?.();
+        let sampler;
+        const read = this.#deps.rss ?? readRssMb;
+        const sample = async (pid) => {
+            const rss = await read(pid);
+            if (rss === undefined)
+                return;
+            memory.peakMb = Math.max(memory.peakMb, rss);
+            runtime.peakRssMb = Math.round(memory.peakMb);
+            if (rss > settings.memoryMb && !memory.hit) {
+                memory.hit = {
+                    code: "MEMORY_LIMIT",
+                    message: `the wasm runtime used ${Math.round(rss)} MB, over analysis.wasm.memoryMb (${settings.memoryMb} MB)`,
+                };
+                killProcessTree(pid);
+            }
+        };
+        const result = runProcess(node.path, args, {
+            cwd: work,
+            signal: AbortSignal.any([job.signal, stop.signal]),
+            timeoutMs: job.backstopMs,
+            maxBytes: job.maxBytes,
+            onOverflow: "truncate",
+            logFiles: job.logFiles,
+            maxLogBytes: job.maxLogBytes,
+            // Kill, don't ask: Pyodide has no cooperative interrupt (PD13).
+            killGraceMs: 1,
+            env: childEnv({ ...job.env, ALISIO_EXECUTION_ID: job.executionId }),
+            onSpawn: (pid) => {
+                sampler = setInterval(() => void sample(pid), this.#deps.sampleMs ?? SAMPLE_MS);
+                sampler.unref?.();
+            },
+            onData: sink.onData,
+        }).finally(async () => {
+            clearInterval(sampler);
+            clearInterval(watchReady);
+            await readStartup();
+            if (scratch)
+                await rm(scratchCopy, { recursive: true, force: true }).catch(() => undefined);
+        });
+        return {
+            result,
+            stop,
+            runtime,
+            limit: () => memory.hit,
+        };
+    }
+    async terminate(process, _reason) {
+        const started = Date.now();
+        process.stop.abort();
+        return settle(process, "kill", started);
+    }
+}

package/dist/forge/python/pyodide-install.d.ts

@@ -0,0 +1,41 @@
+import { type PyodideFile, type PyodideManifest, type PyodideProfile } from "./pyodide-manifest.ts";
+/** `<state root>/analysis/pyodide/<version>`. */
+export declare const pyodideDir: (stateRoot: string, manifest?: PyodideManifest) => string;
+/** The files of a profile: the runtime plus the wheels of its closure, in load order. */
+export declare function profileFiles(manifest: PyodideManifest, profile: PyodideProfile): Array<PyodideFile & {
+    key?: string;
+}>;
+export declare const downloadSize: (manifest: PyodideManifest, profile: PyodideProfile) => number;
+export type PyodideInspection = {
+    ok: true;
+    dir: string;
+    profile: PyodideProfile;
+} | {
+    ok: false;
+    reason: string;
+};
+/** Is the distribution there, complete for what it claims, and of this manifest's version? */
+export declare function inspectPyodide(dir: string, manifest?: PyodideManifest): Promise<PyodideInspection>;
+/** Hashes the given files of an installation; returns the first that does not match its pin. */
+export declare function verifyPyodideFiles(dir: string, files: readonly PyodideFile[]): Promise<string | undefined>;
+export interface InstallPyodideOptions {
+    /** `<state root>/analysis/pyodide/<version>`. */
+    dir: string;
+    profile: PyodideProfile;
+    manifest?: PyodideManifest;
+    /** Replaceable in tests; the global `fetch` otherwise. */
+    fetch?: typeof fetch;
+    signal?: AbortSignal;
+    onProgress?: (event: {
+        file: string;
+        index: number;
+        total: number;
+    }) => void;
+}
+/** A download whose bytes are not the pinned ones. Nothing from it is kept. */
+export declare class PyodideIntegrityError extends Error {
+    readonly file: string;
+    constructor(file: string, message: string);
+}
+/** Downloads, verifies and installs the profile. Throws on any failure, leaving no partial files. */
+export declare function installPyodide(options: InstallPyodideOptions): Promise<PyodideInspection>;

package/dist/forge/python/pyodide-install.js

@@ -0,0 +1,167 @@
+/**
+ * The local Pyodide distribution (Python spec §8.1, PD7). `installPyodide` is the ONLY code that
+ * touches the network for the `wasm` backend (`alisio analysis setup --wasm`): it downloads the
+ * files of a profile into a staging folder, verifies the SHA-256 and the size of every one and only
+ * then moves them into `<analysis dir>/pyodide/<version>/` and writes the readiness marker. Any
+ * mismatch deletes what this run downloaded and throws; a distribution that was complete before
+ * stays as it was. `inspectPyodide` is the cheap check the backend does before every run.
+ * It never downloads and never reads the hash of a file (`verifyPyodideFiles` does that).
+ */
+import { createHash, randomBytes } from "node:crypto";
+import { createReadStream } from "node:fs";
+import { mkdir, open, readdir, readFile, rename, rm, stat, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { PYODIDE_MANIFEST, PYODIDE_PROFILE_ORDER, } from "./pyodide-manifest.js";
+const MARKER = "ready.json";
+/** Staging folders are `.download-<random>`: every run has its own, so runs never touch each other. */
+const STAGING = ".download";
+const STALE_STAGING_MS = 60 * 60 * 1000;
+/** `<state root>/analysis/pyodide/<version>`. */
+export const pyodideDir = (stateRoot, manifest = PYODIDE_MANIFEST) => join(stateRoot, "analysis", "pyodide", manifest.version);
+/** The files of a profile: the runtime plus the wheels of its closure, in load order. */
+export function profileFiles(manifest, profile) {
+    return [
+        ...manifest.core,
+        ...manifest.profiles[profile].map((key) => {
+            const pkg = manifest.packages[key];
+            if (!pkg)
+                throw new Error(`manifest lists ${key} in a profile but has no file for it`);
+            return { ...pkg, key };
+        }),
+    ];
+}
+export const downloadSize = (manifest, profile) => profileFiles(manifest, profile).reduce((sum, f) => sum + f.size, 0);
+const rank = (profile) => PYODIDE_PROFILE_ORDER.indexOf(profile);
+/** Is the distribution there, complete for what it claims, and of this manifest's version? */
+export async function inspectPyodide(dir, manifest = PYODIDE_MANIFEST) {
+    let marker;
+    try {
+        marker = JSON.parse(await readFile(join(dir, MARKER), "utf8"));
+    }
+    catch {
+        return { ok: false, reason: "Pyodide is not installed" };
+    }
+    if (marker.version !== manifest.version || !(marker.profile in manifest.profiles))
+        return { ok: false, reason: `the installed Pyodide is not ${manifest.version}` };
+    for (const file of profileFiles(manifest, marker.profile)) {
+        const info = await stat(join(dir, file.name)).catch(() => undefined);
+        if (!info?.isFile() || info.size !== file.size)
+            return { ok: false, reason: `Pyodide file ${file.name} is missing or damaged` };
+    }
+    return { ok: true, dir, profile: marker.profile };
+}
+async function sha256OfFile(path) {
+    const hash = createHash("sha256");
+    for await (const chunk of createReadStream(path))
+        hash.update(chunk);
+    return hash.digest("hex");
+}
+/** Hashes the given files of an installation; returns the first that does not match its pin. */
+export async function verifyPyodideFiles(dir, files) {
+    for (const file of files) {
+        const actual = await sha256OfFile(join(dir, file.name)).catch(() => undefined);
+        if (actual !== file.sha256)
+            return file.name;
+    }
+    return undefined;
+}
+/** A download whose bytes are not the pinned ones. Nothing from it is kept. */
+export class PyodideIntegrityError extends Error {
+    file;
+    constructor(file, message) {
+        super(message);
+        this.name = "PyodideIntegrityError";
+        this.file = file;
+    }
+}
+async function download(file, target, fetcher, signal) {
+    const response = await fetcher(file.url, { ...(signal ? { signal } : {}) });
+    if (!response.ok || !response.body)
+        throw new Error(`download of ${file.name} failed: HTTP ${response.status} from ${file.url}`);
+    const hash = createHash("sha256");
+    let size = 0;
+    const handle = await open(target, "w", 0o600);
+    try {
+        for await (const chunk of response.body) {
+            size += chunk.byteLength;
+            // A server that sends more than was pinned is stopped before it fills the disk.
+            if (size > file.size)
+                throw new PyodideIntegrityError(file.name, `${file.name} is larger than the pinned ${file.size} bytes`);
+            hash.update(chunk);
+            await handle.write(chunk);
+        }
+    }
+    finally {
+        await handle.close();
+    }
+    if (size !== file.size)
+        throw new PyodideIntegrityError(file.name, `${file.name} is ${size} bytes, the pin says ${file.size}`);
+    if (hash.digest("hex") !== file.sha256)
+        throw new PyodideIntegrityError(file.name, `${file.name} does not match its pinned SHA-256`);
+}
+/** Downloads, verifies and installs the profile. Throws on any failure, leaving no partial files. */
+export async function installPyodide(options) {
+    const manifest = options.manifest ?? PYODIDE_MANIFEST;
+    const fetcher = options.fetch ?? fetch;
+    const { dir } = options;
+    const readMarker = () => readFile(join(dir, MARKER), "utf8")
+        .then((text) => JSON.parse(text))
+        .catch(() => undefined);
+    const previous = await readMarker();
+    await mkdir(dir, { recursive: true });
+    // Leftovers of a run that was killed, long gone by now; a live run's folder is far younger.
+    for (const name of await readdir(dir).catch(() => [])) {
+        if (!name.startsWith(`${STAGING}-`))
+            continue;
+        const info = await stat(join(dir, name)).catch(() => undefined);
+        if (info && Date.now() - info.mtimeMs > STALE_STAGING_MS)
+            await rm(join(dir, name), { recursive: true, force: true });
+    }
+    const staging = join(dir, `${STAGING}-${randomBytes(6).toString("hex")}`);
+    await mkdir(staging, { recursive: true, mode: 0o700 });
+    const files = profileFiles(manifest, options.profile);
+    try {
+        let index = 0;
+        for (const file of files) {
+            options.signal?.throwIfAborted();
+            index++;
+            // A file that an earlier install put there and still hashes right is kept.
+            const present = await sha256OfFile(join(dir, file.name)).catch(() => undefined);
+            if (present === file.sha256)
+                continue;
+            options.onProgress?.({ file: file.name, index, total: files.length });
+            await download(file, join(staging, file.name), fetcher, options.signal);
+        }
+        for (const file of files) {
+            const staged = join(staging, file.name);
+            if (await stat(staged).catch(() => undefined))
+                await rename(staged, join(dir, file.name));
+        }
+        // Never downgrade a larger installation that is still there (read again: another run may
+        // have finished a larger one while this one downloaded).
+        const current = await readMarker();
+        const keep = current &&
+            current.version === manifest.version &&
+            rank(current.profile) > rank(options.profile)
+            ? current.profile
+            : options.profile;
+        const marker = {
+            version: manifest.version,
+            profile: keep,
+            files: Object.fromEntries(profileFiles(manifest, keep).map((f) => [f.name, f.size])),
+        };
+        const temporaryMarker = join(dir, `${MARKER}.${randomBytes(6).toString("hex")}.tmp`);
+        await writeFile(temporaryMarker, `${JSON.stringify(marker)}\n`);
+        await rename(temporaryMarker, join(dir, MARKER));
+        return { ok: true, dir, profile: keep };
+    }
+    catch (error) {
+        // Nothing of this run stays; a distribution that was complete before is untouched.
+        if (!previous && !(await readMarker()))
+            await rm(dir, { recursive: true, force: true });
+        throw error;
+    }
+    finally {
+        await rm(staging, { recursive: true, force: true });
+    }
+}

package/dist/forge/python/pyodide-manifest.d.ts

@@ -0,0 +1,42 @@
+/**
+ * Pinned Pyodide distribution for the `wasm` Python backend (Python spec §8.1, PD7). A TypeScript
+ * constant, not JSON, because `tsc` does not copy JSON into `dist`. Every file carries its
+ * SHA-256 and size; `alisio analysis setup --wasm` is the only code that fetches them.
+ *
+ * - `pyodide` files come from the official jsDelivr mirror of the 314.0.7 release; the hashes of
+ *   the wheels are the ones of `pyodide-lock.json`, and the core files were checked against the
+ *   mirror (Phase P0, 2026-10-12).
+ * - `pypi` files are pure wheels that the Pyodide lock does not ship; their hashes are the ones
+ *   PyPI publishes. A raw wheel URL does not resolve dependencies, so `depends` lists them.
+ * - `sqlite3` is part of the standard library build and is deliberately not listed.
+ */
+export declare const PYODIDE_VERSION = "314.0.7";
+export declare const PYTHON_VERSION = "3.14.2";
+export type PyodideProfile = "base" | "analysis" | "science";
+export interface PyodideFile {
+    name: string;
+    url: string;
+    sha256: string;
+    size: number;
+    source: "pyodide" | "pypi";
+}
+export interface PyodidePackage extends PyodideFile {
+    /** Direct dependencies by manifest key; they load first. */
+    depends: readonly string[];
+}
+/** Files every profile needs (the runtime itself). */
+export declare const PYODIDE_CORE_FILES: readonly PyodideFile[];
+/** Wheels by manifest key (the distribution name, normalized with dashes). */
+export declare const PYODIDE_PACKAGES: Readonly<Record<string, PyodidePackage>>;
+/** Explicit dependency closure per profile, in load order. `science` includes `analysis`. */
+export declare const PYODIDE_PROFILES: Readonly<Record<PyodideProfile, readonly string[]>>;
+/** The shape `setup --wasm` and the backend work with; tests hand in a tiny one of their own. */
+export interface PyodideManifest {
+    version: string;
+    python: string;
+    core: readonly PyodideFile[];
+    packages: Readonly<Record<string, PyodidePackage>>;
+    profiles: Readonly<Record<PyodideProfile, readonly string[]>>;
+}
+export declare const PYODIDE_MANIFEST: PyodideManifest;
+export declare const PYODIDE_PROFILE_ORDER: readonly PyodideProfile[];