@alisio/core 0.5.2 → 0.6.0

package/dist/forge/python/import-scan.js

@@ -0,0 +1,133 @@
+/**
+ * Static import scan of a Python script (Python spec §6.2). Pure and interpreter-free: it reads the
+ * top-level module names a script imports so the backend selector can tell whether the Pyodide
+ * profiles that were downloaded cover it. It is NOT a security control: `__import__`,
+ * `importlib` and `exec` are invisible to it, and a wrong guess only means the run fails with
+ * `PACKAGE_UNAVAILABLE` and is never retried elsewhere (PD5).
+ */
+import { PYODIDE_PROFILES } from "./pyodide-manifest.js";
+/** Source with comments and string contents removed; newlines and line structure stay. */
+function stripStringsAndComments(source) {
+    let out = "";
+    let i = 0;
+    const n = source.length;
+    while (i < n) {
+        const c = source.charAt(i);
+        if (c === "#") {
+            while (i < n && source.charAt(i) !== "\n")
+                i++;
+            continue;
+        }
+        if (c === '"' || c === "'") {
+            const triple = source.startsWith(c.repeat(3), i);
+            const close = triple ? c.repeat(3) : c;
+            i += close.length;
+            while (i < n && !source.startsWith(close, i)) {
+                if (source.charAt(i) === "\\")
+                    i++;
+                // A single-quoted string ends at the line, even when it was never closed.
+                else if (!triple && source.charAt(i) === "\n")
+                    break;
+                i++;
+            }
+            if (source.startsWith(close, i))
+                i += close.length;
+            out += '""';
+            continue;
+        }
+        out += c;
+        i++;
+    }
+    return out;
+}
+const IMPORT = /^\s*(?:\w[^:]*:\s*)?import\s+(.+)$/;
+const FROM = /^\s*(?:\w[^:]*:\s*)?from\s+(\.*)([\w.]*)\s+import\b/;
+/** Top-level module names imported by `source`, in order of appearance, without repeats. */
+export function scanImports(source) {
+    const found = new Set();
+    const code = stripStringsAndComments(source).replace(/\\\r?\n/g, " ");
+    for (const line of code.split(/\r?\n/)) {
+        for (const statement of line.split(";")) {
+            const plain = IMPORT.exec(statement);
+            if (plain) {
+                for (const part of (plain[1] ?? "").split(",")) {
+                    const name = /^\s*([A-Za-z_]\w*)/.exec(part)?.[1];
+                    if (name)
+                        found.add(name);
+                }
+                continue;
+            }
+            const from = FROM.exec(statement);
+            if (from && from[1] === "") {
+                const name = (from[2] ?? "").split(".")[0];
+                if (name)
+                    found.add(name);
+            }
+        }
+    }
+    return [...found];
+}
+/** Import name -> package name, for the packages the Pyodide profiles carry. */
+const MODULE_PACKAGES = {
+    numpy: "numpy",
+    pandas: "pandas",
+    dateutil: "python-dateutil",
+    pytz: "pytz",
+    six: "six",
+    matplotlib: "matplotlib",
+    mpl_toolkits: "matplotlib",
+    pylab: "matplotlib",
+    contourpy: "contourpy",
+    cycler: "cycler",
+    fontTools: "fonttools",
+    kiwisolver: "kiwisolver",
+    packaging: "packaging",
+    PIL: "pillow",
+    pyparsing: "pyparsing",
+    jinja2: "jinja2",
+    markupsafe: "markupsafe",
+    et_xmlfile: "et-xmlfile",
+    openpyxl: "openpyxl",
+    lxml: "lxml",
+    typing_extensions: "typing-extensions",
+    docx: "python-docx",
+    narwhals: "narwhals",
+    plotly: "plotly",
+    charset_normalizer: "charset-normalizer",
+    reportlab: "reportlab",
+    scipy: "scipy",
+    patsy: "patsy",
+    statsmodels: "statsmodels",
+    joblib: "joblib",
+    threadpoolctl: "threadpoolctl",
+    sklearn: "scikit-learn",
+};
+/** Top-level standard library modules of CPython 3.10-3.14 (a name list, no interpreter). */
+const STDLIB = new Set("__future__ __main__ _thread abc aifc annotationlib antigravity argparse array ast asynchat asyncio asyncore atexit audioop base64 bdb binascii binhex bisect builtins bz2 cProfile calendar cgi cgitb chunk cmath cmd code codecs codeop collections colorsys compileall compression concurrent configparser contextlib contextvars copy copyreg crypt csv ctypes curses dataclasses datetime dbm decimal difflib dis distutils doctest email encodings ensurepip enum errno faulthandler fcntl filecmp fileinput fnmatch fractions ftplib functools gc genericpath getopt getpass gettext glob graphlib grp gzip hashlib heapq hmac html http idlelib imaplib imghdr imp importlib inspect io ipaddress itertools json keyword lib2to3 linecache locale logging lzma mailbox mailcap marshal math mimetypes mmap modulefinder msilib msvcrt multiprocessing netrc nis nntplib nt ntpath nturl2path numbers opcode operator optparse os ossaudiodev pathlib pdb pickle pickletools pipes pkgutil platform plistlib poplib posix posixpath pprint profile pstats pty pwd py_compile pyclbr pydoc pydoc_data pyexpat queue quopri random re readline reprlib resource rlcompleter runpy sched secrets select selectors shelve shlex shutil signal site smtpd smtplib sndhdr socket socketserver spwd sqlite3 sre_compile sre_constants sre_parse ssl stat statistics string stringprep struct subprocess sunau symtable sys sysconfig syslog tabnanny tarfile telnetlib tempfile termios textwrap this threading time timeit tkinter token tokenize tomllib trace traceback tracemalloc tty turtle turtledemo types typing unicodedata unittest urllib uu uuid venv warnings wave weakref webbrowser winreg winsound wsgiref xdrlib xml xmlrpc zipapp zipfile zipimport zlib zoneinfo".split(" "));
+/** The helpers Alisio copies next to every script. */
+const RUNTIME_MODULE = "alisio_runtime";
+/** Modules Pyodide itself provides (the JavaScript bridge); nothing to download for them. */
+const PYODIDE_MODULES = new Set(["js", "pyodide", "pyodide_js", "_pyodide"]);
+/** Package name for an import name, when a Pyodide profile ships it. */
+export const packageOfModule = (module) => MODULE_PACKAGES[module];
+/** `science` when a package only that profile carries is imported, else `analysis`, else `base`. */
+export function classifyModules(modules) {
+    const packages = [];
+    const unknown = [];
+    for (const module of modules) {
+        if (STDLIB.has(module) || PYODIDE_MODULES.has(module) || module === RUNTIME_MODULE)
+            continue;
+        const pkg = MODULE_PACKAGES[module];
+        if (!pkg)
+            unknown.push(module);
+        else if (!packages.includes(pkg))
+            packages.push(pkg);
+    }
+    const inAnalysis = new Set(PYODIDE_PROFILES.analysis);
+    const profile = packages.some((p) => !inAnalysis.has(p))
+        ? "science"
+        : packages.length
+            ? "analysis"
+            : "base";
+    return { profile, packages, unknown };
+}

package/dist/forge/python/managed-backend.d.ts

@@ -0,0 +1,30 @@
+/**
+ * `ManagedBackend` (Python spec §4.3): the discovered host Python, as `python_run` has always run
+ * it. NOT a sandbox (`isolation: "none"`): the script has the user's permissions. This wraps
+ * `AnalysisRuntimeManager.interpreter` and `runProcess` without changing what they do.
+ */
+import type { EngineDescriptor } from "@alisio/sdk";
+import type { AnalysisRuntimeManager } from "../../analysis/runtime-manager.ts";
+import type { TerminationReason, TerminationReport } from "../ports.ts";
+import type { BackendReadiness, PreparedJob, ProcessSink, PythonBackend, RunningProcess, SupportVerdict } from "./backend.ts";
+/** Environment of the script: the usual whitelist, HOME/TMP inside work/, and the job paths. */
+export declare function scriptEnv(job: {
+    input: string;
+    staging: string;
+    work: string;
+    id: string;
+}): Record<string, string>;
+export interface ManagedBackendDeps {
+    /** Discovery of the interpreter; the installed extras environment is part of its answer. */
+    runtime: Pick<AnalysisRuntimeManager, "interpreter">;
+}
+export declare class ManagedBackend implements PythonBackend {
+    #private;
+    readonly id: "managed";
+    constructor(deps: ManagedBackendDeps);
+    describe(): Promise<EngineDescriptor>;
+    check(signal?: AbortSignal): Promise<BackendReadiness>;
+    supports(): SupportVerdict;
+    launch(job: PreparedJob, sink: ProcessSink): Promise<RunningProcess>;
+    terminate(process: RunningProcess, _reason: TerminationReason): Promise<TerminationReport>;
+}

package/dist/forge/python/managed-backend.js

@@ -0,0 +1,111 @@
+import { runProcess } from "../../runtime/process.js";
+import { settle } from "./settle.js";
+/** Environment of the script: the usual whitelist, HOME/TMP inside work/, and the job paths. */
+export function scriptEnv(job) {
+    const keep = [
+        "PATH",
+        "LANG",
+        "LC_ALL",
+        "SystemRoot",
+        "COMSPEC",
+        "PATHEXT",
+        "SYSTEMDRIVE",
+        "WINDIR",
+    ];
+    return {
+        ...Object.fromEntries(keep.flatMap((k) => (process.env[k] ? [[k, process.env[k]]] : []))),
+        HOME: job.work,
+        USERPROFILE: job.work,
+        TMPDIR: job.work,
+        TEMP: job.work,
+        TMP: job.work,
+        ALISIO_INPUT_DIR: job.input,
+        ALISIO_OUTPUT_DIR: job.staging,
+        ALISIO_WORK_DIR: job.work,
+        ALISIO_EXECUTION_ID: job.id,
+        MPLBACKEND: "Agg",
+        PYTHONDONTWRITEBYTECODE: "1",
+        PYTHONUNBUFFERED: "1",
+        PYTHONNOUSERSITE: "1",
+        PYTHONIOENCODING: "utf-8",
+    };
+}
+export class ManagedBackend {
+    id = "managed";
+    #deps;
+    constructor(deps) {
+        this.#deps = deps;
+    }
+    async describe() {
+        return {
+            id: "python-managed",
+            language: "python",
+            version: "host python",
+            isolation: "none",
+            ready: true,
+            capabilities: {
+                cooperativeInterrupt: false,
+                hardTermination: true,
+                hardMemoryLimit: false,
+                networkDeniedByOS: false,
+                toolCalls: false,
+            },
+        };
+    }
+    async check(signal) {
+        const resolved = await this.#deps.runtime.interpreter(signal);
+        return resolved.ok
+            ? { ready: true }
+            : { ready: false, reason: resolved.reason, detail: resolved };
+    }
+    supports() {
+        return { ok: true };
+    }
+    async launch(job, sink) {
+        const interpreter = await this.#deps.runtime.interpreter(job.signal);
+        if (!interpreter.ok)
+            throw new Error(`runtime_unavailable: ${interpreter.reason}`);
+        const stop = new AbortController();
+        const args = [
+            ...(interpreter.prefixArgs ?? []),
+            "-E",
+            "-s",
+            "-B",
+            "-u",
+            "-X",
+            "utf8",
+            job.paths.main,
+        ];
+        const result = runProcess(interpreter.executable, args, {
+            cwd: job.paths.work,
+            signal: AbortSignal.any([job.signal, stop.signal]),
+            timeoutMs: job.backstopMs,
+            maxBytes: job.maxBytes,
+            onOverflow: "truncate",
+            logFiles: job.logFiles,
+            maxLogBytes: job.maxLogBytes,
+            env: {
+                ...scriptEnv(job.paths),
+                ...job.env,
+                // The scratchpad is the user's own folder: the variable names it, nothing enforces read-only.
+                ...(job.scratch ? { ALISIO_SCRATCH_DIR: job.scratch } : {}),
+            },
+            onData: sink.onData,
+        });
+        return {
+            result,
+            stop,
+            runtime: {
+                mode: "managed",
+                python: interpreter.version,
+                extras: interpreter.extras ?? [],
+                ...(interpreter.runtimeVersion ? { runtimeVersion: interpreter.runtimeVersion } : {}),
+            },
+        };
+    }
+    async terminate(process, _reason) {
+        const started = Date.now();
+        process.stop.abort();
+        return settle(process, "kill", started);
+    }
+}

package/dist/forge/python/oci-backend.d.ts

@@ -0,0 +1,27 @@
+/**
+ * `OciBackend` (Python spec §4.3): Docker or Podman with a digest-pinned image, as `python_run` has
+ * always run it (`ociRunArgs`, `OciRuntime.run`). `isolation: "os-container"`: no network, a
+ * read-only root, no capabilities, memory and process limits; not a hardened sandbox.
+ */
+import type { EngineDescriptor } from "@alisio/sdk";
+import { type OciRuntime } from "../../analysis/oci.ts";
+import type { TerminationReason, TerminationReport } from "../ports.ts";
+import type { BackendReadiness, PreparedJob, ProcessSink, PythonBackend, RunningProcess, SupportVerdict } from "./backend.ts";
+export interface OciBackendDeps {
+    /** Read on every call: the settings can change between calls. `mode` is `analysis.runtime`. */
+    oci: {
+        runtime(): OciRuntime;
+        mode?(): string;
+    };
+}
+export declare class OciBackend implements PythonBackend {
+    #private;
+    readonly id: "oci";
+    constructor(deps: OciBackendDeps);
+    describe(): Promise<EngineDescriptor>;
+    check(signal?: AbortSignal): Promise<BackendReadiness>;
+    supports(): SupportVerdict;
+    launch(job: PreparedJob, sink: ProcessSink): Promise<RunningProcess>;
+    /** Aborting the run kills the container by name (`OciRuntime.run` does it in its `finally`). */
+    terminate(process: RunningProcess, _reason: TerminationReason): Promise<TerminationReport>;
+}

package/dist/forge/python/oci-backend.js

@@ -0,0 +1,84 @@
+import { mountSource } from "../../analysis/oci.js";
+import { settle } from "./settle.js";
+export class OciBackend {
+    id = "oci";
+    #deps;
+    constructor(deps) {
+        this.#deps = deps;
+    }
+    async describe() {
+        const { settings } = this.#deps.oci.runtime();
+        return {
+            id: "python-oci",
+            language: "python",
+            version: settings.image ?? settings.engine,
+            isolation: "os-container",
+            ready: true,
+            capabilities: {
+                cooperativeInterrupt: false,
+                hardTermination: true,
+                hardMemoryLimit: true,
+                networkDeniedByOS: true,
+                toolCalls: false,
+            },
+        };
+    }
+    async check(signal) {
+        const reason = await this.#deps.oci
+            .runtime()
+            .unavailable(signal, { auto: this.#deps.oci.mode?.() === "auto" });
+        return reason === undefined ? { ready: true } : { ready: false, reason };
+    }
+    supports() {
+        return { ok: true };
+    }
+    async launch(job, sink) {
+        const runtime = this.#deps.oci.runtime();
+        const stop = new AbortController();
+        // A scratchpad path the engine would misparse as mount syntax is simply not exposed.
+        const scratch = job.scratch && mountable(job.scratch) ? job.scratch : undefined;
+        const result = runtime.run({
+            executionId: job.executionId,
+            mounts: {
+                script: job.paths.script,
+                input: job.paths.input,
+                work: job.paths.work,
+                staging: job.paths.staging,
+                ...(scratch ? { scratch } : {}),
+            },
+            signal: AbortSignal.any([job.signal, stop.signal]),
+            timeoutMs: job.backstopMs,
+            maxBytes: job.maxBytes,
+            logFiles: job.logFiles,
+            maxLogBytes: job.maxLogBytes,
+            onData: sink.onData,
+            env: job.env,
+        });
+        const { settings } = runtime;
+        return {
+            result,
+            stop,
+            runtime: {
+                mode: "oci",
+                engine: settings.engine,
+                ...(settings.image ? { image: settings.image } : {}),
+                extras: [],
+            },
+        };
+    }
+    /** Aborting the run kills the container by name (`OciRuntime.run` does it in its `finally`). */
+    async terminate(process, _reason) {
+        const started = Date.now();
+        process.stop.abort();
+        return settle(process, "container-kill", started);
+    }
+}
+function mountable(path) {
+    try {
+        mountSource(path, process.platform);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}

package/dist/forge/python/pyodide-backend.d.ts

@@ -0,0 +1,40 @@
+import type { EngineDescriptor } from "@alisio/sdk";
+import type { TerminationReason, TerminationReport } from "../ports.ts";
+import type { BackendReadiness, PreparedJob, ProcessSink, PythonBackend, PythonRequirements, RunningProcess, SupportVerdict } from "./backend.ts";
+import { type PyodideFile, type PyodideManifest, type PyodideProfile } from "./pyodide-manifest.ts";
+import { type RssReader } from "./rss.ts";
+import { type NodeEnvironment } from "./wasm-node.ts";
+/** `analysis.wasm`, read on every call: the settings can change between calls. */
+export interface WasmSettings {
+    memoryMb: number;
+    startupMs: number;
+    maxInputBytes: number;
+    nodePath: string | null;
+}
+export interface PyodideBackendDeps {
+    /** The state root; the distribution lives in `<root>/analysis/pyodide/<version>`. */
+    stateRoot: string;
+    settings: () => WasmSettings;
+    manifest?: PyodideManifest;
+    environment?: () => NodeEnvironment;
+    /** Replaceable in tests. */
+    rss?: RssReader;
+    sampleMs?: number;
+}
+/** Keys needed to load `wanted` (and what they depend on), in the manifest's load order. */
+export declare function loadPlan(manifest: PyodideManifest, profile: PyodideProfile, wanted: readonly string[]): {
+    packages: string[];
+    wheels: string[];
+    files: PyodideFile[];
+};
+export declare class PyodideBackend implements PythonBackend {
+    #private;
+    readonly id: "wasm";
+    constructor(deps: PyodideBackendDeps);
+    check(_signal?: AbortSignal): Promise<BackendReadiness>;
+    /** Static facts plus the observed network answer; `ready` is the live state of `check()`. */
+    describe(): Promise<EngineDescriptor>;
+    supports(requirements: PythonRequirements): SupportVerdict;
+    launch(job: PreparedJob, sink: ProcessSink): Promise<RunningProcess>;
+    terminate(process: RunningProcess, _reason: TerminationReason): Promise<TerminationReport>;
+}