@alisio/core 0.1.0-alpha.9 → 0.1.1

package/dist/analysis/oci.js

@@ -0,0 +1,330 @@
+/**
+ * Optional container runtime for `python_run` (`analysis.runtime: "oci"`): Docker or Podman with a
+ * digest-pinned image. Nothing is built or pulled during a run (`alisio analysis setup --oci` pulls
+ * once); the run only starts `<engine> run --rm …`. The isolation is partial and documented: the
+ * container sees only the job folders, has no network, a read-only root, no capabilities and
+ * memory/CPU/process limits, but a container is not a boundary against a hostile kernel exploit.
+ *
+ * Every argument goes in an array (no shell), so paths with spaces and non-ASCII names are fine.
+ * Host paths that the engine would parse as mount syntax (a `:` outside a Windows drive prefix) are
+ * refused with a clear message instead of being passed through.
+ */
+import { readFile } from "node:fs/promises";
+import { OCI_IMAGE_PATTERN } from "../config.js";
+import { which } from "../runtime/fs.js";
+import { runProcess } from "../runtime/process.js";
+/** Paths inside the container (spec §8.3). */
+export const CONTAINER = {
+    script: "/job/script",
+    input: "/job/input",
+    work: "/job/work",
+    out: "/job/out",
+};
+export const PIDS_LIMIT = 256;
+export const TMPFS_SIZE = "256m";
+/** `alisio-exec_01J…`: the container of one execution. */
+export function containerName(executionId) {
+    const safe = executionId.replace(/[^a-zA-Z0-9_.-]/g, "-");
+    return `alisio-${safe}`;
+}
+/** Windows drive prefix (`C:\` or `C:/`), which a `-v` source may legitimately contain. */
+const DRIVE = /^[A-Za-z]:[\\/]/;
+/** A host path that can be passed to `-v`; throws when the engine would misparse it. */
+export function mountSource(path, platform) {
+    const rest = platform === "win32" && DRIVE.test(path) ? path.slice(2) : path;
+    if (rest.includes(":") || /[\r\n\0]/.test(path))
+        throw new Error(`The job folder ${path} contains a character the container runtime cannot mount (":"). ` +
+            "Move the Alisio state folder (ALISIO_STATE_HOME or --db) to a path without it, or use the managed runtime.");
+    return path;
+}
+/**
+ * `<engine> run` arguments for one execution (spec §8.3). The container name is
+ * `alisio-<executionId>` so a cancellation can `kill` it by name.
+ */
+export function ociRunArgs(input) {
+    const { settings, host } = input;
+    const mount = (source, target, mode) => `${mountSource(source, host.platform)}:${target}:${mode}${host.selinux ? ",z" : ""}`;
+    const args = [
+        "run",
+        "--rm",
+        "--name",
+        containerName(input.executionId),
+        "--label",
+        "alisio=analysis",
+        "--network=none",
+        "--read-only",
+        "--cap-drop=ALL",
+        "--security-opt=no-new-privileges",
+        `--pids-limit=${PIDS_LIMIT}`,
+        `--memory=${settings.memoryMb}m`,
+        `--memory-swap=${settings.memoryMb}m`,
+        `--cpus=${settings.cpus}`,
+    ];
+    // Files the script writes must belong to the user on Linux. Docker Desktop (macOS, Windows)
+    // maps ownership itself, so `--user` is omitted there; a rootless engine already maps the user
+    // to root inside the container (Podman needs keep-id to keep the owner on the mounts).
+    if (host.platform === "linux" && host.uid !== undefined && host.gid !== undefined) {
+        if (settings.engine === "podman" && host.rootless && host.uid !== 0)
+            args.push("--userns=keep-id", "--user", `${host.uid}:${host.gid}`);
+        else if (!host.rootless && host.uid !== 0)
+            args.push("--user", `${host.uid}:${host.gid}`);
+    }
+    args.push("-v", mount(input.mounts.script, CONTAINER.script, "ro"), "-v", mount(input.mounts.input, CONTAINER.input, "ro"), "-v", mount(input.mounts.work, CONTAINER.work, "rw"), "-v", mount(input.mounts.staging, CONTAINER.out, "rw"), "--tmpfs", `/tmp:size=${TMPFS_SIZE}`);
+    const env = {
+        ALISIO_INPUT_DIR: CONTAINER.input,
+        ALISIO_OUTPUT_DIR: CONTAINER.out,
+        ALISIO_WORK_DIR: CONTAINER.work,
+        ALISIO_EXECUTION_ID: input.executionId,
+        HOME: CONTAINER.work,
+        TMPDIR: "/tmp",
+        MPLCONFIGDIR: `${CONTAINER.work}/.matplotlib`,
+        MPLBACKEND: "Agg",
+        PYTHONDONTWRITEBYTECODE: "1",
+        PYTHONUNBUFFERED: "1",
+        PYTHONNOUSERSITE: "1",
+        PYTHONIOENCODING: "utf-8",
+        ...input.env,
+    };
+    for (const [key, value] of Object.entries(env))
+        args.push("-e", `${key}=${value}`);
+    args.push("-w", CONTAINER.work, settings.image, "python", "-E", "-s", "-B", "-u", "-X", "utf8", `${CONTAINER.script}/main.py`);
+    return args;
+}
+async function selinuxEnforcing() {
+    try {
+        return (await readFile("/sys/fs/selinux/enforce", "utf8")).trim() === "1";
+    }
+    catch {
+        return false;
+    }
+}
+export function defaultOciDeps() {
+    return {
+        locate: (engine) => which(engine),
+        async host() {
+            return {
+                platform: process.platform,
+                ...(typeof process.getuid === "function" ? { uid: process.getuid() } : {}),
+                ...(typeof process.getgid === "function" ? { gid: process.getgid() } : {}),
+                ...(process.platform === "linux" ? { selinux: await selinuxEnforcing() } : {}),
+            };
+        },
+        async exec(command, args, options) {
+            const result = await runProcess(command, args, {
+                cwd: process.cwd(),
+                signal: options.signal,
+                timeoutMs: options.timeoutMs,
+                maxBytes: 256 * 1024,
+                onOverflow: "truncate",
+            });
+            return { stdout: result.stdout, stderr: result.stderr, exitCode: result.exitCode };
+        },
+    };
+}
+const other = (engine) => (engine === "docker" ? "podman" : "docker");
+/** Container runtime of `python_run`; all engine calls go through `OciDeps`. */
+export class OciRuntime {
+    settings;
+    deps;
+    constructor(settings, deps = {}) {
+        this.settings = settings;
+        this.deps = { ...defaultOciDeps(), ...deps };
+    }
+    async command() {
+        return this.deps.locate(this.settings.engine);
+    }
+    async call(args, options = {}) {
+        const command = await this.command();
+        if (!command)
+            throw new Error(`${this.settings.engine} is not installed`);
+        return this.deps.exec(command, [...(this.deps.prefixArgs ?? []), ...args], {
+            signal: options.signal ?? AbortSignal.timeout(options.timeoutMs ?? 30_000),
+            timeoutMs: options.timeoutMs ?? 30_000,
+        });
+    }
+    /**
+     * Is the engine installed and its daemon reachable? Optional by design: a missing engine is a
+     * status, not an error of Alisio. Also reports rootless mode, which changes the `--user` flags.
+     */
+    async status(signal) {
+        const engine = this.settings.engine;
+        const path = await this.command();
+        if (!path) {
+            const alternative = (await this.deps.locate(other(engine))) ? other(engine) : undefined;
+            return {
+                ok: false,
+                engine,
+                reason: `${engine} was not found on PATH`,
+                ...(alternative ? { alternative } : {}),
+            };
+        }
+        try {
+            const version = await this.call(["version", "--format", engine === "docker" ? "{{.Server.Version}}" : "{{.Version}}"], signal ? { signal } : {});
+            if (version.exitCode !== 0)
+                return {
+                    ok: false,
+                    engine,
+                    reason: `${engine} is installed but not usable: ${(version.stderr || version.stdout).trim().split(/\r?\n/)[0] ?? "no answer"}`,
+                };
+            let rootless = false;
+            try {
+                const info = await this.call([
+                    "info",
+                    "--format",
+                    engine === "docker" ? "{{json .SecurityOptions}}" : "{{.Host.Security.Rootless}}",
+                ], signal ? { signal } : {});
+                rootless =
+                    info.exitCode === 0 &&
+                        (engine === "docker" ? /name=rootless/.test(info.stdout) : info.stdout.trim() === "true");
+            }
+            catch {
+                /* Rootless detection is best effort; the default flags are the rootful ones. */
+            }
+            return { ok: true, engine, path, version: version.stdout.trim(), rootless };
+        }
+        catch (error) {
+            return {
+                ok: false,
+                engine,
+                reason: `${engine} did not answer: ${error instanceof Error ? error.message : String(error)}`,
+            };
+        }
+    }
+    /** The reason a run cannot start (`undefined` = ready), checked before any job is created. */
+    async unavailable(signal) {
+        const { image } = this.settings;
+        if (!image)
+            return ('analysis.runtime is "oci" but analysis.oci.image is not set. Put a digest-pinned image ' +
+                "(name@sha256:…) in your user configuration; `alisio analysis setup --oci --image <name>` prints the value.");
+        if (!OCI_IMAGE_PATTERN.test(image))
+            return "analysis.oci.image must be pinned by digest";
+        const status = await this.status(signal);
+        if (!status.ok)
+            return `${status.reason}${status.alternative ? `; ${status.alternative} is installed (set analysis.oci.engine to "${status.alternative}")` : ""}. The container runtime is optional: remove analysis.runtime "oci" to use managed Python.`;
+        return undefined;
+    }
+    /**
+     * Runs the job's script in a container. Cancelling or timing out `kills` the container by name
+     * (killing the client process does not always stop it) and no `alisio-*` container is left.
+     */
+    async run(input) {
+        const image = this.settings.image;
+        if (!image)
+            throw new Error("analysis.oci.image is not set");
+        const command = await this.command();
+        if (!command)
+            throw new Error(`${this.settings.engine} is not installed`);
+        const status = await this.status(input.signal);
+        const host = await this.deps.host();
+        const args = ociRunArgs({
+            settings: { ...this.settings, image },
+            executionId: input.executionId,
+            mounts: input.mounts,
+            host: { ...host, rootless: status.ok ? status.rootless : false },
+        });
+        const name = containerName(input.executionId);
+        const stop = AbortSignal.any([input.signal, AbortSignal.timeout(input.timeoutMs)]);
+        const onAbort = () => void this.kill(name);
+        stop.addEventListener("abort", onAbort, { once: true });
+        try {
+            const result = await runProcess(command, [...(this.deps.prefixArgs ?? []), ...args], {
+                cwd: input.mounts.work,
+                signal: input.signal,
+                timeoutMs: input.timeoutMs,
+                maxBytes: input.maxBytes,
+                onOverflow: "truncate",
+                logFiles: input.logFiles,
+                maxLogBytes: input.maxLogBytes,
+                // The engine client only needs to find its own daemon: no variables of the script.
+                env: engineEnv(),
+                ...(input.onData ? { onData: input.onData } : {}),
+            });
+            return { ...result, containerName: name };
+        }
+        finally {
+            stop.removeEventListener("abort", onAbort);
+            // After an abort the client may have exited before the daemon stopped the container.
+            if (stop.aborted)
+                await this.kill(name);
+        }
+    }
+    /** `<engine> kill <name>`; a container that is already gone is not an error. */
+    async kill(name) {
+        try {
+            await this.call(["kill", name], { timeoutMs: 15_000 });
+        }
+        catch {
+            /* Already gone, or the engine stopped answering: nothing else to do. */
+        }
+    }
+    /** Names of the `alisio-*` containers that still exist (running or not). */
+    async leftovers() {
+        const result = await this.call([
+            "ps",
+            "-a",
+            "--filter",
+            "label=alisio=analysis",
+            "--format",
+            "{{.Names}}",
+        ]);
+        return result.stdout
+            .split(/\r?\n/)
+            .map((line) => line.trim())
+            .filter(Boolean);
+    }
+    /** Pulls the image (once, from `alisio analysis setup --oci`); a digest reference verifies itself. */
+    async pull(image, signal) {
+        const pulled = await this.call(["pull", image], { signal, timeoutMs: 30 * 60_000 });
+        if (pulled.exitCode !== 0)
+            throw new Error(`${this.settings.engine} pull ${image} failed: ${(pulled.stderr || pulled.stdout).trim().slice(-2000)}`);
+        const inspected = await this.call(["image", "inspect", "--format", "{{json .RepoDigests}}", image], { signal });
+        const digests = parseDigests(inspected.stdout);
+        return { ...(digests[0] ? { digest: digests[0] } : {}) };
+    }
+    /** True when the image is already in the local store (so a run never needs the network). */
+    async hasImage(image, signal) {
+        try {
+            const result = await this.call(["image", "inspect", "--format", "{{.Id}}", image], signal ? { signal } : {});
+            return result.exitCode === 0;
+        }
+        catch {
+            return false;
+        }
+    }
+}
+/** `["repo@sha256:…"]` → `["repo@sha256:…"]` (JSON from `--format {{json .RepoDigests}}`). */
+export function parseDigests(output) {
+    try {
+        const value = JSON.parse(output.trim());
+        return Array.isArray(value)
+            ? value.filter((v) => typeof v === "string" && OCI_IMAGE_PATTERN.test(v))
+            : [];
+    }
+    catch {
+        return [];
+    }
+}
+/** The environment of the engine client: where to find the daemon, nothing else. */
+function engineEnv() {
+    const keep = [
+        "PATH",
+        "HOME",
+        "USERPROFILE",
+        "TMPDIR",
+        "TEMP",
+        "SystemRoot",
+        "COMSPEC",
+        "LANG",
+        "DOCKER_HOST",
+        "DOCKER_CONTEXT",
+        "DOCKER_CONFIG",
+        "XDG_RUNTIME_DIR",
+        "CONTAINER_HOST",
+        "APPDATA",
+        "LOCALAPPDATA",
+        "ProgramData",
+        "ProgramFiles",
+        "DBUS_SESSION_BUS_ADDRESS",
+    ];
+    return Object.fromEntries(keep.flatMap((key) => (process.env[key] ? [[key, process.env[key]]] : [])));
+}

package/dist/analysis/python/sources.d.ts

@@ -0,0 +1,2 @@
+/** The pure-Python `alisio_runtime` package copied next to every python_run script. */
+export declare const ALISIO_RUNTIME_FILES: Record<string, string>;

package/dist/analysis/python/sources.js

@@ -0,0 +1,10 @@
+// Generated by scripts/analysis-runtime-sources.ts from alisio_runtime/*.py. Do not edit.
+/** The pure-Python `alisio_runtime` package copied next to every python_run script. */
+export const ALISIO_RUNTIME_FILES = {
+    "__init__.py": '"""Helpers for scripts run by Alisio\'s python_run tool (standard library only).\n\nEverything a script writes to ``output_dir()`` ($ALISIO_OUTPUT_DIR) is published as a\ndownloadable artifact when the script exits with code 0. Intermediate files belong in\n``work_dir()`` (the current directory), never in the output folder.\n"""\n\nimport os\nfrom pathlib import Path\n\n__all__ = ["output_dir", "input_dir", "work_dir", "execution_id", "outputs", "html", "svg", "datasets"]\n__version__ = "1"\n\n\ndef _dir(name: str, fallback: str) -> Path:\n    return Path(os.environ.get(name) or fallback)\n\n\ndef output_dir() -> Path:\n    """Folder whose files are published as artifacts ($ALISIO_OUTPUT_DIR)."""\n    return _dir("ALISIO_OUTPUT_DIR", "out")\n\n\ndef input_dir() -> Path:\n    """Read-only copies of the inputs given to python_run ($ALISIO_INPUT_DIR)."""\n    return _dir("ALISIO_INPUT_DIR", "input")\n\n\ndef work_dir() -> Path:\n    """Scratch folder for intermediate files; also the current directory."""\n    return _dir("ALISIO_WORK_DIR", ".")\n\n\ndef execution_id() -> str:\n    return os.environ.get("ALISIO_EXECUTION_ID", "")\n',
+    "datasets.py": '"""Read-only access to the datasets Alisio prepared (one SQLite file each) and any .sqlite input.\n\n``open("sales")`` returns a read-only ``sqlite3`` connection to ``input/sales.sqlite``; the data\nis in the table ``data`` (CSV, TSV, JSON, JSONL) or ``s_<sheet>`` (XLSX). Values are stored\nwithout loss: columns have no declared type, numbers are numbers and everything else (``007``,\n``1,234``, ``N/A``) is the exact text. ``columns("sales")`` lists the columns with the type\nhints and statistics computed at ingestion. Standard library only; with the optional extras,\n``pandas.read_sql_query("SELECT * FROM data", open("sales"))`` works on the same connection.\n"""\n\nimport json\nimport sqlite3\nfrom pathlib import Path\nfrom typing import Any, Dict, List, Optional, Union\n\nfrom . import input_dir\n\n__all__ = ["connect", "open", "tables", "columns", "inputs"]\n\n\ndef _resolve(name: Union[str, Path]) -> Path:\n    target = Path(name)\n    if not target.is_absolute():\n        target = input_dir() / target\n    if not target.exists() and target.suffix != ".sqlite":\n        alt = target.with_name(target.name + ".sqlite")\n        if alt.exists():\n            target = alt\n    return target.resolve()\n\n\ndef connect(path: Union[str, Path]) -> sqlite3.Connection:\n    """Opens a SQLite file read-only (``mode=ro``); writes raise ``sqlite3.OperationalError``."""\n    return sqlite3.connect(_resolve(path).as_uri() + "?mode=ro&immutable=1", uri=True)\n\n\ndef open(name: Union[str, Path]) -> sqlite3.Connection:  # noqa: A001 - mirrors the documented API\n    """The dataset ``name`` (``input/<name>.sqlite``) as a read-only connection."""\n    return connect(name)\n\n\ndef tables(name: Union[str, Path]) -> List[Dict[str, Any]]:\n    """Sheets of a dataset: ``[{"name", "table", "rows", "columns"}]``."""\n    with connect(name) as db:\n        rows = db.execute(\n            "SELECT name, table_name, rows, columns FROM _alisio_sheets ORDER BY ordinal"\n        ).fetchall()\n    return [{"name": r[0], "table": r[1], "rows": r[2], "columns": r[3]} for r in rows]\n\n\ndef columns(name: Union[str, Path], table: Optional[str] = None) -> List[Dict[str, Any]]:\n    """Columns with type hint and statistics (``nulls``, ``distinct``, ``min``, ``max``, ``mean``,\n    ``text_fallbacks``, ``top``); ``table`` defaults to the first sheet."""\n    with connect(name) as db:\n        if table is None:\n            first = db.execute("SELECT table_name FROM _alisio_sheets ORDER BY ordinal LIMIT 1").fetchone()\n            if first is None:\n                return []\n            table = first[0]\n        rows = db.execute(\n            "SELECT name, label, inferred_type, nulls, distinct_count, distinct_exact, min, max, mean,"\n            " text_fallbacks, top_values FROM _alisio_columns WHERE table_name=? ORDER BY ordinal",\n            (table,),\n        ).fetchall()\n    keys = ["name", "label", "type", "nulls", "distinct", "distinct_exact", "min", "max", "mean",\n            "text_fallbacks", "top"]\n    result = []\n    for row in rows:\n        item = dict(zip(keys, row))\n        item["top"] = json.loads(item["top"] or "[]")\n        result.append(item)\n    return result\n\n\ndef inputs() -> List[Dict[str, Any]]:\n    """What python_run placed in the input folder (``input/inputs.json``)."""\n    listing = input_dir() / "inputs.json"\n    if not listing.exists():\n        return []\n    return json.loads(listing.read_text(encoding="utf-8"))\n',
+    "html.py": '"""Self-contained HTML pages. The Alisio viewer blocks network access: inline every asset."""\n\nimport html as _html\nfrom pathlib import Path\nfrom typing import Optional\n\nfrom . import outputs\n\n_CSS = """\n:root { color-scheme: light; font-family: system-ui, -apple-system, Segoe UI, Roboto, sans-serif; }\nbody { margin: 0; padding: 24px; background: #fff; color: #1b1f24; line-height: 1.5; }\nh1, h2, h3 { line-height: 1.25; }\ntable { border-collapse: collapse; margin: 12px 0; }\nth, td { border: 1px solid #d0d7de; padding: 6px 10px; text-align: left; }\nth { background: #f6f8fa; }\n.grid { display: grid; gap: 16px; grid-template-columns: repeat(auto-fit, minmax(280px, 1fr)); }\n.card { border: 1px solid #d0d7de; border-radius: 8px; padding: 16px; }\n"""\n\n\ndef escape(text: object) -> str:\n    return _html.escape(str(text), quote=True)\n\n\ndef table(rows, headers=None) -> str:\n    """An HTML table from an iterable of rows (values are escaped)."""\n    head = ""\n    if headers:\n        head = "<thead><tr>" + "".join(f"<th>{escape(h)}</th>" for h in headers) + "</tr></thead>"\n    body = "".join("<tr>" + "".join(f"<td>{escape(c)}</td>" for c in row) + "</tr>" for row in rows)\n    return f"<table>{head}<tbody>{body}</tbody></table>"\n\n\ndef page(title: str, body: str, css: Optional[str] = None) -> str:\n    """A complete HTML document; ``body`` is trusted HTML written by the script."""\n    style = _CSS + (css or "")\n    return (\n        "<!doctype html>\\n<html lang=\\"en\\"><head><meta charset=\\"utf-8\\">"\n        "<meta name=\\"viewport\\" content=\\"width=device-width, initial-scale=1\\">"\n        f"<title>{escape(title)}</title><style>{style}</style></head>"\n        f"<body>{body}</body></html>\\n"\n    )\n\n\ndef write(name: str, title: str, body: str, css: Optional[str] = None) -> Path:\n    """Writes a page to the output folder and returns its path."""\n    target = outputs.path(name)\n    target.write_text(page(title, body, css), encoding="utf-8")\n    return target\n\n\ndef inline_plotly() -> str:\n    """A <script> with the plotly.js bundled in the ``plotly`` package (needs the analysis extras)."""\n    try:\n        from plotly.offline import get_plotlyjs  # type: ignore\n    except ImportError as error:\n        raise ImportError(\n            "plotly is not installed; run `alisio analysis setup --extras analysis` or use alisio_runtime.svg"\n        ) from error\n    return f"<script>{get_plotlyjs()}</script>"\n',
+    "outputs.py": '"""Declare published outputs explicitly (optional ``outputs.json``)."""\n\nimport json\nfrom pathlib import Path\nfrom typing import Optional\n\nfrom . import output_dir\n\n\ndef path(name: str) -> Path:\n    """Absolute path for a new output file, creating parent folders."""\n    target = output_dir() / name\n    target.parent.mkdir(parents=True, exist_ok=True)\n    return target\n\n\ndef declare(name: str, title: Optional[str] = None, entry: Optional[str] = None) -> None:\n    """Adds an artifact to outputs.json. Once the file exists, ONLY declared outputs are published."""\n    manifest_path = output_dir() / "outputs.json"\n    manifest = {"artifacts": []}\n    if manifest_path.exists():\n        manifest = json.loads(manifest_path.read_text(encoding="utf-8"))\n    item = {"path": name}\n    if title:\n        item["title"] = title\n    if entry:\n        item["entry"] = entry\n    manifest["artifacts"] = [a for a in manifest.get("artifacts", []) if a.get("path") != name]\n    manifest["artifacts"].append(item)\n    manifest_path.write_text(json.dumps(manifest, indent=2), encoding="utf-8")\n',
+    "svg.py": '"""Dependency-free SVG charts (bar, line, scatter) to embed in HTML or save as .svg files."""\n\nimport html as _html\nfrom typing import Iterable, List, Optional, Sequence\n\n_PALETTE = ["#2f6fdb", "#e0692b", "#2a9d63", "#c2417b", "#7a5bd6", "#a8862b"]\n\n\ndef _esc(text: object) -> str:\n    return _html.escape(str(text), quote=True)\n\n\ndef _num(value: float) -> str:\n    return f"{value:.2f}".rstrip("0").rstrip(".")\n\n\ndef _scale(values: Sequence[float]):\n    low = min(0.0, min(values)) if values else 0.0\n    high = max(values) if values else 1.0\n    if high == low:\n        high = low + 1.0\n    return low, high\n\n\ndef _frame(width: int, height: int, title: Optional[str], body: List[str]) -> str:\n    heading = f\'<text x="{width / 2}" y="20" text-anchor="middle" font-size="14" font-weight="600">{_esc(title)}</text>\' if title else ""\n    return (\n        f\'<svg xmlns="http://www.w3.org/2000/svg" width="{width}" height="{height}" \'\n        f\'viewBox="0 0 {width} {height}" font-family="system-ui, sans-serif" role="img">\'\n        f\'<rect width="100%" height="100%" fill="#ffffff"/>{heading}{"".join(body)}</svg>\'\n    )\n\n\ndef _axes(left: int, top: int, right: int, bottom: int, low: float, high: float) -> List[str]:\n    parts = [\n        f\'<line x1="{left}" y1="{bottom}" x2="{right}" y2="{bottom}" stroke="#57606a"/>\',\n        f\'<line x1="{left}" y1="{top}" x2="{left}" y2="{bottom}" stroke="#57606a"/>\',\n    ]\n    for i in range(5):\n        value = low + (high - low) * i / 4\n        y = bottom - (bottom - top) * i / 4\n        parts.append(f\'<text x="{left - 6}" y="{y + 4}" text-anchor="end" font-size="10" fill="#57606a">{_esc(_num(value))}</text>\')\n        parts.append(f\'<line x1="{left}" y1="{y}" x2="{right}" y2="{y}" stroke="#eaeef2"/>\')\n    return parts\n\n\ndef bar(labels: Iterable[object], values: Iterable[float], title: Optional[str] = None, width: int = 640, height: int = 360) -> str:\n    labels = [str(l) for l in labels]\n    values = [float(v) for v in values]\n    left, top, right, bottom = 56, 36, width - 16, height - 48\n    low, high = _scale(values)\n    parts = _axes(left, top, right, bottom, low, high)\n    step = (right - left) / max(1, len(values))\n    zero = bottom - (bottom - top) * (0 - low) / (high - low)\n    for i, (label, value) in enumerate(zip(labels, values)):\n        x = left + step * i + step * 0.15\n        y = bottom - (bottom - top) * (value - low) / (high - low)\n        parts.append(f\'<rect x="{_num(x)}" y="{_num(min(y, zero))}" width="{_num(step * 0.7)}" height="{_num(abs(zero - y))}" fill="{_PALETTE[0]}"><title>{_esc(label)}: {_esc(_num(value))}</title></rect>\')\n        parts.append(f\'<text x="{_num(x + step * 0.35)}" y="{bottom + 16}" text-anchor="middle" font-size="10">{_esc(label)}</text>\')\n    return _frame(width, height, title, parts)\n\n\ndef _xy(xs, ys, title, width, height, connect: bool) -> str:\n    xs = [float(x) for x in xs]\n    ys = [float(y) for y in ys]\n    left, top, right, bottom = 56, 36, width - 16, height - 40\n    low, high = _scale(ys)\n    xlow, xhigh = (min(xs), max(xs)) if xs else (0.0, 1.0)\n    if xhigh == xlow:\n        xhigh = xlow + 1.0\n    parts = _axes(left, top, right, bottom, low, high)\n    points = [(left + (right - left) * (x - xlow) / (xhigh - xlow), bottom - (bottom - top) * (y - low) / (high - low)) for x, y in zip(xs, ys)]\n    if connect and points:\n        path = " ".join(f"{_num(x)},{_num(y)}" for x, y in points)\n        parts.append(f\'<polyline points="{path}" fill="none" stroke="{_PALETTE[0]}" stroke-width="2"/>\')\n    for (x, y), xv, yv in zip(points, xs, ys):\n        parts.append(f\'<circle cx="{_num(x)}" cy="{_num(y)}" r="3" fill="{_PALETTE[1] if not connect else _PALETTE[0]}"><title>{_esc(_num(xv))}, {_esc(_num(yv))}</title></circle>\')\n    parts.append(f\'<text x="{left}" y="{bottom + 16}" font-size="10">{_esc(_num(xlow))}</text>\')\n    parts.append(f\'<text x="{right}" y="{bottom + 16}" text-anchor="end" font-size="10">{_esc(_num(xhigh))}</text>\')\n    return _frame(width, height, title, parts)\n\n\ndef line(xs: Iterable[float], ys: Iterable[float], title: Optional[str] = None, width: int = 640, height: int = 360) -> str:\n    return _xy(list(xs), list(ys), title, width, height, True)\n\n\ndef scatter(xs: Iterable[float], ys: Iterable[float], title: Optional[str] = None, width: int = 640, height: int = 360) -> str:\n    return _xy(list(xs), list(ys), title, width, height, False)\n',
+    "xlsx_to_sqlite.py": '"""Converts an .xlsx workbook into Alisio\'s dataset SQLite file (standard library only).\n\nUsage: python -m alisio_runtime.xlsx_to_sqlite INPUT.xlsx OUTPUT.sqlite\n           [--max-rows N] [--max-columns N] [--max-cell-chars N]\n\nWrites one table ``s_<sheet>`` per worksheet plus ``_alisio_sheets`` and ``_alisio_columns``\n(name and label only: Alisio\'s host computes types and statistics afterwards, so the result is\nidentical to a CSV export of the same sheet). Cell values follow the same lossless rules as the\nNode ingestion: a cell becomes a number only when its text is a plain number (no leading zeros or\nseparators, safe integer range); everything else keeps its exact text. Dates become ISO 8601\ntext, booleans ``TRUE``/``FALSE``, formulas keep their cached value, merged cells keep the value\nof their top-left cell. Rows with no values are skipped. Errors exit with status 2 and one line\n``alisio-xlsx: <reason>`` on stderr.\n"""\n\nimport math\nimport re\nimport sqlite3\nimport sys\nimport unicodedata\nimport zipfile\nfrom datetime import datetime, timedelta\nfrom xml.etree import ElementTree as ET\n\nMAX_UNCOMPRESSED = 1024 * 1024 * 1024  # 1 GiB\nMAX_RATIO = 100  # uncompressed : compressed\nBATCH_ROWS = 5000\nRESERVED = ("rowid", "oid", "_rowid_")\n\n_NUMBER = re.compile(r"-?(?:0|[1-9]\\d*)(?:\\.\\d+)?(?:[eE][+-]?\\d+)?")\n_INTEGER = re.compile(r"-?(?:0|[1-9]\\d*)")\n_REF = re.compile(r"([A-Z]+)(\\d+)")\n_DIMENSION = re.compile(r"^[A-Z]+\\d+:([A-Z]+)\\d+$")\n\nMETA_SCHEMA = """\nCREATE TABLE IF NOT EXISTS _alisio_meta(key TEXT PRIMARY KEY, value TEXT);\nCREATE TABLE IF NOT EXISTS _alisio_sheets(ordinal INTEGER PRIMARY KEY, name TEXT NOT NULL,\n  table_name TEXT NOT NULL, rows INTEGER NOT NULL, columns INTEGER NOT NULL);\nCREATE TABLE IF NOT EXISTS _alisio_columns(table_name TEXT NOT NULL, ordinal INTEGER NOT NULL,\n  name TEXT NOT NULL, label TEXT NOT NULL, inferred_type TEXT NOT NULL,\n  nulls INTEGER, distinct_count INTEGER, distinct_exact INTEGER,\n  min TEXT, max TEXT, mean REAL, text_fallbacks INTEGER, top_values TEXT,\n  PRIMARY KEY(table_name, ordinal));\n"""\n\n\nclass XlsxError(Exception):\n    """A problem with the workbook that the user can understand."""\n\n\ndef convert_text(text):\n    """The lossless cell conversion shared with the Node ingestion (see infer.ts)."""\n    if text == "":\n        return None\n    if _NUMBER.fullmatch(text) is None:\n        return text\n    if _INTEGER.fullmatch(text) is not None:\n        if text == "-0":\n            return text\n        n = int(text)\n        return n if abs(n) <= 9007199254740991 else text\n    value = float(text)\n    return value if math.isfinite(value) else text\n\n\ndef sanitize(label, fallback):\n    text = unicodedata.normalize("NFKD", label)\n    text = "".join(c for c in text if not "̀" <= c <= "ͯ").lower()\n    text = re.sub(r"[^a-z0-9]+", "_", text).strip("_")[:48]\n    if not text:\n        return fallback\n    return "_" + text if text[0].isdigit() else text\n\n\ndef unique(name, taken):\n    candidate, n = name, 2\n    while candidate in taken:\n        candidate = f"{name}_{n}"\n        n += 1\n    taken.add(candidate)\n    return candidate\n\n\ndef looks_like_data(cells):\n    non_empty = [c for c in cells if c is not None and c != ""]\n    return bool(non_empty) and all(_NUMBER.fullmatch(c) is not None for c in non_empty)\n\n\ndef _local(tag):\n    return tag.rsplit("}", 1)[-1]\n\n\ndef _column_index(letters):\n    n = 0\n    for ch in letters:\n        n = n * 26 + (ord(ch) - 64)\n    return n - 1\n\n\nclass _Budget:\n    """Counts the bytes read from the archive (headers can lie about their sizes)."""\n\n    def __init__(self, limit):\n        self.left = limit\n\n    def wrap(self, handle):\n        budget = self\n\n        class Reader:\n            def read(self, size=-1):\n                data = handle.read(size)\n                budget.left -= len(data)\n                if budget.left < 0:\n                    raise XlsxError("The workbook expands to more than 1 GiB (possible zip bomb)")\n                return data\n\n        return Reader()\n\n\ndef _open_xml(archive, name, budget):\n    handle = archive.open(name)\n    head = handle.read(2048)\n    if b"<!DOCTYPE" in head or b"<!ENTITY" in head:\n        raise XlsxError(f"{name} declares a DOCTYPE or entities, which are not supported")\n    handle.close()\n    return budget.wrap(archive.open(name))\n\n\ndef _shared_strings(archive, budget):\n    if "xl/sharedStrings.xml" not in archive.namelist():\n        return []\n    strings = []\n    for _, elem in ET.iterparse(_open_xml(archive, "xl/sharedStrings.xml", budget)):\n        if _local(elem.tag) == "si":\n            parts = []\n            for child in elem:\n                tag = _local(child.tag)\n                if tag == "t":\n                    parts.append(child.text or "")\n                elif tag == "r":\n                    # Rich text runs; phonetic runs (rPh) hold furigana, not the cell text.\n                    parts.extend(t.text or "" for t in child if _local(t.tag) == "t")\n            strings.append("".join(parts))\n            elem.clear()\n    return strings\n\n\n_BUILTIN_DATE_IDS = set(range(14, 23)) | {27, 28, 29, 30, 31, 32, 33, 34, 35, 36} | {45, 46, 47}\n_BUILTIN_DATE_IDS |= set(range(50, 59))\n\n\ndef _is_date_format(code):\n    lowered = code.lower()\n    if lowered in ("general", "standard"):\n        return False\n    stripped = re.sub(r\'"[^"]*"|\\\\.|_.|\\*.|\\[[^\\]]*\\]\', "", lowered)\n    return re.search(r"[dmyhs]", stripped) is not None\n\n\ndef _date_styles(archive, budget):\n    """Indexes of cell styles (`s`) that format their number as a date or time."""\n    if "xl/styles.xml" not in archive.namelist():\n        return set()\n    custom = {}\n    xfs = []\n    in_xfs = False\n    for event, elem in ET.iterparse(_open_xml(archive, "xl/styles.xml", budget), events=("start", "end")):\n        name = _local(elem.tag)\n        if event == "start" and name == "cellXfs":\n            in_xfs = True\n        elif event == "end" and name == "cellXfs":\n            in_xfs = False\n        elif event == "end" and name == "numFmt":\n            custom[int(elem.get("numFmtId", "0"))] = elem.get("formatCode", "")\n        elif event == "end" and name == "xf" and in_xfs:\n            xfs.append(int(elem.get("numFmtId", "0")))\n    result = set()\n    for index, fmt in enumerate(xfs):\n        if fmt in custom:\n            if _is_date_format(custom[fmt]):\n                result.add(index)\n        elif fmt in _BUILTIN_DATE_IDS:\n            result.add(index)\n    return result\n\n\ndef serial_to_text(serial, date1904):\n    """An Excel serial date as ISO 8601 text (``None`` when it is not a representable date)."""\n    if serial < 0 or serial >= 2958466:\n        return None\n    days = int(math.floor(serial))\n    fraction = serial - days\n    seconds = int(round(fraction * 86400))\n    if seconds >= 86400:\n        days += 1\n        seconds = 0\n    clock = f"{seconds // 3600:02d}:{seconds % 3600 // 60:02d}:{seconds % 60:02d}"\n    if days == 0 and not date1904:\n        return clock\n    if date1904:\n        base = datetime(1904, 1, 1) + timedelta(days=days)\n    elif days == 60:\n        return "1900-02-29" + ("T" + clock if seconds else "")\n    elif days < 60:\n        base = datetime(1899, 12, 31) + timedelta(days=days)\n    else:\n        base = datetime(1899, 12, 30) + timedelta(days=days)\n    date = base.strftime("%Y-%m-%d")\n    return f"{date}T{clock}" if seconds else date\n\n\ndef _workbook(archive, budget):\n    """Worksheets in workbook order as (name, part path) and whether dates use the 1904 epoch."""\n    date1904 = False\n    sheets = []\n    for _, elem in ET.iterparse(_open_xml(archive, "xl/workbook.xml", budget)):\n        name = _local(elem.tag)\n        if name == "workbookPr" and elem.get("date1904") in ("1", "true"):\n            date1904 = True\n        elif name == "sheet":\n            rid = next((v for k, v in elem.attrib.items() if _local(k) == "id"), None)\n            sheets.append((elem.get("name", ""), rid))\n    targets = {}\n    rels = "xl/_rels/workbook.xml.rels"\n    if rels in archive.namelist():\n        for _, elem in ET.iterparse(_open_xml(archive, rels, budget)):\n            if _local(elem.tag) == "Relationship" and elem.get("Type", "").endswith("/worksheet"):\n                target = elem.get("Target", "")\n                targets[elem.get("Id")] = target[1:] if target.startswith("/") else "xl/" + target\n    result = [(name, targets[rid]) for name, rid in sheets if rid in targets]\n    if not result:\n        raise XlsxError("The workbook has no worksheets")\n    return result, date1904\n\n\ndef _rows(archive, part, budget, strings, date_styles, date1904, max_cell_chars):\n    """Yields (width_hint, row) where row is a dict column -> cell text, one per non-empty row."""\n    width = None\n    for event, elem in ET.iterparse(_open_xml(archive, part, budget), events=("start", "end")):\n        name = _local(elem.tag)\n        if event == "start":\n            if name == "dimension":\n                match = _DIMENSION.match(elem.get("ref", ""))\n                if match:\n                    width = _column_index(match.group(1)) + 1\n            continue\n        if name != "row":\n            continue\n        cells = {}\n        position = 0\n        for c in elem:\n            if _local(c.tag) != "c":\n                continue\n            ref = _REF.match(c.get("r", ""))\n            index = _column_index(ref.group(1)) if ref else position\n            position = index + 1\n            kind = c.get("t", "n")\n            value = None\n            text = None\n            for child in c:\n                tag = _local(child.tag)\n                if tag == "v":\n                    value = child.text\n                elif tag == "is":\n                    text = "".join(t.text or "" for t in child.iter() if _local(t.tag) == "t")\n            if kind == "inlineStr":\n                cell = text\n            elif kind == "s":\n                cell = strings[int(value)] if value not in (None, "") else None\n            elif kind == "b":\n                cell = None if value in (None, "") else ("TRUE" if value.strip() in ("1", "true") else "FALSE")\n            elif kind in ("str", "e", "d"):\n                cell = value\n            else:\n                cell = value\n                if cell not in (None, "") and int(c.get("s", "0") or 0) in date_styles:\n                    try:\n                        converted = serial_to_text(float(cell), date1904)\n                    except ValueError:\n                        converted = None\n                    if converted is not None:\n                        cell = converted\n            if cell is None or cell == "":\n                continue\n            if len(cell) > max_cell_chars:\n                raise XlsxError(f"A cell in row {elem.get(\'r\', \'?\')} exceeds {max_cell_chars} characters")\n            cells[index] = cell\n        elem.clear()\n        if cells:\n            yield width, cells\n\n\ndef convert(source, target, max_rows=5_000_000, max_columns=1000, max_cell_chars=1024 * 1024):\n    try:\n        archive = zipfile.ZipFile(source)\n    except (zipfile.BadZipFile, OSError) as error:\n        raise XlsxError(f"Not a valid .xlsx file: {error}") from error\n    with archive:\n        infos = archive.infolist()\n        total = sum(i.file_size for i in infos)\n        packed = max(1, sum(i.compress_size for i in infos))\n        if total > MAX_UNCOMPRESSED:\n            raise XlsxError("The workbook expands to more than 1 GiB (possible zip bomb)")\n        if total > 64 * 1024 * 1024 and total / packed > MAX_RATIO:\n            raise XlsxError("The workbook compresses more than 100:1 (possible zip bomb)")\n        budget = _Budget(MAX_UNCOMPRESSED)\n        sheets, date1904 = _workbook(archive, budget)\n        strings = _shared_strings(archive, budget)\n        date_styles = _date_styles(archive, budget)\n        db = sqlite3.connect(target)\n        try:\n            db.execute("PRAGMA journal_mode=OFF")\n            db.execute("PRAGMA synchronous=OFF")\n            db.executescript(META_SCHEMA)\n            taken_tables = set()\n            for ordinal, (sheet_name, part) in enumerate(sheets, start=1):\n                table = unique("s_" + sanitize(sheet_name, f"sheet_{ordinal}"), taken_tables)\n                _write_sheet(db, archive, part, budget, strings, date_styles, date1904,\n                             ordinal, sheet_name, table, max_rows, max_columns, max_cell_chars)\n            db.commit()\n        finally:\n            db.close()\n\n\ndef _write_sheet(db, archive, part, budget, strings, date_styles, date1904,\n                 ordinal, sheet_name, table, max_rows, max_columns, max_cell_chars):\n    rows = _rows(archive, part, budget, strings, date_styles, date1904, max_cell_chars)\n    pending = []\n    width = 0\n    count = 0\n    insert = None\n    db.execute("BEGIN")\n    for hint, cells in rows:\n        if insert is None:\n            width = max(hint or 0, max(cells) + 1)\n            if width > max_columns:\n                raise XlsxError(f"Sheet {sheet_name!r} has {width} columns; the limit is {max_columns}")\n            first = [cells.get(i) for i in range(width)]\n            header = not looks_like_data(first)\n            labels = [(first[i] if header and first[i] else f"column_{i + 1}") for i in range(width)]\n            taken = set(RESERVED)\n            names = [unique(sanitize(label, f"column_{i + 1}"), taken) for i, label in enumerate(labels)]\n            quoted = ", ".join(\'"\' + n.replace(\'"\', \'""\') + \'"\' for n in names)\n            db.execute(f\'CREATE TABLE "{table}"({quoted})\')\n            db.execute("INSERT INTO _alisio_sheets(ordinal,name,table_name,rows,columns) VALUES(?,?,?,0,?)",\n                       (ordinal, sheet_name, table, width))\n            db.executemany(\n                "INSERT INTO _alisio_columns(table_name,ordinal,name,label,inferred_type) VALUES(?,?,?,?,\'text\')",\n                [(table, i + 1, names[i], labels[i]) for i in range(width)])\n            insert = f\'INSERT INTO "{table}" VALUES ({",".join("?" * width)})\'\n            if header:\n                continue\n        count += 1\n        if count > max_rows:\n            raise XlsxError(f"Sheet {sheet_name!r} has more than {max_rows} rows")\n        pending.append([convert_text(cells[i]) if i in cells else None for i in range(width)])\n        if len(pending) >= BATCH_ROWS:\n            db.executemany(insert, pending)\n            pending = []\n    if insert is None:\n        # An empty sheet: keep it listed with no columns of its own (nothing to query).\n        db.execute(f\'CREATE TABLE "{table}"("column_1")\')\n        db.execute("INSERT INTO _alisio_sheets(ordinal,name,table_name,rows,columns) VALUES(?,?,?,0,1)",\n                   (ordinal, sheet_name, table))\n        db.execute("INSERT INTO _alisio_columns(table_name,ordinal,name,label,inferred_type) VALUES(?,1,\'column_1\',\'column_1\',\'text\')",\n                   (table,))\n    elif pending:\n        db.executemany(insert, pending)\n    db.execute("COMMIT")\n\n\ndef main(argv):\n    args = list(argv)\n    limits = {"max_rows": 5_000_000, "max_columns": 1000, "max_cell_chars": 1024 * 1024}\n    positional = []\n    i = 0\n    while i < len(args):\n        if args[i] in ("--max-rows", "--max-columns", "--max-cell-chars"):\n            limits[args[i][2:].replace("-", "_")] = int(args[i + 1])\n            i += 2\n        else:\n            positional.append(args[i])\n            i += 1\n    if len(positional) != 2:\n        print("alisio-xlsx: usage: xlsx_to_sqlite INPUT.xlsx OUTPUT.sqlite", file=sys.stderr)\n        return 2\n    try:\n        convert(positional[0], positional[1], **limits)\n    except XlsxError as error:\n        print(f"alisio-xlsx: {error}", file=sys.stderr)\n        return 2\n    except (ET.ParseError, KeyError, zipfile.BadZipFile) as error:\n        print(f"alisio-xlsx: The workbook is damaged or unsupported ({type(error).__name__}: {error})", file=sys.stderr)\n        return 2\n    return 0\n\n\nif __name__ == "__main__":\n    sys.exit(main(sys.argv[1:]))\n',
+};

package/dist/analysis/requirements.d.ts

@@ -0,0 +1,32 @@
+/**
+ * Pinned, hash-locked requirements of the optional Python extras (decision D5, recommendation
+ * adopted pending owner confirmation). Generated with uv, wheels only, for every platform:
+ *
+ *   uv pip compile <set>.in --universal --generate-hashes --python-version 3.10 \
+ *     --only-binary :all: -o <set>.txt
+ *
+ * where `analysis.in` lists pandas, numpy, matplotlib, openpyxl, python-docx, reportlab, plotly
+ * and jinja2, and `science.in` adds scipy, statsmodels and scikit-learn. Comment lines are
+ * stripped. Kept as strings so the npm package and the standalone binary carry them.
+ */
+export declare const EXTRAS_PACKAGES: {
+    readonly analysis: readonly ["pandas", "numpy", "matplotlib", "openpyxl", "docx", "reportlab", "plotly", "jinja2"];
+    readonly science: readonly ["pandas", "numpy", "matplotlib", "openpyxl", "docx", "reportlab", "plotly", "jinja2", "scipy", "statsmodels", "sklearn"];
+};
+/** The distributions the user recognizes (the lockfile also pins their dependencies). */
+export declare const EXTRAS_DISTRIBUTIONS: {
+    readonly analysis: readonly ["pandas", "numpy", "matplotlib", "openpyxl", "python-docx", "reportlab", "plotly", "jinja2"];
+    readonly science: readonly ["pandas", "numpy", "matplotlib", "openpyxl", "python-docx", "reportlab", "plotly", "jinja2", "scipy", "statsmodels", "scikit-learn"];
+};
+/**
+ * Download size of the wheels, rounded up from a measurement on Linux x86-64 with Python 3.10
+ * (`pip download --only-binary=:all:`: 68.9 MiB and 126.5 MiB). Other platforms differ a little; the
+ * approval says "about".
+ */
+export declare const EXTRAS_ESTIMATED_BYTES: {
+    readonly analysis: number;
+    readonly science: number;
+};
+/** Distinct pinned distributions of a lockfile (versions per Python marker count once). */
+export declare function lockedDistributionCount(extras: "analysis" | "science"): number;
+export declare const EXTRAS_LOCK: Record<"analysis" | "science", string>;