@alisio/core 0.5.2 → 0.6.0

package/dist/forge/guides/provider.js

@@ -0,0 +1,96 @@
+/**
+ * `GuideProvider` (spec §8.3, D12, Lazy Load): layer 2 of the incremental context. Each topic is
+ * generated on request from the live state of the engines, so a guide never describes a limit, a
+ * package or a guarantee the engine does not have right now. Nothing is built at construction.
+ * Every topic is at most 4 KiB and carries an `etag` (truncated SHA-256 of its content); a caller
+ * that already holds the same etag gets `{ unchanged: true }`.
+ */
+import { createHash } from "node:crypto";
+import { jsApiGuide, jsPatternsGuide, jsToolsGuide, } from "./javascript.js";
+import { pythonDashboardsGuide, pythonDataGuide, pythonEnvironmentGuide, pythonMlGuide, pythonRuntimeGuide, } from "./python.js";
+export const JS_GUIDE_TOPICS = ["js.api", "js.patterns", "js.tools"];
+export const PYTHON_GUIDE_TOPICS = [
+    "python.runtime",
+    "python.data",
+    "python.ml",
+    "python.dashboards",
+    "python.environment",
+];
+export const GUIDE_TOPICS = [...JS_GUIDE_TOPICS, ...PYTHON_GUIDE_TOPICS];
+/** Hard bound of one topic (spec §8.3). */
+export const GUIDE_MAX_BYTES = 4096;
+/** Characters of the SHA-256 kept as the etag. */
+export const ETAG_LENGTH = 16;
+export class UnknownGuideTopicError extends Error {
+    constructor(topic, available) {
+        super(`Unknown guide topic "${topic}". Available: ${available.join(", ")}`);
+        this.name = "UnknownGuideTopicError";
+    }
+}
+export const etagOf = (content) => createHash("sha256").update(content).digest("hex").slice(0, ETAG_LENGTH);
+/** Cuts to the byte bound on a character boundary, saying so. */
+function bounded(content) {
+    if (Buffer.byteLength(content) <= GUIDE_MAX_BYTES)
+        return content;
+    const note = "\n[guide cut at 4 KiB]";
+    let out = content;
+    while (Buffer.byteLength(out) + Buffer.byteLength(note) > GUIDE_MAX_BYTES)
+        out = out.slice(0, -64);
+    return out + note;
+}
+export class GuideProvider {
+    #sources;
+    constructor(sources) {
+        this.#sources = sources;
+    }
+    /** The topics that exist now: the JavaScript ones with a ready engine, the Python ones with `python_run`. */
+    topics() {
+        return [
+            ...(this.#sources.javascript ? JS_GUIDE_TOPICS : []),
+            ...(this.#sources.python ? PYTHON_GUIDE_TOPICS : []),
+        ];
+    }
+    async get(request) {
+        const available = this.topics();
+        const topic = available.find((candidate) => candidate === request.topic);
+        if (!topic)
+            throw new UnknownGuideTopicError(request.topic, available);
+        const content = bounded(await this.#render(topic, request));
+        const etag = etagOf(content);
+        if (request.ifNoneMatch !== undefined && request.ifNoneMatch === etag)
+            return { unchanged: true, etag };
+        return { unchanged: false, topic, etag, content };
+    }
+    async #render(topic, request) {
+        switch (topic) {
+            case "js.api":
+                return jsApiGuide(await this.#js());
+            case "js.patterns":
+                return jsPatternsGuide(await this.#js());
+            case "js.tools":
+                return jsToolsGuide(request.catalog);
+            case "python.runtime":
+                return pythonRuntimeGuide(await this.#py());
+            case "python.data":
+                return pythonDataGuide(await this.#py());
+            case "python.ml":
+                return pythonMlGuide(await this.#py());
+            case "python.dashboards":
+                return pythonDashboardsGuide(await this.#py());
+            case "python.environment":
+                return pythonEnvironmentGuide(await this.#py());
+        }
+    }
+    #js() {
+        const source = this.#sources.javascript;
+        if (!source)
+            throw new Error("the JavaScript engine is not available");
+        return source();
+    }
+    #py() {
+        const source = this.#sources.python;
+        if (!source)
+            throw new Error("python_run is not available");
+        return source();
+    }
+}

package/dist/forge/guides/python-environment.d.ts

@@ -0,0 +1,15 @@
+import type { PythonBackend, PythonBackendId } from "../python/backend.ts";
+import { type IsolationPolicy, type PythonRuntimeSetting } from "../python/backend-selector.ts";
+import type { PythonEnvironment } from "./python.ts";
+export interface PythonEnvironmentDeps {
+    backends: Partial<Record<PythonBackendId, PythonBackend>>;
+    /** `analysis.runtime`, read live. */
+    runtime(): PythonRuntimeSetting;
+    /** `analysis.isolation`, read live. */
+    isolation(): IsolationPolicy;
+    /** The extras installed in the managed virtual environment (`analysis`, `science`). */
+    managedExtras(): Promise<readonly string[]>;
+    /** `analysis.smartDashboard`. */
+    smartDashboard(): boolean;
+}
+export declare function probePythonEnvironment(deps: PythonEnvironmentDeps, signal?: AbortSignal): Promise<PythonEnvironment>;

package/dist/forge/guides/python-environment.js

@@ -0,0 +1,81 @@
+/**
+ * Gathers the `PythonEnvironment` snapshot the Python guides are written from: the backend the
+ * selector would choose RIGHT NOW (same chain, same settings as `python_run`, no script known) and
+ * the packages that backend really has. It probes readiness like a call would and never starts a
+ * script or installs anything.
+ */
+import { EXTRAS_DISTRIBUTIONS } from "../../analysis/requirements.js";
+import { isForgeError } from "../errors.js";
+import { createPythonBackendSelector, neededBackends, } from "../python/backend-selector.js";
+import { PYODIDE_PROFILE_ORDER, PYODIDE_PROFILES } from "../python/pyodide-manifest.js";
+const selector = createPythonBackendSelector();
+async function packagesOf(backend, deps) {
+    if (backend.id === "wasm") {
+        const installed = (await backend.describe()).profiles ?? [];
+        const top = PYODIDE_PROFILE_ORDER.filter((profile) => installed.includes(profile)).at(-1);
+        return top
+            ? {
+                known: true,
+                packages: [...PYODIDE_PROFILES[top]],
+                detail: `Pyodide profile "${top}" is installed (profiles: ${installed.join(", ")}).`,
+            }
+            : { known: true, packages: [], detail: "No Pyodide package profile is installed (base)." };
+    }
+    if (backend.id === "managed") {
+        const extras = await deps.managedExtras();
+        const top = extras.includes("science")
+            ? "science"
+            : extras.includes("analysis")
+                ? "analysis"
+                : undefined;
+        return top
+            ? {
+                known: true,
+                packages: [...EXTRAS_DISTRIBUTIONS[top]],
+                detail: `The "${top}" extras are installed in the managed environment.`,
+            }
+            : {
+                known: true,
+                packages: [],
+                detail: "No extras are installed; pass `extras` to ask the user to install them.",
+            };
+    }
+    return {
+        known: false,
+        detail: "A container has the packages of its image; scripts cannot install anything (no network).",
+    };
+}
+export async function probePythonEnvironment(deps, signal) {
+    const runtime = deps.runtime();
+    const isolation = deps.isolation();
+    const smartDashboard = deps.smartDashboard();
+    const wanted = neededBackends({ runtime, isolation });
+    const readiness = {};
+    const networkDenied = {};
+    await Promise.all(wanted.map(async (id) => {
+        const backend = deps.backends[id];
+        readiness[id] = backend
+            ? await backend.check(signal)
+            : { ready: false, reason: `${id} is not available here` };
+        if (backend && readiness[id]?.ready && isolation === "strict")
+            networkDenied[id] = (await backend.describe()).capabilities.networkDeniedByOS;
+    }));
+    try {
+        const chosen = selector.select({ runtime, isolation, readiness, networkDenied });
+        const backend = deps.backends[chosen.engine];
+        if (!backend)
+            return { available: false, reason: `${chosen.engine} is not available here`, smartDashboard };
+        return {
+            available: true,
+            backend: chosen.engine,
+            reason: chosen.reason,
+            packages: await packagesOf(backend, deps),
+            smartDashboard,
+        };
+    }
+    catch (error) {
+        if (!isForgeError(error))
+            throw error;
+        return { available: false, reason: error.message, smartDashboard };
+    }
+}

package/dist/forge/guides/python.d.ts

@@ -0,0 +1,38 @@
+/**
+ * The Python topics of `forge_guide` (spec §8.3; Python spec §4.3, §6, §7, §8): `python.runtime`,
+ * `python.data`, `python.ml`, `python.dashboards` and `python.environment`. Pure functions of a
+ * `PythonEnvironment` snapshot: the backend the selector would choose right now and what that
+ * backend really provides. No guide ever claims a guarantee the chosen backend does not give.
+ */
+import type { PythonBackendId } from "../python/backend.ts";
+export type PackageKnowledge = 
+/** The packages this backend provides right now (a distribution name each). */
+{
+    known: true;
+    packages: string[];
+    detail: string;
+}
+/** The backend's packages cannot be listed from here (a container image). */
+ | {
+    known: false;
+    detail: string;
+};
+export type PythonEnvironment = {
+    available: true;
+    backend: PythonBackendId;
+    /** Why the selector chose it (its own `reason`). */
+    reason: string;
+    packages: PackageKnowledge;
+    /** `analysis.smartDashboard`: `dashboard_generate` exists. */
+    smartDashboard: boolean;
+} | {
+    available: false;
+    /** The selector's message: every backend's reason and the shortest fix. */
+    reason: string;
+    smartDashboard: boolean;
+};
+export declare function pythonRuntimeGuide(env: PythonEnvironment): string;
+export declare function pythonDataGuide(env: PythonEnvironment): string;
+export declare function pythonMlGuide(env: PythonEnvironment): string;
+export declare function pythonDashboardsGuide(env: PythonEnvironment): string;
+export declare function pythonEnvironmentGuide(env: PythonEnvironment): string;

package/dist/forge/guides/python.js

@@ -0,0 +1,170 @@
+/** Python spec §4.3, one row per backend; the guide never states more than this. */
+const GUARANTEES = {
+    managed: [
+        "Isolation: none. The script runs with your user permissions: it can read your files and use the network.",
+        "Termination: the process is killed on timeout. Memory: no hard limit.",
+    ],
+    oci: [
+        "Isolation: os-container. Network off (`--network=none`); only the job folders are mounted (the scratchpad read-only).",
+        "Termination: the container is killed on timeout. Memory: hard limit (cgroup).",
+    ],
+    wasm: [
+        "Isolation: wasm-process (Pyodide in a restricted Node process). It reads only the Pyodide distribution and the job, and writes only its work/ and out/ folders.",
+        "Network: NOT blocked by the OS on Node 22/24; do not rely on it being off. Memory: soft limit (RSS watchdog). Termination: the process is killed on timeout.",
+    ],
+};
+const SETUP_NOTE = "Setup commands are run by the user, never by you: `alisio analysis setup --wasm [--profile analysis|science]` or `alisio analysis setup --oci`.";
+const has = (env, pkg) => {
+    if (!env.available)
+        return false;
+    if (!env.packages.known)
+        return undefined;
+    return env.packages.packages.includes(pkg);
+};
+/** What the model should know about one package group right now. */
+function availability(env, pkgs) {
+    const states = pkgs.map((pkg) => has(env, pkg));
+    if (states.every((s) => s === true))
+        return "available";
+    if (states.every((s) => s === undefined))
+        return "depends on the container image (not listed here)";
+    const missing = pkgs.filter((_, i) => states[i] === false);
+    return `NOT available (${missing.join(", ")} missing)`;
+}
+function installAdvice(env, profile) {
+    if (env.backend === "wasm")
+        return `ask the user to run \`alisio analysis setup --wasm --profile ${profile}\``;
+    if (env.backend === "managed")
+        return `pass \`extras: ["${profile}"]\` (the user approves the download once)`;
+    return "use an image that includes them (`analysis.oci.image`)";
+}
+export function pythonRuntimeGuide(env) {
+    return [
+        "# python_run runtime",
+        "",
+        "Your script runs once in a job folder outside the repository, with `main.py` as the entry. The helper package `alisio_runtime` (standard library only) is importable.",
+        "",
+        "## Folders (environment variables)",
+        "- `ALISIO_OUTPUT_DIR`: everything written here is published as a downloadable artifact only if the script exits 0 (`publishOnError` keeps partial output). Publication is validated by the host: paths, sizes and types. An `outputs.json` limits publishing to the files it declares.",
+        "- `ALISIO_INPUT_DIR`: read-only copies of `inputs`. `ALISIO_WORK_DIR`: scratch, also the current directory.",
+        "- `ALISIO_SCRATCH_DIR`: the session scratchpad, READ-ONLY (write scripts there with the file tools, run them with `codePath`).",
+        "- `ALISIO_EXECUTION_ID` identifies the run. `ALISIO_FORGE_*` belong to the protocol: never print them or fake their lines.",
+        "",
+        "## alisio_runtime",
+        "- `output_dir()`, `input_dir()`, `work_dir()`, `execution_id()`; `outputs.path(name)` makes a new output path, `outputs.declare(name, title)` registers it.",
+        "- `forge.progress(message, completed=, total=)`, `forge.text(value)` (explicit output for you), `forge.result(value)` (the final JSON value), `forge.state.get/set/delete` (small JSON shared with js_run; kept only if the run succeeds).",
+        "- `datasets`, `html`, `charts`, `svg`: see `python.data` and `python.dashboards`.",
+        "",
+        "## Rules",
+        "- Print little: stdout and stderr return only their last 4 KB. Use `forge.text`/`forge.result` for what you need to read.",
+        "- No nested tool calls and no JavaScript from Python. Chain tools yourself, or compose with `js_run` first.",
+        "- Keep each script short; save long ones to the scratchpad. Rerun an earlier analysis unchanged with `rerunOf`.",
+        `- Packages depend on the backend: see \`python.environment\` (now: ${env.available ? env.backend : "no backend can run"}).`,
+    ].join("\n");
+}
+export function pythonDataGuide(env) {
+    const pandas = availability(env, ["pandas"]);
+    return [
+        "# Data in python_run",
+        "",
+        "## Which tool",
+        "- A SQL question on a loaded dataset: `data_query` (no code, read-only). Use `data_inspect` first for tables and columns.",
+        "- Joins across files, statistics, models, exports, or anything SQL cannot do: `python_run`.",
+        "",
+        "## Inputs",
+        "- `inputs: [{ datasetId }]` copies a dataset to `$ALISIO_INPUT_DIR/<name>.sqlite`; `inputs: [{ path }]` copies a workspace file.",
+        "- `from alisio_runtime import datasets`: `datasets.inputs()` lists what was placed; `datasets.open(name)` returns a read-only `sqlite3` connection (table `data`, or `s_<sheet>` for workbooks); `datasets.tables(name)` and `datasets.columns(name, table)` describe them.",
+        "- XLSX files are converted to SQLite by `alisio_runtime/xlsx_to_sqlite.py` (standard library `sqlite3`); you rarely need to call it yourself.",
+        "",
+        "## pandas",
+        `- pandas is ${pandas} on the backend that would run now.`,
+        "- Without pandas use `sqlite3`, `csv`, `statistics` and `json` from the standard library.",
+        '- `pandas.read_sql_query("SELECT * FROM data", datasets.open(name))` loads a dataset.',
+        ...(env.available && env.backend === "wasm"
+            ? ["- The wasm backend limits the total input size (`analysis.wasm.maxInputBytes`)."]
+            : []),
+    ].join("\n");
+}
+export function pythonMlGuide(env) {
+    const science = availability(env, ["scipy", "statsmodels", "scikit-learn"]);
+    const where = env.available
+        ? `Backend that would run now: ${env.backend}.`
+        : "No Python backend can run right now (see `python.environment`).";
+    return [
+        "# Statistics and machine learning",
+        "",
+        where,
+        `- scikit-learn, statsmodels and scipy are ${science}.`,
+        ...(env.available && has(env, "scikit-learn") === false
+            ? [`- To get them: ${installAdvice(env, "science")}.`]
+            : []),
+        "",
+        "## Patterns",
+        "- Split first: `train_test_split(X, y, test_size=0.2, random_state=0)`; fix `random_state` so a rerun gives the same numbers.",
+        "- Tabular baseline: `sklearn.ensemble.RandomForestClassifier` or `sklearn.linear_model.LogisticRegression`; report accuracy AND a baseline (majority class) with `forge.result({...})`.",
+        "- Inference and tests: `scipy.stats.ttest_ind`, `chi2_contingency`; regression with coefficients and p-values: `statsmodels.api.OLS(y, sm.add_constant(X)).fit().summary()`.",
+        '- Return numbers, not objects: `forge.result({"r2": float(r2), "n": int(n)})`. JSON cannot hold NaN or Infinity; convert them first.',
+        "- Keep a time budget in mind: heavy training can exceed the call's timeout. Subsample first, then scale up.",
+        "- Pandas first, then models; without the science packages use the standard library `statistics` for descriptive work.",
+    ].join("\n");
+}
+export function pythonDashboardsGuide(env) {
+    const plotly = availability(env, ["plotly"]);
+    const matplotlib = availability(env, ["matplotlib"]);
+    return [
+        "# Charts and dashboards",
+        "",
+        ...(env.smartDashboard
+            ? [
+                "- A standard dashboard from a dataset: `dashboard_generate` (no code). Use `python_run` only for visuals its catalog does not cover.",
+            ]
+            : ["- `dashboard_generate` is not enabled here: build dashboards with `alisio_runtime`."]),
+        "",
+        "## alisio_runtime (no extra packages)",
+        "- `charts`: Chart.js is bundled. `charts.pie/donut/bar/hbar/line/area/scatter(labels, values, title=, fmt='currency:USD', locale=)` return HTML; compose with `charts.card(title, chart)`, `charts.grid(*cards)`, `charts.kpis((label, value), ...)`, then `charts.write('dashboard.html', 'Title', body)`.",
+        "- `html`: `html.page/table/write` for plain reports. `svg`: static SVG, no script.",
+        "- Never hand-write SVG arcs, never load a CDN: HTML is shown offline, so inline every script and style. One chart per card; pie only for 2 to 5 parts.",
+        "",
+        "## Static images",
+        `- matplotlib is ${matplotlib}. It runs with the Agg backend: save a PNG with \`fig.savefig(output_dir() / 'chart.png', dpi=150)\`; \`plt.show()\` does nothing.`,
+        `- plotly is ${plotly}. Write a self-contained file with \`fig.write_html(path, include_plotlyjs=True)\` (several MB); prefer \`charts\` when it is enough.`,
+        "",
+        "A folder under the output directory with an `index.html` is published as one multi-file dashboard.",
+    ].join("\n");
+}
+export function pythonEnvironmentGuide(env) {
+    if (!env.available)
+        return [
+            "# Python environment",
+            "",
+            "No Python backend can run a script right now.",
+            "",
+            env.reason,
+            "",
+            SETUP_NOTE,
+            "Until one is ready, use `js_run` or the dataset tools (`data_inspect`, `data_query`).",
+        ].join("\n");
+    const packages = env.packages.known
+        ? env.packages.packages.length
+            ? env.packages.packages.join(", ")
+            : "none beyond the standard library"
+        : "those baked into the container image (not listed here)";
+    return [
+        "# Python environment",
+        "",
+        `Backend that runs your next script: ${env.backend} (${env.reason}).`,
+        "",
+        "## Guarantees of this backend",
+        ...GUARANTEES[env.backend].map((line) => `- ${line}`),
+        "",
+        "## Packages here",
+        `- Standard library and \`alisio_runtime\` always. ${env.packages.detail}`,
+        `- Installed: ${packages}.`,
+        "- Anything else fails with `PACKAGE_UNAVAILABLE`/`ModuleNotFoundError` and is never retried on another backend.",
+        ...(env.packages.known && !env.packages.packages.includes("scikit-learn")
+            ? ["", `For scikit-learn, statsmodels and scipy: ${installAdvice(env, "science")}.`]
+            : []),
+        "",
+        SETUP_NOTE,
+    ].join("\n");
+}

package/dist/forge/javascript/engine.d.ts

@@ -0,0 +1,44 @@
+import type { EngineDescriptor } from "@alisio/sdk";
+import type { EngineOutcome, ExecutionEngine, ExecutionRequest, IsolateHost, IsolateSession, KernelChannel, PreparedExecution, TerminationReason, TerminationReport, TrustedExecutionContext } from "../ports.ts";
+import type { IsolateHostFactory } from "./hosts/factory.ts";
+export interface PreparedJavaScript extends PreparedExecution {
+    readonly source: string;
+    /** The state snapshot the script starts with. */
+    readonly state: Record<string, unknown>;
+    readonly catalogSize: number;
+    readonly host: IsolateHost;
+    session?: IsolateSession;
+    /** Ends the run with a host verdict; set by `run`, used when a stop cannot be confirmed. */
+    forceEnd?: () => void;
+}
+export interface JavaScriptEngineOptions {
+    hosts: IsolateHostFactory;
+    /** The verified module; defaults to the process-wide cache. */
+    loadModule?: () => Promise<WebAssembly.Module>;
+}
+export declare class JavaScriptEngine implements ExecutionEngine {
+    #private;
+    readonly id: "javascript-quickjs";
+    constructor(options: JavaScriptEngineOptions);
+    /**
+     * Cheap self-check (runs at startup): the WASM matches its pinned hash and compiles, and the
+     * files the chosen host starts exist. The full spin-up of an isolate is `probe()`.
+     */
+    describe(): Promise<EngineDescriptor>;
+    /** The kind of host that runs the isolates (`forge status`). */
+    get hostKind(): "worker" | "process";
+    prepare(request: ExecutionRequest, ctx: TrustedExecutionContext): Promise<PreparedJavaScript>;
+    run(prepared: PreparedExecution, channel: KernelChannel): Promise<EngineOutcome>;
+    terminate(prepared: PreparedExecution, reason: TerminationReason): Promise<TerminationReport>;
+    dispose(prepared: PreparedExecution): Promise<void>;
+    /**
+     * Full self-test (`alisio forge status`): starts a real isolate, runs `return 1` and reports how
+     * long it took. Nothing here is on the hot path of a normal execution.
+     */
+    probe(): Promise<{
+        ok: boolean;
+        reason?: string;
+        bootMs?: number;
+        host: "worker" | "process";
+    }>;
+}