@alisio/core 0.5.2 → 0.6.0

package/dist/forge/python/engine.js

@@ -0,0 +1,279 @@
+/**
+ * `PythonEngine` (Python spec §4, PD1): the `ExecutionEngine` that runs a `python_run` script
+ * through a `PythonBackend`. It reuses what `python_run` always did, only moved behind the kernel's
+ * lifecycle: `prepare` creates the job folders and copies the authorized inputs, `run` launches the
+ * script and reads its stdout frames, `collect` publishes the files it left (before the shared
+ * state is committed), `terminate` stops the process.
+ *
+ * One instance per backend (`python-managed`, `python-oci`). Everything that belongs to ONE call
+ * (the code, the inputs, the publisher, where output goes) travels in the request payload as a
+ * `PythonRunPlan`; the engine keeps nothing between calls. `PythonRunReport` is how the tool gets
+ * back what the kernel's `ForgeResult` does not carry (the raw process result, the artifacts).
+ */
+import { randomBytes } from "node:crypto";
+import { basename } from "node:path";
+import { ArtifactRejected } from "../../artifacts/store.js";
+import { FORGE_ERROR_CODES, ForgeError } from "../errors.js";
+import { stripFrames } from "./frame-parser.js";
+import { StdoutFrameTransport } from "./frame-transport.js";
+/** Key of the plan in `ExecutionRequest.payload`. */
+export const PYTHON_PLAN_KEY = "python";
+/** The terminal text the model and the UI read is bounded to this many bytes. */
+export const TERMINAL_BYTES = 256 * 1024;
+/** A process that ignores the kernel's stop is killed by its own backstop this much later. */
+const BACKSTOP_SLACK_MS = 15_000;
+export const newPythonRunReport = () => ({
+    inputs: [],
+    published: [],
+    warnings: [],
+});
+const KNOWN_CODES = new Set(FORGE_ERROR_CODES);
+export class PythonEngine {
+    id;
+    #backend;
+    #jobs;
+    constructor(deps) {
+        this.#backend = deps.backend;
+        this.#jobs = deps.jobs;
+        this.id = `python-${deps.backend.id}`;
+    }
+    describe() {
+        return this.#backend.describe();
+    }
+    async prepare(request, ctx) {
+        const plan = request.payload?.[PYTHON_PLAN_KEY];
+        if (!plan)
+            throw new ForgeError("INVALID_INPUT", "python_run did not provide its run plan");
+        const { report } = plan;
+        const jobs = this.#jobs;
+        const rerun = plan.rerun;
+        let job;
+        try {
+            job = await jobs.create({
+                sessionId: plan.sessionId,
+                rootSessionId: plan.rootSessionId,
+                workspace: plan.workspace,
+                ...(plan.runId ? { runId: plan.runId } : {}),
+                ...(plan.callId ? { callId: plan.callId } : {}),
+                code: plan.code,
+                runtime: this.#backend.id,
+                ...(rerun ? { rerunOf: rerun.executionId } : {}),
+            });
+            report.job = job;
+            const inputs = report.inputs;
+            let manifest = [];
+            if (rerun) {
+                // The verified copies of the original run, under the same names.
+                for (const item of rerun.inputs) {
+                    const added = await jobs.addInput(job, item.source, item.name);
+                    if (added.sha256 !== (item.copySha256 ?? item.sha256) && !item.datasetId)
+                        throw new Error(`Input ${item.name} changed while the rerun started`);
+                    inputs.push({
+                        ...added,
+                        sha256: item.sha256,
+                        copySha256: added.sha256,
+                        ...(item.datasetId ? { datasetId: item.datasetId } : {}),
+                    });
+                }
+                manifest = rerun.manifest;
+            }
+            else
+                for (const item of plan.inputs) {
+                    if ((item.path === undefined) === (item.datasetId === undefined))
+                        throw new Error("Each input needs exactly one of path or datasetId");
+                    if (item.datasetId !== undefined) {
+                        if (!plan.datasets)
+                            throw new Error("Datasets are not available here");
+                        const record = plan.datasets.getFor(item.datasetId, plan.rootSessionId);
+                        const added = await jobs.addInput(job, plan.datasets.file(record), `${basename(record.name)
+                            .replace(/\.[^.]*$/, "")
+                            .replace(/[^\w.-]+/g, "_") || "dataset"}.sqlite`);
+                        // Provenance names the original content, so a rerun can verify it by hash.
+                        inputs.push({
+                            ...added,
+                            sha256: record.sha256,
+                            copySha256: added.sha256,
+                            datasetId: record.id,
+                        });
+                        manifest.push({
+                            name: added.name,
+                            kind: "dataset",
+                            datasetId: record.id,
+                            source: record.name,
+                            format: record.format,
+                            sheets: record.sheets,
+                        });
+                        continue;
+                    }
+                    const resolved = await plan.resolvePath(item.path);
+                    const added = await jobs.addInput(job, resolved);
+                    inputs.push({ ...added, copySha256: added.sha256 });
+                    manifest.push({ name: added.name, kind: "file", bytes: added.bytes });
+                }
+            if (manifest.length)
+                await jobs.writeInputsManifest(job, manifest);
+            await jobs.recordInputs(job, {
+                inputs,
+                manifest,
+                ...(plan.extrasAsked.length ? { extras: plan.extrasAsked } : {}),
+            });
+            const state = request.payload?.state;
+            await jobs.writeForgeState(job, state ?? {});
+        }
+        catch (error) {
+            report.prepareError = error;
+            if (job)
+                jobs.finish(job.id, "failed", {
+                    error: error instanceof Error ? error.message : String(error),
+                });
+            throw error;
+        }
+        const nonce = randomBytes(16).toString("hex");
+        report.nonce = nonce;
+        const prepared = {
+            executionId: ctx.executionId,
+            engine: this.id,
+            limits: ctx.limits,
+            analysisExecutionId: job.id,
+            plan,
+            job,
+            nonce,
+            abort: new AbortController(),
+            finished: false,
+        };
+        return prepared;
+    }
+    async run(prepared, channel) {
+        const p = prepared;
+        const { plan, job } = p;
+        const { report } = plan;
+        let failure;
+        const transport = new StdoutFrameTransport({
+            nonce: p.nonce,
+            receive: (message) => channel.receive(message),
+            onText: plan.onText,
+            onFailure: (error) => {
+                failure = error;
+                p.abort.abort();
+            },
+        });
+        const signal = AbortSignal.any([channel.signal, p.abort.signal]);
+        const started = Date.now();
+        const launchJob = {
+            executionId: job.id,
+            paths: job,
+            env: { ALISIO_FORGE_NONCE: p.nonce, ALISIO_FORGE_ENGINE: this.id },
+            ...(plan.scratch ? { scratch: plan.scratch } : {}),
+            signal,
+            backstopMs: p.limits.timeoutMs + BACKSTOP_SLACK_MS,
+            maxBytes: TERMINAL_BYTES,
+            maxLogBytes: plan.maxLogBytes,
+            logFiles: {
+                stdout: `${job.logs}/stdout.log`,
+                stderr: `${job.logs}/stderr.log`,
+            },
+        };
+        try {
+            const running = await this.#backend.launch(launchJob, {
+                onData: (chunk, stream) => stream === "stdout" ? transport.feed(chunk) : plan.onText(chunk),
+            });
+            p.running = running;
+            report.runtime = running.runtime;
+            const result = await running.result;
+            report.process = { ...result, stdout: stripFrames(result.stdout, p.nonce) };
+        }
+        catch (error) {
+            if (!signal.aborted)
+                report.crash = error;
+        }
+        finally {
+            report.durationMs = Date.now() - started;
+            await transport.end();
+            report.warnings.push(...transport.warnings);
+        }
+        if (failure)
+            return { kind: "failed", code: failure.code, message: failure.message };
+        // A limit the backend enforced by killing the process (the wasm memory watchdog).
+        const limit = p.running?.limit?.();
+        if (limit && !signal.aborted)
+            return { kind: "failed", code: limit.code, message: limit.message };
+        if (signal.aborted)
+            return { kind: "failed", code: "ISOLATE_FAILURE", message: "the script was stopped" };
+        if (report.crash)
+            return {
+                kind: "failed",
+                code: "ISOLATE_FAILURE",
+                message: report.crash instanceof Error ? report.crash.message : String(report.crash),
+            };
+        const exitCode = report.process?.exitCode ?? 1;
+        p.finished = true;
+        const fatal = transport.fatal;
+        if (fatal)
+            return {
+                kind: "failed",
+                code: (KNOWN_CODES.has(fatal.code) ? fatal.code : "SCRIPT_RUNTIME"),
+                message: fatal.message,
+            };
+        if (exitCode !== 0)
+            return {
+                kind: "failed",
+                code: "SCRIPT_RUNTIME",
+                message: `the script exited with code ${exitCode}`,
+            };
+        const value = transport.completion?.value;
+        return { kind: "completed", ...(value !== undefined ? { value } : {}) };
+    }
+    /** Publishes the files the script left, when the run earned it (exit 0, or `publishOnError`). */
+    async collect(prepared, outcome) {
+        const p = prepared;
+        const { plan, job } = p;
+        const { report } = plan;
+        if (!p.finished || !p.running)
+            return outcome;
+        const failed = outcome.kind === "failed";
+        if (failed && !plan.publishOnError)
+            return outcome;
+        try {
+            report.published = await plan.publisher.publishOutputs(job.staging, {
+                executionId: job.id,
+                ...(plan.title ? { title: plan.title } : {}),
+                ...(failed ? { partial: true } : {}),
+                provenance: {
+                    runtime: p.running.runtime,
+                    scriptSha256: job.scriptSha256,
+                    inputs: report.inputs.map((i) => ({
+                        name: i.name,
+                        sha256: i.sha256,
+                        ...(i.datasetId ? { datasetId: i.datasetId } : {}),
+                    })),
+                    ...(plan.rerun
+                        ? {
+                            rerunOf: plan.rerun.executionId,
+                            ...(plan.rerun.artifactId ? { rerunOfArtifact: plan.rerun.artifactId } : {}),
+                        }
+                        : {}),
+                    engine: plan.engine.id,
+                    isolation: plan.engine.isolation,
+                    selectionReason: plan.engine.selectionReason,
+                },
+            });
+        }
+        catch (error) {
+            if (!(error instanceof ArtifactRejected))
+                throw error;
+            report.publishError = error.message;
+            return { kind: "failed", code: "PUBLICATION_REJECTED", message: error.message };
+        }
+        return outcome;
+    }
+    async terminate(prepared, reason) {
+        const p = prepared;
+        p.abort.abort();
+        if (!p.running)
+            return { confirmed: true, method: "kill", elapsedMs: 0 };
+        return this.#backend.terminate(p.running, reason);
+    }
+    async dispose(prepared) {
+        prepared.abort.abort();
+    }
+}

package/dist/forge/python/frame-parser.d.ts

@@ -0,0 +1,24 @@
+export declare const FRAME_MARK = "\u001EALISIO-FORGE:";
+/** The longest frame body kept, in characters (a character is at least one byte of the cap). */
+export declare const MAX_FRAME_CHARS: number;
+export type FrameEvent = {
+    type: "text";
+    text: string;
+} | {
+    type: "frame";
+    json: string;
+}
+/** A frame line longer than the cap: its body was dropped. */
+ | {
+    type: "oversize";
+};
+export declare const framePrefix: (nonce: string) => string;
+export declare class FrameDecoder {
+    #private;
+    constructor(nonce: string);
+    feed(chunk: string): FrameEvent[];
+    /** The stream ended: whatever is held was never completed as a frame line. */
+    flush(): FrameEvent[];
+}
+/** `text` without its frame lines (the form of stdout the model and the terminal see). */
+export declare function stripFrames(text: string, nonce: string): string;

package/dist/forge/python/frame-parser.js

@@ -0,0 +1,107 @@
+/**
+ * Pure line framing of the Python stdout transport (Python spec §7). Nothing here does I/O.
+ *
+ * A frame is a stdout line that STARTS with `\u001eALISIO-FORGE:<nonce>:` followed by JSON. Every
+ * other byte of stdout is ordinary script output and is handed back unchanged, in order, as soon as
+ * it is known not to be a frame (a plain line streams chunk by chunk; only a line that still looks
+ * like the start of a frame is held). The nonce only tells frames from output: it grants nothing.
+ */
+import { MAX_MESSAGE_BYTES } from "../protocol.js";
+export const FRAME_MARK = "\u001eALISIO-FORGE:";
+/** The longest frame body kept, in characters (a character is at least one byte of the cap). */
+export const MAX_FRAME_CHARS = MAX_MESSAGE_BYTES + 512;
+export const framePrefix = (nonce) => `${FRAME_MARK}${nonce}:`;
+export class FrameDecoder {
+    #prefix;
+    #mode = "start";
+    /** Characters of the current line that matched the prefix so far (mode `start`). */
+    #held = "";
+    #body = "";
+    constructor(nonce) {
+        this.#prefix = framePrefix(nonce);
+    }
+    feed(chunk) {
+        const events = [];
+        let rest = chunk;
+        while (rest.length > 0) {
+            if (this.#mode === "text") {
+                const newline = rest.indexOf("\n");
+                if (newline < 0) {
+                    events.push({ type: "text", text: rest });
+                    return events;
+                }
+                events.push({ type: "text", text: rest.slice(0, newline + 1) });
+                rest = rest.slice(newline + 1);
+                this.#mode = "start";
+            }
+            else if (this.#mode === "drop") {
+                const newline = rest.indexOf("\n");
+                if (newline < 0)
+                    return events;
+                rest = rest.slice(newline + 1);
+                this.#mode = "start";
+            }
+            else if (this.#mode === "frame") {
+                const newline = rest.indexOf("\n");
+                this.#body += newline < 0 ? rest : rest.slice(0, newline);
+                if (this.#body.length > MAX_FRAME_CHARS) {
+                    events.push({ type: "oversize" });
+                    this.#body = "";
+                    this.#mode = "drop";
+                    rest = newline < 0 ? "" : rest.slice(newline);
+                    continue;
+                }
+                if (newline < 0)
+                    return events;
+                events.push({ type: "frame", json: this.#body.replace(/\r$/, "") });
+                this.#body = "";
+                this.#mode = "start";
+                rest = rest.slice(newline + 1);
+            }
+            else
+                rest = this.#start(rest, events);
+        }
+        return events;
+    }
+    /** The stream ended: whatever is held was never completed as a frame line. */
+    flush() {
+        const events = [];
+        if (this.#mode === "start" && this.#held)
+            events.push({ type: "text", text: this.#held });
+        if (this.#mode === "frame" && this.#body)
+            events.push({ type: "frame", json: this.#body.replace(/\r$/, "") });
+        this.#mode = "start";
+        this.#held = "";
+        this.#body = "";
+        return events;
+    }
+    /** Reads the beginning of a line; returns what is left of `rest` for the next mode. */
+    #start(rest, events) {
+        for (let i = 0; i < rest.length; i++) {
+            const ch = rest[i];
+            if (ch === this.#prefix[this.#held.length]) {
+                this.#held += ch;
+                if (this.#held.length === this.#prefix.length) {
+                    this.#held = "";
+                    this.#mode = "frame";
+                    return rest.slice(i + 1);
+                }
+                continue;
+            }
+            // The line is ordinary output: release what matched and let the text mode stream the rest.
+            if (this.#held)
+                events.push({ type: "text", text: this.#held });
+            this.#held = "";
+            this.#mode = "text";
+            return rest.slice(i);
+        }
+        return "";
+    }
+}
+/** `text` without its frame lines (the form of stdout the model and the terminal see). */
+export function stripFrames(text, nonce) {
+    const decoder = new FrameDecoder(nonce);
+    return [...decoder.feed(text), ...decoder.flush()]
+        .flatMap((event) => (event.type === "text" ? [event.text] : []))
+        .join("");
+}

package/dist/forge/python/frame-transport.d.ts

@@ -0,0 +1,40 @@
+/**
+ * `StdoutFrameTransport` (Python spec §7, PD9): turns the frames of a script's stdout into common
+ * protocol messages for the kernel. The guest is untrusted: every frame is validated like any
+ * guest message (schema, sequence, size, depth, forbidden keys) with tool calls switched off, and a
+ * frame the host cannot accept ends the execution with `PROTOCOL_VIOLATION`.
+ *
+ * Messages reach the kernel one at a time, in the order they were written. Past `MAX_FRAMES`
+ * messages the rest is ignored with one warning, and the run goes on. Ordinary output is passed
+ * to `onText` exactly as it came.
+ */
+import { ForgeError } from "../errors.ts";
+import { type GuestMessage } from "../protocol.ts";
+export declare const MAX_FRAMES = 1000;
+export interface FrameTransportOptions {
+    nonce: string;
+    /** The kernel's side. Rejecting with a `ForgeError` ends the execution with that code. */
+    receive: (message: GuestMessage) => Promise<unknown>;
+    /** Output that is not a frame, in order. */
+    onText: (text: string) => void;
+    /** Called once, when the execution must end because of what the guest sent. */
+    onFailure: (error: ForgeError) => void;
+    maxMessages?: number;
+}
+export declare class StdoutFrameTransport {
+    #private;
+    readonly warnings: string[];
+    constructor(options: FrameTransportOptions);
+    /** The value of the guest's `complete` message, when it sent one. */
+    get completion(): {
+        value?: unknown;
+    } | undefined;
+    /** The guest's own `fatal` message, when it sent one. */
+    get fatal(): {
+        code: string;
+        message: string;
+    } | undefined;
+    feed(chunk: string): void;
+    /** Flushes what is held and resolves once every accepted message was handed to the kernel. */
+    end(): Promise<void>;
+}

package/dist/forge/python/frame-transport.js

@@ -0,0 +1,101 @@
+/**
+ * `StdoutFrameTransport` (Python spec §7, PD9): turns the frames of a script's stdout into common
+ * protocol messages for the kernel. The guest is untrusted: every frame is validated like any
+ * guest message (schema, sequence, size, depth, forbidden keys) with tool calls switched off, and a
+ * frame the host cannot accept ends the execution with `PROTOCOL_VIOLATION`.
+ *
+ * Messages reach the kernel one at a time, in the order they were written. Past `MAX_FRAMES`
+ * messages the rest is ignored with one warning, and the run goes on. Ordinary output is passed
+ * to `onText` exactly as it came.
+ */
+import { ForgeError, isForgeError } from "../errors.js";
+import { GuestMessageValidator } from "../protocol.js";
+import { FrameDecoder } from "./frame-parser.js";
+export const MAX_FRAMES = 1000;
+export class StdoutFrameTransport {
+    #options;
+    #decoder;
+    #validator = new GuestMessageValidator({ toolCalls: false });
+    #max;
+    #queue = Promise.resolve();
+    #seen = 0;
+    #failed = false;
+    #completion;
+    #fatal;
+    warnings = [];
+    constructor(options) {
+        this.#options = options;
+        this.#decoder = new FrameDecoder(options.nonce);
+        this.#max = options.maxMessages ?? MAX_FRAMES;
+    }
+    /** The value of the guest's `complete` message, when it sent one. */
+    get completion() {
+        return this.#completion;
+    }
+    /** The guest's own `fatal` message, when it sent one. */
+    get fatal() {
+        return this.#fatal;
+    }
+    feed(chunk) {
+        this.#handle(this.#decoder.feed(chunk));
+    }
+    /** Flushes what is held and resolves once every accepted message was handed to the kernel. */
+    async end() {
+        this.#handle(this.#decoder.flush());
+        await this.#queue;
+    }
+    #handle(events) {
+        for (const event of events) {
+            if (event.type === "text")
+                this.#options.onText(event.text);
+            else
+                this.#frame(event);
+        }
+    }
+    #fail(error) {
+        if (this.#failed)
+            return;
+        this.#failed = true;
+        this.#options.onFailure(error);
+    }
+    #frame(event) {
+        if (this.#failed)
+            return;
+        if (++this.#seen > this.#max) {
+            if (this.warnings.length === 0)
+                this.warnings.push(`the script sent more than ${this.#max} protocol messages; the rest was ignored`);
+            return;
+        }
+        if (event.type === "oversize")
+            return this.#fail(new ForgeError("PROTOCOL_VIOLATION", "protocol message larger than allowed"));
+        let raw;
+        try {
+            raw = JSON.parse(event.json);
+        }
+        catch {
+            return this.#fail(new ForgeError("PROTOCOL_VIOLATION", "protocol frame is not valid JSON"));
+        }
+        const checked = this.#validator.accept(raw);
+        if (!checked.ok)
+            return this.#fail(checked.error);
+        const message = checked.message;
+        if (message.type === "complete")
+            this.#completion = { ...message.payload };
+        else if (message.type === "fatal")
+            this.#fatal = message.payload;
+        if (message.type === "complete" || message.type === "fatal" || message.type === "ready")
+            return;
+        this.#queue = this.#queue.then(async () => {
+            if (this.#failed)
+                return;
+            try {
+                await this.#options.receive(message);
+            }
+            catch (error) {
+                this.#fail(isForgeError(error)
+                    ? error
+                    : new ForgeError("ISOLATE_FAILURE", error instanceof Error ? error.message : String(error)));
+            }
+        });
+    }
+}

package/dist/forge/python/import-scan.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Static import scan of a Python script (Python spec §6.2). Pure and interpreter-free: it reads the
+ * top-level module names a script imports so the backend selector can tell whether the Pyodide
+ * profiles that were downloaded cover it. It is NOT a security control: `__import__`,
+ * `importlib` and `exec` are invisible to it, and a wrong guess only means the run fails with
+ * `PACKAGE_UNAVAILABLE` and is never retried elsewhere (PD5).
+ */
+import { type PyodideProfile } from "./pyodide-manifest.ts";
+/** Top-level module names imported by `source`, in order of appearance, without repeats. */
+export declare function scanImports(source: string): string[];
+/** Package name for an import name, when a Pyodide profile ships it. */
+export declare const packageOfModule: (module: string) => string | undefined;
+export interface ModuleClassification {
+    /** The smallest profile that covers every module the wheels provide. */
+    profile: PyodideProfile;
+    /** Packages the imports need, without repeats. */
+    packages: string[];
+    /** Modules neither in the standard library nor in any profile. */
+    unknown: string[];
+}
+/** `science` when a package only that profile carries is imported, else `analysis`, else `base`. */
+export declare function classifyModules(modules: readonly string[]): ModuleClassification;