@alisio/core 0.1.0-alpha.20 → 0.1.0-alpha.22

package/dist/analysis/install-hints.js

@@ -0,0 +1,144 @@
+/**
+ * Install instructions for Python ≥ 3.10 on the machine Alisio runs on. A pure function of the
+ * detected host (injected in tests); the host detection reads `/etc/os-release` and looks for
+ * `uv`, `winget` and `brew` on PATH without launching any process. Commands are shown, never run.
+ */
+import { readFile } from "node:fs/promises";
+import { which } from "../runtime/fs.js";
+const UV = "uv python install 3.12";
+const PYTHON_ORG = "Download the installer from https://www.python.org/downloads/";
+/** Parses `KEY=value` / `KEY="value"` lines of an os-release file. */
+export function parseOsRelease(text) {
+    const out = {};
+    for (const line of text.split(/\r?\n/)) {
+        const match = /^([A-Z_]+)=(.*)$/.exec(line.trim());
+        if (match?.[1])
+            out[match[1]] = (match[2] ?? "").replace(/^["']|["']$/g, "");
+    }
+    return out;
+}
+function systemLabel(host) {
+    const os = host.platform === "win32"
+        ? "Windows"
+        : host.platform === "darwin"
+            ? "macOS"
+            : host.platform === "linux"
+                ? "Linux"
+                : host.platform;
+    const distro = host.osRelease?.PRETTY_NAME;
+    return [os, distro, host.arch].filter(Boolean).join(" · ");
+}
+export function pythonInstallHints(host) {
+    const system = systemLabel(host);
+    const heading = host.found
+        ? host.found.version
+            ? `Found Python ${host.found.version} at ${host.found.path}; Alisio needs 3.10 or newer.`
+            : `The Python at ${host.found.path} is not usable${host.found.reason ? ` (${host.found.reason})` : ""}.`
+        : undefined;
+    const notes = [];
+    if (host.found)
+        notes.push("If another Python 3.10+ is installed, start Alisio with --python <path>.");
+    let primary;
+    let alternatives = [];
+    if (host.platform === "win32") {
+        primary = "winget install Python.Python.3.12";
+        alternatives = [`${PYTHON_ORG} (check "Add python.exe to PATH")`];
+        if (host.hasUv)
+            alternatives.push(UV);
+        notes.push("The Microsoft Store alias (WindowsApps\\python.exe) does not count as an installed Python.", "Open a new terminal after installing so the updated PATH is visible.");
+    }
+    else if (host.platform === "darwin") {
+        if (host.hasBrew) {
+            primary = "brew install python@3.12";
+            alternatives = [
+                "Download the macOS installer from https://www.python.org/downloads/macos/",
+                UV,
+            ];
+        }
+        else {
+            primary = "Download the macOS installer from https://www.python.org/downloads/macos/";
+            alternatives = [
+                UV,
+                "xcode-select --install (Command Line Tools Python; it may be older than 3.10)",
+            ];
+        }
+    }
+    else if (host.platform === "linux") {
+        const ids = `${host.osRelease?.ID ?? ""} ${host.osRelease?.ID_LIKE ?? ""}`.toLowerCase();
+        const has = (id) => ids.split(/\s+/).includes(id);
+        if (has("debian") || has("ubuntu")) {
+            primary = "sudo apt install python3";
+            alternatives = [`If your distribution ships a version older than 3.10: ${UV}`];
+        }
+        else if (has("fedora") || has("rhel") || has("centos")) {
+            primary = "sudo dnf install python3";
+            alternatives = [UV];
+        }
+        else if (has("arch")) {
+            primary = "sudo pacman -S python";
+        }
+        else if (ids.includes("suse")) {
+            primary = "sudo zypper install python3";
+        }
+        else if (has("alpine")) {
+            primary = "sudo apk add python3";
+        }
+        else {
+            primary = "Install python3 with your system package manager";
+            alternatives = [UV];
+        }
+    }
+    else {
+        primary = "Install python3 with your system package manager";
+        alternatives = [UV];
+    }
+    return { system, ...(heading ? { heading } : {}), primary, alternatives, notes };
+}
+/** One paragraph for the model and terminals (English, like every tool message). */
+export function installHintsText(hints) {
+    const lines = [
+        ...(hints.heading ? [hints.heading] : []),
+        `Python 3.10+ was not found on this machine (${hints.system}). To install it: ${hints.primary}`,
+        ...hints.alternatives.map((alternative) => `Alternatively: ${alternative}`),
+        ...hints.notes,
+        "After installing, ask again; no Alisio restart is needed. To use a specific interpreter, start Alisio with --python <path>.",
+    ];
+    return lines.join("\n");
+}
+/** The same guidance as Markdown (web and TUI render it as a `markdown` block). */
+export function installHintsMarkdown(hints) {
+    return [
+        "**Python 3.10+ is not available**",
+        "",
+        ...(hints.heading ? [hints.heading, ""] : []),
+        `Detected system: ${hints.system}`,
+        "",
+        "```sh",
+        hints.primary,
+        "```",
+        ...(hints.alternatives.length
+            ? ["", "Alternatives:", ...hints.alternatives.map((a) => `- ${a}`)]
+            : []),
+        ...(hints.notes.length ? ["", ...hints.notes.map((n) => `- ${n}`)] : []),
+        "",
+        "After installing, ask again; no restart is needed. Start Alisio with `--python <path>` to pin an interpreter.",
+    ].join("\n");
+}
+/** Detects the host for `pythonInstallHints` (reads files and PATH only; launches nothing). */
+export async function detectHintHost(found) {
+    let osRelease;
+    if (process.platform === "linux")
+        osRelease = await readFile("/etc/os-release", "utf8")
+            .then(parseOsRelease)
+            .catch(() => undefined);
+    const [uv, winget, brew] = await Promise.all([which("uv"), which("winget"), which("brew")]);
+    return {
+        platform: process.platform,
+        arch: process.arch,
+        ...(osRelease ? { osRelease } : {}),
+        hasUv: !!uv,
+        hasWinget: !!winget,
+        hasBrew: !!brew,
+        ...(found ? { found } : {}),
+    };
+}

package/dist/analysis/janitor.d.ts

@@ -0,0 +1,60 @@
+import type { SqlDatabase } from "@alisio/sdk";
+import type { ArtifactStore } from "../artifacts/store.ts";
+import type { BlobStore } from "../runtime/blobs.ts";
+import type { DatasetService } from "./data/datasets.ts";
+export declare const SWEEP_INTERVAL_MS = 86400000;
+export interface RetentionPolicy {
+    jobsDays: number;
+    intermediateDays: number;
+    artifactsDays: number;
+}
+export interface SweepReport {
+    /** Why nothing ran: swept less than 24 h ago, or another process is sweeping. */
+    skipped?: "recent" | "locked";
+    workDeleted: number;
+    jobsTrimmed: number;
+    jobsDeleted: number;
+    artifactsExpired: number;
+    datasetsDeleted: number;
+    uploadsDeleted: number;
+    bytesFreed: number;
+    errors: number;
+}
+export declare class AnalysisJanitor {
+    private options;
+    private timer?;
+    private stopped;
+    constructor(options: {
+        /** State root (`stateHome()` or the folder of `--db`). */
+        root: string;
+        db: SqlDatabase;
+        artifacts: ArtifactStore;
+        datasets?: DatasetService;
+        blobs?: BlobStore;
+        /** Read at every sweep, so a changed setting applies without a restart. */
+        retention: () => RetentionPolicy;
+        /** The clock (injected in tests). */
+        now?: () => number;
+        /** Delay before the first background sweep after `start()` (default 30 s). */
+        initialDelayMs?: number;
+    });
+    private now;
+    private get dir();
+    /** When the last sweep started (epoch ms), if there was one. */
+    lastSweep(): Promise<number | undefined>;
+    /** Launches the sweep in the background; never delays the caller and never keeps Node alive. */
+    start(): void;
+    stop(): void;
+    private arm;
+    /** One sweep (see the module comment). `force` ignores the 24 h mark. */
+    sweep(options?: {
+        force?: boolean;
+    }): Promise<SweepReport>;
+    /** Exclusive lock between processes sharing the state folder; undefined when someone holds it. */
+    private lock;
+    private expireArtifacts;
+    private trimJobs;
+    /** Removes the empty session and workspace folders a deleted job leaves behind. */
+    private pruneEmptyParents;
+    private dropDatasets;
+}

package/dist/analysis/janitor.js

@@ -0,0 +1,264 @@
+/**
+ * Retention of analysis data (spec §19): `AnalysisJanitor.sweep()` runs at most once every 24 hours
+ * (the mark is `<state>/analysis/.last-sweep`), launched in the background shortly after startup
+ * with an unreferenced timer, so it never delays the start nor keeps a process alive.
+ *
+ *  - `work/` of a job goes after `intermediateDays`; `logs/` and `staging/` after `jobsDays`.
+ *  - `script/`, `input/` and `job.json` stay while any artifact of that execution is `ready`
+ *    (a rerun needs them); once none is, the whole job folder goes after `jobsDays`.
+ *  - With `artifactsDays > 0`, older artifacts become `expired` (folder deleted, row kept).
+ *  - Datasets unused for `jobsDays` are deleted with their uploaded original in `blobs/`; an
+ *    upload without a dataset (a failed ingestion) goes after the same time.
+ *  - `0` disables the deletion of that class. Running executions are never touched.
+ *
+ * Deletions are best effort and independent: one that fails is counted and the sweep goes on.
+ */
+import { mkdir, readFile, rm, rmdir, stat, writeFile } from "node:fs/promises";
+import { dirname, isAbsolute, join, relative } from "node:path";
+const DAY = 86_400_000;
+export const SWEEP_INTERVAL_MS = DAY;
+/** A lock older than this belongs to a dead process. */
+const STALE_LOCK_MS = 60 * 60_000;
+/** A job still `running` after this long was left by a crashed process. */
+const STALE_RUNNING_MS = DAY;
+const empty = () => ({
+    workDeleted: 0,
+    jobsTrimmed: 0,
+    jobsDeleted: 0,
+    artifactsExpired: 0,
+    datasetsDeleted: 0,
+    uploadsDeleted: 0,
+    bytesFreed: 0,
+    errors: 0,
+});
+/** Size in bytes of a file or folder (0 when it does not exist). */
+async function sizeOf(path) {
+    let info;
+    try {
+        info = await stat(path);
+    }
+    catch {
+        return 0;
+    }
+    if (!info.isDirectory())
+        return info.size;
+    const { readdir } = await import("node:fs/promises");
+    let total = 0;
+    for (const entry of await readdir(path).catch(() => []))
+        total += await sizeOf(join(path, entry));
+    return total;
+}
+async function exists(path) {
+    try {
+        await stat(path);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+export class AnalysisJanitor {
+    options;
+    timer;
+    stopped = false;
+    constructor(options) {
+        this.options = options;
+    }
+    now() {
+        return (this.options.now ?? Date.now)();
+    }
+    get dir() {
+        return join(this.options.root, "analysis");
+    }
+    /** When the last sweep started (epoch ms), if there was one. */
+    async lastSweep() {
+        try {
+            const value = Number((await readFile(join(this.dir, ".last-sweep"), "utf8")).trim());
+            return Number.isFinite(value) ? value : undefined;
+        }
+        catch {
+            return undefined;
+        }
+    }
+    /** Launches the sweep in the background; never delays the caller and never keeps Node alive. */
+    start() {
+        this.stopped = false;
+        this.arm(this.options.initialDelayMs ?? 30_000);
+    }
+    stop() {
+        this.stopped = true;
+        if (this.timer)
+            clearTimeout(this.timer);
+        this.timer = undefined;
+    }
+    arm(delay) {
+        if (this.stopped)
+            return;
+        this.timer = setTimeout(() => {
+            void this.sweep()
+                .catch(() => undefined)
+                .finally(() => this.arm(SWEEP_INTERVAL_MS));
+        }, delay);
+        this.timer.unref();
+    }
+    /** One sweep (see the module comment). `force` ignores the 24 h mark. */
+    async sweep(options = {}) {
+        const report = empty();
+        const now = this.now();
+        const last = await this.lastSweep();
+        if (!options.force && last !== undefined && now - last < SWEEP_INTERVAL_MS)
+            return { ...report, skipped: "recent" };
+        const release = await this.lock();
+        if (!release)
+            return { ...report, skipped: "locked" };
+        try {
+            // The mark is written first: a crash mid-sweep does not make every start retry at once.
+            await writeFile(join(this.dir, ".last-sweep"), `${now}\n`);
+            const policy = this.options.retention();
+            await this.expireArtifacts(policy, now, report);
+            await this.trimJobs(policy, now, report);
+            await this.dropDatasets(policy, now, report);
+        }
+        finally {
+            await release();
+        }
+        return report;
+    }
+    /** Exclusive lock between processes sharing the state folder; undefined when someone holds it. */
+    async lock() {
+        await mkdir(this.dir, { recursive: true });
+        const file = join(this.dir, ".sweep.lock");
+        for (let attempt = 0; attempt < 2; attempt++)
+            try {
+                await writeFile(file, `${process.pid}\n`, { flag: "wx" });
+                return () => rm(file, { force: true });
+            }
+            catch {
+                const info = await stat(file).catch(() => undefined);
+                // Real time on purpose: the lock belongs to a live process, not to the injected clock.
+                if (info && Date.now() - info.mtimeMs < STALE_LOCK_MS)
+                    return undefined;
+                await rm(file, { force: true });
+            }
+        return undefined;
+    }
+    async expireArtifacts(policy, now, report) {
+        if (policy.artifactsDays <= 0)
+            return;
+        for (const record of this.options.artifacts.readyBefore(now - policy.artifactsDays * DAY))
+            try {
+                const bytes = await sizeOf(this.options.artifacts.folder(record));
+                if (await this.options.artifacts.markExpired(record.id)) {
+                    report.artifactsExpired++;
+                    report.bytesFreed += bytes;
+                }
+            }
+            catch {
+                report.errors++;
+            }
+    }
+    async trimJobs(policy, now, report) {
+        if (policy.jobsDays <= 0 && policy.intermediateDays <= 0)
+            return;
+        const rows = this.options.db
+            .prepare("SELECT id, status, rel_dir, created_at, ended_at FROM analysis_executions")
+            .all();
+        const ready = this.options.db.prepare("SELECT count(*) AS n FROM artifacts WHERE execution_id=? AND status='ready'");
+        for (const row of rows) {
+            const stamp = row.ended_at ?? row.created_at;
+            if (row.status === "running" && now - row.created_at < STALE_RUNNING_MS)
+                continue;
+            const age = now - stamp;
+            const folder = join(this.options.root, row.rel_dir);
+            // Only folders under analysis/jobs are ever deleted, whatever a row says.
+            const inside = relative(join(this.options.root, "analysis", "jobs"), folder);
+            if (!inside || inside.startsWith("..") || isAbsolute(inside))
+                continue;
+            if (!(await exists(folder)))
+                continue;
+            try {
+                if (policy.intermediateDays > 0 && age > policy.intermediateDays * DAY) {
+                    const target = join(folder, "work");
+                    if (await exists(target)) {
+                        report.bytesFreed += await sizeOf(target);
+                        await rm(target, { recursive: true, force: true });
+                        report.workDeleted++;
+                    }
+                }
+                if (policy.jobsDays > 0 && age > policy.jobsDays * DAY) {
+                    for (const name of ["logs", "staging"]) {
+                        const target = join(folder, name);
+                        if (await exists(target)) {
+                            report.bytesFreed += await sizeOf(target);
+                            await rm(target, { recursive: true, force: true });
+                        }
+                    }
+                    const { n } = ready.get(row.id);
+                    if (n === 0) {
+                        report.bytesFreed += await sizeOf(folder);
+                        await rm(folder, { recursive: true, force: true });
+                        report.jobsDeleted++;
+                        await this.pruneEmptyParents(folder);
+                    }
+                    else
+                        report.jobsTrimmed++;
+                }
+            }
+            catch {
+                report.errors++;
+            }
+        }
+    }
+    /** Removes the empty session and workspace folders a deleted job leaves behind. */
+    async pruneEmptyParents(folder) {
+        const stop = join(this.options.root, "analysis", "jobs");
+        let dir = dirname(folder);
+        while (dir.startsWith(stop) && dir !== stop)
+            try {
+                await rmdir(dir);
+                dir = dirname(dir);
+            }
+            catch {
+                return; // not empty (or gone): done
+            }
+    }
+    async dropDatasets(policy, now, report) {
+        if (policy.jobsDays <= 0)
+            return;
+        const { datasets, blobs } = this.options;
+        const cutoff = now - policy.jobsDays * DAY;
+        if (datasets)
+            for (const record of datasets.all())
+                try {
+                    const info = await stat(datasets.file(record)).catch(() => undefined);
+                    // The file's modification time moves when the dataset is used (DatasetService.touch).
+                    const lastUse = info ? Math.max(record.createdAt, info.mtimeMs) : record.createdAt;
+                    if (lastUse >= cutoff)
+                        continue;
+                    const bytes = info?.size ?? 0;
+                    await datasets.remove(record);
+                    report.datasetsDeleted++;
+                    report.bytesFreed += bytes;
+                }
+                catch {
+                    report.errors++;
+                }
+        if (!blobs)
+            return;
+        // Uploaded originals: those no dataset refers to any more (and failed ingestions) go too.
+        const referenced = new Set(this.options.db
+            .prepare("SELECT blob_hash FROM datasets WHERE blob_hash IS NOT NULL")
+            .all().map((r) => r.blob_hash));
+        for (const hash of blobs.uploadsBefore(cutoff))
+            try {
+                if (referenced.has(hash))
+                    continue;
+                const freed = blobs.removeUpload(hash);
+                report.uploadsDeleted++;
+                report.bytesFreed += freed;
+            }
+            catch {
+                report.errors++;
+            }
+    }
+}

package/dist/analysis/jobs.d.ts

@@ -0,0 +1,94 @@
+import type { SqlDatabase } from "@alisio/sdk";
+export type ExecutionStatus = "running" | "completed" | "failed" | "cancelled" | "timed_out";
+export interface JobPaths {
+    id: string;
+    root: string;
+    script: string;
+    main: string;
+    input: string;
+    work: string;
+    staging: string;
+    logs: string;
+}
+export interface JobInput {
+    /** Name inside `input/`. */
+    name: string;
+    /** Hash of the content named in the provenance (for a dataset: the original file). */
+    sha256: string;
+    bytes: number;
+    /** Set for a dataset (the `.sqlite` file of §17): its id and the hash of the original file. */
+    datasetId?: string;
+    /** Hash of the copy inside `input/` (differs from `sha256` for a dataset's SQLite file). */
+    copySha256?: string;
+}
+/** What `job.json` records about the inputs, so a rerun can verify them (internal; never public). */
+export interface RecordedInputs {
+    inputs: JobInput[];
+    /** The `input/inputs.json` entries the script reads. */
+    manifest: unknown[];
+    /** Extras the run asked for (`python_run { extras }`), so a rerun asks for the same. */
+    extras?: string[];
+}
+export interface ExecutionDetails {
+    id: string;
+    sessionId: string;
+    rootSessionId: string;
+    workspace: string;
+    runtime: "managed" | "oci";
+    status: ExecutionStatus;
+    scriptSha256: string;
+    exitCode?: number;
+    relDir: string;
+    rerunOf?: string;
+    createdAt: number;
+    endedAt?: number;
+}
+/** Executions of `python_run` (`analysis_executions`) and their job folders. */
+export declare class AnalysisJobs {
+    private options;
+    constructor(options: {
+        root: string;
+        db: SqlDatabase;
+        now?: () => number;
+    });
+    private now;
+    /** Creates the job folder (script, runtime helpers, empty input/work/staging/logs) and its row. */
+    create(input: {
+        sessionId: string;
+        rootSessionId: string;
+        workspace: string;
+        runId?: string;
+        callId?: string;
+        code: string;
+        runtime: "managed" | "oci";
+        /** The execution this one repeats (a rerun); recorded in the row and in `job.json`. */
+        rerunOf?: string;
+    }): Promise<JobPaths & {
+        scriptSha256: string;
+    }>;
+    /**
+     * Copies an input file (already resolved by the path policy) into `input/`, as `name` (default:
+     * its own name). Copy-on-write clones are used where the file system has them; the copy is
+     * never a link, so a script cannot alter the original.
+     */
+    addInput(job: JobPaths, source: string, preferred?: string): Promise<JobInput>;
+    /** `input/inputs.json`: what each input is (the script reads it with `alisio_runtime.datasets.inputs()`). */
+    writeInputsManifest(job: JobPaths, entries: unknown[]): Promise<void>;
+    /** Adds the inputs to `job.json` (atomically), once they are copied into `input/`. */
+    recordInputs(job: JobPaths, recorded: RecordedInputs): Promise<void>;
+    /** The inputs a finished job recorded; `undefined` when `job.json` is missing or has none. */
+    recordedInputs(relDir: string): Promise<RecordedInputs | undefined>;
+    /** Absolute folder of an execution (`rel_dir` is always relative to the state root). */
+    folder(relDir: string): string;
+    /** The whole row of an execution. */
+    details(id: string): ExecutionDetails | undefined;
+    finish(id: string, status: ExecutionStatus, details?: {
+        exitCode?: number;
+        error?: string;
+    }): void;
+    get(id: string): {
+        status: ExecutionStatus;
+        exitCode?: number;
+        relDir: string;
+    } | undefined;
+}