@alisio/core 0.1.0-alpha.21 → 0.1.0-alpha.22

package/dist/analysis/data/datasets.js

@@ -0,0 +1,441 @@
+/**
+ * Datasets of a session (spec §17): one SQLite file per dataset under
+ * `<state>/analysis/datasets/<workspaceKey>/<rootSession>/<datasetId>.sqlite`, the `datasets` rows
+ * that describe them, ingestion (Node engine for CSV/TSV/JSON/JSONL, the Python standard-library
+ * helper for XLSX), read-only queries, and the pages `SpreadsheetView` reads. Files are
+ * regenerable derivations: the original (a blob or a workspace/artifact file) is never altered.
+ */
+import { createHash } from "node:crypto";
+import { createReadStream } from "node:fs";
+import { chmod, mkdir, rename, rm, stat, utimes, writeFile } from "node:fs/promises";
+import { basename, extname, join, relative } from "node:path";
+import { newId } from "../../runtime/ids.js";
+import { workspaceKey } from "../../runtime/paths.js";
+import { runProcess } from "../../runtime/process.js";
+import { openReadOnlyDatabase } from "../../runtime/sqlite.js";
+import { ALISIO_RUNTIME_FILES } from "../python/sources.js";
+import { describeDataset, promptSummary } from "./describe.js";
+import { DataEngine, DataError } from "./engine-client.js";
+import { buildCountQuery, buildPageQuery, encodeCursor, MAX_ALL_COLUMNS_FILTER_ROWS, MAX_PAGE_ROWS, PageParamError, } from "./rows.js";
+import { guardSql } from "./sql-guard.js";
+import { INGEST_VERSION } from "./stats.js";
+export { DataError, PageParamError };
+export const DEFAULT_DATASET_LIMITS = {
+    maxUploadBytes: 200 * 1024 * 1024,
+    maxRows: 5_000_000,
+    queryTimeoutMs: 5000,
+    maxInteractiveRows: 1_000_000,
+};
+const MAX_COLUMNS = 1000;
+const MAX_CELL_CHARS = 1024 * 1024;
+const XLSX_TIMEOUT_MS = 10 * 60_000;
+export const XLSX_REMEDY = "Export the sheet as CSV, or install Python 3.10+.";
+/** Heap soft cap of SQLite in the query process (`PRAGMA hard_heap_limit`). */
+const QUERY_HEAP_BYTES = 512 * 1024 * 1024;
+const FORMATS = {
+    ".csv": "csv",
+    ".tsv": "tsv",
+    ".tab": "tsv",
+    ".json": "json",
+    ".jsonl": "jsonl",
+    ".ndjson": "jsonl",
+    ".xlsx": "xlsx",
+};
+/** The format of a file name, or undefined when it is not a supported dataset. */
+export function datasetFormat(name) {
+    return FORMATS[extname(name).toLowerCase()];
+}
+const toRecord = (row) => ({
+    id: row.id,
+    sessionId: row.session,
+    rootSessionId: row.root_session,
+    name: row.name,
+    format: row.format,
+    sha256: row.sha256,
+    bytes: row.bytes,
+    ...(row.blob_hash ? { blobHash: row.blob_hash } : {}),
+    ...(row.source_path ? { sourcePath: row.source_path } : {}),
+    dbRelPath: row.db_rel_path,
+    ingestVersion: row.ingest_version,
+    sheets: JSON.parse(row.sheets),
+    createdAt: row.created_at,
+});
+export const toDatasetRef = (record) => ({
+    id: record.id,
+    name: record.name,
+    format: record.format,
+    bytes: record.bytes,
+    sha256: record.sha256,
+    sheets: record.sheets.map((s) => ({ ...s })),
+});
+async function sha256Of(path, signal) {
+    const hash = createHash("sha256");
+    for await (const chunk of createReadStream(path)) {
+        signal?.throwIfAborted();
+        hash.update(chunk);
+    }
+    return hash.digest("hex");
+}
+const message = (error) => (error instanceof Error ? error.message : String(error));
+export class DatasetService {
+    options;
+    limits;
+    engine;
+    inflight = new Map();
+    constructor(options) {
+        this.options = options;
+        this.limits = { ...DEFAULT_DATASET_LIMITS, ...options.limits };
+        this.engine = options.engine ?? new DataEngine({ root: options.root });
+    }
+    now() {
+        return (this.options.now ?? Date.now)();
+    }
+    /** Absolute path of a dataset file. */
+    file(record) {
+        return join(this.options.root, record.dbRelPath);
+    }
+    get(id) {
+        const row = this.options.db.prepare("SELECT * FROM datasets WHERE id=?").get(id);
+        const record = row ? toRecord(row) : undefined;
+        if (record)
+            this.touch(record);
+        return record;
+    }
+    touched = new Map();
+    /**
+     * Records that a dataset was used, for the retention sweep ("unused for `jobsDays`"): the
+     * modification time of its SQLite file (nothing else changes it after the ingestion), at most
+     * once every ten minutes per dataset. Best effort: a failure never affects the caller.
+     */
+    touch(record) {
+        const now = this.now();
+        if (now - (this.touched.get(record.id) ?? 0) < 10 * 60_000)
+            return;
+        this.touched.set(record.id, now);
+        const when = new Date(now);
+        void utimes(this.file(record), when, when).catch(() => undefined);
+    }
+    /** Every dataset (all sessions), for the retention sweep; does not count as a use. */
+    all() {
+        return this.options.db.prepare("SELECT * FROM datasets").all().map(toRecord);
+    }
+    /** A dataset of this root session; another session's id is `not_found`. */
+    getFor(id, rootSessionId) {
+        const record = this.get(id);
+        if (!record || record.rootSessionId !== rootSessionId)
+            throw new DataError("not_found", `Dataset ${id} not found in this session`);
+        return record;
+    }
+    list(rootSessionId) {
+        return this.options.db
+            .prepare("SELECT * FROM datasets WHERE root_session=? ORDER BY created_at DESC, id DESC")
+            .all(rootSessionId).map(toRecord);
+    }
+    bySha(rootSessionId, sha256) {
+        const row = this.options.db
+            .prepare("SELECT * FROM datasets WHERE root_session=? AND sha256=? ORDER BY created_at LIMIT 1")
+            .get(rootSessionId, sha256);
+        return row ? toRecord(row) : undefined;
+    }
+    /**
+     * Ingests a file into a dataset, or returns the session's existing dataset with the same
+     * content (`reused`). Concurrent calls for the same content share one ingestion. A failure
+     * leaves no row and no file.
+     */
+    async ingest(input) {
+        const name = input.name ?? basename(input.path);
+        const format = datasetFormat(name);
+        if (!format)
+            throw new DataError("dataset_unsupported", `Unsupported file type ${extname(name) || "(none)"}. Supported: CSV, TSV, JSON, JSONL and XLSX.`);
+        const info = await stat(input.path).catch(() => undefined);
+        if (!info?.isFile())
+            throw new DataError("not_found", `File not found: ${name}`);
+        if (info.size > this.limits.maxUploadBytes)
+            throw new DataError("ingest_limit", `${name} is ${(info.size / (1024 * 1024)).toFixed(1)} MiB; the limit is ${Math.round(this.limits.maxUploadBytes / (1024 * 1024))} MiB (analysis.data.maxUploadBytes)`);
+        if (info.size === 0)
+            throw new DataError("ingest_invalid", `${name} is empty`);
+        const sha256 = await sha256Of(input.path, input.signal);
+        const existing = this.bySha(input.rootSessionId, sha256);
+        if (existing)
+            return { record: existing, reused: true };
+        const key = `${input.rootSessionId}:${sha256}`;
+        let pending = this.inflight.get(key);
+        if (!pending) {
+            pending = this.build(input, name, format, sha256, info.size).finally(() => this.inflight.delete(key));
+            this.inflight.set(key, pending);
+        }
+        return pending;
+    }
+    async build(input, name, format, sha256, bytes) {
+        const id = newId("ds", this.now());
+        const dir = join(this.options.root, "analysis", "datasets", workspaceKey(input.workspace), input.rootSessionId);
+        await mkdir(dir, { recursive: true, mode: 0o700 });
+        const target = join(dir, `${id}.sqlite`);
+        const temp = `${target}.tmp`;
+        try {
+            let result;
+            if (format === "xlsx")
+                result = await this.ingestXlsx(input, name, sha256, temp);
+            else
+                result = await this.engine.ingest({
+                    source: input.path,
+                    format,
+                    target: temp,
+                    sourceName: name,
+                    sourceSha256: sha256,
+                    limits: {
+                        maxRows: this.limits.maxRows,
+                        maxColumns: MAX_COLUMNS,
+                        maxCellChars: MAX_CELL_CHARS,
+                    },
+                }, {
+                    ...(input.signal ? { signal: input.signal } : {}),
+                    ...(input.onProgress ? { onProgress: input.onProgress } : {}),
+                });
+            // Windows cannot rename over an existing file; nothing else holds this fresh path.
+            await rm(target, { force: true });
+            await rename(temp, target);
+            await chmod(target, 0o400).catch(() => undefined);
+            const record = {
+                id,
+                sessionId: input.sessionId,
+                rootSessionId: input.rootSessionId,
+                name,
+                format,
+                sha256,
+                bytes,
+                ...(input.blobHash ? { blobHash: input.blobHash } : {}),
+                ...(input.sourcePath ? { sourcePath: input.sourcePath } : {}),
+                dbRelPath: relative(this.options.root, target),
+                ingestVersion: INGEST_VERSION,
+                sheets: result.sheets,
+                createdAt: this.now(),
+            };
+            this.options.db
+                .prepare(`INSERT INTO datasets(id,session,root_session,name,format,sha256,bytes,blob_hash,source_path,
+             db_rel_path,ingest_version,sheets,created_at) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)`)
+                .run(record.id, record.sessionId, record.rootSessionId, record.name, record.format, record.sha256, record.bytes, record.blobHash ?? null, record.sourcePath ?? null, record.dbRelPath, record.ingestVersion, JSON.stringify(record.sheets), record.createdAt);
+            return { record, reused: false };
+        }
+        catch (error) {
+            await rm(temp, { force: true }).catch(() => undefined);
+            await rm(target, { force: true }).catch(() => undefined);
+            throw error;
+        }
+    }
+    /** The XLSX route (spec §17.6, option B): the Python standard-library helper, then statistics. */
+    async ingestXlsx(input, name, sha256, temp) {
+        const python = this.options.python;
+        const signal = input.signal ?? new AbortController().signal;
+        const runtime = python ? await python.interpreter(signal) : undefined;
+        if (!runtime?.ok)
+            throw new DataError("dataset_unsupported", `${name} is an Excel workbook, which needs Python 3.10+ to be read. ${XLSX_REMEDY}`);
+        const helper = await this.helperDir();
+        await rm(temp, { force: true });
+        let run;
+        try {
+            run = await runProcess(runtime.executable, [
+                ...(runtime.prefixArgs ?? []),
+                "-E",
+                "-s",
+                "-B",
+                "-u",
+                "-X",
+                "utf8",
+                "-m",
+                "alisio_runtime.xlsx_to_sqlite",
+                input.path,
+                temp,
+                "--max-rows",
+                String(this.limits.maxRows),
+                "--max-columns",
+                String(MAX_COLUMNS),
+                "--max-cell-chars",
+                String(MAX_CELL_CHARS),
+            ], { cwd: helper, signal, timeoutMs: XLSX_TIMEOUT_MS, maxBytes: 16 * 1024 });
+        }
+        catch (error) {
+            if (signal.aborted)
+                throw new DataError("cancelled", "Cancelled");
+            throw new DataError("engine_error", `The XLSX helper failed: ${message(error)}`);
+        }
+        if (run.exitCode !== 0) {
+            const line = run.stderr
+                .split(/\r?\n/)
+                .reverse()
+                .find((l) => l.startsWith("alisio-xlsx:")) ?? "";
+            throw new DataError("ingest_invalid", line
+                ? line.replace(/^alisio-xlsx:\s*/, "")
+                : `The XLSX helper failed (exit ${run.exitCode})`);
+        }
+        return this.engine.finalize({ target: temp, sourceName: name, sourceSha256: sha256 }, {
+            ...(input.signal ? { signal: input.signal } : {}),
+        });
+    }
+    /** The `alisio_runtime` package on disk for `python -m` (content-addressed, written once). */
+    async helperDir() {
+        const hash = createHash("sha256")
+            .update(JSON.stringify(ALISIO_RUNTIME_FILES))
+            .digest("hex")
+            .slice(0, 12);
+        const dir = join(this.options.root, "runtimes", "python", "helper", hash);
+        const marker = join(dir, ".complete");
+        if (await stat(marker).then(() => true, () => false))
+            return dir;
+        const package_ = join(dir, "alisio_runtime");
+        await mkdir(package_, { recursive: true, mode: 0o700 });
+        for (const [file, text] of Object.entries(ALISIO_RUNTIME_FILES))
+            await writeFile(join(package_, file), text);
+        await writeFile(marker, "");
+        return dir;
+    }
+    /** Schema and statistics of every sheet (`GET /api/datasets/:did`). */
+    detail(record) {
+        const db = openReadOnlyDatabase(this.file(record));
+        try {
+            const meta = Object.fromEntries(db.prepare("SELECT key, value FROM _alisio_meta").all().map((r) => [r.key, r.value]));
+            const sheets = db
+                .prepare("SELECT name, table_name, rows FROM _alisio_sheets ORDER BY ordinal")
+                .all();
+            const columns = db.prepare(`SELECT name, label, inferred_type, nulls, distinct_count, distinct_exact, min, max, mean,
+           text_fallbacks, top_values FROM _alisio_columns WHERE table_name=? ORDER BY ordinal`);
+            return {
+                ...toDatasetRef(record),
+                ingestVersion: record.ingestVersion,
+                ...(meta.encoding ? { encoding: meta.encoding } : {}),
+                ...(meta.delimiter ? { delimiter: meta.delimiter } : {}),
+                maxInteractiveRows: this.limits.maxInteractiveRows,
+                sheetDetails: sheets.map((sheet) => ({
+                    name: sheet.name,
+                    table: sheet.table_name,
+                    rows: Number(sheet.rows),
+                    columns: columns.all(sheet.table_name).map((c) => ({
+                        name: String(c.name),
+                        label: String(c.label),
+                        type: String(c.inferred_type),
+                        nulls: Number(c.nulls ?? 0),
+                        distinct: Number(c.distinct_count ?? 0),
+                        distinctExact: Number(c.distinct_exact ?? 1) === 1,
+                        ...(c.min !== null && c.min !== undefined ? { min: String(c.min) } : {}),
+                        ...(c.max !== null && c.max !== undefined ? { max: String(c.max) } : {}),
+                        ...(c.mean !== null && c.mean !== undefined ? { mean: Number(c.mean) } : {}),
+                        textFallbacks: Number(c.text_fallbacks ?? 0),
+                        top: JSON.parse(String(c.top_values ?? "[]")),
+                    })),
+                })),
+            };
+        }
+        finally {
+            db.close();
+        }
+    }
+    /** First rows of a table, for `data_inspect` and the prompt summary. */
+    async sample(record, table, rows, signal) {
+        const result = await this.engine.runQuery({
+            db: this.file(record),
+            sql: `SELECT * FROM "${table.replaceAll('"', '""')}" LIMIT ${Math.max(0, Math.floor(rows))}`,
+            maxRows: rows,
+            cellChars: 200,
+            guard: false,
+        }, { timeoutMs: this.limits.queryTimeoutMs, ...(signal ? { signal } : {}) });
+        return { table, columns: result.columns, rows: result.rows };
+    }
+    /** The description for the model: `data_inspect` text. */
+    async describe(record, sampleRows, maxChars, signal) {
+        const detail = this.detail(record);
+        const samples = new Map();
+        for (const sheet of detail.sheetDetails.slice(0, 5))
+            samples.set(sheet.table, await this.sample(record, sheet.table, sampleRows, signal));
+        return { detail, samples, text: describeDataset(detail, samples, { maxChars }) };
+    }
+    /** The ≤ 4 KB summary appended to a prompt that attaches the dataset. */
+    async summary(record, signal) {
+        const detail = this.detail(record);
+        const samples = new Map();
+        for (const sheet of detail.sheetDetails.slice(0, 3))
+            samples.set(sheet.table, await this.sample(record, sheet.table, 5, signal));
+        return promptSummary(detail, samples);
+    }
+    /**
+     * Runs a model-written read-only query (`data_query`): lexical guard, then a read-only
+     * connection in the engine process, with a row limit and a time limit.
+     */
+    async query(record, sql, options = {}) {
+        const guarded = guardSql(sql);
+        if (!guarded.ok)
+            throw new DataError("query_rejected", guarded.reason);
+        return this.engine.runQuery({
+            db: this.file(record),
+            sql: guarded.sql,
+            maxRows: Math.min(Math.max(1, options.maxRows ?? 200), 1000),
+            cellChars: 2048,
+            guard: true,
+            heapBytes: QUERY_HEAP_BYTES,
+        }, {
+            timeoutMs: this.limits.queryTimeoutMs,
+            ...(options.signal ? { signal: options.signal } : {}),
+        });
+    }
+    /** One page of a sheet for `SpreadsheetView` (spec §17.4). */
+    async page(record, request, signal) {
+        const detail = this.detail(record);
+        const sheet = detail.sheetDetails.find((s) => s.name === request.sheet || s.table === request.sheet) ??
+            (request.sheet === undefined ? detail.sheetDetails[0] : undefined);
+        if (!sheet)
+            throw new PageParamError(`Unknown sheet: ${request.sheet}`);
+        const interactive = sheet.rows <= this.limits.maxInteractiveRows;
+        const names = sheet.columns.map((c) => c.name);
+        const filterText = interactive ? (request.filter ?? "") : "";
+        if (filterText && request.column === undefined && sheet.rows > MAX_ALL_COLUMNS_FILTER_ROWS)
+            throw new PageParamError(`Choose a column to filter: this sheet has more than ${MAX_ALL_COLUMNS_FILTER_ROWS} rows`);
+        const params = {
+            table: sheet.table,
+            columns: names,
+            limit: Math.min(Math.max(1, request.limit ?? 200), MAX_PAGE_ROWS),
+            ...(interactive && request.sort
+                ? { sort: { column: request.sort, dir: request.dir ?? "asc" } }
+                : {}),
+            ...(filterText
+                ? {
+                    filter: {
+                        text: filterText,
+                        ...(request.column !== undefined ? { column: request.column } : {}),
+                    },
+                }
+                : {}),
+            ...(request.after ? { after: request.after } : {}),
+            ...(request.offset ? { offset: request.offset } : {}),
+        };
+        const built = buildPageQuery(params);
+        const run = (sql, values, maxRows) => this.engine.runQuery({ db: this.file(record), sql, params: values, maxRows, cellChars: 4096, guard: false }, { timeoutMs: Math.max(this.limits.queryTimeoutMs, 30_000), ...(signal ? { signal } : {}) });
+        const result = await run(built.sql, built.params, params.limit + 1);
+        const hasNext = result.rows.length > params.limit;
+        const page = hasNext ? result.rows.slice(0, params.limit) : result.rows;
+        const rowids = page.map((r) => Number(r[0]));
+        let matched;
+        if (filterText && !request.after && !request.offset) {
+            const count = buildCountQuery({ table: sheet.table, columns: names, filter: params.filter });
+            const counted = await run(count.sql, count.params, 1);
+            matched = Number(counted.rows[0]?.[0] ?? 0);
+        }
+        const last = rowids.at(-1);
+        return {
+            columns: sheet.columns.map((c) => ({ name: c.name, label: c.label, type: c.type })),
+            rows: page.map((r) => r.slice(1)),
+            rowids,
+            ...(hasNext && last !== undefined ? { next: encodeCursor(last) } : {}),
+            total: sheet.rows,
+            ...(matched !== undefined ? { matched } : {}),
+            interactive,
+        };
+    }
+    /** Deletes a dataset file and row (connections are closed first, for Windows). */
+    async remove(record) {
+        await this.engine.closeFile(this.file(record));
+        await rm(this.file(record), { force: true });
+        this.options.db.prepare("DELETE FROM datasets WHERE id=?").run(record.id);
+    }
+    close() {
+        this.engine.close();
+    }
+}

package/dist/analysis/data/describe.d.ts

@@ -0,0 +1,27 @@
+/**
+ * Text and table renderings of a dataset for the model and the terminal (spec §12 and the
+ * prompt summary of §21 phase 3). Pure: given the dataset detail and sample rows, produces a
+ * bounded text (the model never sees more than `maxChars`).
+ */
+import type { DatasetDetailWire } from "@alisio/sdk";
+export declare const formatCount: (n: number) => string;
+export declare function formatBytes(bytes: number): string;
+/** One cell as plain, single-line text of at most `max` characters. */
+export declare function cellText(value: string | number | null | undefined, max?: number): string;
+/** `ds_… "sales.csv" · csv · 1.2 MB · 12,480 rows × 6 columns` (one line per dataset). */
+export declare function datasetHeadline(detail: DatasetDetailWire): string;
+export interface SampleRows {
+    table: string;
+    columns: string[];
+    rows: Array<Array<string | number | null>>;
+}
+/**
+ * The description a model reads: headline, usage note, and for each sheet its columns with
+ * statistics and a few sample rows. Fits `maxChars` by dropping column detail first, then sample
+ * columns (never silently: a line says what was left out).
+ */
+export declare function describeDataset(detail: DatasetDetailWire, samples: Map<string, SampleRows>, options: {
+    maxChars: number;
+}): string;
+/** The summary appended to the user's prompt for an attached dataset (≤ 4 KB). */
+export declare function promptSummary(detail: DatasetDetailWire, samples: Map<string, SampleRows>): string;

package/dist/analysis/data/describe.js

@@ -0,0 +1,103 @@
+export const formatCount = (n) => n.toLocaleString("en-US");
+export function formatBytes(bytes) {
+    if (bytes < 1024)
+        return `${bytes} B`;
+    if (bytes < 1024 * 1024)
+        return `${Math.round(bytes / 1024)} KB`;
+    return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
+}
+/** One cell as plain, single-line text of at most `max` characters. */
+export function cellText(value, max = 40) {
+    if (value === null || value === undefined)
+        return "";
+    const text = String(value).replace(/\r?\n/g, "\\n").replace(/\t/g, " ");
+    return text.length > max ? `${text.slice(0, Math.max(1, max - 1))}…` : text;
+}
+function columnLine(column) {
+    const parts = [`${column.name} (${column.type})`];
+    if (column.label !== column.name)
+        parts.push(`header "${cellText(column.label, 30)}"`);
+    parts.push(`nulls ${formatCount(column.nulls)}`);
+    parts.push(`distinct ${formatCount(column.distinct)}${column.distinctExact ? "" : "~"}`);
+    if (column.min !== undefined && column.max !== undefined)
+        parts.push(`min ${cellText(column.min, 24)}, max ${cellText(column.max, 24)}`);
+    if (column.mean !== undefined)
+        parts.push(`mean ${Number(column.mean.toPrecision(6))}`);
+    if (column.textFallbacks > 0)
+        parts.push(`${formatCount(column.textFallbacks)} cells kept as exact text`);
+    if (column.top.length)
+        parts.push(`top: ${column.top
+            .slice(0, 3)
+            .map((t) => `${cellText(t.value, 20)} (${formatCount(t.count)})`)
+            .join(", ")}`);
+    return `- ${parts.join(" · ")}`;
+}
+/** `ds_… "sales.csv" · csv · 1.2 MB · 12,480 rows × 6 columns` (one line per dataset). */
+export function datasetHeadline(detail) {
+    const sheet = detail.sheets[0];
+    const shape = detail.sheets.length === 1 && sheet
+        ? `${formatCount(sheet.rows)} rows × ${sheet.columns} columns`
+        : `${detail.sheets.length} sheets`;
+    return `${detail.id} "${detail.name}" · ${detail.format} · ${formatBytes(detail.bytes)} · ${shape}`;
+}
+function sampleLines(sample, columnLimit) {
+    const names = sample.columns.slice(0, columnLimit);
+    const lines = [names.join(" | ")];
+    for (const row of sample.rows)
+        lines.push(row
+            .slice(0, columnLimit)
+            .map((v) => cellText(v, 24))
+            .join(" | "));
+    if (sample.columns.length > columnLimit)
+        lines.push(`… ${sample.columns.length - columnLimit} more columns`);
+    return lines;
+}
+/**
+ * The description a model reads: headline, usage note, and for each sheet its columns with
+ * statistics and a few sample rows. Fits `maxChars` by dropping column detail first, then sample
+ * columns (never silently: a line says what was left out).
+ */
+export function describeDataset(detail, samples, options) {
+    const header = [
+        `Dataset ${datasetHeadline(detail)}`,
+        "Stored in SQLite: query it with data_query (SELECT/WITH only, quote column names) or pass { datasetId } to python_run (alisio_runtime.datasets.open). Values are exact: numbers are numbers, everything else (e.g. 007, 1,234, N/A) is the original text.",
+    ];
+    const blocks = detail.sheetDetails.map((sheet) => {
+        const lines = [
+            `Table "${sheet.table}" (sheet "${sheet.name}"): ${formatCount(sheet.rows)} rows`,
+        ];
+        return { sheet, lines };
+    });
+    const build = (columnCap, sampleCols) => {
+        const out = [...header];
+        for (const { sheet, lines } of blocks) {
+            out.push(...lines);
+            for (const column of sheet.columns.slice(0, columnCap))
+                out.push(columnLine(column));
+            if (sheet.columns.length > columnCap)
+                out.push(`… ${sheet.columns.length - columnCap} more columns (see the table in data_inspect)`);
+            const sample = samples.get(sheet.table);
+            if (sample?.rows.length) {
+                out.push(`First ${sample.rows.length} rows:`);
+                out.push(...sampleLines(sample, sampleCols));
+            }
+        }
+        return out.join("\n");
+    };
+    for (const [columns, sampleCols] of [
+        [60, 12],
+        [30, 10],
+        [16, 8],
+        [8, 6],
+        [4, 4],
+    ]) {
+        const text = build(columns, sampleCols);
+        if (text.length <= options.maxChars)
+            return text;
+    }
+    return `${build(2, 3).slice(0, Math.max(0, options.maxChars - 1))}…`;
+}
+/** The summary appended to the user's prompt for an attached dataset (≤ 4 KB). */
+export function promptSummary(detail, samples) {
+    return `[Attached ${describeDataset(detail, samples, { maxChars: 3900 })}]`;
+}

package/dist/analysis/data/engine-client.d.ts

@@ -0,0 +1,58 @@
+import type { FinalizeRequest, IngestRequest, IngestResult, QueryRequest, QueryResult } from "./engine-types.ts";
+export type DataErrorCode = "dataset_unsupported" | "query_rejected" | "query_timeout" | "not_found" | "ingest_limit" | "ingest_invalid" | "sql_error" | "engine_error" | "cancelled";
+/** A failure with a stable code that tools and routes map to their own error shapes. */
+export declare class DataError extends Error {
+    readonly code: DataErrorCode;
+    constructor(code: DataErrorCode, message: string);
+}
+/** The engine script on disk (`<root>/analysis/engine/data-engine-<hash>.mjs`). */
+export declare function ensureEngineScript(root: string): Promise<string>;
+export interface DataEngineOptions {
+    /** State folder (`<root>/analysis/engine/…` holds the script). */
+    root: string;
+    /** Heap limit of the child process (Node only), in MiB. */
+    memoryMb?: number;
+    /** A persistent query process exits after this much inactivity. */
+    idleMs?: number;
+    /** Ingestions running at once. */
+    concurrency?: number;
+}
+/** Ingestion and query access to dataset files through the engine process. */
+export declare class DataEngine {
+    private readonly options;
+    private script;
+    private query;
+    private queryQueue;
+    private idle;
+    private running;
+    private waiting;
+    private active;
+    private closed;
+    constructor(options: DataEngineOptions);
+    private scriptPath;
+    private memory;
+    private slot;
+    /** Runs one request in a fresh process (ingestion and finalization). */
+    private oneShot;
+    ingest(request: Omit<IngestRequest, "id" | "op">, options?: {
+        signal?: AbortSignal;
+        onProgress?: (rows: number) => void;
+    }): Promise<IngestResult>;
+    finalize(request: Omit<FinalizeRequest, "id" | "op">, options?: {
+        signal?: AbortSignal;
+    }): Promise<IngestResult>;
+    /**
+     * Runs one read-only statement. Requests are served one at a time by a persistent process;
+     * exceeding `timeoutMs` kills it (the next request starts a new one) and fails with
+     * `query_timeout`.
+     */
+    runQuery(request: Omit<QueryRequest, "id" | "op">, options: {
+        timeoutMs: number;
+        signal?: AbortSignal;
+    }): Promise<QueryResult>;
+    private armIdle;
+    /** Closes the engine's connection to a file (before the host deletes or replaces it). */
+    closeFile(db?: string): Promise<void>;
+    /** Kills every process; later requests fail. */
+    close(): void;
+}