@alisio/core 0.1.0-alpha.21 → 0.1.0-alpha.23

package/dist/analysis/capabilities.d.ts

@@ -0,0 +1,48 @@
+/**
+ * Persisted capability decisions (`capability_grants`). A `session` allow of the ROOT session is
+ * the only state that grants anything; `once` allows and every `deny` are audit rows only (a deny
+ * is never sticky: the next call asks again). Revoking stamps `revoked_at`; nothing is deleted.
+ */
+import type { AnalysisCapability, CapabilityGrantWire, SqlDatabase } from "@alisio/sdk";
+export type GrantSource = CapabilityGrantWire["source"];
+export interface GrantRecord extends CapabilityGrantWire {
+    workspace: string;
+    callId?: string;
+    runId?: string;
+    correlationId?: string;
+    revokedBy?: string;
+}
+/** Wire shape (no workspace path or call ids). */
+export declare const toGrantWire: (record: GrantRecord) => CapabilityGrantWire;
+export declare class CapabilityGrants {
+    private options;
+    constructor(options: {
+        db: SqlDatabase;
+        /** Root of a session (grants always belong to the root). */
+        rootOf: (sessionId: string) => string;
+        now?: () => number;
+    });
+    rootOf(sessionId: string): string;
+    /** The live `session` allow of `capability` for the root of `sessionId`, if any. */
+    active(capability: AnalysisCapability, sessionId: string): GrantRecord | undefined;
+    granted(capability: AnalysisCapability, sessionId: string): boolean;
+    /** Records one decision (audit row; a `session` allow is also the live grant). */
+    record(input: {
+        capability: AnalysisCapability;
+        sessionId: string;
+        workspace: string;
+        scope: "once" | "session";
+        decision: "allow" | "deny";
+        source: GrantSource;
+        callId?: string;
+        runId?: string;
+        correlationId?: string;
+    }): GrantRecord;
+    get(id: string): GrantRecord | undefined;
+    /** Live session grants first, then the latest `limit` audit rows of the root session. */
+    list(sessionId: string, limit?: number): GrantRecord[];
+    /** Live grants of the root session only. */
+    live(sessionId: string): GrantRecord[];
+    /** Revokes a live grant of `sessionId`'s root. False when unknown, foreign or already revoked. */
+    revoke(sessionId: string, grantId: string, by: string): boolean;
+}

package/dist/analysis/capabilities.js

@@ -0,0 +1,90 @@
+import { newId } from "../runtime/ids.js";
+const toRecord = (r) => ({
+    id: String(r.id),
+    capability: String(r.capability),
+    sessionId: String(r.session),
+    workspace: String(r.workspace),
+    scope: String(r.scope),
+    decision: String(r.decision),
+    source: String(r.source),
+    createdAt: Number(r.created_at),
+    ...(r.revoked_at != null ? { revokedAt: Number(r.revoked_at) } : {}),
+    ...(r.revoked_by != null ? { revokedBy: String(r.revoked_by) } : {}),
+    ...(r.call_id != null ? { callId: String(r.call_id) } : {}),
+    ...(r.run_id != null ? { runId: String(r.run_id) } : {}),
+    ...(r.correlation_id != null ? { correlationId: String(r.correlation_id) } : {}),
+});
+/** Wire shape (no workspace path or call ids). */
+export const toGrantWire = (record) => ({
+    id: record.id,
+    capability: record.capability,
+    sessionId: record.sessionId,
+    scope: record.scope,
+    decision: record.decision,
+    source: record.source,
+    createdAt: record.createdAt,
+    ...(record.revokedAt !== undefined ? { revokedAt: record.revokedAt } : {}),
+});
+export class CapabilityGrants {
+    options;
+    constructor(options) {
+        this.options = options;
+    }
+    rootOf(sessionId) {
+        return this.options.rootOf(sessionId);
+    }
+    /** The live `session` allow of `capability` for the root of `sessionId`, if any. */
+    active(capability, sessionId) {
+        const row = this.options.db
+            .prepare(`SELECT * FROM capability_grants WHERE capability=? AND session=? AND scope='session'
+           AND decision='allow' AND revoked_at IS NULL ORDER BY created_at DESC LIMIT 1`)
+            .get(capability, this.rootOf(sessionId));
+        return row ? toRecord(row) : undefined;
+    }
+    granted(capability, sessionId) {
+        return !!this.active(capability, sessionId);
+    }
+    /** Records one decision (audit row; a `session` allow is also the live grant). */
+    record(input) {
+        const id = newId("grant");
+        const createdAt = (this.options.now ?? Date.now)();
+        this.options.db
+            .prepare(`INSERT INTO capability_grants(id,capability,session,workspace,scope,decision,source,call_id,
+           run_id,correlation_id,created_at) VALUES(?,?,?,?,?,?,?,?,?,?,?)`)
+            .run(id, input.capability, this.rootOf(input.sessionId), input.workspace, input.scope, input.decision, input.source, input.callId ?? null, input.runId ?? null, input.correlationId ?? null, createdAt);
+        return this.get(id);
+    }
+    get(id) {
+        const row = this.options.db.prepare("SELECT * FROM capability_grants WHERE id=?").get(id);
+        return row ? toRecord(row) : undefined;
+    }
+    /** Live session grants first, then the latest `limit` audit rows of the root session. */
+    list(sessionId, limit = 50) {
+        const root = this.rootOf(sessionId);
+        const live = this.options.db
+            .prepare(`SELECT * FROM capability_grants WHERE session=? AND scope='session' AND decision='allow'
+             AND revoked_at IS NULL ORDER BY created_at DESC`)
+            .all(root).map(toRecord);
+        const ids = new Set(live.map((g) => g.id));
+        const audit = this.options.db
+            .prepare("SELECT * FROM capability_grants WHERE session=? ORDER BY created_at DESC LIMIT ?")
+            .all(root, limit)
+            .map(toRecord)
+            .filter((g) => !ids.has(g.id));
+        return [...live, ...audit];
+    }
+    /** Live grants of the root session only. */
+    live(sessionId) {
+        return this.list(sessionId, 0).filter((g) => g.revokedAt === undefined && g.scope === "session");
+    }
+    /** Revokes a live grant of `sessionId`'s root. False when unknown, foreign or already revoked. */
+    revoke(sessionId, grantId, by) {
+        const grant = this.get(grantId);
+        if (!grant || grant.sessionId !== this.rootOf(sessionId) || grant.revokedAt !== undefined)
+            return false;
+        this.options.db
+            .prepare("UPDATE capability_grants SET revoked_at=?, revoked_by=? WHERE id=?")
+            .run((this.options.now ?? Date.now)(), by, grantId);
+        return true;
+    }
+}

package/dist/analysis/data/csv.d.ts

@@ -0,0 +1,46 @@
+/**
+ * Streaming CSV/TSV parser (RFC 4180, spec §17.2): quoted fields, doubled quotes, line breaks
+ * inside quotes, CRLF/LF/CR rows. No dependencies and no I/O: callers push decoded text chunks
+ * (any split, even inside a quote or between CR and LF) and receive rows through a callback.
+ * Blank lines are skipped. Lenient on purpose: a stray quote inside an unquoted field is literal
+ * and text after a closing quote is appended.
+ */
+export declare class CsvError extends Error {
+    readonly code: "cell_too_large";
+    constructor(message: string, code: "cell_too_large");
+}
+export declare class CsvParser {
+    private readonly onRow;
+    private readonly maxCellChars;
+    private state;
+    private field;
+    private fields;
+    private skipLineFeed;
+    /** The current field was opened with a quote (so `""` is a value, not a blank line). */
+    private quotedField;
+    private readonly delimiter;
+    /** Rows delivered so far (blank lines excluded). */
+    rows: number;
+    constructor(delimiter: string, onRow: (fields: string[]) => void, maxCellChars?: number);
+    private add;
+    private endField;
+    private endRow;
+    write(chunk: string): void;
+    /** Flushes the last row when the input has no final line break. */
+    end(): void;
+}
+/** Parses a whole string (tests and small inputs). */
+export declare function parseCsv(text: string, delimiter?: string): string[][];
+/**
+ * Detects the delimiter from a text sample (the first 64 KiB): the candidate that splits most
+ * rows into the same number (> 1) of fields wins; ties keep the order `,` `;` TAB `|`.
+ */
+export declare function detectDelimiter(sample: string): string;
+/**
+ * Decodes a byte sample as the ingestion does: UTF-8 (BOM removed), UTF-16 by BOM, or
+ * windows-1252 when the bytes are not valid UTF-8. A sample cut inside a character is tolerated.
+ */
+export declare function decodeTextBytes(bytes: Uint8Array, truncated?: boolean): {
+    text: string;
+    encoding: string;
+};

package/dist/analysis/data/csv.js

@@ -0,0 +1,240 @@
+/**
+ * Streaming CSV/TSV parser (RFC 4180, spec §17.2): quoted fields, doubled quotes, line breaks
+ * inside quotes, CRLF/LF/CR rows. No dependencies and no I/O: callers push decoded text chunks
+ * (any split, even inside a quote or between CR and LF) and receive rows through a callback.
+ * Blank lines are skipped. Lenient on purpose: a stray quote inside an unquoted field is literal
+ * and text after a closing quote is appended.
+ */
+const QUOTE = 34;
+const CR = 13;
+const LF = 10;
+var State;
+(function (State) {
+    /** Before the first character of a field. */
+    State[State["Start"] = 0] = "Start";
+    State[State["Unquoted"] = 1] = "Unquoted";
+    State[State["Quoted"] = 2] = "Quoted";
+    /** Right after a `"` inside a quoted field: either an escaped quote or the closing one. */
+    State[State["QuoteInQuoted"] = 3] = "QuoteInQuoted";
+})(State || (State = {}));
+export class CsvError extends Error {
+    code;
+    constructor(message, code) {
+        super(message);
+        this.code = code;
+    }
+}
+export class CsvParser {
+    onRow;
+    maxCellChars;
+    state = State.Start;
+    field = "";
+    fields = [];
+    skipLineFeed = false;
+    /** The current field was opened with a quote (so `""` is a value, not a blank line). */
+    quotedField = false;
+    delimiter;
+    /** Rows delivered so far (blank lines excluded). */
+    rows = 0;
+    constructor(delimiter, onRow, maxCellChars = 1024 * 1024) {
+        this.onRow = onRow;
+        this.maxCellChars = maxCellChars;
+        if (delimiter.length !== 1)
+            throw new Error("The CSV delimiter must be one character");
+        this.delimiter = delimiter.charCodeAt(0);
+    }
+    add(text) {
+        this.field += text;
+        if (this.field.length > this.maxCellChars)
+            throw new CsvError(`A cell in row ${this.rows + 1} exceeds ${this.maxCellChars} characters`, "cell_too_large");
+    }
+    endField() {
+        this.fields.push(this.field);
+        this.field = "";
+        this.quotedField = false;
+    }
+    endRow() {
+        const blank = this.fields.length === 0 && this.field === "" && !this.quotedField;
+        if (!blank) {
+            this.endField();
+            this.rows++;
+            const row = this.fields;
+            this.fields = [];
+            this.state = State.Start;
+            this.onRow(row);
+            return;
+        }
+        this.state = State.Start;
+        this.quotedField = false;
+    }
+    write(chunk) {
+        const n = chunk.length;
+        let i = 0;
+        if (this.skipLineFeed && n > 0) {
+            if (chunk.charCodeAt(0) === LF)
+                i = 1;
+            this.skipLineFeed = false;
+        }
+        while (i < n) {
+            switch (this.state) {
+                case State.Start: {
+                    if (chunk.charCodeAt(i) === QUOTE) {
+                        this.state = State.Quoted;
+                        this.quotedField = true;
+                        i++;
+                    }
+                    else
+                        this.state = State.Unquoted;
+                    break;
+                }
+                case State.Unquoted: {
+                    let j = i;
+                    let c = 0;
+                    while (j < n) {
+                        c = chunk.charCodeAt(j);
+                        if (c === this.delimiter || c === LF || c === CR)
+                            break;
+                        j++;
+                    }
+                    if (j > i)
+                        this.add(chunk.slice(i, j));
+                    i = j;
+                    if (j >= n)
+                        break;
+                    if (c === this.delimiter) {
+                        this.endField();
+                        this.state = State.Start;
+                        i++;
+                    }
+                    else {
+                        i++;
+                        if (c === CR) {
+                            if (i < n && chunk.charCodeAt(i) === LF)
+                                i++;
+                            else if (i >= n)
+                                this.skipLineFeed = true;
+                        }
+                        this.endRow();
+                    }
+                    break;
+                }
+                case State.Quoted: {
+                    const quote = chunk.indexOf('"', i);
+                    if (quote < 0) {
+                        this.add(chunk.slice(i));
+                        i = n;
+                    }
+                    else {
+                        if (quote > i)
+                            this.add(chunk.slice(i, quote));
+                        i = quote + 1;
+                        this.state = State.QuoteInQuoted;
+                    }
+                    break;
+                }
+                case State.QuoteInQuoted: {
+                    const c = chunk.charCodeAt(i);
+                    if (c === QUOTE) {
+                        this.add('"');
+                        this.state = State.Quoted;
+                        i++;
+                    }
+                    else if (c === this.delimiter) {
+                        this.endField();
+                        this.state = State.Start;
+                        i++;
+                    }
+                    else if (c === LF || c === CR) {
+                        i++;
+                        if (c === CR) {
+                            if (i < n && chunk.charCodeAt(i) === LF)
+                                i++;
+                            else if (i >= n)
+                                this.skipLineFeed = true;
+                        }
+                        // A quoted-empty field ("") is a real, empty value: the row is not blank.
+                        this.state = State.Unquoted;
+                        this.endRow();
+                    }
+                    else {
+                        // Text after the closing quote: keep it (lenient).
+                        this.state = State.Unquoted;
+                    }
+                    break;
+                }
+            }
+        }
+    }
+    /** Flushes the last row when the input has no final line break. */
+    end() {
+        if (this.fields.length === 0 && this.field === "" && !this.quotedField)
+            return;
+        this.endRow();
+    }
+}
+/** Parses a whole string (tests and small inputs). */
+export function parseCsv(text, delimiter = ",") {
+    const rows = [];
+    const parser = new CsvParser(delimiter, (row) => rows.push(row));
+    parser.write(text);
+    parser.end();
+    return rows;
+}
+const CANDIDATES = [",", ";", "\t", "|"];
+/**
+ * Detects the delimiter from a text sample (the first 64 KiB): the candidate that splits most
+ * rows into the same number (> 1) of fields wins; ties keep the order `,` `;` TAB `|`.
+ */
+export function detectDelimiter(sample) {
+    let best = ",";
+    let bestScore = 0;
+    for (const candidate of CANDIDATES) {
+        const counts = new Map();
+        let rows = 0;
+        const parser = new CsvParser(candidate, (row) => {
+            rows++;
+            counts.set(row.length, (counts.get(row.length) ?? 0) + 1);
+        });
+        try {
+            parser.write(sample);
+        }
+        catch {
+            continue;
+        }
+        // The last row may be cut by the sample boundary: it is never flushed (no end()).
+        if (rows < 1)
+            continue;
+        let width = 0;
+        let frequency = 0;
+        for (const [w, count] of counts)
+            if (w > 1 && (count > frequency || (count === frequency && w > width))) {
+                width = w;
+                frequency = count;
+            }
+        const score = frequency * 1000 + width;
+        if (score > bestScore) {
+            best = candidate;
+            bestScore = score;
+        }
+    }
+    return best;
+}
+/**
+ * Decodes a byte sample as the ingestion does: UTF-8 (BOM removed), UTF-16 by BOM, or
+ * windows-1252 when the bytes are not valid UTF-8. A sample cut inside a character is tolerated.
+ */
+export function decodeTextBytes(bytes, truncated = false) {
+    if (bytes[0] === 0xff && bytes[1] === 0xfe)
+        return { text: new TextDecoder("utf-16le").decode(bytes), encoding: "utf-16le" };
+    if (bytes[0] === 0xfe && bytes[1] === 0xff)
+        return { text: new TextDecoder("utf-16be").decode(bytes), encoding: "utf-16be" };
+    try {
+        return {
+            text: new TextDecoder("utf-8", { fatal: true }).decode(bytes, { stream: truncated }),
+            encoding: "utf-8",
+        };
+    }
+    catch {
+        return { text: new TextDecoder("windows-1252").decode(bytes), encoding: "windows-1252" };
+    }
+}

package/dist/analysis/data/datasets.d.ts

@@ -0,0 +1,132 @@
+import type { DatasetDetailWire, DatasetFormat, DatasetRef, DatasetRowsPage, SqlDatabase } from "@alisio/sdk";
+import type { PythonResolution } from "../runtime-manager.ts";
+import { type SampleRows } from "./describe.ts";
+import { DataEngine, DataError } from "./engine-client.ts";
+import type { SheetSummary } from "./engine-types.ts";
+import { PageParamError } from "./rows.ts";
+export { DataError, PageParamError };
+export interface DatasetLimits {
+    maxUploadBytes: number;
+    maxRows: number;
+    queryTimeoutMs: number;
+    maxInteractiveRows: number;
+}
+export declare const DEFAULT_DATASET_LIMITS: DatasetLimits;
+export declare const XLSX_REMEDY = "Export the sheet as CSV, or install Python 3.10+.";
+/** The format of a file name, or undefined when it is not a supported dataset. */
+export declare function datasetFormat(name: string): DatasetFormat | undefined;
+export interface DatasetRecord {
+    id: string;
+    /** The session that created it. */
+    sessionId: string;
+    rootSessionId: string;
+    name: string;
+    format: DatasetFormat;
+    sha256: string;
+    bytes: number;
+    blobHash?: string;
+    sourcePath?: string;
+    dbRelPath: string;
+    ingestVersion: number;
+    sheets: SheetSummary[];
+    createdAt: number;
+}
+export declare const toDatasetRef: (record: DatasetRecord) => DatasetRef;
+export interface IngestInput {
+    rootSessionId: string;
+    sessionId: string;
+    workspace: string;
+    /** Absolute path of the file to read (resolved and authorised by the caller). */
+    path: string;
+    /** Display name; defaults to the file name. */
+    name?: string;
+    blobHash?: string;
+    /** Recorded as `source_path` (a workspace or artifact file); never sent to clients. */
+    sourcePath?: string;
+    signal?: AbortSignal;
+    onProgress?: (rows: number) => void;
+}
+export interface DatasetServiceOptions {
+    /** State folder (`stateHome()` or the folder of `--db`). */
+    root: string;
+    db: SqlDatabase;
+    limits?: Partial<DatasetLimits>;
+    /** The Python interpreter (XLSX only); absent means XLSX is unsupported. */
+    python?: {
+        interpreter(signal: AbortSignal): Promise<PythonResolution>;
+    };
+    engine?: DataEngine;
+    now?: () => number;
+}
+export declare class DatasetService {
+    private readonly options;
+    readonly limits: DatasetLimits;
+    readonly engine: DataEngine;
+    private inflight;
+    constructor(options: DatasetServiceOptions);
+    private now;
+    /** Absolute path of a dataset file. */
+    file(record: DatasetRecord): string;
+    get(id: string): DatasetRecord | undefined;
+    private touched;
+    /**
+     * Records that a dataset was used, for the retention sweep ("unused for `jobsDays`"): the
+     * modification time of its SQLite file (nothing else changes it after the ingestion), at most
+     * once every ten minutes per dataset. Best effort: a failure never affects the caller.
+     */
+    private touch;
+    /** Every dataset (all sessions), for the retention sweep; does not count as a use. */
+    all(): DatasetRecord[];
+    /** A dataset of this root session; another session's id is `not_found`. */
+    getFor(id: string, rootSessionId: string): DatasetRecord;
+    list(rootSessionId: string): DatasetRecord[];
+    private bySha;
+    /**
+     * Ingests a file into a dataset, or returns the session's existing dataset with the same
+     * content (`reused`). Concurrent calls for the same content share one ingestion. A failure
+     * leaves no row and no file.
+     */
+    ingest(input: IngestInput): Promise<{
+        record: DatasetRecord;
+        reused: boolean;
+    }>;
+    private build;
+    /** The XLSX route (spec §17.6, option B): the Python standard-library helper, then statistics. */
+    private ingestXlsx;
+    /** The `alisio_runtime` package on disk for `python -m` (content-addressed, written once). */
+    private helperDir;
+    /** Schema and statistics of every sheet (`GET /api/datasets/:did`). */
+    detail(record: DatasetRecord): DatasetDetailWire;
+    /** First rows of a table, for `data_inspect` and the prompt summary. */
+    sample(record: DatasetRecord, table: string, rows: number, signal?: AbortSignal): Promise<SampleRows>;
+    /** The description for the model: `data_inspect` text. */
+    describe(record: DatasetRecord, sampleRows: number, maxChars: number, signal?: AbortSignal): Promise<{
+        detail: DatasetDetailWire;
+        samples: Map<string, SampleRows>;
+        text: string;
+    }>;
+    /** The ≤ 4 KB summary appended to a prompt that attaches the dataset. */
+    summary(record: DatasetRecord, signal?: AbortSignal): Promise<string>;
+    /**
+     * Runs a model-written read-only query (`data_query`): lexical guard, then a read-only
+     * connection in the engine process, with a row limit and a time limit.
+     */
+    query(record: DatasetRecord, sql: string, options?: {
+        maxRows?: number;
+        signal?: AbortSignal;
+    }): Promise<import("./engine-types.ts").QueryResult>;
+    /** One page of a sheet for `SpreadsheetView` (spec §17.4). */
+    page(record: DatasetRecord, request: {
+        sheet?: string;
+        after?: string;
+        offset?: number;
+        limit?: number;
+        sort?: string;
+        dir?: "asc" | "desc";
+        filter?: string;
+        column?: string;
+    }, signal?: AbortSignal): Promise<DatasetRowsPage>;
+    /** Deletes a dataset file and row (connections are closed first, for Windows). */
+    remove(record: DatasetRecord): Promise<void>;
+    close(): void;
+}