@alisio/core 0.1.0-alpha.8 → 0.1.0

package/dist/analysis/data/csv.js

@@ -0,0 +1,240 @@
+/**
+ * Streaming CSV/TSV parser (RFC 4180, spec §17.2): quoted fields, doubled quotes, line breaks
+ * inside quotes, CRLF/LF/CR rows. No dependencies and no I/O: callers push decoded text chunks
+ * (any split, even inside a quote or between CR and LF) and receive rows through a callback.
+ * Blank lines are skipped. Lenient on purpose: a stray quote inside an unquoted field is literal
+ * and text after a closing quote is appended.
+ */
+const QUOTE = 34;
+const CR = 13;
+const LF = 10;
+var State;
+(function (State) {
+    /** Before the first character of a field. */
+    State[State["Start"] = 0] = "Start";
+    State[State["Unquoted"] = 1] = "Unquoted";
+    State[State["Quoted"] = 2] = "Quoted";
+    /** Right after a `"` inside a quoted field: either an escaped quote or the closing one. */
+    State[State["QuoteInQuoted"] = 3] = "QuoteInQuoted";
+})(State || (State = {}));
+export class CsvError extends Error {
+    code;
+    constructor(message, code) {
+        super(message);
+        this.code = code;
+    }
+}
+export class CsvParser {
+    onRow;
+    maxCellChars;
+    state = State.Start;
+    field = "";
+    fields = [];
+    skipLineFeed = false;
+    /** The current field was opened with a quote (so `""` is a value, not a blank line). */
+    quotedField = false;
+    delimiter;
+    /** Rows delivered so far (blank lines excluded). */
+    rows = 0;
+    constructor(delimiter, onRow, maxCellChars = 1024 * 1024) {
+        this.onRow = onRow;
+        this.maxCellChars = maxCellChars;
+        if (delimiter.length !== 1)
+            throw new Error("The CSV delimiter must be one character");
+        this.delimiter = delimiter.charCodeAt(0);
+    }
+    add(text) {
+        this.field += text;
+        if (this.field.length > this.maxCellChars)
+            throw new CsvError(`A cell in row ${this.rows + 1} exceeds ${this.maxCellChars} characters`, "cell_too_large");
+    }
+    endField() {
+        this.fields.push(this.field);
+        this.field = "";
+        this.quotedField = false;
+    }
+    endRow() {
+        const blank = this.fields.length === 0 && this.field === "" && !this.quotedField;
+        if (!blank) {
+            this.endField();
+            this.rows++;
+            const row = this.fields;
+            this.fields = [];
+            this.state = State.Start;
+            this.onRow(row);
+            return;
+        }
+        this.state = State.Start;
+        this.quotedField = false;
+    }
+    write(chunk) {
+        const n = chunk.length;
+        let i = 0;
+        if (this.skipLineFeed && n > 0) {
+            if (chunk.charCodeAt(0) === LF)
+                i = 1;
+            this.skipLineFeed = false;
+        }
+        while (i < n) {
+            switch (this.state) {
+                case State.Start: {
+                    if (chunk.charCodeAt(i) === QUOTE) {
+                        this.state = State.Quoted;
+                        this.quotedField = true;
+                        i++;
+                    }
+                    else
+                        this.state = State.Unquoted;
+                    break;
+                }
+                case State.Unquoted: {
+                    let j = i;
+                    let c = 0;
+                    while (j < n) {
+                        c = chunk.charCodeAt(j);
+                        if (c === this.delimiter || c === LF || c === CR)
+                            break;
+                        j++;
+                    }
+                    if (j > i)
+                        this.add(chunk.slice(i, j));
+                    i = j;
+                    if (j >= n)
+                        break;
+                    if (c === this.delimiter) {
+                        this.endField();
+                        this.state = State.Start;
+                        i++;
+                    }
+                    else {
+                        i++;
+                        if (c === CR) {
+                            if (i < n && chunk.charCodeAt(i) === LF)
+                                i++;
+                            else if (i >= n)
+                                this.skipLineFeed = true;
+                        }
+                        this.endRow();
+                    }
+                    break;
+                }
+                case State.Quoted: {
+                    const quote = chunk.indexOf('"', i);
+                    if (quote < 0) {
+                        this.add(chunk.slice(i));
+                        i = n;
+                    }
+                    else {
+                        if (quote > i)
+                            this.add(chunk.slice(i, quote));
+                        i = quote + 1;
+                        this.state = State.QuoteInQuoted;
+                    }
+                    break;
+                }
+                case State.QuoteInQuoted: {
+                    const c = chunk.charCodeAt(i);
+                    if (c === QUOTE) {
+                        this.add('"');
+                        this.state = State.Quoted;
+                        i++;
+                    }
+                    else if (c === this.delimiter) {
+                        this.endField();
+                        this.state = State.Start;
+                        i++;
+                    }
+                    else if (c === LF || c === CR) {
+                        i++;
+                        if (c === CR) {
+                            if (i < n && chunk.charCodeAt(i) === LF)
+                                i++;
+                            else if (i >= n)
+                                this.skipLineFeed = true;
+                        }
+                        // A quoted-empty field ("") is a real, empty value: the row is not blank.
+                        this.state = State.Unquoted;
+                        this.endRow();
+                    }
+                    else {
+                        // Text after the closing quote: keep it (lenient).
+                        this.state = State.Unquoted;
+                    }
+                    break;
+                }
+            }
+        }
+    }
+    /** Flushes the last row when the input has no final line break. */
+    end() {
+        if (this.fields.length === 0 && this.field === "" && !this.quotedField)
+            return;
+        this.endRow();
+    }
+}
+/** Parses a whole string (tests and small inputs). */
+export function parseCsv(text, delimiter = ",") {
+    const rows = [];
+    const parser = new CsvParser(delimiter, (row) => rows.push(row));
+    parser.write(text);
+    parser.end();
+    return rows;
+}
+const CANDIDATES = [",", ";", "\t", "|"];
+/**
+ * Detects the delimiter from a text sample (the first 64 KiB): the candidate that splits most
+ * rows into the same number (> 1) of fields wins; ties keep the order `,` `;` TAB `|`.
+ */
+export function detectDelimiter(sample) {
+    let best = ",";
+    let bestScore = 0;
+    for (const candidate of CANDIDATES) {
+        const counts = new Map();
+        let rows = 0;
+        const parser = new CsvParser(candidate, (row) => {
+            rows++;
+            counts.set(row.length, (counts.get(row.length) ?? 0) + 1);
+        });
+        try {
+            parser.write(sample);
+        }
+        catch {
+            continue;
+        }
+        // The last row may be cut by the sample boundary: it is never flushed (no end()).
+        if (rows < 1)
+            continue;
+        let width = 0;
+        let frequency = 0;
+        for (const [w, count] of counts)
+            if (w > 1 && (count > frequency || (count === frequency && w > width))) {
+                width = w;
+                frequency = count;
+            }
+        const score = frequency * 1000 + width;
+        if (score > bestScore) {
+            best = candidate;
+            bestScore = score;
+        }
+    }
+    return best;
+}
+/**
+ * Decodes a byte sample as the ingestion does: UTF-8 (BOM removed), UTF-16 by BOM, or
+ * windows-1252 when the bytes are not valid UTF-8. A sample cut inside a character is tolerated.
+ */
+export function decodeTextBytes(bytes, truncated = false) {
+    if (bytes[0] === 0xff && bytes[1] === 0xfe)
+        return { text: new TextDecoder("utf-16le").decode(bytes), encoding: "utf-16le" };
+    if (bytes[0] === 0xfe && bytes[1] === 0xff)
+        return { text: new TextDecoder("utf-16be").decode(bytes), encoding: "utf-16be" };
+    try {
+        return {
+            text: new TextDecoder("utf-8", { fatal: true }).decode(bytes, { stream: truncated }),
+            encoding: "utf-8",
+        };
+    }
+    catch {
+        return { text: new TextDecoder("windows-1252").decode(bytes), encoding: "windows-1252" };
+    }
+}

package/dist/analysis/data/datasets.d.ts

@@ -0,0 +1,132 @@
+import type { DatasetDetailWire, DatasetFormat, DatasetRef, DatasetRowsPage, SqlDatabase } from "@alisio/sdk";
+import type { PythonResolution } from "../runtime-manager.ts";
+import { type SampleRows } from "./describe.ts";
+import { DataEngine, DataError } from "./engine-client.ts";
+import type { SheetSummary } from "./engine-types.ts";
+import { PageParamError } from "./rows.ts";
+export { DataError, PageParamError };
+export interface DatasetLimits {
+    maxUploadBytes: number;
+    maxRows: number;
+    queryTimeoutMs: number;
+    maxInteractiveRows: number;
+}
+export declare const DEFAULT_DATASET_LIMITS: DatasetLimits;
+export declare const XLSX_REMEDY = "Export the sheet as CSV, or install Python 3.10+.";
+/** The format of a file name, or undefined when it is not a supported dataset. */
+export declare function datasetFormat(name: string): DatasetFormat | undefined;
+export interface DatasetRecord {
+    id: string;
+    /** The session that created it. */
+    sessionId: string;
+    rootSessionId: string;
+    name: string;
+    format: DatasetFormat;
+    sha256: string;
+    bytes: number;
+    blobHash?: string;
+    sourcePath?: string;
+    dbRelPath: string;
+    ingestVersion: number;
+    sheets: SheetSummary[];
+    createdAt: number;
+}
+export declare const toDatasetRef: (record: DatasetRecord) => DatasetRef;
+export interface IngestInput {
+    rootSessionId: string;
+    sessionId: string;
+    workspace: string;
+    /** Absolute path of the file to read (resolved and authorised by the caller). */
+    path: string;
+    /** Display name; defaults to the file name. */
+    name?: string;
+    blobHash?: string;
+    /** Recorded as `source_path` (a workspace or artifact file); never sent to clients. */
+    sourcePath?: string;
+    signal?: AbortSignal;
+    onProgress?: (rows: number) => void;
+}
+export interface DatasetServiceOptions {
+    /** State folder (`stateHome()` or the folder of `--db`). */
+    root: string;
+    db: SqlDatabase;
+    limits?: Partial<DatasetLimits>;
+    /** The Python interpreter (XLSX only); absent means XLSX is unsupported. */
+    python?: {
+        interpreter(signal: AbortSignal): Promise<PythonResolution>;
+    };
+    engine?: DataEngine;
+    now?: () => number;
+}
+export declare class DatasetService {
+    private readonly options;
+    readonly limits: DatasetLimits;
+    readonly engine: DataEngine;
+    private inflight;
+    constructor(options: DatasetServiceOptions);
+    private now;
+    /** Absolute path of a dataset file. */
+    file(record: DatasetRecord): string;
+    get(id: string): DatasetRecord | undefined;
+    private touched;
+    /**
+     * Records that a dataset was used, for the retention sweep ("unused for `jobsDays`"): the
+     * modification time of its SQLite file (nothing else changes it after the ingestion), at most
+     * once every ten minutes per dataset. Best effort: a failure never affects the caller.
+     */
+    private touch;
+    /** Every dataset (all sessions), for the retention sweep; does not count as a use. */
+    all(): DatasetRecord[];
+    /** A dataset of this root session; another session's id is `not_found`. */
+    getFor(id: string, rootSessionId: string): DatasetRecord;
+    list(rootSessionId: string): DatasetRecord[];
+    private bySha;
+    /**
+     * Ingests a file into a dataset, or returns the session's existing dataset with the same
+     * content (`reused`). Concurrent calls for the same content share one ingestion. A failure
+     * leaves no row and no file.
+     */
+    ingest(input: IngestInput): Promise<{
+        record: DatasetRecord;
+        reused: boolean;
+    }>;
+    private build;
+    /** The XLSX route (spec §17.6, option B): the Python standard-library helper, then statistics. */
+    private ingestXlsx;
+    /** The `alisio_runtime` package on disk for `python -m` (content-addressed, written once). */
+    private helperDir;
+    /** Schema and statistics of every sheet (`GET /api/datasets/:did`). */
+    detail(record: DatasetRecord): DatasetDetailWire;
+    /** First rows of a table, for `data_inspect` and the prompt summary. */
+    sample(record: DatasetRecord, table: string, rows: number, signal?: AbortSignal): Promise<SampleRows>;
+    /** The description for the model: `data_inspect` text. */
+    describe(record: DatasetRecord, sampleRows: number, maxChars: number, signal?: AbortSignal): Promise<{
+        detail: DatasetDetailWire;
+        samples: Map<string, SampleRows>;
+        text: string;
+    }>;
+    /** The ≤ 4 KB summary appended to a prompt that attaches the dataset. */
+    summary(record: DatasetRecord, signal?: AbortSignal): Promise<string>;
+    /**
+     * Runs a model-written read-only query (`data_query`): lexical guard, then a read-only
+     * connection in the engine process, with a row limit and a time limit.
+     */
+    query(record: DatasetRecord, sql: string, options?: {
+        maxRows?: number;
+        signal?: AbortSignal;
+    }): Promise<import("./engine-types.ts").QueryResult>;
+    /** One page of a sheet for `SpreadsheetView` (spec §17.4). */
+    page(record: DatasetRecord, request: {
+        sheet?: string;
+        after?: string;
+        offset?: number;
+        limit?: number;
+        sort?: string;
+        dir?: "asc" | "desc";
+        filter?: string;
+        column?: string;
+    }, signal?: AbortSignal): Promise<DatasetRowsPage>;
+    /** Deletes a dataset file and row (connections are closed first, for Windows). */
+    remove(record: DatasetRecord): Promise<void>;
+    close(): void;
+}