@alisio/core 0.1.0-alpha.21 → 0.1.0-alpha.22

package/dist/analysis/data/engine-client.js

@@ -0,0 +1,304 @@
+/**
+ * Host side of the data engine (spec §17): runs ingestion and read-only queries in a child
+ * process with the current JavaScript runtime and talks to it in newline-delimited JSON. A child
+ * process, not a `worker_thread`, because `worker.terminate()` cannot interrupt a native SQLite
+ * call (verified on Node 22.19 and Bun 1.4.2: a recursive CTE keeps the thread alive), while a
+ * killed process always stops. Portable: `process.execPath` is `node`, `bun`, or the compiled
+ * binary, which acts as `bun` with `BUN_BE_BUN=1`. The engine script is written once to the state
+ * folder (content-addressed) because the bundled binary has no loose files to point a child to.
+ */
+import { spawn } from "node:child_process";
+import { createHash } from "node:crypto";
+import { mkdir, rename, rm, stat, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { ENGINE_SOURCE } from "./engine-source.js";
+import { LineSplitter } from "./json.js";
+/** A failure with a stable code that tools and routes map to their own error shapes. */
+export class DataError extends Error {
+    code;
+    constructor(code, message) {
+        super(message);
+        this.code = code;
+    }
+}
+/** The engine script on disk (`<root>/analysis/engine/data-engine-<hash>.mjs`). */
+export async function ensureEngineScript(root) {
+    const hash = createHash("sha256").update(ENGINE_SOURCE).digest("hex").slice(0, 12);
+    const dir = join(root, "analysis", "engine");
+    const file = join(dir, `data-engine-${hash}.mjs`);
+    try {
+        if ((await stat(file)).isFile())
+            return file;
+    }
+    catch {
+        /* write it below */
+    }
+    await mkdir(dir, { recursive: true, mode: 0o700 });
+    const temp = `${file}.${process.pid}.${Date.now()}.tmp`;
+    await writeFile(temp, ENGINE_SOURCE, { mode: 0o600 });
+    await rename(temp, file).catch(async (error) => {
+        await rm(temp, { force: true });
+        // Another process may have written the same file first (Windows refuses to replace it).
+        if (!(await stat(file)
+            .then((s) => s.isFile())
+            .catch(() => false)))
+            throw error;
+    });
+    return file;
+}
+const ENV_KEYS = [
+    "PATH",
+    "HOME",
+    "USERPROFILE",
+    "TMPDIR",
+    "TEMP",
+    "TMP",
+    "SystemRoot",
+    "SYSTEMROOT",
+    "WINDIR",
+    "LANG",
+];
+/** One engine child process and its pending request. */
+class EngineProcess {
+    child;
+    nextId = 1;
+    pending = new Map();
+    stderr = "";
+    dead = false;
+    exitError;
+    constructor(script, memoryMb) {
+        const bun = !!process.versions.bun;
+        this.child = spawn(process.execPath, [
+            ...(bun
+                ? []
+                : [`--max-old-space-size=${memoryMb}`, "--disable-warning=ExperimentalWarning"]),
+            script,
+        ], {
+            env: {
+                ...Object.fromEntries(ENV_KEYS.flatMap((k) => (process.env[k] ? [[k, process.env[k]]] : []))),
+                BUN_BE_BUN: "1",
+            },
+            stdio: ["pipe", "pipe", "pipe"],
+            windowsHide: true,
+        });
+        const lines = new LineSplitter((line) => this.onLine(line));
+        this.child.stdout?.setEncoding("utf8");
+        this.child.stdout?.on("data", (text) => lines.write(text));
+        this.child.stderr?.setEncoding("utf8");
+        this.child.stderr?.on("data", (text) => {
+            this.stderr = `${this.stderr}${text}`.slice(-2000);
+        });
+        this.child.stdin?.on("error", () => {
+            /* the exit handler reports a dead process */
+        });
+        this.child.once("error", (error) => this.fail(error));
+        this.child.once("close", (code, signal) => this.fail(new DataError("engine_error", `The data engine stopped unexpectedly (${signal ?? `exit code ${code}`})${this.stderr.trim() ? `: ${this.stderr.trim().split("\n").at(-1)}` : ""}`)));
+    }
+    fail(error) {
+        this.dead = true;
+        this.exitError ??= error;
+        for (const waiting of this.pending.values())
+            waiting.reject(this.exitError);
+        this.pending.clear();
+    }
+    onLine(line) {
+        let message;
+        try {
+            message = JSON.parse(line);
+        }
+        catch {
+            return;
+        }
+        const waiting = this.pending.get(message.id);
+        if (!waiting)
+            return;
+        if (message.type === "progress") {
+            waiting.onProgress?.(message.rows);
+            return;
+        }
+        this.pending.delete(message.id);
+        if (message.type === "result")
+            waiting.resolve(message.result);
+        else
+            waiting.reject(new DataError(asCode(message.code), message.message));
+    }
+    request(request, onProgress) {
+        if (this.dead)
+            return Promise.reject(this.exitError ?? new DataError("engine_error", "The data engine is not running"));
+        const id = this.nextId++;
+        return new Promise((resolve, reject) => {
+            this.pending.set(id, {
+                resolve: resolve,
+                reject,
+                ...(onProgress ? { onProgress } : {}),
+            });
+            this.child.stdin?.write(`${JSON.stringify({ ...request, id })}\n`, (error) => {
+                if (error)
+                    this.fail(new DataError("engine_error", "The data engine is not running"));
+            });
+        });
+    }
+    /** Stops the process for good (a stuck statement, a cancelled ingestion, shutdown). */
+    kill() {
+        this.dead = true;
+        this.exitError ??= new DataError("cancelled", "The data engine was stopped");
+        try {
+            this.child.kill("SIGKILL");
+        }
+        catch {
+            /* already gone */
+        }
+    }
+    /** Lets the process finish what it is doing and exit by itself. */
+    end() {
+        this.dead = true;
+        this.child.stdin?.end();
+    }
+}
+function asCode(code) {
+    return ["query_rejected", "ingest_limit", "ingest_invalid", "sql_error"].includes(code)
+        ? code
+        : "engine_error";
+}
+/** Ingestion and query access to dataset files through the engine process. */
+export class DataEngine {
+    options;
+    script;
+    query;
+    queryQueue = Promise.resolve();
+    idle;
+    running = new Set();
+    waiting = [];
+    active = 0;
+    closed = false;
+    constructor(options) {
+        this.options = options;
+    }
+    scriptPath() {
+        this.script ??= ensureEngineScript(this.options.root);
+        return this.script;
+    }
+    memory() {
+        return this.options.memoryMb ?? 1024;
+    }
+    async slot(signal) {
+        const limit = this.options.concurrency ?? 2;
+        while (this.active >= limit) {
+            await new Promise((resolve, reject) => {
+                this.waiting.push(resolve);
+                signal?.addEventListener("abort", () => reject(new DataError("cancelled", "Cancelled")), {
+                    once: true,
+                });
+            });
+            signal?.throwIfAborted();
+        }
+        this.active++;
+        return () => {
+            this.active--;
+            this.waiting.shift()?.();
+        };
+    }
+    /** Runs one request in a fresh process (ingestion and finalization). */
+    async oneShot(request, options) {
+        if (this.closed)
+            throw new DataError("cancelled", "The data engine is closed");
+        options.signal?.throwIfAborted();
+        const release = await this.slot(options.signal);
+        const child = new EngineProcess(await this.scriptPath(), this.memory());
+        this.running.add(child);
+        const stop = () => child.kill();
+        options.signal?.addEventListener("abort", stop, { once: true });
+        const timer = options.timeoutMs ? setTimeout(stop, options.timeoutMs) : undefined;
+        try {
+            return await child.request(request, options.onProgress);
+        }
+        catch (error) {
+            if (options.signal?.aborted)
+                throw new DataError("cancelled", "Cancelled");
+            throw error;
+        }
+        finally {
+            clearTimeout(timer);
+            options.signal?.removeEventListener("abort", stop);
+            child.end();
+            // A process that does not exit by itself shortly after must not linger.
+            setTimeout(() => child.kill(), 2000).unref();
+            this.running.delete(child);
+            release();
+        }
+    }
+    ingest(request, options = {}) {
+        return this.oneShot({ op: "ingest", ...request }, options);
+    }
+    finalize(request, options = {}) {
+        return this.oneShot({ op: "finalize", ...request }, options);
+    }
+    /**
+     * Runs one read-only statement. Requests are served one at a time by a persistent process;
+     * exceeding `timeoutMs` kills it (the next request starts a new one) and fails with
+     * `query_timeout`.
+     */
+    runQuery(request, options) {
+        const run = async () => {
+            if (this.closed)
+                throw new DataError("cancelled", "The data engine is closed");
+            options.signal?.throwIfAborted();
+            clearTimeout(this.idle);
+            if (!this.query || this.query.dead) {
+                this.query = new EngineProcess(await this.scriptPath(), this.memory());
+            }
+            const child = this.query;
+            let timedOut = false;
+            const timer = setTimeout(() => {
+                timedOut = true;
+                child.kill();
+            }, options.timeoutMs);
+            const abort = () => child.kill();
+            options.signal?.addEventListener("abort", abort, { once: true });
+            try {
+                return await child.request({ op: "query", ...request });
+            }
+            catch (error) {
+                if (timedOut)
+                    throw new DataError("query_timeout", `The query took longer than ${options.timeoutMs} ms and was stopped; simplify it or use python_run`);
+                if (options.signal?.aborted)
+                    throw new DataError("cancelled", "Cancelled");
+                throw error;
+            }
+            finally {
+                clearTimeout(timer);
+                options.signal?.removeEventListener("abort", abort);
+                this.armIdle();
+            }
+        };
+        const next = this.queryQueue.then(run, run);
+        this.queryQueue = next.catch(() => undefined);
+        return next;
+    }
+    armIdle() {
+        clearTimeout(this.idle);
+        this.idle = setTimeout(() => {
+            this.query?.end();
+            this.query = undefined;
+        }, this.options.idleMs ?? 30_000);
+        this.idle.unref();
+    }
+    /** Closes the engine's connection to a file (before the host deletes or replaces it). */
+    async closeFile(db) {
+        const child = this.query;
+        if (!child || child.dead)
+            return;
+        const request = { op: "close", ...(db ? { db } : {}) };
+        await this.queryQueue.then(() => child.request(request)).catch(() => undefined);
+    }
+    /** Kills every process; later requests fail. */
+    close() {
+        this.closed = true;
+        clearTimeout(this.idle);
+        this.query?.kill();
+        this.query = undefined;
+        for (const child of this.running)
+            child.kill();
+        this.running.clear();
+    }
+}

package/dist/analysis/data/engine-entry.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/analysis/data/engine-entry.js

@@ -0,0 +1,59 @@
+import { EngineError, runFinalize, runIngest } from "./ingest-worker.js";
+import { LineSplitter } from "./json.js";
+import { closeConnections, QueryRejected, runQuery } from "./query-worker.js";
+const send = (message) => process.stdout.write(`${JSON.stringify(message)}\n`);
+async function handle(req) {
+    try {
+        switch (req.op) {
+            case "ingest": {
+                const result = await runIngest(req, (rows) => send({ id: req.id, type: "progress", rows }));
+                send({ id: req.id, type: "result", result });
+                return;
+            }
+            case "finalize":
+                send({
+                    id: req.id,
+                    type: "result",
+                    result: runFinalize(req.target, req.sourceName, req.sourceSha256),
+                });
+                return;
+            case "query":
+                send({ id: req.id, type: "result", result: runQuery(req) });
+                return;
+            case "close":
+                closeConnections(req.db);
+                send({ id: req.id, type: "result", result: {} });
+                return;
+        }
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        const code = error instanceof EngineError
+            ? error.code
+            : error instanceof QueryRejected
+                ? "query_rejected"
+                : req.op === "query"
+                    ? "sql_error"
+                    : "engine_error";
+        send({ id: req.id, type: "error", code, message });
+    }
+}
+let chain = Promise.resolve();
+const lines = new LineSplitter((line) => {
+    if (!line.trim())
+        return;
+    let request;
+    try {
+        request = JSON.parse(line);
+    }
+    catch {
+        return;
+    }
+    chain = chain.then(() => handle(request));
+});
+process.stdin.setEncoding("utf8");
+process.stdin.on("data", (text) => lines.write(text));
+process.stdin.on("end", () => {
+    lines.end();
+    chain.then(() => closeConnections());
+});

package/dist/analysis/data/engine-source.d.ts

@@ -0,0 +1,4 @@
+/** Hash of the engine inputs this bundle was built from. */
+export declare const ENGINE_INPUT_SHA256 = "4c473f13a258d72c20d901fdec010302c8f54b21d3edd07be6ae4e1ab87e7cf5";
+/** The data engine as one ES module script (run with node, bun or the compiled binary). */
+export declare const ENGINE_SOURCE: string;

package/dist/analysis/data/engine-source.js

@@ -0,0 +1,6 @@
+// Generated by scripts/analysis-data-engine.ts from the modules listed in
+// scripts/analysis-data-engine-inputs.ts. Do not edit.
+/** Hash of the engine inputs this bundle was built from. */
+export const ENGINE_INPUT_SHA256 = "4c473f13a258d72c20d901fdec010302c8f54b21d3edd07be6ae4e1ab87e7cf5";
+/** The data engine as one ES module script (run with node, bun or the compiled binary). */
+export const ENGINE_SOURCE = '// packages/core/src/analysis/data/ingest-worker.ts\nimport {\n  closeSync,\n  createReadStream,\n  openSync,\n  readFileSync,\n  readSync,\n  rmSync,\n  statSync\n} from "node:fs";\n\n// packages/core/src/analysis/data/csv.ts\nvar QUOTE = 34;\nvar CR = 13;\nvar LF = 10;\nclass CsvError extends Error {\n  code;\n  constructor(message, code) {\n    super(message);\n    this.code = code;\n  }\n}\n\nclass CsvParser {\n  onRow;\n  maxCellChars;\n  state = 0 /* Start */;\n  field = "";\n  fields = [];\n  skipLineFeed = false;\n  quotedField = false;\n  delimiter;\n  rows = 0;\n  constructor(delimiter, onRow, maxCellChars = 1024 * 1024) {\n    this.onRow = onRow;\n    this.maxCellChars = maxCellChars;\n    if (delimiter.length !== 1)\n      throw new Error("The CSV delimiter must be one character");\n    this.delimiter = delimiter.charCodeAt(0);\n  }\n  add(text) {\n    this.field += text;\n    if (this.field.length > this.maxCellChars)\n      throw new CsvError(`A cell in row ${this.rows + 1} exceeds ${this.maxCellChars} characters`, "cell_too_large");\n  }\n  endField() {\n    this.fields.push(this.field);\n    this.field = "";\n    this.quotedField = false;\n  }\n  endRow() {\n    const blank = this.fields.length === 0 && this.field === "" && !this.quotedField;\n    if (!blank) {\n      this.endField();\n      this.rows++;\n      const row = this.fields;\n      this.fields = [];\n      this.state = 0 /* Start */;\n      this.onRow(row);\n      return;\n    }\n    this.state = 0 /* Start */;\n    this.quotedField = false;\n  }\n  write(chunk) {\n    const n = chunk.length;\n    let i = 0;\n    if (this.skipLineFeed && n > 0) {\n      if (chunk.charCodeAt(0) === LF)\n        i = 1;\n      this.skipLineFeed = false;\n    }\n    while (i < n) {\n      switch (this.state) {\n        case 0 /* Start */: {\n          if (chunk.charCodeAt(i) === QUOTE) {\n            this.state = 2 /* Quoted */;\n            this.quotedField = true;\n            i++;\n          } else\n            this.state = 1 /* Unquoted */;\n          break;\n        }\n        case 1 /* Unquoted */: {\n          let j = i;\n          let c = 0;\n          while (j < n) {\n            c = chunk.charCodeAt(j);\n            if (c === this.delimiter || c === LF || c === CR)\n              break;\n            j++;\n          }\n          if (j > i)\n            this.add(chunk.slice(i, j));\n          i = j;\n          if (j >= n)\n            break;\n          if (c === this.delimiter) {\n            this.endField();\n            this.state = 0 /* Start */;\n            i++;\n          } else {\n            i++;\n            if (c === CR) {\n              if (i < n && chunk.charCodeAt(i) === LF)\n                i++;\n              else if (i >= n)\n                this.skipLineFeed = true;\n            }\n            this.endRow();\n          }\n          break;\n        }\n        case 2 /* Quoted */: {\n          const quote = chunk.indexOf(\'"\', i);\n          if (quote < 0) {\n            this.add(chunk.slice(i));\n            i = n;\n          } else {\n            if (quote > i)\n              this.add(chunk.slice(i, quote));\n            i = quote + 1;\n            this.state = 3 /* QuoteInQuoted */;\n          }\n          break;\n        }\n        case 3 /* QuoteInQuoted */: {\n          const c = chunk.charCodeAt(i);\n          if (c === QUOTE) {\n            this.add(\'"\');\n            this.state = 2 /* Quoted */;\n            i++;\n          } else if (c === this.delimiter) {\n            this.endField();\n            this.state = 0 /* Start */;\n            i++;\n          } else if (c === LF || c === CR) {\n            i++;\n            if (c === CR) {\n              if (i < n && chunk.charCodeAt(i) === LF)\n                i++;\n              else if (i >= n)\n                this.skipLineFeed = true;\n            }\n            this.state = 1 /* Unquoted */;\n            this.endRow();\n          } else {\n            this.state = 1 /* Unquoted */;\n          }\n          break;\n        }\n      }\n    }\n  }\n  end() {\n    if (this.fields.length === 0 && this.field === "" && !this.quotedField)\n      return;\n    this.endRow();\n  }\n}\nvar CANDIDATES = [",", ";", "\\t", "|"];\nfunction detectDelimiter(sample) {\n  let best = ",";\n  let bestScore = 0;\n  for (const candidate of CANDIDATES) {\n    const counts = new Map;\n    let rows = 0;\n    const parser = new CsvParser(candidate, (row) => {\n      rows++;\n      counts.set(row.length, (counts.get(row.length) ?? 0) + 1);\n    });\n    try {\n      parser.write(sample);\n    } catch {\n      continue;\n    }\n    if (rows < 1)\n      continue;\n    let width = 0;\n    let frequency = 0;\n    for (const [w, count] of counts)\n      if (w > 1 && (count > frequency || count === frequency && w > width)) {\n        width = w;\n        frequency = count;\n      }\n    const score = frequency * 1000 + width;\n    if (score > bestScore) {\n      best = candidate;\n      bestScore = score;\n    }\n  }\n  return best;\n}\n\n// packages/core/src/analysis/data/infer.ts\nvar NUMBER = /^-?(?:0|[1-9]\\d*)(?:\\.\\d+)?(?:[eE][+-]?\\d+)?$/;\nvar INTEGER = /^-?(?:0|[1-9]\\d*)$/;\nvar ISO_DATE = /^\\d{4}-\\d{2}-\\d{2}(?:[T ]\\d{2}:\\d{2}(?::\\d{2}(?:\\.\\d+)?)?(?:Z|[+-]\\d{2}:?\\d{2})?)?$/;\nvar BOOLEAN = /^(?:true|false)$/i;\nfunction convertCell(raw) {\n  if (raw === "")\n    return null;\n  if (!NUMBER.test(raw))\n    return raw;\n  if (INTEGER.test(raw)) {\n    if (raw === "-0")\n      return raw;\n    const n = Number(raw);\n    return Number.isSafeInteger(n) ? BigInt(raw) : raw;\n  }\n  const n = Number(raw);\n  return Number.isFinite(n) ? n : raw;\n}\nfunction convertJson(value) {\n  if (value === null || value === undefined)\n    return null;\n  if (typeof value === "number") {\n    if (!Number.isFinite(value))\n      return String(value);\n    return Number.isSafeInteger(value) && !Object.is(value, -0) ? BigInt(value) : value;\n  }\n  if (typeof value === "string")\n    return value;\n  if (typeof value === "boolean")\n    return value ? "true" : "false";\n  return JSON.stringify(value);\n}\nfunction looksLikeData(row) {\n  const cells = row.filter((cell) => cell !== "");\n  return cells.length > 0 && cells.every((cell) => NUMBER.test(cell));\n}\nfunction sanitizeIdentifier(label, fallback) {\n  let id = label.normalize("NFKD").replace(/[̀-ͯ]/g, "").toLowerCase().replace(/[^a-z0-9]+/g, "_").replace(/^_+|_+$/g, "").slice(0, 48);\n  if (!id)\n    return fallback;\n  if (/^[0-9]/.test(id))\n    id = `_${id}`;\n  return id;\n}\nfunction uniqueName(name, taken) {\n  let candidate = name;\n  for (let n = 2;taken.has(candidate); n++)\n    candidate = `${name}_${n}`;\n  taken.add(candidate);\n  return candidate;\n}\nfunction inferType(sample) {\n  let integers = 0, reals = 0, texts = 0, dates = 0, booleans = 0;\n  for (const { kind, value } of sample) {\n    if (kind === "null")\n      continue;\n    if (kind === "integer")\n      integers++;\n    else if (kind === "real")\n      reals++;\n    else if (kind === "text") {\n      texts++;\n      const text = String(value);\n      if (ISO_DATE.test(text))\n        dates++;\n      else if (BOOLEAN.test(text))\n        booleans++;\n    } else\n      return "text";\n  }\n  const total = integers + reals + texts;\n  if (!total)\n    return "text";\n  if (texts === 0)\n    return reals === 0 ? "integer" : "real";\n  if (integers + reals > 0)\n    return "text";\n  if (dates === texts)\n    return "date";\n  if (booleans === texts)\n    return "boolean";\n  return "text";\n}\nvar RESERVED_NAMES = ["rowid", "oid", "_rowid_"];\nvar newTakenNames = () => new Set(RESERVED_NAMES);\nvar quoteIdent = (name) => `"${name.replaceAll(\'"\', \'""\')}"`;\n\n// packages/core/src/analysis/data/json.ts\nclass LineSplitter {\n  onLine;\n  rest = "";\n  constructor(onLine) {\n    this.onLine = onLine;\n  }\n  write(chunk) {\n    const text = this.rest + chunk;\n    let start = 0;\n    for (;; ) {\n      const end = text.indexOf(`\n`, start);\n      if (end < 0)\n        break;\n      const stop = end > start && text.charCodeAt(end - 1) === 13 ? end - 1 : end;\n      this.onLine(text.slice(start, stop));\n      start = end + 1;\n    }\n    this.rest = start === 0 ? text : text.slice(start);\n  }\n  end() {\n    if (this.rest)\n      this.onLine(this.rest.endsWith("\\r") ? this.rest.slice(0, -1) : this.rest);\n    this.rest = "";\n  }\n}\nfunction toRecord(value) {\n  return value !== null && typeof value === "object" && !Array.isArray(value) ? value : { value };\n}\nfunction jsonDocumentRecords(text) {\n  const parsed = JSON.parse(text);\n  return Array.isArray(parsed) ? parsed.map(toRecord) : [toRecord(parsed)];\n}\n\n// packages/core/src/analysis/data/stats.ts\nvar INGEST_VERSION = 1;\nvar EXACT_DISTINCT_ROWS = 1e6;\nvar SAMPLE_ROWS = 200000;\nvar TYPE_SAMPLE = 1000;\nvar META_SCHEMA = `\nCREATE TABLE IF NOT EXISTS _alisio_meta(key TEXT PRIMARY KEY, value TEXT);\nCREATE TABLE IF NOT EXISTS _alisio_sheets(ordinal INTEGER PRIMARY KEY, name TEXT NOT NULL,\n  table_name TEXT NOT NULL, rows INTEGER NOT NULL, columns INTEGER NOT NULL);\nCREATE TABLE IF NOT EXISTS _alisio_columns(table_name TEXT NOT NULL, ordinal INTEGER NOT NULL,\n  name TEXT NOT NULL, label TEXT NOT NULL, inferred_type TEXT NOT NULL,\n  nulls INTEGER, distinct_count INTEGER, distinct_exact INTEGER,\n  min TEXT, max TEXT, mean REAL, text_fallbacks INTEGER, top_values TEXT,\n  PRIMARY KEY(table_name, ordinal));\n`;\nfunction finalizeDatabase(db) {\n  const sheets = db.prepare("SELECT ordinal, table_name FROM _alisio_sheets ORDER BY ordinal").all();\n  db.exec("BEGIN");\n  try {\n    for (const sheet of sheets) {\n      const table = quoteIdent(sheet.table_name);\n      const rows = db.prepare(`SELECT count(*) AS n FROM ${table}`).get().n;\n      const columns = db.prepare("SELECT table_name, ordinal, name FROM _alisio_columns WHERE table_name=? ORDER BY ordinal").all(sheet.table_name);\n      db.prepare("UPDATE _alisio_sheets SET rows=?, columns=? WHERE ordinal=?").run(rows, columns.length, sheet.ordinal);\n      const exact = rows <= EXACT_DISTINCT_ROWS;\n      const stride = exact ? 1 : Math.ceil(rows / SAMPLE_ROWS);\n      const andWhere = exact ? "" : `AND rowid % ${stride} = 0`;\n      for (const column of columns) {\n        const col = quoteIdent(column.name);\n        const sample = db.prepare(`SELECT typeof(${col}) AS kind, ${col} AS value FROM ${table} LIMIT ${TYPE_SAMPLE}`).all();\n        const type = inferType(sample);\n        const agg = db.prepare(`SELECT sum(${col} IS NULL) AS nulls, min(${col}) AS mn, max(${col}) AS mx,\n               sum(typeof(${col}) IN (\'integer\',\'real\')) AS nums,\n               sum(typeof(${col})=\'text\') AS texts,\n               sum(typeof(${col})=\'text\' AND ${col} GLOB \'*[0-9]*\' AND NOT ${col} GLOB \'*[A-Za-z]*\') AS numlike,\n               avg(CASE WHEN typeof(${col}) IN (\'integer\',\'real\') THEN ${col} END) AS mean\n             FROM ${table}`).get();\n        const groups = db.prepare(`WITH g AS MATERIALIZED (\n               SELECT ${col} AS v, count(*) AS n FROM ${table}\n               WHERE ${col} IS NOT NULL ${andWhere} GROUP BY ${col})\n             SELECT (SELECT count(*) FROM g) AS d, CAST(v AS TEXT) AS v, n FROM g\n             ORDER BY n DESC, v LIMIT 5`).all();\n        const nums = Number(agg.nums ?? 0);\n        const texts = Number(agg.texts ?? 0);\n        const fallbacks = nums > 0 ? texts : type === "text" ? Number(agg.numlike ?? 0) : 0;\n        const numeric = type === "integer" || type === "real";\n        db.prepare(`UPDATE _alisio_columns SET inferred_type=?, nulls=?, distinct_count=?, distinct_exact=?,\n             min=?, max=?, mean=?, text_fallbacks=?, top_values=? WHERE table_name=? AND ordinal=?`).run(type, Number(agg.nulls ?? 0), Number(groups[0]?.d ?? 0), exact ? 1 : 0, agg.mn === null || agg.mn === undefined ? null : String(agg.mn), agg.mx === null || agg.mx === undefined ? null : String(agg.mx), numeric && agg.mean !== null ? Number(agg.mean) : null, fallbacks, JSON.stringify(groups.map((g) => ({ value: g.v, count: Number(g.n) }))), sheet.table_name, column.ordinal);\n      }\n    }\n    db.exec("COMMIT");\n  } catch (error) {\n    db.exec("ROLLBACK");\n    throw error;\n  }\n}\n\n// packages/core/src/analysis/data/ingest-worker.ts\nvar sqlite = process.getBuiltinModule("node:sqlite");\nvar MAX_JSON_BYTES = 50 * 1024 * 1024;\nvar BATCH_ROWS = 500;\nvar MAX_PARAMS = 32766;\nvar COMMIT_ROWS = 5000;\n\nclass EngineError extends Error {\n  code;\n  constructor(code, message) {\n    super(message);\n    this.code = code;\n  }\n}\nfunction openWritable(path) {\n  rmSync(path, { force: true });\n  const db = new sqlite.DatabaseSync(path);\n  db.exec("PRAGMA journal_mode=OFF; PRAGMA synchronous=OFF; PRAGMA cache_size=-65536; PRAGMA temp_store=FILE;");\n  return db;\n}\nfunction head(path, bytes) {\n  const fd = openSync(path, "r");\n  try {\n    const buffer = Buffer.alloc(bytes);\n    const read = readSync(fd, buffer, 0, bytes, 0);\n    return buffer.subarray(0, read);\n  } finally {\n    closeSync(fd);\n  }\n}\nasync function* chunks(path) {\n  for await (const chunk of createReadStream(path, { highWaterMark: 1024 * 1024 }))\n    yield chunk;\n}\nasync function detectEncoding(path) {\n  const start = head(path, 3);\n  if (start[0] === 239 && start[1] === 187 && start[2] === 191)\n    return "utf-8";\n  if (start[0] === 255 && start[1] === 254)\n    return "utf-16le";\n  if (start[0] === 254 && start[1] === 255)\n    return "utf-16be";\n  const decoder = new TextDecoder("utf-8", { fatal: true });\n  try {\n    for await (const chunk of chunks(path))\n      decoder.decode(chunk, { stream: true });\n    decoder.decode();\n    return "utf-8";\n  } catch {\n    return "windows-1252";\n  }\n}\nasync function* textChunks(path, encoding) {\n  const decoder = new TextDecoder(encoding);\n  for await (const chunk of chunks(path)) {\n    const text = decoder.decode(chunk, { stream: true });\n    if (text)\n      yield text;\n  }\n  const rest = decoder.decode();\n  if (rest)\n    yield rest;\n}\n\nclass TableWriter {\n  db;\n  table;\n  columns;\n  maxRows;\n  onRows;\n  pending = [];\n  statements = new Map;\n  sinceCommit = 0;\n  rows = 0;\n  constructor(db, table, columns, maxRows, onRows) {\n    this.db = db;\n    this.table = table;\n    this.columns = columns;\n    this.maxRows = maxRows;\n    this.onRows = onRows;\n    db.exec("BEGIN");\n    db.exec(`CREATE TABLE ${quoteIdent(table)}(${columns.map(quoteIdent).join(", ")})`);\n  }\n  statement(count) {\n    let statement = this.statements.get(count);\n    if (!statement) {\n      const group = `(${this.columns.map(() => "?").join(",")})`;\n      statement = this.db.prepare(`INSERT INTO ${quoteIdent(this.table)} VALUES ${Array(count).fill(group).join(",")}`);\n      this.statements.set(count, statement);\n    }\n    return statement;\n  }\n  add(row) {\n    if (this.rows >= this.maxRows)\n      throw new EngineError("ingest_limit", `The file has more than ${this.maxRows} rows`);\n    this.pending.push(row);\n    this.rows++;\n    if (this.pending.length >= Math.min(BATCH_ROWS, Math.floor(MAX_PARAMS / this.columns.length)))\n      this.flush();\n  }\n  flush() {\n    if (!this.pending.length)\n      return;\n    const width = this.columns.length;\n    const params = [];\n    for (const row of this.pending) {\n      for (let i = 0;i < width; i++)\n        params.push(row[i] ?? null);\n    }\n    this.statement(this.pending.length).run(...params);\n    this.sinceCommit += this.pending.length;\n    this.pending = [];\n    if (this.sinceCommit >= COMMIT_ROWS) {\n      this.db.exec("COMMIT");\n      this.db.exec("BEGIN");\n      this.sinceCommit = 0;\n      this.onRows(this.rows);\n    }\n  }\n  addColumn(name) {\n    this.flush();\n    this.db.exec(`ALTER TABLE ${quoteIdent(this.table)} ADD COLUMN ${quoteIdent(name)}`);\n    this.columns = [...this.columns, name];\n    this.statements.clear();\n  }\n  finish() {\n    this.flush();\n    this.db.exec("COMMIT");\n  }\n}\nfunction register(db, sheetName, ordinal, table, names, labels) {\n  db.prepare("INSERT INTO _alisio_sheets(ordinal,name,table_name,rows,columns) VALUES(?,?,?,0,?)").run(ordinal, sheetName, table, names.length);\n  const insert = db.prepare("INSERT INTO _alisio_columns(table_name,ordinal,name,label,inferred_type) VALUES(?,?,?,?,\'text\')");\n  names.forEach((name, i) => {\n    insert.run(table, i + 1, name, labels[i] ?? name);\n  });\n}\nasync function ingestDelimited(db, req, encoding, onRows) {\n  let sample = "";\n  for await (const text of textChunks(req.source, encoding)) {\n    sample = text.slice(0, 64 * 1024);\n    break;\n  }\n  const delimiter = req.format === "tsv" ? "\\t" : detectDelimiter(sample);\n  let writer;\n  let names = [];\n  let labels = [];\n  let irregular = 0;\n  const addRow = (fields) => {\n    if (fields.length !== names.length)\n      irregular++;\n    const values = [];\n    for (let i = 0;i < names.length; i++)\n      values.push(convertCell(fields[i] ?? ""));\n    writer.add(values);\n  };\n  const parser = new CsvParser(delimiter, (fields) => {\n    if (!writer) {\n      if (fields.length > req.limits.maxColumns)\n        throw new EngineError("ingest_limit", `The file has ${fields.length} columns; the limit is ${req.limits.maxColumns}`);\n      const header = !looksLikeData(fields);\n      const taken = newTakenNames();\n      labels = fields.map((cell, i) => header && cell !== "" ? cell : `column_${i + 1}`);\n      names = labels.map((label, i) => uniqueName(sanitizeIdentifier(label, `column_${i + 1}`), taken));\n      writer = new TableWriter(db, "data", names, req.limits.maxRows, onRows);\n      if (!header)\n        addRow(fields);\n      return;\n    }\n    addRow(fields);\n  }, req.limits.maxCellChars);\n  try {\n    for await (const text of textChunks(req.source, encoding))\n      parser.write(text);\n    parser.end();\n  } catch (error) {\n    if (error instanceof CsvError)\n      throw new EngineError("ingest_limit", error.message);\n    throw error;\n  }\n  if (!writer)\n    throw new EngineError("ingest_invalid", "The file has no rows");\n  writer.finish();\n  return {\n    registered: { table: "data", names, labels, rows: writer.rows },\n    delimiter,\n    irregular\n  };\n}\nasync function ingestJson(db, req, encoding, onRows) {\n  const names = [];\n  const labels = [];\n  const taken = newTakenNames();\n  const index = new Map;\n  let writer;\n  const addRecord = (record) => {\n    const keys = Object.keys(record);\n    const fresh = keys.filter((key) => !index.has(key));\n    if (fresh.length && names.length + fresh.length > req.limits.maxColumns)\n      throw new EngineError("ingest_limit", `The file has more than ${req.limits.maxColumns} distinct keys`);\n    for (const key of fresh) {\n      const name = uniqueName(sanitizeIdentifier(key, `column_${names.length + 1}`), taken);\n      index.set(key, names.length);\n      names.push(name);\n      labels.push(key);\n      if (writer)\n        writer.addColumn(name);\n    }\n    if (!writer) {\n      if (!names.length) {\n        return;\n      }\n      writer = new TableWriter(db, "data", [...names], req.limits.maxRows, onRows);\n    }\n    const values = Array(names.length).fill(null);\n    for (const key of keys)\n      values[index.get(key)] = convertJson(record[key]);\n    writer.add(values);\n  };\n  const lineHandler = () => {\n    let n = 0;\n    return (line) => {\n      n++;\n      if (!line.trim())\n        return;\n      let value;\n      try {\n        value = JSON.parse(line);\n      } catch {\n        throw new EngineError("ingest_invalid", `Line ${n} is not valid JSON`);\n      }\n      addRecord(toRecord(value));\n    };\n  };\n  if (req.format === "jsonl") {\n    const splitter = new LineSplitter(lineHandler());\n    for await (const text of textChunks(req.source, encoding))\n      splitter.write(text);\n    splitter.end();\n  } else {\n    if (statSync(req.source).size > MAX_JSON_BYTES)\n      throw new EngineError("ingest_limit", `A JSON document is read in memory up to ${MAX_JSON_BYTES / (1024 * 1024)} MiB; use JSON Lines for larger files`);\n    const text = new TextDecoder(encoding).decode(readFileSync(req.source));\n    let records;\n    try {\n      records = jsonDocumentRecords(text);\n    } catch {\n      const splitter = new LineSplitter(lineHandler());\n      splitter.write(text);\n      splitter.end();\n      records = [];\n    }\n    for (const record of records)\n      addRecord(record);\n  }\n  if (!writer)\n    throw new EngineError("ingest_invalid", "The file has no records");\n  writer.finish();\n  return {\n    registered: { table: "data", names, labels, rows: writer.rows },\n    irregular: 0\n  };\n}\nasync function runIngest(req, progress) {\n  const encoding = await detectEncoding(req.source);\n  const db = openWritable(req.target);\n  let lastProgress = 0;\n  const onRows = (rows) => {\n    const now = Date.now();\n    if (now - lastProgress >= 250) {\n      lastProgress = now;\n      progress(rows);\n    }\n  };\n  try {\n    db.exec(META_SCHEMA);\n    let delimiter;\n    let irregular = 0;\n    let registered;\n    if (req.format === "csv" || req.format === "tsv") {\n      const done = await ingestDelimited(db, req, encoding, onRows);\n      registered = done.registered;\n      delimiter = done.delimiter;\n      irregular = done.irregular;\n    } else {\n      const done = await ingestJson(db, req, encoding, onRows);\n      registered = done.registered;\n    }\n    register(db, "data", 1, registered.table, registered.names, registered.labels);\n    const meta = db.prepare("INSERT OR REPLACE INTO _alisio_meta(key,value) VALUES(?,?)");\n    for (const [key, value] of Object.entries({\n      ingest_version: String(INGEST_VERSION),\n      format: req.format,\n      source_name: req.sourceName,\n      source_sha256: req.sourceSha256,\n      encoding,\n      ...delimiter ? { delimiter } : {},\n      irregular_rows: String(irregular),\n      created_at: new Date().toISOString()\n    }))\n      meta.run(key, value);\n    finalizeDatabase(db);\n    const sheets = db.prepare(\'SELECT name, table_name AS "table", rows, columns FROM _alisio_sheets ORDER BY ordinal\').all();\n    return {\n      sheets: sheets.map((s) => ({ ...s })),\n      encoding,\n      ...delimiter ? { delimiter } : {},\n      irregularRows: irregular\n    };\n  } finally {\n    db.close();\n  }\n}\nfunction runFinalize(target, sourceName, sourceSha256) {\n  const db = new sqlite.DatabaseSync(target);\n  try {\n    db.exec("PRAGMA journal_mode=OFF; PRAGMA synchronous=OFF;");\n    db.exec(META_SCHEMA);\n    const meta = db.prepare("INSERT OR REPLACE INTO _alisio_meta(key,value) VALUES(?,?)");\n    for (const [key, value] of Object.entries({\n      ingest_version: String(INGEST_VERSION),\n      format: "xlsx",\n      source_name: sourceName,\n      source_sha256: sourceSha256,\n      encoding: "utf-8",\n      created_at: new Date().toISOString()\n    }))\n      meta.run(key, value);\n    finalizeDatabase(db);\n    const sheets = db.prepare(\'SELECT name, table_name AS "table", rows, columns FROM _alisio_sheets ORDER BY ordinal\').all();\n    return { sheets: sheets.map((s) => ({ ...s })), encoding: "utf-8", irregularRows: 0 };\n  } finally {\n    db.close();\n  }\n}\n\n// packages/core/src/analysis/data/sql-guard.ts\nvar FORBIDDEN = new Set([\n  "ATTACH",\n  "DETACH",\n  "PRAGMA",\n  "INSERT",\n  "UPDATE",\n  "DELETE",\n  "REPLACE",\n  "CREATE",\n  "DROP",\n  "ALTER",\n  "VACUUM",\n  "REINDEX",\n  "ANALYZE",\n  "BEGIN",\n  "COMMIT",\n  "ROLLBACK",\n  "SAVEPOINT",\n  "RELEASE",\n  "LOAD_EXTENSION"\n]);\nvar isWordStart = (c) => /[A-Za-z_]/.test(c);\nvar isWordChar = (c) => /[A-Za-z0-9_$]/.test(c);\nfunction tokenize(sql) {\n  const tokens = [];\n  const n = sql.length;\n  let i = 0;\n  while (i < n) {\n    const c = sql[i];\n    if (c === "\'" || c === \'"\' || c === "`") {\n      let j = i + 1;\n      for (;; ) {\n        if (j >= n)\n          return { error: `Unterminated ${c === "\'" ? "string" : "identifier"}` };\n        if (sql[j] === c) {\n          if (sql[j + 1] === c) {\n            j += 2;\n            continue;\n          }\n          break;\n        }\n        j++;\n      }\n      tokens.push({ type: "punct", text: c });\n      i = j + 1;\n    } else if (c === "[") {\n      const end = sql.indexOf("]", i + 1);\n      if (end < 0)\n        return { error: "Unterminated identifier" };\n      tokens.push({ type: "punct", text: "[" });\n      i = end + 1;\n    } else if (c === "-" && sql[i + 1] === "-") {\n      const end = sql.indexOf(`\n`, i + 2);\n      i = end < 0 ? n : end + 1;\n    } else if (c === "/" && sql[i + 1] === "*") {\n      const end = sql.indexOf("*/", i + 2);\n      if (end < 0)\n        return { error: "Unterminated comment" };\n      i = end + 2;\n    } else if (/\\s/.test(c)) {\n      i++;\n    } else if (isWordStart(c)) {\n      let j = i + 1;\n      while (j < n && isWordChar(sql[j]))\n        j++;\n      tokens.push({ type: "word", text: sql.slice(i, j) });\n      i = j;\n    } else {\n      tokens.push({ type: "punct", text: c });\n      i++;\n    }\n  }\n  return { tokens };\n}\nfunction guardSql(sql) {\n  const scanned = tokenize(sql);\n  if ("error" in scanned)\n    return { ok: false, reason: scanned.error };\n  const { tokens } = scanned;\n  if (tokens.at(-1)?.text === ";")\n    tokens.pop();\n  if (!tokens.length)\n    return { ok: false, reason: "The query is empty" };\n  if (tokens.some((t) => t.type === "punct" && t.text === ";"))\n    return { ok: false, reason: "Only one statement is allowed (no `;` between statements)" };\n  const first = tokens[0];\n  if (first.type !== "word" || !["SELECT", "WITH"].includes(first.text.toUpperCase()))\n    return { ok: false, reason: "Only SELECT and WITH queries are allowed" };\n  for (let i = 0;i < tokens.length; i++) {\n    const token = tokens[i];\n    if (token.type !== "word")\n      continue;\n    const word = token.text.toUpperCase();\n    if (!FORBIDDEN.has(word))\n      continue;\n    if (word === "REPLACE" && tokens[i + 1]?.text === "(")\n      continue;\n    return {\n      ok: false,\n      reason: `\\`${token.text}\\` is not allowed in a read-only query (quote column names that look like keywords, e.g. "${token.text}")`\n    };\n  }\n  return { ok: true, sql };\n}\n\n// packages/core/src/analysis/data/query-worker.ts\nvar sqlite2 = process.getBuiltinModule("node:sqlite");\nvar connections = new Map;\nfunction connect(path, heapBytes) {\n  let db = connections.get(path);\n  if (!db) {\n    db = new sqlite2.DatabaseSync(path, { readOnly: true });\n    db.exec("PRAGMA query_only=ON; PRAGMA trusted_schema=OFF;");\n    if (heapBytes)\n      db.exec(`PRAGMA hard_heap_limit=${Math.floor(heapBytes)};`);\n    connections.set(path, db);\n  }\n  return db;\n}\nfunction closeConnections(path) {\n  for (const [key, db] of connections) {\n    if (path && key !== path)\n      continue;\n    try {\n      db.close();\n    } catch {}\n    connections.delete(key);\n  }\n}\n\nclass QueryRejected extends Error {\n}\nfunction runQuery(req) {\n  let sql = req.sql;\n  if (req.guard) {\n    const guarded = guardSql(sql);\n    if (!guarded.ok)\n      throw new QueryRejected(guarded.reason);\n    sql = guarded.sql;\n  }\n  const db = connect(req.db, req.heapBytes);\n  const statement = db.prepare(sql);\n  statement.setReadBigInts(true);\n  statement.setReturnArrays(true);\n  const columns = statement.columns().map((c) => c.name);\n  const rows = [];\n  let truncated = false;\n  let clipped = false;\n  const cell = (value) => {\n    if (value === null || value === undefined)\n      return null;\n    if (typeof value === "bigint") {\n      const n = Number(value);\n      return Number.isSafeInteger(n) ? n : value.toString();\n    }\n    if (typeof value === "number")\n      return value;\n    if (value instanceof Uint8Array)\n      return `<blob ${value.byteLength} bytes>`;\n    const text = String(value);\n    if (text.length > req.cellChars) {\n      clipped = true;\n      return `${text.slice(0, req.cellChars)}…`;\n    }\n    return text;\n  };\n  const bound = (req.params ?? []).map((value) => typeof value === "number" && Number.isSafeInteger(value) ? BigInt(value) : value);\n  for (const row of statement.iterate(...bound)) {\n    if (rows.length >= req.maxRows) {\n      truncated = true;\n      break;\n    }\n    rows.push(row.map(cell));\n  }\n  return { columns, rows, truncated, clipped };\n}\n\n// packages/core/src/analysis/data/engine-entry.ts\nvar send = (message) => process.stdout.write(`${JSON.stringify(message)}\n`);\nasync function handle(req) {\n  try {\n    switch (req.op) {\n      case "ingest": {\n        const result = await runIngest(req, (rows) => send({ id: req.id, type: "progress", rows }));\n        send({ id: req.id, type: "result", result });\n        return;\n      }\n      case "finalize":\n        send({\n          id: req.id,\n          type: "result",\n          result: runFinalize(req.target, req.sourceName, req.sourceSha256)\n        });\n        return;\n      case "query":\n        send({ id: req.id, type: "result", result: runQuery(req) });\n        return;\n      case "close":\n        closeConnections(req.db);\n        send({ id: req.id, type: "result", result: {} });\n        return;\n    }\n  } catch (error) {\n    const message = error instanceof Error ? error.message : String(error);\n    const code = error instanceof EngineError ? error.code : error instanceof QueryRejected ? "query_rejected" : req.op === "query" ? "sql_error" : "engine_error";\n    send({ id: req.id, type: "error", code, message });\n  }\n}\nvar chain = Promise.resolve();\nvar lines = new LineSplitter((line) => {\n  if (!line.trim())\n    return;\n  let request;\n  try {\n    request = JSON.parse(line);\n  } catch {\n    return;\n  }\n  chain = chain.then(() => handle(request));\n});\nprocess.stdin.setEncoding("utf8");\nprocess.stdin.on("data", (text) => lines.write(text));\nprocess.stdin.on("end", () => {\n  lines.end();\n  chain.then(() => closeConnections());\n});\n';

package/dist/analysis/data/engine-types.d.ts

@@ -0,0 +1,86 @@
+/**
+ * Messages between the host and the data engine, a separate process (`node` or `bun`) that runs
+ * `node:sqlite` so that a stuck statement can be stopped with a kill (`worker.terminate()` cannot
+ * interrupt a native SQLite call; verified on Node 22.19 and Bun 1.4.2). Newline-delimited JSON
+ * over stdin/stdout. Types only: no runtime code, safe to import from anywhere.
+ */
+export interface DataLimits {
+    maxRows: number;
+    maxColumns: number;
+    /** Characters per cell. */
+    maxCellChars: number;
+}
+export interface IngestRequest {
+    op: "ingest";
+    id: number;
+    source: string;
+    format: "csv" | "tsv" | "json" | "jsonl";
+    /** Temporary database file to create (the host renames it). */
+    target: string;
+    sourceName: string;
+    sourceSha256: string;
+    limits: DataLimits;
+}
+/** Computes the statistics of a database the Python helper wrote. */
+export interface FinalizeRequest {
+    op: "finalize";
+    id: number;
+    target: string;
+    sourceName: string;
+    sourceSha256: string;
+}
+export interface QueryRequest {
+    op: "query";
+    id: number;
+    db: string;
+    sql: string;
+    params?: Array<string | number | null>;
+    /** Rows returned; one more is read to report truncation. */
+    maxRows: number;
+    /** Cells longer than this many characters are cut (and marked). */
+    cellChars: number;
+    /** Apply the lexical guard in the engine too (model-written SQL). */
+    guard: boolean;
+    /** Soft cap of SQLite's heap, in bytes. */
+    heapBytes?: number;
+}
+export interface CloseRequest {
+    op: "close";
+    id: number;
+    /** A database file to close, or every open one when absent. */
+    db?: string;
+}
+export type EngineRequest = IngestRequest | FinalizeRequest | QueryRequest | CloseRequest;
+export interface SheetSummary {
+    name: string;
+    table: string;
+    rows: number;
+    columns: number;
+}
+export interface IngestResult {
+    sheets: SheetSummary[];
+    encoding: string;
+    delimiter?: string;
+    irregularRows: number;
+}
+export interface QueryResult {
+    columns: string[];
+    rows: Array<Array<string | number | null>>;
+    truncated: boolean;
+    /** Some cell was cut at `cellChars`. */
+    clipped: boolean;
+}
+export type EngineResponse = {
+    id: number;
+    type: "result";
+    result: IngestResult | QueryResult | Record<string, never>;
+} | {
+    id: number;
+    type: "progress";
+    rows: number;
+} | {
+    id: number;
+    type: "error";
+    code: string;
+    message: string;
+};

package/dist/analysis/data/engine-types.js

@@ -0,0 +1,7 @@
+/**
+ * Messages between the host and the data engine, a separate process (`node` or `bun`) that runs
+ * `node:sqlite` so that a stuck statement can be stopped with a kill (`worker.terminate()` cannot
+ * interrupt a native SQLite call; verified on Node 22.19 and Bun 1.4.2). Newline-delimited JSON
+ * over stdin/stdout. Types only: no runtime code, safe to import from anywhere.
+ */
+export {};

package/dist/analysis/data/infer.d.ts

@@ -0,0 +1,42 @@
+/**
+ * Cell conversion, identifier sanitising and type hints of the dataset ingestion (spec §17.2).
+ * Pure functions with no imports, shared by the ingestion engine (a separate process) and the
+ * tests. Values are stored without loss: a cell becomes a number only when its text is a plain
+ * number (no leading zeros, no separators, safe range); otherwise the exact text is kept.
+ */
+/** A value ready to bind: `bigint` (stored as INTEGER), `number` (REAL), text or `null`. */
+export type StoredValue = bigint | number | string | null;
+export interface ConvertedCell {
+    value: StoredValue;
+}
+/**
+ * Converts one CSV/XLSX text cell. Empty → `null`; plain integers in the safe range → INTEGER
+ * (`bigint`, because node:sqlite binds JS numbers as REAL); plain decimals → REAL; anything else
+ * (`007`, `1,234`, `$12`, `N/A`, `-0`, `1e999`, unsafe integers) keeps its exact text. The only
+ * documented normalisation is `1.50` → `1.5`.
+ */
+export declare function convertCell(raw: string): StoredValue;
+/** A JSON value stored without loss: numbers stay numbers, nested values become JSON text. */
+export declare function convertJson(value: unknown): StoredValue;
+/** Whether every non-empty cell of a row is a plain number (then the row is data, not a header). */
+export declare function looksLikeData(row: string[]): boolean;
+/** `region`, `año` → `ano`, `Total ($)` → `total`; never empty, never starts with a digit. */
+export declare function sanitizeIdentifier(label: string, fallback: string): string;
+/** Deduplicates names in order (`x`, `x_2`, `x_3`); `taken` carries names across calls. */
+export declare function uniqueName(name: string, taken: Set<string>): string;
+export type InferredType = "integer" | "real" | "date" | "boolean" | "text";
+/**
+ * Type hint of a column from the storage class and value of its first cells (`kind` is SQLite's
+ * `typeof`). All integers → `integer`; all numbers → `real`; all dates / booleans → those;
+ * anything mixed or empty → `text`.
+ */
+export declare function inferType(sample: Array<{
+    kind: string;
+    value: unknown;
+}>): InferredType;
+/** Names SQLite reserves for the row id: a column named so would shadow it. */
+export declare const RESERVED_NAMES: readonly ["rowid", "oid", "_rowid_"];
+/** A fresh set of taken column names, pre-seeded with the reserved ones. */
+export declare const newTakenNames: () => Set<string>;
+/** Double-quotes an SQL identifier. */
+export declare const quoteIdent: (name: string) => string;