@alisio/core 0.1.0-alpha.20 → 0.1.0-alpha.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agents/active.d.ts +19 -0
- package/dist/agents/active.js +41 -0
- package/dist/agents/capabilities.d.ts +26 -0
- package/dist/agents/capabilities.js +120 -0
- package/dist/agents/create-agent-skill.d.ts +11 -0
- package/dist/agents/create-agent-skill.js +60 -0
- package/dist/agents/definitions.d.ts +88 -0
- package/dist/agents/definitions.js +404 -0
- package/dist/agents/draft.d.ts +41 -0
- package/dist/agents/draft.js +133 -0
- package/dist/agents/service.d.ts +57 -0
- package/dist/agents/service.js +86 -0
- package/dist/agents/templates.d.ts +8 -0
- package/dist/agents/templates.js +102 -0
- package/dist/analysis/capabilities.d.ts +48 -0
- package/dist/analysis/capabilities.js +90 -0
- package/dist/analysis/data/csv.d.ts +46 -0
- package/dist/analysis/data/csv.js +240 -0
- package/dist/analysis/data/datasets.d.ts +132 -0
- package/dist/analysis/data/datasets.js +441 -0
- package/dist/analysis/data/describe.d.ts +27 -0
- package/dist/analysis/data/describe.js +103 -0
- package/dist/analysis/data/engine-client.d.ts +58 -0
- package/dist/analysis/data/engine-client.js +304 -0
- package/dist/analysis/data/engine-entry.d.ts +1 -0
- package/dist/analysis/data/engine-entry.js +59 -0
- package/dist/analysis/data/engine-source.d.ts +4 -0
- package/dist/analysis/data/engine-source.js +6 -0
- package/dist/analysis/data/engine-types.d.ts +86 -0
- package/dist/analysis/data/engine-types.js +7 -0
- package/dist/analysis/data/infer.d.ts +42 -0
- package/dist/analysis/data/infer.js +118 -0
- package/dist/analysis/data/ingest-worker.d.ts +17 -0
- package/dist/analysis/data/ingest-worker.js +365 -0
- package/dist/analysis/data/json.d.ts +19 -0
- package/dist/analysis/data/json.js +44 -0
- package/dist/analysis/data/query-worker.d.ts +12 -0
- package/dist/analysis/data/query-worker.js +75 -0
- package/dist/analysis/data/rows.d.ts +39 -0
- package/dist/analysis/data/rows.js +120 -0
- package/dist/analysis/data/sql-guard.d.ts +20 -0
- package/dist/analysis/data/sql-guard.js +129 -0
- package/dist/analysis/data/stats.d.ts +18 -0
- package/dist/analysis/data/stats.js +77 -0
- package/dist/analysis/install-hints.d.ts +33 -0
- package/dist/analysis/install-hints.js +144 -0
- package/dist/analysis/janitor.d.ts +60 -0
- package/dist/analysis/janitor.js +264 -0
- package/dist/analysis/jobs.d.ts +94 -0
- package/dist/analysis/jobs.js +165 -0
- package/dist/analysis/oci.d.ts +129 -0
- package/dist/analysis/oci.js +330 -0
- package/dist/analysis/python/sources.d.ts +2 -0
- package/dist/analysis/python/sources.js +10 -0
- package/dist/analysis/requirements.d.ts +32 -0
- package/dist/analysis/requirements.js +73 -0
- package/dist/analysis/rerun.d.ts +40 -0
- package/dist/analysis/rerun.js +136 -0
- package/dist/analysis/runtime-manager.d.ts +146 -0
- package/dist/analysis/runtime-manager.js +472 -0
- package/dist/analysis/status.d.ts +21 -0
- package/dist/analysis/status.js +52 -0
- package/dist/application.d.ts +48 -0
- package/dist/application.js +228 -0
- package/dist/artifacts/kinds.d.ts +35 -0
- package/dist/artifacts/kinds.js +177 -0
- package/dist/artifacts/publisher.d.ts +7 -0
- package/dist/artifacts/publisher.js +29 -0
- package/dist/artifacts/store.d.ts +119 -0
- package/dist/artifacts/store.js +581 -0
- package/dist/commands/builtins.js +13 -0
- package/dist/commands/catalog.js +13 -1
- package/dist/config.d.ts +102 -0
- package/dist/config.js +167 -6
- package/dist/core/contracts.d.ts +69 -1
- package/dist/core/runner.d.ts +34 -4
- package/dist/core/runner.js +388 -130
- package/dist/index.d.ts +30 -5
- package/dist/index.js +29 -4
- package/dist/mcp/rich.js +4 -0
- package/dist/plugins/host.js +14 -6
- package/dist/runtime/blobs.d.ts +22 -0
- package/dist/runtime/blobs.js +84 -1
- package/dist/runtime/ids.d.ts +8 -0
- package/dist/runtime/ids.js +32 -0
- package/dist/runtime/paths.d.ts +5 -0
- package/dist/runtime/paths.js +6 -0
- package/dist/runtime/process.d.ts +12 -0
- package/dist/runtime/process.js +64 -4
- package/dist/runtime/sqlite.d.ts +2 -0
- package/dist/runtime/sqlite.js +18 -0
- package/dist/runtime/store.d.ts +5 -0
- package/dist/runtime/store.js +75 -0
- package/dist/runtime/zip.d.ts +24 -0
- package/dist/runtime/zip.js +126 -0
- package/dist/sessions/children.js +4 -0
- package/dist/tools/analysis.d.ts +37 -0
- package/dist/tools/analysis.js +572 -0
- package/dist/tools/artifacts.d.ts +14 -0
- package/dist/tools/artifacts.js +143 -0
- package/dist/tools/data.d.ts +7 -0
- package/dist/tools/data.js +160 -0
- package/dist/tools/execute.js +6 -2
- package/package.json +2 -2
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* JSON and JSON Lines helpers of the dataset ingestion (spec §17.2). Pure: the ingestion engine
|
|
3
|
+
* feeds decoded text and gets records back. Nested values are stored as JSON text by the caller.
|
|
4
|
+
*/
|
|
5
|
+
/** Splits streamed text into lines (`\n` or `\r\n`); the last partial line waits for more text. */
|
|
6
|
+
export class LineSplitter {
|
|
7
|
+
onLine;
|
|
8
|
+
rest = "";
|
|
9
|
+
constructor(onLine) {
|
|
10
|
+
this.onLine = onLine;
|
|
11
|
+
}
|
|
12
|
+
write(chunk) {
|
|
13
|
+
const text = this.rest + chunk;
|
|
14
|
+
let start = 0;
|
|
15
|
+
for (;;) {
|
|
16
|
+
const end = text.indexOf("\n", start);
|
|
17
|
+
if (end < 0)
|
|
18
|
+
break;
|
|
19
|
+
const stop = end > start && text.charCodeAt(end - 1) === 13 ? end - 1 : end;
|
|
20
|
+
this.onLine(text.slice(start, stop));
|
|
21
|
+
start = end + 1;
|
|
22
|
+
}
|
|
23
|
+
this.rest = start === 0 ? text : text.slice(start);
|
|
24
|
+
}
|
|
25
|
+
end() {
|
|
26
|
+
if (this.rest)
|
|
27
|
+
this.onLine(this.rest.endsWith("\r") ? this.rest.slice(0, -1) : this.rest);
|
|
28
|
+
this.rest = "";
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
/** A plain object stays as it is; any other JSON value becomes `{ value }`. */
|
|
32
|
+
export function toRecord(value) {
|
|
33
|
+
return value !== null && typeof value === "object" && !Array.isArray(value)
|
|
34
|
+
? value
|
|
35
|
+
: { value };
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* The records of a JSON document: an array (each element a record) or a single object (one
|
|
39
|
+
* record). Throws a `SyntaxError` for invalid JSON.
|
|
40
|
+
*/
|
|
41
|
+
export function jsonDocumentRecords(text) {
|
|
42
|
+
const parsed = JSON.parse(text);
|
|
43
|
+
return Array.isArray(parsed) ? parsed.map(toRecord) : [toRecord(parsed)];
|
|
44
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Query side of the data engine (spec §17.3): runs one read-only statement on a dataset file.
|
|
3
|
+
* The connection is opened `readOnly`, then `query_only=ON` and `trusted_schema=OFF`; extension
|
|
4
|
+
* loading stays disabled (the `node:sqlite` default). The host kills the process when a statement
|
|
5
|
+
* exceeds its time limit: `node:sqlite` offers no interrupt, progress handler or authorizer.
|
|
6
|
+
*/
|
|
7
|
+
import type { QueryRequest, QueryResult } from "./engine-types.ts";
|
|
8
|
+
/** Closes one connection (or all of them): the host does this before deleting a file. */
|
|
9
|
+
export declare function closeConnections(path?: string): void;
|
|
10
|
+
export declare class QueryRejected extends Error {
|
|
11
|
+
}
|
|
12
|
+
export declare function runQuery(req: QueryRequest): QueryResult;
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import { guardSql } from "./sql-guard.js";
|
|
2
|
+
const sqlite = process.getBuiltinModule("node:sqlite");
|
|
3
|
+
const connections = new Map();
|
|
4
|
+
function connect(path, heapBytes) {
|
|
5
|
+
let db = connections.get(path);
|
|
6
|
+
if (!db) {
|
|
7
|
+
db = new sqlite.DatabaseSync(path, { readOnly: true });
|
|
8
|
+
db.exec("PRAGMA query_only=ON; PRAGMA trusted_schema=OFF;");
|
|
9
|
+
if (heapBytes)
|
|
10
|
+
db.exec(`PRAGMA hard_heap_limit=${Math.floor(heapBytes)};`);
|
|
11
|
+
connections.set(path, db);
|
|
12
|
+
}
|
|
13
|
+
return db;
|
|
14
|
+
}
|
|
15
|
+
/** Closes one connection (or all of them): the host does this before deleting a file. */
|
|
16
|
+
export function closeConnections(path) {
|
|
17
|
+
for (const [key, db] of connections) {
|
|
18
|
+
if (path && key !== path)
|
|
19
|
+
continue;
|
|
20
|
+
try {
|
|
21
|
+
db.close();
|
|
22
|
+
}
|
|
23
|
+
catch {
|
|
24
|
+
/* already closed */
|
|
25
|
+
}
|
|
26
|
+
connections.delete(key);
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
export class QueryRejected extends Error {
|
|
30
|
+
}
|
|
31
|
+
export function runQuery(req) {
|
|
32
|
+
let sql = req.sql;
|
|
33
|
+
if (req.guard) {
|
|
34
|
+
const guarded = guardSql(sql);
|
|
35
|
+
if (!guarded.ok)
|
|
36
|
+
throw new QueryRejected(guarded.reason);
|
|
37
|
+
sql = guarded.sql;
|
|
38
|
+
}
|
|
39
|
+
const db = connect(req.db, req.heapBytes);
|
|
40
|
+
const statement = db.prepare(sql);
|
|
41
|
+
statement.setReadBigInts(true);
|
|
42
|
+
statement.setReturnArrays(true);
|
|
43
|
+
const columns = statement.columns().map((c) => c.name);
|
|
44
|
+
const rows = [];
|
|
45
|
+
let truncated = false;
|
|
46
|
+
let clipped = false;
|
|
47
|
+
const cell = (value) => {
|
|
48
|
+
if (value === null || value === undefined)
|
|
49
|
+
return null;
|
|
50
|
+
if (typeof value === "bigint") {
|
|
51
|
+
const n = Number(value);
|
|
52
|
+
return Number.isSafeInteger(n) ? n : value.toString();
|
|
53
|
+
}
|
|
54
|
+
if (typeof value === "number")
|
|
55
|
+
return value;
|
|
56
|
+
if (value instanceof Uint8Array)
|
|
57
|
+
return `<blob ${value.byteLength} bytes>`;
|
|
58
|
+
const text = String(value);
|
|
59
|
+
if (text.length > req.cellChars) {
|
|
60
|
+
clipped = true;
|
|
61
|
+
return `${text.slice(0, req.cellChars)}…`;
|
|
62
|
+
}
|
|
63
|
+
return text;
|
|
64
|
+
};
|
|
65
|
+
// JS numbers bind as REAL, which `LIMIT`/`OFFSET` reject: integral values bind as INTEGER.
|
|
66
|
+
const bound = (req.params ?? []).map((value) => typeof value === "number" && Number.isSafeInteger(value) ? BigInt(value) : value);
|
|
67
|
+
for (const row of statement.iterate(...bound)) {
|
|
68
|
+
if (rows.length >= req.maxRows) {
|
|
69
|
+
truncated = true;
|
|
70
|
+
break;
|
|
71
|
+
}
|
|
72
|
+
rows.push(row.map(cell));
|
|
73
|
+
}
|
|
74
|
+
return { columns, rows, truncated, clipped };
|
|
75
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
export declare const MAX_PAGE_ROWS = 500;
|
|
2
|
+
export declare const MAX_OFFSET = 100000;
|
|
3
|
+
/** Filtering every column at once (no `column`) is only allowed up to this many rows. */
|
|
4
|
+
export declare const MAX_ALL_COLUMNS_FILTER_ROWS = 100000;
|
|
5
|
+
export interface PageParams {
|
|
6
|
+
table: string;
|
|
7
|
+
columns: string[];
|
|
8
|
+
limit: number;
|
|
9
|
+
sort?: {
|
|
10
|
+
column: string;
|
|
11
|
+
dir: "asc" | "desc";
|
|
12
|
+
};
|
|
13
|
+
/** Substring filter on one column (or on every column when `column` is absent). */
|
|
14
|
+
filter?: {
|
|
15
|
+
column?: string;
|
|
16
|
+
text: string;
|
|
17
|
+
};
|
|
18
|
+
/** Opaque cursor returned by the previous page. */
|
|
19
|
+
after?: string;
|
|
20
|
+
/** Rows to skip (jump). */
|
|
21
|
+
offset?: number;
|
|
22
|
+
}
|
|
23
|
+
export declare class PageParamError extends Error {
|
|
24
|
+
}
|
|
25
|
+
/** The cursor of a row: base64 of `[rowid]`. */
|
|
26
|
+
export declare const encodeCursor: (rowid: number) => string;
|
|
27
|
+
export declare function decodeCursor(cursor: string): number;
|
|
28
|
+
type Param = string | number | null;
|
|
29
|
+
/** Statement and parameters of one page (`limit + 1` rows, to know whether a next page exists). */
|
|
30
|
+
export declare function buildPageQuery(params: PageParams): {
|
|
31
|
+
sql: string;
|
|
32
|
+
params: Param[];
|
|
33
|
+
};
|
|
34
|
+
/** `SELECT count(*)` of the filter (shown on the first page of a filtered view). */
|
|
35
|
+
export declare function buildCountQuery(params: Pick<PageParams, "table" | "columns" | "filter">): {
|
|
36
|
+
sql: string;
|
|
37
|
+
params: Param[];
|
|
38
|
+
};
|
|
39
|
+
export {};
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SQL of `SpreadsheetView` pages (spec §17.4). Pure: builds trusted statements for the engine.
|
|
3
|
+
* Paging is keyset-based on `rowid` (dense, in file order), so a page never scans from the start
|
|
4
|
+
* for the unsorted view and never duplicates or skips rows while sorting. `LIMIT/OFFSET` only
|
|
5
|
+
* serves jumps to a position when a sort or filter is active (≤ 100 000 rows in).
|
|
6
|
+
*/
|
|
7
|
+
import { quoteIdent } from "./infer.js";
|
|
8
|
+
export const MAX_PAGE_ROWS = 500;
|
|
9
|
+
export const MAX_OFFSET = 100_000;
|
|
10
|
+
/** Filtering every column at once (no `column`) is only allowed up to this many rows. */
|
|
11
|
+
export const MAX_ALL_COLUMNS_FILTER_ROWS = 100_000;
|
|
12
|
+
export class PageParamError extends Error {
|
|
13
|
+
}
|
|
14
|
+
/** The cursor of a row: base64 of `[rowid]`. */
|
|
15
|
+
export const encodeCursor = (rowid) => Buffer.from(JSON.stringify([rowid])).toString("base64url");
|
|
16
|
+
export function decodeCursor(cursor) {
|
|
17
|
+
try {
|
|
18
|
+
const value = JSON.parse(Buffer.from(cursor, "base64url").toString("utf8"));
|
|
19
|
+
if (Array.isArray(value) && Number.isSafeInteger(value[0]) && value[0] >= 0)
|
|
20
|
+
return value[0];
|
|
21
|
+
}
|
|
22
|
+
catch {
|
|
23
|
+
/* falls through */
|
|
24
|
+
}
|
|
25
|
+
throw new PageParamError("Invalid cursor");
|
|
26
|
+
}
|
|
27
|
+
const escapeLike = (text) => text.replace(/[\\%_]/g, (c) => `\\${c}`);
|
|
28
|
+
/** Statement and parameters of one page (`limit + 1` rows, to know whether a next page exists). */
|
|
29
|
+
export function buildPageQuery(params) {
|
|
30
|
+
const table = quoteIdent(params.table);
|
|
31
|
+
const known = new Set(params.columns);
|
|
32
|
+
const values = [];
|
|
33
|
+
const where = [];
|
|
34
|
+
const sortColumn = params.sort?.column;
|
|
35
|
+
if (sortColumn !== undefined && !known.has(sortColumn))
|
|
36
|
+
throw new PageParamError(`Unknown column: ${sortColumn}`);
|
|
37
|
+
if (params.filter?.text) {
|
|
38
|
+
const like = `%${escapeLike(params.filter.text)}%`;
|
|
39
|
+
if (params.filter.column !== undefined) {
|
|
40
|
+
if (!known.has(params.filter.column))
|
|
41
|
+
throw new PageParamError(`Unknown column: ${params.filter.column}`);
|
|
42
|
+
where.push(`${quoteIdent(params.filter.column)} LIKE ? ESCAPE '\\'`);
|
|
43
|
+
values.push(like);
|
|
44
|
+
}
|
|
45
|
+
else {
|
|
46
|
+
where.push(`(${params.columns.map((c) => `${quoteIdent(c)} LIKE ? ESCAPE '\\'`).join(" OR ")})`);
|
|
47
|
+
for (let i = 0; i < params.columns.length; i++)
|
|
48
|
+
values.push(like);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
const filtered = where.length > 0;
|
|
52
|
+
let order = "rowid";
|
|
53
|
+
let offset = 0;
|
|
54
|
+
if (sortColumn === undefined) {
|
|
55
|
+
if (params.after) {
|
|
56
|
+
where.push("rowid > ?");
|
|
57
|
+
values.push(decodeCursor(params.after));
|
|
58
|
+
}
|
|
59
|
+
else if (params.offset) {
|
|
60
|
+
if (!filtered) {
|
|
61
|
+
// rowid is dense (1..N in file order): jumping needs no OFFSET at all.
|
|
62
|
+
where.push("rowid > ?");
|
|
63
|
+
values.push(params.offset);
|
|
64
|
+
}
|
|
65
|
+
else
|
|
66
|
+
offset = params.offset;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
else {
|
|
70
|
+
const k = quoteIdent(sortColumn);
|
|
71
|
+
const asc = params.sort?.dir !== "desc";
|
|
72
|
+
order = `${k} ${asc ? "ASC" : "DESC"}, rowid ASC`;
|
|
73
|
+
if (params.after) {
|
|
74
|
+
const r = decodeCursor(params.after);
|
|
75
|
+
const v = `(SELECT ${k} FROM ${table} WHERE rowid = ?)`;
|
|
76
|
+
// NULLs sort first ascending and last descending; ties break on rowid ascending.
|
|
77
|
+
const next = asc
|
|
78
|
+
? `((${v} IS NULL AND (${k} IS NOT NULL OR rowid > ?)) OR ` +
|
|
79
|
+
`(${v} IS NOT NULL AND (${k} > ${v} OR (${k} = ${v} AND rowid > ?))))`
|
|
80
|
+
: `((${v} IS NULL AND ${k} IS NULL AND rowid > ?) OR ` +
|
|
81
|
+
`(${v} IS NOT NULL AND (${k} < ${v} OR (${k} = ${v} AND rowid > ?) OR ${k} IS NULL)))`;
|
|
82
|
+
where.push(next);
|
|
83
|
+
for (let i = 0; i < 6; i++)
|
|
84
|
+
values.push(r);
|
|
85
|
+
}
|
|
86
|
+
else if (params.offset)
|
|
87
|
+
offset = params.offset;
|
|
88
|
+
}
|
|
89
|
+
if (offset > MAX_OFFSET)
|
|
90
|
+
throw new PageParamError(`Jumping is limited to the first ${MAX_OFFSET} rows when sorting or filtering`);
|
|
91
|
+
const select = ["rowid", ...params.columns.map(quoteIdent)].join(", ");
|
|
92
|
+
values.push(Math.min(params.limit, MAX_PAGE_ROWS) + 1);
|
|
93
|
+
const sql = `SELECT ${select} FROM ${table}` +
|
|
94
|
+
(where.length ? ` WHERE ${where.join(" AND ")}` : "") +
|
|
95
|
+
` ORDER BY ${order} LIMIT ?` +
|
|
96
|
+
(offset ? ` OFFSET ${Math.floor(offset)}` : "");
|
|
97
|
+
return { sql, params: values };
|
|
98
|
+
}
|
|
99
|
+
/** `SELECT count(*)` of the filter (shown on the first page of a filtered view). */
|
|
100
|
+
export function buildCountQuery(params) {
|
|
101
|
+
const table = quoteIdent(params.table);
|
|
102
|
+
const text = params.filter?.text;
|
|
103
|
+
if (!text)
|
|
104
|
+
return { sql: `SELECT count(*) FROM ${table}`, params: [] };
|
|
105
|
+
const like = `%${escapeLike(text)}%`;
|
|
106
|
+
if (params.filter?.column !== undefined) {
|
|
107
|
+
if (!params.columns.includes(params.filter.column))
|
|
108
|
+
throw new PageParamError(`Unknown column: ${params.filter.column}`);
|
|
109
|
+
return {
|
|
110
|
+
sql: `SELECT count(*) FROM ${table} WHERE ${quoteIdent(params.filter.column)} LIKE ? ESCAPE '\\'`,
|
|
111
|
+
params: [like],
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
return {
|
|
115
|
+
sql: `SELECT count(*) FROM ${table} WHERE (${params.columns
|
|
116
|
+
.map((c) => `${quoteIdent(c)} LIKE ? ESCAPE '\\'`)
|
|
117
|
+
.join(" OR ")})`,
|
|
118
|
+
params: params.columns.map(() => like),
|
|
119
|
+
};
|
|
120
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Lexical guard of `data_query` (spec §17.3). `node:sqlite` has no authorizer or progress
|
|
3
|
+
* handler and `DatabaseSync.prepare()` silently ignores text after the first statement, so the
|
|
4
|
+
* guard checks the SQL itself: exactly one statement, starting with SELECT or WITH, without
|
|
5
|
+
* write, schema, transaction or extension keywords outside literals and comments. The read-only
|
|
6
|
+
* connection (`readOnly`, `query_only`, extensions off) is the second line of defence.
|
|
7
|
+
*/
|
|
8
|
+
export type GuardResult = {
|
|
9
|
+
ok: true;
|
|
10
|
+
sql: string;
|
|
11
|
+
} | {
|
|
12
|
+
ok: false;
|
|
13
|
+
reason: string;
|
|
14
|
+
};
|
|
15
|
+
/**
|
|
16
|
+
* Validates a model-written query. Accepts one `SELECT`/`WITH` statement (a single trailing `;`
|
|
17
|
+
* is allowed and removed). Keywords inside literals, comments and quoted identifiers are fine;
|
|
18
|
+
* `replace(…)` as a function is fine (REPLACE INTO is not).
|
|
19
|
+
*/
|
|
20
|
+
export declare function guardSql(sql: string): GuardResult;
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Lexical guard of `data_query` (spec §17.3). `node:sqlite` has no authorizer or progress
|
|
3
|
+
* handler and `DatabaseSync.prepare()` silently ignores text after the first statement, so the
|
|
4
|
+
* guard checks the SQL itself: exactly one statement, starting with SELECT or WITH, without
|
|
5
|
+
* write, schema, transaction or extension keywords outside literals and comments. The read-only
|
|
6
|
+
* connection (`readOnly`, `query_only`, extensions off) is the second line of defence.
|
|
7
|
+
*/
|
|
8
|
+
const FORBIDDEN = new Set([
|
|
9
|
+
"ATTACH",
|
|
10
|
+
"DETACH",
|
|
11
|
+
"PRAGMA",
|
|
12
|
+
"INSERT",
|
|
13
|
+
"UPDATE",
|
|
14
|
+
"DELETE",
|
|
15
|
+
"REPLACE",
|
|
16
|
+
"CREATE",
|
|
17
|
+
"DROP",
|
|
18
|
+
"ALTER",
|
|
19
|
+
"VACUUM",
|
|
20
|
+
"REINDEX",
|
|
21
|
+
"ANALYZE",
|
|
22
|
+
"BEGIN",
|
|
23
|
+
"COMMIT",
|
|
24
|
+
"ROLLBACK",
|
|
25
|
+
"SAVEPOINT",
|
|
26
|
+
"RELEASE",
|
|
27
|
+
"LOAD_EXTENSION",
|
|
28
|
+
]);
|
|
29
|
+
const isWordStart = (c) => /[A-Za-z_]/.test(c);
|
|
30
|
+
const isWordChar = (c) => /[A-Za-z0-9_$]/.test(c);
|
|
31
|
+
/**
|
|
32
|
+
* Splits SQL into significant tokens, skipping whitespace, comments and the contents of string
|
|
33
|
+
* literals and quoted identifiers. Returns an error text for an unterminated literal or comment.
|
|
34
|
+
*/
|
|
35
|
+
function tokenize(sql) {
|
|
36
|
+
const tokens = [];
|
|
37
|
+
const n = sql.length;
|
|
38
|
+
let i = 0;
|
|
39
|
+
while (i < n) {
|
|
40
|
+
const c = sql[i];
|
|
41
|
+
if (c === "'" || c === '"' || c === "`") {
|
|
42
|
+
let j = i + 1;
|
|
43
|
+
for (;;) {
|
|
44
|
+
if (j >= n)
|
|
45
|
+
return { error: `Unterminated ${c === "'" ? "string" : "identifier"}` };
|
|
46
|
+
if (sql[j] === c) {
|
|
47
|
+
if (sql[j + 1] === c) {
|
|
48
|
+
j += 2;
|
|
49
|
+
continue;
|
|
50
|
+
}
|
|
51
|
+
break;
|
|
52
|
+
}
|
|
53
|
+
j++;
|
|
54
|
+
}
|
|
55
|
+
// A quoted identifier or string is a value, not a keyword; it still counts as a token so
|
|
56
|
+
// that "SELECT" 1 is not mistaken for an empty statement.
|
|
57
|
+
tokens.push({ type: "punct", text: c });
|
|
58
|
+
i = j + 1;
|
|
59
|
+
}
|
|
60
|
+
else if (c === "[") {
|
|
61
|
+
const end = sql.indexOf("]", i + 1);
|
|
62
|
+
if (end < 0)
|
|
63
|
+
return { error: "Unterminated identifier" };
|
|
64
|
+
tokens.push({ type: "punct", text: "[" });
|
|
65
|
+
i = end + 1;
|
|
66
|
+
}
|
|
67
|
+
else if (c === "-" && sql[i + 1] === "-") {
|
|
68
|
+
const end = sql.indexOf("\n", i + 2);
|
|
69
|
+
i = end < 0 ? n : end + 1;
|
|
70
|
+
}
|
|
71
|
+
else if (c === "/" && sql[i + 1] === "*") {
|
|
72
|
+
const end = sql.indexOf("*/", i + 2);
|
|
73
|
+
if (end < 0)
|
|
74
|
+
return { error: "Unterminated comment" };
|
|
75
|
+
i = end + 2;
|
|
76
|
+
}
|
|
77
|
+
else if (/\s/.test(c)) {
|
|
78
|
+
i++;
|
|
79
|
+
}
|
|
80
|
+
else if (isWordStart(c)) {
|
|
81
|
+
let j = i + 1;
|
|
82
|
+
while (j < n && isWordChar(sql[j]))
|
|
83
|
+
j++;
|
|
84
|
+
tokens.push({ type: "word", text: sql.slice(i, j) });
|
|
85
|
+
i = j;
|
|
86
|
+
}
|
|
87
|
+
else {
|
|
88
|
+
tokens.push({ type: "punct", text: c });
|
|
89
|
+
i++;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
return { tokens };
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Validates a model-written query. Accepts one `SELECT`/`WITH` statement (a single trailing `;`
|
|
96
|
+
* is allowed and removed). Keywords inside literals, comments and quoted identifiers are fine;
|
|
97
|
+
* `replace(…)` as a function is fine (REPLACE INTO is not).
|
|
98
|
+
*/
|
|
99
|
+
export function guardSql(sql) {
|
|
100
|
+
const scanned = tokenize(sql);
|
|
101
|
+
if ("error" in scanned)
|
|
102
|
+
return { ok: false, reason: scanned.error };
|
|
103
|
+
const { tokens } = scanned;
|
|
104
|
+
// Drop one trailing semicolon; any other one separates statements.
|
|
105
|
+
if (tokens.at(-1)?.text === ";")
|
|
106
|
+
tokens.pop();
|
|
107
|
+
if (!tokens.length)
|
|
108
|
+
return { ok: false, reason: "The query is empty" };
|
|
109
|
+
if (tokens.some((t) => t.type === "punct" && t.text === ";"))
|
|
110
|
+
return { ok: false, reason: "Only one statement is allowed (no `;` between statements)" };
|
|
111
|
+
const first = tokens[0];
|
|
112
|
+
if (first.type !== "word" || !["SELECT", "WITH"].includes(first.text.toUpperCase()))
|
|
113
|
+
return { ok: false, reason: "Only SELECT and WITH queries are allowed" };
|
|
114
|
+
for (let i = 0; i < tokens.length; i++) {
|
|
115
|
+
const token = tokens[i];
|
|
116
|
+
if (token.type !== "word")
|
|
117
|
+
continue;
|
|
118
|
+
const word = token.text.toUpperCase();
|
|
119
|
+
if (!FORBIDDEN.has(word))
|
|
120
|
+
continue;
|
|
121
|
+
if (word === "REPLACE" && tokens[i + 1]?.text === "(")
|
|
122
|
+
continue;
|
|
123
|
+
return {
|
|
124
|
+
ok: false,
|
|
125
|
+
reason: `\`${token.text}\` is not allowed in a read-only query (quote column names that look like keywords, e.g. "${token.text}")`,
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
return { ok: true, sql };
|
|
129
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/** Minimal view of `node:sqlite`'s `DatabaseSync` (the engine runs outside the type roots). */
|
|
2
|
+
export interface Db {
|
|
3
|
+
exec(sql: string): void;
|
|
4
|
+
prepare(sql: string): {
|
|
5
|
+
run(...params: unknown[]): unknown;
|
|
6
|
+
get(...params: unknown[]): unknown;
|
|
7
|
+
all(...params: unknown[]): unknown[];
|
|
8
|
+
};
|
|
9
|
+
}
|
|
10
|
+
export declare const INGEST_VERSION = 1;
|
|
11
|
+
/** Distinct counts and top values are exact up to this many rows; above, a sample is used. */
|
|
12
|
+
export declare const EXACT_DISTINCT_ROWS = 1000000;
|
|
13
|
+
export declare const META_SCHEMA = "\nCREATE TABLE IF NOT EXISTS _alisio_meta(key TEXT PRIMARY KEY, value TEXT);\nCREATE TABLE IF NOT EXISTS _alisio_sheets(ordinal INTEGER PRIMARY KEY, name TEXT NOT NULL,\n table_name TEXT NOT NULL, rows INTEGER NOT NULL, columns INTEGER NOT NULL);\nCREATE TABLE IF NOT EXISTS _alisio_columns(table_name TEXT NOT NULL, ordinal INTEGER NOT NULL,\n name TEXT NOT NULL, label TEXT NOT NULL, inferred_type TEXT NOT NULL,\n nulls INTEGER, distinct_count INTEGER, distinct_exact INTEGER,\n min TEXT, max TEXT, mean REAL, text_fallbacks INTEGER, top_values TEXT,\n PRIMARY KEY(table_name, ordinal));\n";
|
|
14
|
+
/**
|
|
15
|
+
* Computes `_alisio_columns` statistics for every registered column and refreshes the row and
|
|
16
|
+
* column counts of `_alisio_sheets`. Column rows (table, ordinal, name, label) must exist.
|
|
17
|
+
*/
|
|
18
|
+
export declare function finalizeDatabase(db: Db): void;
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Metadata tables of a dataset file and the per-column statistics (spec §17.2). The same
|
|
3
|
+
* `finalizeDatabase` runs after Node parses a CSV/TSV/JSON/JSONL file and after the Python helper
|
|
4
|
+
* converts an XLSX workbook, so both routes produce identical types and statistics.
|
|
5
|
+
*/
|
|
6
|
+
import { inferType, quoteIdent } from "./infer.js";
|
|
7
|
+
export const INGEST_VERSION = 1;
|
|
8
|
+
/** Distinct counts and top values are exact up to this many rows; above, a sample is used. */
|
|
9
|
+
export const EXACT_DISTINCT_ROWS = 1_000_000;
|
|
10
|
+
const SAMPLE_ROWS = 200_000;
|
|
11
|
+
const TYPE_SAMPLE = 1000;
|
|
12
|
+
export const META_SCHEMA = `
|
|
13
|
+
CREATE TABLE IF NOT EXISTS _alisio_meta(key TEXT PRIMARY KEY, value TEXT);
|
|
14
|
+
CREATE TABLE IF NOT EXISTS _alisio_sheets(ordinal INTEGER PRIMARY KEY, name TEXT NOT NULL,
|
|
15
|
+
table_name TEXT NOT NULL, rows INTEGER NOT NULL, columns INTEGER NOT NULL);
|
|
16
|
+
CREATE TABLE IF NOT EXISTS _alisio_columns(table_name TEXT NOT NULL, ordinal INTEGER NOT NULL,
|
|
17
|
+
name TEXT NOT NULL, label TEXT NOT NULL, inferred_type TEXT NOT NULL,
|
|
18
|
+
nulls INTEGER, distinct_count INTEGER, distinct_exact INTEGER,
|
|
19
|
+
min TEXT, max TEXT, mean REAL, text_fallbacks INTEGER, top_values TEXT,
|
|
20
|
+
PRIMARY KEY(table_name, ordinal));
|
|
21
|
+
`;
|
|
22
|
+
/**
|
|
23
|
+
* Computes `_alisio_columns` statistics for every registered column and refreshes the row and
|
|
24
|
+
* column counts of `_alisio_sheets`. Column rows (table, ordinal, name, label) must exist.
|
|
25
|
+
*/
|
|
26
|
+
export function finalizeDatabase(db) {
|
|
27
|
+
const sheets = db
|
|
28
|
+
.prepare("SELECT ordinal, table_name FROM _alisio_sheets ORDER BY ordinal")
|
|
29
|
+
.all();
|
|
30
|
+
db.exec("BEGIN");
|
|
31
|
+
try {
|
|
32
|
+
for (const sheet of sheets) {
|
|
33
|
+
const table = quoteIdent(sheet.table_name);
|
|
34
|
+
const rows = db.prepare(`SELECT count(*) AS n FROM ${table}`).get().n;
|
|
35
|
+
const columns = db
|
|
36
|
+
.prepare("SELECT table_name, ordinal, name FROM _alisio_columns WHERE table_name=? ORDER BY ordinal")
|
|
37
|
+
.all(sheet.table_name);
|
|
38
|
+
db.prepare("UPDATE _alisio_sheets SET rows=?, columns=? WHERE ordinal=?").run(rows, columns.length, sheet.ordinal);
|
|
39
|
+
const exact = rows <= EXACT_DISTINCT_ROWS;
|
|
40
|
+
const stride = exact ? 1 : Math.ceil(rows / SAMPLE_ROWS);
|
|
41
|
+
const andWhere = exact ? "" : `AND rowid % ${stride} = 0`;
|
|
42
|
+
for (const column of columns) {
|
|
43
|
+
const col = quoteIdent(column.name);
|
|
44
|
+
const sample = db
|
|
45
|
+
.prepare(`SELECT typeof(${col}) AS kind, ${col} AS value FROM ${table} LIMIT ${TYPE_SAMPLE}`)
|
|
46
|
+
.all();
|
|
47
|
+
const type = inferType(sample);
|
|
48
|
+
const agg = db
|
|
49
|
+
.prepare(`SELECT sum(${col} IS NULL) AS nulls, min(${col}) AS mn, max(${col}) AS mx,
|
|
50
|
+
sum(typeof(${col}) IN ('integer','real')) AS nums,
|
|
51
|
+
sum(typeof(${col})='text') AS texts,
|
|
52
|
+
sum(typeof(${col})='text' AND ${col} GLOB '*[0-9]*' AND NOT ${col} GLOB '*[A-Za-z]*') AS numlike,
|
|
53
|
+
avg(CASE WHEN typeof(${col}) IN ('integer','real') THEN ${col} END) AS mean
|
|
54
|
+
FROM ${table}`)
|
|
55
|
+
.get();
|
|
56
|
+
const groups = db
|
|
57
|
+
.prepare(`WITH g AS MATERIALIZED (
|
|
58
|
+
SELECT ${col} AS v, count(*) AS n FROM ${table}
|
|
59
|
+
WHERE ${col} IS NOT NULL ${andWhere} GROUP BY ${col})
|
|
60
|
+
SELECT (SELECT count(*) FROM g) AS d, CAST(v AS TEXT) AS v, n FROM g
|
|
61
|
+
ORDER BY n DESC, v LIMIT 5`)
|
|
62
|
+
.all();
|
|
63
|
+
const nums = Number(agg.nums ?? 0);
|
|
64
|
+
const texts = Number(agg.texts ?? 0);
|
|
65
|
+
const fallbacks = nums > 0 ? texts : type === "text" ? Number(agg.numlike ?? 0) : 0;
|
|
66
|
+
const numeric = type === "integer" || type === "real";
|
|
67
|
+
db.prepare(`UPDATE _alisio_columns SET inferred_type=?, nulls=?, distinct_count=?, distinct_exact=?,
|
|
68
|
+
min=?, max=?, mean=?, text_fallbacks=?, top_values=? WHERE table_name=? AND ordinal=?`).run(type, Number(agg.nulls ?? 0), Number(groups[0]?.d ?? 0), exact ? 1 : 0, agg.mn === null || agg.mn === undefined ? null : String(agg.mn), agg.mx === null || agg.mx === undefined ? null : String(agg.mx), numeric && agg.mean !== null ? Number(agg.mean) : null, fallbacks, JSON.stringify(groups.map((g) => ({ value: g.v, count: Number(g.n) }))), sheet.table_name, column.ordinal);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
db.exec("COMMIT");
|
|
72
|
+
}
|
|
73
|
+
catch (error) {
|
|
74
|
+
db.exec("ROLLBACK");
|
|
75
|
+
throw error;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
export interface HintHost {
|
|
2
|
+
platform: NodeJS.Platform;
|
|
3
|
+
arch: string;
|
|
4
|
+
/** Linux `/etc/os-release` fields (`ID`, `ID_LIKE`, `PRETTY_NAME`, `VERSION_ID`). */
|
|
5
|
+
osRelease?: Record<string, string>;
|
|
6
|
+
hasUv?: boolean;
|
|
7
|
+
hasWinget?: boolean;
|
|
8
|
+
hasBrew?: boolean;
|
|
9
|
+
/** A Python that was found but is too old (or a `--python` that failed). */
|
|
10
|
+
found?: {
|
|
11
|
+
version?: string;
|
|
12
|
+
path: string;
|
|
13
|
+
reason?: string;
|
|
14
|
+
};
|
|
15
|
+
}
|
|
16
|
+
export interface InstallHints {
|
|
17
|
+
/** `Linux · Ubuntu 22.04 · x86_64` */
|
|
18
|
+
system: string;
|
|
19
|
+
/** Optional heading line (found Python too old, `--python` not usable). */
|
|
20
|
+
heading?: string;
|
|
21
|
+
primary: string;
|
|
22
|
+
alternatives: string[];
|
|
23
|
+
notes: string[];
|
|
24
|
+
}
|
|
25
|
+
/** Parses `KEY=value` / `KEY="value"` lines of an os-release file. */
|
|
26
|
+
export declare function parseOsRelease(text: string): Record<string, string>;
|
|
27
|
+
export declare function pythonInstallHints(host: HintHost): InstallHints;
|
|
28
|
+
/** One paragraph for the model and terminals (English, like every tool message). */
|
|
29
|
+
export declare function installHintsText(hints: InstallHints): string;
|
|
30
|
+
/** The same guidance as Markdown (web and TUI render it as a `markdown` block). */
|
|
31
|
+
export declare function installHintsMarkdown(hints: InstallHints): string;
|
|
32
|
+
/** Detects the host for `pythonInstallHints` (reads files and PATH only; launches nothing). */
|
|
33
|
+
export declare function detectHintHost(found?: HintHost["found"]): Promise<HintHost>;
|