bunqueue-dashboard 0.0.32 → 0.0.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +44 -15
- package/agent/db.ts +537 -64
- package/agent/dbQueryWorker.ts +34 -11
- package/agent/manager.ts +186 -45
- package/agent/server.ts +124 -35
- package/dist/.vite/manifest.json +212 -157
- package/dist/assets/AddJob-Cih9F84t.js +3 -0
- package/dist/assets/Alerts-Dp4Vclez.js +1 -0
- package/dist/assets/{AreaChart-2gM0YoZT.js → AreaChart-REK4ZsTC.js} +1 -1
- package/dist/assets/Benchmark-X_gdZun5.js +1 -0
- package/dist/assets/BulkAddJobs-CXTpYMZf.js +5 -0
- package/dist/assets/CopilotPanel-CIXpcwkH.js +44 -0
- package/dist/assets/Cron-DUHkse77.js +1 -0
- package/dist/assets/CronManager-BeYrLK2E.js +1 -0
- package/dist/assets/Database-CXPb3XGr.js +3 -0
- package/dist/assets/Diagnostics-DDtoiDYT.js +1 -0
- package/dist/assets/Dlq-CX_uRblW.js +1 -0
- package/dist/assets/DlqControl-CazFmSEd.js +1 -0
- package/dist/assets/DlqPro-HSffdtLJ.js +1 -0
- package/dist/assets/Flows-CGmyghKF.js +1 -0
- package/dist/assets/JobInspector-B9i1HnG5.js +2 -0
- package/dist/assets/Jobs-BZGtMnJI.js +1 -0
- package/dist/assets/JobsPro-tmEN8f-P.js +1 -0
- package/dist/assets/Logs-DxUD9V2A.js +1 -0
- package/dist/assets/LogsPro-BTQhvPKj.js +1 -0
- package/dist/assets/McpServer-D5rwBg74.js +22 -0
- package/dist/assets/Metrics-CSPpBf1m.js +1 -0
- package/dist/assets/MetricsPro-DSn-3qNT.js +1 -0
- package/dist/assets/Overview-StvHYRZ5.js +1 -0
- package/dist/assets/{Pagination-B9JDi5MI.js → Pagination-eotgGQf9.js} +1 -1
- package/dist/assets/QueueActions-CibRLzrz.js +1 -0
- package/dist/assets/QueueControl-DwYNcMfI.js +1 -0
- package/dist/assets/QueueDetail-NMfKwj6E.js +1 -0
- package/dist/assets/QueueDetailPro-C8SBjX09.js +1 -0
- package/dist/assets/Queues-BeTAi8V8.js +1 -0
- package/dist/assets/QueuesOverview-BfQO8NS4.js +1 -0
- package/dist/assets/S3Backup-BYkAtONl.js +1 -0
- package/dist/assets/S3BackupPro-BP1Ch54R.js +2 -0
- package/dist/assets/ServerControl-C85jYgyN.js +3 -0
- package/dist/assets/Settings-DjP28c_5.js +1 -0
- package/dist/assets/Usage-DECeEsO2.js +1 -0
- package/dist/assets/UsagePro-CD7dgt3d.js +1 -0
- package/dist/assets/Webhooks-Ct-x8DD1.js +1 -0
- package/dist/assets/Workers-lP1BXHFq.js +1 -0
- package/dist/assets/WorkersPro-CAUqy3Cp.js +1 -0
- package/dist/assets/connectionStore-Cd9-3Fw0.js +1 -0
- package/dist/assets/flowMutationSafety-DVD1IMr-.js +1 -0
- package/dist/assets/form-DsUq0w8s.js +1 -0
- package/dist/assets/index-BWoUj86M.css +1 -0
- package/dist/assets/index-Dr__4DsT.js +8 -0
- package/dist/assets/install-DPCcfOYo.js +3 -0
- package/dist/assets/jobActions-LQD4KfCc.js +1 -0
- package/dist/assets/promisePool-BVfQ9n9Q.js +1 -0
- package/dist/assets/{react-vendor-BcouBxpZ.js → react-vendor-Bjui4wS4.js} +1 -1
- package/dist/assets/upstreamPaths-nG8N1XAz.js +1 -0
- package/dist/assets/useServerActionGuard-CItj3hvh.js +1 -0
- package/dist/assets/useThroughputSeries-Bpyg6Jaq.js +1 -0
- package/dist/index.html +5 -4
- package/package.json +11 -3
- package/scripts/embedded.gen.ts +132 -126
- package/scripts/serve.ts +259 -67
- package/dist/assets/AddJob-Dvpp--yq.js +0 -3
- package/dist/assets/Alerts-CR9jgCCs.js +0 -1
- package/dist/assets/Benchmark-CkL-Ee-M.js +0 -1
- package/dist/assets/BulkAddJobs-wI6cRZnX.js +0 -5
- package/dist/assets/CopilotPanel-DPZV9opm.js +0 -44
- package/dist/assets/Cron-gWVkYWad.js +0 -1
- package/dist/assets/CronManager-BvfW219u.js +0 -1
- package/dist/assets/Database-D2T_ZIGA.js +0 -3
- package/dist/assets/Diagnostics-C2goqq5h.js +0 -1
- package/dist/assets/Dlq-Q4TmmZrK.js +0 -1
- package/dist/assets/DlqControl-C5BfE7Gq.js +0 -1
- package/dist/assets/DlqPro-ClPNmw-X.js +0 -1
- package/dist/assets/Flows-DbhMbvSG.js +0 -1
- package/dist/assets/JobInspector-CO_zKjSX.js +0 -2
- package/dist/assets/Jobs-D2rDHPM_.js +0 -1
- package/dist/assets/JobsPro-Drdgdj3s.js +0 -1
- package/dist/assets/Logs-DjwGuRLi.js +0 -1
- package/dist/assets/LogsPro-7uTBjPWX.js +0 -1
- package/dist/assets/McpServer-DAklQDB5.js +0 -22
- package/dist/assets/Metrics-Dup9Zinh.js +0 -1
- package/dist/assets/MetricsPro-PP3TymeP.js +0 -1
- package/dist/assets/Overview-C7RCDE6r.js +0 -1
- package/dist/assets/QueueActions-DJTw0h2g.js +0 -1
- package/dist/assets/QueueControl-CxKjUjF-.js +0 -1
- package/dist/assets/QueueDetail-bf2n-r2h.js +0 -1
- package/dist/assets/QueueDetailPro-C-XLWFBa.js +0 -1
- package/dist/assets/Queues-CMQW89J9.js +0 -1
- package/dist/assets/QueuesOverview-x7-akDzf.js +0 -1
- package/dist/assets/S3Backup-1jAySC9R.js +0 -1
- package/dist/assets/S3BackupPro-BG_0OHW7.js +0 -1
- package/dist/assets/ServerControl-DohTB--h.js +0 -3
- package/dist/assets/Settings-CnS5PjNe.js +0 -1
- package/dist/assets/Usage-C3H0hMSA.js +0 -1
- package/dist/assets/UsagePro-CdnQkP9o.js +0 -1
- package/dist/assets/Webhooks-CAwVmIEZ.js +0 -1
- package/dist/assets/Workers-lvujKZwF.js +0 -1
- package/dist/assets/WorkersPro-CIJKl7U1.js +0 -1
- package/dist/assets/connectionStore-DO6xQY7h.js +0 -1
- package/dist/assets/form-DGinTEEb.js +0 -1
- package/dist/assets/index-DcVOpnSj.css +0 -1
- package/dist/assets/index-r1ZMLIZ9.js +0 -8
- package/dist/assets/install-BIVfix28.js +0 -3
- package/dist/assets/jobActions-CuOdMglt.js +0 -1
- package/dist/assets/promisePool-Daxkc1OB.js +0 -1
- package/dist/assets/useThroughputSeries-CVKzWzdv.js +0 -1
package/agent/db.ts
CHANGED
|
@@ -18,12 +18,11 @@
|
|
|
18
18
|
* - results are streamed with an early break at MAX_ROWS, so a huge or
|
|
19
19
|
* row-bomb query can't materialize the whole set in agent memory;
|
|
20
20
|
* - queryWithTimeout() runs it in a disposable Worker raced against a wall
|
|
21
|
-
* clock, so an O(N^k) scan can't pin the agent's
|
|
22
|
-
* process-control endpoints
|
|
23
|
-
*
|
|
24
|
-
* the
|
|
25
|
-
*
|
|
26
|
-
* MAX_CONCURRENT_QUERIES instead of accumulating one pinned core per request.
|
|
21
|
+
* clock, so an O(N^k) scan can't pin the agent's event loop and freeze the
|
|
22
|
+
* process-control endpoints. Worker startup failures surface instead of
|
|
23
|
+
* falling back to a synchronous scan. The timeout aborts the response, not
|
|
24
|
+
* the query — a synchronous sqlite3_step never yields to terminate() — so
|
|
25
|
+
* live worker threads are counted and capped by MAX_CONCURRENT_QUERIES.
|
|
27
26
|
*/
|
|
28
27
|
import { existsSync } from 'node:fs';
|
|
29
28
|
import { Database } from 'bun:sqlite';
|
|
@@ -34,6 +33,18 @@ export const MAX_ROWS = 500;
|
|
|
34
33
|
const MAX_CELL = 2000;
|
|
35
34
|
/** Full single-cell fetch (detail view) hard cap — bounds one pathological blob. */
|
|
36
35
|
const MAX_FULL_CELL = 1_000_000;
|
|
36
|
+
/** Full-table CSV export cap. The cursor probes one extra row to report truncation exactly. */
|
|
37
|
+
export const DB_EXPORT_MAX_ROWS = 200_000;
|
|
38
|
+
/** Maximum CSV response body. The exporter writes into one fixed-size buffer. */
|
|
39
|
+
export const DB_EXPORT_MAX_BYTES = 16 * 1024 * 1024;
|
|
40
|
+
/** Refuse pathological schemas before one row can create excessive transient allocations. */
|
|
41
|
+
export const DB_EXPORT_MAX_COLUMNS = 256;
|
|
42
|
+
/** Bound SQL/header construction even for a hostile or corrupt local schema. */
|
|
43
|
+
export const DB_EXPORT_MAX_IDENTIFIER_CHARS = 1024;
|
|
44
|
+
/** Bound a user-supplied filter before it is retained by SQLite for a long export. */
|
|
45
|
+
export const DB_EXPORT_MAX_FILTER_CHARS = 4096;
|
|
46
|
+
/** Response deadline; timed-out SQLite work remains isolated and accounted for in its Worker. */
|
|
47
|
+
export const DB_EXPORT_TIMEOUT_MS = 15_000;
|
|
37
48
|
/** Wall-clock budget for an arbitrary query before it is aborted. */
|
|
38
49
|
export const QUERY_TIMEOUT_MS = 5000;
|
|
39
50
|
|
|
@@ -42,6 +53,12 @@ export class MissingDbError extends Error {
|
|
|
42
53
|
readonly missing = true;
|
|
43
54
|
}
|
|
44
55
|
|
|
56
|
+
/** Maps a saturated export worker pool to HTTP 429. */
|
|
57
|
+
export class DbExportBusyError extends Error {}
|
|
58
|
+
|
|
59
|
+
/** Maps export worker startup/crash/timeout failures to HTTP 503. */
|
|
60
|
+
export class DbExportUnavailableError extends Error {}
|
|
61
|
+
|
|
45
62
|
export interface DbTableInfo {
|
|
46
63
|
name: string;
|
|
47
64
|
rows: number;
|
|
@@ -59,7 +76,7 @@ export interface DbRowsPage {
|
|
|
59
76
|
columns: string[];
|
|
60
77
|
rows: unknown[][];
|
|
61
78
|
/** rowid per row (null for WITHOUT ROWID tables) — keys the full-cell fetch. */
|
|
62
|
-
rowids: (
|
|
79
|
+
rowids: (DbRowId | null)[];
|
|
63
80
|
/** Which cells were truncated server-side, so the UI can flag them. */
|
|
64
81
|
truncatedCells: boolean[][];
|
|
65
82
|
total: number;
|
|
@@ -70,6 +87,34 @@ export interface DbRowsPage {
|
|
|
70
87
|
filter: DbFilter | null;
|
|
71
88
|
}
|
|
72
89
|
|
|
90
|
+
export type DbExportCap = 'rows' | 'bytes' | null;
|
|
91
|
+
|
|
92
|
+
/** Bounded CSV generated from one SQLite read transaction. */
|
|
93
|
+
export interface DbCsvExport {
|
|
94
|
+
table: string;
|
|
95
|
+
content: Uint8Array<ArrayBuffer>;
|
|
96
|
+
rowCount: number;
|
|
97
|
+
bytes: number;
|
|
98
|
+
cap: DbExportCap;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/** Tests may lower a cap, but callers can never raise the production ceiling. */
|
|
102
|
+
export interface DbExportLimits {
|
|
103
|
+
maxRows?: number;
|
|
104
|
+
maxBytes?: number;
|
|
105
|
+
/** Test synchronization only: runs after the transaction's first read pins its snapshot. */
|
|
106
|
+
onSnapshot?: () => void;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/** Test synchronization for proving that a browse page is one SQLite snapshot. */
|
|
110
|
+
export interface DbRowsOptions {
|
|
111
|
+
/** Runs after COUNT, while the same read transaction remains open for SELECT. */
|
|
112
|
+
afterCount?: () => void;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/** JSON/query-string-safe SQLite rowid. Unsafe integers are represented as decimal strings. */
|
|
116
|
+
export type DbRowId = number | string;
|
|
117
|
+
|
|
73
118
|
export interface DbColumnInfo {
|
|
74
119
|
name: string;
|
|
75
120
|
type: string;
|
|
@@ -165,6 +210,28 @@ function bigintCell(v: bigint): number | string {
|
|
|
165
210
|
: v.toString();
|
|
166
211
|
}
|
|
167
212
|
|
|
213
|
+
const MIN_SQLITE_INTEGER = -(1n << 63n);
|
|
214
|
+
const MAX_SQLITE_INTEGER = (1n << 63n) - 1n;
|
|
215
|
+
|
|
216
|
+
/**
|
|
217
|
+
* Validate a rowid without converting an unsafe decimal string through Number.
|
|
218
|
+
* Strings stay strings when bound; SQLite applies INTEGER affinity exactly.
|
|
219
|
+
*/
|
|
220
|
+
function boundRowid(rowid: DbRowId): DbRowId {
|
|
221
|
+
if (typeof rowid === 'number') {
|
|
222
|
+
if (!Number.isSafeInteger(rowid)) throw new Error('Invalid rowid');
|
|
223
|
+
return rowid;
|
|
224
|
+
}
|
|
225
|
+
if (rowid.length === 0 || rowid.length > 20 || !/^-?\d+$/.test(rowid)) {
|
|
226
|
+
throw new Error('Invalid rowid');
|
|
227
|
+
}
|
|
228
|
+
const integer = BigInt(rowid);
|
|
229
|
+
if (integer < MIN_SQLITE_INTEGER || integer > MAX_SQLITE_INTEGER) {
|
|
230
|
+
throw new Error('Invalid rowid');
|
|
231
|
+
}
|
|
232
|
+
return rowid;
|
|
233
|
+
}
|
|
234
|
+
|
|
168
235
|
/**
|
|
169
236
|
* bun:sqlite exposes Statement.safeIntegers() at runtime but does not type it.
|
|
170
237
|
* Enabled per statement (never connection-wide) so the COUNT(*) reads that are
|
|
@@ -225,6 +292,188 @@ function buildFilter(
|
|
|
225
292
|
};
|
|
226
293
|
}
|
|
227
294
|
|
|
295
|
+
function boundedExportLimit(value: number | undefined, hardCap: number, label: string): number {
|
|
296
|
+
if (value === undefined) return hardCap;
|
|
297
|
+
if (!Number.isSafeInteger(value) || value < 1) {
|
|
298
|
+
throw new Error(`${label} must be a positive safe integer`);
|
|
299
|
+
}
|
|
300
|
+
return Math.min(value, hardCap);
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
/** RFC-4180 cell with spreadsheet-formula neutralization for textual values. */
|
|
304
|
+
function csvCell(value: unknown): string {
|
|
305
|
+
let text: string;
|
|
306
|
+
let textual = false;
|
|
307
|
+
if (value == null) text = '';
|
|
308
|
+
else if (typeof value === 'string') {
|
|
309
|
+
text = value;
|
|
310
|
+
textual = true;
|
|
311
|
+
} else if (typeof value === 'bigint' || typeof value === 'number' || typeof value === 'boolean') {
|
|
312
|
+
text = String(value);
|
|
313
|
+
} else if (value instanceof Uint8Array) {
|
|
314
|
+
// The SELECT below converts BLOBs before they cross into JS. Keep this
|
|
315
|
+
// fallback bounded and deterministic if SQLite changes the expression type.
|
|
316
|
+
text = `<blob ${value.byteLength} B>`;
|
|
317
|
+
textual = true;
|
|
318
|
+
} else {
|
|
319
|
+
throw new Error('Database export produced an unsupported SQLite value');
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
// Excel and Sheets may execute strings beginning with these characters. A
|
|
323
|
+
// leading TAB/CR is dangerous too because spreadsheet import strips it first.
|
|
324
|
+
const safe = textual && /^[=+\-@\t\r]/.test(text) ? `'${text}` : text;
|
|
325
|
+
return /[",\r\n]/.test(safe) ? `"${safe.replaceAll('"', '""')}"` : safe;
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
/**
|
|
329
|
+
* Export one filtered/sorted table view from a single SQLite snapshot.
|
|
330
|
+
*
|
|
331
|
+
* The readonly connection and explicit deferred transaction pin sqlite_master,
|
|
332
|
+
* schema, filter and row iteration to one WAL snapshot. Unlike client-side
|
|
333
|
+
* OFFSET pagination, concurrent inserts/deletes cannot duplicate or omit rows.
|
|
334
|
+
* The cursor is streamed into a fixed-size buffer and probes at most one row
|
|
335
|
+
* beyond the row cap, so both retained output and cardinality are bounded.
|
|
336
|
+
*/
|
|
337
|
+
export function dbExportCsv(
|
|
338
|
+
path: string,
|
|
339
|
+
table: string,
|
|
340
|
+
orderBy?: string,
|
|
341
|
+
dir: 'asc' | 'desc' = 'asc',
|
|
342
|
+
filter?: DbFilter,
|
|
343
|
+
limits: Readonly<DbExportLimits> = {}
|
|
344
|
+
): DbCsvExport {
|
|
345
|
+
if (typeof table !== 'string' || table.length === 0) throw new Error('Table is required');
|
|
346
|
+
if (table.length > DB_EXPORT_MAX_IDENTIFIER_CHARS) {
|
|
347
|
+
throw new Error(`Table name exceeds ${DB_EXPORT_MAX_IDENTIFIER_CHARS} characters`);
|
|
348
|
+
}
|
|
349
|
+
if (dir !== 'asc' && dir !== 'desc') throw new Error('Invalid export sort direction');
|
|
350
|
+
if (filter) {
|
|
351
|
+
if (
|
|
352
|
+
typeof filter.column !== 'string' ||
|
|
353
|
+
!filter.column ||
|
|
354
|
+
(filter.op !== 'contains' && filter.op !== 'eq' && filter.op !== 'ne') ||
|
|
355
|
+
typeof filter.value !== 'string'
|
|
356
|
+
) {
|
|
357
|
+
throw new Error('Invalid database export filter');
|
|
358
|
+
}
|
|
359
|
+
if (filter.column.length > DB_EXPORT_MAX_IDENTIFIER_CHARS) {
|
|
360
|
+
throw new Error(`Filter column exceeds ${DB_EXPORT_MAX_IDENTIFIER_CHARS} characters`);
|
|
361
|
+
}
|
|
362
|
+
if (filter.value.length > DB_EXPORT_MAX_FILTER_CHARS) {
|
|
363
|
+
throw new Error(
|
|
364
|
+
`Database export filter exceeds ${DB_EXPORT_MAX_FILTER_CHARS} characters`
|
|
365
|
+
);
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
const maxRows = boundedExportLimit(limits.maxRows, DB_EXPORT_MAX_ROWS, 'Export row limit');
|
|
370
|
+
const maxBytes = boundedExportLimit(limits.maxBytes, DB_EXPORT_MAX_BYTES, 'Export byte limit');
|
|
371
|
+
const db = open(path);
|
|
372
|
+
try {
|
|
373
|
+
const snapshot = db.transaction((): DbCsvExport => {
|
|
374
|
+
const name = knownTable(db, table);
|
|
375
|
+
const columns = (
|
|
376
|
+
db.query(`PRAGMA table_info(${ident(name)})`).all() as { name: string }[]
|
|
377
|
+
).map((column) => column.name);
|
|
378
|
+
if (columns.length === 0) throw new Error(`Table has no exportable columns: ${name}`);
|
|
379
|
+
if (columns.length > DB_EXPORT_MAX_COLUMNS) {
|
|
380
|
+
throw new Error(
|
|
381
|
+
`Table has ${columns.length} columns; database export supports at most ${DB_EXPORT_MAX_COLUMNS}`
|
|
382
|
+
);
|
|
383
|
+
}
|
|
384
|
+
if (columns.some((column) => column.length > DB_EXPORT_MAX_IDENTIFIER_CHARS)) {
|
|
385
|
+
throw new Error(
|
|
386
|
+
`Database export column names must not exceed ${DB_EXPORT_MAX_IDENTIFIER_CHARS} characters`
|
|
387
|
+
);
|
|
388
|
+
}
|
|
389
|
+
limits.onSnapshot?.();
|
|
390
|
+
|
|
391
|
+
let order = '';
|
|
392
|
+
if (orderBy !== undefined) {
|
|
393
|
+
if (typeof orderBy !== 'string' || !orderBy) {
|
|
394
|
+
throw new Error('Invalid database export sort column');
|
|
395
|
+
}
|
|
396
|
+
if (orderBy.length > DB_EXPORT_MAX_IDENTIFIER_CHARS) {
|
|
397
|
+
throw new Error(`Sort column exceeds ${DB_EXPORT_MAX_IDENTIFIER_CHARS} characters`);
|
|
398
|
+
}
|
|
399
|
+
const sortColumn = columns.find((column) => column === orderBy);
|
|
400
|
+
if (!sortColumn) throw new Error(`No such column: ${orderBy}`);
|
|
401
|
+
order = ` ORDER BY ${ident(sortColumn)} ${dir === 'desc' ? 'DESC' : 'ASC'}`;
|
|
402
|
+
}
|
|
403
|
+
const { clause, params } = buildFilter(columns, filter);
|
|
404
|
+
|
|
405
|
+
// Generated aliases avoid object-property hazards for legal SQLite column
|
|
406
|
+
// names such as "__proto__". Text and BLOB expressions ensure one cell
|
|
407
|
+
// cannot materialize an unbounded JS value before the byte cap is checked.
|
|
408
|
+
const aliases = columns.map((_, index) => `__bq_export_${index}`);
|
|
409
|
+
const selected = columns.map((column, index) => {
|
|
410
|
+
const quoted = ident(column);
|
|
411
|
+
return (
|
|
412
|
+
`CASE ` +
|
|
413
|
+
`WHEN typeof(${quoted}) = 'blob' THEN '<blob ' || length(${quoted}) || ' B>' ` +
|
|
414
|
+
`WHEN typeof(${quoted}) = 'text' AND length(CAST(${quoted} AS BLOB)) > ${MAX_CELL} ` +
|
|
415
|
+
`THEN substr(${quoted}, 1, ${MAX_CELL}) || '…' ` +
|
|
416
|
+
`ELSE ${quoted} END AS ${ident(aliases[index])}`
|
|
417
|
+
);
|
|
418
|
+
});
|
|
419
|
+
const stmt = db.query<Record<string, unknown>, Array<string | number>>(
|
|
420
|
+
`SELECT ${selected.join(', ')} FROM ${ident(name)}${clause}${order} LIMIT ?`
|
|
421
|
+
);
|
|
422
|
+
safeIntegers(stmt);
|
|
423
|
+
|
|
424
|
+
const encoder = new TextEncoder();
|
|
425
|
+
const buffer = new Uint8Array(maxBytes);
|
|
426
|
+
let used = 0;
|
|
427
|
+
const append = (text: string): boolean => {
|
|
428
|
+
const result = encoder.encodeInto(text, buffer.subarray(used));
|
|
429
|
+
if (result.read !== text.length) return false;
|
|
430
|
+
used += result.written;
|
|
431
|
+
return true;
|
|
432
|
+
};
|
|
433
|
+
if (!append(columns.map(csvCell).join(','))) {
|
|
434
|
+
throw new Error(`Database export header exceeds the ${maxBytes}-byte export limit`);
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
let rowCount = 0;
|
|
438
|
+
let cap: DbExportCap = null;
|
|
439
|
+
const bindings = [...params, maxRows + 1] as Array<string | number>;
|
|
440
|
+
const iterator = stmt.iterate(...bindings);
|
|
441
|
+
try {
|
|
442
|
+
for (;;) {
|
|
443
|
+
const next = iterator.next();
|
|
444
|
+
if (next.done) break;
|
|
445
|
+
if (rowCount >= maxRows) {
|
|
446
|
+
cap = 'rows';
|
|
447
|
+
break;
|
|
448
|
+
}
|
|
449
|
+
const line = `\r\n${aliases.map((alias) => csvCell(next.value[alias])).join(',')}`;
|
|
450
|
+
if (!append(line)) {
|
|
451
|
+
cap = 'bytes';
|
|
452
|
+
break;
|
|
453
|
+
}
|
|
454
|
+
rowCount++;
|
|
455
|
+
}
|
|
456
|
+
} finally {
|
|
457
|
+
iterator.return?.();
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
// Slice releases the unused portion of the fixed working buffer before
|
|
461
|
+
// the Response is retained by the HTTP server. Peak memory remains a
|
|
462
|
+
// small constant multiple of DB_EXPORT_MAX_BYTES, never table size.
|
|
463
|
+
const content = buffer.slice(0, used);
|
|
464
|
+
return { table: name, content, rowCount, bytes: content.byteLength, cap };
|
|
465
|
+
});
|
|
466
|
+
// On a readonly connection BEGIN DEFERRED is a read transaction. Bun's
|
|
467
|
+
// wrapper commits on success and rolls back on every thrown validation,
|
|
468
|
+
// cursor or encoding error.
|
|
469
|
+
return snapshot.deferred();
|
|
470
|
+
} catch (error) {
|
|
471
|
+
throw translateError(error);
|
|
472
|
+
} finally {
|
|
473
|
+
db.close();
|
|
474
|
+
}
|
|
475
|
+
}
|
|
476
|
+
|
|
228
477
|
/**
|
|
229
478
|
* One page of a table's rows. Table, orderBy and filter-column identifiers are
|
|
230
479
|
* validated against sqlite_master / table_info — never interpolated from raw
|
|
@@ -237,7 +486,8 @@ export function dbRows(
|
|
|
237
486
|
offset: number,
|
|
238
487
|
orderBy?: string,
|
|
239
488
|
dir: 'asc' | 'desc' = 'asc',
|
|
240
|
-
filter?: DbFilter
|
|
489
|
+
filter?: DbFilter,
|
|
490
|
+
options: Readonly<DbRowsOptions> = {}
|
|
241
491
|
): DbRowsPage {
|
|
242
492
|
const lim = Math.max(1, Math.min(MAX_ROWS, Math.floor(limit) || 50));
|
|
243
493
|
// Non-finite / out-of-range offsets must be clamped here: SQLite rejects a
|
|
@@ -247,67 +497,83 @@ export function dbRows(
|
|
|
247
497
|
: 0;
|
|
248
498
|
const db = open(path);
|
|
249
499
|
try {
|
|
250
|
-
const
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
500
|
+
const snapshot = db.transaction((): DbRowsPage => {
|
|
501
|
+
const name = knownTable(db, table);
|
|
502
|
+
const columns = (
|
|
503
|
+
db.query(`PRAGMA table_info(${ident(name)})`).all() as { name: string }[]
|
|
504
|
+
).map((c) => c.name);
|
|
254
505
|
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
506
|
+
let order = '';
|
|
507
|
+
let sortCol: string | null = null;
|
|
508
|
+
if (orderBy) {
|
|
509
|
+
sortCol = columns.find((c) => c === orderBy) ?? null;
|
|
510
|
+
if (!sortCol) throw new Error(`No such column: ${orderBy}`);
|
|
511
|
+
order = ` ORDER BY ${ident(sortCol)} ${dir === 'desc' ? 'DESC' : 'ASC'}`;
|
|
512
|
+
}
|
|
262
513
|
|
|
263
|
-
|
|
514
|
+
const { clause, params, applied } = buildFilter(columns, filter);
|
|
264
515
|
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
516
|
+
const { c: total } = db
|
|
517
|
+
.query(`SELECT COUNT(*) AS c FROM ${ident(name)}${clause}`)
|
|
518
|
+
.get(...(params as [])) as { c: number };
|
|
519
|
+
options.afterCount?.();
|
|
268
520
|
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
521
|
+
// Prefer selecting rowid so the detail view can fetch untruncated cells;
|
|
522
|
+
// WITHOUT ROWID tables have no rowid column — fall back to a plain select.
|
|
523
|
+
let rowids: (DbRowId | null)[] = [];
|
|
524
|
+
let rawRows: unknown[][];
|
|
525
|
+
try {
|
|
526
|
+
const stmt = db.query(
|
|
527
|
+
`SELECT rowid AS __rid, * FROM ${ident(name)}${clause}${order} LIMIT ? OFFSET ?`
|
|
528
|
+
);
|
|
529
|
+
safeIntegers(stmt);
|
|
530
|
+
const withRowid = stmt.values(...(params as []), lim, off) as unknown[][];
|
|
531
|
+
rowids = withRowid.map((r) =>
|
|
532
|
+
typeof r[0] === 'bigint'
|
|
533
|
+
? bigintCell(r[0])
|
|
534
|
+
: typeof r[0] === 'number' && Number.isSafeInteger(r[0])
|
|
535
|
+
? r[0]
|
|
536
|
+
: null
|
|
537
|
+
);
|
|
538
|
+
rawRows = withRowid.map((r) => r.slice(1));
|
|
539
|
+
} catch {
|
|
540
|
+
const stmt = db.query(`SELECT * FROM ${ident(name)}${clause}${order} LIMIT ? OFFSET ?`);
|
|
541
|
+
safeIntegers(stmt);
|
|
542
|
+
rawRows = stmt.values(...(params as []), lim, off) as unknown[][];
|
|
543
|
+
rowids = rawRows.map(() => null);
|
|
544
|
+
}
|
|
289
545
|
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
546
|
+
const truncatedCells = rawRows.map((r) => r.map(isTruncated));
|
|
547
|
+
return {
|
|
548
|
+
table: name,
|
|
549
|
+
columns,
|
|
550
|
+
rows: sanitize(rawRows),
|
|
551
|
+
rowids,
|
|
552
|
+
truncatedCells,
|
|
553
|
+
total,
|
|
554
|
+
limit: lim,
|
|
555
|
+
offset: off,
|
|
556
|
+
orderBy: sortCol,
|
|
557
|
+
dir: dir === 'desc' ? 'desc' : 'asc',
|
|
558
|
+
filter: applied,
|
|
559
|
+
};
|
|
560
|
+
});
|
|
561
|
+
// COUNT and page SELECT must describe the same live view. Without this read
|
|
562
|
+
// transaction, a concurrent WAL commit between them could make a valid
|
|
563
|
+
// response internally contradictory (or shift rows across page boundaries).
|
|
564
|
+
return snapshot.deferred();
|
|
304
565
|
} finally {
|
|
305
566
|
db.close();
|
|
306
567
|
}
|
|
307
568
|
}
|
|
308
569
|
|
|
309
570
|
/** Full, untruncated value of a single cell, keyed by rowid (for the detail view). */
|
|
310
|
-
export function dbCell(
|
|
571
|
+
export function dbCell(
|
|
572
|
+
path: string,
|
|
573
|
+
table: string,
|
|
574
|
+
rowid: DbRowId,
|
|
575
|
+
column: string
|
|
576
|
+
): { value: unknown } {
|
|
311
577
|
const db = open(path);
|
|
312
578
|
try {
|
|
313
579
|
const name = knownTable(db, table);
|
|
@@ -318,7 +584,7 @@ export function dbCell(path: string, table: string, rowid: number, column: strin
|
|
|
318
584
|
if (!col) throw new Error(`No such column: ${column}`);
|
|
319
585
|
const stmt = db.query(`SELECT ${ident(col)} AS v FROM ${ident(name)} WHERE rowid = ?`);
|
|
320
586
|
safeIntegers(stmt);
|
|
321
|
-
const row = stmt.get(rowid) as { v: unknown } | null;
|
|
587
|
+
const row = stmt.get(boundRowid(rowid)) as { v: unknown } | null;
|
|
322
588
|
if (!row) throw new Error('Row not found');
|
|
323
589
|
let v = row.v;
|
|
324
590
|
if (v instanceof Uint8Array) v = `<blob ${v.byteLength} B — binary, not shown>`;
|
|
@@ -517,16 +783,16 @@ function translateError(e: unknown): Error {
|
|
|
517
783
|
}
|
|
518
784
|
|
|
519
785
|
/**
|
|
520
|
-
*
|
|
521
|
-
*
|
|
522
|
-
*
|
|
786
|
+
* Shared URL of the disposable database worker. It accepts both arbitrary
|
|
787
|
+
* query and atomic CSV-export messages; scripts/serve.ts points this at the
|
|
788
|
+
* secondary embedded entrypoint in compiled builds.
|
|
523
789
|
*
|
|
524
790
|
* The timeout aborts the RESPONSE, not the query: terminate() is delivered to
|
|
525
791
|
* the worker's event loop, which a synchronous sqlite3_step never yields back
|
|
526
792
|
* to, so a runaway scan keeps burning its own thread until SQLite finishes it.
|
|
527
793
|
* That thread is therefore accounted for (a slot is held until the worker really
|
|
528
|
-
* exits) and capped by
|
|
529
|
-
*
|
|
794
|
+
* exits) and capped by the respective query/export pool, so abandoned work can
|
|
795
|
+
* pin only the configured number of cores instead of one new core per request.
|
|
530
796
|
*/
|
|
531
797
|
let queryWorkerUrl = new URL('./dbQueryWorker.ts', import.meta.url).href;
|
|
532
798
|
|
|
@@ -560,6 +826,213 @@ export function setQueryWorkerFactory(factory: (() => Worker) | null): void {
|
|
|
560
826
|
queryWorkerFactory = factory ?? spawnRealWorker;
|
|
561
827
|
}
|
|
562
828
|
|
|
829
|
+
/** Only one potentially expensive table scan/sort may run at a time. */
|
|
830
|
+
export const MAX_CONCURRENT_EXPORTS = 1;
|
|
831
|
+
let liveExportWorkers = 0;
|
|
832
|
+
|
|
833
|
+
/** Export workers alive now, including timed-out/aborted SQLite calls still unwinding. */
|
|
834
|
+
export const exportWorkerLoad = (): number => liveExportWorkers;
|
|
835
|
+
|
|
836
|
+
const spawnRealExportWorker = (): Worker => new Worker(queryWorkerUrl, { type: 'module' });
|
|
837
|
+
let exportWorkerFactory: () => Worker = spawnRealExportWorker;
|
|
838
|
+
|
|
839
|
+
/** Test-only export-worker factory; pass null to restore the real worker. */
|
|
840
|
+
export function setExportWorkerFactory(factory: (() => Worker) | null): void {
|
|
841
|
+
exportWorkerFactory = factory ?? spawnRealExportWorker;
|
|
842
|
+
}
|
|
843
|
+
|
|
844
|
+
function validatedWorkerExport(value: unknown, expectedTable: string): DbCsvExport {
|
|
845
|
+
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
|
846
|
+
throw new DbExportUnavailableError('Database export worker returned a malformed result');
|
|
847
|
+
}
|
|
848
|
+
const result = value as Record<string, unknown>;
|
|
849
|
+
if (
|
|
850
|
+
typeof result.table !== 'string' ||
|
|
851
|
+
result.table !== expectedTable ||
|
|
852
|
+
!(result.content instanceof Uint8Array) ||
|
|
853
|
+
!Number.isSafeInteger(result.rowCount) ||
|
|
854
|
+
(result.rowCount as number) < 0 ||
|
|
855
|
+
(result.rowCount as number) > DB_EXPORT_MAX_ROWS ||
|
|
856
|
+
!Number.isSafeInteger(result.bytes) ||
|
|
857
|
+
(result.bytes as number) < 1 ||
|
|
858
|
+
(result.bytes as number) > DB_EXPORT_MAX_BYTES ||
|
|
859
|
+
result.content.byteLength !== result.bytes ||
|
|
860
|
+
(result.cap !== null && result.cap !== 'rows' && result.cap !== 'bytes') ||
|
|
861
|
+
(result.cap === 'rows' && result.rowCount !== DB_EXPORT_MAX_ROWS)
|
|
862
|
+
) {
|
|
863
|
+
throw new DbExportUnavailableError('Database export worker returned a malformed result');
|
|
864
|
+
}
|
|
865
|
+
const content =
|
|
866
|
+
result.content.buffer instanceof ArrayBuffer
|
|
867
|
+
? (result.content as Uint8Array<ArrayBuffer>)
|
|
868
|
+
: Uint8Array.from(result.content);
|
|
869
|
+
return {
|
|
870
|
+
table: result.table as string,
|
|
871
|
+
content,
|
|
872
|
+
rowCount: result.rowCount as number,
|
|
873
|
+
bytes: result.bytes as number,
|
|
874
|
+
cap: result.cap as DbExportCap,
|
|
875
|
+
};
|
|
876
|
+
}
|
|
877
|
+
|
|
878
|
+
/**
|
|
879
|
+
* Run an atomic CSV export off the control-agent event loop.
|
|
880
|
+
*
|
|
881
|
+
* A timeout or caller abort abandons the response, not sqlite3_step: the Worker
|
|
882
|
+
* keeps its only slot until `close`, preventing repeated abandoned scans from
|
|
883
|
+
* pinning additional cores. Row/cell/byte caps remain enforced inside
|
|
884
|
+
* dbExportCsv before its ArrayBuffer is transferred back without JSON expansion.
|
|
885
|
+
*/
|
|
886
|
+
export async function exportWithTimeout(
|
|
887
|
+
path: string,
|
|
888
|
+
table: string,
|
|
889
|
+
orderBy?: string,
|
|
890
|
+
dir: 'asc' | 'desc' = 'asc',
|
|
891
|
+
filter?: DbFilter,
|
|
892
|
+
signal?: AbortSignal
|
|
893
|
+
): Promise<DbCsvExport> {
|
|
894
|
+
signal?.throwIfAborted();
|
|
895
|
+
if (liveExportWorkers >= MAX_CONCURRENT_EXPORTS) {
|
|
896
|
+
throw new DbExportBusyError(
|
|
897
|
+
'A database export is already running. Wait for it to finish, or restart the agent if it was abandoned.'
|
|
898
|
+
);
|
|
899
|
+
}
|
|
900
|
+
|
|
901
|
+
let worker: Worker;
|
|
902
|
+
try {
|
|
903
|
+
worker = exportWorkerFactory();
|
|
904
|
+
} catch (error) {
|
|
905
|
+
throw new DbExportUnavailableError(
|
|
906
|
+
`Database export worker unavailable: ${(error as Error).message ?? String(error)}`
|
|
907
|
+
);
|
|
908
|
+
}
|
|
909
|
+
liveExportWorkers++;
|
|
910
|
+
let released = false;
|
|
911
|
+
const release = () => {
|
|
912
|
+
if (released) return;
|
|
913
|
+
released = true;
|
|
914
|
+
liveExportWorkers--;
|
|
915
|
+
};
|
|
916
|
+
|
|
917
|
+
try {
|
|
918
|
+
return await new Promise<DbCsvExport>((resolve, reject) => {
|
|
919
|
+
let settled = false;
|
|
920
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
921
|
+
const cleanup = () => {
|
|
922
|
+
if (timer !== undefined) clearTimeout(timer);
|
|
923
|
+
signal?.removeEventListener('abort', onAbort);
|
|
924
|
+
};
|
|
925
|
+
const abandon = (error: Error) => {
|
|
926
|
+
if (settled) return;
|
|
927
|
+
settled = true;
|
|
928
|
+
cleanup();
|
|
929
|
+
worker.terminate();
|
|
930
|
+
// Do NOT release here. A synchronous sqlite3_step may still be running;
|
|
931
|
+
// the close event is the proof that its thread has really exited.
|
|
932
|
+
reject(error);
|
|
933
|
+
};
|
|
934
|
+
const onAbort = () =>
|
|
935
|
+
abandon(
|
|
936
|
+
signal?.reason instanceof Error
|
|
937
|
+
? signal.reason
|
|
938
|
+
: new Error('Database export request was aborted')
|
|
939
|
+
);
|
|
940
|
+
|
|
941
|
+
worker.addEventListener('close', () => {
|
|
942
|
+
release();
|
|
943
|
+
if (!settled) {
|
|
944
|
+
settled = true;
|
|
945
|
+
cleanup();
|
|
946
|
+
reject(new DbExportUnavailableError('Database export worker exited unexpectedly'));
|
|
947
|
+
}
|
|
948
|
+
});
|
|
949
|
+
worker.addEventListener('message', (event: MessageEvent) => {
|
|
950
|
+
if (settled) return;
|
|
951
|
+
settled = true;
|
|
952
|
+
cleanup();
|
|
953
|
+
release();
|
|
954
|
+
const data = event.data as
|
|
955
|
+
| {
|
|
956
|
+
ok?: unknown;
|
|
957
|
+
export?: unknown;
|
|
958
|
+
error?: unknown;
|
|
959
|
+
missing?: unknown;
|
|
960
|
+
}
|
|
961
|
+
| null;
|
|
962
|
+
if (!data || typeof data !== 'object' || Array.isArray(data)) {
|
|
963
|
+
reject(
|
|
964
|
+
new DbExportUnavailableError('Database export worker returned a malformed response')
|
|
965
|
+
);
|
|
966
|
+
return;
|
|
967
|
+
}
|
|
968
|
+
const response: {
|
|
969
|
+
ok?: unknown;
|
|
970
|
+
export?: unknown;
|
|
971
|
+
error?: unknown;
|
|
972
|
+
missing?: unknown;
|
|
973
|
+
} = data;
|
|
974
|
+
if (response.ok === true) {
|
|
975
|
+
try {
|
|
976
|
+
resolve(validatedWorkerExport(response.export, table));
|
|
977
|
+
} catch (error) {
|
|
978
|
+
reject(error);
|
|
979
|
+
}
|
|
980
|
+
} else if (response.missing === true) {
|
|
981
|
+
reject(
|
|
982
|
+
new MissingDbError(
|
|
983
|
+
typeof response.error === 'string' ? response.error : 'Database not found'
|
|
984
|
+
)
|
|
985
|
+
);
|
|
986
|
+
} else {
|
|
987
|
+
reject(
|
|
988
|
+
new Error(
|
|
989
|
+
typeof response.error === 'string' ? response.error : 'Database export failed'
|
|
990
|
+
)
|
|
991
|
+
);
|
|
992
|
+
}
|
|
993
|
+
});
|
|
994
|
+
worker.addEventListener('error', (event: ErrorEvent) => {
|
|
995
|
+
if (settled) return;
|
|
996
|
+
event.preventDefault?.();
|
|
997
|
+
abandon(
|
|
998
|
+
new DbExportUnavailableError(
|
|
999
|
+
`Database export worker failed: ${event.message || 'unknown worker error'}`
|
|
1000
|
+
)
|
|
1001
|
+
);
|
|
1002
|
+
});
|
|
1003
|
+
|
|
1004
|
+
timer = setTimeout(
|
|
1005
|
+
() =>
|
|
1006
|
+
abandon(
|
|
1007
|
+
new DbExportUnavailableError(
|
|
1008
|
+
`Database export exceeded the ${DB_EXPORT_TIMEOUT_MS / 1000}s time limit and was abandoned (SQLite may keep scanning until the worker exits).`
|
|
1009
|
+
)
|
|
1010
|
+
),
|
|
1011
|
+
DB_EXPORT_TIMEOUT_MS
|
|
1012
|
+
);
|
|
1013
|
+
signal?.addEventListener('abort', onAbort, { once: true });
|
|
1014
|
+
if (signal?.aborted) {
|
|
1015
|
+
onAbort();
|
|
1016
|
+
return;
|
|
1017
|
+
}
|
|
1018
|
+
try {
|
|
1019
|
+
worker.postMessage({ kind: 'export', path, table, orderBy, dir, filter });
|
|
1020
|
+
} catch (error) {
|
|
1021
|
+
settled = true;
|
|
1022
|
+
cleanup();
|
|
1023
|
+
release();
|
|
1024
|
+
reject(
|
|
1025
|
+
new DbExportUnavailableError(
|
|
1026
|
+
`Could not start database export: ${(error as Error).message ?? String(error)}`
|
|
1027
|
+
)
|
|
1028
|
+
);
|
|
1029
|
+
}
|
|
1030
|
+
});
|
|
1031
|
+
} finally {
|
|
1032
|
+
worker.terminate();
|
|
1033
|
+
}
|
|
1034
|
+
}
|
|
1035
|
+
|
|
563
1036
|
export async function queryWithTimeout(path: string, sql: string): Promise<DbQueryResult> {
|
|
564
1037
|
if (liveQueryWorkers >= MAX_CONCURRENT_QUERIES) {
|
|
565
1038
|
throw new Error(
|