bunqueue-dashboard 0.0.31 → 0.0.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +44 -15
- package/agent/db.ts +682 -68
- package/agent/dbQueryWorker.ts +41 -10
- package/agent/index.ts +4 -1
- package/agent/manager.ts +267 -41
- package/agent/server.ts +131 -36
- package/dist/.vite/manifest.json +215 -159
- package/dist/assets/AddJob-ytpBLTCJ.js +3 -0
- package/dist/assets/Alerts-C39RrGbQ.js +1 -0
- package/dist/assets/{AreaChart-2gM0YoZT.js → AreaChart-REK4ZsTC.js} +1 -1
- package/dist/assets/Benchmark-CM60tHbf.js +1 -0
- package/dist/assets/BulkAddJobs-BjL-xcQy.js +5 -0
- package/dist/assets/CopilotPanel-BGK1rw-V.js +44 -0
- package/dist/assets/Cron-s1e-jgOo.js +1 -0
- package/dist/assets/CronManager-BostU7BX.js +1 -0
- package/dist/assets/Database-ZluStWMz.js +3 -0
- package/dist/assets/Diagnostics-C0N5z5j-.js +1 -0
- package/dist/assets/Dlq-CtcLzs-b.js +1 -0
- package/dist/assets/DlqControl-B_Z8KKn5.js +1 -0
- package/dist/assets/DlqPro-B68en6AG.js +1 -0
- package/dist/assets/Flows-CD_bkCBM.js +1 -0
- package/dist/assets/JobInspector-DwRtyMz0.js +2 -0
- package/dist/assets/Jobs-BUtcBKKk.js +1 -0
- package/dist/assets/JobsPro-BK_xd4ji.js +1 -0
- package/dist/assets/Logs-CmVODYCI.js +1 -0
- package/dist/assets/LogsPro-CJ-Il3LP.js +1 -0
- package/dist/assets/McpServer-dMjyJKgT.js +22 -0
- package/dist/assets/Metrics-Dsbeplz-.js +1 -0
- package/dist/assets/MetricsPro-BqxurxrJ.js +1 -0
- package/dist/assets/Overview-DH9C2dsf.js +1 -0
- package/dist/assets/{Pagination-D4ki-pGz.js → Pagination-DpxiBnt8.js} +1 -1
- package/dist/assets/QueueActions-D_QGBaVm.js +1 -0
- package/dist/assets/QueueControl-V0DJ6qk6.js +1 -0
- package/dist/assets/QueueDetail-BgKAMJqf.js +1 -0
- package/dist/assets/QueueDetailPro-BkHYzGiq.js +1 -0
- package/dist/assets/Queues-2_v7DcNi.js +1 -0
- package/dist/assets/QueuesOverview-BySSkmB4.js +1 -0
- package/dist/assets/S3Backup-CPADzKQE.js +1 -0
- package/dist/assets/S3BackupPro-CSrQYIDL.js +2 -0
- package/dist/assets/ServerControl-C0tI1VDx.js +3 -0
- package/dist/assets/Settings-CSzoM5up.js +1 -0
- package/dist/assets/Usage-B4B7tmrc.js +1 -0
- package/dist/assets/UsagePro-CCBpgSDy.js +1 -0
- package/dist/assets/Webhooks-aJz-GisQ.js +1 -0
- package/dist/assets/Workers-Bg8xKjkg.js +1 -0
- package/dist/assets/WorkersPro-BOOCrXf8.js +1 -0
- package/dist/assets/connectionStore-Cd9-3Fw0.js +1 -0
- package/dist/assets/{exportFile-tdUeW5m5.js → exportFile-C7xhE8Yh.js} +1 -1
- package/dist/assets/flowMutationSafety-DVD1IMr-.js +1 -0
- package/dist/assets/form-DsUq0w8s.js +1 -0
- package/dist/assets/index-BWoUj86M.css +1 -0
- package/dist/assets/index-Bn2R06qm.js +8 -0
- package/dist/assets/{install-DhTYJCUC.js → install-CUS8Kijb.js} +2 -2
- package/dist/assets/jobActions-LQD4KfCc.js +1 -0
- package/dist/assets/promisePool-BVfQ9n9Q.js +1 -0
- package/dist/assets/{react-vendor-BcouBxpZ.js → react-vendor-Bjui4wS4.js} +1 -1
- package/dist/assets/upstreamPaths-nG8N1XAz.js +1 -0
- package/dist/assets/useServerActionGuard-CItj3hvh.js +1 -0
- package/dist/assets/useThroughputSeries-tb26_lkb.js +1 -0
- package/dist/index.html +5 -4
- package/package.json +11 -3
- package/scripts/embedded.gen.ts +132 -126
- package/scripts/serve.ts +501 -108
- package/dist/assets/AddJob-CAhdg38n.js +0 -3
- package/dist/assets/Alerts-70bxf-PT.js +0 -1
- package/dist/assets/Benchmark-CFZ3WCvj.js +0 -1
- package/dist/assets/BulkAddJobs-BzgzpGg_.js +0 -5
- package/dist/assets/CopilotPanel-DZUGIf1q.js +0 -44
- package/dist/assets/Cron-ltjSoTjj.js +0 -1
- package/dist/assets/CronManager-BcFALDeT.js +0 -1
- package/dist/assets/Database-CV2yt5O2.js +0 -3
- package/dist/assets/Diagnostics-D-EK4GEi.js +0 -1
- package/dist/assets/Dlq-fZAKwQoC.js +0 -1
- package/dist/assets/DlqControl-eTrrQCrn.js +0 -1
- package/dist/assets/DlqPro-CF79339e.js +0 -1
- package/dist/assets/Flows-Ba8G7mcC.js +0 -1
- package/dist/assets/JobInspector--d_8ca9p.js +0 -2
- package/dist/assets/Jobs-DEqAc1A5.js +0 -1
- package/dist/assets/JobsPro-BcaokZIf.js +0 -1
- package/dist/assets/Logs-DBTukPIX.js +0 -1
- package/dist/assets/LogsPro-Wl_e6hZM.js +0 -1
- package/dist/assets/McpServer-CnWJsKw7.js +0 -22
- package/dist/assets/Metrics-DzeIhNL-.js +0 -1
- package/dist/assets/MetricsPro-O9eVJSH0.js +0 -1
- package/dist/assets/Overview-Bnu7rWqt.js +0 -1
- package/dist/assets/QueueActions-DHoSkWRz.js +0 -1
- package/dist/assets/QueueControl-DKqEXnkM.js +0 -1
- package/dist/assets/QueueDetail-DLRow04B.js +0 -1
- package/dist/assets/QueueDetailPro-BYmOVd4T.js +0 -1
- package/dist/assets/Queues-CoyjRFN7.js +0 -1
- package/dist/assets/QueuesOverview-BeWdJpKi.js +0 -1
- package/dist/assets/S3Backup-BaLFgl9d.js +0 -1
- package/dist/assets/S3BackupPro-DJB4oWgz.js +0 -1
- package/dist/assets/ServerControl-_0lVB33E.js +0 -3
- package/dist/assets/Settings-zKFJTtnq.js +0 -1
- package/dist/assets/Usage-CKGkvl0j.js +0 -1
- package/dist/assets/UsagePro-B-TJ6fp4.js +0 -1
- package/dist/assets/Webhooks-C3zmvY-_.js +0 -1
- package/dist/assets/Workers-jKhhXl3W.js +0 -1
- package/dist/assets/WorkersPro-mvY5VX8I.js +0 -1
- package/dist/assets/connectionStore-Ci04lJmR.js +0 -1
- package/dist/assets/form-DGinTEEb.js +0 -1
- package/dist/assets/index-BOcegTHa.js +0 -8
- package/dist/assets/index-DWV3tgrk.css +0 -1
- package/dist/assets/jobActions-CuOdMglt.js +0 -1
- package/dist/assets/promisePool-Daxkc1OB.js +0 -1
- package/dist/assets/useThroughputSeries-Chp5OLx3.js +0 -1
package/agent/db.ts
CHANGED
|
@@ -9,14 +9,20 @@
|
|
|
9
9
|
*
|
|
10
10
|
* Defense in depth for the arbitrary-query endpoint (dbQuery):
|
|
11
11
|
* - a positive statement allowlist rejects ATTACH/DETACH/CREATE-TEMP/etc.
|
|
12
|
-
*
|
|
13
|
-
*
|
|
12
|
+
* before execution — but it inspects only the FIRST keyword, so it does NOT
|
|
13
|
+
* stop a trailing statement smuggled in after a legal one. Everything here
|
|
14
|
+
* must therefore compile SQL through db.query() (one statement, the rest
|
|
15
|
+
* inert) and never db.run() (executes them all). Do not weaken this comment
|
|
16
|
+
* into "the allowlist makes the store safe": believing that is what once
|
|
17
|
+
* turned the dup-column recovery path into an arbitrary file write;
|
|
14
18
|
* - results are streamed with an early break at MAX_ROWS, so a huge or
|
|
15
19
|
* row-bomb query can't materialize the whole set in agent memory;
|
|
16
20
|
* - queryWithTimeout() runs it in a disposable Worker raced against a wall
|
|
17
|
-
* clock, so an O(N^k) scan can't pin the agent's
|
|
18
|
-
* process-control endpoints
|
|
19
|
-
*
|
|
21
|
+
* clock, so an O(N^k) scan can't pin the agent's event loop and freeze the
|
|
22
|
+
* process-control endpoints. Worker startup failures surface instead of
|
|
23
|
+
* falling back to a synchronous scan. The timeout aborts the response, not
|
|
24
|
+
* the query — a synchronous sqlite3_step never yields to terminate() — so
|
|
25
|
+
* live worker threads are counted and capped by MAX_CONCURRENT_QUERIES.
|
|
20
26
|
*/
|
|
21
27
|
import { existsSync } from 'node:fs';
|
|
22
28
|
import { Database } from 'bun:sqlite';
|
|
@@ -27,6 +33,18 @@ export const MAX_ROWS = 500;
|
|
|
27
33
|
const MAX_CELL = 2000;
|
|
28
34
|
/** Full single-cell fetch (detail view) hard cap — bounds one pathological blob. */
|
|
29
35
|
const MAX_FULL_CELL = 1_000_000;
|
|
36
|
+
/** Full-table CSV export cap. The cursor probes one extra row to report truncation exactly. */
|
|
37
|
+
export const DB_EXPORT_MAX_ROWS = 200_000;
|
|
38
|
+
/** Maximum CSV response body. The exporter writes into one fixed-size buffer. */
|
|
39
|
+
export const DB_EXPORT_MAX_BYTES = 16 * 1024 * 1024;
|
|
40
|
+
/** Refuse pathological schemas before one row can create excessive transient allocations. */
|
|
41
|
+
export const DB_EXPORT_MAX_COLUMNS = 256;
|
|
42
|
+
/** Bound SQL/header construction even for a hostile or corrupt local schema. */
|
|
43
|
+
export const DB_EXPORT_MAX_IDENTIFIER_CHARS = 1024;
|
|
44
|
+
/** Bound a user-supplied filter before it is retained by SQLite for a long export. */
|
|
45
|
+
export const DB_EXPORT_MAX_FILTER_CHARS = 4096;
|
|
46
|
+
/** Response deadline; timed-out SQLite work remains isolated and accounted for in its Worker. */
|
|
47
|
+
export const DB_EXPORT_TIMEOUT_MS = 15_000;
|
|
30
48
|
/** Wall-clock budget for an arbitrary query before it is aborted. */
|
|
31
49
|
export const QUERY_TIMEOUT_MS = 5000;
|
|
32
50
|
|
|
@@ -35,6 +53,12 @@ export class MissingDbError extends Error {
|
|
|
35
53
|
readonly missing = true;
|
|
36
54
|
}
|
|
37
55
|
|
|
56
|
+
/** Maps a saturated export worker pool to HTTP 429. */
|
|
57
|
+
export class DbExportBusyError extends Error {}
|
|
58
|
+
|
|
59
|
+
/** Maps export worker startup/crash/timeout failures to HTTP 503. */
|
|
60
|
+
export class DbExportUnavailableError extends Error {}
|
|
61
|
+
|
|
38
62
|
export interface DbTableInfo {
|
|
39
63
|
name: string;
|
|
40
64
|
rows: number;
|
|
@@ -52,7 +76,7 @@ export interface DbRowsPage {
|
|
|
52
76
|
columns: string[];
|
|
53
77
|
rows: unknown[][];
|
|
54
78
|
/** rowid per row (null for WITHOUT ROWID tables) — keys the full-cell fetch. */
|
|
55
|
-
rowids: (
|
|
79
|
+
rowids: (DbRowId | null)[];
|
|
56
80
|
/** Which cells were truncated server-side, so the UI can flag them. */
|
|
57
81
|
truncatedCells: boolean[][];
|
|
58
82
|
total: number;
|
|
@@ -63,6 +87,34 @@ export interface DbRowsPage {
|
|
|
63
87
|
filter: DbFilter | null;
|
|
64
88
|
}
|
|
65
89
|
|
|
90
|
+
export type DbExportCap = 'rows' | 'bytes' | null;
|
|
91
|
+
|
|
92
|
+
/** Bounded CSV generated from one SQLite read transaction. */
|
|
93
|
+
export interface DbCsvExport {
|
|
94
|
+
table: string;
|
|
95
|
+
content: Uint8Array<ArrayBuffer>;
|
|
96
|
+
rowCount: number;
|
|
97
|
+
bytes: number;
|
|
98
|
+
cap: DbExportCap;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/** Tests may lower a cap, but callers can never raise the production ceiling. */
|
|
102
|
+
export interface DbExportLimits {
|
|
103
|
+
maxRows?: number;
|
|
104
|
+
maxBytes?: number;
|
|
105
|
+
/** Test synchronization only: runs after the transaction's first read pins its snapshot. */
|
|
106
|
+
onSnapshot?: () => void;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/** Test synchronization for proving that a browse page is one SQLite snapshot. */
|
|
110
|
+
export interface DbRowsOptions {
|
|
111
|
+
/** Runs after COUNT, while the same read transaction remains open for SELECT. */
|
|
112
|
+
afterCount?: () => void;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/** JSON/query-string-safe SQLite rowid. Unsafe integers are represented as decimal strings. */
|
|
116
|
+
export type DbRowId = number | string;
|
|
117
|
+
|
|
66
118
|
export interface DbColumnInfo {
|
|
67
119
|
name: string;
|
|
68
120
|
type: string;
|
|
@@ -144,10 +196,51 @@ function isTruncated(v: unknown): boolean {
|
|
|
144
196
|
function cell(v: unknown): unknown {
|
|
145
197
|
if (v instanceof Uint8Array) return `<blob ${v.byteLength} B>`;
|
|
146
198
|
if (typeof v === 'string' && v.length > MAX_CELL) return `${v.slice(0, MAX_CELL)}…`;
|
|
147
|
-
|
|
199
|
+
// Read statements run with safeIntegers, so INTEGERs arrive as bigint: keep the
|
|
200
|
+
// familiar number shape when it round-trips exactly, and fall back to a string
|
|
201
|
+
// beyond 2^53 (where a double would silently round the stored value).
|
|
202
|
+
if (typeof v === 'bigint') return bigintCell(v);
|
|
148
203
|
return v;
|
|
149
204
|
}
|
|
150
205
|
|
|
206
|
+
/** Exact-value-preserving JSON form of a SQLite INTEGER read with safeIntegers. */
|
|
207
|
+
function bigintCell(v: bigint): number | string {
|
|
208
|
+
return v >= BigInt(Number.MIN_SAFE_INTEGER) && v <= BigInt(Number.MAX_SAFE_INTEGER)
|
|
209
|
+
? Number(v)
|
|
210
|
+
: v.toString();
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
const MIN_SQLITE_INTEGER = -(1n << 63n);
|
|
214
|
+
const MAX_SQLITE_INTEGER = (1n << 63n) - 1n;
|
|
215
|
+
|
|
216
|
+
/**
|
|
217
|
+
* Validate a rowid without converting an unsafe decimal string through Number.
|
|
218
|
+
* Strings stay strings when bound; SQLite applies INTEGER affinity exactly.
|
|
219
|
+
*/
|
|
220
|
+
function boundRowid(rowid: DbRowId): DbRowId {
|
|
221
|
+
if (typeof rowid === 'number') {
|
|
222
|
+
if (!Number.isSafeInteger(rowid)) throw new Error('Invalid rowid');
|
|
223
|
+
return rowid;
|
|
224
|
+
}
|
|
225
|
+
if (rowid.length === 0 || rowid.length > 20 || !/^-?\d+$/.test(rowid)) {
|
|
226
|
+
throw new Error('Invalid rowid');
|
|
227
|
+
}
|
|
228
|
+
const integer = BigInt(rowid);
|
|
229
|
+
if (integer < MIN_SQLITE_INTEGER || integer > MAX_SQLITE_INTEGER) {
|
|
230
|
+
throw new Error('Invalid rowid');
|
|
231
|
+
}
|
|
232
|
+
return rowid;
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
/**
|
|
236
|
+
* bun:sqlite exposes Statement.safeIntegers() at runtime but does not type it.
|
|
237
|
+
* Enabled per statement (never connection-wide) so the COUNT(*) reads that are
|
|
238
|
+
* consumed as numbers keep returning numbers.
|
|
239
|
+
*/
|
|
240
|
+
function safeIntegers(stmt: unknown): void {
|
|
241
|
+
(stmt as { safeIntegers?: (on: boolean) => void }).safeIntegers?.(true);
|
|
242
|
+
}
|
|
243
|
+
|
|
151
244
|
const sanitize = (rows: unknown[][]): unknown[][] => rows.map((r) => r.map(cell));
|
|
152
245
|
|
|
153
246
|
/** All user tables with row + column counts. */
|
|
@@ -189,8 +282,196 @@ function buildFilter(
|
|
|
189
282
|
const q = ident(col);
|
|
190
283
|
if (filter.op === 'eq') return { clause: ` WHERE ${q} = ?`, params: [filter.value], applied: filter };
|
|
191
284
|
if (filter.op === 'ne') return { clause: ` WHERE ${q} <> ?`, params: [filter.value], applied: filter };
|
|
192
|
-
// contains (default)
|
|
193
|
-
|
|
285
|
+
// contains (default) — escape LIKE metacharacters so a value containing % or _
|
|
286
|
+
// matches literally instead of turning into a wildcard pattern.
|
|
287
|
+
const needle = filter.value.replaceAll('\\', '\\\\').replaceAll('%', '\\%').replaceAll('_', '\\_');
|
|
288
|
+
return {
|
|
289
|
+
clause: ` WHERE ${q} LIKE ? ESCAPE '\\'`,
|
|
290
|
+
params: [`%${needle}%`],
|
|
291
|
+
applied: { ...filter, op: 'contains' },
|
|
292
|
+
};
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
function boundedExportLimit(value: number | undefined, hardCap: number, label: string): number {
|
|
296
|
+
if (value === undefined) return hardCap;
|
|
297
|
+
if (!Number.isSafeInteger(value) || value < 1) {
|
|
298
|
+
throw new Error(`${label} must be a positive safe integer`);
|
|
299
|
+
}
|
|
300
|
+
return Math.min(value, hardCap);
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
/** RFC-4180 cell with spreadsheet-formula neutralization for textual values. */
|
|
304
|
+
function csvCell(value: unknown): string {
|
|
305
|
+
let text: string;
|
|
306
|
+
let textual = false;
|
|
307
|
+
if (value == null) text = '';
|
|
308
|
+
else if (typeof value === 'string') {
|
|
309
|
+
text = value;
|
|
310
|
+
textual = true;
|
|
311
|
+
} else if (typeof value === 'bigint' || typeof value === 'number' || typeof value === 'boolean') {
|
|
312
|
+
text = String(value);
|
|
313
|
+
} else if (value instanceof Uint8Array) {
|
|
314
|
+
// The SELECT below converts BLOBs before they cross into JS. Keep this
|
|
315
|
+
// fallback bounded and deterministic if SQLite changes the expression type.
|
|
316
|
+
text = `<blob ${value.byteLength} B>`;
|
|
317
|
+
textual = true;
|
|
318
|
+
} else {
|
|
319
|
+
throw new Error('Database export produced an unsupported SQLite value');
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
// Excel and Sheets may execute strings beginning with these characters. A
|
|
323
|
+
// leading TAB/CR is dangerous too because spreadsheet import strips it first.
|
|
324
|
+
const safe = textual && /^[=+\-@\t\r]/.test(text) ? `'${text}` : text;
|
|
325
|
+
return /[",\r\n]/.test(safe) ? `"${safe.replaceAll('"', '""')}"` : safe;
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
/**
|
|
329
|
+
* Export one filtered/sorted table view from a single SQLite snapshot.
|
|
330
|
+
*
|
|
331
|
+
* The readonly connection and explicit deferred transaction pin sqlite_master,
|
|
332
|
+
* schema, filter and row iteration to one WAL snapshot. Unlike client-side
|
|
333
|
+
* OFFSET pagination, concurrent inserts/deletes cannot duplicate or omit rows.
|
|
334
|
+
* The cursor is streamed into a fixed-size buffer and probes at most one row
|
|
335
|
+
* beyond the row cap, so both retained output and cardinality are bounded.
|
|
336
|
+
*/
|
|
337
|
+
export function dbExportCsv(
|
|
338
|
+
path: string,
|
|
339
|
+
table: string,
|
|
340
|
+
orderBy?: string,
|
|
341
|
+
dir: 'asc' | 'desc' = 'asc',
|
|
342
|
+
filter?: DbFilter,
|
|
343
|
+
limits: Readonly<DbExportLimits> = {}
|
|
344
|
+
): DbCsvExport {
|
|
345
|
+
if (typeof table !== 'string' || table.length === 0) throw new Error('Table is required');
|
|
346
|
+
if (table.length > DB_EXPORT_MAX_IDENTIFIER_CHARS) {
|
|
347
|
+
throw new Error(`Table name exceeds ${DB_EXPORT_MAX_IDENTIFIER_CHARS} characters`);
|
|
348
|
+
}
|
|
349
|
+
if (dir !== 'asc' && dir !== 'desc') throw new Error('Invalid export sort direction');
|
|
350
|
+
if (filter) {
|
|
351
|
+
if (
|
|
352
|
+
typeof filter.column !== 'string' ||
|
|
353
|
+
!filter.column ||
|
|
354
|
+
(filter.op !== 'contains' && filter.op !== 'eq' && filter.op !== 'ne') ||
|
|
355
|
+
typeof filter.value !== 'string'
|
|
356
|
+
) {
|
|
357
|
+
throw new Error('Invalid database export filter');
|
|
358
|
+
}
|
|
359
|
+
if (filter.column.length > DB_EXPORT_MAX_IDENTIFIER_CHARS) {
|
|
360
|
+
throw new Error(`Filter column exceeds ${DB_EXPORT_MAX_IDENTIFIER_CHARS} characters`);
|
|
361
|
+
}
|
|
362
|
+
if (filter.value.length > DB_EXPORT_MAX_FILTER_CHARS) {
|
|
363
|
+
throw new Error(
|
|
364
|
+
`Database export filter exceeds ${DB_EXPORT_MAX_FILTER_CHARS} characters`
|
|
365
|
+
);
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
const maxRows = boundedExportLimit(limits.maxRows, DB_EXPORT_MAX_ROWS, 'Export row limit');
|
|
370
|
+
const maxBytes = boundedExportLimit(limits.maxBytes, DB_EXPORT_MAX_BYTES, 'Export byte limit');
|
|
371
|
+
const db = open(path);
|
|
372
|
+
try {
|
|
373
|
+
const snapshot = db.transaction((): DbCsvExport => {
|
|
374
|
+
const name = knownTable(db, table);
|
|
375
|
+
const columns = (
|
|
376
|
+
db.query(`PRAGMA table_info(${ident(name)})`).all() as { name: string }[]
|
|
377
|
+
).map((column) => column.name);
|
|
378
|
+
if (columns.length === 0) throw new Error(`Table has no exportable columns: ${name}`);
|
|
379
|
+
if (columns.length > DB_EXPORT_MAX_COLUMNS) {
|
|
380
|
+
throw new Error(
|
|
381
|
+
`Table has ${columns.length} columns; database export supports at most ${DB_EXPORT_MAX_COLUMNS}`
|
|
382
|
+
);
|
|
383
|
+
}
|
|
384
|
+
if (columns.some((column) => column.length > DB_EXPORT_MAX_IDENTIFIER_CHARS)) {
|
|
385
|
+
throw new Error(
|
|
386
|
+
`Database export column names must not exceed ${DB_EXPORT_MAX_IDENTIFIER_CHARS} characters`
|
|
387
|
+
);
|
|
388
|
+
}
|
|
389
|
+
limits.onSnapshot?.();
|
|
390
|
+
|
|
391
|
+
let order = '';
|
|
392
|
+
if (orderBy !== undefined) {
|
|
393
|
+
if (typeof orderBy !== 'string' || !orderBy) {
|
|
394
|
+
throw new Error('Invalid database export sort column');
|
|
395
|
+
}
|
|
396
|
+
if (orderBy.length > DB_EXPORT_MAX_IDENTIFIER_CHARS) {
|
|
397
|
+
throw new Error(`Sort column exceeds ${DB_EXPORT_MAX_IDENTIFIER_CHARS} characters`);
|
|
398
|
+
}
|
|
399
|
+
const sortColumn = columns.find((column) => column === orderBy);
|
|
400
|
+
if (!sortColumn) throw new Error(`No such column: ${orderBy}`);
|
|
401
|
+
order = ` ORDER BY ${ident(sortColumn)} ${dir === 'desc' ? 'DESC' : 'ASC'}`;
|
|
402
|
+
}
|
|
403
|
+
const { clause, params } = buildFilter(columns, filter);
|
|
404
|
+
|
|
405
|
+
// Generated aliases avoid object-property hazards for legal SQLite column
|
|
406
|
+
// names such as "__proto__". Text and BLOB expressions ensure one cell
|
|
407
|
+
// cannot materialize an unbounded JS value before the byte cap is checked.
|
|
408
|
+
const aliases = columns.map((_, index) => `__bq_export_${index}`);
|
|
409
|
+
const selected = columns.map((column, index) => {
|
|
410
|
+
const quoted = ident(column);
|
|
411
|
+
return (
|
|
412
|
+
`CASE ` +
|
|
413
|
+
`WHEN typeof(${quoted}) = 'blob' THEN '<blob ' || length(${quoted}) || ' B>' ` +
|
|
414
|
+
`WHEN typeof(${quoted}) = 'text' AND length(CAST(${quoted} AS BLOB)) > ${MAX_CELL} ` +
|
|
415
|
+
`THEN substr(${quoted}, 1, ${MAX_CELL}) || '…' ` +
|
|
416
|
+
`ELSE ${quoted} END AS ${ident(aliases[index])}`
|
|
417
|
+
);
|
|
418
|
+
});
|
|
419
|
+
const stmt = db.query<Record<string, unknown>, Array<string | number>>(
|
|
420
|
+
`SELECT ${selected.join(', ')} FROM ${ident(name)}${clause}${order} LIMIT ?`
|
|
421
|
+
);
|
|
422
|
+
safeIntegers(stmt);
|
|
423
|
+
|
|
424
|
+
const encoder = new TextEncoder();
|
|
425
|
+
const buffer = new Uint8Array(maxBytes);
|
|
426
|
+
let used = 0;
|
|
427
|
+
const append = (text: string): boolean => {
|
|
428
|
+
const result = encoder.encodeInto(text, buffer.subarray(used));
|
|
429
|
+
if (result.read !== text.length) return false;
|
|
430
|
+
used += result.written;
|
|
431
|
+
return true;
|
|
432
|
+
};
|
|
433
|
+
if (!append(columns.map(csvCell).join(','))) {
|
|
434
|
+
throw new Error(`Database export header exceeds the ${maxBytes}-byte export limit`);
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
let rowCount = 0;
|
|
438
|
+
let cap: DbExportCap = null;
|
|
439
|
+
const bindings = [...params, maxRows + 1] as Array<string | number>;
|
|
440
|
+
const iterator = stmt.iterate(...bindings);
|
|
441
|
+
try {
|
|
442
|
+
for (;;) {
|
|
443
|
+
const next = iterator.next();
|
|
444
|
+
if (next.done) break;
|
|
445
|
+
if (rowCount >= maxRows) {
|
|
446
|
+
cap = 'rows';
|
|
447
|
+
break;
|
|
448
|
+
}
|
|
449
|
+
const line = `\r\n${aliases.map((alias) => csvCell(next.value[alias])).join(',')}`;
|
|
450
|
+
if (!append(line)) {
|
|
451
|
+
cap = 'bytes';
|
|
452
|
+
break;
|
|
453
|
+
}
|
|
454
|
+
rowCount++;
|
|
455
|
+
}
|
|
456
|
+
} finally {
|
|
457
|
+
iterator.return?.();
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
// Slice releases the unused portion of the fixed working buffer before
|
|
461
|
+
// the Response is retained by the HTTP server. Peak memory remains a
|
|
462
|
+
// small constant multiple of DB_EXPORT_MAX_BYTES, never table size.
|
|
463
|
+
const content = buffer.slice(0, used);
|
|
464
|
+
return { table: name, content, rowCount, bytes: content.byteLength, cap };
|
|
465
|
+
});
|
|
466
|
+
// On a readonly connection BEGIN DEFERRED is a read transaction. Bun's
|
|
467
|
+
// wrapper commits on success and rolls back on every thrown validation,
|
|
468
|
+
// cursor or encoding error.
|
|
469
|
+
return snapshot.deferred();
|
|
470
|
+
} catch (error) {
|
|
471
|
+
throw translateError(error);
|
|
472
|
+
} finally {
|
|
473
|
+
db.close();
|
|
474
|
+
}
|
|
194
475
|
}
|
|
195
476
|
|
|
196
477
|
/**
|
|
@@ -205,69 +486,94 @@ export function dbRows(
|
|
|
205
486
|
offset: number,
|
|
206
487
|
orderBy?: string,
|
|
207
488
|
dir: 'asc' | 'desc' = 'asc',
|
|
208
|
-
filter?: DbFilter
|
|
489
|
+
filter?: DbFilter,
|
|
490
|
+
options: Readonly<DbRowsOptions> = {}
|
|
209
491
|
): DbRowsPage {
|
|
210
492
|
const lim = Math.max(1, Math.min(MAX_ROWS, Math.floor(limit) || 50));
|
|
211
|
-
|
|
493
|
+
// Non-finite / out-of-range offsets must be clamped here: SQLite rejects a
|
|
494
|
+
// bound non-safe-integer double with a raw "datatype mismatch".
|
|
495
|
+
const off = Number.isFinite(offset)
|
|
496
|
+
? Math.max(0, Math.min(Number.MAX_SAFE_INTEGER, Math.floor(offset) || 0))
|
|
497
|
+
: 0;
|
|
212
498
|
const db = open(path);
|
|
213
499
|
try {
|
|
214
|
-
const
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
500
|
+
const snapshot = db.transaction((): DbRowsPage => {
|
|
501
|
+
const name = knownTable(db, table);
|
|
502
|
+
const columns = (
|
|
503
|
+
db.query(`PRAGMA table_info(${ident(name)})`).all() as { name: string }[]
|
|
504
|
+
).map((c) => c.name);
|
|
218
505
|
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
506
|
+
let order = '';
|
|
507
|
+
let sortCol: string | null = null;
|
|
508
|
+
if (orderBy) {
|
|
509
|
+
sortCol = columns.find((c) => c === orderBy) ?? null;
|
|
510
|
+
if (!sortCol) throw new Error(`No such column: ${orderBy}`);
|
|
511
|
+
order = ` ORDER BY ${ident(sortCol)} ${dir === 'desc' ? 'DESC' : 'ASC'}`;
|
|
512
|
+
}
|
|
226
513
|
|
|
227
|
-
|
|
514
|
+
const { clause, params, applied } = buildFilter(columns, filter);
|
|
228
515
|
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
516
|
+
const { c: total } = db
|
|
517
|
+
.query(`SELECT COUNT(*) AS c FROM ${ident(name)}${clause}`)
|
|
518
|
+
.get(...(params as [])) as { c: number };
|
|
519
|
+
options.afterCount?.();
|
|
232
520
|
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
521
|
+
// Prefer selecting rowid so the detail view can fetch untruncated cells;
|
|
522
|
+
// WITHOUT ROWID tables have no rowid column — fall back to a plain select.
|
|
523
|
+
let rowids: (DbRowId | null)[] = [];
|
|
524
|
+
let rawRows: unknown[][];
|
|
525
|
+
try {
|
|
526
|
+
const stmt = db.query(
|
|
527
|
+
`SELECT rowid AS __rid, * FROM ${ident(name)}${clause}${order} LIMIT ? OFFSET ?`
|
|
528
|
+
);
|
|
529
|
+
safeIntegers(stmt);
|
|
530
|
+
const withRowid = stmt.values(...(params as []), lim, off) as unknown[][];
|
|
531
|
+
rowids = withRowid.map((r) =>
|
|
532
|
+
typeof r[0] === 'bigint'
|
|
533
|
+
? bigintCell(r[0])
|
|
534
|
+
: typeof r[0] === 'number' && Number.isSafeInteger(r[0])
|
|
535
|
+
? r[0]
|
|
536
|
+
: null
|
|
537
|
+
);
|
|
538
|
+
rawRows = withRowid.map((r) => r.slice(1));
|
|
539
|
+
} catch {
|
|
540
|
+
const stmt = db.query(`SELECT * FROM ${ident(name)}${clause}${order} LIMIT ? OFFSET ?`);
|
|
541
|
+
safeIntegers(stmt);
|
|
542
|
+
rawRows = stmt.values(...(params as []), lim, off) as unknown[][];
|
|
543
|
+
rowids = rawRows.map(() => null);
|
|
544
|
+
}
|
|
249
545
|
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
546
|
+
const truncatedCells = rawRows.map((r) => r.map(isTruncated));
|
|
547
|
+
return {
|
|
548
|
+
table: name,
|
|
549
|
+
columns,
|
|
550
|
+
rows: sanitize(rawRows),
|
|
551
|
+
rowids,
|
|
552
|
+
truncatedCells,
|
|
553
|
+
total,
|
|
554
|
+
limit: lim,
|
|
555
|
+
offset: off,
|
|
556
|
+
orderBy: sortCol,
|
|
557
|
+
dir: dir === 'desc' ? 'desc' : 'asc',
|
|
558
|
+
filter: applied,
|
|
559
|
+
};
|
|
560
|
+
});
|
|
561
|
+
// COUNT and page SELECT must describe the same live view. Without this read
|
|
562
|
+
// transaction, a concurrent WAL commit between them could make a valid
|
|
563
|
+
// response internally contradictory (or shift rows across page boundaries).
|
|
564
|
+
return snapshot.deferred();
|
|
264
565
|
} finally {
|
|
265
566
|
db.close();
|
|
266
567
|
}
|
|
267
568
|
}
|
|
268
569
|
|
|
269
570
|
/** Full, untruncated value of a single cell, keyed by rowid (for the detail view). */
|
|
270
|
-
export function dbCell(
|
|
571
|
+
export function dbCell(
|
|
572
|
+
path: string,
|
|
573
|
+
table: string,
|
|
574
|
+
rowid: DbRowId,
|
|
575
|
+
column: string
|
|
576
|
+
): { value: unknown } {
|
|
271
577
|
const db = open(path);
|
|
272
578
|
try {
|
|
273
579
|
const name = knownTable(db, table);
|
|
@@ -276,13 +582,13 @@ export function dbCell(path: string, table: string, rowid: number, column: strin
|
|
|
276
582
|
);
|
|
277
583
|
const col = columns.find((c) => c === column);
|
|
278
584
|
if (!col) throw new Error(`No such column: ${column}`);
|
|
279
|
-
const
|
|
280
|
-
|
|
281
|
-
|
|
585
|
+
const stmt = db.query(`SELECT ${ident(col)} AS v FROM ${ident(name)} WHERE rowid = ?`);
|
|
586
|
+
safeIntegers(stmt);
|
|
587
|
+
const row = stmt.get(boundRowid(rowid)) as { v: unknown } | null;
|
|
282
588
|
if (!row) throw new Error('Row not found');
|
|
283
589
|
let v = row.v;
|
|
284
590
|
if (v instanceof Uint8Array) v = `<blob ${v.byteLength} B — binary, not shown>`;
|
|
285
|
-
else if (typeof v === 'bigint') v = v
|
|
591
|
+
else if (typeof v === 'bigint') v = bigintCell(v);
|
|
286
592
|
else if (typeof v === 'string' && v.length > MAX_FULL_CELL) v = `${v.slice(0, MAX_FULL_CELL)}…`;
|
|
287
593
|
return { value: v };
|
|
288
594
|
} finally {
|
|
@@ -375,6 +681,38 @@ function leadingKeyword(sql: string): string {
|
|
|
375
681
|
|
|
376
682
|
const READ_ONLY_LEAD = /^(?:SELECT|WITH|EXPLAIN|VALUES|PRAGMA)\b/;
|
|
377
683
|
|
|
684
|
+
/** Temp-view name used to recover colliding output column names (see dbQuery). */
|
|
685
|
+
const DUP_VIEW = '__bq_query_columns';
|
|
686
|
+
|
|
687
|
+
/**
|
|
688
|
+
* Re-prepare a statement whose output column names collide through a TEMP VIEW,
|
|
689
|
+
* so SQLite disambiguates them itself ("id", "id:1") and every column survives
|
|
690
|
+
* the by-name row rebuild. The view lives in the private temp schema (nothing is
|
|
691
|
+
* written to the readonly store) and dies with the connection. Returns null when
|
|
692
|
+
* the statement can't be a view (PRAGMA / EXPLAIN — neither can collide anyway).
|
|
693
|
+
*/
|
|
694
|
+
function disambiguateColumns(
|
|
695
|
+
db: Database,
|
|
696
|
+
sql: string
|
|
697
|
+
): { stmt: ReturnType<Database['query']>; cols: string[] } | null {
|
|
698
|
+
try {
|
|
699
|
+
// MUST be query().run() (single prepared statement), never db.run(): db.run
|
|
700
|
+
// executes EVERY statement in the string, and the front-door allowlist only
|
|
701
|
+
// inspects the FIRST keyword. `SELECT 1 a, 2 a; VACUUM INTO '/tmp/x'` would
|
|
702
|
+
// otherwise reach a readonly connection as an arbitrary-file WRITE, and
|
|
703
|
+
// `…; ATTACH '/any.db' AS l; DROP VIEW …; CREATE TEMP VIEW … AS SELECT * FROM l.x`
|
|
704
|
+
// as an arbitrary-SQLite-file READ. query() compiles only the first
|
|
705
|
+
// statement, so the trailing payload is inert.
|
|
706
|
+
db.query(`CREATE TEMP VIEW ${ident(DUP_VIEW)} AS ${sql.replace(/;\s*$/, '')}`).run();
|
|
707
|
+
const cols = (
|
|
708
|
+
db.query(`PRAGMA table_info(${ident(DUP_VIEW)})`).all() as { name: string }[]
|
|
709
|
+
).map((c) => c.name);
|
|
710
|
+
return { stmt: db.query(`SELECT * FROM ${ident(DUP_VIEW)}`), cols };
|
|
711
|
+
} catch {
|
|
712
|
+
return null;
|
|
713
|
+
}
|
|
714
|
+
}
|
|
715
|
+
|
|
378
716
|
/**
|
|
379
717
|
* Run an arbitrary read-only query. Rejects non-read statements up front,
|
|
380
718
|
* streams rows with an early break at MAX_ROWS, and translates the raw SQLite
|
|
@@ -396,7 +734,18 @@ export function dbQuery(path: string, sql: string): DbQueryResult {
|
|
|
396
734
|
} catch (e) {
|
|
397
735
|
throw translateError(e);
|
|
398
736
|
}
|
|
399
|
-
|
|
737
|
+
let cols = stmt.columnNames;
|
|
738
|
+
// Bun de-duplicates columnNames while columnTypes keeps the true arity, and
|
|
739
|
+
// rows are rebuilt BY NAME below — so without this, a statement with colliding
|
|
740
|
+
// output names (`SELECT a.id, b.id`, any self-join) silently loses columns.
|
|
741
|
+
if (stmt.columnTypes.length > cols.length) {
|
|
742
|
+
const wide = disambiguateColumns(db, trimmed);
|
|
743
|
+
if (wide) {
|
|
744
|
+
stmt = wide.stmt;
|
|
745
|
+
cols = wide.cols;
|
|
746
|
+
}
|
|
747
|
+
}
|
|
748
|
+
safeIntegers(stmt);
|
|
400
749
|
const rows: unknown[][] = [];
|
|
401
750
|
let truncated = false;
|
|
402
751
|
try {
|
|
@@ -434,12 +783,26 @@ function translateError(e: unknown): Error {
|
|
|
434
783
|
}
|
|
435
784
|
|
|
436
785
|
/**
|
|
437
|
-
*
|
|
438
|
-
*
|
|
439
|
-
*
|
|
786
|
+
* Shared URL of the disposable database worker. It accepts both arbitrary
|
|
787
|
+
* query and atomic CSV-export messages; scripts/serve.ts points this at the
|
|
788
|
+
* secondary embedded entrypoint in compiled builds.
|
|
789
|
+
*
|
|
790
|
+
* The timeout aborts the RESPONSE, not the query: terminate() is delivered to
|
|
791
|
+
* the worker's event loop, which a synchronous sqlite3_step never yields back
|
|
792
|
+
* to, so a runaway scan keeps burning its own thread until SQLite finishes it.
|
|
793
|
+
* That thread is therefore accounted for (a slot is held until the worker really
|
|
794
|
+
* exits) and capped by the respective query/export pool, so abandoned work can
|
|
795
|
+
* pin only the configured number of cores instead of one new core per request.
|
|
440
796
|
*/
|
|
441
797
|
let queryWorkerUrl = new URL('./dbQueryWorker.ts', import.meta.url).href;
|
|
442
798
|
|
|
799
|
+
/** Hard cap on query worker threads alive at once (see queryWithTimeout). */
|
|
800
|
+
export const MAX_CONCURRENT_QUERIES = 2;
|
|
801
|
+
let liveQueryWorkers = 0;
|
|
802
|
+
|
|
803
|
+
/** Query worker threads currently alive — includes timed-out, still-running ones. */
|
|
804
|
+
export const queryWorkerLoad = (): number => liveQueryWorkers;
|
|
805
|
+
|
|
443
806
|
/**
|
|
444
807
|
* The standalone executable's bundle root is scripts/serve.ts, so it supplies
|
|
445
808
|
* the embedded worker URL relative to that entrypoint. Source-mode agent runs
|
|
@@ -449,13 +812,249 @@ export function setQueryWorkerUrl(url: string): void {
|
|
|
449
812
|
queryWorkerUrl = url;
|
|
450
813
|
}
|
|
451
814
|
|
|
815
|
+
const spawnRealWorker = (): Worker => new Worker(queryWorkerUrl, { type: 'module' });
|
|
816
|
+
let queryWorkerFactory: () => Worker = spawnRealWorker;
|
|
817
|
+
|
|
818
|
+
/**
|
|
819
|
+
* Swap how a query worker is constructed. Tests only: spawning a real Bun
|
|
820
|
+
* Worker in a process where happy-dom's globals are installed panics the
|
|
821
|
+
* runtime (allocator abort), so the timeout/accounting behaviour below is
|
|
822
|
+
* exercised through a stand-in worker instead of a real thread. Pass null to
|
|
823
|
+
* restore the real one.
|
|
824
|
+
*/
|
|
825
|
+
export function setQueryWorkerFactory(factory: (() => Worker) | null): void {
|
|
826
|
+
queryWorkerFactory = factory ?? spawnRealWorker;
|
|
827
|
+
}
|
|
828
|
+
|
|
829
|
+
/** Only one potentially expensive table scan/sort may run at a time. */
|
|
830
|
+
export const MAX_CONCURRENT_EXPORTS = 1;
|
|
831
|
+
let liveExportWorkers = 0;
|
|
832
|
+
|
|
833
|
+
/** Export workers alive now, including timed-out/aborted SQLite calls still unwinding. */
|
|
834
|
+
export const exportWorkerLoad = (): number => liveExportWorkers;
|
|
835
|
+
|
|
836
|
+
const spawnRealExportWorker = (): Worker => new Worker(queryWorkerUrl, { type: 'module' });
|
|
837
|
+
let exportWorkerFactory: () => Worker = spawnRealExportWorker;
|
|
838
|
+
|
|
839
|
+
/** Test-only export-worker factory; pass null to restore the real worker. */
|
|
840
|
+
export function setExportWorkerFactory(factory: (() => Worker) | null): void {
|
|
841
|
+
exportWorkerFactory = factory ?? spawnRealExportWorker;
|
|
842
|
+
}
|
|
843
|
+
|
|
844
|
+
function validatedWorkerExport(value: unknown, expectedTable: string): DbCsvExport {
|
|
845
|
+
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
|
846
|
+
throw new DbExportUnavailableError('Database export worker returned a malformed result');
|
|
847
|
+
}
|
|
848
|
+
const result = value as Record<string, unknown>;
|
|
849
|
+
if (
|
|
850
|
+
typeof result.table !== 'string' ||
|
|
851
|
+
result.table !== expectedTable ||
|
|
852
|
+
!(result.content instanceof Uint8Array) ||
|
|
853
|
+
!Number.isSafeInteger(result.rowCount) ||
|
|
854
|
+
(result.rowCount as number) < 0 ||
|
|
855
|
+
(result.rowCount as number) > DB_EXPORT_MAX_ROWS ||
|
|
856
|
+
!Number.isSafeInteger(result.bytes) ||
|
|
857
|
+
(result.bytes as number) < 1 ||
|
|
858
|
+
(result.bytes as number) > DB_EXPORT_MAX_BYTES ||
|
|
859
|
+
result.content.byteLength !== result.bytes ||
|
|
860
|
+
(result.cap !== null && result.cap !== 'rows' && result.cap !== 'bytes') ||
|
|
861
|
+
(result.cap === 'rows' && result.rowCount !== DB_EXPORT_MAX_ROWS)
|
|
862
|
+
) {
|
|
863
|
+
throw new DbExportUnavailableError('Database export worker returned a malformed result');
|
|
864
|
+
}
|
|
865
|
+
const content =
|
|
866
|
+
result.content.buffer instanceof ArrayBuffer
|
|
867
|
+
? (result.content as Uint8Array<ArrayBuffer>)
|
|
868
|
+
: Uint8Array.from(result.content);
|
|
869
|
+
return {
|
|
870
|
+
table: result.table as string,
|
|
871
|
+
content,
|
|
872
|
+
rowCount: result.rowCount as number,
|
|
873
|
+
bytes: result.bytes as number,
|
|
874
|
+
cap: result.cap as DbExportCap,
|
|
875
|
+
};
|
|
876
|
+
}
|
|
877
|
+
|
|
878
|
+
/**
|
|
879
|
+
* Run an atomic CSV export off the control-agent event loop.
|
|
880
|
+
*
|
|
881
|
+
* A timeout or caller abort abandons the response, not sqlite3_step: the Worker
|
|
882
|
+
* keeps its only slot until `close`, preventing repeated abandoned scans from
|
|
883
|
+
* pinning additional cores. Row/cell/byte caps remain enforced inside
|
|
884
|
+
* dbExportCsv before its ArrayBuffer is transferred back without JSON expansion.
|
|
885
|
+
*/
|
|
886
|
+
export async function exportWithTimeout(
|
|
887
|
+
path: string,
|
|
888
|
+
table: string,
|
|
889
|
+
orderBy?: string,
|
|
890
|
+
dir: 'asc' | 'desc' = 'asc',
|
|
891
|
+
filter?: DbFilter,
|
|
892
|
+
signal?: AbortSignal
|
|
893
|
+
): Promise<DbCsvExport> {
|
|
894
|
+
signal?.throwIfAborted();
|
|
895
|
+
if (liveExportWorkers >= MAX_CONCURRENT_EXPORTS) {
|
|
896
|
+
throw new DbExportBusyError(
|
|
897
|
+
'A database export is already running. Wait for it to finish, or restart the agent if it was abandoned.'
|
|
898
|
+
);
|
|
899
|
+
}
|
|
900
|
+
|
|
901
|
+
let worker: Worker;
|
|
902
|
+
try {
|
|
903
|
+
worker = exportWorkerFactory();
|
|
904
|
+
} catch (error) {
|
|
905
|
+
throw new DbExportUnavailableError(
|
|
906
|
+
`Database export worker unavailable: ${(error as Error).message ?? String(error)}`
|
|
907
|
+
);
|
|
908
|
+
}
|
|
909
|
+
liveExportWorkers++;
|
|
910
|
+
let released = false;
|
|
911
|
+
const release = () => {
|
|
912
|
+
if (released) return;
|
|
913
|
+
released = true;
|
|
914
|
+
liveExportWorkers--;
|
|
915
|
+
};
|
|
916
|
+
|
|
917
|
+
try {
|
|
918
|
+
return await new Promise<DbCsvExport>((resolve, reject) => {
|
|
919
|
+
let settled = false;
|
|
920
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
921
|
+
const cleanup = () => {
|
|
922
|
+
if (timer !== undefined) clearTimeout(timer);
|
|
923
|
+
signal?.removeEventListener('abort', onAbort);
|
|
924
|
+
};
|
|
925
|
+
const abandon = (error: Error) => {
|
|
926
|
+
if (settled) return;
|
|
927
|
+
settled = true;
|
|
928
|
+
cleanup();
|
|
929
|
+
worker.terminate();
|
|
930
|
+
// Do NOT release here. A synchronous sqlite3_step may still be running;
|
|
931
|
+
// the close event is the proof that its thread has really exited.
|
|
932
|
+
reject(error);
|
|
933
|
+
};
|
|
934
|
+
const onAbort = () =>
|
|
935
|
+
abandon(
|
|
936
|
+
signal?.reason instanceof Error
|
|
937
|
+
? signal.reason
|
|
938
|
+
: new Error('Database export request was aborted')
|
|
939
|
+
);
|
|
940
|
+
|
|
941
|
+
worker.addEventListener('close', () => {
|
|
942
|
+
release();
|
|
943
|
+
if (!settled) {
|
|
944
|
+
settled = true;
|
|
945
|
+
cleanup();
|
|
946
|
+
reject(new DbExportUnavailableError('Database export worker exited unexpectedly'));
|
|
947
|
+
}
|
|
948
|
+
});
|
|
949
|
+
worker.addEventListener('message', (event: MessageEvent) => {
|
|
950
|
+
if (settled) return;
|
|
951
|
+
settled = true;
|
|
952
|
+
cleanup();
|
|
953
|
+
release();
|
|
954
|
+
const data = event.data as
|
|
955
|
+
| {
|
|
956
|
+
ok?: unknown;
|
|
957
|
+
export?: unknown;
|
|
958
|
+
error?: unknown;
|
|
959
|
+
missing?: unknown;
|
|
960
|
+
}
|
|
961
|
+
| null;
|
|
962
|
+
if (!data || typeof data !== 'object' || Array.isArray(data)) {
|
|
963
|
+
reject(
|
|
964
|
+
new DbExportUnavailableError('Database export worker returned a malformed response')
|
|
965
|
+
);
|
|
966
|
+
return;
|
|
967
|
+
}
|
|
968
|
+
const response: {
|
|
969
|
+
ok?: unknown;
|
|
970
|
+
export?: unknown;
|
|
971
|
+
error?: unknown;
|
|
972
|
+
missing?: unknown;
|
|
973
|
+
} = data;
|
|
974
|
+
if (response.ok === true) {
|
|
975
|
+
try {
|
|
976
|
+
resolve(validatedWorkerExport(response.export, table));
|
|
977
|
+
} catch (error) {
|
|
978
|
+
reject(error);
|
|
979
|
+
}
|
|
980
|
+
} else if (response.missing === true) {
|
|
981
|
+
reject(
|
|
982
|
+
new MissingDbError(
|
|
983
|
+
typeof response.error === 'string' ? response.error : 'Database not found'
|
|
984
|
+
)
|
|
985
|
+
);
|
|
986
|
+
} else {
|
|
987
|
+
reject(
|
|
988
|
+
new Error(
|
|
989
|
+
typeof response.error === 'string' ? response.error : 'Database export failed'
|
|
990
|
+
)
|
|
991
|
+
);
|
|
992
|
+
}
|
|
993
|
+
});
|
|
994
|
+
worker.addEventListener('error', (event: ErrorEvent) => {
|
|
995
|
+
if (settled) return;
|
|
996
|
+
event.preventDefault?.();
|
|
997
|
+
abandon(
|
|
998
|
+
new DbExportUnavailableError(
|
|
999
|
+
`Database export worker failed: ${event.message || 'unknown worker error'}`
|
|
1000
|
+
)
|
|
1001
|
+
);
|
|
1002
|
+
});
|
|
1003
|
+
|
|
1004
|
+
timer = setTimeout(
|
|
1005
|
+
() =>
|
|
1006
|
+
abandon(
|
|
1007
|
+
new DbExportUnavailableError(
|
|
1008
|
+
`Database export exceeded the ${DB_EXPORT_TIMEOUT_MS / 1000}s time limit and was abandoned (SQLite may keep scanning until the worker exits).`
|
|
1009
|
+
)
|
|
1010
|
+
),
|
|
1011
|
+
DB_EXPORT_TIMEOUT_MS
|
|
1012
|
+
);
|
|
1013
|
+
signal?.addEventListener('abort', onAbort, { once: true });
|
|
1014
|
+
if (signal?.aborted) {
|
|
1015
|
+
onAbort();
|
|
1016
|
+
return;
|
|
1017
|
+
}
|
|
1018
|
+
try {
|
|
1019
|
+
worker.postMessage({ kind: 'export', path, table, orderBy, dir, filter });
|
|
1020
|
+
} catch (error) {
|
|
1021
|
+
settled = true;
|
|
1022
|
+
cleanup();
|
|
1023
|
+
release();
|
|
1024
|
+
reject(
|
|
1025
|
+
new DbExportUnavailableError(
|
|
1026
|
+
`Could not start database export: ${(error as Error).message ?? String(error)}`
|
|
1027
|
+
)
|
|
1028
|
+
);
|
|
1029
|
+
}
|
|
1030
|
+
});
|
|
1031
|
+
} finally {
|
|
1032
|
+
worker.terminate();
|
|
1033
|
+
}
|
|
1034
|
+
}
|
|
1035
|
+
|
|
452
1036
|
export async function queryWithTimeout(path: string, sql: string): Promise<DbQueryResult> {
|
|
1037
|
+
if (liveQueryWorkers >= MAX_CONCURRENT_QUERIES) {
|
|
1038
|
+
throw new Error(
|
|
1039
|
+
`Too many queries running (${liveQueryWorkers}/${MAX_CONCURRENT_QUERIES}). A previous query timed out and is still running inside SQLite — wait for it to finish, or restart the agent.`
|
|
1040
|
+
);
|
|
1041
|
+
}
|
|
453
1042
|
let worker: Worker;
|
|
454
1043
|
try {
|
|
455
|
-
worker =
|
|
1044
|
+
worker = queryWorkerFactory();
|
|
456
1045
|
} catch (e) {
|
|
457
1046
|
throw new Error(`Query worker unavailable: ${(e as Error).message ?? String(e)}`);
|
|
458
1047
|
}
|
|
1048
|
+
liveQueryWorkers++;
|
|
1049
|
+
let released = false;
|
|
1050
|
+
const release = () => {
|
|
1051
|
+
if (released) return;
|
|
1052
|
+
released = true;
|
|
1053
|
+
liveQueryWorkers--;
|
|
1054
|
+
};
|
|
1055
|
+
// A worker that never answers still holds its slot until the thread really
|
|
1056
|
+
// exits ('close' fires when SQLite finally returns) — that is the whole point.
|
|
1057
|
+
worker.addEventListener('close', release);
|
|
459
1058
|
try {
|
|
460
1059
|
return await new Promise<DbQueryResult>((resolve, reject) => {
|
|
461
1060
|
let settled = false;
|
|
@@ -463,20 +1062,35 @@ export async function queryWithTimeout(path: string, sql: string): Promise<DbQue
|
|
|
463
1062
|
if (settled) return;
|
|
464
1063
|
settled = true;
|
|
465
1064
|
worker.terminate();
|
|
466
|
-
reject(
|
|
1065
|
+
reject(
|
|
1066
|
+
new Error(
|
|
1067
|
+
`Query exceeded the ${QUERY_TIMEOUT_MS / 1000}s time limit and was abandoned (it may keep running inside SQLite until it completes).`
|
|
1068
|
+
)
|
|
1069
|
+
);
|
|
467
1070
|
}, QUERY_TIMEOUT_MS);
|
|
468
1071
|
worker.addEventListener('message', (ev: MessageEvent) => {
|
|
469
1072
|
if (settled) return;
|
|
470
1073
|
settled = true;
|
|
471
1074
|
clearTimeout(timer);
|
|
472
|
-
|
|
1075
|
+
// The query is done — its thread is idle, so free the slot immediately.
|
|
1076
|
+
release();
|
|
1077
|
+
const d = ev.data as {
|
|
1078
|
+
ok: boolean;
|
|
1079
|
+
result?: DbQueryResult;
|
|
1080
|
+
error?: string;
|
|
1081
|
+
missing?: boolean;
|
|
1082
|
+
};
|
|
473
1083
|
if (d.ok && d.result) resolve(d.result);
|
|
1084
|
+
// The worker boundary is a string channel, so the error CLASS is carried
|
|
1085
|
+
// as a flag — server.ts maps MissingDbError to 404, everything else 400.
|
|
1086
|
+
else if (d.missing) reject(new MissingDbError(d.error ?? 'Database not found'));
|
|
474
1087
|
else reject(new Error(d.error ?? 'Query failed'));
|
|
475
1088
|
});
|
|
476
1089
|
worker.addEventListener('error', (ev: ErrorEvent) => {
|
|
477
1090
|
if (settled) return;
|
|
478
1091
|
settled = true;
|
|
479
1092
|
clearTimeout(timer);
|
|
1093
|
+
release();
|
|
480
1094
|
ev.preventDefault?.();
|
|
481
1095
|
reject(new Error(`Query worker failed: ${ev.message || 'unknown worker error'}`));
|
|
482
1096
|
});
|