@gscdump/engine-duckdb-wasm 1.4.0 → 1.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs CHANGED
@@ -1,1557 +1,11 @@
1
- import { arrowToRows } from "@gscdump/engine/arrow";
2
- import { DefaultLogger, NoopLogger, entityKind } from "drizzle-orm";
3
- import { PgAsyncDatabase, PgAsyncPreparedQuery, PgAsyncSession, PgDialect } from "drizzle-orm/pg-core";
4
- import { buildRelations } from "drizzle-orm/relations";
5
- import { createScopedHelpers } from "@gscdump/engine/scope";
6
- import { countries, dates, drizzleSchema as schema, hourly_pages, page_queries, pages, queries } from "@gscdump/engine/schema";
7
- import { runAnalyzerFromSource } from "@gscdump/engine/analyzer";
8
- import { pgResolverAdapter } from "@gscdump/engine/resolver";
9
- import { createAttachedTableSource } from "@gscdump/engine/source";
10
- import { sqlEscape } from "@gscdump/engine/sql";
11
- import { err, ok, unwrapResult } from "gscdump/result";
1
+ import { compileArchetypeSql, tableForArchetype } from "./archetype-sql.mjs";
2
+ import { createClient } from "./drizzle-adapter/client.mjs";
3
+ import { DuckDBWasmDatabase, drizzle } from "./drizzle-adapter/driver.mjs";
4
+ import "./drizzle-adapter/index.mjs";
5
+ import { overlayViewBody } from "./overlay-view.mjs";
6
+ import { OpfsQuotaExceededError, attachOpfsParquetTables, clearOpfsSnapshotCache, estimateOpfsStorage, readOpfsSnapshotFile, requestPersistentStorage } from "./opfs.mjs";
7
+ import { countries, dates, hourly_pages, page_queries, pages, queries, schema } from "./schema.mjs";
8
+ import { createInsightRunner, mergeScope, scopeFor } from "./runner.mjs";
9
+ import { BrowserAttachBudgetExceededError, attachParquetTables, attachParquetUrlTables, attachParquetUrlTablesResult, bootDuckDBWasm, browserAttachErrors, createBrowserAnalysisRuntime, isBrowserAttachError } from "./runtime.mjs";
12
10
  import { resolveWindow } from "@gscdump/engine/period";
13
- const METRIC_SQL = {
14
- clicks: "SUM(clicks)",
15
- impressions: "SUM(impressions)",
16
- ctr: "CASE WHEN SUM(impressions) = 0 THEN 0 ELSE SUM(clicks) * 1.0 / SUM(impressions) END",
17
- position: "CASE WHEN SUM(impressions) = 0 THEN NULL ELSE SUM(sum_position) * 1.0 / SUM(impressions) + 1 END"
18
- };
19
- const DIM_COLUMN = {
20
- page: "url",
21
- query: "query",
22
- country: "country",
23
- date: "date",
24
- searchAppearance: "search_appearance"
25
- };
26
- function quoteIdent(value) {
27
- return `"${value.replace(/"/g, "\"\"")}"`;
28
- }
29
- function qualifiedColumn(table, column) {
30
- return `${quoteIdent(table)}.${quoteIdent(column)}`;
31
- }
32
- function queryCanonicalExpr(table) {
33
- const queryRef = qualifiedColumn(table, "query");
34
- return `COALESCE((SELECT qd.query_canonical FROM query_dim qd WHERE qd.query = ${queryRef} LIMIT 1), ${queryRef})`;
35
- }
36
- function dimExpr(dim, table) {
37
- return dim === "queryCanonical" ? queryCanonicalExpr(table) : DIM_COLUMN[dim] ?? dim;
38
- }
39
- function tableForDimensions(dims) {
40
- const set = new Set(dims.filter((d) => d !== "date"));
41
- if (set.has("page") && (set.has("query") || set.has("queryCanonical"))) return "page_queries";
42
- if (set.has("query") || set.has("queryCanonical")) return "queries";
43
- if (set.has("country")) return "countries";
44
- if (set.has("device")) return "dates";
45
- return "pages";
46
- }
47
- function tableForTopNBreakdown(query) {
48
- const dims = [query.dimension];
49
- for (const facet of query.facets ?? []) {
50
- const hasPage = query.dimension === "page" || facet.column === "page";
51
- const hasQuery = query.dimension === "query" || query.dimension === "queryCanonical" || facet.column === "query" || facet.column === "queryCanonical";
52
- if (hasPage && hasQuery) dims.push(facet.column);
53
- }
54
- return tableForDimensions(dims);
55
- }
56
- function metricExpr(metric) {
57
- const expr = METRIC_SQL[metric];
58
- if (!expr) throw new Error(`[archetype-sql] unknown metric: ${metric}`);
59
- return expr;
60
- }
61
- function metricSelectList(metrics) {
62
- return metrics.map((m) => `${metricExpr(m)} AS ${m}`).join(", ");
63
- }
64
- const DEVICE_VALUES = [
65
- "DESKTOP",
66
- "MOBILE",
67
- "TABLET"
68
- ];
69
- const DEVICE_SUFFIX = {
70
- DESKTOP: "desktop",
71
- MOBILE: "mobile",
72
- TABLET: "tablet"
73
- };
74
- function deviceMetricExpr(metric, suffix) {
75
- switch (metric) {
76
- case "clicks": return `SUM(clicks_${suffix})`;
77
- case "impressions": return `SUM(impressions_${suffix})`;
78
- case "ctr": return `CASE WHEN SUM(impressions_${suffix}) = 0 THEN 0 ELSE SUM(clicks_${suffix}) * 1.0 / SUM(impressions_${suffix}) END`;
79
- case "position": return `CASE WHEN SUM(impressions_${suffix}) = 0 THEN NULL ELSE SUM(sum_position_${suffix}) * 1.0 / SUM(impressions_${suffix}) + 1 END`;
80
- default: throw new Error(`[archetype-sql] unknown metric: ${metric}`);
81
- }
82
- }
83
- function deviceUnpivotSql(metrics, whereSql, byDate) {
84
- const dateCol = byDate ? "date, " : "";
85
- const dateGroup = byDate ? " GROUP BY date" : "";
86
- return DEVICE_VALUES.map((dv) => {
87
- const suffix = DEVICE_SUFFIX[dv];
88
- const mcols = metrics.map((m) => `${deviceMetricExpr(m, suffix)} AS ${m}`).join(", ");
89
- return `SELECT ${dateCol}'${dv}' AS device, ${mcols} FROM dates WHERE ${whereSql}${dateGroup}`;
90
- }).join(" UNION ALL ");
91
- }
92
- function rangePredicate(q) {
93
- return {
94
- sql: "date BETWEEN ? AND ?",
95
- params: [q.range.start, q.range.end]
96
- };
97
- }
98
- function compareRangePredicate(q) {
99
- if (!q.compareRange) return null;
100
- return {
101
- sql: "date BETWEEN ? AND ?",
102
- params: [q.compareRange.start, q.compareRange.end]
103
- };
104
- }
105
- const STD_METRICS = [
106
- "clicks",
107
- "impressions",
108
- "ctr",
109
- "position"
110
- ];
111
- function coalesceMetric(metric, src, alias) {
112
- const ref = `${src}.${metric}`;
113
- return metric === "clicks" || metric === "impressions" ? `CAST(COALESCE(${ref}, 0) AS DOUBLE) AS ${alias}` : `COALESCE(${ref}, 0) AS ${alias}`;
114
- }
115
- function prevAlias(metric) {
116
- return `prev${metric.charAt(0).toUpperCase()}${metric.slice(1)}`;
117
- }
118
- function moverClause(movers) {
119
- const curClicks = "COALESCE(c.clicks, 0)";
120
- const prevClicks = "COALESCE(p.clicks, 0)";
121
- const curImpr = "COALESCE(c.impressions, 0)";
122
- const prevImpr = "COALESCE(p.impressions, 0)";
123
- switch (movers) {
124
- case "improving": return {
125
- where: `${curClicks} > ${prevClicks}`,
126
- order: `(${curClicks} - ${prevClicks}) DESC`
127
- };
128
- case "declining": return {
129
- where: `${curClicks} < ${prevClicks}`,
130
- order: `(${curClicks} - ${prevClicks}) ASC`
131
- };
132
- case "new": return {
133
- where: `${prevImpr} = 0 AND ${curImpr} > 0`,
134
- order: `${curClicks} DESC, ${curImpr} DESC`
135
- };
136
- case "lost": return {
137
- where: `${curImpr} = 0 AND ${prevImpr} > 0`,
138
- order: `${prevImpr} DESC`
139
- };
140
- default: throw new Error(`[archetype-sql] unknown movers mode: ${movers}`);
141
- }
142
- }
143
- function facetPredicate(query, table) {
144
- const facets = query.facets;
145
- if (!facets?.length) return {
146
- sql: "",
147
- params: []
148
- };
149
- const parts = [];
150
- const params = [];
151
- for (const f of facets) {
152
- const col = dimExpr(f.column, table);
153
- switch (f.op) {
154
- case "eq":
155
- parts.push(`${col} = ?`);
156
- params.push(f.value);
157
- break;
158
- case "regex":
159
- parts.push(`regexp_matches(LOWER(${col}), ?)`);
160
- params.push(f.value);
161
- break;
162
- case "notRegex":
163
- parts.push(`NOT regexp_matches(LOWER(${col}), ?)`);
164
- params.push(f.value);
165
- break;
166
- default: throw new Error(`[archetype-sql] unknown facet op: ${f.op}`);
167
- }
168
- }
169
- return {
170
- sql: parts.length ? ` AND ${parts.join(" AND ")}` : "",
171
- params
172
- };
173
- }
174
- function compileArchetypeSql(query) {
175
- switch (query.archetype) {
176
- case "site-daily-timeseries": {
177
- const table = "dates";
178
- const where = rangePredicate(query);
179
- return {
180
- table,
181
- sql: `SELECT date, ${query.metrics.map((m) => `${metricExpr(m)} AS ${m}`).join(", ")} FROM ${table} WHERE ${where.sql} GROUP BY date ORDER BY date`,
182
- params: where.params
183
- };
184
- }
185
- case "entity-daily-timeseries": {
186
- const table = tableForDimensions([query.entity.dimension]);
187
- const col = dimExpr(query.entity.dimension, table);
188
- const where = rangePredicate(query);
189
- return {
190
- table,
191
- sql: `SELECT date, ${metricSelectList(query.metrics)} FROM ${table} WHERE ${where.sql} AND ${col} = ? GROUP BY date ORDER BY date`,
192
- params: [...where.params, query.entity.value]
193
- };
194
- }
195
- case "entity-daily-sparkline": {
196
- const table = tableForDimensions([query.dimension]);
197
- const col = dimExpr(query.dimension, table);
198
- const where = rangePredicate(query);
199
- if (query.entities.length === 0) throw new Error("[archetype-sql] entity-daily-sparkline requires resolved entities");
200
- const placeholders = query.entities.map(() => "?").join(", ");
201
- return {
202
- table,
203
- sql: `SELECT ${col} AS entity, date, ${metricExpr(query.metric)} AS ${query.metric} FROM ${table} WHERE ${where.sql} AND ${col} IN (${placeholders}) GROUP BY ${col}, date ORDER BY ${col}, date`,
204
- params: [...where.params, ...query.entities]
205
- };
206
- }
207
- case "top-n-breakdown": {
208
- const table = tableForTopNBreakdown(query);
209
- const where = rangePredicate(query);
210
- const cmp = compareRangePredicate(query);
211
- const dir = query.orderBy.dir === "asc" ? "ASC" : "DESC";
212
- const compareOrder = `COALESCE(c.${query.orderBy.metric}, 0) ${dir}`;
213
- const metricList = query.metrics.includes(query.orderBy.metric) ? query.metrics : [...query.metrics, query.orderBy.metric];
214
- const variantSel = query.dimension === "queryCanonical" ? ", COUNT(DISTINCT query) AS variantCount" : "";
215
- if (query.dimension === "device") {
216
- if (cmp) {
217
- const curCols = metricList.map((m) => coalesceMetric(m, "c", m)).join(", ");
218
- const prevCols = STD_METRICS.map((m) => coalesceMetric(m, "p", prevAlias(m))).join(", ");
219
- let sql = `WITH cur AS (${deviceUnpivotSql(metricList, where.sql, false)}), prev AS (${deviceUnpivotSql(STD_METRICS, cmp.sql, false)}) SELECT COALESCE(c.device, p.device) AS device, ${curCols}, ${prevCols} FROM cur c FULL OUTER JOIN prev p ON c.device = p.device ORDER BY ${compareOrder} LIMIT ?`;
220
- const params = [
221
- ...where.params,
222
- ...where.params,
223
- ...where.params,
224
- ...cmp.params,
225
- ...cmp.params,
226
- ...cmp.params,
227
- query.limit
228
- ];
229
- if (query.offset && query.offset > 0) {
230
- sql += " OFFSET ?";
231
- params.push(query.offset);
232
- }
233
- return {
234
- table,
235
- sql,
236
- params
237
- };
238
- }
239
- let sql = `SELECT device, ${metricList.join(", ")} FROM (${deviceUnpivotSql(metricList, where.sql, false)}) ORDER BY ${query.orderBy.metric} ${dir} LIMIT ?`;
240
- const params = [
241
- ...where.params,
242
- ...where.params,
243
- ...where.params,
244
- query.limit
245
- ];
246
- if (query.offset && query.offset > 0) {
247
- sql += " OFFSET ?";
248
- params.push(query.offset);
249
- }
250
- return {
251
- table,
252
- sql,
253
- params
254
- };
255
- }
256
- const col = dimExpr(query.dimension, table);
257
- const facet = facetPredicate(query, table);
258
- const totalCol = query.includeTotal ? ", COUNT(*) OVER() AS __total" : "";
259
- if (cmp) {
260
- const curCols = metricList.map((m) => coalesceMetric(m, "c", m)).join(", ");
261
- const prevCols = STD_METRICS.map((m) => coalesceMetric(m, "p", prevAlias(m))).join(", ");
262
- const variantOut = query.dimension === "queryCanonical" ? ", c.variantCount AS variantCount" : "";
263
- const mover = query.movers ? moverClause(query.movers) : null;
264
- const moverWhere = mover ? `WHERE ${mover.where} ` : "";
265
- const orderSql = mover ? mover.order : compareOrder;
266
- let sql = `WITH cur AS (SELECT ${col} AS k, ${metricSelectList(metricList)}${variantSel} FROM ${table} WHERE ${where.sql}${facet.sql} GROUP BY ${col}), prev AS (SELECT ${col} AS k, ${metricSelectList(STD_METRICS)} FROM ${table} WHERE ${cmp.sql}${facet.sql} GROUP BY ${col}) SELECT COALESCE(c.k, p.k) AS ${query.dimension}, ${curCols}, ${prevCols}${variantOut}${totalCol} FROM cur c FULL OUTER JOIN prev p ON c.k = p.k ${moverWhere}ORDER BY ${orderSql} LIMIT ?`;
267
- const params = [
268
- ...where.params,
269
- ...facet.params,
270
- ...cmp.params,
271
- ...facet.params,
272
- query.limit
273
- ];
274
- if (query.offset && query.offset > 0) {
275
- sql += " OFFSET ?";
276
- params.push(query.offset);
277
- }
278
- return {
279
- table,
280
- sql,
281
- params
282
- };
283
- }
284
- let sql = `SELECT ${col} AS ${query.dimension}, ${metricSelectList(query.metrics)}${variantSel}${totalCol} FROM ${table} WHERE ${where.sql}${facet.sql} GROUP BY ${col} ORDER BY ${query.orderBy.metric} ${dir} LIMIT ?`;
285
- const params = [
286
- ...where.params,
287
- ...facet.params,
288
- query.limit
289
- ];
290
- if (query.offset && query.offset > 0) {
291
- sql += " OFFSET ?";
292
- params.push(query.offset);
293
- }
294
- return {
295
- table,
296
- sql,
297
- params
298
- };
299
- }
300
- case "single-row-lookup": {
301
- const table = tableForDimensions(Object.keys(query.match));
302
- const where = rangePredicate(query);
303
- const matchParts = [];
304
- const matchParams = [];
305
- for (const [dim, value] of Object.entries(query.match)) {
306
- const col = dimExpr(dim, table);
307
- if (!col) throw new Error(`[archetype-sql] single-row-lookup: unknown dimension ${dim}`);
308
- matchParts.push(`${col} = ?`);
309
- matchParams.push(value);
310
- }
311
- const matchSql = matchParts.length ? ` AND ${matchParts.join(" AND ")}` : "";
312
- return {
313
- table,
314
- sql: `SELECT ${metricSelectList(query.metrics)} FROM ${table} WHERE ${where.sql}${matchSql}`,
315
- params: [...where.params, ...matchParams]
316
- };
317
- }
318
- case "multi-series-stacked-daily": {
319
- const table = tableForDimensions([query.seriesDimension]);
320
- const where = rangePredicate(query);
321
- if (query.seriesDimension === "device") return {
322
- table,
323
- sql: `SELECT date, device, ${query.metric} FROM (${deviceUnpivotSql([query.metric], where.sql, true)}) ORDER BY date, device`,
324
- params: [
325
- ...where.params,
326
- ...where.params,
327
- ...where.params
328
- ]
329
- };
330
- const col = dimExpr(query.seriesDimension, table);
331
- return {
332
- table,
333
- sql: `SELECT date, ${col} AS ${query.seriesDimension}, ${metricExpr(query.metric)} AS ${query.metric} FROM ${table} WHERE ${where.sql} GROUP BY date, ${col} ORDER BY date, ${col}`,
334
- params: where.params
335
- };
336
- }
337
- case "two-dimension-detail": {
338
- const table = "page_queries";
339
- const where = rangePredicate(query);
340
- const facet = facetPredicate(query, table);
341
- const filterParts = [];
342
- const filterParams = [];
343
- if (query.filter?.page) {
344
- filterParts.push("url = ?");
345
- filterParams.push(query.filter.page);
346
- }
347
- if (query.filter?.query) {
348
- filterParts.push("query = ?");
349
- filterParams.push(query.filter.query);
350
- }
351
- const filterSql = filterParts.length ? ` AND ${filterParts.join(" AND ")}` : "";
352
- let sql = `SELECT url AS page, query, ${metricSelectList(query.metrics)} FROM ${table} WHERE ${where.sql}${filterSql}${facet.sql} GROUP BY url, query`;
353
- const params = [
354
- ...where.params,
355
- ...filterParams,
356
- ...facet.params
357
- ];
358
- if (query.orderBy) {
359
- const dir = query.orderBy.dir === "asc" ? "ASC" : "DESC";
360
- sql += ` ORDER BY ${query.orderBy.metric} ${dir}`;
361
- }
362
- if (typeof query.limit === "number") {
363
- sql += " LIMIT ?";
364
- params.push(query.limit);
365
- }
366
- return {
367
- table,
368
- sql,
369
- params
370
- };
371
- }
372
- case "arbitrary-sql": return {
373
- table: "page_queries",
374
- sql: query.sql,
375
- params: [...query.params ?? []]
376
- };
377
- case "aux-cloud-only": throw new Error("[archetype-sql] aux-cloud-only is not an Iceberg query — route to the cloud endpoint");
378
- default: throw new Error(`[archetype-sql] unhandled archetype: ${query.archetype}`);
379
- }
380
- }
381
- function tableForArchetype(query) {
382
- if (query.archetype === "aux-cloud-only") return null;
383
- return compileArchetypeSql(query).table;
384
- }
385
- async function createClient(db, conn) {
386
- return {
387
- db,
388
- conn,
389
- async query(sql, params = []) {
390
- if (params.length === 0) return arrowToRows(await conn.query(sql));
391
- const stmt = await conn.prepare(sql);
392
- try {
393
- return arrowToRows(await stmt.query(...params));
394
- } finally {
395
- await stmt.close();
396
- }
397
- },
398
- async close() {
399
- await conn.close();
400
- }
401
- };
402
- }
403
- var DuckDBWasmSession = class extends PgAsyncSession {
404
- client;
405
- options;
406
- static [entityKind] = "DuckDBWasmSession";
407
- logger;
408
- cache;
409
- constructor(client, dialect, options = {}) {
410
- super(dialect);
411
- this.client = client;
412
- this.options = options;
413
- this.logger = options.logger ?? new NoopLogger();
414
- this.cache = options.cache;
415
- }
416
- prepareQuery(query, mode, _name, mapper, queryMetadata, cacheConfig) {
417
- const client = this.client;
418
- const executor = async (params = []) => {
419
- const rows = await (await client).query(query.sql, params);
420
- return mode === "arrays" ? rows.map((row) => Object.values(row)) : rows;
421
- };
422
- return new PgAsyncPreparedQuery(executor, query, mapper, mode, this.logger, this.cache, queryMetadata, cacheConfig);
423
- }
424
- transaction() {
425
- throw new Error("Transactions are not supported by the DuckDB-WASM drizzle adapter. The analytics workload is read-only; if transactions become necessary, implement PgAsyncSession.transaction() in @gscdump/engine-duckdb-wasm.");
426
- }
427
- };
428
- var DuckDBWasmDatabase = class extends PgAsyncDatabase {
429
- static [entityKind] = "DuckDBWasmDatabase";
430
- };
431
- function drizzle(client, config = {}) {
432
- const dialect = new PgDialect();
433
- const clientPromise = Promise.resolve(client);
434
- let logger;
435
- if (config.logger === true) logger = new DefaultLogger();
436
- else if (config.logger !== false) logger = config.logger;
437
- const db = new DuckDBWasmDatabase(dialect, new DuckDBWasmSession(clientPromise, dialect, {
438
- logger,
439
- cache: config.cache
440
- }), config.relations ?? (config.schema ? buildRelations(config.schema, {}) : {}));
441
- db.$client = clientPromise;
442
- return db;
443
- }
444
- function createOpfsHandleRegistry(backend) {
445
- const entries = /* @__PURE__ */ new Map();
446
- const pending = /* @__PURE__ */ new Map();
447
- function reportCleanupFailure(operation, error) {
448
- console.warn(`[gscdump/engine-duckdb-wasm] ${operation} failed`, error);
449
- }
450
- async function acquire(name, openHandle) {
451
- const existing = entries.get(name);
452
- if (existing) {
453
- existing.refs++;
454
- return;
455
- }
456
- const inflight = pending.get(name);
457
- if (inflight) {
458
- await inflight.then(() => void 0, () => void 0);
459
- const settled = entries.get(name);
460
- if (settled) {
461
- settled.refs++;
462
- return;
463
- }
464
- return acquire(name, openHandle);
465
- }
466
- const registration = (async () => {
467
- const handle = await openHandle();
468
- await backend.register(name, handle);
469
- entries.set(name, {
470
- handle,
471
- refs: 1
472
- });
473
- })();
474
- pending.set(name, registration);
475
- try {
476
- await registration;
477
- } finally {
478
- pending.delete(name);
479
- }
480
- }
481
- async function release(names) {
482
- for (const name of names) {
483
- const entry = entries.get(name);
484
- if (!entry) continue;
485
- entry.refs--;
486
- if (entry.refs > 0) continue;
487
- entries.delete(name);
488
- await backend.drop(name).catch((error) => {
489
- reportCleanupFailure(`dropping OPFS handle ${name}`, error);
490
- });
491
- }
492
- }
493
- const viewRefs = /* @__PURE__ */ new Map();
494
- function acquireView(key, signature) {
495
- const existing = viewRefs.get(key);
496
- if (existing) {
497
- if (existing.signature !== signature) throw new Error(`view ${key} is already attached with a different signature`);
498
- existing.refs++;
499
- return;
500
- }
501
- viewRefs.set(key, {
502
- refs: 1,
503
- signature
504
- });
505
- }
506
- async function releaseView(key, drop) {
507
- const existing = viewRefs.get(key);
508
- const next = (existing?.refs ?? 0) - 1;
509
- if (next > 0) {
510
- viewRefs.set(key, {
511
- refs: next,
512
- signature: existing?.signature
513
- });
514
- return;
515
- }
516
- viewRefs.delete(key);
517
- await drop().catch((error) => {
518
- reportCleanupFailure(`dropping view ${key}`, error);
519
- });
520
- }
521
- return {
522
- acquire,
523
- release,
524
- size: () => entries.size,
525
- refs: (name) => entries.get(name)?.refs ?? 0,
526
- acquireView,
527
- releaseView,
528
- viewRefs: (key) => viewRefs.get(key)?.refs ?? 0,
529
- viewSignature: (key) => viewRefs.get(key)?.signature
530
- };
531
- }
532
- function overlayViewBody(args) {
533
- const { lakeSelect, overlaySelect } = args;
534
- const materialize = args.materializeLake ?? true;
535
- if (!lakeSelect && !overlaySelect) return null;
536
- if (!overlaySelect) return lakeSelect;
537
- if (!lakeSelect) return overlaySelect;
538
- if (materialize) return `WITH lake AS MATERIALIZED (${lakeSelect}), lake_dates AS (SELECT DISTINCT date FROM lake) SELECT * FROM lake UNION ALL BY NAME SELECT * FROM (${overlaySelect}) AS overlay WHERE overlay.date NOT IN (SELECT date FROM lake_dates)`;
539
- return `${lakeSelect} UNION ALL BY NAME SELECT * FROM (${overlaySelect}) AS overlay WHERE overlay.date NOT IN (SELECT DISTINCT date FROM (${lakeSelect}))`;
540
- }
541
- var OpfsQuotaExceededError = class extends Error {
542
- name = "OpfsQuotaExceededError";
543
- degradedTables;
544
- constructor(message, degradedTables) {
545
- super(message);
546
- this.degradedTables = degradedTables;
547
- }
548
- };
549
- const DEFAULT_CONCURRENCY = 2;
550
- const OPFS_PREFIX = "gscdump-snapshot__";
551
- const RECOVER_BUFFER_PREFIX = "gscdump-recover__";
552
- const MAX_CONTENT_HASH_SLUG_CACHE = 4096;
553
- const contentHashSlugCache = /* @__PURE__ */ new Map();
554
- const IDENT_RE = /^[A-Z_][\w$]*$/i;
555
- function nowMs() {
556
- return globalThis.performance?.now?.() ?? Date.now();
557
- }
558
- function isNotFoundError(error) {
559
- return typeof error === "object" && error !== null && error.name === "NotFoundError";
560
- }
561
- function reportBestEffortFailure(operation, error) {
562
- console.warn(`[gscdump/engine-duckdb-wasm] ${operation} failed`, error);
563
- }
564
- async function attemptCleanup(operation, cleanup) {
565
- try {
566
- await cleanup();
567
- } catch (error) {
568
- reportBestEffortFailure(operation, error);
569
- }
570
- }
571
- function emitTiming(onTiming, info) {
572
- try {
573
- onTiming?.(info);
574
- } catch (error) {
575
- reportBestEffortFailure("timing callback", error);
576
- }
577
- }
578
- async function timed(onTiming, stage, meta, fn) {
579
- const started = nowMs();
580
- try {
581
- return await fn();
582
- } finally {
583
- emitTiming(onTiming, {
584
- ...meta,
585
- stage,
586
- durationMs: nowMs() - started
587
- });
588
- }
589
- }
590
- const dbRegistries = /* @__PURE__ */ new WeakMap();
591
- function getOpfsRegistry(db, protocol) {
592
- let registry = dbRegistries.get(db);
593
- if (!registry) {
594
- registry = createOpfsHandleRegistry({
595
- register: (name, handle) => db.registerFileHandle(name, handle, protocol, true),
596
- drop: async (name) => {
597
- await db.dropFile(name);
598
- }
599
- });
600
- dbRegistries.set(db, registry);
601
- }
602
- return registry;
603
- }
604
- const dbBufferRegistries = /* @__PURE__ */ new WeakMap();
605
- function getBufferRegistry(db) {
606
- let registry = dbBufferRegistries.get(db);
607
- if (!registry) {
608
- registry = createOpfsHandleRegistry({
609
- register: (name, handle) => db.registerFileBuffer(name, handle),
610
- drop: async (name) => {
611
- await db.dropFile(name);
612
- }
613
- });
614
- dbBufferRegistries.set(db, registry);
615
- }
616
- return registry;
617
- }
618
- function assertSqlIdentifier(kind, value) {
619
- if (!IDENT_RE.test(value)) throw new TypeError(`[engine-duckdb-wasm/opfs] invalid ${kind} identifier ${JSON.stringify(value)}`);
620
- }
621
- const DOWNLOAD_DEADLINE_MS = 12e4;
622
- function isQuotaError(err) {
623
- if (typeof err !== "object" || err === null) return false;
624
- return err.name === "QuotaExceededError" || err.code === 22;
625
- }
626
- function isAbortError$1(err) {
627
- return typeof err === "object" && err !== null && err.name === "AbortError";
628
- }
629
- function isOpfsAccessHandleConflict(err) {
630
- const msg = err instanceof Error ? err.message : String(err);
631
- return /createSyncAccessHandle|Access Handle/i.test(msg);
632
- }
633
- function isOpfsWriteConflict(err) {
634
- if (typeof err !== "object" || err === null) return false;
635
- const name = err.name;
636
- const msg = err instanceof Error ? err.message : String(err);
637
- return name === "NoModificationAllowedError" || err.code === 7 || /createWritable|modifications are not allowed/i.test(msg);
638
- }
639
- function opfsFileName(table, hashSlug, index) {
640
- return hashSlug ? `${OPFS_PREFIX}${table}_${hashSlug}.parquet` : `${OPFS_PREFIX}${table}_${index}.parquet`;
641
- }
642
- function escapeRegExp(s) {
643
- return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
644
- }
645
- function tableEntryMatcher(table) {
646
- return new RegExp(`^${escapeRegExp(`${OPFS_PREFIX}${table}_`)}(?:[0-9a-f]{16}|\\d+(?:_[0-9a-f]{16})?)\\.parquet$`);
647
- }
648
- async function sweepStaleEntries(root, registry, expectedByTable) {
649
- const dir = root;
650
- if (!dir.keys) return;
651
- const matchers = [...expectedByTable].map(([table, expected]) => ({
652
- expected,
653
- re: tableEntryMatcher(table)
654
- }));
655
- for await (const name of dir.keys()) {
656
- const m = matchers.find((m) => m.re.test(name));
657
- if (!m || m.expected.has(name) || registry.refs(name) > 0) continue;
658
- try {
659
- await root.removeEntry(name);
660
- } catch (error) {
661
- if (!isNotFoundError(error)) reportBestEffortFailure(`removing stale OPFS entry ${name}`, error);
662
- }
663
- }
664
- }
665
- async function requestPersistentStorage() {
666
- const storage = globalThis.navigator?.storage;
667
- if (!storage?.persist) return false;
668
- if (await storage.persisted?.().catch(() => false)) return true;
669
- return storage.persist().catch(() => false);
670
- }
671
- async function estimateOpfsStorage() {
672
- const estimate = globalThis.navigator?.storage?.estimate;
673
- if (!estimate) return {};
674
- const est = await estimate.call(globalThis.navigator.storage).catch(() => null);
675
- return est ? {
676
- usageBytes: est.usage,
677
- quotaBytes: est.quota
678
- } : {};
679
- }
680
- async function contentHashSlug(contentHash) {
681
- const cached = contentHashSlugCache.get(contentHash);
682
- if (cached) return cached;
683
- if (contentHashSlugCache.size >= MAX_CONTENT_HASH_SLUG_CACHE) {
684
- const oldest = contentHashSlugCache.keys().next().value;
685
- if (oldest !== void 0) contentHashSlugCache.delete(oldest);
686
- }
687
- const promise = deriveContentHashSlug(contentHash).catch((err) => {
688
- contentHashSlugCache.delete(contentHash);
689
- throw err;
690
- });
691
- contentHashSlugCache.set(contentHash, promise);
692
- return promise;
693
- }
694
- async function deriveContentHashSlug(contentHash) {
695
- const digest = await globalThis.crypto.subtle.digest("SHA-256", new TextEncoder().encode(contentHash));
696
- const bytes = new Uint8Array(digest);
697
- let hex = "";
698
- for (let i = 0; i < 8; i++) hex += bytes[i].toString(16).padStart(2, "0");
699
- return hex;
700
- }
701
- async function getOpfsRoot() {
702
- const dir = globalThis.navigator?.storage?.getDirectory;
703
- if (!dir) throw new Error("[engine-duckdb-wasm/opfs] OPFS unavailable: navigator.storage.getDirectory missing");
704
- return dir.call(globalThis.navigator.storage);
705
- }
706
- async function readOpfsSnapshotFile(table, contentHash, index, expectedBytes) {
707
- try {
708
- const root = await getOpfsRoot();
709
- const name = opfsFileName(table, contentHash ? await contentHashSlug(contentHash) : void 0, index);
710
- const file = await (await root.getFileHandle(name)).getFile();
711
- if (file.size !== expectedBytes) return null;
712
- return new Uint8Array(await file.arrayBuffer());
713
- } catch (error) {
714
- if (!isNotFoundError(error)) reportBestEffortFailure(`reading OPFS snapshot ${table}`, error);
715
- return null;
716
- }
717
- }
718
- async function materialiseFile(root, name, file, fetchImpl, fetchInit, signal) {
719
- signal?.throwIfAborted();
720
- let handle;
721
- try {
722
- handle = await root.getFileHandle(name);
723
- if ((await handle.getFile()).size === file.bytes) return {
724
- handle,
725
- outcome: "cache-hit"
726
- };
727
- } catch (error) {
728
- if (!isNotFoundError(error)) throw error;
729
- }
730
- signal?.throwIfAborted();
731
- const deadline = AbortSignal.timeout(DOWNLOAD_DEADLINE_MS);
732
- const fetchSignal = signal ? AbortSignal.any([signal, deadline]) : deadline;
733
- const resp = await fetchImpl(file.url, {
734
- ...fetchInit,
735
- signal: fetchSignal
736
- });
737
- if (!resp.ok) throw new Error(`[engine-duckdb-wasm/opfs] download ${file.url} failed: ${resp.status}`);
738
- handle = await root.getFileHandle(name, { create: true });
739
- let writable;
740
- try {
741
- writable = await handle.createWritable();
742
- } catch (err) {
743
- await resp.body?.cancel().catch(() => void 0);
744
- throw err;
745
- }
746
- let reader;
747
- let bytesWritten = 0;
748
- try {
749
- if (resp.body) {
750
- reader = resp.body.getReader();
751
- while (true) {
752
- const { done, value } = await reader.read();
753
- if (done) break;
754
- bytesWritten += value.byteLength;
755
- if (bytesWritten > file.bytes) throw new Error(`[engine-duckdb-wasm/opfs] download ${file.url} byte length mismatch: expected ${file.bytes}, got more than ${file.bytes}`);
756
- await writable.write(value);
757
- }
758
- } else {
759
- const buf = await resp.arrayBuffer();
760
- bytesWritten = buf.byteLength;
761
- await writable.write(buf);
762
- }
763
- if (bytesWritten !== file.bytes) throw new Error(`[engine-duckdb-wasm/opfs] download ${file.url} byte length mismatch: expected ${file.bytes}, got ${bytesWritten}`);
764
- await writable.close();
765
- } catch (err) {
766
- await reader?.cancel().catch(() => void 0);
767
- if (writable.abort) await attemptCleanup(`aborting partial OPFS write ${name}`, () => writable.abort());
768
- await attemptCleanup(`removing partial OPFS file ${name}`, () => root.removeEntry(name));
769
- throw err;
770
- } finally {
771
- reader?.releaseLock();
772
- }
773
- return {
774
- handle,
775
- outcome: "downloaded"
776
- };
777
- }
778
- function quoteList(files) {
779
- return files.map((f) => `'${f.replace(/'/g, "''")}'`).join(", ");
780
- }
781
- function lakeSelect(files) {
782
- return `SELECT * REPLACE (CAST(date AS DATE) AS date) FROM read_parquet([${quoteList(files)}], union_by_name = true)`;
783
- }
784
- function readParquetViewSql$1(schema, table, files) {
785
- return `CREATE OR REPLACE VIEW ${schema}.${table} AS ${lakeSelect(files)}`;
786
- }
787
- function readParquetViewWithOverlaySql(schema, table, lakeFiles, overlayFile) {
788
- const overlay = `SELECT * REPLACE (CAST(date AS DATE) AS date) FROM read_parquet(['${overlayFile.replace(/'/g, "''")}'], union_by_name = true)`;
789
- return `CREATE OR REPLACE VIEW ${schema}.${table} AS ${overlayViewBody({
790
- lakeSelect: lakeFiles.length === 0 ? null : lakeSelect(lakeFiles),
791
- overlaySelect: overlay,
792
- materializeLake: false
793
- })}`;
794
- }
795
- function viewKey(schema, table) {
796
- return `${schema}.${table}`;
797
- }
798
- function viewSignature(lakeFiles, overlayFile) {
799
- return JSON.stringify({
800
- lake: [...lakeFiles].sort(),
801
- overlay: overlayFile ?? null
802
- });
803
- }
804
- async function ensureView(registry, conn, schema, table, signature, sql) {
805
- const key = viewKey(schema, table);
806
- if (registry.viewSignature(key) === signature) {
807
- registry.acquireView(key, signature);
808
- return;
809
- }
810
- if (registry.viewRefs(key) > 0) throw new Error(`[engine-duckdb-wasm/opfs] view ${key} is already attached with a different fileset`);
811
- await conn.query(sql);
812
- registry.acquireView(key, signature);
813
- }
814
- async function recoveredBufferName(table, file, index) {
815
- const slug = file.contentHash ? await contentHashSlug(file.contentHash) : String(index);
816
- return `${RECOVER_BUFFER_PREFIX}${table}_${slug}.parquet`;
817
- }
818
- async function runWithConcurrency$1(items, concurrency, fn) {
819
- let next = 0;
820
- let failed;
821
- async function worker() {
822
- while (failed === void 0 && next < items.length) {
823
- const index = next++;
824
- try {
825
- await fn(items[index], index);
826
- } catch (err) {
827
- failed = err;
828
- throw err;
829
- }
830
- }
831
- }
832
- const rejected = (await Promise.allSettled(Array.from({ length: Math.min(concurrency, items.length) }, worker))).find((r) => r.status === "rejected");
833
- if (rejected) throw rejected.reason;
834
- }
835
- async function attachOpfsParquetTables(options) {
836
- const { db, conn, tables, schema = "main", fetch: fetchImpl = globalThis.fetch.bind(globalThis), fetchInit, fetchConcurrency = DEFAULT_CONCURRENCY, signal, version, onFileProgress, onTiming, withDb = (fn) => fn(), recoverContention = true } = options;
837
- assertSqlIdentifier("schema", schema);
838
- for (const table of tables) assertSqlIdentifier("table", table.table);
839
- const totalStarted = nowMs();
840
- await timed(onTiming, "persist", {}, requestPersistentStorage);
841
- const root = await timed(onTiming, "root", {}, getOpfsRoot);
842
- const flat = [];
843
- const expectedByTable = /* @__PURE__ */ new Map();
844
- await timed(onTiming, "plan", { tables: tables.length }, async () => {
845
- for (const t of tables) {
846
- const expected = /* @__PURE__ */ new Set();
847
- for (let i = 0; i < t.files.length; i++) {
848
- const file = t.files[i];
849
- const slug = file.contentHash ? await contentHashSlug(file.contentHash) : void 0;
850
- const name = opfsFileName(t.table, slug, i);
851
- flat.push({
852
- table: t.table,
853
- file,
854
- name
855
- });
856
- expected.add(name);
857
- }
858
- if (t.overlay) {
859
- const slug = t.overlay.contentHash ? await contentHashSlug(t.overlay.contentHash) : void 0;
860
- const name = opfsFileName(t.table, slug, t.files.length);
861
- flat.push({
862
- table: t.table,
863
- file: t.overlay,
864
- name,
865
- overlay: true
866
- });
867
- expected.add(name);
868
- }
869
- expectedByTable.set(t.table, expected);
870
- }
871
- });
872
- const total = flat.length;
873
- const overlayNames = /* @__PURE__ */ new Map();
874
- const expectedCount = new Map(tables.map((t) => [t.table, t.files.length + (t.overlay ? 1 : 0)]));
875
- const { DuckDBDataProtocol } = await timed(onTiming, "duckdb-import", {}, () => import("@duckdb/duckdb-wasm"));
876
- const registry = getOpfsRegistry(db, DuckDBDataProtocol.BROWSER_FSACCESS);
877
- const bufferRegistry = getBufferRegistry(db);
878
- await timed(onTiming, "sweep", {
879
- files: total,
880
- tables: tables.length
881
- }, () => sweepStaleEntries(root, registry, expectedByTable)).catch((error) => {
882
- reportBestEffortFailure("stale OPFS cache sweep", error);
883
- });
884
- const tableFiles = /* @__PURE__ */ new Map();
885
- const degraded = /* @__PURE__ */ new Set();
886
- const degradeReason = /* @__PURE__ */ new Map();
887
- const acquiredNames = /* @__PURE__ */ new Set();
888
- const bufferFiles = [];
889
- const bufferViews = [];
890
- let bytesAttached = 0;
891
- const releaseTable = async (table) => {
892
- const names = (tableFiles.get(table) ?? []).map((f) => f.name);
893
- for (const n of names) acquiredNames.delete(n);
894
- await withDb(() => registry.release(names));
895
- };
896
- const runDownloads = () => runWithConcurrency$1(flat, Math.max(1, fetchConcurrency), async (item, index) => {
897
- if (degraded.has(item.table)) return;
898
- signal?.throwIfAborted();
899
- const fileStarted = nowMs();
900
- const name = item.name;
901
- let result;
902
- let materialiseMs = 0;
903
- let registerMs = 0;
904
- try {
905
- const materialiseStarted = nowMs();
906
- result = await materialiseFile(root, name, item.file, fetchImpl, fetchInit, signal);
907
- materialiseMs = nowMs() - materialiseStarted;
908
- emitTiming(onTiming, {
909
- stage: "materialise",
910
- durationMs: materialiseMs,
911
- table: item.table,
912
- file: name,
913
- bytes: item.file.bytes,
914
- outcome: result.outcome
915
- });
916
- } catch (err) {
917
- if (isAbortError$1(err)) throw err;
918
- if (isQuotaError(err)) {
919
- degraded.add(item.table);
920
- degradeReason.set(item.table, "quota");
921
- return;
922
- }
923
- if (isOpfsWriteConflict(err)) {
924
- degraded.add(item.table);
925
- degradeReason.set(item.table, "contention");
926
- return;
927
- }
928
- throw err;
929
- }
930
- try {
931
- const registerStarted = nowMs();
932
- await withDb(() => registry.acquire(name, () => result.handle));
933
- registerMs = nowMs() - registerStarted;
934
- emitTiming(onTiming, {
935
- stage: "register",
936
- durationMs: registerMs,
937
- table: item.table,
938
- file: name,
939
- bytes: item.file.bytes,
940
- outcome: result.outcome
941
- });
942
- acquiredNames.add(name);
943
- } catch (err) {
944
- if (isAbortError$1(err)) throw err;
945
- if (isOpfsAccessHandleConflict(err)) {
946
- degraded.add(item.table);
947
- degradeReason.set(item.table, "contention");
948
- return;
949
- }
950
- throw err;
951
- }
952
- const list = tableFiles.get(item.table) ?? [];
953
- list.push({
954
- name,
955
- handle: result.handle
956
- });
957
- tableFiles.set(item.table, list);
958
- if (item.overlay) overlayNames.set(item.table, name);
959
- bytesAttached += item.file.bytes;
960
- onFileProgress?.({
961
- table: item.table,
962
- file: name,
963
- index,
964
- total,
965
- bytes: item.file.bytes,
966
- outcome: result.outcome,
967
- materialiseMs,
968
- registerMs,
969
- totalMs: nowMs() - fileStarted
970
- });
971
- });
972
- try {
973
- await timed(onTiming, "downloads", {
974
- files: total,
975
- tables: tables.length
976
- }, runDownloads);
977
- } catch (err) {
978
- await attemptCleanup("releasing handles after download failure", () => withDb(() => registry.release([...acquiredNames])));
979
- throw err;
980
- }
981
- const attached = [];
982
- const registeredNames = [];
983
- for (const t of tables) {
984
- if (degraded.has(t.table)) {
985
- await releaseTable(t.table);
986
- continue;
987
- }
988
- const files = tableFiles.get(t.table) ?? [];
989
- if (files.length !== (expectedCount.get(t.table) ?? t.files.length)) {
990
- degraded.add(t.table);
991
- degradeReason.set(t.table, degradeReason.get(t.table) ?? "incomplete");
992
- await releaseTable(t.table);
993
- continue;
994
- }
995
- const overlayName = overlayNames.get(t.table);
996
- const lakeNames = (overlayName ? files.map((f) => f.name).filter((n) => n !== overlayName) : files.map((f) => f.name)).sort();
997
- const signature = viewSignature(lakeNames, overlayName);
998
- try {
999
- signal?.throwIfAborted();
1000
- await timed(onTiming, "view", {
1001
- table: t.table,
1002
- files: files.length
1003
- }, () => withDb(() => ensureView(registry, conn, schema, t.table, signature, overlayName ? readParquetViewWithOverlaySql(schema, t.table, lakeNames, overlayName) : readParquetViewSql$1(schema, t.table, lakeNames))));
1004
- } catch (err) {
1005
- if (isAbortError$1(err)) {
1006
- await attemptCleanup("detaching OPFS resources after abort", () => withDb(() => detachOpfs(registry, conn, schema, attached, [...acquiredNames])));
1007
- throw err;
1008
- }
1009
- if (isOpfsAccessHandleConflict(err)) {
1010
- if (registry.viewRefs(viewKey(schema, t.table)) === 0) await attemptCleanup(`dropping incomplete view ${schema}.${t.table}`, () => withDb(() => conn.query(`DROP VIEW IF EXISTS ${schema}.${t.table}`)));
1011
- degraded.add(t.table);
1012
- degradeReason.set(t.table, "contention");
1013
- await releaseTable(t.table);
1014
- continue;
1015
- }
1016
- await attemptCleanup("detaching OPFS resources after view failure", () => withDb(() => detachOpfs(registry, conn, schema, attached, [...acquiredNames])));
1017
- throw err;
1018
- }
1019
- attached.push(t.table);
1020
- for (const f of files) registeredNames.push(f.name);
1021
- }
1022
- const recoverTableToBuffers = async (t) => {
1023
- return timed(onTiming, "recovery", {
1024
- table: t.table,
1025
- files: t.files.length + (t.overlay ? 1 : 0)
1026
- }, async () => {
1027
- const readOne = async (file, index) => {
1028
- const cached = await readOpfsSnapshotFile(t.table, file.contentHash, index, file.bytes);
1029
- if (cached) return cached;
1030
- signal?.throwIfAborted();
1031
- const resp = await fetchImpl(file.url, {
1032
- ...fetchInit,
1033
- signal
1034
- });
1035
- if (!resp.ok) throw new Error(`[engine-duckdb-wasm/opfs] recover ${file.url} failed: ${resp.status}`);
1036
- return new Uint8Array(await resp.arrayBuffer());
1037
- };
1038
- const lakeNames = [];
1039
- for (let i = 0; i < t.files.length; i++) {
1040
- const file = t.files[i];
1041
- const name = await recoveredBufferName(t.table, file, i);
1042
- const buf = await readOne(file, i);
1043
- await withDb(() => bufferRegistry.acquire(name, () => buf));
1044
- bufferFiles.push(name);
1045
- lakeNames.push(name);
1046
- bytesAttached += file.bytes;
1047
- }
1048
- let overlayName;
1049
- if (t.overlay) {
1050
- overlayName = await recoveredBufferName(t.table, t.overlay, t.files.length);
1051
- const buf = await readOne(t.overlay, t.files.length);
1052
- await withDb(() => bufferRegistry.acquire(overlayName, () => buf));
1053
- bufferFiles.push(overlayName);
1054
- bytesAttached += t.overlay.bytes;
1055
- }
1056
- lakeNames.sort();
1057
- const body = overlayViewBody({
1058
- lakeSelect: lakeNames.length ? lakeSelect(lakeNames) : null,
1059
- overlaySelect: overlayName ? `SELECT * REPLACE (CAST(date AS DATE) AS date) FROM read_parquet(['${overlayName.replace(/'/g, "''")}'], union_by_name = true)` : null,
1060
- materializeLake: false
1061
- });
1062
- if (!body) return false;
1063
- await withDb(() => ensureView(registry, conn, schema, t.table, viewSignature(lakeNames, overlayName), `CREATE OR REPLACE VIEW ${schema}.${t.table} AS ${body}`));
1064
- bufferViews.push(t.table);
1065
- return true;
1066
- });
1067
- };
1068
- if (recoverContention && !signal?.aborted) for (const t of tables) {
1069
- if (!degraded.has(t.table) || degradeReason.get(t.table) !== "contention") continue;
1070
- const before = bufferFiles.length;
1071
- if (await recoverTableToBuffers(t).catch(() => false)) degraded.delete(t.table);
1072
- else {
1073
- await attemptCleanup(`releasing recovery buffers for ${t.table}`, () => withDb(() => bufferRegistry.release(bufferFiles.slice(before))));
1074
- bufferFiles.length = before;
1075
- }
1076
- }
1077
- emitTiming(onTiming, {
1078
- stage: "total",
1079
- durationMs: nowMs() - totalStarted,
1080
- files: total,
1081
- tables: tables.length,
1082
- bytes: bytesAttached
1083
- });
1084
- let detached = false;
1085
- return {
1086
- version,
1087
- tables: [...attached, ...bufferViews],
1088
- schema,
1089
- bytesAttached,
1090
- degradedTables: [...degraded],
1091
- degradeReasons: Object.fromEntries(degradeReason),
1092
- async detach() {
1093
- if (detached) return;
1094
- detached = true;
1095
- await detachOpfs(registry, conn, schema, attached, registeredNames);
1096
- for (const v of bufferViews) await registry.releaseView(viewKey(schema, v), () => conn.query(`DROP VIEW IF EXISTS ${schema}.${v}`).then(() => {}));
1097
- await bufferRegistry.release(bufferFiles);
1098
- }
1099
- };
1100
- }
1101
- async function detachOpfs(registry, conn, schema, tables, files) {
1102
- for (const table of tables) await registry.releaseView(`${schema}.${table}`, () => conn.query(`DROP VIEW IF EXISTS ${schema}.${table}`).then(() => {}));
1103
- await registry.release(files);
1104
- }
1105
- async function clearOpfsSnapshotCache() {
1106
- const root = await getOpfsRoot().catch((error) => {
1107
- reportBestEffortFailure("opening OPFS cache for clearing", error);
1108
- return null;
1109
- });
1110
- if (!root) return;
1111
- const removable = [];
1112
- const dir = root;
1113
- if (dir.keys) {
1114
- for await (const name of dir.keys()) if (name.startsWith(OPFS_PREFIX)) removable.push(name);
1115
- }
1116
- for (const name of removable) try {
1117
- await root.removeEntry(name);
1118
- } catch (error) {
1119
- if (!isNotFoundError(error)) reportBestEffortFailure(`clearing OPFS cache entry ${name}`, error);
1120
- }
1121
- }
1122
- async function createInsightRunner(opts) {
1123
- const client = await createClient(opts.db, opts.conn);
1124
- const clientPromise = Promise.resolve(client);
1125
- return {
1126
- db: drizzle(clientPromise, {
1127
- schema,
1128
- logger: opts.logger
1129
- }),
1130
- client: clientPromise,
1131
- close: () => client.close()
1132
- };
1133
- }
1134
- const { scopeFor, mergeScope } = createScopedHelpers(schema);
1135
- const DEFAULT_ATTACH_FETCH_CONCURRENCY = 2;
1136
- const DEFAULT_ATTACH_MAX_FILES = 32;
1137
- const DEFAULT_ATTACH_MAX_BYTES = 16 * 1024 * 1024;
1138
- let nextAttachId = 0;
1139
- var BrowserAttachBudgetExceededError = class extends Error {
1140
- name = "BrowserAttachBudgetExceededError";
1141
- };
1142
- const browserAttachErrors = {
1143
- maxFilesExceeded(files, maxFiles) {
1144
- return {
1145
- kind: "browser-attach-budget-exceeded",
1146
- budget: "maxFiles",
1147
- message: `browser parquet attach requires ${files} files, above maxFiles=${maxFiles}`
1148
- };
1149
- },
1150
- hintedBytesExceeded(hintedBytes, maxBytes) {
1151
- return {
1152
- kind: "browser-attach-budget-exceeded",
1153
- budget: "maxBytes",
1154
- message: `browser parquet attach requires ${hintedBytes} hinted bytes, above maxBytes=${maxBytes}`
1155
- };
1156
- },
1157
- plannedBytesExceeded(plannedBytes, maxBytes) {
1158
- return {
1159
- kind: "browser-attach-budget-exceeded",
1160
- budget: "maxBytes",
1161
- message: `browser parquet attach planned ${plannedBytes} bytes, above maxBytes=${maxBytes}`
1162
- };
1163
- }
1164
- };
1165
- function isBrowserAttachError(value) {
1166
- return typeof value === "object" && value !== null && value.kind === "browser-attach-budget-exceeded" && typeof value.message === "string";
1167
- }
1168
- function browserAttachErrorToException(error) {
1169
- const exception = new BrowserAttachBudgetExceededError(error.message);
1170
- exception.browserAttachError = error;
1171
- return exception;
1172
- }
1173
- function fileName(table, index, provided) {
1174
- return provided ?? `${table}_${index}.parquet`;
1175
- }
1176
- function attachFileName(attachId, table, index) {
1177
- return `__gscdump_attach_${attachId}_${fileName(table, index)}`;
1178
- }
1179
- function readParquetViewSql(schema, table, files) {
1180
- return `CREATE OR REPLACE VIEW ${schema}.${table} AS SELECT * REPLACE (CAST(date AS DATE) AS date) FROM read_parquet([${files.map((name) => `'${sqlEscape(name)}'`).join(", ")}], union_by_name = true)`;
1181
- }
1182
- function positiveInteger(value, fallback, label) {
1183
- const raw = value ?? fallback;
1184
- if (!Number.isFinite(raw) || raw < 1) throw new Error(`${label} must be a positive integer`);
1185
- return Math.floor(raw);
1186
- }
1187
- async function runWithConcurrency(items, concurrency, fn) {
1188
- let next = 0;
1189
- let failed = false;
1190
- async function worker() {
1191
- while (!failed && next < items.length) {
1192
- const index = next++;
1193
- try {
1194
- await fn(items[index], index);
1195
- } catch (err) {
1196
- failed = true;
1197
- throw err;
1198
- }
1199
- }
1200
- }
1201
- const rejected = (await Promise.allSettled(Array.from({ length: Math.min(concurrency, items.length) }, worker))).find((result) => result.status === "rejected");
1202
- if (rejected) throw rejected.reason;
1203
- }
1204
- function sizeHintFromUrl(url) {
1205
- try {
1206
- const base = typeof globalThis.location?.href === "string" ? globalThis.location.href : "http://localhost";
1207
- const raw = new URL(url, base).searchParams.get("s")?.split(".")[0];
1208
- if (!raw) return null;
1209
- const size = Number(raw);
1210
- return Number.isFinite(size) && size >= 0 ? size : null;
1211
- } catch {
1212
- return null;
1213
- }
1214
- }
1215
- function mergeAbortSignals(primary, secondary) {
1216
- if (!primary) return secondary;
1217
- if (!secondary) return primary;
1218
- if (primary.aborted) return primary;
1219
- if (secondary.aborted) return secondary;
1220
- const controller = new AbortController();
1221
- const abort = (signal) => {
1222
- controller.abort(signal.reason);
1223
- };
1224
- primary.addEventListener("abort", () => abort(primary), { once: true });
1225
- secondary.addEventListener("abort", () => abort(secondary), { once: true });
1226
- return controller.signal;
1227
- }
1228
- function fetchInitFor(fetchInit, method, signal, extraHeaders) {
1229
- const { body: _body, method: _method, signal: initSignal, headers: initHeaders, ...rest } = fetchInit ?? {};
1230
- const headers = new Headers(initHeaders);
1231
- for (const [key, value] of Object.entries(extraHeaders ?? {})) headers.set(key, value);
1232
- return {
1233
- ...rest,
1234
- method,
1235
- headers,
1236
- signal: mergeAbortSignals(signal, initSignal ?? void 0)
1237
- };
1238
- }
1239
- function isAbortError(err) {
1240
- return typeof err === "object" && err !== null && err.name === "AbortError";
1241
- }
1242
- function parseContentLength(headers) {
1243
- const raw = headers.get("content-length");
1244
- if (!raw) return null;
1245
- const size = Number(raw);
1246
- return Number.isFinite(size) && size >= 0 ? size : null;
1247
- }
1248
- function parseContentRangeSize(headers) {
1249
- const raw = headers.get("content-range");
1250
- if (!raw) return null;
1251
- const match = /^bytes\s+\d+-\d+\/(\d+)$/i.exec(raw);
1252
- if (!match) return null;
1253
- const size = Number(match[1]);
1254
- return Number.isFinite(size) && size >= 0 ? size : null;
1255
- }
1256
- function supportsRangeReads(headers) {
1257
- return headers.get("accept-ranges")?.toLowerCase().split(",").map((v) => v.trim()).includes("bytes") === true;
1258
- }
1259
- async function cancelBody(response) {
1260
- await response.body?.cancel().catch(() => void 0);
1261
- }
1262
- async function preflightHttpUrl(url, fetchImpl, fetchInit, signal) {
1263
- const head = await fetchImpl(url, fetchInitFor(fetchInit, "HEAD", signal));
1264
- if (head.ok) {
1265
- const size = parseContentLength(head.headers);
1266
- if (size === null) throw new Error(`HEAD ${url} missing Content-Length`);
1267
- if (!supportsRangeReads(head.headers)) throw new Error(`HEAD ${url} missing Accept-Ranges: bytes`);
1268
- return size;
1269
- }
1270
- await cancelBody(head);
1271
- if (![
1272
- 403,
1273
- 405,
1274
- 501
1275
- ].includes(head.status)) throw new Error(`HEAD ${url} failed: ${head.status}`);
1276
- const probe = await fetchImpl(url, fetchInitFor(fetchInit, "GET", signal, { Range: "bytes=0-0" }));
1277
- try {
1278
- if (probe.status !== 206) throw new Error(`range probe ${url} failed: ${probe.status}`);
1279
- const size = parseContentRangeSize(probe.headers);
1280
- if (size === null) throw new Error(`range probe ${url} missing Content-Range size`);
1281
- return size;
1282
- } finally {
1283
- await cancelBody(probe);
1284
- }
1285
- }
1286
- function rangeOnlyConfig(config) {
1287
- return {
1288
- ...config ?? {},
1289
- filesystem: {
1290
- reliableHeadRequests: true,
1291
- allowFullHTTPReads: false,
1292
- ...config?.filesystem ?? {},
1293
- forceFullHTTPReads: false
1294
- }
1295
- };
1296
- }
1297
- async function dropAttachedResources(db, conn, schema, tables, files) {
1298
- for (const table of tables) await conn.query(`DROP VIEW IF EXISTS ${schema}.${table}`);
1299
- if (files.length > 0) await db.dropFiles([...files]);
1300
- }
1301
- function parseMemoryLimit(value) {
1302
- const trimmed = value.trim();
1303
- if (!/^\d+(?:\.\d+)?\s*(?:B|KB|MB|GB|TB)?$/i.test(trimmed)) throw new Error(`invalid memoryLimit '${value}' — expected e.g. '512MB' or '2GB'`);
1304
- return trimmed;
1305
- }
1306
- async function bootDuckDBWasm(options = {}) {
1307
- const { getJsDelivrBundles, selectBundle, AsyncDuckDB, ConsoleLogger, LogLevel } = await import("@duckdb/duckdb-wasm");
1308
- const bundle = await selectBundle(options.bundles ?? getJsDelivrBundles());
1309
- const workerUrl = URL.createObjectURL(new Blob([`importScripts("${bundle.mainWorker}");`], { type: "text/javascript" }));
1310
- const worker = new Worker(workerUrl);
1311
- const db = new AsyncDuckDB(options.logger ?? new ConsoleLogger(LogLevel.WARNING), worker);
1312
- try {
1313
- await db.instantiate(bundle.mainModule, bundle.pthreadWorker);
1314
- await db.open(rangeOnlyConfig(options.config));
1315
- const conn = await db.connect();
1316
- if (options.memoryLimit !== void 0) await conn.query(`SET memory_limit='${parseMemoryLimit(options.memoryLimit)}'`);
1317
- return {
1318
- db,
1319
- conn
1320
- };
1321
- } catch (err) {
1322
- worker.terminate();
1323
- throw err;
1324
- } finally {
1325
- URL.revokeObjectURL(workerUrl);
1326
- }
1327
- }
1328
- async function attachParquetTables(options) {
1329
- const { db, conn, tables, schema = "main" } = options;
1330
- for (const table of tables) {
1331
- const names = [];
1332
- for (let i = 0; i < table.files.length; i++) {
1333
- const file = table.files[i];
1334
- const name = fileName(table.table, i, file.name);
1335
- names.push(name);
1336
- await db.registerFileBuffer(name, file.bytes);
1337
- }
1338
- await conn.query(readParquetViewSql(schema, table.table, names));
1339
- }
1340
- }
1341
- async function attachParquetUrlTablesResult(options) {
1342
- const { db, conn, tables, fetch: fetchImpl = globalThis.fetch.bind(globalThis), schema = "main", fetchInit, fetchConcurrency, maxFiles, maxBytes, signal, version, onFileAttached, attachMode = "http", trustSizeHint = true } = options;
1343
- const concurrency = positiveInteger(fetchConcurrency, DEFAULT_ATTACH_FETCH_CONCURRENCY, "fetchConcurrency");
1344
- const fileBudget = positiveInteger(maxFiles, DEFAULT_ATTACH_MAX_FILES, "maxFiles");
1345
- const byteBudget = positiveInteger(maxBytes, DEFAULT_ATTACH_MAX_BYTES, "maxBytes");
1346
- const attachId = nextAttachId++;
1347
- const flat = [];
1348
- const counts = {};
1349
- for (const [table, urls] of tables.map((t) => [t.table, t.urls])) {
1350
- if (urls.length === 0) continue;
1351
- counts[table] = urls.length;
1352
- for (let i = 0; i < urls.length; i++) flat.push({
1353
- table,
1354
- url: urls[i],
1355
- index: i
1356
- });
1357
- }
1358
- if (flat.length > fileBudget) return err(browserAttachErrors.maxFilesExceeded(flat.length, fileBudget));
1359
- const hintedBytes = flat.reduce((acc, item) => {
1360
- const hint = sizeHintFromUrl(item.url);
1361
- return hint === null ? acc : acc + hint;
1362
- }, 0);
1363
- if (hintedBytes > byteBudget) return err(browserAttachErrors.hintedBytesExceeded(hintedBytes, byteBudget));
1364
- const tableFailures = /* @__PURE__ */ new Map();
1365
- const budgetController = new AbortController();
1366
- const effectiveSignal = mergeAbortSignals(signal, budgetController.signal);
1367
- let plannedBytes = 0;
1368
- const total = flat.length;
1369
- const prepared = [];
1370
- try {
1371
- await runWithConcurrency(flat, concurrency, async ({ table, url, index }) => {
1372
- if (tableFailures.has(table)) return;
1373
- effectiveSignal?.throwIfAborted();
1374
- try {
1375
- let bytes = null;
1376
- let body = null;
1377
- if (attachMode === "buffer") {
1378
- const res = await fetchImpl(url, fetchInitFor(fetchInit, "GET", effectiveSignal));
1379
- if (!res.ok) throw new Error(`GET ${url} failed: ${res.status}`);
1380
- const buf = new Uint8Array(await res.arrayBuffer());
1381
- body = buf;
1382
- bytes = buf.byteLength;
1383
- } else if (trustSizeHint && sizeHintFromUrl(url) !== null) bytes = sizeHintFromUrl(url);
1384
- else bytes = await preflightHttpUrl(url, fetchImpl, fetchInit, effectiveSignal);
1385
- plannedBytes += bytes;
1386
- if (plannedBytes > byteBudget) {
1387
- const budgetErr = new BrowserAttachBudgetExceededError(`browser parquet attach planned ${plannedBytes} bytes, above maxBytes=${byteBudget}`);
1388
- budgetController.abort(budgetErr);
1389
- throw budgetErr;
1390
- }
1391
- effectiveSignal?.throwIfAborted();
1392
- prepared.push({
1393
- table,
1394
- url,
1395
- index,
1396
- name: attachFileName(attachId, table, index),
1397
- bytes,
1398
- body
1399
- });
1400
- } catch (err) {
1401
- if (effectiveSignal?.aborted || err instanceof BrowserAttachBudgetExceededError || isAbortError(err)) throw err;
1402
- tableFailures.set(table, err instanceof Error ? err : new Error(String(err)));
1403
- }
1404
- });
1405
- } catch (downloadErr) {
1406
- if (downloadErr instanceof BrowserAttachBudgetExceededError && !signal?.aborted) return err(browserAttachErrors.plannedBytesExceeded(plannedBytes, byteBudget));
1407
- throw downloadErr;
1408
- }
1409
- const { DuckDBDataProtocol } = await import("@duckdb/duckdb-wasm");
1410
- const attached = [];
1411
- const registeredFiles = [];
1412
- try {
1413
- for (const file of prepared) {
1414
- if (tableFailures.has(file.table)) continue;
1415
- effectiveSignal?.throwIfAborted();
1416
- if (file.body !== null) await db.registerFileBuffer(file.name, file.body);
1417
- else await db.registerFileURL(file.name, file.url, DuckDBDataProtocol.HTTP, false);
1418
- registeredFiles.push(file.name);
1419
- onFileAttached?.({
1420
- table: file.table,
1421
- index: file.index,
1422
- total
1423
- });
1424
- }
1425
- for (const table of Object.keys(counts)) {
1426
- if (tableFailures.has(table)) continue;
1427
- const files = prepared.filter((file) => file.table === table).sort((a, b) => a.index - b.index);
1428
- if (files.length !== counts[table]) continue;
1429
- effectiveSignal?.throwIfAborted();
1430
- await conn.query(readParquetViewSql(schema, table, files.map((file) => file.name)));
1431
- attached.push(table);
1432
- }
1433
- } catch (err) {
1434
- await dropAttachedResources(db, conn, schema, attached, registeredFiles).catch((cleanupErr) => {
1435
- console.warn("[gscdump/engine-duckdb-wasm] cleanup after failed attach failed", cleanupErr);
1436
- });
1437
- throw err;
1438
- }
1439
- if (tableFailures.size > 0) for (const [table, err] of tableFailures) console.warn(`[gscdump/engine-duckdb-wasm] dropped table "${table}" — ${err.message}`);
1440
- let detached = false;
1441
- return ok({
1442
- version,
1443
- tables: attached,
1444
- schema,
1445
- async detach() {
1446
- if (detached) return;
1447
- detached = true;
1448
- await dropAttachedResources(db, conn, schema, attached, registeredFiles);
1449
- }
1450
- });
1451
- }
1452
- async function attachParquetUrlTables(options) {
1453
- return unwrapResult(await attachParquetUrlTablesResult(options), browserAttachErrorToException);
1454
- }
1455
- function createBrowserAnalysisRuntime(boot, options = {}) {
1456
- const { db, conn } = boot;
1457
- const schema = options.schema ?? "main";
1458
- let version = options.version;
1459
- let attachedTables = options.attachedTables;
1460
- let chain = Promise.resolve();
1461
- function abortError(signal) {
1462
- return signal.reason ?? new DOMException("aborted", "AbortError");
1463
- }
1464
- function raceSignal(promise, signal) {
1465
- if (!signal) return promise;
1466
- if (signal.aborted) return Promise.reject(abortError(signal));
1467
- return new Promise((resolve, reject) => {
1468
- const onAbort = () => reject(abortError(signal));
1469
- signal.addEventListener("abort", onAbort, { once: true });
1470
- promise.then((value) => {
1471
- signal.removeEventListener("abort", onAbort);
1472
- resolve(value);
1473
- }, (err) => {
1474
- signal.removeEventListener("abort", onAbort);
1475
- reject(err);
1476
- });
1477
- });
1478
- }
1479
- function runExclusive(signal, work) {
1480
- const next = chain.then(work, work);
1481
- chain = next.then(() => void 0, () => void 0);
1482
- return raceSignal(next, signal);
1483
- }
1484
- async function cancelOnAbort(signal, work) {
1485
- if (!signal) return work;
1486
- if (signal.aborted) throw abortError(signal);
1487
- const onAbort = () => {
1488
- conn.cancelSent().catch((error) => {
1489
- console.warn("[gscdump/engine-duckdb-wasm] failed to cancel aborted query", error);
1490
- });
1491
- };
1492
- signal.addEventListener("abort", onAbort, { once: true });
1493
- try {
1494
- return await work;
1495
- } finally {
1496
- signal.removeEventListener("abort", onAbort);
1497
- }
1498
- }
1499
- async function runParameterizedDirect(sql, params, signal) {
1500
- signal?.throwIfAborted();
1501
- return cancelOnAbort(signal, (async () => {
1502
- if (!params || params.length === 0) return conn.query(sql);
1503
- const stmt = await conn.prepare(sql);
1504
- try {
1505
- return await stmt.query(...params);
1506
- } finally {
1507
- await stmt.close();
1508
- }
1509
- })());
1510
- }
1511
- return {
1512
- db,
1513
- conn,
1514
- async query(sql, params, signal) {
1515
- const t0 = performance.now();
1516
- return {
1517
- rows: arrowToRows(await runExclusive(signal, () => runParameterizedDirect(sql, params, signal))),
1518
- queryMs: performance.now() - t0
1519
- };
1520
- },
1521
- async analyze(params, registry, options = {}) {
1522
- const signal = options.signal;
1523
- const run = async () => {
1524
- signal?.throwIfAborted();
1525
- const t0 = performance.now();
1526
- const result = await runAnalyzerFromSource(createAttachedTableSource({ query: async (sql, bindParams, innerSignal) => {
1527
- return arrowToRows(await runParameterizedDirect(sql, bindParams, innerSignal ?? signal));
1528
- } }, {
1529
- schema,
1530
- signal,
1531
- attachedTables,
1532
- adapter: pgResolverAdapter
1533
- }), params, registry);
1534
- return {
1535
- results: result.results,
1536
- meta: result.meta,
1537
- queryMs: performance.now() - t0
1538
- };
1539
- };
1540
- return runExclusive(signal, run);
1541
- },
1542
- isStale(expected) {
1543
- return expected !== version;
1544
- },
1545
- setVersion(next) {
1546
- version = next;
1547
- },
1548
- setAttachedTables(next) {
1549
- attachedTables = next;
1550
- },
1551
- async close() {
1552
- await conn.close();
1553
- await db.terminate();
1554
- }
1555
- };
1556
- }
1557
11
  export { BrowserAttachBudgetExceededError, DuckDBWasmDatabase, OpfsQuotaExceededError, attachOpfsParquetTables, attachParquetTables, attachParquetUrlTables, attachParquetUrlTablesResult, bootDuckDBWasm, browserAttachErrors, clearOpfsSnapshotCache, compileArchetypeSql, countries, createBrowserAnalysisRuntime, createClient, createInsightRunner, dates, drizzle, estimateOpfsStorage, hourly_pages, isBrowserAttachError, mergeScope, overlayViewBody, page_queries, pages, queries, readOpfsSnapshotFile, requestPersistentStorage, resolveWindow, schema, scopeFor, tableForArchetype };