@gscdump/nuxt 0.21.3 → 0.22.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,500 @@
1
+ // Per-site DuckDB-WASM analyzer over the Iceberg fact tables.
2
+ //
3
+ // One analyzer instance per `(siteId, searchType, range)`, cached on the
4
+ // NuxtApp so every subpage under `/sites/[id]/*` shares the same DuckDB boot
5
+ // + attached parquet. The site overview pays the cold cost once; tabbing
6
+ // across Queries / Pages / Countries / etc is essentially free — DuckDB has
7
+ // the data resident in memory and each tab just runs a different SQL query.
8
+ //
9
+ // Tables are attached LAZILY: the first call that needs `queries` triggers
10
+ // the parquet fetch for that table; sibling tabs reuse the attach. This
11
+ // keeps cold-tab latency low and avoids attaching parquet for tables a
12
+ // session never touches (`countries` if the user only looks at Queries).
13
+ //
14
+ // Iceberg fact tables follow the same shape:
15
+ // pages — url, date, clicks, impressions, sum_position
16
+ // queries — query, query_canonical, date, …metrics
17
+ // countries — country, date, …metrics
18
+ // dates — date, …metrics + anonymized_impressions_pct + 9-col device pivot
19
+ // page_queries — url, query, query_canonical, date, …metrics
20
+ // search_appearance — searchAppearance, date, …metrics
21
+ // search_appearance_* — searchAppearance plus page/query context
22
+
23
+ import type { AsyncDuckDB, AsyncDuckDBConnection } from '@duckdb/duckdb-wasm'
24
+ import type { AnalysisParams, AnalysisResult } from '@gscdump/analysis'
25
+ import type { FileResolutionResponse } from '@gscdump/contracts'
26
+ import type { OpfsAttachedHandle } from '@gscdump/engine-duckdb-wasm'
27
+ import { defaultAnalyzerRegistry } from '@gscdump/analysis'
28
+ import { runAnalyzerFromSource } from '@gscdump/engine/analyzer'
29
+ import { createAttachedTableSource } from '@gscdump/engine/source'
30
+ import { useGscFetch } from '#imports'
31
+ import { attachParquetWithFallback } from '../utils/duckdb-wasm'
32
+
33
+ export type GscFactTable = 'pages' | 'queries' | 'countries' | 'dates' | 'page_queries' | 'search_appearance' | 'search_appearance_pages' | 'search_appearance_queries' | 'search_appearance_page_queries'
34
+
35
+ export type GscTableStage = 'idle' | 'resolving' | 'downloading' | 'attaching' | 'ready' | 'error' | 'unavailable'
36
+
37
+ export interface GscTableStatus {
38
+ stage: GscTableStage
39
+ filesAttached: number
40
+ filesTotal: number
41
+ startedAt?: number
42
+ endedAt?: number
43
+ error?: string
44
+ }
45
+
46
+ export interface UseGscSiteAnalyzerReturn {
47
+ /** Per-table stage + file counts. `<GscBootProgress />` reads from the layer-wide map (we mirror there); this is the local view for in-page hints. */
48
+ tables: Readonly<Ref<Record<GscFactTable, GscTableStatus>>>
49
+ /** True once DuckDB-WASM has booted. Queries can dispatch as soon as the table they need is in `'ready'`. */
50
+ ready: Readonly<Ref<boolean>>
51
+ /** First fatal error (boot or any table). Per-table errors land on `tables[<name>].error`. */
52
+ error: Readonly<Ref<Error | null>>
53
+ /**
54
+ * Run SQL against the attached tables. Awaits the requested tables' attach,
55
+ * triggering on-demand attach if they haven't been started yet.
56
+ */
57
+ query: <T = Record<string, unknown>>(opts: { sql: string, needs: readonly GscFactTable[], params?: readonly unknown[] }) => Promise<T[]>
58
+ /**
59
+ * Positional-form `query` returning `{ rows, queryMs }`. Convenience for
60
+ * callers that want raw SQL access without naming `needs` themselves; the
61
+ * required tables are inferred from `FROM`/`JOIN` references.
62
+ */
63
+ runQuery: <T = Record<string, unknown>>(sql: string, params?: readonly unknown[]) => Promise<{ rows: T[], queryMs: number }>
64
+ /**
65
+ * Dispatch an analyzer from `defaultAnalyzerRegistry` against the attached
66
+ * tables. Builds an `attached-table` SQL source that proxies back through
67
+ * `query()`, so the per-site DuckDB-WASM boot and the on-demand attach
68
+ * lifecycle are reused.
69
+ */
70
+ analyze: (params: AnalysisParams, opts?: { signal?: AbortSignal }) => Promise<AnalysisResult & { queryMs: number }>
71
+ }
72
+
73
+ interface BootedAnalyzer {
74
+ /** Inert proxy used until the real DuckDB-WASM boots. */
75
+ bootPromise: Promise<{ db: AsyncDuckDB, conn: AsyncDuckDBConnection }>
76
+ /** Latched once attach has been started for a table; resolves when the table is queryable. */
77
+ tablePromises: Map<GscFactTable, Promise<void>>
78
+ /** Reactive per-table status (mirrored to the layer-wide progress map). */
79
+ tables: Ref<Record<GscFactTable, GscTableStatus>>
80
+ ready: Ref<boolean>
81
+ error: Ref<Error | null>
82
+ /** Held mutex around DuckDB DDL/query calls — WASM DuckDB is single-connection. */
83
+ withDb: <T>(fn: (conn: AsyncDuckDBConnection) => Promise<T>) => Promise<T>
84
+ /** Per-instance refcount — bumped on every consumer hookup, decremented on dispose. */
85
+ refs: number
86
+ /** Cleanup; called when refs hits 0. */
87
+ dispose: () => Promise<void>
88
+ }
89
+
90
+ const ALL_TABLES: readonly GscFactTable[] = [
91
+ 'pages',
92
+ 'queries',
93
+ 'countries',
94
+ 'dates',
95
+ 'page_queries',
96
+ 'search_appearance',
97
+ 'search_appearance_pages',
98
+ 'search_appearance_queries',
99
+ 'search_appearance_page_queries',
100
+ ] as const
101
+
102
+ const FACT_TABLE_NAMES = ALL_TABLES.join('|')
103
+ const TABLE_RE = new RegExp(`\\b(?:FROM|JOIN)\\s+(?:main\\.)?(${FACT_TABLE_NAMES})\\b`, 'gi')
104
+
105
+ function tablesFromSql(sql: string): readonly GscFactTable[] {
106
+ const out = new Set<GscFactTable>()
107
+ for (const m of sql.matchAll(TABLE_RE))
108
+ out.add(m[1]!.toLowerCase() as GscFactTable)
109
+ return out.size > 0 ? Array.from(out) : ALL_TABLES
110
+ }
111
+
112
+ function sqlIdent(s: string): string {
113
+ return s.replace(/\W/g, '_')
114
+ }
115
+
116
+ function emptyStatus(): GscTableStatus {
117
+ return { stage: 'idle', filesAttached: 0, filesTotal: 0 }
118
+ }
119
+
120
+ function emptyTables(): Record<GscFactTable, GscTableStatus> {
121
+ return Object.fromEntries(ALL_TABLES.map(t => [t, emptyStatus()])) as Record<GscFactTable, GscTableStatus>
122
+ }
123
+
124
+ // Per-NuxtApp instance cache keyed by `siteId|searchType|start|end`. WeakMap
125
+ // against the NuxtApp so SSR isolation holds (each request gets its own cache)
126
+ // and client-side instances live as long as the app does.
127
+ const instanceCaches = new WeakMap<object, Map<string, BootedAnalyzer>>()
128
+
129
+ function getCache(): Map<string, BootedAnalyzer> {
130
+ const app = useNuxtApp()
131
+ let bag = instanceCaches.get(app)
132
+ if (!bag) {
133
+ bag = new Map()
134
+ instanceCaches.set(app, bag)
135
+ }
136
+ return bag
137
+ }
138
+
139
+ function instanceKey(siteId: string, searchType: string, range: { start: string, end: string }): string {
140
+ return `${siteId}|${searchType}|${range.start}|${range.end}`
141
+ }
142
+
143
+ export function useGscSiteAnalyzer(
144
+ siteId: MaybeRefOrGetter<string | null | undefined>,
145
+ range: MaybeRefOrGetter<{ start: string, end: string } | null | undefined>,
146
+ options: { searchType?: 'web' | 'image' | 'video' | 'news' | 'discover' | 'googleNews', useOpfsCache?: boolean } = {},
147
+ ): UseGscSiteAnalyzerReturn {
148
+ const searchType = options.searchType ?? 'web'
149
+ const useOpfsCache = options.useOpfsCache ?? true
150
+
151
+ // The reactive return shape — proxies the currently-selected instance.
152
+ // We rebind every time the key changes (e.g. range moves outside the
153
+ // cached attach window).
154
+ const tables = ref<Record<GscFactTable, GscTableStatus>>(emptyTables())
155
+ const ready = ref(false)
156
+ const error = ref<Error | null>(null)
157
+ let active: BootedAnalyzer | null = null
158
+
159
+ function bind(instance: BootedAnalyzer | null): void {
160
+ // Release the previous instance's refcount; pick up the new one. Skipping
161
+ // shutdown on rebind means tab navigation never tears DuckDB down — the
162
+ // instance lives until every consumer unmounts and the cache drops it.
163
+ const prev = active
164
+ active = instance
165
+ if (instance) {
166
+ instance.refs++
167
+ tables.value = instance.tables.value
168
+ ready.value = instance.ready.value
169
+ error.value = instance.error.value
170
+ // Re-link reactive refs to the live instance.
171
+ watchEffect(() => {
172
+ tables.value = instance.tables.value
173
+ })
174
+ watchEffect(() => {
175
+ ready.value = instance.ready.value
176
+ })
177
+ watchEffect(() => {
178
+ error.value = instance.error.value
179
+ })
180
+ }
181
+ else {
182
+ tables.value = emptyTables()
183
+ ready.value = false
184
+ error.value = null
185
+ }
186
+ if (prev) {
187
+ prev.refs--
188
+ if (prev.refs <= 0)
189
+ prev.dispose().catch(() => {})
190
+ }
191
+ }
192
+
193
+ function obtain(sid: string, r: { start: string, end: string }): BootedAnalyzer {
194
+ const cache = getCache()
195
+ const key = instanceKey(sid, searchType, r)
196
+ const hit = cache.get(key)
197
+ if (hit)
198
+ return hit
199
+ const created = createAnalyzer({ siteId: sid, searchType, range: r, useOpfsCache })
200
+ cache.set(key, created)
201
+ return created
202
+ }
203
+
204
+ watch(
205
+ () => {
206
+ const sid = toValue(siteId) ?? null
207
+ const r = toValue(range) ?? null
208
+ return sid && r?.start && r?.end ? { sid, r } : null
209
+ },
210
+ (val) => {
211
+ if (!val || !import.meta.client) {
212
+ bind(null)
213
+ return
214
+ }
215
+ bind(obtain(val.sid, val.r))
216
+ },
217
+ { immediate: true },
218
+ )
219
+
220
+ onScopeDispose(() => bind(null))
221
+
222
+ async function query<T = Record<string, unknown>>(opts: { sql: string, needs: readonly GscFactTable[], params?: readonly unknown[] }): Promise<T[]> {
223
+ if (!active)
224
+ throw new Error('useGscSiteAnalyzer: no active instance (site/range not set)')
225
+ const instance = active
226
+ await instance.bootPromise
227
+ // Kick off (or join) per-table attaches for every table the query needs.
228
+ await Promise.all(opts.needs.map(t => ensureTable(instance, t)))
229
+ return instance.withDb(async (conn) => {
230
+ const result = opts.params && opts.params.length > 0
231
+ ? await (async () => {
232
+ const stmt = await conn.prepare(opts.sql)
233
+ try {
234
+ return await stmt.query(...(opts.params as unknown[]))
235
+ }
236
+ finally {
237
+ await stmt.close()
238
+ }
239
+ })()
240
+ : await conn.query(opts.sql)
241
+ // Apache Arrow returns `Proxy(StructRow)` rows — Vue's reactivity
242
+ // can't make those reactive (the Arrow proxy's `isExtensible` trap
243
+ // throws a TypeError when the reactive proxy probes it). Materialise
244
+ // into plain objects at the seam so every consumer is free to assign
245
+ // results directly into a `ref`/`reactive` without surprise.
246
+ const arr = result.toArray() as unknown as Array<Record<string, unknown>>
247
+ return arr.map(row => ({ ...row })) as unknown as T[]
248
+ })
249
+ }
250
+
251
+ async function runQuery<T = Record<string, unknown>>(sql: string, params?: readonly unknown[]): Promise<{ rows: T[], queryMs: number }> {
252
+ const t0 = performance.now()
253
+ const rows = await query<T>({ sql, needs: tablesFromSql(sql), params })
254
+ return { rows, queryMs: performance.now() - t0 }
255
+ }
256
+
257
+ async function analyze(params: AnalysisParams, opts?: { signal?: AbortSignal }): Promise<AnalysisResult & { queryMs: number }> {
258
+ const t0 = performance.now()
259
+ const source = createAttachedTableSource(
260
+ {
261
+ query: async (sql, bindParams) => {
262
+ return query<Record<string, unknown>>({ sql, needs: tablesFromSql(sql), params: bindParams })
263
+ },
264
+ },
265
+ {
266
+ schema: 'main',
267
+ signal: opts?.signal,
268
+ attachedTables: ALL_TABLES as readonly string[],
269
+ },
270
+ )
271
+ const result = await runAnalyzerFromSource(source, params, defaultAnalyzerRegistry)
272
+ return {
273
+ results: result.results as AnalysisResult['results'],
274
+ meta: result.meta as AnalysisResult['meta'],
275
+ queryMs: performance.now() - t0,
276
+ }
277
+ }
278
+
279
+ return {
280
+ tables: tables as Readonly<Ref<Record<GscFactTable, GscTableStatus>>>,
281
+ ready: ready as Readonly<Ref<boolean>>,
282
+ error: error as Readonly<Ref<Error | null>>,
283
+ query,
284
+ runQuery,
285
+ analyze,
286
+ }
287
+ }
288
+
289
+ // ── implementation ──────────────────────────────────────────────────────────
290
+
291
+ interface AnalyzerArgs {
292
+ siteId: string
293
+ searchType: string
294
+ range: { start: string, end: string }
295
+ useOpfsCache: boolean
296
+ }
297
+
298
+ function createAnalyzer(args: AnalyzerArgs): BootedAnalyzer {
299
+ const tables = ref<Record<GscFactTable, GscTableStatus>>(emptyTables())
300
+ const ready = ref(false)
301
+ const error = ref<Error | null>(null)
302
+ const tablePromises = new Map<GscFactTable, Promise<void>>()
303
+ const opfsHandles = new Map<GscFactTable, OpfsAttachedHandle>()
304
+
305
+ // Pin the configured fetcher + analytics config at construction — these
306
+ // need a NuxtApp context, which is available here (the composable's
307
+ // setup) but not inside the async work spawned below.
308
+ const $gscFetch = useGscFetch()
309
+ const analyticsConfig = useGscAnalyticsConfig()
310
+ const analyticsCtx = useGscAnalyticsContext()
311
+ const apiBase = (analyticsConfig.apiBase ?? '').replace(/\/+$/, '')
312
+
313
+ // Single resolve call — `analysis-sources` already returns all 5 tables
314
+ // for a given site+range, so we hit it once and demux per-table later.
315
+ let resolvePromise: Promise<FileResolutionResponse | null> | null = null
316
+ function getResolution(): Promise<FileResolutionResponse | null> {
317
+ if (resolvePromise)
318
+ return resolvePromise
319
+ const qs = `searchType=${args.searchType}&start=${args.range.start}&end=${args.range.end}`
320
+ resolvePromise = $gscFetch<FileResolutionResponse | { canUseBrowser?: false }>(
321
+ `${apiBase}/api/__gsc/sites/${args.siteId}/analysis-sources?${qs}`,
322
+ )
323
+ .then((res) => {
324
+ if (!res || (res as { canUseBrowser?: boolean }).canUseBrowser === false)
325
+ return null
326
+ return res as FileResolutionResponse
327
+ })
328
+ .catch(() => null)
329
+ return resolvePromise
330
+ }
331
+
332
+ // Boot DuckDB-WASM up front — the resolve and the WASM bundle download in
333
+ // parallel, so by the time a query asks for a table, the DB is ready.
334
+ const bootPromise: Promise<{ db: AsyncDuckDB, conn: AsyncDuckDBConnection }> = (async () => {
335
+ // Shared boot: navigating home → site detail (or between sites) reuses
336
+ // the same DuckDB-WASM instance instead of paying ~800ms cold-boot again.
337
+ const bundleBase = (analyticsConfig as { duckdbBundleBase?: string }).duckdbBundleBase
338
+ const boot = await sharedGscDuckDBWasm(bundleBase)
339
+ ready.value = true
340
+ return boot
341
+ })()
342
+ bootPromise.catch((err) => {
343
+ error.value = err instanceof Error ? err : new Error(String(err))
344
+ })
345
+
346
+ // DuckDB-WASM single-connection mutex — DDL and queries serialise; downloads
347
+ // run free in parallel outside the lock.
348
+ let dbLock: Promise<unknown> = Promise.resolve()
349
+ function withDb<T>(fn: (conn: AsyncDuckDBConnection) => Promise<T>): Promise<T> {
350
+ const next = dbLock.then(async () => {
351
+ const { conn } = await bootPromise
352
+ return fn(conn)
353
+ })
354
+ dbLock = next.catch(() => {})
355
+ return next
356
+ }
357
+
358
+ // Per-table attach kicker. Stores its promise so concurrent callers join
359
+ // the same in-flight work; the layer-wide `patchProgress` map is updated
360
+ // as each phase advances so `<GscBootProgress />` shows the site's table
361
+ // load alongside other sites.
362
+ function patchSelf(table: GscFactTable, patch: Partial<GscTableStatus>): void {
363
+ tables.value = { ...tables.value, [table]: { ...tables.value[table], ...patch } }
364
+ }
365
+ function patchLayer(table: GscFactTable, stage: GscTableStage, extras: Partial<GscTableStatus> = {}): void {
366
+ // Reuse the layer's SiteLoadProgress shape, namespacing by table so
367
+ // multiple tables for the same site don't collide on the boot bar.
368
+ const siteSlot = `${args.siteId}#${table}`
369
+ const layerStage = stage === 'unavailable' ? 'ready' : stage === 'downloading' || stage === 'attaching' ? 'attach' : stage === 'resolving' ? 'manifest' : stage === 'ready' ? 'ready' : stage === 'error' ? 'error' : 'idle'
370
+ analyticsCtx.patchProgress(siteSlot, {
371
+ stage: layerStage as never,
372
+ source: 'duckdb',
373
+ filesAttached: extras.filesAttached ?? tables.value[table].filesAttached,
374
+ filesTotal: extras.filesTotal ?? tables.value[table].filesTotal,
375
+ startedAt: extras.startedAt ?? tables.value[table].startedAt ?? Date.now(),
376
+ endedAt: extras.endedAt,
377
+ error: extras.error,
378
+ })
379
+ }
380
+
381
+ async function attachTable(table: GscFactTable): Promise<void> {
382
+ const startedAt = Date.now()
383
+ patchSelf(table, { stage: 'resolving', startedAt, endedAt: undefined, error: undefined })
384
+ patchLayer(table, 'resolving', { startedAt })
385
+
386
+ const full = await getResolution()
387
+ if (!full) {
388
+ patchSelf(table, { stage: 'unavailable', endedAt: Date.now() })
389
+ patchLayer(table, 'unavailable', { endedAt: Date.now() })
390
+ return
391
+ }
392
+ const tableEntry = full.tables.find(t => t.table === table)
393
+ const browserFiles = tableEntry?.mode === 'browser' ? tableEntry.files : []
394
+ if (browserFiles.length === 0) {
395
+ patchSelf(table, { stage: 'unavailable', endedAt: Date.now() })
396
+ patchLayer(table, 'unavailable', { endedAt: Date.now() })
397
+ return
398
+ }
399
+
400
+ patchSelf(table, { stage: 'downloading', filesTotal: browserFiles.length })
401
+ patchLayer(table, 'downloading', { filesTotal: browserFiles.length })
402
+
403
+ const sid = sqlIdent(args.siteId)
404
+ const tableIdent = sqlIdent(table)
405
+ // Per-(site,table) view name keeps OPFS file names unique across sites
406
+ // sharing this DuckDB instance.
407
+ const viewName = `${tableIdent}_${sid}`
408
+
409
+ let filesAttached = 0
410
+ const { db, conn } = await bootPromise
411
+ const opfsHandle = await attachParquetWithFallback({
412
+ db,
413
+ conn,
414
+ viewName,
415
+ files: browserFiles,
416
+ version: full.snapshotVersion,
417
+ useOpfsCache: args.useOpfsCache,
418
+ withDb: fn => withDb(() => fn()),
419
+ onFileProgress: () => {
420
+ filesAttached++
421
+ patchSelf(table, { filesAttached })
422
+ patchLayer(table, 'downloading', { filesAttached })
423
+ },
424
+ }).catch((err) => {
425
+ const msg = err instanceof Error ? err.message : String(err)
426
+ patchSelf(table, { stage: 'error', error: msg, endedAt: Date.now() })
427
+ patchLayer(table, 'error', { error: msg, endedAt: Date.now() })
428
+ throw err
429
+ })
430
+
431
+ // Query SQL references the canonical table name (`pages`, etc.); the
432
+ // underlying view is per-(site,table) so OPFS file names don't collide
433
+ // across sites that may share a DuckDB instance in future.
434
+ if (viewName !== tableIdent) {
435
+ await withDb(async (conn) => {
436
+ await conn.query(`CREATE OR REPLACE VIEW ${tableIdent} AS SELECT * FROM ${viewName}`)
437
+ })
438
+ }
439
+
440
+ if (opfsHandle)
441
+ opfsHandles.set(table, opfsHandle)
442
+
443
+ patchSelf(table, { stage: 'ready', endedAt: Date.now() })
444
+ patchLayer(table, 'ready', { endedAt: Date.now() })
445
+ }
446
+
447
+ function ensureTableInner(table: GscFactTable): Promise<void> {
448
+ const existing = tablePromises.get(table)
449
+ if (existing)
450
+ return existing
451
+ const p = attachTable(table)
452
+ tablePromises.set(table, p)
453
+ return p
454
+ }
455
+
456
+ const instance: BootedAnalyzer = {
457
+ bootPromise,
458
+ tablePromises,
459
+ tables,
460
+ ready,
461
+ error,
462
+ withDb,
463
+ refs: 0,
464
+ dispose: async () => {
465
+ // Pull the cache entry first so subsequent obtains create a fresh one
466
+ // (we can't dispose mid-attach without leaking promises).
467
+ const cache = instanceCaches.get(useNuxtApp())
468
+ if (cache) {
469
+ for (const [k, v] of cache.entries()) {
470
+ if (v === instance) {
471
+ cache.delete(k)
472
+ break
473
+ }
474
+ }
475
+ }
476
+ try {
477
+ for (const h of opfsHandles.values())
478
+ await h.detach().catch(() => {})
479
+ opfsHandles.clear()
480
+ // Drop the per-site views we created so a re-attach is clean. Do NOT
481
+ // terminate the DB — it's shared via `sharedGscDuckDBWasm`.
482
+ const { conn } = await bootPromise
483
+ for (const t of ALL_TABLES)
484
+ await conn.query(`DROP VIEW IF EXISTS ${sqlIdent(t)}`).catch(() => {})
485
+ }
486
+ catch {
487
+ // Boot failed — nothing to tear down.
488
+ }
489
+ },
490
+ }
491
+ // Attach the ensureTable closure to the instance so the outer composable
492
+ // can reach it without a globalThis hack.
493
+ ;(instance as BootedAnalyzer & { ensureTable: typeof ensureTableInner }).ensureTable = ensureTableInner
494
+ return instance
495
+ }
496
+
497
+ async function ensureTable(instance: BootedAnalyzer, table: GscFactTable): Promise<void> {
498
+ const ensure = (instance as BootedAnalyzer & { ensureTable: (t: GscFactTable) => Promise<void> }).ensureTable
499
+ return ensure(table)
500
+ }
@@ -0,0 +1,166 @@
1
+ // Shared browser-DuckDB-WASM helpers used by per-site analyzers and the
2
+ // home-page multi-site fanout. Both consumers boot the same WASM bundle and
3
+ // run the same in-memory parquet attach path; OPFS attach lives in the caller
4
+ // because the two lifecycle models (refcounted long-lived vs ephemeral
5
+ // per-run) need different handle ownership.
6
+
7
+ import type { AsyncDuckDB, AsyncDuckDBConnection } from '@duckdb/duckdb-wasm'
8
+ import type { DuckDBWasmBootResult, OpfsAttachedHandle } from '@gscdump/engine-duckdb-wasm'
9
+
10
+ /**
11
+ * Boot DuckDB-WASM using the layer's configured bundle base. The dynamic
12
+ * `import('@gscdump/engine-duckdb-wasm')` keeps the WASM bundle out of the
13
+ * SSR build; callers should also gate the call behind `import.meta.client`.
14
+ *
15
+ * The `filesystem` config disables HEAD-request preflight and forces full
16
+ * HTTP reads — Cloudflare Workers strips `Content-Length` on streamed
17
+ * responses, which makes DuckDB's default range-request path 404.
18
+ */
19
+ export async function bootGscDuckDBWasm(bundleBase?: string): Promise<DuckDBWasmBootResult> {
20
+ const { bootDuckDBWasm } = await import('@gscdump/engine-duckdb-wasm')
21
+ return bootDuckDBWasm({
22
+ ...(bundleBase
23
+ ? {
24
+ bundles: {
25
+ mvp: { mainModule: `${bundleBase}/duckdb-mvp.wasm`, mainWorker: `${bundleBase}/duckdb-browser-mvp.worker.js` },
26
+ eh: { mainModule: `${bundleBase}/duckdb-eh.wasm`, mainWorker: `${bundleBase}/duckdb-browser-eh.worker.js` },
27
+ },
28
+ }
29
+ : {}),
30
+ config: { filesystem: { reliableHeadRequests: false, allowFullHTTPReads: true } },
31
+ })
32
+ }
33
+
34
+ /**
35
+ * Process-wide shared boot. The ~800ms WASM boot is the dominant cost on the
36
+ * warm path (OPFS cache-hit means downloads are skipped); paying it once per
37
+ * session instead of per refresh turns date-range changes / search-type
38
+ * swaps into near-instant updates.
39
+ *
40
+ * Callers that want their own throwaway instance can keep using
41
+ * {@link bootGscDuckDBWasm}; those that want session-wide reuse — the home
42
+ * page fanout and the per-site analyzer — go through this.
43
+ *
44
+ * Browser-only. The `bundleBase` is captured from the first call; subsequent
45
+ * calls reuse the promise and ignore the argument (it's a deployment-wide
46
+ * constant in practice).
47
+ */
48
+ let sharedBootPromise: Promise<DuckDBWasmBootResult> | null = null
49
+ let sharedBootColdMs: number | undefined
50
+ export function sharedGscDuckDBWasm(bundleBase?: string): Promise<DuckDBWasmBootResult> {
51
+ if (!sharedBootPromise) {
52
+ if (typeof performance !== 'undefined')
53
+ performance.mark('gsc:duckdb:boot:start')
54
+ const start = typeof performance !== 'undefined' ? performance.now() : 0
55
+ sharedBootPromise = bootGscDuckDBWasm(bundleBase).then((b) => {
56
+ if (typeof performance !== 'undefined') {
57
+ sharedBootColdMs = performance.now() - start
58
+ performance.mark('gsc:duckdb:boot:end')
59
+ }
60
+ return b
61
+ })
62
+ }
63
+ else if (typeof performance !== 'undefined') {
64
+ performance.mark('gsc:duckdb:boot:reuse')
65
+ }
66
+ return sharedBootPromise
67
+ }
68
+
69
+ /** Diagnostics: ms paid for the one-time cold boot, or undefined if not yet booted. */
70
+ export function sharedGscDuckDBWasmColdMs(): number | undefined {
71
+ return sharedBootColdMs
72
+ }
73
+
74
+ /**
75
+ * Fetch every URL into a `Uint8Array` in parallel. `onFileFetched` ticks
76
+ * once per successful download. Separate from `registerParquetView` so
77
+ * callers can run fetches outside any connection mutex.
78
+ */
79
+ export async function fetchParquetBuffers(
80
+ urls: readonly string[],
81
+ onFileFetched?: () => void,
82
+ ): Promise<Uint8Array[]> {
83
+ return Promise.all(urls.map(async (url) => {
84
+ const r = await fetch(url, { credentials: 'omit' })
85
+ if (!r.ok)
86
+ throw new Error(`GET ${url} failed: ${r.status}`)
87
+ const buf = new Uint8Array(await r.arrayBuffer())
88
+ onFileFetched?.()
89
+ return buf
90
+ }))
91
+ }
92
+
93
+ /**
94
+ * Register pre-fetched parquet buffers as virtual files, then build a single
95
+ * view that unions them. `CAST(date AS DATE)` canonicalises the legacy-VARCHAR
96
+ * / new-DATE encodings. File-name uniqueness is the caller's job; pass a
97
+ * `viewName` that already encodes any site/table salt so concurrent attaches
98
+ * across sites don't collide on the DuckDB virtual filesystem. Call under the
99
+ * DuckDB-WASM single-connection mutex.
100
+ */
101
+ export async function registerParquetView(opts: {
102
+ db: AsyncDuckDB
103
+ conn: AsyncDuckDBConnection
104
+ viewName: string
105
+ buffers: readonly Uint8Array[]
106
+ }): Promise<void> {
107
+ const { db, conn, viewName, buffers } = opts
108
+ const fileNames = buffers.map((_, i) => `${viewName}_${i}.parquet`)
109
+ for (let i = 0; i < buffers.length; i++)
110
+ await db.registerFileBuffer(fileNames[i]!, buffers[i]!)
111
+ const fileList = fileNames.map(n => `'${n.replace(/'/g, '\'\'')}'`).join(',')
112
+ await conn.query(`CREATE OR REPLACE VIEW ${viewName} AS SELECT * REPLACE (CAST(date AS DATE) AS date) FROM read_parquet([${fileList}], union_by_name = true)`)
113
+ }
114
+
115
+ /**
116
+ * Attach a parquet file set into DuckDB-WASM, preferring the OPFS-backed
117
+ * path (cache-hit skips download) and falling back to in-memory buffers when
118
+ * OPFS is opted out, unavailable, or quota-exhausted.
119
+ *
120
+ * Returns the OPFS handle when the OPFS path succeeded — caller must keep
121
+ * the handle alive and `detach()` on cleanup. Returns `null` when the
122
+ * fallback path was used; nothing to detach. `onFileProgress` ticks once per
123
+ * downloaded file in either path so the caller can drive a progress UI.
124
+ *
125
+ * `withDb` wraps the DDL/file-register calls when the caller serialises
126
+ * against other DuckDB activity (e.g. a shared connection); pass `undefined`
127
+ * when the caller already owns a private connection.
128
+ */
129
+ export async function attachParquetWithFallback(args: {
130
+ db: AsyncDuckDB
131
+ conn: AsyncDuckDBConnection
132
+ viewName: string
133
+ files: readonly { url: string, bytes: number, contentHash: string }[]
134
+ version: string
135
+ useOpfsCache: boolean
136
+ fetchConcurrency?: number
137
+ onFileProgress?: () => void
138
+ withDb?: <T>(fn: () => Promise<T>) => Promise<T>
139
+ }): Promise<OpfsAttachedHandle | null> {
140
+ const { db, conn, viewName, files, version, useOpfsCache, onFileProgress } = args
141
+ const lock = args.withDb ?? (<T>(fn: () => Promise<T>) => fn())
142
+ const fetchConcurrency = args.fetchConcurrency ?? Math.min(files.length, 4)
143
+
144
+ if (useOpfsCache) {
145
+ const { attachOpfsParquetTables } = await import('@gscdump/engine-duckdb-wasm')
146
+ const handle = await lock(() => attachOpfsParquetTables({
147
+ db,
148
+ conn,
149
+ tables: [{
150
+ table: viewName,
151
+ files: files.map(f => ({ url: f.url, bytes: f.bytes, contentHash: f.contentHash })),
152
+ }],
153
+ version,
154
+ fetchConcurrency,
155
+ onFileProgress,
156
+ }))
157
+ if (!handle.degradedTables.includes(viewName))
158
+ return handle
159
+ // OPFS quota / failure — fall through to buffer path. Caller's progress
160
+ // counter restarts; both paths report one tick per fetched file.
161
+ }
162
+
163
+ const buffers = await fetchParquetBuffers(files.map(f => f.url), onFileProgress)
164
+ await lock(() => registerParquetView({ db, conn, viewName, buffers }))
165
+ return null
166
+ }
package/nuxt.config.ts CHANGED
@@ -14,10 +14,15 @@ import { fileURLToPath } from 'node:url'
14
14
  export default defineNuxtConfig({
15
15
  // Register the layer's module so hooks + runtime config land on host apps
16
16
  // without them having to wire each module individually.
17
+ //
18
+ // `nuxt-use-query` provides `useNuxtQuery` / `useQueryCache` — TanStack-Query-
19
+ // shaped wrapper over `useFetch`. Composables in this layer fetch through
20
+ // it for SWR + dedup.
17
21
  modules: [
18
22
  fileURLToPath(new URL('./module.ts', import.meta.url)),
19
23
  '@nuxt/ui',
20
24
  '@vueuse/nuxt',
25
+ 'nuxt-use-query',
21
26
  ],
22
27
 
23
28
  css: [