@gscdump/nuxt 0.21.3 → 0.22.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -18
- package/app/composables/_useGscQueryDispatcher.ts +5 -4
- package/app/composables/useGscAnalytics.ts +11 -3
- package/app/composables/useGscAnalyzerBatch.ts +2 -2
- package/app/composables/useGscPanelContext.ts +9 -2
- package/app/composables/useGscParquetTable.ts +3 -4
- package/app/composables/useGscQueryDispatcher.ts +14 -122
- package/app/composables/useGscResource.ts +11 -114
- package/app/composables/useGscSiteAnalyzer.ts +500 -0
- package/app/utils/duckdb-wasm.ts +166 -0
- package/nuxt.config.ts +5 -0
- package/package.json +9 -8
- package/types.ts +8 -3
- package/app/composables/useGscEngine.ts +0 -16
- package/app/composables/useGscRollupTable.ts +0 -78
- package/app/composables/useGscRowQuery.ts +0 -56
- package/app/composables/useGscSnapshotAnalyzer.contract.ts +0 -150
- package/app/composables/useGscSnapshotAnalyzer.routing.ts +0 -87
- package/app/composables/useGscSnapshotAnalyzer.ts +0 -559
- package/app/utils/setup-gsc-fetch-auth.ts +0 -62
|
@@ -0,0 +1,500 @@
|
|
|
1
|
+
// Per-site DuckDB-WASM analyzer over the Iceberg fact tables.
|
|
2
|
+
//
|
|
3
|
+
// One analyzer instance per `(siteId, searchType, range)`, cached on the
|
|
4
|
+
// NuxtApp so every subpage under `/sites/[id]/*` shares the same DuckDB boot
|
|
5
|
+
// + attached parquet. The site overview pays the cold cost once; tabbing
|
|
6
|
+
// across Queries / Pages / Countries / etc is essentially free — DuckDB has
|
|
7
|
+
// the data resident in memory and each tab just runs a different SQL query.
|
|
8
|
+
//
|
|
9
|
+
// Tables are attached LAZILY: the first call that needs `queries` triggers
|
|
10
|
+
// the parquet fetch for that table; sibling tabs reuse the attach. This
|
|
11
|
+
// keeps cold-tab latency low and avoids attaching parquet for tables a
|
|
12
|
+
// session never touches (`countries` if the user only looks at Queries).
|
|
13
|
+
//
|
|
14
|
+
// Iceberg fact tables follow the same shape:
|
|
15
|
+
// pages — url, date, clicks, impressions, sum_position
|
|
16
|
+
// queries — query, query_canonical, date, …metrics
|
|
17
|
+
// countries — country, date, …metrics
|
|
18
|
+
// dates — date, …metrics + anonymized_impressions_pct + 9-col device pivot
|
|
19
|
+
// page_queries — url, query, query_canonical, date, …metrics
|
|
20
|
+
// search_appearance — searchAppearance, date, …metrics
|
|
21
|
+
// search_appearance_* — searchAppearance plus page/query context
|
|
22
|
+
|
|
23
|
+
import type { AsyncDuckDB, AsyncDuckDBConnection } from '@duckdb/duckdb-wasm'
|
|
24
|
+
import type { AnalysisParams, AnalysisResult } from '@gscdump/analysis'
|
|
25
|
+
import type { FileResolutionResponse } from '@gscdump/contracts'
|
|
26
|
+
import type { OpfsAttachedHandle } from '@gscdump/engine-duckdb-wasm'
|
|
27
|
+
import { defaultAnalyzerRegistry } from '@gscdump/analysis'
|
|
28
|
+
import { runAnalyzerFromSource } from '@gscdump/engine/analyzer'
|
|
29
|
+
import { createAttachedTableSource } from '@gscdump/engine/source'
|
|
30
|
+
import { useGscFetch } from '#imports'
|
|
31
|
+
import { attachParquetWithFallback } from '../utils/duckdb-wasm'
|
|
32
|
+
|
|
33
|
+
export type GscFactTable = 'pages' | 'queries' | 'countries' | 'dates' | 'page_queries' | 'search_appearance' | 'search_appearance_pages' | 'search_appearance_queries' | 'search_appearance_page_queries'
|
|
34
|
+
|
|
35
|
+
export type GscTableStage = 'idle' | 'resolving' | 'downloading' | 'attaching' | 'ready' | 'error' | 'unavailable'
|
|
36
|
+
|
|
37
|
+
export interface GscTableStatus {
|
|
38
|
+
stage: GscTableStage
|
|
39
|
+
filesAttached: number
|
|
40
|
+
filesTotal: number
|
|
41
|
+
startedAt?: number
|
|
42
|
+
endedAt?: number
|
|
43
|
+
error?: string
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export interface UseGscSiteAnalyzerReturn {
|
|
47
|
+
/** Per-table stage + file counts. `<GscBootProgress />` reads from the layer-wide map (we mirror there); this is the local view for in-page hints. */
|
|
48
|
+
tables: Readonly<Ref<Record<GscFactTable, GscTableStatus>>>
|
|
49
|
+
/** True once DuckDB-WASM has booted. Queries can dispatch as soon as the table they need is in `'ready'`. */
|
|
50
|
+
ready: Readonly<Ref<boolean>>
|
|
51
|
+
/** First fatal error (boot or any table). Per-table errors land on `tables[<name>].error`. */
|
|
52
|
+
error: Readonly<Ref<Error | null>>
|
|
53
|
+
/**
|
|
54
|
+
* Run SQL against the attached tables. Awaits the requested tables' attach,
|
|
55
|
+
* triggering on-demand attach if they haven't been started yet.
|
|
56
|
+
*/
|
|
57
|
+
query: <T = Record<string, unknown>>(opts: { sql: string, needs: readonly GscFactTable[], params?: readonly unknown[] }) => Promise<T[]>
|
|
58
|
+
/**
|
|
59
|
+
* Positional-form `query` returning `{ rows, queryMs }`. Convenience for
|
|
60
|
+
* callers that want raw SQL access without naming `needs` themselves; the
|
|
61
|
+
* required tables are inferred from `FROM`/`JOIN` references.
|
|
62
|
+
*/
|
|
63
|
+
runQuery: <T = Record<string, unknown>>(sql: string, params?: readonly unknown[]) => Promise<{ rows: T[], queryMs: number }>
|
|
64
|
+
/**
|
|
65
|
+
* Dispatch an analyzer from `defaultAnalyzerRegistry` against the attached
|
|
66
|
+
* tables. Builds an `attached-table` SQL source that proxies back through
|
|
67
|
+
* `query()`, so the per-site DuckDB-WASM boot and the on-demand attach
|
|
68
|
+
* lifecycle are reused.
|
|
69
|
+
*/
|
|
70
|
+
analyze: (params: AnalysisParams, opts?: { signal?: AbortSignal }) => Promise<AnalysisResult & { queryMs: number }>
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
interface BootedAnalyzer {
|
|
74
|
+
/** Inert proxy used until the real DuckDB-WASM boots. */
|
|
75
|
+
bootPromise: Promise<{ db: AsyncDuckDB, conn: AsyncDuckDBConnection }>
|
|
76
|
+
/** Latched once attach has been started for a table; resolves when the table is queryable. */
|
|
77
|
+
tablePromises: Map<GscFactTable, Promise<void>>
|
|
78
|
+
/** Reactive per-table status (mirrored to the layer-wide progress map). */
|
|
79
|
+
tables: Ref<Record<GscFactTable, GscTableStatus>>
|
|
80
|
+
ready: Ref<boolean>
|
|
81
|
+
error: Ref<Error | null>
|
|
82
|
+
/** Held mutex around DuckDB DDL/query calls — WASM DuckDB is single-connection. */
|
|
83
|
+
withDb: <T>(fn: (conn: AsyncDuckDBConnection) => Promise<T>) => Promise<T>
|
|
84
|
+
/** Per-instance refcount — bumped on every consumer hookup, decremented on dispose. */
|
|
85
|
+
refs: number
|
|
86
|
+
/** Cleanup; called when refs hits 0. */
|
|
87
|
+
dispose: () => Promise<void>
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
const ALL_TABLES: readonly GscFactTable[] = [
|
|
91
|
+
'pages',
|
|
92
|
+
'queries',
|
|
93
|
+
'countries',
|
|
94
|
+
'dates',
|
|
95
|
+
'page_queries',
|
|
96
|
+
'search_appearance',
|
|
97
|
+
'search_appearance_pages',
|
|
98
|
+
'search_appearance_queries',
|
|
99
|
+
'search_appearance_page_queries',
|
|
100
|
+
] as const
|
|
101
|
+
|
|
102
|
+
const FACT_TABLE_NAMES = ALL_TABLES.join('|')
|
|
103
|
+
const TABLE_RE = new RegExp(`\\b(?:FROM|JOIN)\\s+(?:main\\.)?(${FACT_TABLE_NAMES})\\b`, 'gi')
|
|
104
|
+
|
|
105
|
+
function tablesFromSql(sql: string): readonly GscFactTable[] {
|
|
106
|
+
const out = new Set<GscFactTable>()
|
|
107
|
+
for (const m of sql.matchAll(TABLE_RE))
|
|
108
|
+
out.add(m[1]!.toLowerCase() as GscFactTable)
|
|
109
|
+
return out.size > 0 ? Array.from(out) : ALL_TABLES
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
function sqlIdent(s: string): string {
|
|
113
|
+
return s.replace(/\W/g, '_')
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
function emptyStatus(): GscTableStatus {
|
|
117
|
+
return { stage: 'idle', filesAttached: 0, filesTotal: 0 }
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
function emptyTables(): Record<GscFactTable, GscTableStatus> {
|
|
121
|
+
return Object.fromEntries(ALL_TABLES.map(t => [t, emptyStatus()])) as Record<GscFactTable, GscTableStatus>
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
// Per-NuxtApp instance cache keyed by `siteId|searchType|start|end`. WeakMap
|
|
125
|
+
// against the NuxtApp so SSR isolation holds (each request gets its own cache)
|
|
126
|
+
// and client-side instances live as long as the app does.
|
|
127
|
+
const instanceCaches = new WeakMap<object, Map<string, BootedAnalyzer>>()
|
|
128
|
+
|
|
129
|
+
function getCache(): Map<string, BootedAnalyzer> {
|
|
130
|
+
const app = useNuxtApp()
|
|
131
|
+
let bag = instanceCaches.get(app)
|
|
132
|
+
if (!bag) {
|
|
133
|
+
bag = new Map()
|
|
134
|
+
instanceCaches.set(app, bag)
|
|
135
|
+
}
|
|
136
|
+
return bag
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
function instanceKey(siteId: string, searchType: string, range: { start: string, end: string }): string {
|
|
140
|
+
return `${siteId}|${searchType}|${range.start}|${range.end}`
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
export function useGscSiteAnalyzer(
|
|
144
|
+
siteId: MaybeRefOrGetter<string | null | undefined>,
|
|
145
|
+
range: MaybeRefOrGetter<{ start: string, end: string } | null | undefined>,
|
|
146
|
+
options: { searchType?: 'web' | 'image' | 'video' | 'news' | 'discover' | 'googleNews', useOpfsCache?: boolean } = {},
|
|
147
|
+
): UseGscSiteAnalyzerReturn {
|
|
148
|
+
const searchType = options.searchType ?? 'web'
|
|
149
|
+
const useOpfsCache = options.useOpfsCache ?? true
|
|
150
|
+
|
|
151
|
+
// The reactive return shape — proxies the currently-selected instance.
|
|
152
|
+
// We rebind every time the key changes (e.g. range moves outside the
|
|
153
|
+
// cached attach window).
|
|
154
|
+
const tables = ref<Record<GscFactTable, GscTableStatus>>(emptyTables())
|
|
155
|
+
const ready = ref(false)
|
|
156
|
+
const error = ref<Error | null>(null)
|
|
157
|
+
let active: BootedAnalyzer | null = null
|
|
158
|
+
|
|
159
|
+
function bind(instance: BootedAnalyzer | null): void {
|
|
160
|
+
// Release the previous instance's refcount; pick up the new one. Skipping
|
|
161
|
+
// shutdown on rebind means tab navigation never tears DuckDB down — the
|
|
162
|
+
// instance lives until every consumer unmounts and the cache drops it.
|
|
163
|
+
const prev = active
|
|
164
|
+
active = instance
|
|
165
|
+
if (instance) {
|
|
166
|
+
instance.refs++
|
|
167
|
+
tables.value = instance.tables.value
|
|
168
|
+
ready.value = instance.ready.value
|
|
169
|
+
error.value = instance.error.value
|
|
170
|
+
// Re-link reactive refs to the live instance.
|
|
171
|
+
watchEffect(() => {
|
|
172
|
+
tables.value = instance.tables.value
|
|
173
|
+
})
|
|
174
|
+
watchEffect(() => {
|
|
175
|
+
ready.value = instance.ready.value
|
|
176
|
+
})
|
|
177
|
+
watchEffect(() => {
|
|
178
|
+
error.value = instance.error.value
|
|
179
|
+
})
|
|
180
|
+
}
|
|
181
|
+
else {
|
|
182
|
+
tables.value = emptyTables()
|
|
183
|
+
ready.value = false
|
|
184
|
+
error.value = null
|
|
185
|
+
}
|
|
186
|
+
if (prev) {
|
|
187
|
+
prev.refs--
|
|
188
|
+
if (prev.refs <= 0)
|
|
189
|
+
prev.dispose().catch(() => {})
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
function obtain(sid: string, r: { start: string, end: string }): BootedAnalyzer {
|
|
194
|
+
const cache = getCache()
|
|
195
|
+
const key = instanceKey(sid, searchType, r)
|
|
196
|
+
const hit = cache.get(key)
|
|
197
|
+
if (hit)
|
|
198
|
+
return hit
|
|
199
|
+
const created = createAnalyzer({ siteId: sid, searchType, range: r, useOpfsCache })
|
|
200
|
+
cache.set(key, created)
|
|
201
|
+
return created
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
watch(
|
|
205
|
+
() => {
|
|
206
|
+
const sid = toValue(siteId) ?? null
|
|
207
|
+
const r = toValue(range) ?? null
|
|
208
|
+
return sid && r?.start && r?.end ? { sid, r } : null
|
|
209
|
+
},
|
|
210
|
+
(val) => {
|
|
211
|
+
if (!val || !import.meta.client) {
|
|
212
|
+
bind(null)
|
|
213
|
+
return
|
|
214
|
+
}
|
|
215
|
+
bind(obtain(val.sid, val.r))
|
|
216
|
+
},
|
|
217
|
+
{ immediate: true },
|
|
218
|
+
)
|
|
219
|
+
|
|
220
|
+
onScopeDispose(() => bind(null))
|
|
221
|
+
|
|
222
|
+
async function query<T = Record<string, unknown>>(opts: { sql: string, needs: readonly GscFactTable[], params?: readonly unknown[] }): Promise<T[]> {
|
|
223
|
+
if (!active)
|
|
224
|
+
throw new Error('useGscSiteAnalyzer: no active instance (site/range not set)')
|
|
225
|
+
const instance = active
|
|
226
|
+
await instance.bootPromise
|
|
227
|
+
// Kick off (or join) per-table attaches for every table the query needs.
|
|
228
|
+
await Promise.all(opts.needs.map(t => ensureTable(instance, t)))
|
|
229
|
+
return instance.withDb(async (conn) => {
|
|
230
|
+
const result = opts.params && opts.params.length > 0
|
|
231
|
+
? await (async () => {
|
|
232
|
+
const stmt = await conn.prepare(opts.sql)
|
|
233
|
+
try {
|
|
234
|
+
return await stmt.query(...(opts.params as unknown[]))
|
|
235
|
+
}
|
|
236
|
+
finally {
|
|
237
|
+
await stmt.close()
|
|
238
|
+
}
|
|
239
|
+
})()
|
|
240
|
+
: await conn.query(opts.sql)
|
|
241
|
+
// Apache Arrow returns `Proxy(StructRow)` rows — Vue's reactivity
|
|
242
|
+
// can't make those reactive (the Arrow proxy's `isExtensible` trap
|
|
243
|
+
// throws a TypeError when the reactive proxy probes it). Materialise
|
|
244
|
+
// into plain objects at the seam so every consumer is free to assign
|
|
245
|
+
// results directly into a `ref`/`reactive` without surprise.
|
|
246
|
+
const arr = result.toArray() as unknown as Array<Record<string, unknown>>
|
|
247
|
+
return arr.map(row => ({ ...row })) as unknown as T[]
|
|
248
|
+
})
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
async function runQuery<T = Record<string, unknown>>(sql: string, params?: readonly unknown[]): Promise<{ rows: T[], queryMs: number }> {
|
|
252
|
+
const t0 = performance.now()
|
|
253
|
+
const rows = await query<T>({ sql, needs: tablesFromSql(sql), params })
|
|
254
|
+
return { rows, queryMs: performance.now() - t0 }
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
async function analyze(params: AnalysisParams, opts?: { signal?: AbortSignal }): Promise<AnalysisResult & { queryMs: number }> {
|
|
258
|
+
const t0 = performance.now()
|
|
259
|
+
const source = createAttachedTableSource(
|
|
260
|
+
{
|
|
261
|
+
query: async (sql, bindParams) => {
|
|
262
|
+
return query<Record<string, unknown>>({ sql, needs: tablesFromSql(sql), params: bindParams })
|
|
263
|
+
},
|
|
264
|
+
},
|
|
265
|
+
{
|
|
266
|
+
schema: 'main',
|
|
267
|
+
signal: opts?.signal,
|
|
268
|
+
attachedTables: ALL_TABLES as readonly string[],
|
|
269
|
+
},
|
|
270
|
+
)
|
|
271
|
+
const result = await runAnalyzerFromSource(source, params, defaultAnalyzerRegistry)
|
|
272
|
+
return {
|
|
273
|
+
results: result.results as AnalysisResult['results'],
|
|
274
|
+
meta: result.meta as AnalysisResult['meta'],
|
|
275
|
+
queryMs: performance.now() - t0,
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
return {
|
|
280
|
+
tables: tables as Readonly<Ref<Record<GscFactTable, GscTableStatus>>>,
|
|
281
|
+
ready: ready as Readonly<Ref<boolean>>,
|
|
282
|
+
error: error as Readonly<Ref<Error | null>>,
|
|
283
|
+
query,
|
|
284
|
+
runQuery,
|
|
285
|
+
analyze,
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
// ── implementation ──────────────────────────────────────────────────────────
|
|
290
|
+
|
|
291
|
+
interface AnalyzerArgs {
|
|
292
|
+
siteId: string
|
|
293
|
+
searchType: string
|
|
294
|
+
range: { start: string, end: string }
|
|
295
|
+
useOpfsCache: boolean
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
function createAnalyzer(args: AnalyzerArgs): BootedAnalyzer {
|
|
299
|
+
const tables = ref<Record<GscFactTable, GscTableStatus>>(emptyTables())
|
|
300
|
+
const ready = ref(false)
|
|
301
|
+
const error = ref<Error | null>(null)
|
|
302
|
+
const tablePromises = new Map<GscFactTable, Promise<void>>()
|
|
303
|
+
const opfsHandles = new Map<GscFactTable, OpfsAttachedHandle>()
|
|
304
|
+
|
|
305
|
+
// Pin the configured fetcher + analytics config at construction — these
|
|
306
|
+
// need a NuxtApp context, which is available here (the composable's
|
|
307
|
+
// setup) but not inside the async work spawned below.
|
|
308
|
+
const $gscFetch = useGscFetch()
|
|
309
|
+
const analyticsConfig = useGscAnalyticsConfig()
|
|
310
|
+
const analyticsCtx = useGscAnalyticsContext()
|
|
311
|
+
const apiBase = (analyticsConfig.apiBase ?? '').replace(/\/+$/, '')
|
|
312
|
+
|
|
313
|
+
// Single resolve call — `analysis-sources` already returns all 5 tables
|
|
314
|
+
// for a given site+range, so we hit it once and demux per-table later.
|
|
315
|
+
let resolvePromise: Promise<FileResolutionResponse | null> | null = null
|
|
316
|
+
function getResolution(): Promise<FileResolutionResponse | null> {
|
|
317
|
+
if (resolvePromise)
|
|
318
|
+
return resolvePromise
|
|
319
|
+
const qs = `searchType=${args.searchType}&start=${args.range.start}&end=${args.range.end}`
|
|
320
|
+
resolvePromise = $gscFetch<FileResolutionResponse | { canUseBrowser?: false }>(
|
|
321
|
+
`${apiBase}/api/__gsc/sites/${args.siteId}/analysis-sources?${qs}`,
|
|
322
|
+
)
|
|
323
|
+
.then((res) => {
|
|
324
|
+
if (!res || (res as { canUseBrowser?: boolean }).canUseBrowser === false)
|
|
325
|
+
return null
|
|
326
|
+
return res as FileResolutionResponse
|
|
327
|
+
})
|
|
328
|
+
.catch(() => null)
|
|
329
|
+
return resolvePromise
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
// Boot DuckDB-WASM up front — the resolve and the WASM bundle download in
|
|
333
|
+
// parallel, so by the time a query asks for a table, the DB is ready.
|
|
334
|
+
const bootPromise: Promise<{ db: AsyncDuckDB, conn: AsyncDuckDBConnection }> = (async () => {
|
|
335
|
+
// Shared boot: navigating home → site detail (or between sites) reuses
|
|
336
|
+
// the same DuckDB-WASM instance instead of paying ~800ms cold-boot again.
|
|
337
|
+
const bundleBase = (analyticsConfig as { duckdbBundleBase?: string }).duckdbBundleBase
|
|
338
|
+
const boot = await sharedGscDuckDBWasm(bundleBase)
|
|
339
|
+
ready.value = true
|
|
340
|
+
return boot
|
|
341
|
+
})()
|
|
342
|
+
bootPromise.catch((err) => {
|
|
343
|
+
error.value = err instanceof Error ? err : new Error(String(err))
|
|
344
|
+
})
|
|
345
|
+
|
|
346
|
+
// DuckDB-WASM single-connection mutex — DDL and queries serialise; downloads
|
|
347
|
+
// run free in parallel outside the lock.
|
|
348
|
+
let dbLock: Promise<unknown> = Promise.resolve()
|
|
349
|
+
function withDb<T>(fn: (conn: AsyncDuckDBConnection) => Promise<T>): Promise<T> {
|
|
350
|
+
const next = dbLock.then(async () => {
|
|
351
|
+
const { conn } = await bootPromise
|
|
352
|
+
return fn(conn)
|
|
353
|
+
})
|
|
354
|
+
dbLock = next.catch(() => {})
|
|
355
|
+
return next
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
// Per-table attach kicker. Stores its promise so concurrent callers join
|
|
359
|
+
// the same in-flight work; the layer-wide `patchProgress` map is updated
|
|
360
|
+
// as each phase advances so `<GscBootProgress />` shows the site's table
|
|
361
|
+
// load alongside other sites.
|
|
362
|
+
function patchSelf(table: GscFactTable, patch: Partial<GscTableStatus>): void {
|
|
363
|
+
tables.value = { ...tables.value, [table]: { ...tables.value[table], ...patch } }
|
|
364
|
+
}
|
|
365
|
+
function patchLayer(table: GscFactTable, stage: GscTableStage, extras: Partial<GscTableStatus> = {}): void {
|
|
366
|
+
// Reuse the layer's SiteLoadProgress shape, namespacing by table so
|
|
367
|
+
// multiple tables for the same site don't collide on the boot bar.
|
|
368
|
+
const siteSlot = `${args.siteId}#${table}`
|
|
369
|
+
const layerStage = stage === 'unavailable' ? 'ready' : stage === 'downloading' || stage === 'attaching' ? 'attach' : stage === 'resolving' ? 'manifest' : stage === 'ready' ? 'ready' : stage === 'error' ? 'error' : 'idle'
|
|
370
|
+
analyticsCtx.patchProgress(siteSlot, {
|
|
371
|
+
stage: layerStage as never,
|
|
372
|
+
source: 'duckdb',
|
|
373
|
+
filesAttached: extras.filesAttached ?? tables.value[table].filesAttached,
|
|
374
|
+
filesTotal: extras.filesTotal ?? tables.value[table].filesTotal,
|
|
375
|
+
startedAt: extras.startedAt ?? tables.value[table].startedAt ?? Date.now(),
|
|
376
|
+
endedAt: extras.endedAt,
|
|
377
|
+
error: extras.error,
|
|
378
|
+
})
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
async function attachTable(table: GscFactTable): Promise<void> {
|
|
382
|
+
const startedAt = Date.now()
|
|
383
|
+
patchSelf(table, { stage: 'resolving', startedAt, endedAt: undefined, error: undefined })
|
|
384
|
+
patchLayer(table, 'resolving', { startedAt })
|
|
385
|
+
|
|
386
|
+
const full = await getResolution()
|
|
387
|
+
if (!full) {
|
|
388
|
+
patchSelf(table, { stage: 'unavailable', endedAt: Date.now() })
|
|
389
|
+
patchLayer(table, 'unavailable', { endedAt: Date.now() })
|
|
390
|
+
return
|
|
391
|
+
}
|
|
392
|
+
const tableEntry = full.tables.find(t => t.table === table)
|
|
393
|
+
const browserFiles = tableEntry?.mode === 'browser' ? tableEntry.files : []
|
|
394
|
+
if (browserFiles.length === 0) {
|
|
395
|
+
patchSelf(table, { stage: 'unavailable', endedAt: Date.now() })
|
|
396
|
+
patchLayer(table, 'unavailable', { endedAt: Date.now() })
|
|
397
|
+
return
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
patchSelf(table, { stage: 'downloading', filesTotal: browserFiles.length })
|
|
401
|
+
patchLayer(table, 'downloading', { filesTotal: browserFiles.length })
|
|
402
|
+
|
|
403
|
+
const sid = sqlIdent(args.siteId)
|
|
404
|
+
const tableIdent = sqlIdent(table)
|
|
405
|
+
// Per-(site,table) view name keeps OPFS file names unique across sites
|
|
406
|
+
// sharing this DuckDB instance.
|
|
407
|
+
const viewName = `${tableIdent}_${sid}`
|
|
408
|
+
|
|
409
|
+
let filesAttached = 0
|
|
410
|
+
const { db, conn } = await bootPromise
|
|
411
|
+
const opfsHandle = await attachParquetWithFallback({
|
|
412
|
+
db,
|
|
413
|
+
conn,
|
|
414
|
+
viewName,
|
|
415
|
+
files: browserFiles,
|
|
416
|
+
version: full.snapshotVersion,
|
|
417
|
+
useOpfsCache: args.useOpfsCache,
|
|
418
|
+
withDb: fn => withDb(() => fn()),
|
|
419
|
+
onFileProgress: () => {
|
|
420
|
+
filesAttached++
|
|
421
|
+
patchSelf(table, { filesAttached })
|
|
422
|
+
patchLayer(table, 'downloading', { filesAttached })
|
|
423
|
+
},
|
|
424
|
+
}).catch((err) => {
|
|
425
|
+
const msg = err instanceof Error ? err.message : String(err)
|
|
426
|
+
patchSelf(table, { stage: 'error', error: msg, endedAt: Date.now() })
|
|
427
|
+
patchLayer(table, 'error', { error: msg, endedAt: Date.now() })
|
|
428
|
+
throw err
|
|
429
|
+
})
|
|
430
|
+
|
|
431
|
+
// Query SQL references the canonical table name (`pages`, etc.); the
|
|
432
|
+
// underlying view is per-(site,table) so OPFS file names don't collide
|
|
433
|
+
// across sites that may share a DuckDB instance in future.
|
|
434
|
+
if (viewName !== tableIdent) {
|
|
435
|
+
await withDb(async (conn) => {
|
|
436
|
+
await conn.query(`CREATE OR REPLACE VIEW ${tableIdent} AS SELECT * FROM ${viewName}`)
|
|
437
|
+
})
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
if (opfsHandle)
|
|
441
|
+
opfsHandles.set(table, opfsHandle)
|
|
442
|
+
|
|
443
|
+
patchSelf(table, { stage: 'ready', endedAt: Date.now() })
|
|
444
|
+
patchLayer(table, 'ready', { endedAt: Date.now() })
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
function ensureTableInner(table: GscFactTable): Promise<void> {
|
|
448
|
+
const existing = tablePromises.get(table)
|
|
449
|
+
if (existing)
|
|
450
|
+
return existing
|
|
451
|
+
const p = attachTable(table)
|
|
452
|
+
tablePromises.set(table, p)
|
|
453
|
+
return p
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
const instance: BootedAnalyzer = {
|
|
457
|
+
bootPromise,
|
|
458
|
+
tablePromises,
|
|
459
|
+
tables,
|
|
460
|
+
ready,
|
|
461
|
+
error,
|
|
462
|
+
withDb,
|
|
463
|
+
refs: 0,
|
|
464
|
+
dispose: async () => {
|
|
465
|
+
// Pull the cache entry first so subsequent obtains create a fresh one
|
|
466
|
+
// (we can't dispose mid-attach without leaking promises).
|
|
467
|
+
const cache = instanceCaches.get(useNuxtApp())
|
|
468
|
+
if (cache) {
|
|
469
|
+
for (const [k, v] of cache.entries()) {
|
|
470
|
+
if (v === instance) {
|
|
471
|
+
cache.delete(k)
|
|
472
|
+
break
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
}
|
|
476
|
+
try {
|
|
477
|
+
for (const h of opfsHandles.values())
|
|
478
|
+
await h.detach().catch(() => {})
|
|
479
|
+
opfsHandles.clear()
|
|
480
|
+
// Drop the per-site views we created so a re-attach is clean. Do NOT
|
|
481
|
+
// terminate the DB — it's shared via `sharedGscDuckDBWasm`.
|
|
482
|
+
const { conn } = await bootPromise
|
|
483
|
+
for (const t of ALL_TABLES)
|
|
484
|
+
await conn.query(`DROP VIEW IF EXISTS ${sqlIdent(t)}`).catch(() => {})
|
|
485
|
+
}
|
|
486
|
+
catch {
|
|
487
|
+
// Boot failed — nothing to tear down.
|
|
488
|
+
}
|
|
489
|
+
},
|
|
490
|
+
}
|
|
491
|
+
// Attach the ensureTable closure to the instance so the outer composable
|
|
492
|
+
// can reach it without a globalThis hack.
|
|
493
|
+
;(instance as BootedAnalyzer & { ensureTable: typeof ensureTableInner }).ensureTable = ensureTableInner
|
|
494
|
+
return instance
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
async function ensureTable(instance: BootedAnalyzer, table: GscFactTable): Promise<void> {
|
|
498
|
+
const ensure = (instance as BootedAnalyzer & { ensureTable: (t: GscFactTable) => Promise<void> }).ensureTable
|
|
499
|
+
return ensure(table)
|
|
500
|
+
}
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
// Shared browser-DuckDB-WASM helpers used by per-site analyzers and the
|
|
2
|
+
// home-page multi-site fanout. Both consumers boot the same WASM bundle and
|
|
3
|
+
// run the same in-memory parquet attach path; OPFS attach lives in the caller
|
|
4
|
+
// because the two lifecycle models (refcounted long-lived vs ephemeral
|
|
5
|
+
// per-run) need different handle ownership.
|
|
6
|
+
|
|
7
|
+
import type { AsyncDuckDB, AsyncDuckDBConnection } from '@duckdb/duckdb-wasm'
|
|
8
|
+
import type { DuckDBWasmBootResult, OpfsAttachedHandle } from '@gscdump/engine-duckdb-wasm'
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Boot DuckDB-WASM using the layer's configured bundle base. The dynamic
|
|
12
|
+
* `import('@gscdump/engine-duckdb-wasm')` keeps the WASM bundle out of the
|
|
13
|
+
* SSR build; callers should also gate the call behind `import.meta.client`.
|
|
14
|
+
*
|
|
15
|
+
* The `filesystem` config disables HEAD-request preflight and forces full
|
|
16
|
+
* HTTP reads — Cloudflare Workers strips `Content-Length` on streamed
|
|
17
|
+
* responses, which makes DuckDB's default range-request path 404.
|
|
18
|
+
*/
|
|
19
|
+
export async function bootGscDuckDBWasm(bundleBase?: string): Promise<DuckDBWasmBootResult> {
|
|
20
|
+
const { bootDuckDBWasm } = await import('@gscdump/engine-duckdb-wasm')
|
|
21
|
+
return bootDuckDBWasm({
|
|
22
|
+
...(bundleBase
|
|
23
|
+
? {
|
|
24
|
+
bundles: {
|
|
25
|
+
mvp: { mainModule: `${bundleBase}/duckdb-mvp.wasm`, mainWorker: `${bundleBase}/duckdb-browser-mvp.worker.js` },
|
|
26
|
+
eh: { mainModule: `${bundleBase}/duckdb-eh.wasm`, mainWorker: `${bundleBase}/duckdb-browser-eh.worker.js` },
|
|
27
|
+
},
|
|
28
|
+
}
|
|
29
|
+
: {}),
|
|
30
|
+
config: { filesystem: { reliableHeadRequests: false, allowFullHTTPReads: true } },
|
|
31
|
+
})
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Process-wide shared boot. The ~800ms WASM boot is the dominant cost on the
|
|
36
|
+
* warm path (OPFS cache-hit means downloads are skipped); paying it once per
|
|
37
|
+
* session instead of per refresh turns date-range changes / search-type
|
|
38
|
+
* swaps into near-instant updates.
|
|
39
|
+
*
|
|
40
|
+
* Callers that want their own throwaway instance can keep using
|
|
41
|
+
* {@link bootGscDuckDBWasm}; those that want session-wide reuse — the home
|
|
42
|
+
* page fanout and the per-site analyzer — go through this.
|
|
43
|
+
*
|
|
44
|
+
* Browser-only. The `bundleBase` is captured from the first call; subsequent
|
|
45
|
+
* calls reuse the promise and ignore the argument (it's a deployment-wide
|
|
46
|
+
* constant in practice).
|
|
47
|
+
*/
|
|
48
|
+
let sharedBootPromise: Promise<DuckDBWasmBootResult> | null = null
|
|
49
|
+
let sharedBootColdMs: number | undefined
|
|
50
|
+
export function sharedGscDuckDBWasm(bundleBase?: string): Promise<DuckDBWasmBootResult> {
|
|
51
|
+
if (!sharedBootPromise) {
|
|
52
|
+
if (typeof performance !== 'undefined')
|
|
53
|
+
performance.mark('gsc:duckdb:boot:start')
|
|
54
|
+
const start = typeof performance !== 'undefined' ? performance.now() : 0
|
|
55
|
+
sharedBootPromise = bootGscDuckDBWasm(bundleBase).then((b) => {
|
|
56
|
+
if (typeof performance !== 'undefined') {
|
|
57
|
+
sharedBootColdMs = performance.now() - start
|
|
58
|
+
performance.mark('gsc:duckdb:boot:end')
|
|
59
|
+
}
|
|
60
|
+
return b
|
|
61
|
+
})
|
|
62
|
+
}
|
|
63
|
+
else if (typeof performance !== 'undefined') {
|
|
64
|
+
performance.mark('gsc:duckdb:boot:reuse')
|
|
65
|
+
}
|
|
66
|
+
return sharedBootPromise
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** Diagnostics: ms paid for the one-time cold boot, or undefined if not yet booted. */
|
|
70
|
+
export function sharedGscDuckDBWasmColdMs(): number | undefined {
|
|
71
|
+
return sharedBootColdMs
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Fetch every URL into a `Uint8Array` in parallel. `onFileFetched` ticks
|
|
76
|
+
* once per successful download. Separate from `registerParquetView` so
|
|
77
|
+
* callers can run fetches outside any connection mutex.
|
|
78
|
+
*/
|
|
79
|
+
export async function fetchParquetBuffers(
|
|
80
|
+
urls: readonly string[],
|
|
81
|
+
onFileFetched?: () => void,
|
|
82
|
+
): Promise<Uint8Array[]> {
|
|
83
|
+
return Promise.all(urls.map(async (url) => {
|
|
84
|
+
const r = await fetch(url, { credentials: 'omit' })
|
|
85
|
+
if (!r.ok)
|
|
86
|
+
throw new Error(`GET ${url} failed: ${r.status}`)
|
|
87
|
+
const buf = new Uint8Array(await r.arrayBuffer())
|
|
88
|
+
onFileFetched?.()
|
|
89
|
+
return buf
|
|
90
|
+
}))
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Register pre-fetched parquet buffers as virtual files, then build a single
|
|
95
|
+
* view that unions them. `CAST(date AS DATE)` canonicalises the legacy-VARCHAR
|
|
96
|
+
* / new-DATE encodings. File-name uniqueness is the caller's job; pass a
|
|
97
|
+
* `viewName` that already encodes any site/table salt so concurrent attaches
|
|
98
|
+
* across sites don't collide on the DuckDB virtual filesystem. Call under the
|
|
99
|
+
* DuckDB-WASM single-connection mutex.
|
|
100
|
+
*/
|
|
101
|
+
export async function registerParquetView(opts: {
|
|
102
|
+
db: AsyncDuckDB
|
|
103
|
+
conn: AsyncDuckDBConnection
|
|
104
|
+
viewName: string
|
|
105
|
+
buffers: readonly Uint8Array[]
|
|
106
|
+
}): Promise<void> {
|
|
107
|
+
const { db, conn, viewName, buffers } = opts
|
|
108
|
+
const fileNames = buffers.map((_, i) => `${viewName}_${i}.parquet`)
|
|
109
|
+
for (let i = 0; i < buffers.length; i++)
|
|
110
|
+
await db.registerFileBuffer(fileNames[i]!, buffers[i]!)
|
|
111
|
+
const fileList = fileNames.map(n => `'${n.replace(/'/g, '\'\'')}'`).join(',')
|
|
112
|
+
await conn.query(`CREATE OR REPLACE VIEW ${viewName} AS SELECT * REPLACE (CAST(date AS DATE) AS date) FROM read_parquet([${fileList}], union_by_name = true)`)
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* Attach a parquet file set into DuckDB-WASM, preferring the OPFS-backed
|
|
117
|
+
* path (cache-hit skips download) and falling back to in-memory buffers when
|
|
118
|
+
* OPFS is opted out, unavailable, or quota-exhausted.
|
|
119
|
+
*
|
|
120
|
+
* Returns the OPFS handle when the OPFS path succeeded — caller must keep
|
|
121
|
+
* the handle alive and `detach()` on cleanup. Returns `null` when the
|
|
122
|
+
* fallback path was used; nothing to detach. `onFileProgress` ticks once per
|
|
123
|
+
* downloaded file in either path so the caller can drive a progress UI.
|
|
124
|
+
*
|
|
125
|
+
* `withDb` wraps the DDL/file-register calls when the caller serialises
|
|
126
|
+
* against other DuckDB activity (e.g. a shared connection); pass `undefined`
|
|
127
|
+
* when the caller already owns a private connection.
|
|
128
|
+
*/
|
|
129
|
+
export async function attachParquetWithFallback(args: {
|
|
130
|
+
db: AsyncDuckDB
|
|
131
|
+
conn: AsyncDuckDBConnection
|
|
132
|
+
viewName: string
|
|
133
|
+
files: readonly { url: string, bytes: number, contentHash: string }[]
|
|
134
|
+
version: string
|
|
135
|
+
useOpfsCache: boolean
|
|
136
|
+
fetchConcurrency?: number
|
|
137
|
+
onFileProgress?: () => void
|
|
138
|
+
withDb?: <T>(fn: () => Promise<T>) => Promise<T>
|
|
139
|
+
}): Promise<OpfsAttachedHandle | null> {
|
|
140
|
+
const { db, conn, viewName, files, version, useOpfsCache, onFileProgress } = args
|
|
141
|
+
const lock = args.withDb ?? (<T>(fn: () => Promise<T>) => fn())
|
|
142
|
+
const fetchConcurrency = args.fetchConcurrency ?? Math.min(files.length, 4)
|
|
143
|
+
|
|
144
|
+
if (useOpfsCache) {
|
|
145
|
+
const { attachOpfsParquetTables } = await import('@gscdump/engine-duckdb-wasm')
|
|
146
|
+
const handle = await lock(() => attachOpfsParquetTables({
|
|
147
|
+
db,
|
|
148
|
+
conn,
|
|
149
|
+
tables: [{
|
|
150
|
+
table: viewName,
|
|
151
|
+
files: files.map(f => ({ url: f.url, bytes: f.bytes, contentHash: f.contentHash })),
|
|
152
|
+
}],
|
|
153
|
+
version,
|
|
154
|
+
fetchConcurrency,
|
|
155
|
+
onFileProgress,
|
|
156
|
+
}))
|
|
157
|
+
if (!handle.degradedTables.includes(viewName))
|
|
158
|
+
return handle
|
|
159
|
+
// OPFS quota / failure — fall through to buffer path. Caller's progress
|
|
160
|
+
// counter restarts; both paths report one tick per fetched file.
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
const buffers = await fetchParquetBuffers(files.map(f => f.url), onFileProgress)
|
|
164
|
+
await lock(() => registerParquetView({ db, conn, viewName, buffers }))
|
|
165
|
+
return null
|
|
166
|
+
}
|
package/nuxt.config.ts
CHANGED
|
@@ -14,10 +14,15 @@ import { fileURLToPath } from 'node:url'
|
|
|
14
14
|
export default defineNuxtConfig({
|
|
15
15
|
// Register the layer's module so hooks + runtime config land on host apps
|
|
16
16
|
// without them having to wire each module individually.
|
|
17
|
+
//
|
|
18
|
+
// `nuxt-use-query` provides `useNuxtQuery` / `useQueryCache` — TanStack-Query-
|
|
19
|
+
// shaped wrapper over `useFetch`. Composables in this layer fetch through
|
|
20
|
+
// it for SWR + dedup.
|
|
17
21
|
modules: [
|
|
18
22
|
fileURLToPath(new URL('./module.ts', import.meta.url)),
|
|
19
23
|
'@nuxt/ui',
|
|
20
24
|
'@vueuse/nuxt',
|
|
25
|
+
'nuxt-use-query',
|
|
21
26
|
],
|
|
22
27
|
|
|
23
28
|
css: [
|