@gscdump/nuxt 0.20.3 → 0.21.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CONTRACT — consolidated analyzer composable public API (Wave-2, frozen).
|
|
3
|
+
*
|
|
4
|
+
* The single composable that replaces BOTH `useGscAnalyzer.ts` (pkg) and
|
|
5
|
+
* `useBrowserAnalyzer.ts` (app). One OPFS-backed, attach-once, browser-
|
|
6
|
+
* eligibility-aware analyzer.
|
|
7
|
+
*
|
|
8
|
+
* This file is the INTERFACE ONLY — the Wave-2 implementation agent writes
|
|
9
|
+
* `useGscSnapshotAnalyzer.ts` against these types, then deletes the two old
|
|
10
|
+
* composables. Naming follows the package convention (`useGsc*`).
|
|
11
|
+
*
|
|
12
|
+
* Behaviour the types lock in:
|
|
13
|
+
* - ATTACH-ONCE — DuckDB-WASM boots once; OPFS-cached parquet attaches once
|
|
14
|
+
* per `(site, searchType)`. Filter / date-range changes within the attached
|
|
15
|
+
* span re-query, they do NOT re-attach.
|
|
16
|
+
* - OPFS-backed — files resolved via the file-resolution endpoint are cached
|
|
17
|
+
* in OPFS (content-hash verified); steady-state queries are zero-network.
|
|
18
|
+
* - RESULT LRU — identical archetype queries replay from an in-memory LRU.
|
|
19
|
+
* - BROWSER-ELIGIBILITY AWARE — when the file-resolution endpoint returns a
|
|
20
|
+
* server-tail directive for a table, `query()` for that table transparently
|
|
21
|
+
* routes server-side; the consumer does not branch.
|
|
22
|
+
*
|
|
23
|
+
* TYPES ONLY — no composable body.
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
import type { GscSearchType } from '@gscdump/contracts'
|
|
27
|
+
import type {
|
|
28
|
+
ArchetypeQuery,
|
|
29
|
+
ArchetypeResult,
|
|
30
|
+
ArchetypeResultRow,
|
|
31
|
+
} from '@gscdump/sdk'
|
|
32
|
+
import type { MaybeRefOrGetter, Ref } from 'vue'
|
|
33
|
+
|
|
34
|
+
/** Date window the analyzer is attached over. */
|
|
35
|
+
export interface AnalyzerRange {
|
|
36
|
+
start: string
|
|
37
|
+
end: string
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** Boot / attach lifecycle phase, surfaced for the progress UI. */
|
|
41
|
+
export type AnalyzerPhase
|
|
42
|
+
= | 'idle'
|
|
43
|
+
| 'resolving' // calling the file-resolution endpoint
|
|
44
|
+
| 'booting' // DuckDB-WASM boot
|
|
45
|
+
| 'downloading' // fetching parquet into OPFS
|
|
46
|
+
| 'attaching' // registering OPFS files as DuckDB views
|
|
47
|
+
| 'ready'
|
|
48
|
+
| 'error'
|
|
49
|
+
|
|
50
|
+
/** Progressive-load progress, drives `<GscBootProgress>`. */
|
|
51
|
+
export interface AnalyzerProgress {
|
|
52
|
+
phase: AnalyzerPhase
|
|
53
|
+
/** Files fetched into OPFS so far. */
|
|
54
|
+
filesReady: number
|
|
55
|
+
/** Total files to fetch for the current attach. */
|
|
56
|
+
filesTotal: number
|
|
57
|
+
/** Bytes fetched into OPFS so far. */
|
|
58
|
+
bytesReady: number
|
|
59
|
+
bytesTotal: number
|
|
60
|
+
startedAt?: number
|
|
61
|
+
endedAt?: number
|
|
62
|
+
error?: string
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** OPFS storage health, surfaced so the UI can warn before eviction. */
|
|
66
|
+
export interface AnalyzerStorageState {
|
|
67
|
+
/** `navigator.storage.persist()` result. `false` => eviction-eligible. */
|
|
68
|
+
persisted: boolean
|
|
69
|
+
/** Estimated bytes used by this origin's OPFS. */
|
|
70
|
+
usageBytes?: number
|
|
71
|
+
/** Estimated quota. */
|
|
72
|
+
quotaBytes?: number
|
|
73
|
+
/** True after a `QuotaExceededError` forced a degraded (server-tail) path. */
|
|
74
|
+
degraded: boolean
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Per-table routing the analyzer resolved. `'browser'` tables answer locally
|
|
79
|
+
* from OPFS; `'server'` tables route to the server tail. Consumers may read
|
|
80
|
+
* this to show "deep history served from the cloud" UX.
|
|
81
|
+
*/
|
|
82
|
+
export type AnalyzerTableRouting = Record<string, 'browser' | 'server'>
|
|
83
|
+
|
|
84
|
+
/** Options for `useGscSnapshotAnalyzer`. */
|
|
85
|
+
export interface GscSnapshotAnalyzerOptions {
|
|
86
|
+
/** Max entries in the result LRU. Default implementation-defined. */
|
|
87
|
+
resultCacheSize?: number
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* The consolidated analyzer instance. Refs are reactive over the currently
|
|
92
|
+
* bound `(site, searchType, range)` — switching any of them rebinds and the
|
|
93
|
+
* refs track the new attach with no manual mirroring.
|
|
94
|
+
*/
|
|
95
|
+
export interface GscSnapshotAnalyzer {
|
|
96
|
+
/** True once the analyzer can serve `query()` (browser attached OR server-tail ready). */
|
|
97
|
+
ready: Ref<boolean>
|
|
98
|
+
/** True while resolving / booting / attaching. */
|
|
99
|
+
initializing: Ref<boolean>
|
|
100
|
+
error: Ref<Error | null>
|
|
101
|
+
progress: Ref<AnalyzerProgress>
|
|
102
|
+
storage: Ref<AnalyzerStorageState>
|
|
103
|
+
/** Per-table browser vs server routing for the current attach. */
|
|
104
|
+
routing: Ref<AnalyzerTableRouting>
|
|
105
|
+
/** Snapshot version of the currently-attached file set (re-attach trigger). */
|
|
106
|
+
snapshotVersion: Ref<string | undefined>
|
|
107
|
+
/** The site currently bound. */
|
|
108
|
+
currentSiteId: Ref<string | null>
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* Run a typed archetype query. Routes automatically:
|
|
112
|
+
* - browser-eligible tables → DuckDB-WASM over OPFS files.
|
|
113
|
+
* - server-tail tables → R2 SQL or server DuckDB per the directive.
|
|
114
|
+
* Identical queries replay from the result LRU. Honours `signal`.
|
|
115
|
+
*/
|
|
116
|
+
query: <R extends ArchetypeResultRow = ArchetypeResultRow>(
|
|
117
|
+
q: ArchetypeQuery,
|
|
118
|
+
opts?: { signal?: AbortSignal },
|
|
119
|
+
) => Promise<ArchetypeResult<R>>
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* Re-probe the file-resolution endpoint; if `snapshotVersion` changed,
|
|
123
|
+
* detach stale views and re-attach the fresher parquet in place (the
|
|
124
|
+
* DuckDB-WASM runtime stays alive). Resolves true if it re-attached.
|
|
125
|
+
* No-op for fully server-tail-routed sites.
|
|
126
|
+
*/
|
|
127
|
+
refresh: () => Promise<boolean>
|
|
128
|
+
|
|
129
|
+
/** Drop the result LRU without detaching. Cheap cache bust. */
|
|
130
|
+
clearCache: () => void
|
|
131
|
+
|
|
132
|
+
/** Detach views, close the runtime, release OPFS handles. Idempotent. */
|
|
133
|
+
dispose: () => Promise<void>
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* Get (or create) the consolidated analyzer for a site. Per-`(site,
|
|
138
|
+
* searchType, range)` cached + refcounted across the app, so panels on the
|
|
139
|
+
* same site share one DuckDB-WASM boot and one OPFS attach.
|
|
140
|
+
*
|
|
141
|
+
* Replaces `useGscAnalyzer` (pkg) and `useBrowserAnalyzer` /
|
|
142
|
+
* `provideBrowserAnalyzer` / `useSharedBrowserAnalyzer` (app) — those are
|
|
143
|
+
* deleted once consumers migrate.
|
|
144
|
+
*/
|
|
145
|
+
export type UseGscSnapshotAnalyzer = (
|
|
146
|
+
siteId: MaybeRefOrGetter<string | null | undefined>,
|
|
147
|
+
searchType?: MaybeRefOrGetter<GscSearchType | undefined>,
|
|
148
|
+
range?: MaybeRefOrGetter<AnalyzerRange | null | undefined>,
|
|
149
|
+
options?: GscSnapshotAnalyzerOptions,
|
|
150
|
+
) => GscSnapshotAnalyzer
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pure routing + caching helpers for `useGscSnapshotAnalyzer`.
|
|
3
|
+
*
|
|
4
|
+
* Extracted into a standalone, Nuxt-free module so the routing decision and
|
|
5
|
+
* the result LRU are unit-testable without a Nuxt runtime (the composable file
|
|
6
|
+
* itself depends on Nuxt auto-imports).
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import type { ArchetypeQuery } from '@gscdump/sdk'
|
|
10
|
+
import type { AnalyzerTableRouting } from './useGscSnapshotAnalyzer.contract'
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Decide where an archetype query runs given the analyzer's per-table routing.
|
|
14
|
+
*
|
|
15
|
+
* - `aux-cloud-only` is always `cloud` (not an Iceberg query).
|
|
16
|
+
* - Otherwise the fact-table the archetype reads decides: a table the resolver
|
|
17
|
+
* marked `browser` runs locally; `server` (or degraded by a quota error)
|
|
18
|
+
* routes to the server tail. An unknown table fails safe to `server`.
|
|
19
|
+
*/
|
|
20
|
+
export function routeArchetype(
|
|
21
|
+
query: ArchetypeQuery,
|
|
22
|
+
routing: AnalyzerTableRouting,
|
|
23
|
+
tableOf: (q: ArchetypeQuery) => string | null,
|
|
24
|
+
): 'browser' | 'server' | 'cloud' {
|
|
25
|
+
if (query.archetype === 'aux-cloud-only')
|
|
26
|
+
return 'cloud'
|
|
27
|
+
const table = tableOf(query)
|
|
28
|
+
if (!table)
|
|
29
|
+
return 'cloud'
|
|
30
|
+
return routing[table] === 'browser' ? 'browser' : 'server'
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Stable hash of an archetype query for the result LRU. Archetype inputs are
|
|
35
|
+
* flat + declarative; keys are sorted so insertion order can't perturb the key.
|
|
36
|
+
*/
|
|
37
|
+
export function archetypeQueryHash(query: ArchetypeQuery): string {
|
|
38
|
+
return JSON.stringify(query, Object.keys(query as unknown as Record<string, unknown>).sort())
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Compose the LRU cache key. Keyed by `snapshotVersion` so a fresh compaction
|
|
43
|
+
* (new snapshot) invalidates every cached result without an explicit bust.
|
|
44
|
+
*/
|
|
45
|
+
export function resultCacheKey(snapshotVersion: string | undefined, query: ArchetypeQuery): string {
|
|
46
|
+
return `${snapshotVersion ?? 'none'}::${archetypeQueryHash(query)}`
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export interface ResultLru<V> {
|
|
50
|
+
get: (key: string) => V | undefined
|
|
51
|
+
set: (key: string, value: V) => void
|
|
52
|
+
clear: () => void
|
|
53
|
+
readonly size: number
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** Minimal insertion-ordered LRU. Re-inserts on `get` to mark recency. */
|
|
57
|
+
export function createResultLru<V>(capacity: number): ResultLru<V> {
|
|
58
|
+
const map = new Map<string, V>()
|
|
59
|
+
const cap = Math.max(1, Math.floor(capacity))
|
|
60
|
+
return {
|
|
61
|
+
get(key) {
|
|
62
|
+
if (!map.has(key))
|
|
63
|
+
return undefined
|
|
64
|
+
const value = map.get(key)!
|
|
65
|
+
map.delete(key)
|
|
66
|
+
map.set(key, value)
|
|
67
|
+
return value
|
|
68
|
+
},
|
|
69
|
+
set(key, value) {
|
|
70
|
+
if (map.has(key))
|
|
71
|
+
map.delete(key)
|
|
72
|
+
map.set(key, value)
|
|
73
|
+
while (map.size > cap) {
|
|
74
|
+
const oldest = map.keys().next().value
|
|
75
|
+
if (oldest === undefined)
|
|
76
|
+
break
|
|
77
|
+
map.delete(oldest)
|
|
78
|
+
}
|
|
79
|
+
},
|
|
80
|
+
clear() {
|
|
81
|
+
map.clear()
|
|
82
|
+
},
|
|
83
|
+
get size() {
|
|
84
|
+
return map.size
|
|
85
|
+
},
|
|
86
|
+
}
|
|
87
|
+
}
|
|
@@ -0,0 +1,554 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* useGscSnapshotAnalyzer — the single consolidated GSC analyzer composable.
|
|
3
|
+
*
|
|
4
|
+
* Replaces `useGscAnalyzer.ts` (pkg) and `useBrowserAnalyzer.ts` (app). One
|
|
5
|
+
* OPFS-backed, attach-once, browser-eligibility-aware analyzer.
|
|
6
|
+
*
|
|
7
|
+
* Implements the frozen contract in `useGscSnapshotAnalyzer.contract.ts`.
|
|
8
|
+
*
|
|
9
|
+
* Flow per `(site, searchType, range)`:
|
|
10
|
+
* 1. RESOLVE — call the file-resolution endpoint (`/analysis-sources`). It
|
|
11
|
+
* returns per-table `mode: browser | server` and, for browser tables, the
|
|
12
|
+
* compacted Iceberg parquet files (content hash + signed URL).
|
|
13
|
+
* 2. BOOT — DuckDB-WASM boots once.
|
|
14
|
+
* 3. DOWNLOAD + ATTACH — browser-mode files download into OPFS (content-hash
|
|
15
|
+
* verified) and attach as views. ATTACH-ONCE: filter / range changes
|
|
16
|
+
* inside the attached span re-query, never re-attach.
|
|
17
|
+
* 4. QUERY — `query(archetype)` routes per-table: browser-mode tables run
|
|
18
|
+
* locally against OPFS; server-mode tables POST to the server tail.
|
|
19
|
+
* Identical queries replay from an in-memory LRU keyed by
|
|
20
|
+
* `snapshotVersion + queryHash`.
|
|
21
|
+
*
|
|
22
|
+
* Quota: a `QuotaExceededError` during OPFS download degrades the affected
|
|
23
|
+
* table to the server tail (`storage.degraded = true`); it never crashes.
|
|
24
|
+
*
|
|
25
|
+
* The instance is cached + refcounted per `(site, searchType, range)` at module
|
|
26
|
+
* scope (keyed by NuxtApp) so panels on the same site share one DuckDB-WASM
|
|
27
|
+
* boot and one OPFS attach.
|
|
28
|
+
*/
|
|
29
|
+
|
|
30
|
+
import type { OpfsFileProgress } from '@gscdump/engine-duckdb-wasm'
|
|
31
|
+
import type {
|
|
32
|
+
ArchetypeQuery,
|
|
33
|
+
ArchetypeResult,
|
|
34
|
+
ArchetypeResultRow,
|
|
35
|
+
ArchetypeResultSource,
|
|
36
|
+
FileResolutionResponse,
|
|
37
|
+
GscSearchType,
|
|
38
|
+
ResolvedTable,
|
|
39
|
+
} from '@gscdump/sdk'
|
|
40
|
+
import type { NuxtApp } from 'nuxt/app'
|
|
41
|
+
import type {
|
|
42
|
+
AnalyzerProgress,
|
|
43
|
+
AnalyzerRange,
|
|
44
|
+
AnalyzerStorageState,
|
|
45
|
+
AnalyzerTableRouting,
|
|
46
|
+
GscSnapshotAnalyzer,
|
|
47
|
+
GscSnapshotAnalyzerOptions,
|
|
48
|
+
UseGscSnapshotAnalyzer,
|
|
49
|
+
} from './useGscSnapshotAnalyzer.contract'
|
|
50
|
+
import { createResultLru, resultCacheKey, routeArchetype } from './useGscSnapshotAnalyzer.routing'
|
|
51
|
+
|
|
52
|
+
const DEFAULT_SEARCH_TYPE: GscSearchType = 'web'
|
|
53
|
+
const DEFAULT_RESULT_CACHE_SIZE = 64
|
|
54
|
+
const DEFAULT_ATTACH_FETCH_CONCURRENCY = 2
|
|
55
|
+
|
|
56
|
+
const IDLE_PROGRESS: AnalyzerProgress = Object.freeze({
|
|
57
|
+
phase: 'idle',
|
|
58
|
+
filesReady: 0,
|
|
59
|
+
filesTotal: 0,
|
|
60
|
+
bytesReady: 0,
|
|
61
|
+
bytesTotal: 0,
|
|
62
|
+
})
|
|
63
|
+
|
|
64
|
+
const IDLE_STORAGE: AnalyzerStorageState = Object.freeze({
|
|
65
|
+
persisted: false,
|
|
66
|
+
degraded: false,
|
|
67
|
+
})
|
|
68
|
+
|
|
69
|
+
// ── instance cache (per NuxtApp, refcounted) ────────────────────────────────
|
|
70
|
+
|
|
71
|
+
interface CacheEntry {
|
|
72
|
+
instance: GscSnapshotAnalyzer
|
|
73
|
+
refs: number
|
|
74
|
+
}
|
|
75
|
+
const instanceBags = new WeakMap<NuxtApp, Map<string, CacheEntry>>()
|
|
76
|
+
|
|
77
|
+
function instanceCacheFor(app: NuxtApp): Map<string, CacheEntry> {
|
|
78
|
+
let bag = instanceBags.get(app)
|
|
79
|
+
if (!bag) {
|
|
80
|
+
bag = new Map()
|
|
81
|
+
instanceBags.set(app, bag)
|
|
82
|
+
}
|
|
83
|
+
return bag
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function instanceKey(siteId: string, searchType: GscSearchType, range: AnalyzerRange | null): string {
|
|
87
|
+
return JSON.stringify([siteId, searchType, range?.start ?? null, range?.end ?? null])
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
function normalizeRange(range: AnalyzerRange | null | undefined): AnalyzerRange | null {
|
|
91
|
+
return range?.start && range?.end ? { start: range.start, end: range.end } : null
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// ── the composable ──────────────────────────────────────────────────────────
|
|
95
|
+
|
|
96
|
+
export const useGscSnapshotAnalyzer: UseGscSnapshotAnalyzer = (
|
|
97
|
+
siteId,
|
|
98
|
+
searchType,
|
|
99
|
+
range,
|
|
100
|
+
options,
|
|
101
|
+
) => {
|
|
102
|
+
const app = useNuxtApp()
|
|
103
|
+
const cache = instanceCacheFor(app)
|
|
104
|
+
|
|
105
|
+
const key = computed(() => {
|
|
106
|
+
const id = toValue(siteId)
|
|
107
|
+
if (!id)
|
|
108
|
+
return null
|
|
109
|
+
return instanceKey(
|
|
110
|
+
id,
|
|
111
|
+
toValue(searchType) ?? DEFAULT_SEARCH_TYPE,
|
|
112
|
+
normalizeRange(toValue(range)),
|
|
113
|
+
)
|
|
114
|
+
})
|
|
115
|
+
|
|
116
|
+
/** Acquire (or create) + refcount the cached instance for the current key. */
|
|
117
|
+
function acquire(k: string): GscSnapshotAnalyzer {
|
|
118
|
+
let entry = cache.get(k)
|
|
119
|
+
if (!entry) {
|
|
120
|
+
const [id, st, start, end] = JSON.parse(k) as [string, GscSearchType, string | null, string | null]
|
|
121
|
+
entry = {
|
|
122
|
+
instance: createSnapshotAnalyzerInstance(
|
|
123
|
+
id,
|
|
124
|
+
st,
|
|
125
|
+
start && end ? { start, end } : null,
|
|
126
|
+
options ?? {},
|
|
127
|
+
),
|
|
128
|
+
refs: 0,
|
|
129
|
+
}
|
|
130
|
+
cache.set(k, entry)
|
|
131
|
+
}
|
|
132
|
+
entry.refs++
|
|
133
|
+
return entry.instance
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
function release(k: string): void {
|
|
137
|
+
const entry = cache.get(k)
|
|
138
|
+
if (!entry)
|
|
139
|
+
return
|
|
140
|
+
entry.refs--
|
|
141
|
+
if (entry.refs <= 0) {
|
|
142
|
+
cache.delete(k)
|
|
143
|
+
entry.instance.dispose().catch((e) => {
|
|
144
|
+
console.error('[useGscSnapshotAnalyzer] dispose failed', e)
|
|
145
|
+
})
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
// Bind to the current key; rebind (release old, acquire new) on change.
|
|
150
|
+
let boundKey: string | null = null
|
|
151
|
+
const bound = shallowRef<GscSnapshotAnalyzer | null>(null)
|
|
152
|
+
|
|
153
|
+
watch(key, (next, prev) => {
|
|
154
|
+
if (next === prev)
|
|
155
|
+
return
|
|
156
|
+
if (boundKey)
|
|
157
|
+
release(boundKey)
|
|
158
|
+
boundKey = next
|
|
159
|
+
bound.value = next ? acquire(next) : null
|
|
160
|
+
}, { immediate: true })
|
|
161
|
+
|
|
162
|
+
tryOnScopeDispose(() => {
|
|
163
|
+
if (boundKey)
|
|
164
|
+
release(boundKey)
|
|
165
|
+
boundKey = null
|
|
166
|
+
})
|
|
167
|
+
|
|
168
|
+
// Reactive views over the bound instance — track both site switch and inner
|
|
169
|
+
// ref updates. A null-bound analyzer reports a safe idle state.
|
|
170
|
+
const ready = computed(() => bound.value?.ready.value ?? false) as Ref<boolean>
|
|
171
|
+
const initializing = computed(() => bound.value?.initializing.value ?? false) as Ref<boolean>
|
|
172
|
+
const error = computed(() => bound.value?.error.value ?? null) as Ref<Error | null>
|
|
173
|
+
const progress = computed<AnalyzerProgress>(() => bound.value?.progress.value ?? IDLE_PROGRESS) as Ref<AnalyzerProgress>
|
|
174
|
+
const storage = computed<AnalyzerStorageState>(() => bound.value?.storage.value ?? IDLE_STORAGE) as Ref<AnalyzerStorageState>
|
|
175
|
+
const routing = computed<AnalyzerTableRouting>(() => bound.value?.routing.value ?? {}) as Ref<AnalyzerTableRouting>
|
|
176
|
+
const snapshotVersion = computed(() => bound.value?.snapshotVersion.value) as Ref<string | undefined>
|
|
177
|
+
const currentSiteId = computed(() => bound.value?.currentSiteId.value ?? null) as Ref<string | null>
|
|
178
|
+
|
|
179
|
+
return {
|
|
180
|
+
ready,
|
|
181
|
+
initializing,
|
|
182
|
+
error,
|
|
183
|
+
progress,
|
|
184
|
+
storage,
|
|
185
|
+
routing,
|
|
186
|
+
snapshotVersion,
|
|
187
|
+
currentSiteId,
|
|
188
|
+
query: async <R extends ArchetypeResultRow = ArchetypeResultRow>(
|
|
189
|
+
q: ArchetypeQuery,
|
|
190
|
+
opts?: { signal?: AbortSignal },
|
|
191
|
+
): Promise<ArchetypeResult<R>> => {
|
|
192
|
+
const inst = bound.value
|
|
193
|
+
if (!inst)
|
|
194
|
+
throw new Error('useGscSnapshotAnalyzer: no site bound')
|
|
195
|
+
return inst.query<R>(q, opts)
|
|
196
|
+
},
|
|
197
|
+
refresh: () => bound.value?.refresh() ?? Promise.resolve(false),
|
|
198
|
+
clearCache: () => bound.value?.clearCache(),
|
|
199
|
+
// Lifecycle owned by the refcounted cache; kept for API compatibility.
|
|
200
|
+
dispose: async () => {},
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
// ── the per-instance implementation ─────────────────────────────────────────
|
|
205
|
+
|
|
206
|
+
function createSnapshotAnalyzerInstance(
|
|
207
|
+
siteId: string,
|
|
208
|
+
searchType: GscSearchType,
|
|
209
|
+
range: AnalyzerRange | null,
|
|
210
|
+
options: GscSnapshotAnalyzerOptions,
|
|
211
|
+
): GscSnapshotAnalyzer {
|
|
212
|
+
const ready = ref(false)
|
|
213
|
+
const initializing = ref(true)
|
|
214
|
+
const error = ref<Error | null>(null)
|
|
215
|
+
const progress = ref<AnalyzerProgress>({ ...IDLE_PROGRESS })
|
|
216
|
+
const storage = ref<AnalyzerStorageState>({ ...IDLE_STORAGE })
|
|
217
|
+
const routing = ref<AnalyzerTableRouting>({})
|
|
218
|
+
const snapshotVersion = ref<string | undefined>(undefined)
|
|
219
|
+
const currentSiteId = ref<string | null>(siteId)
|
|
220
|
+
|
|
221
|
+
const resultLru = createResultLru<ArchetypeResult>(
|
|
222
|
+
options.resultCacheSize ?? DEFAULT_RESULT_CACHE_SIZE,
|
|
223
|
+
)
|
|
224
|
+
|
|
225
|
+
const lifetime = new AbortController()
|
|
226
|
+
|
|
227
|
+
// DuckDB-WASM runtime + OPFS handle — populated by `boot`. Typed loosely
|
|
228
|
+
// because the runtime types come from a dynamically-imported package.
|
|
229
|
+
let db: { conn: unknown, db: unknown } | null = null
|
|
230
|
+
let conn: any = null
|
|
231
|
+
let attachedHandle: { detach: () => Promise<void>, tables: string[] } | null = null
|
|
232
|
+
// The server-tail directive endpoint, when any table routed server-side.
|
|
233
|
+
let serverEndpoint: string | null = null
|
|
234
|
+
|
|
235
|
+
function patchProgress(p: Partial<AnalyzerProgress>): void {
|
|
236
|
+
progress.value = { ...progress.value, ...p }
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
/** Call the file-resolution endpoint. */
|
|
240
|
+
async function resolveFiles(signal: AbortSignal): Promise<FileResolutionResponse> {
|
|
241
|
+
patchProgress({ phase: 'resolving' })
|
|
242
|
+
const qs = new URLSearchParams({ searchType })
|
|
243
|
+
if (range) {
|
|
244
|
+
qs.set('start', range.start)
|
|
245
|
+
qs.set('end', range.end)
|
|
246
|
+
}
|
|
247
|
+
return $fetch<FileResolutionResponse>(`/api/sites/${siteId}/analysis-sources?${qs}`, {
|
|
248
|
+
headers: { 'cache-control': 'no-cache' },
|
|
249
|
+
signal,
|
|
250
|
+
})
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
/** Apply a resolution response to `routing` + return the browser tables. */
|
|
254
|
+
function applyResolution(res: FileResolutionResponse): ResolvedTable[] {
|
|
255
|
+
snapshotVersion.value = res.snapshotVersion
|
|
256
|
+
serverEndpoint = res.serverTail?.endpoint ?? null
|
|
257
|
+
const nextRouting: AnalyzerTableRouting = {}
|
|
258
|
+
for (const t of res.tables)
|
|
259
|
+
nextRouting[t.table] = t.mode
|
|
260
|
+
routing.value = nextRouting
|
|
261
|
+
return res.tables.filter(t => t.mode === 'browser' && t.files.length > 0)
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
const boot = (async () => {
|
|
265
|
+
const signal = lifetime.signal
|
|
266
|
+
patchProgress({ phase: 'resolving', startedAt: Date.now() })
|
|
267
|
+
|
|
268
|
+
const resolution = await resolveFiles(signal)
|
|
269
|
+
const browserTables = applyResolution(resolution)
|
|
270
|
+
|
|
271
|
+
// Fully server-tail-routed site — nothing to attach. Mark ready; queries
|
|
272
|
+
// proxy to the server.
|
|
273
|
+
if (browserTables.length === 0) {
|
|
274
|
+
patchProgress({ phase: 'ready', endedAt: Date.now() })
|
|
275
|
+
ready.value = true
|
|
276
|
+
return
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
// ---- boot DuckDB-WASM -------------------------------------------------
|
|
280
|
+
patchProgress({ phase: 'booting' })
|
|
281
|
+
const cfg = useGscAnalyticsConfig()
|
|
282
|
+
const bundleBase = cfg.duckdbBundleBase as string | undefined
|
|
283
|
+
const { bootDuckDBWasm } = await import('@gscdump/engine-duckdb-wasm')
|
|
284
|
+
db = await bootDuckDBWasm(bundleBase
|
|
285
|
+
? {
|
|
286
|
+
bundles: {
|
|
287
|
+
mvp: { mainModule: `${bundleBase}/duckdb-mvp.wasm`, mainWorker: `${bundleBase}/duckdb-browser-mvp.worker.js` },
|
|
288
|
+
eh: { mainModule: `${bundleBase}/duckdb-eh.wasm`, mainWorker: `${bundleBase}/duckdb-browser-eh.worker.js` },
|
|
289
|
+
},
|
|
290
|
+
}
|
|
291
|
+
: undefined)
|
|
292
|
+
conn = db.conn
|
|
293
|
+
|
|
294
|
+
// ---- download + attach OPFS parquet ----------------------------------
|
|
295
|
+
const { attachOpfsParquetTables, estimateOpfsStorage, requestPersistentStorage } = await import('@gscdump/engine-duckdb-wasm')
|
|
296
|
+
|
|
297
|
+
const persisted = await requestPersistentStorage()
|
|
298
|
+
const est = await estimateOpfsStorage()
|
|
299
|
+
storage.value = { persisted, degraded: false, usageBytes: est.usageBytes, quotaBytes: est.quotaBytes }
|
|
300
|
+
|
|
301
|
+
const opfsTables = browserTables.map(t => ({
|
|
302
|
+
table: t.table,
|
|
303
|
+
files: t.files.map(f => ({
|
|
304
|
+
url: f.url,
|
|
305
|
+
bytes: f.bytes,
|
|
306
|
+
contentHash: f.contentHash,
|
|
307
|
+
rowCount: f.rowCount,
|
|
308
|
+
})),
|
|
309
|
+
}))
|
|
310
|
+
const filesTotal = opfsTables.reduce((n, t) => n + t.files.length, 0)
|
|
311
|
+
const bytesTotal = browserTables.reduce((n, t) => n + t.totalBytes, 0)
|
|
312
|
+
patchProgress({ phase: 'downloading', filesTotal, bytesTotal, filesReady: 0, bytesReady: 0 })
|
|
313
|
+
|
|
314
|
+
let filesReady = 0
|
|
315
|
+
let bytesReady = 0
|
|
316
|
+
const handle = await attachOpfsParquetTables({
|
|
317
|
+
db: db.db as any,
|
|
318
|
+
conn: conn as any,
|
|
319
|
+
tables: opfsTables,
|
|
320
|
+
schema: 'main',
|
|
321
|
+
version: resolution.snapshotVersion,
|
|
322
|
+
fetchInit: { credentials: 'same-origin' },
|
|
323
|
+
fetchConcurrency: DEFAULT_ATTACH_FETCH_CONCURRENCY,
|
|
324
|
+
signal,
|
|
325
|
+
onFileProgress: (info: OpfsFileProgress) => {
|
|
326
|
+
filesReady++
|
|
327
|
+
bytesReady += info.bytes
|
|
328
|
+
patchProgress({ phase: 'downloading', filesReady, bytesReady })
|
|
329
|
+
},
|
|
330
|
+
})
|
|
331
|
+
attachedHandle = { detach: handle.detach, tables: handle.tables }
|
|
332
|
+
|
|
333
|
+
patchProgress({ phase: 'attaching' })
|
|
334
|
+
|
|
335
|
+
// A table that hit a quota error degrades to the server tail. Flip its
|
|
336
|
+
// routing + mark storage degraded so the UI can warn.
|
|
337
|
+
if (handle.degradedTables.length > 0) {
|
|
338
|
+
const next = { ...routing.value }
|
|
339
|
+
for (const t of handle.degradedTables)
|
|
340
|
+
next[t] = 'server'
|
|
341
|
+
routing.value = next
|
|
342
|
+
storage.value = { ...storage.value, degraded: true }
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
const estAfter = await estimateOpfsStorage()
|
|
346
|
+
storage.value = { ...storage.value, usageBytes: estAfter.usageBytes, quotaBytes: estAfter.quotaBytes }
|
|
347
|
+
|
|
348
|
+
patchProgress({ phase: 'ready', endedAt: Date.now(), filesReady, bytesReady })
|
|
349
|
+
ready.value = true
|
|
350
|
+
})()
|
|
351
|
+
.catch((e) => {
|
|
352
|
+
const err = e instanceof Error ? e : new Error(String(e))
|
|
353
|
+
error.value = err
|
|
354
|
+
patchProgress({ phase: 'error', error: err.message, endedAt: Date.now() })
|
|
355
|
+
throw err
|
|
356
|
+
})
|
|
357
|
+
.finally(() => {
|
|
358
|
+
initializing.value = false
|
|
359
|
+
})
|
|
360
|
+
boot.catch(() => {})
|
|
361
|
+
|
|
362
|
+
// ---- query routing -----------------------------------------------------
|
|
363
|
+
|
|
364
|
+
async function runBrowserQuery<R extends ArchetypeResultRow>(
|
|
365
|
+
q: ArchetypeQuery,
|
|
366
|
+
signal?: AbortSignal,
|
|
367
|
+
): Promise<ArchetypeResult<R>> {
|
|
368
|
+
const { compileArchetypeSql } = await import('@gscdump/engine-duckdb-wasm')
|
|
369
|
+
const compiled = compileArchetypeSql(q)
|
|
370
|
+
const t0 = performance.now()
|
|
371
|
+
signal?.throwIfAborted()
|
|
372
|
+
// DuckDB-WASM connection is not concurrency-safe; the shared connection is
|
|
373
|
+
// serialized by callers running queries one-at-a-time here.
|
|
374
|
+
let result: any
|
|
375
|
+
if (compiled.params.length === 0) {
|
|
376
|
+
result = await conn.query(compiled.sql)
|
|
377
|
+
}
|
|
378
|
+
else {
|
|
379
|
+
const stmt = await conn.prepare(compiled.sql)
|
|
380
|
+
try {
|
|
381
|
+
result = await stmt.query(...compiled.params)
|
|
382
|
+
}
|
|
383
|
+
finally {
|
|
384
|
+
await stmt.close()
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
signal?.throwIfAborted()
|
|
388
|
+
const rows = arrowToRows<R>(result)
|
|
389
|
+
return {
|
|
390
|
+
archetype: q.archetype,
|
|
391
|
+
rows,
|
|
392
|
+
source: 'browser',
|
|
393
|
+
meta: { rowCount: rows.length, queryMs: performance.now() - t0 },
|
|
394
|
+
}
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
async function runServerQuery<R extends ArchetypeResultRow>(
|
|
398
|
+
q: ArchetypeQuery,
|
|
399
|
+
where: 'server' | 'cloud',
|
|
400
|
+
signal?: AbortSignal,
|
|
401
|
+
): Promise<ArchetypeResult<R>> {
|
|
402
|
+
const t0 = performance.now()
|
|
403
|
+
// Server tail: POST the archetype to the directive endpoint, or fall back
|
|
404
|
+
// to the conventional per-site analysis endpoint when no directive exists.
|
|
405
|
+
const endpoint = where === 'cloud'
|
|
406
|
+
? `/api/sites/${siteId}/archetype-query`
|
|
407
|
+
: (serverEndpoint ?? `/api/sites/${siteId}/archetype-query`)
|
|
408
|
+
const res = await $fetch<ArchetypeResult<R>>(endpoint, {
|
|
409
|
+
method: 'POST',
|
|
410
|
+
body: { siteId, query: q },
|
|
411
|
+
signal,
|
|
412
|
+
})
|
|
413
|
+
// Trust the server's `source` when present; otherwise tag from routing.
|
|
414
|
+
const source: ArchetypeResultSource = res.source
|
|
415
|
+
?? (where === 'cloud' ? 'cloud' : 'server-r2-sql')
|
|
416
|
+
return {
|
|
417
|
+
...res,
|
|
418
|
+
source,
|
|
419
|
+
meta: { rowCount: res.rows?.length ?? 0, queryMs: performance.now() - t0, ...res.meta },
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
async function query<R extends ArchetypeResultRow = ArchetypeResultRow>(
|
|
424
|
+
q: ArchetypeQuery,
|
|
425
|
+
opts?: { signal?: AbortSignal },
|
|
426
|
+
): Promise<ArchetypeResult<R>> {
|
|
427
|
+
await boot
|
|
428
|
+
opts?.signal?.throwIfAborted()
|
|
429
|
+
|
|
430
|
+
// ---- result LRU — keyed by snapshotVersion + queryHash ----------------
|
|
431
|
+
const cacheKey = resultCacheKey(snapshotVersion.value, q)
|
|
432
|
+
const cached = resultLru.get(cacheKey)
|
|
433
|
+
if (cached)
|
|
434
|
+
return cached as ArchetypeResult<R>
|
|
435
|
+
|
|
436
|
+
const { tableForArchetype } = await import('@gscdump/engine-duckdb-wasm')
|
|
437
|
+
const where = routeArchetype(q, routing.value, tableForArchetype)
|
|
438
|
+
|
|
439
|
+
let result: ArchetypeResult<R>
|
|
440
|
+
if (where === 'browser' && conn) {
|
|
441
|
+
result = await runBrowserQuery<R>(q, opts?.signal)
|
|
442
|
+
}
|
|
443
|
+
else {
|
|
444
|
+
result = await runServerQuery<R>(q, where === 'cloud' ? 'cloud' : 'server', opts?.signal)
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
resultLru.set(cacheKey, result as ArchetypeResult)
|
|
448
|
+
return result
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
async function refresh(): Promise<boolean> {
|
|
452
|
+
await boot.catch(() => {})
|
|
453
|
+
if (!db)
|
|
454
|
+
return false // fully server-tail-routed — nothing to re-attach.
|
|
455
|
+
const resolution = await resolveFiles(lifetime.signal)
|
|
456
|
+
if (resolution.snapshotVersion === snapshotVersion.value)
|
|
457
|
+
return false
|
|
458
|
+
// Snapshot moved — detach the stale views, re-attach the fresher parquet.
|
|
459
|
+
// The DuckDB-WASM runtime (db + conn) stays alive.
|
|
460
|
+
const browserTables = applyResolution(resolution)
|
|
461
|
+
await attachedHandle?.detach().catch((e) => {
|
|
462
|
+
console.warn('[useGscSnapshotAnalyzer] detach during refresh failed', e)
|
|
463
|
+
})
|
|
464
|
+
attachedHandle = null
|
|
465
|
+
resultLru.clear()
|
|
466
|
+
if (browserTables.length === 0)
|
|
467
|
+
return true
|
|
468
|
+
const { attachOpfsParquetTables } = await import('@gscdump/engine-duckdb-wasm')
|
|
469
|
+
const handle = await attachOpfsParquetTables({
|
|
470
|
+
db: (db as any).db,
|
|
471
|
+
conn: conn as any,
|
|
472
|
+
tables: browserTables.map(t => ({
|
|
473
|
+
table: t.table,
|
|
474
|
+
files: t.files.map(f => ({ url: f.url, bytes: f.bytes, contentHash: f.contentHash, rowCount: f.rowCount })),
|
|
475
|
+
})),
|
|
476
|
+
schema: 'main',
|
|
477
|
+
version: resolution.snapshotVersion,
|
|
478
|
+
fetchInit: { credentials: 'same-origin' },
|
|
479
|
+
fetchConcurrency: DEFAULT_ATTACH_FETCH_CONCURRENCY,
|
|
480
|
+
signal: lifetime.signal,
|
|
481
|
+
})
|
|
482
|
+
attachedHandle = { detach: handle.detach, tables: handle.tables }
|
|
483
|
+
if (handle.degradedTables.length > 0) {
|
|
484
|
+
const next = { ...routing.value }
|
|
485
|
+
for (const t of handle.degradedTables)
|
|
486
|
+
next[t] = 'server'
|
|
487
|
+
routing.value = next
|
|
488
|
+
storage.value = { ...storage.value, degraded: true }
|
|
489
|
+
}
|
|
490
|
+
return true
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
function clearCache(): void {
|
|
494
|
+
resultLru.clear()
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
async function dispose(): Promise<void> {
|
|
498
|
+
lifetime.abort()
|
|
499
|
+
await attachedHandle?.detach().catch((e) => {
|
|
500
|
+
console.error('[useGscSnapshotAnalyzer] detach on dispose failed', e)
|
|
501
|
+
})
|
|
502
|
+
if (conn) {
|
|
503
|
+
await conn.close().catch(() => {})
|
|
504
|
+
}
|
|
505
|
+
if (db && (db.db as any)?.terminate) {
|
|
506
|
+
await (db.db as any).terminate().catch(() => {})
|
|
507
|
+
}
|
|
508
|
+
attachedHandle = null
|
|
509
|
+
conn = null
|
|
510
|
+
db = null
|
|
511
|
+
resultLru.clear()
|
|
512
|
+
ready.value = false
|
|
513
|
+
routing.value = {}
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
return {
|
|
517
|
+
ready,
|
|
518
|
+
initializing,
|
|
519
|
+
error,
|
|
520
|
+
progress,
|
|
521
|
+
storage,
|
|
522
|
+
routing,
|
|
523
|
+
snapshotVersion,
|
|
524
|
+
currentSiteId,
|
|
525
|
+
query,
|
|
526
|
+
refresh,
|
|
527
|
+
clearCache,
|
|
528
|
+
dispose,
|
|
529
|
+
}
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
/**
|
|
533
|
+
* Convert an Apache Arrow result (DuckDB-WASM `conn.query` return) to plain
|
|
534
|
+
* row objects. Kept local so the composable has no Arrow type dependency.
|
|
535
|
+
*/
|
|
536
|
+
function arrowToRows<R extends ArchetypeResultRow>(result: unknown): R[] {
|
|
537
|
+
if (!result || typeof result !== 'object')
|
|
538
|
+
return []
|
|
539
|
+
const table = result as { toArray?: () => unknown[] }
|
|
540
|
+
if (typeof table.toArray !== 'function')
|
|
541
|
+
return []
|
|
542
|
+
return table.toArray().map((row) => {
|
|
543
|
+
// Arrow rows expose `toJSON()`; fall back to a shallow copy.
|
|
544
|
+
const r = row as { toJSON?: () => Record<string, unknown> }
|
|
545
|
+
const obj = typeof r.toJSON === 'function' ? r.toJSON() : { ...(row as Record<string, unknown>) }
|
|
546
|
+
// Normalise BigInt (DuckDB SUM returns BigInt) to number for JSON safety.
|
|
547
|
+
for (const k of Object.keys(obj)) {
|
|
548
|
+
const v = obj[k]
|
|
549
|
+
if (typeof v === 'bigint')
|
|
550
|
+
obj[k] = Number(v)
|
|
551
|
+
}
|
|
552
|
+
return obj as R
|
|
553
|
+
})
|
|
554
|
+
}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gscdump/nuxt",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "0.
|
|
4
|
+
"version": "0.21.1",
|
|
5
5
|
"description": "Nuxt layer: GSC analytics UI + DuckDB-WASM integration. Frontend-only; server primitives live in @gscdump/analysis, @gscdump/engine-sqlite, @gscdump/cloudflare, and host apps.",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "Harlan Wilton",
|
|
@@ -55,12 +55,12 @@
|
|
|
55
55
|
"defu": "^6.1.7",
|
|
56
56
|
"ofetch": "^1.5.1",
|
|
57
57
|
"tailwindcss": "^4.3.0",
|
|
58
|
-
"@gscdump/contracts": "0.
|
|
59
|
-
"@gscdump/
|
|
60
|
-
"@gscdump/engine": "0.
|
|
61
|
-
"@gscdump/sdk": "0.
|
|
62
|
-
"@gscdump/
|
|
63
|
-
"gscdump": "0.
|
|
58
|
+
"@gscdump/contracts": "0.21.1",
|
|
59
|
+
"@gscdump/engine": "0.21.1",
|
|
60
|
+
"@gscdump/engine-duckdb-wasm": "0.21.1",
|
|
61
|
+
"@gscdump/sdk": "0.21.1",
|
|
62
|
+
"@gscdump/analysis": "0.21.1",
|
|
63
|
+
"gscdump": "0.21.1"
|
|
64
64
|
},
|
|
65
65
|
"devDependencies": {
|
|
66
66
|
"@nuxt/kit": "^4.4.6",
|