@gscdump/nuxt 0.22.1 → 0.22.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -9
- package/app/components/GscDashboardPage.vue +2 -2
- package/app/components/GscHero.vue +3 -4
- package/app/components/GscLazyTopResult.vue +7 -5
- package/app/components/GscPerformanceChart.vue +4 -4
- package/app/components/GscSourceDebugPanel.vue +2 -1
- package/app/composables/_useGscBackfill.ts +13 -5
- package/app/composables/_useGscResource.ts +144 -56
- package/app/composables/useGscAnalytics.ts +2 -1
- package/app/composables/useGscAnalyticsSourceInfo.ts +11 -5
- package/app/composables/useGscAnalyzer.ts +33 -25
- package/app/composables/useGscAnalyzerDefs.ts +2 -1
- package/app/composables/useGscCountries.ts +5 -6
- package/app/composables/useGscCurrentSite.ts +32 -11
- package/app/composables/useGscInspectionHistory.ts +5 -5
- package/app/composables/useGscInspections.ts +5 -5
- package/app/composables/useGscPanelContext.ts +1 -1
- package/app/composables/useGscQuery.ts +6 -8
- package/app/composables/useGscRequestIndexingInspect.ts +20 -11
- package/app/composables/useGscRollup.ts +66 -81
- package/app/composables/useGscSearchAppearance.ts +5 -6
- package/app/composables/useGscSiteAnalyzer.ts +93 -137
- package/app/composables/useGscSitemapHistory.ts +5 -5
- package/app/composables/useGscSitemaps.ts +5 -5
- package/app/queries/gsc.ts +163 -0
- package/app/utils/gsc-rpc.ts +18 -0
- package/module.ts +1 -1
- package/nuxt.config.ts +20 -0
- package/package.json +139 -8
- package/types.ts +1 -5
- package/app/composables/useGscParquetTable.ts +0 -198
|
@@ -1,36 +1,31 @@
|
|
|
1
|
-
// Per-site DuckDB-WASM analyzer over the Iceberg fact tables.
|
|
1
|
+
// Per-site DuckDB-WASM analyzer over the hosted Iceberg fact tables.
|
|
2
2
|
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
//
|
|
6
|
-
//
|
|
7
|
-
// the data resident in memory and each tab just runs a different SQL query.
|
|
8
|
-
//
|
|
9
|
-
// Tables are attached LAZILY: the first call that needs `queries` triggers
|
|
10
|
-
// the parquet fetch for that table; sibling tabs reuse the attach. This
|
|
11
|
-
// keeps cold-tab latency low and avoids attaching parquet for tables a
|
|
12
|
-
// session never touches (`countries` if the user only looks at Queries).
|
|
13
|
-
//
|
|
14
|
-
// Iceberg fact tables follow the same shape:
|
|
15
|
-
// pages — url, date, clicks, impressions, sum_position
|
|
16
|
-
// queries — query, query_canonical, date, …metrics
|
|
17
|
-
// countries — country, date, …metrics
|
|
18
|
-
// dates — date, …metrics + anonymized_impressions_pct + 9-col device pivot
|
|
19
|
-
// page_queries — url, query, query_canonical, date, …metrics
|
|
20
|
-
// search_appearance — searchAppearance, date, …metrics
|
|
21
|
-
// search_appearance_* — searchAppearance plus page/query context
|
|
3
|
+
// The module owns the browser runtime lifecycle: resolve analysis sources,
|
|
4
|
+
// boot shared DuckDB-WASM, attach parquet tables on demand, mirror progress
|
|
5
|
+
// into the layer-wide map, and materialise Arrow rows into plain objects.
|
|
6
|
+
// Callers keep ownership of SQL, presentation, and page-specific state.
|
|
22
7
|
|
|
23
8
|
import type { AsyncDuckDB, AsyncDuckDBConnection } from '@duckdb/duckdb-wasm'
|
|
24
|
-
import type { AnalysisParams, AnalysisResult } from '@gscdump/analysis'
|
|
25
9
|
import type { FileResolutionResponse } from '@gscdump/contracts'
|
|
26
10
|
import type { OpfsAttachedHandle } from '@gscdump/engine-duckdb-wasm'
|
|
11
|
+
import type { AnalysisParams, AnalysisResult } from '@gscdump/engine/analysis-types'
|
|
27
12
|
import { defaultAnalyzerRegistry } from '@gscdump/analysis'
|
|
28
13
|
import { runAnalyzerFromSource } from '@gscdump/engine/analyzer'
|
|
29
14
|
import { createAttachedTableSource } from '@gscdump/engine/source'
|
|
30
|
-
import {
|
|
31
|
-
import { attachParquetWithFallback } from '../utils/duckdb-wasm'
|
|
32
|
-
|
|
33
|
-
|
|
15
|
+
import { gscQueries } from '../queries/gsc'
|
|
16
|
+
import { attachParquetWithFallback, sharedGscDuckDBWasm } from '../utils/duckdb-wasm'
|
|
17
|
+
import { useGscRpc } from '../utils/gsc-rpc'
|
|
18
|
+
|
|
19
|
+
export type GscFactTable
|
|
20
|
+
= | 'pages'
|
|
21
|
+
| 'queries'
|
|
22
|
+
| 'countries'
|
|
23
|
+
| 'dates'
|
|
24
|
+
| 'page_queries'
|
|
25
|
+
| 'search_appearance'
|
|
26
|
+
| 'search_appearance_pages'
|
|
27
|
+
| 'search_appearance_queries'
|
|
28
|
+
| 'search_appearance_page_queries'
|
|
34
29
|
|
|
35
30
|
export type GscTableStage = 'idle' | 'resolving' | 'downloading' | 'attaching' | 'ready' | 'error' | 'unavailable'
|
|
36
31
|
|
|
@@ -44,49 +39,36 @@ export interface GscTableStatus {
|
|
|
44
39
|
}
|
|
45
40
|
|
|
46
41
|
export interface UseGscSiteAnalyzerReturn {
|
|
47
|
-
/** Per-table stage + file counts. `<GscBootProgress />` reads from the layer-wide map (we mirror there); this is the local view for in-page hints. */
|
|
48
42
|
tables: Readonly<Ref<Record<GscFactTable, GscTableStatus>>>
|
|
49
|
-
/** True once DuckDB-WASM has booted. Queries can dispatch as soon as the table they need is in `'ready'`. */
|
|
50
43
|
ready: Readonly<Ref<boolean>>
|
|
51
|
-
/** First fatal error (boot or any table). Per-table errors land on `tables[<name>].error`. */
|
|
52
44
|
error: Readonly<Ref<Error | null>>
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
/**
|
|
59
|
-
* Positional-form `query` returning `{ rows, queryMs }`. Convenience for
|
|
60
|
-
* callers that want raw SQL access without naming `needs` themselves; the
|
|
61
|
-
* required tables are inferred from `FROM`/`JOIN` references.
|
|
62
|
-
*/
|
|
45
|
+
query: <T = Record<string, unknown>>(opts: {
|
|
46
|
+
sql: string
|
|
47
|
+
needs: readonly GscFactTable[]
|
|
48
|
+
params?: readonly unknown[]
|
|
49
|
+
}) => Promise<T[]>
|
|
63
50
|
runQuery: <T = Record<string, unknown>>(sql: string, params?: readonly unknown[]) => Promise<{ rows: T[], queryMs: number }>
|
|
64
|
-
/**
|
|
65
|
-
* Dispatch an analyzer from `defaultAnalyzerRegistry` against the attached
|
|
66
|
-
* tables. Builds an `attached-table` SQL source that proxies back through
|
|
67
|
-
* `query()`, so the per-site DuckDB-WASM boot and the on-demand attach
|
|
68
|
-
* lifecycle are reused.
|
|
69
|
-
*/
|
|
70
51
|
analyze: (params: AnalysisParams, opts?: { signal?: AbortSignal }) => Promise<AnalysisResult & { queryMs: number }>
|
|
71
52
|
}
|
|
72
53
|
|
|
73
54
|
interface BootedAnalyzer {
|
|
74
|
-
/** Inert proxy used until the real DuckDB-WASM boots. */
|
|
75
55
|
bootPromise: Promise<{ db: AsyncDuckDB, conn: AsyncDuckDBConnection }>
|
|
76
|
-
/** Latched once attach has been started for a table; resolves when the table is queryable. */
|
|
77
56
|
tablePromises: Map<GscFactTable, Promise<void>>
|
|
78
|
-
/** Reactive per-table status (mirrored to the layer-wide progress map). */
|
|
79
57
|
tables: Ref<Record<GscFactTable, GscTableStatus>>
|
|
80
58
|
ready: Ref<boolean>
|
|
81
59
|
error: Ref<Error | null>
|
|
82
|
-
/** Held mutex around DuckDB DDL/query calls — WASM DuckDB is single-connection. */
|
|
83
60
|
withDb: <T>(fn: (conn: AsyncDuckDBConnection) => Promise<T>) => Promise<T>
|
|
84
|
-
/** Per-instance refcount — bumped on every consumer hookup, decremented on dispose. */
|
|
85
61
|
refs: number
|
|
86
|
-
/** Cleanup; called when refs hits 0. */
|
|
87
62
|
dispose: () => Promise<void>
|
|
88
63
|
}
|
|
89
64
|
|
|
65
|
+
interface AnalyzerArgs {
|
|
66
|
+
siteId: string
|
|
67
|
+
searchType: string
|
|
68
|
+
range: { start: string, end: string }
|
|
69
|
+
useOpfsCache: boolean
|
|
70
|
+
}
|
|
71
|
+
|
|
90
72
|
const ALL_TABLES: readonly GscFactTable[] = [
|
|
91
73
|
'pages',
|
|
92
74
|
'queries',
|
|
@@ -101,6 +83,7 @@ const ALL_TABLES: readonly GscFactTable[] = [
|
|
|
101
83
|
|
|
102
84
|
const FACT_TABLE_NAMES = ALL_TABLES.join('|')
|
|
103
85
|
const TABLE_RE = new RegExp(`\\b(?:FROM|JOIN)\\s+(?:main\\.)?(${FACT_TABLE_NAMES})\\b`, 'gi')
|
|
86
|
+
const instanceCaches = new WeakMap<object, Map<string, BootedAnalyzer>>()
|
|
104
87
|
|
|
105
88
|
function tablesFromSql(sql: string): readonly GscFactTable[] {
|
|
106
89
|
const out = new Set<GscFactTable>()
|
|
@@ -121,11 +104,6 @@ function emptyTables(): Record<GscFactTable, GscTableStatus> {
|
|
|
121
104
|
return Object.fromEntries(ALL_TABLES.map(t => [t, emptyStatus()])) as Record<GscFactTable, GscTableStatus>
|
|
122
105
|
}
|
|
123
106
|
|
|
124
|
-
// Per-NuxtApp instance cache keyed by `siteId|searchType|start|end`. WeakMap
|
|
125
|
-
// against the NuxtApp so SSR isolation holds (each request gets its own cache)
|
|
126
|
-
// and client-side instances live as long as the app does.
|
|
127
|
-
const instanceCaches = new WeakMap<object, Map<string, BootedAnalyzer>>()
|
|
128
|
-
|
|
129
107
|
function getCache(): Map<string, BootedAnalyzer> {
|
|
130
108
|
const app = useNuxtApp()
|
|
131
109
|
let bag = instanceCaches.get(app)
|
|
@@ -143,46 +121,47 @@ function instanceKey(siteId: string, searchType: string, range: { start: string,
|
|
|
143
121
|
export function useGscSiteAnalyzer(
|
|
144
122
|
siteId: MaybeRefOrGetter<string | null | undefined>,
|
|
145
123
|
range: MaybeRefOrGetter<{ start: string, end: string } | null | undefined>,
|
|
146
|
-
options: {
|
|
124
|
+
options: {
|
|
125
|
+
searchType?: 'web' | 'image' | 'video' | 'news' | 'discover' | 'googleNews'
|
|
126
|
+
useOpfsCache?: boolean
|
|
127
|
+
} = {},
|
|
147
128
|
): UseGscSiteAnalyzerReturn {
|
|
148
129
|
const searchType = options.searchType ?? 'web'
|
|
149
130
|
const useOpfsCache = options.useOpfsCache ?? true
|
|
150
|
-
|
|
151
|
-
// The reactive return shape — proxies the currently-selected instance.
|
|
152
|
-
// We rebind every time the key changes (e.g. range moves outside the
|
|
153
|
-
// cached attach window).
|
|
154
131
|
const tables = ref<Record<GscFactTable, GscTableStatus>>(emptyTables())
|
|
155
132
|
const ready = ref(false)
|
|
156
133
|
const error = ref<Error | null>(null)
|
|
157
134
|
let active: BootedAnalyzer | null = null
|
|
135
|
+
let stopEffects: (() => void) | null = null
|
|
158
136
|
|
|
159
137
|
function bind(instance: BootedAnalyzer | null): void {
|
|
160
|
-
// Release the previous instance's refcount; pick up the new one. Skipping
|
|
161
|
-
// shutdown on rebind means tab navigation never tears DuckDB down — the
|
|
162
|
-
// instance lives until every consumer unmounts and the cache drops it.
|
|
163
138
|
const prev = active
|
|
139
|
+
stopEffects?.()
|
|
140
|
+
stopEffects = null
|
|
164
141
|
active = instance
|
|
142
|
+
|
|
165
143
|
if (instance) {
|
|
166
144
|
instance.refs++
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
error.value = instance.error.value
|
|
145
|
+
const scope = effectScope()
|
|
146
|
+
scope.run(() => {
|
|
147
|
+
watchEffect(() => {
|
|
148
|
+
tables.value = instance.tables.value
|
|
149
|
+
})
|
|
150
|
+
watchEffect(() => {
|
|
151
|
+
ready.value = instance.ready.value
|
|
152
|
+
})
|
|
153
|
+
watchEffect(() => {
|
|
154
|
+
error.value = instance.error.value
|
|
155
|
+
})
|
|
179
156
|
})
|
|
157
|
+
stopEffects = () => scope.stop()
|
|
180
158
|
}
|
|
181
159
|
else {
|
|
182
160
|
tables.value = emptyTables()
|
|
183
161
|
ready.value = false
|
|
184
162
|
error.value = null
|
|
185
163
|
}
|
|
164
|
+
|
|
186
165
|
if (prev) {
|
|
187
166
|
prev.refs--
|
|
188
167
|
if (prev.refs <= 0)
|
|
@@ -219,12 +198,15 @@ export function useGscSiteAnalyzer(
|
|
|
219
198
|
|
|
220
199
|
onScopeDispose(() => bind(null))
|
|
221
200
|
|
|
222
|
-
async function query<T = Record<string, unknown>>(opts: {
|
|
201
|
+
async function query<T = Record<string, unknown>>(opts: {
|
|
202
|
+
sql: string
|
|
203
|
+
needs: readonly GscFactTable[]
|
|
204
|
+
params?: readonly unknown[]
|
|
205
|
+
}): Promise<T[]> {
|
|
223
206
|
if (!active)
|
|
224
207
|
throw new Error('useGscSiteAnalyzer: no active instance (site/range not set)')
|
|
225
208
|
const instance = active
|
|
226
209
|
await instance.bootPromise
|
|
227
|
-
// Kick off (or join) per-table attaches for every table the query needs.
|
|
228
210
|
await Promise.all(opts.needs.map(t => ensureTable(instance, t)))
|
|
229
211
|
return instance.withDb(async (conn) => {
|
|
230
212
|
const result = opts.params && opts.params.length > 0
|
|
@@ -238,11 +220,6 @@ export function useGscSiteAnalyzer(
|
|
|
238
220
|
}
|
|
239
221
|
})()
|
|
240
222
|
: await conn.query(opts.sql)
|
|
241
|
-
// Apache Arrow returns `Proxy(StructRow)` rows — Vue's reactivity
|
|
242
|
-
// can't make those reactive (the Arrow proxy's `isExtensible` trap
|
|
243
|
-
// throws a TypeError when the reactive proxy probes it). Materialise
|
|
244
|
-
// into plain objects at the seam so every consumer is free to assign
|
|
245
|
-
// results directly into a `ref`/`reactive` without surprise.
|
|
246
223
|
const arr = result.toArray() as unknown as Array<Record<string, unknown>>
|
|
247
224
|
return arr.map(row => ({ ...row })) as unknown as T[]
|
|
248
225
|
})
|
|
@@ -286,15 +263,6 @@ export function useGscSiteAnalyzer(
|
|
|
286
263
|
}
|
|
287
264
|
}
|
|
288
265
|
|
|
289
|
-
// ── implementation ──────────────────────────────────────────────────────────
|
|
290
|
-
|
|
291
|
-
interface AnalyzerArgs {
|
|
292
|
-
siteId: string
|
|
293
|
-
searchType: string
|
|
294
|
-
range: { start: string, end: string }
|
|
295
|
-
useOpfsCache: boolean
|
|
296
|
-
}
|
|
297
|
-
|
|
298
266
|
function createAnalyzer(args: AnalyzerArgs): BootedAnalyzer {
|
|
299
267
|
const tables = ref<Record<GscFactTable, GscTableStatus>>(emptyTables())
|
|
300
268
|
const ready = ref(false)
|
|
@@ -302,38 +270,32 @@ function createAnalyzer(args: AnalyzerArgs): BootedAnalyzer {
|
|
|
302
270
|
const tablePromises = new Map<GscFactTable, Promise<void>>()
|
|
303
271
|
const opfsHandles = new Map<GscFactTable, OpfsAttachedHandle>()
|
|
304
272
|
|
|
305
|
-
|
|
306
|
-
// need a NuxtApp context, which is available here (the composable's
|
|
307
|
-
// setup) but not inside the async work spawned below.
|
|
308
|
-
const $gscFetch = useGscFetch()
|
|
273
|
+
const rpc = useGscRpc()
|
|
309
274
|
const analyticsConfig = useGscAnalyticsConfig()
|
|
310
275
|
const analyticsCtx = useGscAnalyticsContext()
|
|
311
|
-
const apiBase = (analyticsConfig.apiBase ?? '').replace(/\/+$/, '')
|
|
312
276
|
|
|
313
|
-
// Single resolve call — `analysis-sources` already returns all 5 tables
|
|
314
|
-
// for a given site+range, so we hit it once and demux per-table later.
|
|
315
277
|
let resolvePromise: Promise<FileResolutionResponse | null> | null = null
|
|
316
278
|
function getResolution(): Promise<FileResolutionResponse | null> {
|
|
317
279
|
if (resolvePromise)
|
|
318
280
|
return resolvePromise
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
281
|
+
resolvePromise = rpc.query(
|
|
282
|
+
gscQueries.analysisSources(args.siteId, undefined, {
|
|
283
|
+
searchType: args.searchType as 'web' | 'image' | 'video' | 'news' | 'discover' | 'googleNews',
|
|
284
|
+
start: args.range.start,
|
|
285
|
+
end: args.range.end,
|
|
286
|
+
}),
|
|
287
|
+
{ silent: true },
|
|
322
288
|
)
|
|
323
289
|
.then((res) => {
|
|
324
290
|
if (!res || (res as { canUseBrowser?: boolean }).canUseBrowser === false)
|
|
325
291
|
return null
|
|
326
|
-
return res as FileResolutionResponse
|
|
292
|
+
return res as unknown as FileResolutionResponse
|
|
327
293
|
})
|
|
328
294
|
.catch(() => null)
|
|
329
295
|
return resolvePromise
|
|
330
296
|
}
|
|
331
297
|
|
|
332
|
-
// Boot DuckDB-WASM up front — the resolve and the WASM bundle download in
|
|
333
|
-
// parallel, so by the time a query asks for a table, the DB is ready.
|
|
334
298
|
const bootPromise: Promise<{ db: AsyncDuckDB, conn: AsyncDuckDBConnection }> = (async () => {
|
|
335
|
-
// Shared boot: navigating home → site detail (or between sites) reuses
|
|
336
|
-
// the same DuckDB-WASM instance instead of paying ~800ms cold-boot again.
|
|
337
299
|
const bundleBase = (analyticsConfig as { duckdbBundleBase?: string }).duckdbBundleBase
|
|
338
300
|
const boot = await sharedGscDuckDBWasm(bundleBase)
|
|
339
301
|
ready.value = true
|
|
@@ -343,8 +305,6 @@ function createAnalyzer(args: AnalyzerArgs): BootedAnalyzer {
|
|
|
343
305
|
error.value = err instanceof Error ? err : new Error(String(err))
|
|
344
306
|
})
|
|
345
307
|
|
|
346
|
-
// DuckDB-WASM single-connection mutex — DDL and queries serialise; downloads
|
|
347
|
-
// run free in parallel outside the lock.
|
|
348
308
|
let dbLock: Promise<unknown> = Promise.resolve()
|
|
349
309
|
function withDb<T>(fn: (conn: AsyncDuckDBConnection) => Promise<T>): Promise<T> {
|
|
350
310
|
const next = dbLock.then(async () => {
|
|
@@ -355,20 +315,23 @@ function createAnalyzer(args: AnalyzerArgs): BootedAnalyzer {
|
|
|
355
315
|
return next
|
|
356
316
|
}
|
|
357
317
|
|
|
358
|
-
// Per-table attach kicker. Stores its promise so concurrent callers join
|
|
359
|
-
// the same in-flight work; the layer-wide `patchProgress` map is updated
|
|
360
|
-
// as each phase advances so `<GscBootProgress />` shows the site's table
|
|
361
|
-
// load alongside other sites.
|
|
362
318
|
function patchSelf(table: GscFactTable, patch: Partial<GscTableStatus>): void {
|
|
363
319
|
tables.value = { ...tables.value, [table]: { ...tables.value[table], ...patch } }
|
|
364
320
|
}
|
|
321
|
+
|
|
365
322
|
function patchLayer(table: GscFactTable, stage: GscTableStage, extras: Partial<GscTableStatus> = {}): void {
|
|
366
|
-
// Reuse the layer's SiteLoadProgress shape, namespacing by table so
|
|
367
|
-
// multiple tables for the same site don't collide on the boot bar.
|
|
368
323
|
const siteSlot = `${args.siteId}#${table}`
|
|
369
|
-
const layerStage = stage === 'unavailable'
|
|
324
|
+
const layerStage = stage === 'unavailable'
|
|
325
|
+
? 'ready'
|
|
326
|
+
: stage === 'downloading' || stage === 'attaching'
|
|
327
|
+
? 'attach'
|
|
328
|
+
: stage === 'resolving'
|
|
329
|
+
? 'manifest'
|
|
330
|
+
: stage === 'ready'
|
|
331
|
+
? 'ready'
|
|
332
|
+
: stage === 'error' ? 'error' : 'idle'
|
|
370
333
|
analyticsCtx.patchProgress(siteSlot, {
|
|
371
|
-
stage: layerStage
|
|
334
|
+
stage: layerStage,
|
|
372
335
|
source: 'duckdb',
|
|
373
336
|
filesAttached: extras.filesAttached ?? tables.value[table].filesAttached,
|
|
374
337
|
filesTotal: extras.filesTotal ?? tables.value[table].filesTotal,
|
|
@@ -385,15 +348,18 @@ function createAnalyzer(args: AnalyzerArgs): BootedAnalyzer {
|
|
|
385
348
|
|
|
386
349
|
const full = await getResolution()
|
|
387
350
|
if (!full) {
|
|
388
|
-
|
|
389
|
-
|
|
351
|
+
const endedAt = Date.now()
|
|
352
|
+
patchSelf(table, { stage: 'unavailable', endedAt })
|
|
353
|
+
patchLayer(table, 'unavailable', { endedAt })
|
|
390
354
|
return
|
|
391
355
|
}
|
|
356
|
+
|
|
392
357
|
const tableEntry = full.tables.find(t => t.table === table)
|
|
393
358
|
const browserFiles = tableEntry?.mode === 'browser' ? tableEntry.files : []
|
|
394
359
|
if (browserFiles.length === 0) {
|
|
395
|
-
|
|
396
|
-
|
|
360
|
+
const endedAt = Date.now()
|
|
361
|
+
patchSelf(table, { stage: 'unavailable', endedAt })
|
|
362
|
+
patchLayer(table, 'unavailable', { endedAt })
|
|
397
363
|
return
|
|
398
364
|
}
|
|
399
365
|
|
|
@@ -402,10 +368,7 @@ function createAnalyzer(args: AnalyzerArgs): BootedAnalyzer {
|
|
|
402
368
|
|
|
403
369
|
const sid = sqlIdent(args.siteId)
|
|
404
370
|
const tableIdent = sqlIdent(table)
|
|
405
|
-
// Per-(site,table) view name keeps OPFS file names unique across sites
|
|
406
|
-
// sharing this DuckDB instance.
|
|
407
371
|
const viewName = `${tableIdent}_${sid}`
|
|
408
|
-
|
|
409
372
|
let filesAttached = 0
|
|
410
373
|
const { db, conn } = await bootPromise
|
|
411
374
|
const opfsHandle = await attachParquetWithFallback({
|
|
@@ -423,14 +386,12 @@ function createAnalyzer(args: AnalyzerArgs): BootedAnalyzer {
|
|
|
423
386
|
},
|
|
424
387
|
}).catch((err) => {
|
|
425
388
|
const msg = err instanceof Error ? err.message : String(err)
|
|
426
|
-
|
|
427
|
-
|
|
389
|
+
const endedAt = Date.now()
|
|
390
|
+
patchSelf(table, { stage: 'error', error: msg, endedAt })
|
|
391
|
+
patchLayer(table, 'error', { error: msg, endedAt })
|
|
428
392
|
throw err
|
|
429
393
|
})
|
|
430
394
|
|
|
431
|
-
// Query SQL references the canonical table name (`pages`, etc.); the
|
|
432
|
-
// underlying view is per-(site,table) so OPFS file names don't collide
|
|
433
|
-
// across sites that may share a DuckDB instance in future.
|
|
434
395
|
if (viewName !== tableIdent) {
|
|
435
396
|
await withDb(async (conn) => {
|
|
436
397
|
await conn.query(`CREATE OR REPLACE VIEW ${tableIdent} AS SELECT * FROM ${viewName}`)
|
|
@@ -440,8 +401,9 @@ function createAnalyzer(args: AnalyzerArgs): BootedAnalyzer {
|
|
|
440
401
|
if (opfsHandle)
|
|
441
402
|
opfsHandles.set(table, opfsHandle)
|
|
442
403
|
|
|
443
|
-
|
|
444
|
-
|
|
404
|
+
const endedAt = Date.now()
|
|
405
|
+
patchSelf(table, { stage: 'ready', endedAt })
|
|
406
|
+
patchLayer(table, 'ready', { endedAt })
|
|
445
407
|
}
|
|
446
408
|
|
|
447
409
|
function ensureTableInner(table: GscFactTable): Promise<void> {
|
|
@@ -462,8 +424,6 @@ function createAnalyzer(args: AnalyzerArgs): BootedAnalyzer {
|
|
|
462
424
|
withDb,
|
|
463
425
|
refs: 0,
|
|
464
426
|
dispose: async () => {
|
|
465
|
-
// Pull the cache entry first so subsequent obtains create a fresh one
|
|
466
|
-
// (we can't dispose mid-attach without leaking promises).
|
|
467
427
|
const cache = instanceCaches.get(useNuxtApp())
|
|
468
428
|
if (cache) {
|
|
469
429
|
for (const [k, v] of cache.entries()) {
|
|
@@ -477,19 +437,15 @@ function createAnalyzer(args: AnalyzerArgs): BootedAnalyzer {
|
|
|
477
437
|
for (const h of opfsHandles.values())
|
|
478
438
|
await h.detach().catch(() => {})
|
|
479
439
|
opfsHandles.clear()
|
|
480
|
-
// Drop the per-site views we created so a re-attach is clean. Do NOT
|
|
481
|
-
// terminate the DB — it's shared via `sharedGscDuckDBWasm`.
|
|
482
440
|
const { conn } = await bootPromise
|
|
483
441
|
for (const t of ALL_TABLES)
|
|
484
442
|
await conn.query(`DROP VIEW IF EXISTS ${sqlIdent(t)}`).catch(() => {})
|
|
485
443
|
}
|
|
486
444
|
catch {
|
|
487
|
-
// Boot failed
|
|
445
|
+
// Boot failed; nothing to tear down.
|
|
488
446
|
}
|
|
489
447
|
},
|
|
490
448
|
}
|
|
491
|
-
// Attach the ensureTable closure to the instance so the outer composable
|
|
492
|
-
// can reach it without a globalThis hack.
|
|
493
449
|
;(instance as BootedAnalyzer & { ensureTable: typeof ensureTableInner }).ensureTable = ensureTableInner
|
|
494
450
|
return instance
|
|
495
451
|
}
|
|
@@ -4,16 +4,16 @@
|
|
|
4
4
|
import type { SitemapHistoryRecord, SitemapHistoryResponse } from '@gscdump/contracts'
|
|
5
5
|
import type { ComputedRef, Ref } from '@vue/runtime-core'
|
|
6
6
|
import type { GscResourceStatus } from './_useGscResource'
|
|
7
|
+
import { gscQueries } from '../queries/gsc'
|
|
7
8
|
import { useGscResource } from './_useGscResource'
|
|
8
|
-
import { useGscAnalyticsClient } from './useGscAnalyticsClient'
|
|
9
9
|
|
|
10
10
|
export interface UseGscSitemapHistoryReturn {
|
|
11
11
|
response: Readonly<Ref<SitemapHistoryResponse | null>>
|
|
12
12
|
snapshots: ComputedRef<SitemapHistoryRecord[]>
|
|
13
13
|
path: ComputedRef<string | null>
|
|
14
14
|
loading: ComputedRef<boolean>
|
|
15
|
-
status: Ref<GscResourceStatus
|
|
16
|
-
error: Ref<Error | null
|
|
15
|
+
status: Readonly<Ref<GscResourceStatus>>
|
|
16
|
+
error: Readonly<Ref<Error | null>>
|
|
17
17
|
refresh: () => Promise<void>
|
|
18
18
|
}
|
|
19
19
|
|
|
@@ -21,9 +21,9 @@ export function useGscSitemapHistory(
|
|
|
21
21
|
siteId: MaybeRefOrGetter<string | null | undefined>,
|
|
22
22
|
feedpathHash: MaybeRefOrGetter<string | null | undefined>,
|
|
23
23
|
): UseGscSitemapHistoryReturn {
|
|
24
|
-
const { data, status, loading, error, refresh } = useGscResource({
|
|
24
|
+
const { data, status, loading, error, refresh } = useGscResource<[string, string], SitemapHistoryResponse>({
|
|
25
25
|
keys: [siteId, feedpathHash] as const,
|
|
26
|
-
|
|
26
|
+
operation: (id, hash) => gscQueries.sitemapHistory(id, hash),
|
|
27
27
|
})
|
|
28
28
|
|
|
29
29
|
const snapshots = computed(() => data.value?.snapshots ?? [])
|
|
@@ -5,22 +5,22 @@
|
|
|
5
5
|
import type { SitemapHistoryRecord, SitemapIndex } from '@gscdump/contracts'
|
|
6
6
|
import type { ComputedRef, Ref } from '@vue/runtime-core'
|
|
7
7
|
import type { GscResourceStatus } from './_useGscResource'
|
|
8
|
+
import { gscQueries } from '../queries/gsc'
|
|
8
9
|
import { useGscResource } from './_useGscResource'
|
|
9
|
-
import { useGscAnalyticsClient } from './useGscAnalyticsClient'
|
|
10
10
|
|
|
11
11
|
export interface UseGscSitemapsReturn {
|
|
12
12
|
index: Readonly<Ref<SitemapIndex | null>>
|
|
13
13
|
records: ComputedRef<SitemapHistoryRecord[]>
|
|
14
14
|
loading: ComputedRef<boolean>
|
|
15
|
-
status: Ref<GscResourceStatus
|
|
16
|
-
error: Ref<Error | null
|
|
15
|
+
status: Readonly<Ref<GscResourceStatus>>
|
|
16
|
+
error: Readonly<Ref<Error | null>>
|
|
17
17
|
refresh: () => Promise<void>
|
|
18
18
|
}
|
|
19
19
|
|
|
20
20
|
export function useGscSitemaps(siteId: MaybeRefOrGetter<string | null | undefined>): UseGscSitemapsReturn {
|
|
21
|
-
const { data, status, loading, error, refresh } = useGscResource({
|
|
21
|
+
const { data, status, loading, error, refresh } = useGscResource<[string], SitemapIndex>({
|
|
22
22
|
keys: [siteId] as const,
|
|
23
|
-
|
|
23
|
+
operation: (id: string) => gscQueries.sitemaps(id),
|
|
24
24
|
})
|
|
25
25
|
|
|
26
26
|
const records = computed<SitemapHistoryRecord[]>(() => {
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
import type { AnalysisSourcesOptions, BackfillRange, GscApiRange, GscdumpTopAssociationParams, IndexingInspectRequest, SourceInfoOptions } from '@gscdump/contracts'
|
|
2
|
+
import type { AnalysisParams } from '@gscdump/engine/analysis-types'
|
|
3
|
+
import { analyticsRoutes, gscdumpAnalysisParamsSchema, gscdumpAnalysisResponseSchema, partnerEndpointSchemas } from '@gscdump/contracts'
|
|
4
|
+
import { defineNuxtQueryGroup, defineNuxtRpcMutation, defineNuxtRpcQuery } from 'nuxt-use-query/rpc'
|
|
5
|
+
|
|
6
|
+
const DEFAULT_SEARCH_TYPE = 'web'
|
|
7
|
+
|
|
8
|
+
type SearchType = NonNullable<SourceInfoOptions['searchType']>
|
|
9
|
+
|
|
10
|
+
function searchTypeQuery(searchType?: SearchType): Record<string, string> {
|
|
11
|
+
return { searchType: searchType ?? DEFAULT_SEARCH_TYPE }
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
function dateRangeQuery(options: { start?: string, end?: string, startDate?: string, endDate?: string } | undefined): Record<string, string> {
|
|
15
|
+
const query: Record<string, string> = {}
|
|
16
|
+
const start = options?.start ?? options?.startDate
|
|
17
|
+
const end = options?.end ?? options?.endDate
|
|
18
|
+
if (start)
|
|
19
|
+
query.start = start
|
|
20
|
+
if (end)
|
|
21
|
+
query.end = end
|
|
22
|
+
return query
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function sourceInfoQuery(options: SourceInfoOptions | undefined): Record<string, string> {
|
|
26
|
+
return {
|
|
27
|
+
...searchTypeQuery(options?.searchType),
|
|
28
|
+
...dateRangeQuery(options),
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function tablesQuery(
|
|
33
|
+
tablesOrOptions: string[] | string | AnalysisSourcesOptions | undefined,
|
|
34
|
+
options?: { searchType?: SearchType, start?: string, end?: string, startDate?: string, endDate?: string },
|
|
35
|
+
): Record<string, string> {
|
|
36
|
+
const isOptions = !!tablesOrOptions && typeof tablesOrOptions === 'object' && !Array.isArray(tablesOrOptions)
|
|
37
|
+
const source = isOptions ? tablesOrOptions : options
|
|
38
|
+
const tables = isOptions ? tablesOrOptions.tables : tablesOrOptions
|
|
39
|
+
const query = {
|
|
40
|
+
...searchTypeQuery(source?.searchType),
|
|
41
|
+
...dateRangeQuery(source),
|
|
42
|
+
}
|
|
43
|
+
if (tables)
|
|
44
|
+
query.tables = Array.isArray(tables) ? tables.join(',') : tables
|
|
45
|
+
return query
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function withDefaultSearchType<T>(value: T): T {
|
|
49
|
+
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
50
|
+
return value
|
|
51
|
+
return {
|
|
52
|
+
...(value as Record<string, unknown>),
|
|
53
|
+
searchType: (value as { searchType?: SearchType }).searchType ?? DEFAULT_SEARCH_TYPE,
|
|
54
|
+
} as T
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export const gscQueries = defineNuxtQueryGroup('gsc', {
|
|
58
|
+
whoami: () => defineNuxtRpcQuery({
|
|
59
|
+
key: ['gsc', 'whoami'],
|
|
60
|
+
path: analyticsRoutes.whoami,
|
|
61
|
+
response: partnerEndpointSchemas.analyticsWhoami.response,
|
|
62
|
+
}),
|
|
63
|
+
sites: () => defineNuxtRpcQuery({
|
|
64
|
+
key: ['gsc', 'sites'],
|
|
65
|
+
path: analyticsRoutes.sites,
|
|
66
|
+
response: partnerEndpointSchemas.analyticsSites.response,
|
|
67
|
+
}),
|
|
68
|
+
sourceInfo: (siteId: string, options?: SourceInfoOptions) => defineNuxtRpcQuery({
|
|
69
|
+
key: ['gsc', 'source-info', siteId, options?.searchType ?? DEFAULT_SEARCH_TYPE, options?.start ?? options?.startDate ?? '', options?.end ?? options?.endDate ?? ''],
|
|
70
|
+
path: analyticsRoutes.site.sourceInfo(siteId),
|
|
71
|
+
query: sourceInfoQuery(options),
|
|
72
|
+
response: partnerEndpointSchemas.analyticsSourceInfo.response,
|
|
73
|
+
}),
|
|
74
|
+
analysisSources: (
|
|
75
|
+
siteId: string,
|
|
76
|
+
tables?: string[] | string | AnalysisSourcesOptions,
|
|
77
|
+
options?: { searchType?: SearchType, start?: string, end?: string, startDate?: string, endDate?: string },
|
|
78
|
+
) => {
|
|
79
|
+
const source = tables && typeof tables === 'object' && !Array.isArray(tables) ? tables : options
|
|
80
|
+
const tableList = tables && typeof tables === 'object' && !Array.isArray(tables) ? tables.tables : tables
|
|
81
|
+
const tablesCsv = Array.isArray(tableList) ? tableList.join(',') : (tableList ?? '')
|
|
82
|
+
return defineNuxtRpcQuery({
|
|
83
|
+
key: ['gsc', 'analysis-sources', siteId, source?.searchType ?? DEFAULT_SEARCH_TYPE, source?.start ?? source?.startDate ?? '', source?.end ?? source?.endDate ?? '', tablesCsv],
|
|
84
|
+
path: analyticsRoutes.site.analysisSources(siteId),
|
|
85
|
+
query: tablesQuery(tables, options),
|
|
86
|
+
response: partnerEndpointSchemas.analyticsAnalysisSources.response,
|
|
87
|
+
})
|
|
88
|
+
},
|
|
89
|
+
analyze: (siteId: string) => defineNuxtRpcMutation({
|
|
90
|
+
body: gscdumpAnalysisParamsSchema,
|
|
91
|
+
method: 'POST',
|
|
92
|
+
path: analyticsRoutes.site.analyze(siteId),
|
|
93
|
+
response: gscdumpAnalysisResponseSchema,
|
|
94
|
+
}),
|
|
95
|
+
backfill: (siteId: string) => defineNuxtRpcMutation({
|
|
96
|
+
body: partnerEndpointSchemas.analyticsBackfill.body,
|
|
97
|
+
method: 'POST',
|
|
98
|
+
path: analyticsRoutes.site.backfill(siteId),
|
|
99
|
+
response: partnerEndpointSchemas.analyticsBackfill.response,
|
|
100
|
+
}),
|
|
101
|
+
sitemaps: (siteId: string) => defineNuxtRpcQuery({
|
|
102
|
+
key: ['gsc', 'sitemaps', siteId],
|
|
103
|
+
path: analyticsRoutes.site.sitemaps(siteId),
|
|
104
|
+
response: partnerEndpointSchemas.analyticsSitemaps.response,
|
|
105
|
+
}),
|
|
106
|
+
sitemapHistory: (siteId: string, hash: string) => defineNuxtRpcQuery({
|
|
107
|
+
key: ['gsc', 'sitemap-history', siteId, hash],
|
|
108
|
+
path: analyticsRoutes.site.sitemapHistory(siteId, hash),
|
|
109
|
+
response: partnerEndpointSchemas.analyticsSitemapHistory.response,
|
|
110
|
+
}),
|
|
111
|
+
inspections: (siteId: string) => defineNuxtRpcQuery({
|
|
112
|
+
key: ['gsc', 'inspections', siteId],
|
|
113
|
+
path: analyticsRoutes.site.inspections(siteId),
|
|
114
|
+
response: partnerEndpointSchemas.analyticsInspections.response,
|
|
115
|
+
}),
|
|
116
|
+
inspectionHistory: (siteId: string, hash: string) => defineNuxtRpcQuery({
|
|
117
|
+
key: ['gsc', 'inspection-history', siteId, hash],
|
|
118
|
+
path: analyticsRoutes.site.inspectionHistory(siteId, hash),
|
|
119
|
+
response: partnerEndpointSchemas.analyticsInspectionHistory.response,
|
|
120
|
+
}),
|
|
121
|
+
indexingInspect: (siteId: string) => defineNuxtRpcMutation({
|
|
122
|
+
body: partnerEndpointSchemas.analyticsIndexingInspect.body,
|
|
123
|
+
method: 'POST',
|
|
124
|
+
path: analyticsRoutes.site.indexingInspect(siteId),
|
|
125
|
+
response: partnerEndpointSchemas.analyticsIndexingInspect.response,
|
|
126
|
+
}),
|
|
127
|
+
countries: (siteId: string, range: GscApiRange) => defineNuxtRpcQuery({
|
|
128
|
+
key: ['gsc', 'countries', siteId, range.start, range.end],
|
|
129
|
+
path: analyticsRoutes.site.countries(siteId),
|
|
130
|
+
query: range,
|
|
131
|
+
response: partnerEndpointSchemas.analyticsCountries.response,
|
|
132
|
+
}),
|
|
133
|
+
searchAppearance: (siteId: string, range: GscApiRange) => defineNuxtRpcQuery({
|
|
134
|
+
key: ['gsc', 'search-appearance', siteId, range.start, range.end],
|
|
135
|
+
path: analyticsRoutes.site.searchAppearance(siteId),
|
|
136
|
+
query: range,
|
|
137
|
+
response: partnerEndpointSchemas.analyticsSearchAppearance.response,
|
|
138
|
+
}),
|
|
139
|
+
rollup: (siteId: string, rollupId: string, range?: Partial<GscApiRange>) => defineNuxtRpcQuery({
|
|
140
|
+
key: ['gsc', 'rollup', siteId, rollupId, range?.start ?? '', range?.end ?? ''],
|
|
141
|
+
path: analyticsRoutes.site.rollup(siteId, rollupId),
|
|
142
|
+
query: range,
|
|
143
|
+
response: partnerEndpointSchemas.analyticsRollup.response,
|
|
144
|
+
}),
|
|
145
|
+
topAssociation: (siteId: string, query: GscdumpTopAssociationParams) => defineNuxtRpcQuery({
|
|
146
|
+
key: ['gsc', 'top-association', siteId, query.type, query.identifier, query.startDate, query.endDate],
|
|
147
|
+
path: analyticsRoutes.site.topAssociation(siteId),
|
|
148
|
+
query,
|
|
149
|
+
response: partnerEndpointSchemas.getTopAssociation.response,
|
|
150
|
+
}),
|
|
151
|
+
syncProgress: () => defineNuxtRpcQuery({
|
|
152
|
+
key: ['gsc', 'sync-progress'],
|
|
153
|
+
path: analyticsRoutes.syncProgress,
|
|
154
|
+
response: partnerEndpointSchemas.appSyncProgress.response,
|
|
155
|
+
}),
|
|
156
|
+
})
|
|
157
|
+
|
|
158
|
+
export function withDefaultGscSearchType(params: AnalysisParams): AnalysisParams {
|
|
159
|
+
return withDefaultSearchType(params)
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
export type GscIndexingInspectBody = IndexingInspectRequest
|
|
163
|
+
export type GscBackfillBody = BackfillRange
|