@gscdump/nuxt 0.21.3 → 0.22.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +36 -27
- package/app/components/GscDashboardPage.vue +2 -2
- package/app/components/GscHero.vue +3 -4
- package/app/components/GscLazyTopResult.vue +7 -5
- package/app/components/GscPerformanceChart.vue +4 -4
- package/app/components/GscSourceDebugPanel.vue +2 -1
- package/app/composables/_useGscBackfill.ts +13 -5
- package/app/composables/_useGscQueryDispatcher.ts +5 -4
- package/app/composables/_useGscResource.ts +144 -56
- package/app/composables/useGscAnalytics.ts +13 -4
- package/app/composables/useGscAnalyticsSourceInfo.ts +11 -5
- package/app/composables/useGscAnalyzer.ts +33 -25
- package/app/composables/useGscAnalyzerBatch.ts +2 -2
- package/app/composables/useGscAnalyzerDefs.ts +2 -1
- package/app/composables/useGscCountries.ts +5 -6
- package/app/composables/useGscCurrentSite.ts +32 -11
- package/app/composables/useGscInspectionHistory.ts +5 -5
- package/app/composables/useGscInspections.ts +5 -5
- package/app/composables/useGscPanelContext.ts +9 -2
- package/app/composables/useGscQuery.ts +6 -8
- package/app/composables/useGscQueryDispatcher.ts +14 -122
- package/app/composables/useGscRequestIndexingInspect.ts +20 -11
- package/app/composables/useGscResource.ts +11 -114
- package/app/composables/useGscRollup.ts +66 -81
- package/app/composables/useGscSearchAppearance.ts +5 -6
- package/app/composables/useGscSiteAnalyzer.ts +456 -0
- package/app/composables/useGscSitemapHistory.ts +5 -5
- package/app/composables/useGscSitemaps.ts +5 -5
- package/app/queries/gsc.ts +163 -0
- package/app/utils/duckdb-wasm.ts +166 -0
- package/app/utils/gsc-rpc.ts +18 -0
- package/module.ts +1 -1
- package/nuxt.config.ts +25 -0
- package/package.json +140 -8
- package/types.ts +8 -7
- package/app/composables/useGscEngine.ts +0 -16
- package/app/composables/useGscParquetTable.ts +0 -199
- package/app/composables/useGscRollupTable.ts +0 -78
- package/app/composables/useGscRowQuery.ts +0 -56
- package/app/composables/useGscSnapshotAnalyzer.contract.ts +0 -150
- package/app/composables/useGscSnapshotAnalyzer.routing.ts +0 -87
- package/app/composables/useGscSnapshotAnalyzer.ts +0 -559
- package/app/utils/setup-gsc-fetch-auth.ts +0 -62
|
@@ -0,0 +1,456 @@
|
|
|
1
|
+
// Per-site DuckDB-WASM analyzer over the hosted Iceberg fact tables.
|
|
2
|
+
//
|
|
3
|
+
// The module owns the browser runtime lifecycle: resolve analysis sources,
|
|
4
|
+
// boot shared DuckDB-WASM, attach parquet tables on demand, mirror progress
|
|
5
|
+
// into the layer-wide map, and materialise Arrow rows into plain objects.
|
|
6
|
+
// Callers keep ownership of SQL, presentation, and page-specific state.
|
|
7
|
+
|
|
8
|
+
import type { AsyncDuckDB, AsyncDuckDBConnection } from '@duckdb/duckdb-wasm'
|
|
9
|
+
import type { FileResolutionResponse } from '@gscdump/contracts'
|
|
10
|
+
import type { OpfsAttachedHandle } from '@gscdump/engine-duckdb-wasm'
|
|
11
|
+
import type { AnalysisParams, AnalysisResult } from '@gscdump/engine/analysis-types'
|
|
12
|
+
import { defaultAnalyzerRegistry } from '@gscdump/analysis'
|
|
13
|
+
import { runAnalyzerFromSource } from '@gscdump/engine/analyzer'
|
|
14
|
+
import { createAttachedTableSource } from '@gscdump/engine/source'
|
|
15
|
+
import { gscQueries } from '../queries/gsc'
|
|
16
|
+
import { attachParquetWithFallback, sharedGscDuckDBWasm } from '../utils/duckdb-wasm'
|
|
17
|
+
import { useGscRpc } from '../utils/gsc-rpc'
|
|
18
|
+
|
|
19
|
+
export type GscFactTable
|
|
20
|
+
= | 'pages'
|
|
21
|
+
| 'queries'
|
|
22
|
+
| 'countries'
|
|
23
|
+
| 'dates'
|
|
24
|
+
| 'page_queries'
|
|
25
|
+
| 'search_appearance'
|
|
26
|
+
| 'search_appearance_pages'
|
|
27
|
+
| 'search_appearance_queries'
|
|
28
|
+
| 'search_appearance_page_queries'
|
|
29
|
+
|
|
30
|
+
export type GscTableStage = 'idle' | 'resolving' | 'downloading' | 'attaching' | 'ready' | 'error' | 'unavailable'
|
|
31
|
+
|
|
32
|
+
export interface GscTableStatus {
|
|
33
|
+
stage: GscTableStage
|
|
34
|
+
filesAttached: number
|
|
35
|
+
filesTotal: number
|
|
36
|
+
startedAt?: number
|
|
37
|
+
endedAt?: number
|
|
38
|
+
error?: string
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export interface UseGscSiteAnalyzerReturn {
|
|
42
|
+
tables: Readonly<Ref<Record<GscFactTable, GscTableStatus>>>
|
|
43
|
+
ready: Readonly<Ref<boolean>>
|
|
44
|
+
error: Readonly<Ref<Error | null>>
|
|
45
|
+
query: <T = Record<string, unknown>>(opts: {
|
|
46
|
+
sql: string
|
|
47
|
+
needs: readonly GscFactTable[]
|
|
48
|
+
params?: readonly unknown[]
|
|
49
|
+
}) => Promise<T[]>
|
|
50
|
+
runQuery: <T = Record<string, unknown>>(sql: string, params?: readonly unknown[]) => Promise<{ rows: T[], queryMs: number }>
|
|
51
|
+
analyze: (params: AnalysisParams, opts?: { signal?: AbortSignal }) => Promise<AnalysisResult & { queryMs: number }>
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
interface BootedAnalyzer {
|
|
55
|
+
bootPromise: Promise<{ db: AsyncDuckDB, conn: AsyncDuckDBConnection }>
|
|
56
|
+
tablePromises: Map<GscFactTable, Promise<void>>
|
|
57
|
+
tables: Ref<Record<GscFactTable, GscTableStatus>>
|
|
58
|
+
ready: Ref<boolean>
|
|
59
|
+
error: Ref<Error | null>
|
|
60
|
+
withDb: <T>(fn: (conn: AsyncDuckDBConnection) => Promise<T>) => Promise<T>
|
|
61
|
+
refs: number
|
|
62
|
+
dispose: () => Promise<void>
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
interface AnalyzerArgs {
|
|
66
|
+
siteId: string
|
|
67
|
+
searchType: string
|
|
68
|
+
range: { start: string, end: string }
|
|
69
|
+
useOpfsCache: boolean
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
const ALL_TABLES: readonly GscFactTable[] = [
|
|
73
|
+
'pages',
|
|
74
|
+
'queries',
|
|
75
|
+
'countries',
|
|
76
|
+
'dates',
|
|
77
|
+
'page_queries',
|
|
78
|
+
'search_appearance',
|
|
79
|
+
'search_appearance_pages',
|
|
80
|
+
'search_appearance_queries',
|
|
81
|
+
'search_appearance_page_queries',
|
|
82
|
+
] as const
|
|
83
|
+
|
|
84
|
+
const FACT_TABLE_NAMES = ALL_TABLES.join('|')
|
|
85
|
+
const TABLE_RE = new RegExp(`\\b(?:FROM|JOIN)\\s+(?:main\\.)?(${FACT_TABLE_NAMES})\\b`, 'gi')
|
|
86
|
+
const instanceCaches = new WeakMap<object, Map<string, BootedAnalyzer>>()
|
|
87
|
+
|
|
88
|
+
function tablesFromSql(sql: string): readonly GscFactTable[] {
|
|
89
|
+
const out = new Set<GscFactTable>()
|
|
90
|
+
for (const m of sql.matchAll(TABLE_RE))
|
|
91
|
+
out.add(m[1]!.toLowerCase() as GscFactTable)
|
|
92
|
+
return out.size > 0 ? Array.from(out) : ALL_TABLES
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function sqlIdent(s: string): string {
|
|
96
|
+
return s.replace(/\W/g, '_')
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
function emptyStatus(): GscTableStatus {
|
|
100
|
+
return { stage: 'idle', filesAttached: 0, filesTotal: 0 }
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
function emptyTables(): Record<GscFactTable, GscTableStatus> {
|
|
104
|
+
return Object.fromEntries(ALL_TABLES.map(t => [t, emptyStatus()])) as Record<GscFactTable, GscTableStatus>
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function getCache(): Map<string, BootedAnalyzer> {
|
|
108
|
+
const app = useNuxtApp()
|
|
109
|
+
let bag = instanceCaches.get(app)
|
|
110
|
+
if (!bag) {
|
|
111
|
+
bag = new Map()
|
|
112
|
+
instanceCaches.set(app, bag)
|
|
113
|
+
}
|
|
114
|
+
return bag
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
function instanceKey(siteId: string, searchType: string, range: { start: string, end: string }): string {
|
|
118
|
+
return `${siteId}|${searchType}|${range.start}|${range.end}`
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
export function useGscSiteAnalyzer(
|
|
122
|
+
siteId: MaybeRefOrGetter<string | null | undefined>,
|
|
123
|
+
range: MaybeRefOrGetter<{ start: string, end: string } | null | undefined>,
|
|
124
|
+
options: {
|
|
125
|
+
searchType?: 'web' | 'image' | 'video' | 'news' | 'discover' | 'googleNews'
|
|
126
|
+
useOpfsCache?: boolean
|
|
127
|
+
} = {},
|
|
128
|
+
): UseGscSiteAnalyzerReturn {
|
|
129
|
+
const searchType = options.searchType ?? 'web'
|
|
130
|
+
const useOpfsCache = options.useOpfsCache ?? true
|
|
131
|
+
const tables = ref<Record<GscFactTable, GscTableStatus>>(emptyTables())
|
|
132
|
+
const ready = ref(false)
|
|
133
|
+
const error = ref<Error | null>(null)
|
|
134
|
+
let active: BootedAnalyzer | null = null
|
|
135
|
+
let stopEffects: (() => void) | null = null
|
|
136
|
+
|
|
137
|
+
function bind(instance: BootedAnalyzer | null): void {
|
|
138
|
+
const prev = active
|
|
139
|
+
stopEffects?.()
|
|
140
|
+
stopEffects = null
|
|
141
|
+
active = instance
|
|
142
|
+
|
|
143
|
+
if (instance) {
|
|
144
|
+
instance.refs++
|
|
145
|
+
const scope = effectScope()
|
|
146
|
+
scope.run(() => {
|
|
147
|
+
watchEffect(() => {
|
|
148
|
+
tables.value = instance.tables.value
|
|
149
|
+
})
|
|
150
|
+
watchEffect(() => {
|
|
151
|
+
ready.value = instance.ready.value
|
|
152
|
+
})
|
|
153
|
+
watchEffect(() => {
|
|
154
|
+
error.value = instance.error.value
|
|
155
|
+
})
|
|
156
|
+
})
|
|
157
|
+
stopEffects = () => scope.stop()
|
|
158
|
+
}
|
|
159
|
+
else {
|
|
160
|
+
tables.value = emptyTables()
|
|
161
|
+
ready.value = false
|
|
162
|
+
error.value = null
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
if (prev) {
|
|
166
|
+
prev.refs--
|
|
167
|
+
if (prev.refs <= 0)
|
|
168
|
+
prev.dispose().catch(() => {})
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
function obtain(sid: string, r: { start: string, end: string }): BootedAnalyzer {
|
|
173
|
+
const cache = getCache()
|
|
174
|
+
const key = instanceKey(sid, searchType, r)
|
|
175
|
+
const hit = cache.get(key)
|
|
176
|
+
if (hit)
|
|
177
|
+
return hit
|
|
178
|
+
const created = createAnalyzer({ siteId: sid, searchType, range: r, useOpfsCache })
|
|
179
|
+
cache.set(key, created)
|
|
180
|
+
return created
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
watch(
|
|
184
|
+
() => {
|
|
185
|
+
const sid = toValue(siteId) ?? null
|
|
186
|
+
const r = toValue(range) ?? null
|
|
187
|
+
return sid && r?.start && r?.end ? { sid, r } : null
|
|
188
|
+
},
|
|
189
|
+
(val) => {
|
|
190
|
+
if (!val || !import.meta.client) {
|
|
191
|
+
bind(null)
|
|
192
|
+
return
|
|
193
|
+
}
|
|
194
|
+
bind(obtain(val.sid, val.r))
|
|
195
|
+
},
|
|
196
|
+
{ immediate: true },
|
|
197
|
+
)
|
|
198
|
+
|
|
199
|
+
onScopeDispose(() => bind(null))
|
|
200
|
+
|
|
201
|
+
async function query<T = Record<string, unknown>>(opts: {
|
|
202
|
+
sql: string
|
|
203
|
+
needs: readonly GscFactTable[]
|
|
204
|
+
params?: readonly unknown[]
|
|
205
|
+
}): Promise<T[]> {
|
|
206
|
+
if (!active)
|
|
207
|
+
throw new Error('useGscSiteAnalyzer: no active instance (site/range not set)')
|
|
208
|
+
const instance = active
|
|
209
|
+
await instance.bootPromise
|
|
210
|
+
await Promise.all(opts.needs.map(t => ensureTable(instance, t)))
|
|
211
|
+
return instance.withDb(async (conn) => {
|
|
212
|
+
const result = opts.params && opts.params.length > 0
|
|
213
|
+
? await (async () => {
|
|
214
|
+
const stmt = await conn.prepare(opts.sql)
|
|
215
|
+
try {
|
|
216
|
+
return await stmt.query(...(opts.params as unknown[]))
|
|
217
|
+
}
|
|
218
|
+
finally {
|
|
219
|
+
await stmt.close()
|
|
220
|
+
}
|
|
221
|
+
})()
|
|
222
|
+
: await conn.query(opts.sql)
|
|
223
|
+
const arr = result.toArray() as unknown as Array<Record<string, unknown>>
|
|
224
|
+
return arr.map(row => ({ ...row })) as unknown as T[]
|
|
225
|
+
})
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
async function runQuery<T = Record<string, unknown>>(sql: string, params?: readonly unknown[]): Promise<{ rows: T[], queryMs: number }> {
|
|
229
|
+
const t0 = performance.now()
|
|
230
|
+
const rows = await query<T>({ sql, needs: tablesFromSql(sql), params })
|
|
231
|
+
return { rows, queryMs: performance.now() - t0 }
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
async function analyze(params: AnalysisParams, opts?: { signal?: AbortSignal }): Promise<AnalysisResult & { queryMs: number }> {
|
|
235
|
+
const t0 = performance.now()
|
|
236
|
+
const source = createAttachedTableSource(
|
|
237
|
+
{
|
|
238
|
+
query: async (sql, bindParams) => {
|
|
239
|
+
return query<Record<string, unknown>>({ sql, needs: tablesFromSql(sql), params: bindParams })
|
|
240
|
+
},
|
|
241
|
+
},
|
|
242
|
+
{
|
|
243
|
+
schema: 'main',
|
|
244
|
+
signal: opts?.signal,
|
|
245
|
+
attachedTables: ALL_TABLES as readonly string[],
|
|
246
|
+
},
|
|
247
|
+
)
|
|
248
|
+
const result = await runAnalyzerFromSource(source, params, defaultAnalyzerRegistry)
|
|
249
|
+
return {
|
|
250
|
+
results: result.results as AnalysisResult['results'],
|
|
251
|
+
meta: result.meta as AnalysisResult['meta'],
|
|
252
|
+
queryMs: performance.now() - t0,
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
return {
|
|
257
|
+
tables: tables as Readonly<Ref<Record<GscFactTable, GscTableStatus>>>,
|
|
258
|
+
ready: ready as Readonly<Ref<boolean>>,
|
|
259
|
+
error: error as Readonly<Ref<Error | null>>,
|
|
260
|
+
query,
|
|
261
|
+
runQuery,
|
|
262
|
+
analyze,
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
function createAnalyzer(args: AnalyzerArgs): BootedAnalyzer {
|
|
267
|
+
const tables = ref<Record<GscFactTable, GscTableStatus>>(emptyTables())
|
|
268
|
+
const ready = ref(false)
|
|
269
|
+
const error = ref<Error | null>(null)
|
|
270
|
+
const tablePromises = new Map<GscFactTable, Promise<void>>()
|
|
271
|
+
const opfsHandles = new Map<GscFactTable, OpfsAttachedHandle>()
|
|
272
|
+
|
|
273
|
+
const rpc = useGscRpc()
|
|
274
|
+
const analyticsConfig = useGscAnalyticsConfig()
|
|
275
|
+
const analyticsCtx = useGscAnalyticsContext()
|
|
276
|
+
|
|
277
|
+
let resolvePromise: Promise<FileResolutionResponse | null> | null = null
|
|
278
|
+
function getResolution(): Promise<FileResolutionResponse | null> {
|
|
279
|
+
if (resolvePromise)
|
|
280
|
+
return resolvePromise
|
|
281
|
+
resolvePromise = rpc.query(
|
|
282
|
+
gscQueries.analysisSources(args.siteId, undefined, {
|
|
283
|
+
searchType: args.searchType as 'web' | 'image' | 'video' | 'news' | 'discover' | 'googleNews',
|
|
284
|
+
start: args.range.start,
|
|
285
|
+
end: args.range.end,
|
|
286
|
+
}),
|
|
287
|
+
{ silent: true },
|
|
288
|
+
)
|
|
289
|
+
.then((res) => {
|
|
290
|
+
if (!res || (res as { canUseBrowser?: boolean }).canUseBrowser === false)
|
|
291
|
+
return null
|
|
292
|
+
return res as unknown as FileResolutionResponse
|
|
293
|
+
})
|
|
294
|
+
.catch(() => null)
|
|
295
|
+
return resolvePromise
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
const bootPromise: Promise<{ db: AsyncDuckDB, conn: AsyncDuckDBConnection }> = (async () => {
|
|
299
|
+
const bundleBase = (analyticsConfig as { duckdbBundleBase?: string }).duckdbBundleBase
|
|
300
|
+
const boot = await sharedGscDuckDBWasm(bundleBase)
|
|
301
|
+
ready.value = true
|
|
302
|
+
return boot
|
|
303
|
+
})()
|
|
304
|
+
bootPromise.catch((err) => {
|
|
305
|
+
error.value = err instanceof Error ? err : new Error(String(err))
|
|
306
|
+
})
|
|
307
|
+
|
|
308
|
+
let dbLock: Promise<unknown> = Promise.resolve()
|
|
309
|
+
function withDb<T>(fn: (conn: AsyncDuckDBConnection) => Promise<T>): Promise<T> {
|
|
310
|
+
const next = dbLock.then(async () => {
|
|
311
|
+
const { conn } = await bootPromise
|
|
312
|
+
return fn(conn)
|
|
313
|
+
})
|
|
314
|
+
dbLock = next.catch(() => {})
|
|
315
|
+
return next
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
function patchSelf(table: GscFactTable, patch: Partial<GscTableStatus>): void {
|
|
319
|
+
tables.value = { ...tables.value, [table]: { ...tables.value[table], ...patch } }
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
function patchLayer(table: GscFactTable, stage: GscTableStage, extras: Partial<GscTableStatus> = {}): void {
|
|
323
|
+
const siteSlot = `${args.siteId}#${table}`
|
|
324
|
+
const layerStage = stage === 'unavailable'
|
|
325
|
+
? 'ready'
|
|
326
|
+
: stage === 'downloading' || stage === 'attaching'
|
|
327
|
+
? 'attach'
|
|
328
|
+
: stage === 'resolving'
|
|
329
|
+
? 'manifest'
|
|
330
|
+
: stage === 'ready'
|
|
331
|
+
? 'ready'
|
|
332
|
+
: stage === 'error' ? 'error' : 'idle'
|
|
333
|
+
analyticsCtx.patchProgress(siteSlot, {
|
|
334
|
+
stage: layerStage,
|
|
335
|
+
source: 'duckdb',
|
|
336
|
+
filesAttached: extras.filesAttached ?? tables.value[table].filesAttached,
|
|
337
|
+
filesTotal: extras.filesTotal ?? tables.value[table].filesTotal,
|
|
338
|
+
startedAt: extras.startedAt ?? tables.value[table].startedAt ?? Date.now(),
|
|
339
|
+
endedAt: extras.endedAt,
|
|
340
|
+
error: extras.error,
|
|
341
|
+
})
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
async function attachTable(table: GscFactTable): Promise<void> {
|
|
345
|
+
const startedAt = Date.now()
|
|
346
|
+
patchSelf(table, { stage: 'resolving', startedAt, endedAt: undefined, error: undefined })
|
|
347
|
+
patchLayer(table, 'resolving', { startedAt })
|
|
348
|
+
|
|
349
|
+
const full = await getResolution()
|
|
350
|
+
if (!full) {
|
|
351
|
+
const endedAt = Date.now()
|
|
352
|
+
patchSelf(table, { stage: 'unavailable', endedAt })
|
|
353
|
+
patchLayer(table, 'unavailable', { endedAt })
|
|
354
|
+
return
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
const tableEntry = full.tables.find(t => t.table === table)
|
|
358
|
+
const browserFiles = tableEntry?.mode === 'browser' ? tableEntry.files : []
|
|
359
|
+
if (browserFiles.length === 0) {
|
|
360
|
+
const endedAt = Date.now()
|
|
361
|
+
patchSelf(table, { stage: 'unavailable', endedAt })
|
|
362
|
+
patchLayer(table, 'unavailable', { endedAt })
|
|
363
|
+
return
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
patchSelf(table, { stage: 'downloading', filesTotal: browserFiles.length })
|
|
367
|
+
patchLayer(table, 'downloading', { filesTotal: browserFiles.length })
|
|
368
|
+
|
|
369
|
+
const sid = sqlIdent(args.siteId)
|
|
370
|
+
const tableIdent = sqlIdent(table)
|
|
371
|
+
const viewName = `${tableIdent}_${sid}`
|
|
372
|
+
let filesAttached = 0
|
|
373
|
+
const { db, conn } = await bootPromise
|
|
374
|
+
const opfsHandle = await attachParquetWithFallback({
|
|
375
|
+
db,
|
|
376
|
+
conn,
|
|
377
|
+
viewName,
|
|
378
|
+
files: browserFiles,
|
|
379
|
+
version: full.snapshotVersion,
|
|
380
|
+
useOpfsCache: args.useOpfsCache,
|
|
381
|
+
withDb: fn => withDb(() => fn()),
|
|
382
|
+
onFileProgress: () => {
|
|
383
|
+
filesAttached++
|
|
384
|
+
patchSelf(table, { filesAttached })
|
|
385
|
+
patchLayer(table, 'downloading', { filesAttached })
|
|
386
|
+
},
|
|
387
|
+
}).catch((err) => {
|
|
388
|
+
const msg = err instanceof Error ? err.message : String(err)
|
|
389
|
+
const endedAt = Date.now()
|
|
390
|
+
patchSelf(table, { stage: 'error', error: msg, endedAt })
|
|
391
|
+
patchLayer(table, 'error', { error: msg, endedAt })
|
|
392
|
+
throw err
|
|
393
|
+
})
|
|
394
|
+
|
|
395
|
+
if (viewName !== tableIdent) {
|
|
396
|
+
await withDb(async (conn) => {
|
|
397
|
+
await conn.query(`CREATE OR REPLACE VIEW ${tableIdent} AS SELECT * FROM ${viewName}`)
|
|
398
|
+
})
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
if (opfsHandle)
|
|
402
|
+
opfsHandles.set(table, opfsHandle)
|
|
403
|
+
|
|
404
|
+
const endedAt = Date.now()
|
|
405
|
+
patchSelf(table, { stage: 'ready', endedAt })
|
|
406
|
+
patchLayer(table, 'ready', { endedAt })
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
function ensureTableInner(table: GscFactTable): Promise<void> {
|
|
410
|
+
const existing = tablePromises.get(table)
|
|
411
|
+
if (existing)
|
|
412
|
+
return existing
|
|
413
|
+
const p = attachTable(table)
|
|
414
|
+
tablePromises.set(table, p)
|
|
415
|
+
return p
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
const instance: BootedAnalyzer = {
|
|
419
|
+
bootPromise,
|
|
420
|
+
tablePromises,
|
|
421
|
+
tables,
|
|
422
|
+
ready,
|
|
423
|
+
error,
|
|
424
|
+
withDb,
|
|
425
|
+
refs: 0,
|
|
426
|
+
dispose: async () => {
|
|
427
|
+
const cache = instanceCaches.get(useNuxtApp())
|
|
428
|
+
if (cache) {
|
|
429
|
+
for (const [k, v] of cache.entries()) {
|
|
430
|
+
if (v === instance) {
|
|
431
|
+
cache.delete(k)
|
|
432
|
+
break
|
|
433
|
+
}
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
try {
|
|
437
|
+
for (const h of opfsHandles.values())
|
|
438
|
+
await h.detach().catch(() => {})
|
|
439
|
+
opfsHandles.clear()
|
|
440
|
+
const { conn } = await bootPromise
|
|
441
|
+
for (const t of ALL_TABLES)
|
|
442
|
+
await conn.query(`DROP VIEW IF EXISTS ${sqlIdent(t)}`).catch(() => {})
|
|
443
|
+
}
|
|
444
|
+
catch {
|
|
445
|
+
// Boot failed; nothing to tear down.
|
|
446
|
+
}
|
|
447
|
+
},
|
|
448
|
+
}
|
|
449
|
+
;(instance as BootedAnalyzer & { ensureTable: typeof ensureTableInner }).ensureTable = ensureTableInner
|
|
450
|
+
return instance
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
async function ensureTable(instance: BootedAnalyzer, table: GscFactTable): Promise<void> {
|
|
454
|
+
const ensure = (instance as BootedAnalyzer & { ensureTable: (t: GscFactTable) => Promise<void> }).ensureTable
|
|
455
|
+
return ensure(table)
|
|
456
|
+
}
|
|
@@ -4,16 +4,16 @@
|
|
|
4
4
|
import type { SitemapHistoryRecord, SitemapHistoryResponse } from '@gscdump/contracts'
|
|
5
5
|
import type { ComputedRef, Ref } from '@vue/runtime-core'
|
|
6
6
|
import type { GscResourceStatus } from './_useGscResource'
|
|
7
|
+
import { gscQueries } from '../queries/gsc'
|
|
7
8
|
import { useGscResource } from './_useGscResource'
|
|
8
|
-
import { useGscAnalyticsClient } from './useGscAnalyticsClient'
|
|
9
9
|
|
|
10
10
|
export interface UseGscSitemapHistoryReturn {
|
|
11
11
|
response: Readonly<Ref<SitemapHistoryResponse | null>>
|
|
12
12
|
snapshots: ComputedRef<SitemapHistoryRecord[]>
|
|
13
13
|
path: ComputedRef<string | null>
|
|
14
14
|
loading: ComputedRef<boolean>
|
|
15
|
-
status: Ref<GscResourceStatus
|
|
16
|
-
error: Ref<Error | null
|
|
15
|
+
status: Readonly<Ref<GscResourceStatus>>
|
|
16
|
+
error: Readonly<Ref<Error | null>>
|
|
17
17
|
refresh: () => Promise<void>
|
|
18
18
|
}
|
|
19
19
|
|
|
@@ -21,9 +21,9 @@ export function useGscSitemapHistory(
|
|
|
21
21
|
siteId: MaybeRefOrGetter<string | null | undefined>,
|
|
22
22
|
feedpathHash: MaybeRefOrGetter<string | null | undefined>,
|
|
23
23
|
): UseGscSitemapHistoryReturn {
|
|
24
|
-
const { data, status, loading, error, refresh } = useGscResource({
|
|
24
|
+
const { data, status, loading, error, refresh } = useGscResource<[string, string], SitemapHistoryResponse>({
|
|
25
25
|
keys: [siteId, feedpathHash] as const,
|
|
26
|
-
|
|
26
|
+
operation: (id, hash) => gscQueries.sitemapHistory(id, hash),
|
|
27
27
|
})
|
|
28
28
|
|
|
29
29
|
const snapshots = computed(() => data.value?.snapshots ?? [])
|
|
@@ -5,22 +5,22 @@
|
|
|
5
5
|
import type { SitemapHistoryRecord, SitemapIndex } from '@gscdump/contracts'
|
|
6
6
|
import type { ComputedRef, Ref } from '@vue/runtime-core'
|
|
7
7
|
import type { GscResourceStatus } from './_useGscResource'
|
|
8
|
+
import { gscQueries } from '../queries/gsc'
|
|
8
9
|
import { useGscResource } from './_useGscResource'
|
|
9
|
-
import { useGscAnalyticsClient } from './useGscAnalyticsClient'
|
|
10
10
|
|
|
11
11
|
export interface UseGscSitemapsReturn {
|
|
12
12
|
index: Readonly<Ref<SitemapIndex | null>>
|
|
13
13
|
records: ComputedRef<SitemapHistoryRecord[]>
|
|
14
14
|
loading: ComputedRef<boolean>
|
|
15
|
-
status: Ref<GscResourceStatus
|
|
16
|
-
error: Ref<Error | null
|
|
15
|
+
status: Readonly<Ref<GscResourceStatus>>
|
|
16
|
+
error: Readonly<Ref<Error | null>>
|
|
17
17
|
refresh: () => Promise<void>
|
|
18
18
|
}
|
|
19
19
|
|
|
20
20
|
export function useGscSitemaps(siteId: MaybeRefOrGetter<string | null | undefined>): UseGscSitemapsReturn {
|
|
21
|
-
const { data, status, loading, error, refresh } = useGscResource({
|
|
21
|
+
const { data, status, loading, error, refresh } = useGscResource<[string], SitemapIndex>({
|
|
22
22
|
keys: [siteId] as const,
|
|
23
|
-
|
|
23
|
+
operation: (id: string) => gscQueries.sitemaps(id),
|
|
24
24
|
})
|
|
25
25
|
|
|
26
26
|
const records = computed<SitemapHistoryRecord[]>(() => {
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
import type { AnalysisSourcesOptions, BackfillRange, GscApiRange, GscdumpTopAssociationParams, IndexingInspectRequest, SourceInfoOptions } from '@gscdump/contracts'
|
|
2
|
+
import type { AnalysisParams } from '@gscdump/engine/analysis-types'
|
|
3
|
+
import { analyticsRoutes, gscdumpAnalysisParamsSchema, gscdumpAnalysisResponseSchema, partnerEndpointSchemas } from '@gscdump/contracts'
|
|
4
|
+
import { defineNuxtQueryGroup, defineNuxtRpcMutation, defineNuxtRpcQuery } from 'nuxt-use-query/rpc'
|
|
5
|
+
|
|
6
|
+
const DEFAULT_SEARCH_TYPE = 'web'
|
|
7
|
+
|
|
8
|
+
type SearchType = NonNullable<SourceInfoOptions['searchType']>
|
|
9
|
+
|
|
10
|
+
function searchTypeQuery(searchType?: SearchType): Record<string, string> {
|
|
11
|
+
return { searchType: searchType ?? DEFAULT_SEARCH_TYPE }
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
function dateRangeQuery(options: { start?: string, end?: string, startDate?: string, endDate?: string } | undefined): Record<string, string> {
|
|
15
|
+
const query: Record<string, string> = {}
|
|
16
|
+
const start = options?.start ?? options?.startDate
|
|
17
|
+
const end = options?.end ?? options?.endDate
|
|
18
|
+
if (start)
|
|
19
|
+
query.start = start
|
|
20
|
+
if (end)
|
|
21
|
+
query.end = end
|
|
22
|
+
return query
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function sourceInfoQuery(options: SourceInfoOptions | undefined): Record<string, string> {
|
|
26
|
+
return {
|
|
27
|
+
...searchTypeQuery(options?.searchType),
|
|
28
|
+
...dateRangeQuery(options),
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function tablesQuery(
|
|
33
|
+
tablesOrOptions: string[] | string | AnalysisSourcesOptions | undefined,
|
|
34
|
+
options?: { searchType?: SearchType, start?: string, end?: string, startDate?: string, endDate?: string },
|
|
35
|
+
): Record<string, string> {
|
|
36
|
+
const isOptions = !!tablesOrOptions && typeof tablesOrOptions === 'object' && !Array.isArray(tablesOrOptions)
|
|
37
|
+
const source = isOptions ? tablesOrOptions : options
|
|
38
|
+
const tables = isOptions ? tablesOrOptions.tables : tablesOrOptions
|
|
39
|
+
const query = {
|
|
40
|
+
...searchTypeQuery(source?.searchType),
|
|
41
|
+
...dateRangeQuery(source),
|
|
42
|
+
}
|
|
43
|
+
if (tables)
|
|
44
|
+
query.tables = Array.isArray(tables) ? tables.join(',') : tables
|
|
45
|
+
return query
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function withDefaultSearchType<T>(value: T): T {
|
|
49
|
+
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
50
|
+
return value
|
|
51
|
+
return {
|
|
52
|
+
...(value as Record<string, unknown>),
|
|
53
|
+
searchType: (value as { searchType?: SearchType }).searchType ?? DEFAULT_SEARCH_TYPE,
|
|
54
|
+
} as T
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export const gscQueries = defineNuxtQueryGroup('gsc', {
|
|
58
|
+
whoami: () => defineNuxtRpcQuery({
|
|
59
|
+
key: ['gsc', 'whoami'],
|
|
60
|
+
path: analyticsRoutes.whoami,
|
|
61
|
+
response: partnerEndpointSchemas.analyticsWhoami.response,
|
|
62
|
+
}),
|
|
63
|
+
sites: () => defineNuxtRpcQuery({
|
|
64
|
+
key: ['gsc', 'sites'],
|
|
65
|
+
path: analyticsRoutes.sites,
|
|
66
|
+
response: partnerEndpointSchemas.analyticsSites.response,
|
|
67
|
+
}),
|
|
68
|
+
sourceInfo: (siteId: string, options?: SourceInfoOptions) => defineNuxtRpcQuery({
|
|
69
|
+
key: ['gsc', 'source-info', siteId, options?.searchType ?? DEFAULT_SEARCH_TYPE, options?.start ?? options?.startDate ?? '', options?.end ?? options?.endDate ?? ''],
|
|
70
|
+
path: analyticsRoutes.site.sourceInfo(siteId),
|
|
71
|
+
query: sourceInfoQuery(options),
|
|
72
|
+
response: partnerEndpointSchemas.analyticsSourceInfo.response,
|
|
73
|
+
}),
|
|
74
|
+
analysisSources: (
|
|
75
|
+
siteId: string,
|
|
76
|
+
tables?: string[] | string | AnalysisSourcesOptions,
|
|
77
|
+
options?: { searchType?: SearchType, start?: string, end?: string, startDate?: string, endDate?: string },
|
|
78
|
+
) => {
|
|
79
|
+
const source = tables && typeof tables === 'object' && !Array.isArray(tables) ? tables : options
|
|
80
|
+
const tableList = tables && typeof tables === 'object' && !Array.isArray(tables) ? tables.tables : tables
|
|
81
|
+
const tablesCsv = Array.isArray(tableList) ? tableList.join(',') : (tableList ?? '')
|
|
82
|
+
return defineNuxtRpcQuery({
|
|
83
|
+
key: ['gsc', 'analysis-sources', siteId, source?.searchType ?? DEFAULT_SEARCH_TYPE, source?.start ?? source?.startDate ?? '', source?.end ?? source?.endDate ?? '', tablesCsv],
|
|
84
|
+
path: analyticsRoutes.site.analysisSources(siteId),
|
|
85
|
+
query: tablesQuery(tables, options),
|
|
86
|
+
response: partnerEndpointSchemas.analyticsAnalysisSources.response,
|
|
87
|
+
})
|
|
88
|
+
},
|
|
89
|
+
analyze: (siteId: string) => defineNuxtRpcMutation({
|
|
90
|
+
body: gscdumpAnalysisParamsSchema,
|
|
91
|
+
method: 'POST',
|
|
92
|
+
path: analyticsRoutes.site.analyze(siteId),
|
|
93
|
+
response: gscdumpAnalysisResponseSchema,
|
|
94
|
+
}),
|
|
95
|
+
backfill: (siteId: string) => defineNuxtRpcMutation({
|
|
96
|
+
body: partnerEndpointSchemas.analyticsBackfill.body,
|
|
97
|
+
method: 'POST',
|
|
98
|
+
path: analyticsRoutes.site.backfill(siteId),
|
|
99
|
+
response: partnerEndpointSchemas.analyticsBackfill.response,
|
|
100
|
+
}),
|
|
101
|
+
sitemaps: (siteId: string) => defineNuxtRpcQuery({
|
|
102
|
+
key: ['gsc', 'sitemaps', siteId],
|
|
103
|
+
path: analyticsRoutes.site.sitemaps(siteId),
|
|
104
|
+
response: partnerEndpointSchemas.analyticsSitemaps.response,
|
|
105
|
+
}),
|
|
106
|
+
sitemapHistory: (siteId: string, hash: string) => defineNuxtRpcQuery({
|
|
107
|
+
key: ['gsc', 'sitemap-history', siteId, hash],
|
|
108
|
+
path: analyticsRoutes.site.sitemapHistory(siteId, hash),
|
|
109
|
+
response: partnerEndpointSchemas.analyticsSitemapHistory.response,
|
|
110
|
+
}),
|
|
111
|
+
inspections: (siteId: string) => defineNuxtRpcQuery({
|
|
112
|
+
key: ['gsc', 'inspections', siteId],
|
|
113
|
+
path: analyticsRoutes.site.inspections(siteId),
|
|
114
|
+
response: partnerEndpointSchemas.analyticsInspections.response,
|
|
115
|
+
}),
|
|
116
|
+
inspectionHistory: (siteId: string, hash: string) => defineNuxtRpcQuery({
|
|
117
|
+
key: ['gsc', 'inspection-history', siteId, hash],
|
|
118
|
+
path: analyticsRoutes.site.inspectionHistory(siteId, hash),
|
|
119
|
+
response: partnerEndpointSchemas.analyticsInspectionHistory.response,
|
|
120
|
+
}),
|
|
121
|
+
indexingInspect: (siteId: string) => defineNuxtRpcMutation({
|
|
122
|
+
body: partnerEndpointSchemas.analyticsIndexingInspect.body,
|
|
123
|
+
method: 'POST',
|
|
124
|
+
path: analyticsRoutes.site.indexingInspect(siteId),
|
|
125
|
+
response: partnerEndpointSchemas.analyticsIndexingInspect.response,
|
|
126
|
+
}),
|
|
127
|
+
countries: (siteId: string, range: GscApiRange) => defineNuxtRpcQuery({
|
|
128
|
+
key: ['gsc', 'countries', siteId, range.start, range.end],
|
|
129
|
+
path: analyticsRoutes.site.countries(siteId),
|
|
130
|
+
query: range,
|
|
131
|
+
response: partnerEndpointSchemas.analyticsCountries.response,
|
|
132
|
+
}),
|
|
133
|
+
searchAppearance: (siteId: string, range: GscApiRange) => defineNuxtRpcQuery({
|
|
134
|
+
key: ['gsc', 'search-appearance', siteId, range.start, range.end],
|
|
135
|
+
path: analyticsRoutes.site.searchAppearance(siteId),
|
|
136
|
+
query: range,
|
|
137
|
+
response: partnerEndpointSchemas.analyticsSearchAppearance.response,
|
|
138
|
+
}),
|
|
139
|
+
rollup: (siteId: string, rollupId: string, range?: Partial<GscApiRange>) => defineNuxtRpcQuery({
|
|
140
|
+
key: ['gsc', 'rollup', siteId, rollupId, range?.start ?? '', range?.end ?? ''],
|
|
141
|
+
path: analyticsRoutes.site.rollup(siteId, rollupId),
|
|
142
|
+
query: range,
|
|
143
|
+
response: partnerEndpointSchemas.analyticsRollup.response,
|
|
144
|
+
}),
|
|
145
|
+
topAssociation: (siteId: string, query: GscdumpTopAssociationParams) => defineNuxtRpcQuery({
|
|
146
|
+
key: ['gsc', 'top-association', siteId, query.type, query.identifier, query.startDate, query.endDate],
|
|
147
|
+
path: analyticsRoutes.site.topAssociation(siteId),
|
|
148
|
+
query,
|
|
149
|
+
response: partnerEndpointSchemas.getTopAssociation.response,
|
|
150
|
+
}),
|
|
151
|
+
syncProgress: () => defineNuxtRpcQuery({
|
|
152
|
+
key: ['gsc', 'sync-progress'],
|
|
153
|
+
path: analyticsRoutes.syncProgress,
|
|
154
|
+
response: partnerEndpointSchemas.appSyncProgress.response,
|
|
155
|
+
}),
|
|
156
|
+
})
|
|
157
|
+
|
|
158
|
+
export function withDefaultGscSearchType(params: AnalysisParams): AnalysisParams {
|
|
159
|
+
return withDefaultSearchType(params)
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
export type GscIndexingInspectBody = IndexingInspectRequest
|
|
163
|
+
export type GscBackfillBody = BackfillRange
|