@gscdump/analysis 1.0.1 → 1.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,4892 +1,2 @@
1
- import { createAnalyzerRegistry, defineAnalyzer, requireAdapter } from "@gscdump/engine/analyzer";
2
- import { comparisonOf, defaultEndDate, padTimeseries, periodOf } from "@gscdump/engine/period";
3
- import { enumeratePartitions } from "@gscdump/engine/planner";
4
- import { METRIC_EXPR } from "@gscdump/engine/sql-fragments";
5
- import { num } from "@gscdump/engine/analysis-types";
6
- import { err, ok, unwrapResult } from "gscdump/result";
7
- import { between, date, extractDateRange, gsc, page, query } from "gscdump/query";
8
- import { MS_PER_DAY, daysAgoUtc, toIsoDate } from "gscdump/dates";
9
- import { buildExtrasQueries, buildTotalsSql, mergeExtras, resolveComparisonSQL, resolveToSQLOptimized } from "@gscdump/engine/resolver";
10
- function rowNumber(value) {
11
- if (typeof value === "number") return Number.isFinite(value) ? value : 0;
12
- if (typeof value === "bigint") return Number(value);
13
- if (value == null) return 0;
14
- const n = Number(value);
15
- return Number.isFinite(n) ? n : 0;
16
- }
17
- function rowString(value) {
18
- return value == null ? "" : String(value);
19
- }
20
- function rowBoolean(value) {
21
- return value === true || value === 1 || value === "true";
22
- }
23
- function parseJsonRows(value) {
24
- if (Array.isArray(value)) return value;
25
- if (typeof value === "string" && value.length > 0) {
26
- const parsed = JSON.parse(value);
27
- return Array.isArray(parsed) ? parsed : [];
28
- }
29
- return [];
30
- }
31
- const bayesianCtrAnalyzer = defineAnalyzer({
32
- id: "bayesian-ctr",
33
- buildSql(params) {
34
- const { startDate, endDate } = periodOf(params);
35
- const minImpressions = params.minImpressions ?? 50;
36
- const limit = params.limit ?? 300;
37
- const priorMinEntities = 5;
38
- return {
39
- sql: `
40
- WITH entity AS (
41
- SELECT
42
- query,
43
- url,
44
- ${METRIC_EXPR.clicks} AS clicks,
45
- ${METRIC_EXPR.impressions} AS impressions,
46
- ${METRIC_EXPR.ctr} AS observed_ctr,
47
- ${METRIC_EXPR.position} AS position,
48
- CAST(ROUND(LEAST(${METRIC_EXPR.position}, 30)) AS INTEGER) AS bucket
49
- FROM read_parquet({{FILES}}, union_by_name = true)
50
- WHERE date >= ? AND date <= ?
51
- AND query IS NOT NULL AND query <> ''
52
- AND url IS NOT NULL AND url <> ''
53
- GROUP BY query, url
54
- HAVING SUM(impressions) >= ?
55
- AND ${METRIC_EXPR.position} <= 30
56
- ),
57
- bucket_mu AS (
58
- SELECT
59
- bucket,
60
- COUNT(*) AS n_entities,
61
- SUM(observed_ctr * impressions) / NULLIF(SUM(impressions), 0) AS mu,
62
- SUM(impressions) AS total_impressions
63
- FROM entity
64
- GROUP BY bucket
65
- ),
66
- bucket_var AS (
67
- SELECT
68
- e.bucket,
69
- GREATEST(
70
- SUM(e.impressions * POWER(e.observed_ctr - b.mu, 2))
71
- / NULLIF(SUM(e.impressions), 0),
72
- 1e-9
73
- ) AS v
74
- FROM entity e
75
- JOIN bucket_mu b USING (bucket)
76
- GROUP BY e.bucket
77
- ),
78
- priors AS (
79
- SELECT
80
- m.bucket,
81
- m.n_entities,
82
- m.mu,
83
- v.v,
84
- CASE
85
- WHEN m.n_entities >= ${Number(priorMinEntities)}
86
- AND v.v > 0
87
- AND m.mu > 0 AND m.mu < 1
88
- AND (m.mu * (1.0 - m.mu) / v.v - 1.0) > 0
89
- THEN GREATEST(0.5, m.mu * (m.mu * (1.0 - m.mu) / v.v - 1.0))
90
- ELSE 2.0
91
- END AS alpha,
92
- CASE
93
- WHEN m.n_entities >= ${Number(priorMinEntities)}
94
- AND v.v > 0
95
- AND m.mu > 0 AND m.mu < 1
96
- AND (m.mu * (1.0 - m.mu) / v.v - 1.0) > 0
97
- THEN GREATEST(0.5, (1.0 - m.mu) * (m.mu * (1.0 - m.mu) / v.v - 1.0))
98
- ELSE 48.0
99
- END AS beta
100
- FROM bucket_mu m
101
- JOIN bucket_var v USING (bucket)
102
- ),
103
- posterior AS (
104
- SELECT
105
- e.query,
106
- e.url,
107
- e.clicks,
108
- e.impressions,
109
- e.observed_ctr,
110
- e.position,
111
- e.bucket,
112
- p.alpha AS prior_alpha,
113
- p.beta AS prior_beta,
114
- p.mu AS bucket_prior_mean,
115
- p.alpha + e.clicks AS alpha_post,
116
- p.beta + (e.impressions - e.clicks) AS beta_post
117
- FROM entity e
118
- JOIN priors p USING (bucket)
119
- ),
120
- scored AS (
121
- SELECT *,
122
- alpha_post / (alpha_post + beta_post) AS posterior_mean,
123
- SQRT((alpha_post * beta_post)
124
- / (POWER(alpha_post + beta_post, 2) * (alpha_post + beta_post + 1))) AS posterior_sd
125
- FROM posterior
126
- )
127
- SELECT
128
- query AS keyword,
129
- url AS page,
130
- clicks,
131
- impressions,
132
- observed_ctr AS observedCtr,
133
- position,
134
- bucket,
135
- prior_alpha AS priorAlpha,
136
- prior_beta AS priorBeta,
137
- bucket_prior_mean AS bucketPriorMean,
138
- posterior_mean AS posteriorMean,
139
- posterior_sd AS posteriorSd,
140
- GREATEST(0.0, posterior_mean - 1.96 * posterior_sd) AS ciLow,
141
- LEAST(1.0, posterior_mean + 1.96 * posterior_sd) AS ciHigh,
142
- posterior_mean - observed_ctr AS shrinkageDelta,
143
- (posterior_mean - observed_ctr) * impressions AS expectedClicksDelta,
144
- ABS(observed_ctr - posterior_mean) / NULLIF(posterior_sd, 0) AS significance,
145
- CASE
146
- WHEN observed_ctr > LEAST(1.0, posterior_mean + 1.96 * posterior_sd) THEN 'overperforming'
147
- WHEN observed_ctr < GREATEST(0.0, posterior_mean - 1.96 * posterior_sd) THEN 'underperforming'
148
- ELSE 'expected'
149
- END AS classification
150
- FROM scored
151
- ORDER BY significance DESC NULLS LAST
152
- LIMIT ${Number(limit)}
153
- `,
154
- params: [
155
- startDate,
156
- endDate,
157
- minImpressions
158
- ],
159
- current: {
160
- table: "page_queries",
161
- partitions: enumeratePartitions(startDate, endDate)
162
- }
163
- };
164
- },
165
- reduceSql(rows, params) {
166
- const arr = Array.isArray(rows) ? rows : [];
167
- const minImpressions = params.minImpressions ?? 50;
168
- const results = arr.map((r) => ({
169
- keyword: rowString(r.keyword),
170
- page: rowString(r.page),
171
- clicks: rowNumber(r.clicks),
172
- impressions: rowNumber(r.impressions),
173
- observedCtr: rowNumber(r.observedCtr),
174
- position: rowNumber(r.position),
175
- bucket: rowNumber(r.bucket),
176
- priorAlpha: rowNumber(r.priorAlpha),
177
- priorBeta: rowNumber(r.priorBeta),
178
- bucketPriorMean: rowNumber(r.bucketPriorMean),
179
- posteriorMean: rowNumber(r.posteriorMean),
180
- posteriorSd: rowNumber(r.posteriorSd),
181
- ciLow: rowNumber(r.ciLow),
182
- ciHigh: rowNumber(r.ciHigh),
183
- shrinkageDelta: rowNumber(r.shrinkageDelta),
184
- expectedClicksDelta: rowNumber(r.expectedClicksDelta),
185
- significance: rowNumber(r.significance),
186
- classification: rowString(r.classification)
187
- }));
188
- const under = results.filter((r) => r.classification === "underperforming").length;
189
- const over = results.filter((r) => r.classification === "overperforming").length;
190
- return {
191
- results,
192
- meta: {
193
- total: results.length,
194
- underperforming: under,
195
- overperforming: over,
196
- expected: results.length - under - over,
197
- minImpressions
198
- }
199
- };
200
- }
201
- });
202
- const BIPARTITE_PAGERANK_ITERATIONS = 25;
203
- const BIPARTITE_PAGERANK_DAMPING = .85;
204
- const bipartitePagerankAnalyzer = defineAnalyzer({
205
- id: "bipartite-pagerank",
206
- buildSql(params) {
207
- const { startDate, endDate } = periodOf(params);
208
- const minImpressions = params.minImpressions ?? 50;
209
- const topQueries = 1e3;
210
- const topUrls = 500;
211
- const limit = params.limit ?? 50;
212
- const bridgingEdgeThreshold = .05;
213
- const anchoringEdgeThreshold = .05;
214
- const iterations = BIPARTITE_PAGERANK_ITERATIONS;
215
- const d = BIPARTITE_PAGERANK_DAMPING;
216
- const iterCtes = [];
217
- for (let i = 1; i <= iterations; i++) iterCtes.push(`
218
- ranks_${i} AS (
219
- SELECT
220
- 'q' AS kind,
221
- e.qid AS id,
222
- (1.0 - ${d}) / (SELECT n FROM query_count)
223
- + ${d} * SUM(e.w_u_to_q * r.rank) AS rank
224
- FROM u_to_q_weights e
225
- JOIN ranks_${i - 1} r ON r.kind = 'u' AND r.id = e.uid
226
- GROUP BY e.qid
227
- UNION ALL
228
- SELECT
229
- 'u' AS kind,
230
- e.uid AS id,
231
- (1.0 - ${d}) / (SELECT n FROM url_count)
232
- + ${d} * SUM(e.w_q_to_u * r.rank) AS rank
233
- FROM q_to_u_weights e
234
- JOIN ranks_${i - 1} r ON r.kind = 'q' AND r.id = e.qid
235
- GROUP BY e.uid
236
- )`);
237
- const deltaParts = [];
238
- for (let i = 1; i <= iterations; i++) deltaParts.push(`
239
- SELECT ${i} AS step,
240
- (SELECT COALESCE(SUM(ABS(a.rank - b.rank)), 0.0)
241
- FROM ranks_${i} a
242
- JOIN ranks_${i - 1} b USING (kind, id)) AS l1`);
243
- return {
244
- sql: `
245
- WITH edges0 AS (
246
- SELECT
247
- query AS qid,
248
- url AS uid,
249
- CAST(SUM(impressions) AS DOUBLE) AS impressions
250
- FROM read_parquet({{FILES}}, union_by_name = true)
251
- WHERE date >= ? AND date <= ?
252
- AND query IS NOT NULL AND query <> ''
253
- AND url IS NOT NULL AND url <> ''
254
- GROUP BY query, url
255
- HAVING SUM(impressions) >= ?
256
- ),
257
- -- Top-N caps per side keep the iteration tractable.
258
- query_totals AS (
259
- SELECT qid, SUM(impressions) AS tot
260
- FROM edges0 GROUP BY qid
261
- ),
262
- url_totals AS (
263
- SELECT uid, SUM(impressions) AS tot
264
- FROM edges0 GROUP BY uid
265
- ),
266
- top_queries AS (
267
- SELECT qid FROM query_totals
268
- ORDER BY tot DESC, qid ASC LIMIT ${Number(topQueries)}
269
- ),
270
- top_urls AS (
271
- SELECT uid FROM url_totals
272
- ORDER BY tot DESC, uid ASC LIMIT ${Number(topUrls)}
273
- ),
274
- edges AS (
275
- SELECT e.qid, e.uid, e.impressions
276
- FROM edges0 e
277
- JOIN top_queries tq USING (qid)
278
- JOIN top_urls tu USING (uid)
279
- ),
280
- query_nodes AS (SELECT DISTINCT qid FROM edges),
281
- url_nodes AS (SELECT DISTINCT uid FROM edges),
282
- query_count AS (SELECT GREATEST(COUNT(*), 1) AS n FROM query_nodes),
283
- url_count AS (SELECT GREATEST(COUNT(*), 1) AS n FROM url_nodes),
284
- -- Row-stochastic transition weights in each direction. For q->u the
285
- -- weights out of a query sum to 1; symmetric for u->q.
286
- q_out AS (SELECT qid, SUM(impressions) AS s FROM edges GROUP BY qid),
287
- u_out AS (SELECT uid, SUM(impressions) AS s FROM edges GROUP BY uid),
288
- q_to_u_weights AS (
289
- SELECT e.qid, e.uid,
290
- e.impressions / NULLIF(q.s, 0) AS w_q_to_u
291
- FROM edges e JOIN q_out q USING (qid)
292
- ),
293
- u_to_q_weights AS (
294
- SELECT e.qid, e.uid,
295
- e.impressions / NULLIF(u.s, 0) AS w_u_to_q
296
- FROM edges e JOIN u_out u USING (uid)
297
- ),
298
- -- Seed: uniform distribution per side. Total mass = 2 (one unit per side).
299
- ranks_0 AS (
300
- SELECT 'q' AS kind, q.qid AS id, 1.0 / (SELECT n FROM query_count) AS rank
301
- FROM query_nodes q
302
- UNION ALL
303
- SELECT 'u' AS kind, u.uid AS id, 1.0 / (SELECT n FROM url_count) AS rank
304
- FROM url_nodes u
305
- ),
306
- ${iterCtes.join(",\n")},
307
- final_ranks AS (SELECT * FROM ranks_${iterations}),
308
- -- Hub/anchor diagnostics computed from raw edge mass (not rank). A
309
- -- query "bridges" URLs it sends >= ${bridgingEdgeThreshold} of its mass
310
- -- to; a URL "anchors" queries that contribute >= ${anchoringEdgeThreshold}
311
- -- of its incoming mass.
312
- q_bridging AS (
313
- SELECT qid, COUNT(*) AS bridging
314
- FROM q_to_u_weights
315
- WHERE w_q_to_u >= ${bridgingEdgeThreshold}
316
- GROUP BY qid
317
- ),
318
- u_anchoring AS (
319
- SELECT uid, COUNT(*) AS anchoring
320
- FROM u_to_q_weights
321
- WHERE w_u_to_q >= ${anchoringEdgeThreshold}
322
- GROUP BY uid
323
- ),
324
- q_degree AS (
325
- SELECT qid, COUNT(*) AS degree, SUM(impressions) AS impressions
326
- FROM edges GROUP BY qid
327
- ),
328
- u_degree AS (
329
- SELECT uid, COUNT(*) AS degree, SUM(impressions) AS impressions
330
- FROM edges GROUP BY uid
331
- ),
332
- deltas AS (
333
- ${deltaParts.join("\n UNION ALL\n")}
334
- ),
335
- query_rows AS (
336
- SELECT
337
- 'query' AS kind, f.id, f.rank,
338
- COALESCE(b.bridging, 0) AS bridging,
339
- 0 AS anchoring,
340
- COALESCE(qd.degree, 0) AS degree,
341
- COALESCE(qd.impressions, 0) AS impressions
342
- FROM final_ranks f
343
- LEFT JOIN q_bridging b ON b.qid = f.id
344
- LEFT JOIN q_degree qd ON qd.qid = f.id
345
- WHERE f.kind = 'q'
346
- ORDER BY f.rank DESC
347
- LIMIT ${Number(limit)}
348
- ),
349
- url_rows AS (
350
- SELECT
351
- 'url' AS kind, f.id, f.rank,
352
- 0 AS bridging,
353
- COALESCE(a.anchoring, 0) AS anchoring,
354
- COALESCE(ud.degree, 0) AS degree,
355
- COALESCE(ud.impressions, 0) AS impressions
356
- FROM final_ranks f
357
- LEFT JOIN u_anchoring a ON a.uid = f.id
358
- LEFT JOIN u_degree ud ON ud.uid = f.id
359
- WHERE f.kind = 'u'
360
- ORDER BY f.rank DESC
361
- LIMIT ${Number(limit)}
362
- ),
363
- nodes AS (
364
- SELECT * FROM query_rows
365
- UNION ALL
366
- SELECT * FROM url_rows
367
- ),
368
- counts AS (
369
- SELECT
370
- (SELECT n FROM query_count) AS q_count,
371
- (SELECT n FROM url_count) AS u_count
372
- ),
373
- deltas_json AS (
374
- SELECT to_json(list({ 'step': step, 'l1': l1 } ORDER BY step)) AS dj
375
- FROM deltas
376
- )
377
- SELECT
378
- n.kind,
379
- n.id,
380
- n.rank,
381
- n.bridging,
382
- n.anchoring,
383
- n.degree,
384
- n.impressions,
385
- c.q_count AS queryCount,
386
- c.u_count AS urlCount,
387
- dj.dj AS deltasJson
388
- FROM nodes n
389
- CROSS JOIN counts c
390
- CROSS JOIN deltas_json dj
391
- ORDER BY n.kind, n.rank DESC
392
- `,
393
- params: [
394
- startDate,
395
- endDate,
396
- minImpressions
397
- ],
398
- current: {
399
- table: "page_queries",
400
- partitions: enumeratePartitions(startDate, endDate)
401
- }
402
- };
403
- },
404
- reduceSql(rows) {
405
- const arr = Array.isArray(rows) ? rows : [];
406
- const iterations = BIPARTITE_PAGERANK_ITERATIONS;
407
- const d = BIPARTITE_PAGERANK_DAMPING;
408
- const results = arr.map((r) => ({
409
- kind: rowString(r.kind),
410
- id: rowString(r.id),
411
- rank: num(r.rank),
412
- bridging: num(r.bridging),
413
- anchoring: num(r.anchoring),
414
- degree: num(r.degree),
415
- impressions: num(r.impressions)
416
- }));
417
- const first = arr[0] ?? {};
418
- const queryCount = num(first.queryCount);
419
- const urlCount = num(first.urlCount);
420
- const deltas = parseJsonRows(first.deltasJson).map((e) => ({
421
- step: num(e.step),
422
- l1: num(e.l1)
423
- }));
424
- const convergenceDelta = deltas.length > 0 ? deltas[deltas.length - 1].l1 : 0;
425
- return {
426
- results,
427
- meta: {
428
- total: results.length,
429
- convergenceDelta,
430
- iterations,
431
- damping: d,
432
- queryCount,
433
- urlCount,
434
- deltas
435
- }
436
- };
437
- }
438
- });
439
- const DEFAULT_LIMIT$1 = 25e3;
440
- function queriesQueryState(period, limit = DEFAULT_LIMIT$1) {
441
- return gsc.select(query, page).where(between(date, period.startDate, period.endDate)).limit(limit).getState();
442
- }
443
- function pagesQueryState(period, limit = DEFAULT_LIMIT$1) {
444
- return gsc.select(page).where(between(date, period.startDate, period.endDate)).limit(limit).getState();
445
- }
446
- function datesQueryState(period, limit = DEFAULT_LIMIT$1) {
447
- return gsc.select(date).where(between(date, period.startDate, period.endDate)).limit(limit).getState();
448
- }
449
- const analysisErrors = {
450
- missingReportParam(report, param, message) {
451
- return {
452
- kind: "missing-report-param",
453
- report,
454
- param,
455
- message
456
- };
457
- },
458
- missingComparisonWindow(report, message) {
459
- return {
460
- kind: "missing-comparison-window",
461
- report,
462
- message
463
- };
464
- },
465
- missingBrandTerms() {
466
- return {
467
- kind: "missing-brand-terms",
468
- message: "Brand analysis requires brandTerms"
469
- };
470
- },
471
- unknownReport(report, available) {
472
- return {
473
- kind: "unknown-report",
474
- report,
475
- available,
476
- message: `unknown report "${report}"; available: ${available.join(", ")}`
477
- };
478
- },
479
- unknownAnalyzer(analyzer) {
480
- return {
481
- kind: "unknown-analyzer",
482
- analyzer,
483
- message: `unknown analyzer "${analyzer}"`
484
- };
485
- },
486
- requiredStepFailed(report, stepKey, stepError, cause) {
487
- return {
488
- kind: "required-step-failed",
489
- report,
490
- stepKey,
491
- stepError,
492
- cause,
493
- message: `runReport(${report}): required step "${stepKey}" failed: ${stepError}`
494
- };
495
- }
496
- };
497
- function analysisErrorToException(error) {
498
- const exception = new Error(error.message);
499
- if ("cause" in error && error.cause !== void 0) exception.cause = error.cause;
500
- exception.analysisError = error;
501
- return exception;
502
- }
503
- function escapeRegexAlt(s) {
504
- return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
505
- }
506
- function requireBrandTermsResult(brandTerms) {
507
- return brandTerms?.length ? ok(brandTerms) : err(analysisErrors.missingBrandTerms());
508
- }
509
- function requireBrandTerms(brandTerms) {
510
- return unwrapResult(requireBrandTermsResult(brandTerms), analysisErrorToException);
511
- }
512
- function analyzeBrandSegmentation(keywords, options) {
513
- const { brandTerms, minImpressions = 10 } = options;
514
- const lowerBrandTerms = brandTerms.map((t) => t.toLowerCase());
515
- const brand = [];
516
- const nonBrand = [];
517
- let brandClicks = 0;
518
- let nonBrandClicks = 0;
519
- let brandImpressions = 0;
520
- let nonBrandImpressions = 0;
521
- for (const row of keywords) {
522
- const impressions = num(row.impressions);
523
- if (impressions < minImpressions) continue;
524
- const clicks = num(row.clicks);
525
- if ((row.variants?.length ? row.variants : [row.query ?? row.keyword ?? ""]).some((query) => lowerBrandTerms.some((term) => query.toLowerCase().includes(term)))) {
526
- brand.push(row);
527
- brandClicks += clicks;
528
- brandImpressions += impressions;
529
- } else {
530
- nonBrand.push(row);
531
- nonBrandClicks += clicks;
532
- nonBrandImpressions += impressions;
533
- }
534
- }
535
- const totalClicks = brandClicks + nonBrandClicks;
536
- return {
537
- brand,
538
- nonBrand,
539
- summary: {
540
- brandClicks,
541
- nonBrandClicks,
542
- brandShare: totalClicks > 0 ? brandClicks / totalClicks : 0,
543
- brandImpressions,
544
- nonBrandImpressions
545
- }
546
- };
547
- }
548
- const brandAnalyzer = defineAnalyzer({
549
- id: "brand",
550
- buildSql(params) {
551
- const brandTerms = requireBrandTerms(params.brandTerms);
552
- const { startDate, endDate } = periodOf(params);
553
- const minImpressions = params.minImpressions ?? 10;
554
- const limit = params.limit ?? 1e4;
555
- const regex = `(${brandTerms.map((t) => escapeRegexAlt(t.toLowerCase())).join("|")})`;
556
- return {
557
- sql: `
558
- WITH agg AS (
559
- SELECT
560
- query,
561
- url AS page,
562
- ${METRIC_EXPR.clicks} AS clicks,
563
- ${METRIC_EXPR.impressions} AS impressions,
564
- ${METRIC_EXPR.ctr} AS ctr,
565
- ${METRIC_EXPR.position} AS position
566
- FROM read_parquet({{FILES}}, union_by_name = true)
567
- WHERE date >= ? AND date <= ?
568
- GROUP BY query, url
569
- HAVING SUM(impressions) >= ?
570
- )
571
- SELECT
572
- query, page, clicks, impressions, ctr, position,
573
- CASE WHEN regexp_matches(LOWER(query), ?) THEN 'brand' ELSE 'non-brand' END AS segment
574
- FROM agg
575
- ORDER BY clicks DESC
576
- LIMIT ${Number(limit)}
577
- `,
578
- params: [
579
- startDate,
580
- endDate,
581
- minImpressions,
582
- regex
583
- ],
584
- current: {
585
- table: "page_queries",
586
- partitions: enumeratePartitions(startDate, endDate)
587
- }
588
- };
589
- },
590
- reduceSql(rows) {
591
- const normalized = (Array.isArray(rows) ? rows : []).map((r) => ({
592
- query: rowString(r.query),
593
- page: r.page == null ? void 0 : rowString(r.page),
594
- clicks: num(r.clicks),
595
- impressions: num(r.impressions),
596
- ctr: num(r.ctr),
597
- position: num(r.position),
598
- segment: rowString(r.segment)
599
- }));
600
- let brandClicks = 0;
601
- let nonBrandClicks = 0;
602
- let brandImpressions = 0;
603
- let nonBrandImpressions = 0;
604
- for (const r of normalized) if (r.segment === "brand") {
605
- brandClicks += r.clicks;
606
- brandImpressions += r.impressions;
607
- } else {
608
- nonBrandClicks += r.clicks;
609
- nonBrandImpressions += r.impressions;
610
- }
611
- const totalClicks = brandClicks + nonBrandClicks;
612
- return {
613
- results: normalized,
614
- meta: {
615
- total: normalized.length,
616
- summary: {
617
- brandClicks,
618
- nonBrandClicks,
619
- brandShare: totalClicks > 0 ? brandClicks / totalClicks : 0,
620
- brandImpressions,
621
- nonBrandImpressions
622
- }
623
- }
624
- };
625
- },
626
- buildRows(params) {
627
- return { queries: queriesQueryState(periodOf(params), params.limit) };
628
- },
629
- reduceRows(rows, params) {
630
- const brandTerms = requireBrandTerms(params.brandTerms);
631
- const result = analyzeBrandSegmentation(Array.isArray(rows) ? rows : [], {
632
- brandTerms,
633
- minImpressions: params.minImpressions
634
- });
635
- return {
636
- results: [...result.brand.map((r) => ({
637
- ...r,
638
- segment: "brand"
639
- })), ...result.nonBrand.map((r) => ({
640
- ...r,
641
- segment: "non-brand"
642
- }))],
643
- meta: { summary: result.summary }
644
- };
645
- }
646
- });
647
- function buildPeriodMap(rows, key, value, filter) {
648
- const out = /* @__PURE__ */ new Map();
649
- for (const row of rows) {
650
- if (filter && !filter(row)) continue;
651
- out.set(key(row), value(row));
652
- }
653
- return out;
654
- }
655
- function createSorter(getValue, defaultMetric, defaultOrder = "desc") {
656
- return (items, sortBy = defaultMetric, sortOrder = defaultOrder) => {
657
- const mult = sortOrder === "desc" ? -1 : 1;
658
- return [...items].sort((a, b) => (getValue(a, sortBy) - getValue(b, sortBy)) * mult);
659
- };
660
- }
661
- function createMetricSorter(defaultMetric, orderByMetric) {
662
- return (items, sortBy = defaultMetric) => {
663
- const mult = orderByMetric[sortBy] === "desc" ? -1 : 1;
664
- return [...items].sort((a, b) => (a[sortBy] - b[sortBy]) * mult);
665
- };
666
- }
667
- const sortRowResults = createSorter((item, metric) => {
668
- switch (metric) {
669
- case "clicks": return item.totalClicks;
670
- case "impressions": return item.totalImpressions;
671
- case "positionSpread": return item.positionSpread;
672
- case "pageCount": return item.pages.length;
673
- }
674
- }, "clicks");
675
- function analyzeCannibalization(rows, options = {}) {
676
- const { minImpressions = 10, maxPositionSpread = 10, minPages = 2, sortBy = "clicks", sortOrder = "desc" } = options;
677
- const queryMap = /* @__PURE__ */ new Map();
678
- for (const row of rows) {
679
- if (row.impressions < minImpressions) continue;
680
- const pages = queryMap.get(row.query) || [];
681
- pages.push({
682
- page: row.page,
683
- clicks: row.clicks,
684
- impressions: row.impressions,
685
- ctr: row.ctr,
686
- position: row.position
687
- });
688
- queryMap.set(row.query, pages);
689
- }
690
- const results = [];
691
- for (const [query, pages] of queryMap) {
692
- if (pages.length < minPages) continue;
693
- let minPosition = Number.POSITIVE_INFINITY;
694
- let maxPosition = Number.NEGATIVE_INFINITY;
695
- let totalClicks = 0;
696
- let totalImpressions = 0;
697
- for (const page of pages) {
698
- minPosition = Math.min(minPosition, page.position);
699
- maxPosition = Math.max(maxPosition, page.position);
700
- totalClicks += page.clicks;
701
- totalImpressions += page.impressions;
702
- }
703
- const positionSpread = maxPosition - minPosition;
704
- if (positionSpread > maxPositionSpread) continue;
705
- pages.sort((a, b) => b.clicks - a.clicks);
706
- results.push({
707
- query,
708
- pages,
709
- totalClicks,
710
- totalImpressions,
711
- positionSpread
712
- });
713
- }
714
- return sortRowResults(results, sortBy, sortOrder);
715
- }
716
- const cannibalizationAnalyzer = defineAnalyzer({
717
- id: "cannibalization",
718
- buildSql(params) {
719
- const { startDate, endDate } = periodOf(params);
720
- const minImpressions = params.minImpressions ?? 50;
721
- const minCompetitors = 2;
722
- const minQueryImpressions = (params.minImpressions ?? 50) * 2;
723
- const limit = params.limit ?? 200;
724
- return {
725
- sql: `
726
- WITH agg AS (
727
- SELECT
728
- query,
729
- url,
730
- ${METRIC_EXPR.clicks} AS clicks,
731
- ${METRIC_EXPR.impressions} AS impressions,
732
- ${METRIC_EXPR.ctr} AS ctr,
733
- ${METRIC_EXPR.position} AS position
734
- FROM read_parquet({{FILES}}, union_by_name = true)
735
- WHERE date >= ? AND date <= ?
736
- AND query IS NOT NULL AND query <> ''
737
- AND url IS NOT NULL AND url <> ''
738
- GROUP BY query, url
739
- HAVING SUM(impressions) >= ?
740
- ),
741
- query_totals AS (
742
- SELECT
743
- query,
744
- SUM(impressions) AS total_impressions,
745
- SUM(clicks) AS total_clicks,
746
- COUNT(*) AS competitor_count
747
- FROM agg
748
- GROUP BY query
749
- HAVING COUNT(*) >= ? AND SUM(impressions) >= ?
750
- ),
751
- ranked AS (
752
- SELECT
753
- a.query,
754
- a.url,
755
- a.clicks,
756
- a.impressions,
757
- a.ctr,
758
- a.position,
759
- a.impressions / NULLIF(t.total_impressions, 0) AS share,
760
- ROW_NUMBER() OVER (
761
- PARTITION BY a.query
762
- ORDER BY a.impressions DESC, a.clicks DESC, a.url ASC
763
- ) AS rnk
764
- FROM agg a
765
- JOIN query_totals t USING (query)
766
- ),
767
- leader AS (
768
- SELECT query, url AS leader_url, ctr AS leader_ctr, position AS leader_position
769
- FROM ranked WHERE rnk = 1
770
- ),
771
- events AS (
772
- SELECT
773
- r.query,
774
- any_value(l.leader_url) AS leader_url,
775
- any_value(l.leader_ctr) AS leader_ctr,
776
- any_value(l.leader_position) AS leader_position,
777
- SUM(POWER(r.share * 100.0, 2)) AS hhi,
778
- SUM(CASE
779
- WHEN r.rnk > 1 AND l.leader_ctr > r.ctr
780
- THEN (l.leader_ctr - r.ctr) * r.impressions
781
- ELSE 0.0
782
- END) AS stolen_clicks,
783
- to_json(list({
784
- 'url': r.url,
785
- 'clicks': r.clicks,
786
- 'impressions': r.impressions,
787
- 'ctr': r.ctr,
788
- 'position': r.position,
789
- 'share': r.share,
790
- 'rank': r.rnk
791
- } ORDER BY r.rnk)) AS competitors
792
- FROM ranked r
793
- JOIN leader l USING (query)
794
- GROUP BY r.query
795
- )
796
- SELECT
797
- e.query AS keyword,
798
- t.total_impressions AS totalImpressions,
799
- t.total_clicks AS totalClicks,
800
- t.competitor_count AS competitorCount,
801
- e.leader_url AS leaderUrl,
802
- e.leader_ctr AS leaderCtr,
803
- e.leader_position AS leaderPosition,
804
- e.hhi AS hhi,
805
- GREATEST(0.0, 1.0 - e.hhi / 10000.0) AS fragmentation,
806
- e.stolen_clicks AS stolenClicks,
807
- e.competitors AS competitors,
808
- CAST(ROUND(LEAST(100.0,
809
- 100.0 * POWER(
810
- GREATEST(1.0 - e.hhi / 10000.0, 0.0)
811
- * LEAST(e.stolen_clicks / GREATEST(t.total_clicks + e.stolen_clicks, 1.0), 1.0)
812
- * LEAST(LOG10(GREATEST(t.total_impressions, 10.0)) / 5.0, 1.0),
813
- 1.0 / 3.0
814
- )
815
- )) AS DOUBLE) AS severity
816
- FROM events e
817
- JOIN query_totals t USING (query)
818
- ORDER BY severity DESC, stolenClicks DESC
819
- LIMIT ${Number(limit)}
820
- `,
821
- params: [
822
- startDate,
823
- endDate,
824
- minImpressions,
825
- minCompetitors,
826
- minQueryImpressions
827
- ],
828
- current: {
829
- table: "page_queries",
830
- partitions: enumeratePartitions(startDate, endDate)
831
- }
832
- };
833
- },
834
- reduceSql(rows) {
835
- const events = (Array.isArray(rows) ? rows : []).map((r) => ({
836
- keyword: rowString(r.keyword),
837
- totalImpressions: num(r.totalImpressions),
838
- totalClicks: num(r.totalClicks),
839
- competitorCount: num(r.competitorCount),
840
- leaderUrl: rowString(r.leaderUrl),
841
- leaderCtr: num(r.leaderCtr),
842
- leaderPosition: num(r.leaderPosition),
843
- hhi: num(r.hhi),
844
- fragmentation: num(r.fragmentation),
845
- stolenClicks: num(r.stolenClicks),
846
- severity: num(r.severity),
847
- competitors: parseJsonRows(r.competitors).map((c) => ({
848
- url: rowString(c.url),
849
- clicks: num(c.clicks),
850
- impressions: num(c.impressions),
851
- ctr: num(c.ctr),
852
- position: num(c.position),
853
- share: num(c.share),
854
- rank: num(c.rank)
855
- }))
856
- }));
857
- const nodeAgg = /* @__PURE__ */ new Map();
858
- const edgeAgg = /* @__PURE__ */ new Map();
859
- for (const ev of events) {
860
- for (const c of ev.competitors) {
861
- const n = nodeAgg.get(c.url) ?? {
862
- impressions: 0,
863
- clicks: 0,
864
- queries: /* @__PURE__ */ new Set()
865
- };
866
- n.impressions += c.impressions;
867
- n.clicks += c.clicks;
868
- n.queries.add(ev.keyword);
869
- nodeAgg.set(c.url, n);
870
- }
871
- for (let i = 0; i < ev.competitors.length; i++) for (let j = i + 1; j < ev.competitors.length; j++) {
872
- const a = ev.competitors[i];
873
- const b = ev.competitors[j];
874
- const [src, tgt] = a.url < b.url ? [a.url, b.url] : [b.url, a.url];
875
- const key = `${src}${tgt}`;
876
- const weight = Math.min(a.impressions, b.impressions);
877
- const edge = edgeAgg.get(key) ?? {
878
- source: src,
879
- target: tgt,
880
- weight: 0,
881
- queries: 0
882
- };
883
- edge.weight += weight;
884
- edge.queries += 1;
885
- edgeAgg.set(key, edge);
886
- }
887
- }
888
- const nodes = [...nodeAgg.entries()].map(([url, n]) => ({
889
- url,
890
- impressions: n.impressions,
891
- clicks: n.clicks,
892
- queryCount: n.queries.size
893
- }));
894
- const edges = [...edgeAgg.values()];
895
- const avgFragmentation = events.length > 0 ? events.reduce((s, e) => s + e.fragmentation, 0) / events.length : 0;
896
- const totalStolenClicks = events.reduce((s, e) => s + e.stolenClicks, 0);
897
- return {
898
- results: events,
899
- meta: {
900
- total: events.length,
901
- totalStolenClicks,
902
- avgFragmentation,
903
- graph: {
904
- nodes,
905
- edges
906
- }
907
- }
908
- };
909
- },
910
- buildRows(params) {
911
- return { rows: queriesQueryState(periodOf(params), params.limit) };
912
- },
913
- reduceRows(rows, params) {
914
- const results = analyzeCannibalization(Array.isArray(rows) ? rows : [], {
915
- minImpressions: params.minImpressions,
916
- maxPositionSpread: params.maxPositionSpread,
917
- minPages: params.minPages
918
- });
919
- return {
920
- results,
921
- meta: { total: results.length }
922
- };
923
- }
924
- });
925
- const changePointAnalyzer = defineAnalyzer({
926
- id: "change-point",
927
- buildSql(params) {
928
- const endDate = params.endDate ?? defaultEndDate();
929
- const startDate = params.startDate ?? daysAgoUtc(93);
930
- const minDays = 21;
931
- const minSide = 7;
932
- const threshold = params.threshold ?? 10;
933
- const minImpressions = params.minImpressions ?? 50;
934
- const metric = params.metric === "clicks" || params.metric === "impressions" ? params.metric : "position";
935
- const limit = params.limit ?? 100;
936
- const valueExpr = metric === "position" ? METRIC_EXPR.position : `CAST(SUM(${metric}) AS DOUBLE)`;
937
- const weightExpr = metric === "position" ? METRIC_EXPR.impressions : "1.0";
938
- return {
939
- sql: `
940
- WITH daily AS (
941
- SELECT
942
- query,
943
- url AS page,
944
- -- Normalize at the source CTE: union_by_name=true can coerce date to
945
- -- VARCHAR across parquets with mixed schemas, which makes downstream
946
- -- strftime(date, ...) binder-error.
947
- CAST(date AS DATE) AS date,
948
- ${METRIC_EXPR.clicks} AS clicks,
949
- ${METRIC_EXPR.impressions} AS impressions,
950
- ${valueExpr} AS value,
951
- ${weightExpr} AS weight
952
- FROM read_parquet({{FILES}}, union_by_name = true)
953
- WHERE date >= ? AND date <= ?
954
- AND query IS NOT NULL AND query <> ''
955
- AND url IS NOT NULL AND url <> ''
956
- GROUP BY query, url, date
957
- HAVING SUM(impressions) >= 1
958
- ),
959
- entity_stats AS (
960
- SELECT query, page,
961
- COUNT(*) AS n_total,
962
- SUM(impressions) AS total_impressions,
963
- SUM(weight) AS w_total,
964
- SUM(value * weight) AS sum_total,
965
- SUM(value * value * weight) AS sumsq_total
966
- FROM daily
967
- GROUP BY query, page
968
- HAVING COUNT(*) >= ${Number(minDays)}
969
- AND SUM(impressions) >= ?
970
- ),
971
- filtered AS (
972
- SELECT d.*,
973
- e.n_total, e.w_total, e.sum_total, e.sumsq_total, e.total_impressions
974
- FROM daily d
975
- JOIN entity_stats e USING (query, page)
976
- ),
977
- cumulated AS (
978
- SELECT *,
979
- COUNT(*) OVER w AS n_left,
980
- SUM(weight) OVER w AS w_left,
981
- SUM(value * weight) OVER w AS sum_left,
982
- SUM(value * value * weight) OVER w AS sumsq_left
983
- FROM filtered
984
- WINDOW w AS (
985
- PARTITION BY query, page
986
- ORDER BY date
987
- ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
988
- )
989
- ),
990
- llr_scored AS (
991
- SELECT *,
992
- (n_total - n_left) AS n_right,
993
- (w_total - w_left) AS w_right,
994
- (sum_total - sum_left) AS sum_right,
995
- (sumsq_total - sumsq_left) AS sumsq_right,
996
- GREATEST(
997
- (sumsq_left / NULLIF(w_left, 0))
998
- - (sum_left / NULLIF(w_left, 0)) * (sum_left / NULLIF(w_left, 0)),
999
- 1e-9
1000
- ) AS var_left,
1001
- GREATEST(
1002
- ((sumsq_total - sumsq_left) / NULLIF(w_total - w_left, 0))
1003
- - ((sum_total - sum_left) / NULLIF(w_total - w_left, 0))
1004
- * ((sum_total - sum_left) / NULLIF(w_total - w_left, 0)),
1005
- 1e-9
1006
- ) AS var_right,
1007
- GREATEST(
1008
- (sumsq_total / NULLIF(w_total, 0))
1009
- - (sum_total / NULLIF(w_total, 0)) * (sum_total / NULLIF(w_total, 0)),
1010
- 1e-9
1011
- ) AS var_single
1012
- FROM cumulated
1013
- ),
1014
- llr AS (
1015
- SELECT *,
1016
- CASE
1017
- WHEN n_left >= ${Number(minSide)} AND (n_total - n_left) >= ${Number(minSide)}
1018
- THEN n_total * LN(var_single)
1019
- - n_left * LN(var_left)
1020
- - (n_total - n_left) * LN(var_right)
1021
- ELSE NULL
1022
- END AS llr
1023
- FROM llr_scored
1024
- ),
1025
- best AS (
1026
- SELECT query, page, n_total, total_impressions,
1027
- arg_max(date, llr) AS change_date,
1028
- MAX(llr) AS best_llr,
1029
- arg_max(sum_left / NULLIF(w_left, 0), llr) AS left_mean,
1030
- arg_max((sum_total - sum_left) / NULLIF(w_total - w_left, 0), llr) AS right_mean,
1031
- arg_max(sqrt(var_left), llr) AS left_std,
1032
- arg_max(sqrt(var_right), llr) AS right_std
1033
- FROM llr
1034
- WHERE llr IS NOT NULL
1035
- GROUP BY query, page, n_total, total_impressions
1036
- HAVING MAX(llr) > ${Number(threshold)}
1037
- ),
1038
- series AS (
1039
- SELECT query, page,
1040
- to_json(list({
1041
- 'date': strftime(date, '%Y-%m-%d'),
1042
- 'value': value
1043
- } ORDER BY date)) AS seriesJson
1044
- FROM daily
1045
- GROUP BY query, page
1046
- )
1047
- SELECT
1048
- b.query AS keyword,
1049
- b.page,
1050
- CAST(b.n_total AS DOUBLE) AS totalDays,
1051
- CAST(b.total_impressions AS DOUBLE) AS totalImpressions,
1052
- strftime(b.change_date, '%Y-%m-%d') AS changeDate,
1053
- b.best_llr AS llr,
1054
- b.left_mean AS leftMean,
1055
- b.right_mean AS rightMean,
1056
- (b.right_mean - b.left_mean) AS delta,
1057
- b.left_std AS leftStddev,
1058
- b.right_std AS rightStddev,
1059
- s.seriesJson
1060
- FROM best b
1061
- LEFT JOIN series s USING (query, page)
1062
- ORDER BY b.best_llr DESC
1063
- LIMIT ${Number(limit)}
1064
- `,
1065
- params: [
1066
- startDate,
1067
- endDate,
1068
- minImpressions
1069
- ],
1070
- current: {
1071
- table: "page_queries",
1072
- partitions: enumeratePartitions(startDate, endDate)
1073
- }
1074
- };
1075
- },
1076
- reduceSql(rows, params) {
1077
- const arr = Array.isArray(rows) ? rows : [];
1078
- const threshold = params.threshold ?? 10;
1079
- const metric = params.metric === "clicks" || params.metric === "impressions" ? params.metric : "position";
1080
- const lowerIsBetter = metric === "position";
1081
- const results = arr.map((r) => {
1082
- const delta = rowNumber(r.delta);
1083
- const improved = lowerIsBetter ? delta < 0 : delta > 0;
1084
- return {
1085
- keyword: rowString(r.keyword),
1086
- page: rowString(r.page),
1087
- totalDays: rowNumber(r.totalDays),
1088
- totalImpressions: rowNumber(r.totalImpressions),
1089
- changeDate: rowString(r.changeDate),
1090
- llr: rowNumber(r.llr),
1091
- leftMean: rowNumber(r.leftMean),
1092
- rightMean: rowNumber(r.rightMean),
1093
- delta,
1094
- leftStddev: rowNumber(r.leftStddev),
1095
- rightStddev: rowNumber(r.rightStddev),
1096
- direction: improved ? "improved" : "worsened",
1097
- series: parseJsonRows(r.seriesJson).map((s) => ({
1098
- date: rowString(s.date),
1099
- value: rowNumber(s.value)
1100
- }))
1101
- };
1102
- });
1103
- return {
1104
- results,
1105
- meta: {
1106
- total: results.length,
1107
- metric,
1108
- threshold,
1109
- improved: results.filter((r) => r.direction === "improved").length,
1110
- worsened: results.filter((r) => r.direction === "worsened").length
1111
- }
1112
- };
1113
- }
1114
- });
1115
- const INTENT_PREFIXES_REGEX = "^(how to|what is|what are|why is|why do|where to|when to|best|top|vs|versus|compare|review|buy|cheap|free|near me)(\\s|$)";
1116
- const INTENT_PREFIXES = [
1117
- "how to",
1118
- "what is",
1119
- "what are",
1120
- "why is",
1121
- "why do",
1122
- "where to",
1123
- "when to",
1124
- "best",
1125
- "top",
1126
- "vs",
1127
- "versus",
1128
- "compare",
1129
- "review",
1130
- "buy",
1131
- "cheap",
1132
- "free",
1133
- "near me"
1134
- ];
1135
- const WHITESPACE_RE = /\s+/;
1136
- function extractIntentPrefix(keyword) {
1137
- const lower = keyword.toLowerCase();
1138
- for (const prefix of INTENT_PREFIXES) if (lower.startsWith(`${prefix} `) || lower.startsWith(prefix)) return prefix;
1139
- return null;
1140
- }
1141
- function extractWordPrefix(keyword, wordCount = 2) {
1142
- const words = keyword.toLowerCase().split(WHITESPACE_RE).filter(Boolean);
1143
- if (words.length < wordCount + 1) return null;
1144
- return words.slice(0, wordCount).join(" ");
1145
- }
1146
- function analyzeClustering(keywords, options = {}) {
1147
- const { minClusterSize = 2, minImpressions = 10, clusterBy = "both" } = options;
1148
- const filtered = keywords.filter((k) => num(k.impressions) >= minImpressions);
1149
- const clusterMap = /* @__PURE__ */ new Map();
1150
- const clusteredKeywords = /* @__PURE__ */ new Set();
1151
- if (clusterBy === "intent" || clusterBy === "both") for (const kw of filtered) {
1152
- const intent = extractIntentPrefix(kw.query);
1153
- if (intent) {
1154
- const existing = clusterMap.get(intent);
1155
- if (existing) existing.keywords.push(kw);
1156
- else clusterMap.set(intent, {
1157
- type: "intent",
1158
- keywords: [kw]
1159
- });
1160
- clusteredKeywords.add(kw.query);
1161
- }
1162
- }
1163
- if (clusterBy === "prefix" || clusterBy === "both") {
1164
- const prefixMap = /* @__PURE__ */ new Map();
1165
- for (const kw of filtered) {
1166
- if (clusteredKeywords.has(kw.query)) continue;
1167
- const prefix = extractWordPrefix(kw.query);
1168
- if (prefix) {
1169
- const existing = prefixMap.get(prefix);
1170
- if (existing) existing.push(kw);
1171
- else prefixMap.set(prefix, [kw]);
1172
- }
1173
- }
1174
- for (const [prefix, kws] of prefixMap) if (kws.length >= minClusterSize) {
1175
- clusterMap.set(prefix, {
1176
- type: "prefix",
1177
- keywords: kws
1178
- });
1179
- for (const kw of kws) clusteredKeywords.add(kw.query);
1180
- }
1181
- }
1182
- const clusters = [];
1183
- for (const [name, data] of clusterMap) {
1184
- if (data.keywords.length < minClusterSize) continue;
1185
- let totalClicks = 0;
1186
- let totalImpressions = 0;
1187
- let weightedPositionSum = 0;
1188
- let positionWeight = 0;
1189
- let positionSum = 0;
1190
- for (const k of data.keywords) {
1191
- const impressions = num(k.impressions);
1192
- const position = num(k.position);
1193
- totalClicks += num(k.clicks);
1194
- totalImpressions += impressions;
1195
- positionSum += position;
1196
- if (impressions > 0) {
1197
- weightedPositionSum += (position - 1) * impressions;
1198
- positionWeight += impressions;
1199
- }
1200
- }
1201
- const avgPosition = positionWeight > 0 ? weightedPositionSum / positionWeight + 1 : positionSum / data.keywords.length;
1202
- clusters.push({
1203
- clusterName: name,
1204
- clusterType: data.type,
1205
- keywords: data.keywords,
1206
- totalClicks,
1207
- totalImpressions,
1208
- avgPosition,
1209
- keywordCount: data.keywords.length
1210
- });
1211
- }
1212
- clusters.sort((a, b) => b.totalClicks - a.totalClicks);
1213
- return {
1214
- clusters,
1215
- unclustered: filtered.filter((kw) => !clusteredKeywords.has(kw.query))
1216
- };
1217
- }
1218
- const clusteringAnalyzer = defineAnalyzer({
1219
- id: "clustering",
1220
- buildSql(params) {
1221
- const { startDate, endDate } = periodOf(params);
1222
- const minImpressions = params.minImpressions ?? 10;
1223
- const minClusterSize = params.minClusterSize ?? 2;
1224
- const clusterBy = params.clusterBy ?? "both";
1225
- const doIntent = clusterBy === "intent" || clusterBy === "both";
1226
- const doPrefix = clusterBy === "prefix" || clusterBy === "both";
1227
- const intentExpr = doIntent ? `NULLIF(regexp_extract(LOWER(query), '${INTENT_PREFIXES_REGEX}', 1), '')` : `CAST(NULL AS VARCHAR)`;
1228
- const prefixExpr = doPrefix ? `CASE WHEN len(regexp_split_to_array(LOWER(query), '\\s+')) >= 3
1229
- THEN array_to_string(list_slice(regexp_split_to_array(LOWER(query), '\\s+'), 1, 2), ' ')
1230
- ELSE CAST(NULL AS VARCHAR) END` : `CAST(NULL AS VARCHAR)`;
1231
- return {
1232
- sql: `
1233
- WITH agg AS (
1234
- SELECT
1235
- query,
1236
- ${METRIC_EXPR.clicks} AS clicks,
1237
- ${METRIC_EXPR.impressions} AS impressions,
1238
- ${METRIC_EXPR.ctr} AS ctr,
1239
- ${METRIC_EXPR.position} AS position
1240
- FROM read_parquet({{FILES}}, union_by_name = true)
1241
- WHERE date >= ? AND date <= ?
1242
- GROUP BY query
1243
- HAVING SUM(impressions) >= ?
1244
- ),
1245
- classified AS (
1246
- SELECT
1247
- query, clicks, impressions, ctr, position,
1248
- ${intentExpr} AS intent_prefix,
1249
- ${prefixExpr} AS word_prefix
1250
- FROM agg
1251
- ),
1252
- keyed AS (
1253
- SELECT
1254
- query, clicks, impressions, ctr, position,
1255
- COALESCE(intent_prefix, word_prefix) AS cluster_name,
1256
- CASE WHEN intent_prefix IS NOT NULL THEN 'intent' ELSE 'prefix' END AS cluster_type
1257
- FROM classified
1258
- WHERE COALESCE(intent_prefix, word_prefix) IS NOT NULL
1259
- )
1260
- SELECT
1261
- cluster_name AS clusterName,
1262
- any_value(cluster_type) AS clusterType,
1263
- CAST(COUNT(*) AS DOUBLE) AS keywordCount,
1264
- ${METRIC_EXPR.clicks} AS totalClicks,
1265
- ${METRIC_EXPR.impressions} AS totalImpressions,
1266
- SUM((position - 1) * impressions) / NULLIF(SUM(impressions), 0) + 1 AS avgPosition,
1267
- to_json(list({ 'query': query, 'clicks': clicks, 'impressions': impressions, 'ctr': ctr, 'position': position })) AS keywords
1268
- FROM keyed
1269
- GROUP BY cluster_name
1270
- HAVING COUNT(*) >= ?
1271
- ORDER BY totalClicks DESC
1272
- `,
1273
- params: [
1274
- startDate,
1275
- endDate,
1276
- minImpressions,
1277
- minClusterSize
1278
- ],
1279
- current: {
1280
- table: "queries",
1281
- partitions: enumeratePartitions(startDate, endDate)
1282
- }
1283
- };
1284
- },
1285
- reduceSql(rows) {
1286
- const clusters = (Array.isArray(rows) ? rows : []).map((r) => ({
1287
- clusterName: rowString(r.clusterName),
1288
- clusterType: rowString(r.clusterType),
1289
- keywordCount: num(r.keywordCount),
1290
- totalClicks: num(r.totalClicks),
1291
- totalImpressions: num(r.totalImpressions),
1292
- avgPosition: num(r.avgPosition),
1293
- keywords: parseJsonRows(r.keywords).map((k) => ({
1294
- query: rowString(k.query),
1295
- clicks: num(k.clicks),
1296
- impressions: num(k.impressions),
1297
- ctr: num(k.ctr),
1298
- position: num(k.position)
1299
- }))
1300
- }));
1301
- return {
1302
- results: clusters,
1303
- meta: {
1304
- total: clusters.length,
1305
- totalClusters: clusters.length
1306
- }
1307
- };
1308
- },
1309
- buildRows(params) {
1310
- return { queries: queriesQueryState(periodOf(params), params.limit) };
1311
- },
1312
- reduceRows(rows, params) {
1313
- const result = analyzeClustering(Array.isArray(rows) ? rows : [], {
1314
- clusterBy: params.clusterBy,
1315
- minClusterSize: params.minClusterSize,
1316
- minImpressions: params.minImpressions
1317
- });
1318
- return {
1319
- results: result.clusters,
1320
- meta: { totalClusters: result.clusters.length }
1321
- };
1322
- }
1323
- });
1324
- function analyzeConcentration(items, options = {}) {
1325
- const { topN = 10 } = options;
1326
- if (items.length === 0) return {
1327
- giniCoefficient: 0,
1328
- hhi: 0,
1329
- topNConcentration: 0,
1330
- topNItems: [],
1331
- totalItems: 0,
1332
- totalClicks: 0,
1333
- riskLevel: "low"
1334
- };
1335
- const sorted = [...items].sort((a, b) => b.clicks - a.clicks);
1336
- const n = sorted.length;
1337
- let totalClicks = 0;
1338
- let squaredClicks = 0;
1339
- let weightedSum = 0;
1340
- for (let i = 0; i < n; i++) {
1341
- const clicks = sorted[i].clicks;
1342
- totalClicks += clicks;
1343
- squaredClicks += clicks * clicks;
1344
- weightedSum += (n - 2 * i - 1) * clicks;
1345
- }
1346
- const giniCoefficient = totalClicks === 0 ? 0 : weightedSum / (n * totalClicks);
1347
- const hhi = totalClicks > 0 ? squaredClicks / (totalClicks * totalClicks) * 1e4 : 0;
1348
- const topNItems = sorted.slice(0, topN).map((item) => ({
1349
- key: item.key,
1350
- clicks: item.clicks,
1351
- share: totalClicks > 0 ? item.clicks / totalClicks : 0
1352
- }));
1353
- const topNClicks = topNItems.reduce((sum, item) => sum + item.clicks, 0);
1354
- const topNConcentration = totalClicks > 0 ? topNClicks / totalClicks : 0;
1355
- let riskLevel = "low";
1356
- if (hhi > 2500) riskLevel = "high";
1357
- else if (hhi > 1500) riskLevel = "medium";
1358
- return {
1359
- giniCoefficient,
1360
- hhi,
1361
- topNConcentration,
1362
- topNItems,
1363
- totalItems: items.length,
1364
- totalClicks,
1365
- riskLevel
1366
- };
1367
- }
1368
- function analyzePageConcentration(pages, options) {
1369
- return analyzeConcentration(pages.map((p) => ({
1370
- key: p.page,
1371
- clicks: num(p.clicks)
1372
- })), options);
1373
- }
1374
- function analyzeKeywordConcentration(keywords, options) {
1375
- return analyzeConcentration(keywords.map((k) => ({
1376
- key: k.query,
1377
- clicks: num(k.clicks)
1378
- })), options);
1379
- }
1380
- const concentrationAnalyzer = defineAnalyzer({
1381
- id: "concentration",
1382
- buildSql(params) {
1383
- const { startDate, endDate } = periodOf(params);
1384
- const dim = params.dimension || "pages";
1385
- const topN = params.topN ?? 10;
1386
- const table = dim === "keywords" ? "queries" : "pages";
1387
- const keyCol = dim === "keywords" ? "query" : "url";
1388
- return {
1389
- sql: `
1390
- WITH items AS (
1391
- SELECT
1392
- ${keyCol} AS key,
1393
- ${METRIC_EXPR.clicks} AS clicks
1394
- FROM read_parquet({{FILES}}, union_by_name = true)
1395
- WHERE date >= ? AND date <= ?
1396
- GROUP BY ${keyCol}
1397
- HAVING SUM(clicks) > 0
1398
- ),
1399
- totals AS (
1400
- SELECT SUM(clicks) AS total_clicks, COUNT(*) AS total_items FROM items
1401
- ),
1402
- ranked AS (
1403
- SELECT
1404
- i.key, i.clicks,
1405
- i.clicks / NULLIF(t.total_clicks, 0) AS share,
1406
- ROW_NUMBER() OVER (ORDER BY i.clicks DESC, i.key ASC) AS rnk_desc,
1407
- ROW_NUMBER() OVER (ORDER BY i.clicks ASC, i.key ASC) AS rnk_asc,
1408
- t.total_clicks AS tclicks,
1409
- t.total_items AS titems
1410
- FROM items i, totals t
1411
- ),
1412
- gini_num AS (
1413
- SELECT SUM((2.0 * rnk_asc - titems - 1) * clicks) AS weighted_sum FROM ranked
1414
- ),
1415
- hhi_calc AS (
1416
- SELECT SUM(POWER(share * 100, 2)) AS hhi FROM ranked
1417
- ),
1418
- top_list AS (
1419
- SELECT
1420
- list({ 'key': key, 'clicks': clicks, 'share': share } ORDER BY clicks DESC, key ASC) AS items,
1421
- SUM(clicks) AS top_clicks
1422
- FROM ranked WHERE rnk_desc <= ?
1423
- )
1424
- SELECT
1425
- COALESCE(
1426
- (SELECT weighted_sum FROM gini_num)
1427
- / NULLIF((SELECT total_items FROM totals) * (SELECT total_clicks FROM totals), 0),
1428
- 0.0
1429
- ) AS giniCoefficient,
1430
- COALESCE((SELECT hhi FROM hhi_calc), 0.0) AS hhi,
1431
- COALESCE(
1432
- CAST((SELECT top_clicks FROM top_list) AS DOUBLE)
1433
- / NULLIF((SELECT total_clicks FROM totals), 0),
1434
- 0.0
1435
- ) AS topNConcentration,
1436
- COALESCE((SELECT to_json(items) FROM top_list), '[]') AS topNItems,
1437
- COALESCE((SELECT total_items FROM totals), 0) AS totalItems,
1438
- COALESCE((SELECT total_clicks FROM totals), 0.0) AS totalClicks,
1439
- CASE
1440
- WHEN COALESCE((SELECT hhi FROM hhi_calc), 0.0) > 2500 THEN 'high'
1441
- WHEN COALESCE((SELECT hhi FROM hhi_calc), 0.0) > 1500 THEN 'medium'
1442
- ELSE 'low'
1443
- END AS riskLevel
1444
- `,
1445
- params: [
1446
- startDate,
1447
- endDate,
1448
- topN
1449
- ],
1450
- current: {
1451
- table,
1452
- partitions: enumeratePartitions(startDate, endDate)
1453
- }
1454
- };
1455
- },
1456
- reduceSql(rows, params) {
1457
- const r = (Array.isArray(rows) ? rows : [])[0] ?? {};
1458
- const topRaw = parseJsonRows(r.topNItems);
1459
- return {
1460
- results: [{
1461
- giniCoefficient: num(r.giniCoefficient),
1462
- hhi: num(r.hhi),
1463
- topNConcentration: num(r.topNConcentration),
1464
- topNItems: topRaw.map((t) => ({
1465
- key: rowString(t.key),
1466
- clicks: num(t.clicks),
1467
- share: num(t.share)
1468
- })),
1469
- totalItems: num(r.totalItems),
1470
- totalClicks: num(r.totalClicks),
1471
- riskLevel: rowString(r.riskLevel)
1472
- }],
1473
- meta: {
1474
- total: 1,
1475
- dimension: params.dimension || "pages"
1476
- }
1477
- };
1478
- },
1479
- buildRows(params) {
1480
- const dim = params.dimension || "pages";
1481
- const period = periodOf(params);
1482
- const out = {};
1483
- if (dim === "pages") out.pages = pagesQueryState(period, params.limit);
1484
- else out.queries = queriesQueryState(period, params.limit);
1485
- return out;
1486
- },
1487
- reduceRows(rows, params) {
1488
- const dim = params.dimension || "pages";
1489
- const arr = Array.isArray(rows) ? rows : rows[dim === "pages" ? "pages" : "queries"] ?? [];
1490
- return {
1491
- results: [dim === "pages" ? analyzePageConcentration(arr, { topN: params.topN }) : analyzeKeywordConcentration(arr, { topN: params.topN })],
1492
- meta: { dimension: dim }
1493
- };
1494
- }
1495
- });
1496
- const contentVelocityAnalyzer = defineAnalyzer({
1497
- id: "content-velocity",
1498
- buildSql(params) {
1499
- const days = Math.min(Math.max(Number(params.days ?? 90), 7), 365);
1500
- const { endDate } = periodOf(params);
1501
- const start = new Date(endDate);
1502
- start.setUTCDate(start.getUTCDate() - days);
1503
- const startDate = toIsoDate(start);
1504
- return {
1505
- sql: `
1506
- WITH src AS (
1507
- SELECT query, date
1508
- FROM read_parquet({{FILES}}, union_by_name = true)
1509
- WHERE date >= ? AND date <= ? AND impressions > 0
1510
- ),
1511
- first_seen AS (
1512
- SELECT query, MIN(date) AS first_date FROM src GROUP BY query
1513
- ),
1514
- per_week AS (
1515
- SELECT
1516
- strftime(CAST(date AS DATE), '%G-W%V') AS week,
1517
- MIN(date) AS week_start,
1518
- CAST(COUNT(DISTINCT query) AS DOUBLE) AS totalKeywords
1519
- FROM src
1520
- GROUP BY week
1521
- ),
1522
- new_per_week AS (
1523
- SELECT
1524
- strftime(CAST(first_date AS DATE), '%G-W%V') AS week,
1525
- CAST(COUNT(*) AS DOUBLE) AS newKeywords
1526
- FROM first_seen
1527
- GROUP BY week
1528
- )
1529
- SELECT
1530
- pw.week AS week,
1531
- COALESCE(npw.newKeywords, 0) AS newKeywords,
1532
- pw.totalKeywords AS totalKeywords
1533
- FROM per_week pw
1534
- LEFT JOIN new_per_week npw ON pw.week = npw.week
1535
- ORDER BY pw.week ASC
1536
- `,
1537
- params: [startDate, endDate],
1538
- current: {
1539
- table: "queries",
1540
- partitions: enumeratePartitions(startDate, endDate)
1541
- }
1542
- };
1543
- },
1544
- reduceSql(rows, params) {
1545
- const arr = Array.isArray(rows) ? rows : [];
1546
- const days = Math.min(Math.max(Number(params.days ?? 90), 7), 365);
1547
- const { endDate } = periodOf(params);
1548
- const startDateD = new Date(endDate);
1549
- startDateD.setUTCDate(startDateD.getUTCDate() - days);
1550
- const startDate = toIsoDate(startDateD);
1551
- const weekly = arr.map((r) => ({
1552
- week: rowString(r.week),
1553
- newKeywords: rowNumber(r.newKeywords),
1554
- totalKeywords: rowNumber(r.totalKeywords)
1555
- }));
1556
- const total = weekly.reduce((s, w) => s + w.newKeywords, 0);
1557
- const avg = weekly.length > 0 ? total / weekly.length : 0;
1558
- const mid = Math.floor(weekly.length / 2);
1559
- const firstAvg = mid > 0 ? weekly.slice(0, mid).reduce((s, w) => s + w.newKeywords, 0) / mid : 0;
1560
- const diff = (weekly.length - mid > 0 ? weekly.slice(mid).reduce((s, w) => s + w.newKeywords, 0) / (weekly.length - mid) : 0) - firstAvg;
1561
- const threshold = Math.max(1, avg * .15);
1562
- return {
1563
- results: weekly,
1564
- meta: {
1565
- summary: {
1566
- totalNewKeywords: total,
1567
- avgPerWeek: avg,
1568
- trend: diff > threshold ? "accelerating" : diff < -threshold ? "decelerating" : "stable"
1569
- },
1570
- days,
1571
- startDate,
1572
- endDate
1573
- }
1574
- };
1575
- }
1576
- });
1577
- const ctrAnomalyAnalyzer = defineAnalyzer({
1578
- id: "ctr-anomaly",
1579
- buildSql(params) {
1580
- const endDate = params.endDate ?? defaultEndDate();
1581
- const startDate = params.startDate ?? daysAgoUtc(93);
1582
- const minDailyImpressions = params.minImpressions ?? 5;
1583
- const minRollingN = 14;
1584
- const zThreshold = params.threshold ?? 2;
1585
- const maxPositionDelta = 1.5;
1586
- const minBreachDays = 2;
1587
- const limit = params.limit ?? 200;
1588
- return {
1589
- sql: `
1590
- WITH daily AS (
1591
- SELECT
1592
- query,
1593
- url AS page,
1594
- -- Normalize at the source CTE: union_by_name=true can coerce date to
1595
- -- VARCHAR across parquets with mixed schemas, which makes downstream
1596
- -- strftime(date, ...) binder-error.
1597
- CAST(date AS DATE) AS date,
1598
- ${METRIC_EXPR.clicks} AS day_clicks,
1599
- ${METRIC_EXPR.impressions} AS day_impressions,
1600
- ${METRIC_EXPR.ctr} AS day_ctr,
1601
- ${METRIC_EXPR.position} AS day_position
1602
- FROM read_parquet({{FILES}}, union_by_name = true)
1603
- WHERE date >= ? AND date <= ?
1604
- AND query IS NOT NULL AND query <> ''
1605
- AND url IS NOT NULL AND url <> ''
1606
- GROUP BY query, url, date
1607
- HAVING SUM(impressions) >= ?
1608
- ),
1609
- rolled AS (
1610
- SELECT *,
1611
- AVG(day_ctr) OVER w AS rolling_ctr,
1612
- STDDEV_POP(day_ctr) OVER w AS rolling_stddev,
1613
- -- Impression-weighted rolling mean: day_position is already a recovered
1614
- -- 1-based daily weighted mean, so weighting by day impressions directly
1615
- -- is algebraically exact (no -1/+1 dance needed).
1616
- SUM(day_position * day_impressions) OVER w
1617
- / NULLIF(SUM(day_impressions) OVER w, 0) AS rolling_position,
1618
- COUNT(*) OVER w AS rolling_n
1619
- FROM daily
1620
- WINDOW w AS (
1621
- PARTITION BY query, page
1622
- ORDER BY date
1623
- ROWS BETWEEN 28 PRECEDING AND 1 PRECEDING
1624
- )
1625
- ),
1626
- flagged AS (
1627
- SELECT *,
1628
- CASE
1629
- WHEN rolling_n >= ${Number(minRollingN)} AND rolling_stddev > 0
1630
- THEN (day_ctr - rolling_ctr) / rolling_stddev
1631
- ELSE 0.0
1632
- END AS z_score,
1633
- CASE
1634
- WHEN rolling_position IS NULL THEN 0.0
1635
- ELSE ABS(day_position - rolling_position)
1636
- END AS position_delta
1637
- FROM rolled
1638
- ),
1639
- breaches AS (
1640
- SELECT *,
1641
- CASE
1642
- WHEN ABS(z_score) >= ${zThreshold}
1643
- AND position_delta <= ${maxPositionDelta}
1644
- AND rolling_n >= ${Number(minRollingN)}
1645
- THEN true ELSE false
1646
- END AS is_breach
1647
- FROM flagged
1648
- ),
1649
- per_entity AS (
1650
- SELECT
1651
- query, page,
1652
- COUNT(*) FILTER (WHERE is_breach AND z_score < 0) AS breach_days_down,
1653
- COUNT(*) FILTER (WHERE is_breach AND z_score > 0) AS breach_days_up,
1654
- SUM(CASE
1655
- WHEN is_breach AND z_score < 0
1656
- THEN (rolling_ctr - day_ctr) * day_impressions
1657
- ELSE 0.0
1658
- END) AS clicks_lost,
1659
- SUM(CASE
1660
- WHEN is_breach AND z_score < 0
1661
- THEN ABS(z_score) * day_impressions
1662
- ELSE 0.0
1663
- END) AS severity_raw,
1664
- MAX(CASE WHEN is_breach THEN ABS(z_score) ELSE 0.0 END) AS max_z,
1665
- AVG(rolling_ctr) FILTER (WHERE rolling_n >= ${Number(minRollingN)}) AS baseline_ctr,
1666
- SUM(rolling_position * day_impressions) FILTER (WHERE rolling_n >= ${Number(minRollingN)})
1667
- / NULLIF(SUM(day_impressions) FILTER (WHERE rolling_n >= ${Number(minRollingN)}), 0) AS baseline_position,
1668
- SUM(day_impressions) AS total_impressions,
1669
- SUM(day_clicks) AS total_clicks
1670
- FROM breaches
1671
- GROUP BY query, page
1672
- HAVING COUNT(*) FILTER (WHERE is_breach AND z_score < 0) >= ${Number(minBreachDays)}
1673
- ),
1674
- series AS (
1675
- SELECT query, page,
1676
- to_json(list({
1677
- 'date': strftime(date, '%Y-%m-%d'),
1678
- 'ctr': day_ctr,
1679
- 'position': day_position,
1680
- 'impressions': day_impressions,
1681
- 'rollingCtr': rolling_ctr,
1682
- 'rollingStddev': rolling_stddev,
1683
- 'z': z_score,
1684
- 'breach': is_breach AND z_score < 0
1685
- } ORDER BY date)) AS seriesJson
1686
- FROM breaches
1687
- GROUP BY query, page
1688
- )
1689
- SELECT
1690
- e.query AS keyword,
1691
- e.page,
1692
- CAST(e.breach_days_down AS DOUBLE) AS breachDaysDown,
1693
- CAST(e.breach_days_up AS DOUBLE) AS breachDaysUp,
1694
- CAST(ROUND(e.clicks_lost) AS DOUBLE) AS clicksLost,
1695
- e.severity_raw AS severityRaw,
1696
- e.max_z AS maxZ,
1697
- e.baseline_ctr AS baselineCtr,
1698
- e.baseline_position AS baselinePosition,
1699
- e.total_impressions AS totalImpressions,
1700
- e.total_clicks AS totalClicks,
1701
- s.seriesJson
1702
- FROM per_entity e
1703
- LEFT JOIN series s USING (query, page)
1704
- ORDER BY clicksLost DESC
1705
- LIMIT ${Number(limit)}
1706
- `,
1707
- params: [
1708
- startDate,
1709
- endDate,
1710
- minDailyImpressions
1711
- ],
1712
- current: {
1713
- table: "page_queries",
1714
- partitions: enumeratePartitions(startDate, endDate)
1715
- }
1716
- };
1717
- },
1718
- reduceSql(rows, params) {
1719
- const arr = Array.isArray(rows) ? rows : [];
1720
- const minRollingN = 14;
1721
- const zThreshold = params.threshold ?? 2;
1722
- const anomalies = arr.map((r) => ({
1723
- keyword: rowString(r.keyword),
1724
- page: rowString(r.page),
1725
- breachDaysDown: rowNumber(r.breachDaysDown),
1726
- breachDaysUp: rowNumber(r.breachDaysUp),
1727
- clicksLost: rowNumber(r.clicksLost),
1728
- severity: rowNumber(r.severityRaw),
1729
- maxZ: rowNumber(r.maxZ),
1730
- baselineCtr: rowNumber(r.baselineCtr),
1731
- baselinePosition: rowNumber(r.baselinePosition),
1732
- totalImpressions: rowNumber(r.totalImpressions),
1733
- totalClicks: rowNumber(r.totalClicks),
1734
- series: parseJsonRows(r.seriesJson).map((s) => ({
1735
- date: rowString(s.date),
1736
- ctr: rowNumber(s.ctr),
1737
- position: rowNumber(s.position),
1738
- impressions: rowNumber(s.impressions),
1739
- rollingCtr: s.rollingCtr == null ? null : rowNumber(s.rollingCtr),
1740
- rollingStddev: s.rollingStddev == null ? null : rowNumber(s.rollingStddev),
1741
- z: rowNumber(s.z),
1742
- breach: rowBoolean(s.breach)
1743
- }))
1744
- }));
1745
- const totalClicksLost = anomalies.reduce((s, a) => s + a.clicksLost, 0);
1746
- const totalBreachDays = anomalies.reduce((s, a) => s + a.breachDaysDown, 0);
1747
- return {
1748
- results: anomalies,
1749
- meta: {
1750
- total: anomalies.length,
1751
- totalClicksLost,
1752
- totalBreachDays,
1753
- zThreshold,
1754
- minRollingN
1755
- }
1756
- };
1757
- }
1758
- });
1759
- const ctrCurveAnalyzer = defineAnalyzer({
1760
- id: "ctr-curve",
1761
- buildSql(params) {
1762
- const { startDate, endDate } = periodOf(params);
1763
- return {
1764
- sql: `
1765
- WITH src AS (
1766
- SELECT
1767
- query,
1768
- clicks,
1769
- impressions,
1770
- sum_position,
1771
- (sum_position / NULLIF(impressions, 0) + 1) AS avg_pos
1772
- FROM read_parquet({{FILES}}, union_by_name = true)
1773
- WHERE date >= ? AND date <= ? AND impressions > 0
1774
- ),
1775
- curve AS (
1776
- SELECT
1777
- CASE
1778
- WHEN avg_pos <= 1.5 THEN '1'
1779
- WHEN avg_pos <= 2.5 THEN '2'
1780
- WHEN avg_pos <= 3.5 THEN '3'
1781
- WHEN avg_pos <= 5.5 THEN '4-5'
1782
- WHEN avg_pos <= 10.5 THEN '6-10'
1783
- WHEN avg_pos <= 20.5 THEN '11-20'
1784
- ELSE '20+'
1785
- END AS bucket,
1786
- AVG(CAST(clicks AS DOUBLE) / NULLIF(impressions, 0)) AS avgCtr,
1787
- (SUM(sum_position) / NULLIF(SUM(impressions), 0) + 1) AS medianPosition,
1788
- CAST(COUNT(DISTINCT query) AS DOUBLE) AS keywordCount,
1789
- ${METRIC_EXPR.clicks} AS totalClicks,
1790
- ${METRIC_EXPR.impressions} AS totalImpressions
1791
- FROM src
1792
- GROUP BY bucket
1793
- ),
1794
- ks AS (
1795
- SELECT
1796
- query,
1797
- ${METRIC_EXPR.clicks} AS clicks,
1798
- ${METRIC_EXPR.impressions} AS impressions,
1799
- ${METRIC_EXPR.ctr} AS ctr,
1800
- ${METRIC_EXPR.position} AS position,
1801
- CASE
1802
- WHEN ${METRIC_EXPR.position} <= 3.5 THEN 'top3'
1803
- WHEN ${METRIC_EXPR.position} <= 10.5 THEN 'page1'
1804
- WHEN ${METRIC_EXPR.position} <= 20.5 THEN 'page2'
1805
- ELSE 'deep'
1806
- END AS band
1807
- FROM src
1808
- GROUP BY query
1809
- HAVING SUM(impressions) >= 20
1810
- ),
1811
- band_avg AS (
1812
- SELECT band, AVG(ctr) AS band_avg_ctr FROM ks GROUP BY band
1813
- ),
1814
- outliers AS (
1815
- SELECT
1816
- ks.query, ks.clicks, ks.impressions, ks.ctr, ks.position,
1817
- ba.band_avg_ctr AS expectedCtr,
1818
- ks.ctr - ba.band_avg_ctr AS ctrDiff
1819
- FROM ks JOIN band_avg ba ON ks.band = ba.band
1820
- ORDER BY ABS(ks.ctr - ba.band_avg_ctr) DESC
1821
- LIMIT 50
1822
- )
1823
- SELECT
1824
- (SELECT to_json(list({
1825
- 'bucket': bucket,
1826
- 'avgCtr': avgCtr,
1827
- 'medianPosition': medianPosition,
1828
- 'keywordCount': keywordCount,
1829
- 'totalClicks': totalClicks,
1830
- 'totalImpressions': totalImpressions
1831
- })) FROM curve) AS curve_json,
1832
- (SELECT to_json(list({
1833
- 'query': query,
1834
- 'clicks': clicks,
1835
- 'impressions': impressions,
1836
- 'ctr': ctr,
1837
- 'position': position,
1838
- 'expectedCtr': expectedCtr,
1839
- 'ctrDiff': ctrDiff
1840
- })) FROM outliers) AS outliers_json
1841
- `,
1842
- params: [startDate, endDate],
1843
- current: {
1844
- table: "queries",
1845
- partitions: enumeratePartitions(startDate, endDate)
1846
- }
1847
- };
1848
- },
1849
- reduceSql(rows, params) {
1850
- const arr = Array.isArray(rows) ? rows : [];
1851
- const { startDate, endDate } = periodOf(params);
1852
- const row = arr[0] ?? {};
1853
- const curve = parseJsonRows(row.curve_json).map((r) => ({
1854
- bucket: rowString(r.bucket),
1855
- avgCtr: rowNumber(r.avgCtr),
1856
- medianPosition: rowNumber(r.medianPosition),
1857
- keywordCount: rowNumber(r.keywordCount),
1858
- totalClicks: rowNumber(r.totalClicks),
1859
- totalImpressions: rowNumber(r.totalImpressions)
1860
- }));
1861
- const outliers = parseJsonRows(row.outliers_json).map((r) => ({
1862
- query: rowString(r.query),
1863
- clicks: rowNumber(r.clicks),
1864
- impressions: rowNumber(r.impressions),
1865
- ctr: rowNumber(r.ctr),
1866
- position: rowNumber(r.position),
1867
- expectedCtr: rowNumber(r.expectedCtr),
1868
- ctrDiff: rowNumber(r.ctrDiff)
1869
- }));
1870
- return {
1871
- results: curve,
1872
- meta: {
1873
- overperforming: outliers.filter((o) => o.ctrDiff > 0).slice(0, 25),
1874
- underperforming: outliers.filter((o) => o.ctrDiff < 0).slice(0, 25),
1875
- startDate,
1876
- endDate
1877
- }
1878
- };
1879
- }
1880
- });
1881
- const darkTrafficAnalyzer = defineAnalyzer({
1882
- id: "dark-traffic",
1883
- buildSql(params) {
1884
- const { startDate, endDate } = periodOf(params);
1885
- return {
1886
- sql: `
1887
- WITH page_totals AS (
1888
- SELECT SUM(clicks) AS total_clicks, SUM(impressions) AS total_impressions
1889
- FROM read_parquet({{FILES}}, union_by_name = true)
1890
- WHERE date >= ? AND date <= ?
1891
- ),
1892
- kw_totals AS (
1893
- SELECT SUM(clicks) AS total_clicks, SUM(impressions) AS total_impressions
1894
- FROM read_parquet({{FILES_KEYWORDS}}, union_by_name = true)
1895
- WHERE date >= ? AND date <= ?
1896
- ),
1897
- per_page AS (
1898
- SELECT url, SUM(clicks) AS page_clicks
1899
- FROM read_parquet({{FILES}}, union_by_name = true)
1900
- WHERE date >= ? AND date <= ?
1901
- GROUP BY url
1902
- HAVING SUM(clicks) > 0
1903
- ),
1904
- per_page_kw AS (
1905
- SELECT url, SUM(clicks) AS attributed_clicks, COUNT(DISTINCT query) AS kw_count
1906
- FROM read_parquet({{FILES_PAGE_KEYWORDS}}, union_by_name = true)
1907
- WHERE date >= ? AND date <= ?
1908
- GROUP BY url
1909
- ),
1910
- page_rows AS (
1911
- SELECT
1912
- p.url AS url,
1913
- CAST(p.page_clicks AS DOUBLE) AS totalClicks,
1914
- CAST(COALESCE(k.attributed_clicks, 0) AS DOUBLE) AS attributedClicks,
1915
- CAST(p.page_clicks - COALESCE(k.attributed_clicks, 0) AS DOUBLE) AS darkClicks,
1916
- CAST(p.page_clicks - COALESCE(k.attributed_clicks, 0) AS DOUBLE)
1917
- / NULLIF(p.page_clicks, 0) AS darkPercent,
1918
- CAST(COALESCE(k.kw_count, 0) AS DOUBLE) AS keywordCount
1919
- FROM per_page p
1920
- LEFT JOIN per_page_kw k ON p.url = k.url
1921
- WHERE p.page_clicks - COALESCE(k.attributed_clicks, 0) > 0
1922
- ORDER BY darkClicks DESC
1923
- LIMIT 50
1924
- )
1925
- SELECT
1926
- (SELECT to_json({
1927
- 'totalClicks': CAST(total_clicks AS DOUBLE),
1928
- 'totalImpressions': CAST(total_impressions AS DOUBLE)
1929
- }) FROM page_totals) AS page_totals_json,
1930
- (SELECT to_json({
1931
- 'attributedClicks': CAST(total_clicks AS DOUBLE),
1932
- 'attributedImpressions': CAST(total_impressions AS DOUBLE)
1933
- }) FROM kw_totals) AS kw_totals_json,
1934
- (SELECT to_json(list({
1935
- 'url': url,
1936
- 'totalClicks': totalClicks,
1937
- 'attributedClicks': attributedClicks,
1938
- 'darkClicks': darkClicks,
1939
- 'darkPercent': darkPercent,
1940
- 'keywordCount': keywordCount
1941
- })) FROM page_rows) AS pages_json
1942
- `,
1943
- params: [
1944
- startDate,
1945
- endDate,
1946
- startDate,
1947
- endDate,
1948
- startDate,
1949
- endDate,
1950
- startDate,
1951
- endDate
1952
- ],
1953
- current: {
1954
- table: "pages",
1955
- partitions: enumeratePartitions(startDate, endDate)
1956
- },
1957
- extraFiles: {
1958
- KEYWORDS: {
1959
- table: "queries",
1960
- partitions: enumeratePartitions(startDate, endDate)
1961
- },
1962
- PAGE_KEYWORDS: {
1963
- table: "page_queries",
1964
- partitions: enumeratePartitions(startDate, endDate)
1965
- }
1966
- }
1967
- };
1968
- },
1969
- reduceSql(rows, params) {
1970
- const arr = Array.isArray(rows) ? rows : [];
1971
- const { startDate, endDate } = periodOf(params);
1972
- const row = arr[0] ?? {};
1973
- const pageTotals = typeof row.page_totals_json === "string" ? JSON.parse(row.page_totals_json) : row.page_totals_json ?? {};
1974
- const kwTotals = typeof row.kw_totals_json === "string" ? JSON.parse(row.kw_totals_json) : row.kw_totals_json ?? {};
1975
- const totalClicks = num(pageTotals.totalClicks);
1976
- const totalImpressions = num(pageTotals.totalImpressions);
1977
- const attributedClicks = num(kwTotals.attributedClicks);
1978
- const attributedImpressions = num(kwTotals.attributedImpressions);
1979
- const darkClicks = Math.max(0, totalClicks - attributedClicks);
1980
- const darkPercent = totalClicks > 0 ? darkClicks / totalClicks : 0;
1981
- return {
1982
- results: parseJsonRows(row.pages_json).map((r) => ({
1983
- url: rowString(r.url),
1984
- totalClicks: num(r.totalClicks),
1985
- attributedClicks: num(r.attributedClicks),
1986
- darkClicks: num(r.darkClicks),
1987
- darkPercent: num(r.darkPercent),
1988
- keywordCount: num(r.keywordCount)
1989
- })),
1990
- meta: {
1991
- summary: {
1992
- totalClicks,
1993
- attributedClicks,
1994
- darkClicks,
1995
- darkPercent,
1996
- totalImpressions,
1997
- attributedImpressions
1998
- },
1999
- startDate,
2000
- endDate
2001
- }
2002
- };
2003
- }
2004
- });
2005
- function requireBuilderState(input, tool) {
2006
- if (!input || typeof input !== "object" || !("dimensions" in input) || !Array.isArray(input.dimensions)) throw new Error(`${tool}: params.q is required (BuilderState)`);
2007
- return input;
2008
- }
2009
- function optionalBuilderState(input, tool, key) {
2010
- if (input == null) return null;
2011
- if (typeof input !== "object" || !("dimensions" in input) || !Array.isArray(input.dimensions)) throw new Error(`${tool}: params.${key} must be a BuilderState`);
2012
- return input;
2013
- }
2014
- const NUMERIC_METRIC_COLS = [
2015
- "clicks",
2016
- "impressions",
2017
- "ctr",
2018
- "position",
2019
- "prevClicks",
2020
- "prevImpressions",
2021
- "prevCtr",
2022
- "prevPosition",
2023
- "variantCount",
2024
- "totalCount"
2025
- ];
2026
- function coerceNumericCols(row) {
2027
- const out = { ...row };
2028
- for (const col of NUMERIC_METRIC_COLS) if (col in out && out[col] != null) out[col] = Number(out[col]);
2029
- return out;
2030
- }
2031
- function shapeDataQuery(rows, extras, opts) {
2032
- let totalCount;
2033
- let cleaned;
2034
- let totals;
2035
- if (opts.hasPrev) {
2036
- cleaned = rows.map(coerceNumericCols);
2037
- totalCount = Number((extras?.count?.[0])?.total ?? cleaned.length);
2038
- const totalsRow = extras?.totals?.[0] ?? {};
2039
- totals = {
2040
- clicks: Number(totalsRow.clicks ?? 0),
2041
- impressions: Number(totalsRow.impressions ?? 0),
2042
- ctr: Number(totalsRow.ctr ?? 0),
2043
- position: Number(totalsRow.position ?? 0)
2044
- };
2045
- } else {
2046
- const first = rows[0];
2047
- totalCount = Number(first?.totalCount ?? 0);
2048
- totals = {
2049
- clicks: Number(first?.totalClicks ?? 0),
2050
- impressions: Number(first?.totalImpressions ?? 0),
2051
- ctr: Number(first?.totalCtr ?? 0),
2052
- position: Number(first?.totalPosition ?? 0)
2053
- };
2054
- cleaned = rows.map((raw) => {
2055
- const { totalCount: _tc, totalClicks: _tclk, totalImpressions: _timp, totalCtr: _tctr, totalPosition: _tpos, sum_position: _sp, ...rest } = raw;
2056
- return coerceNumericCols(rest);
2057
- });
2058
- }
2059
- const extrasResults = [];
2060
- if (extras?.canonicalExtras) extrasResults.push({
2061
- key: "canonicalExtras",
2062
- results: extras.canonicalExtras
2063
- });
2064
- return {
2065
- results: mergeExtras(cleaned, extrasResults),
2066
- meta: {
2067
- totalCount,
2068
- totals
2069
- }
2070
- };
2071
- }
2072
- function buildDataQueryPlan(params, options) {
2073
- const state = requireBuilderState(params.q, "data-query");
2074
- if (state.dimensions.includes("date")) throw new Error("data-query: date dimension not supported; use data-detail");
2075
- const prev = optionalBuilderState(params.qc, "data-query", "qc");
2076
- const extraQueries = [...buildExtrasQueries(state, options).map((extra) => ({
2077
- name: extra.key,
2078
- sql: extra.sql,
2079
- params: extra.params
2080
- }))];
2081
- const tableKey = options.adapter.inferTable(state.dimensions);
2082
- if (prev) {
2083
- const totals = buildTotalsSql(state, options);
2084
- const comparison = resolveComparisonSQL(state, prev, options, params.comparisonFilter);
2085
- extraQueries.unshift({
2086
- name: "totals",
2087
- sql: totals.sql,
2088
- params: totals.params
2089
- });
2090
- extraQueries.push({
2091
- name: "count",
2092
- sql: comparison.countSql,
2093
- params: comparison.countParams
2094
- });
2095
- return {
2096
- tableKey,
2097
- sql: comparison.sql,
2098
- params: comparison.params,
2099
- extraQueries
2100
- };
2101
- }
2102
- const optimized = resolveToSQLOptimized(state, options);
2103
- return {
2104
- tableKey,
2105
- sql: optimized.sql,
2106
- params: optimized.params,
2107
- extraQueries
2108
- };
2109
- }
2110
- function totalsState(state) {
2111
- return {
2112
- ...state,
2113
- dimensions: [],
2114
- orderBy: void 0,
2115
- rowLimit: 1
2116
- };
2117
- }
2118
- function readTotals(row) {
2119
- return {
2120
- clicks: Number(row?.clicks ?? 0),
2121
- impressions: Number(row?.impressions ?? 0),
2122
- ctr: Number(row?.ctr ?? 0),
2123
- position: Number(row?.position ?? 0)
2124
- };
2125
- }
2126
- function buildDataQueryRows(params) {
2127
- const state = requireBuilderState(params.q, "data-query");
2128
- if (state.dimensions.includes("date")) throw new Error("data-query: date dimension not supported; use data-detail");
2129
- const queries = {
2130
- main: state,
2131
- totals: totalsState(state)
2132
- };
2133
- const prev = optionalBuilderState(params.qc, "data-query", "qc");
2134
- if (prev) {
2135
- queries.prevMain = prev;
2136
- queries.prevTotals = totalsState(prev);
2137
- }
2138
- return queries;
2139
- }
2140
- function shapeDataQueryRowResults(rowMap, params) {
2141
- const main = (rowMap.main ?? []).map((r) => coerceNumericCols(r));
2142
- const totals = readTotals(rowMap.totals?.[0]);
2143
- if (params.qc == null) return {
2144
- results: main,
2145
- meta: {
2146
- totalCount: main.length,
2147
- totals
2148
- }
2149
- };
2150
- const state = requireBuilderState(params.q, "data-query");
2151
- const dims = state.dimensions;
2152
- const keyOf = (row) => dims.map((d) => String(row[d] ?? "")).join("");
2153
- const prevByKey = /* @__PURE__ */ new Map();
2154
- for (const r of rowMap.prevMain ?? []) prevByKey.set(keyOf(r), r);
2155
- const filter = params.comparisonFilter;
2156
- const merged = [];
2157
- const seen = /* @__PURE__ */ new Set();
2158
- for (const cur of main) {
2159
- const key = keyOf(cur);
2160
- seen.add(key);
2161
- const prev = prevByKey.get(key);
2162
- const row = {
2163
- ...cur,
2164
- prevClicks: Number(prev?.clicks ?? 0),
2165
- prevImpressions: Number(prev?.impressions ?? 0),
2166
- prevCtr: Number(prev?.ctr ?? 0),
2167
- prevPosition: Number(prev?.position ?? 0)
2168
- };
2169
- if (passesComparisonFilter(filter, {
2170
- isNew: !prev,
2171
- isLost: false,
2172
- clicksChange: Number(cur.clicks ?? 0) - Number(prev?.clicks ?? 0)
2173
- })) merged.push(row);
2174
- }
2175
- for (const prev of rowMap.prevMain ?? []) {
2176
- const p = prev;
2177
- const key = keyOf(p);
2178
- if (seen.has(key)) continue;
2179
- const row = {
2180
- ...p,
2181
- clicks: 0,
2182
- impressions: 0,
2183
- ctr: 0,
2184
- position: 0,
2185
- prevClicks: Number(p.clicks ?? 0),
2186
- prevImpressions: Number(p.impressions ?? 0),
2187
- prevCtr: Number(p.ctr ?? 0),
2188
- prevPosition: Number(p.position ?? 0)
2189
- };
2190
- if (passesComparisonFilter(filter, {
2191
- isNew: false,
2192
- isLost: true,
2193
- clicksChange: -Number(p.clicks ?? 0)
2194
- })) merged.push(row);
2195
- }
2196
- if (state.orderBy) {
2197
- const { column, dir } = state.orderBy;
2198
- merged.sort((a, b) => {
2199
- const av = Number(a[column]) || 0;
2200
- const bv = Number(b[column]) || 0;
2201
- return dir === "asc" ? av - bv : bv - av;
2202
- });
2203
- }
2204
- return {
2205
- results: merged.map((r) => coerceNumericCols(r)),
2206
- meta: {
2207
- totalCount: merged.length,
2208
- totals
2209
- }
2210
- };
2211
- }
2212
- function passesComparisonFilter(filter, ctx) {
2213
- switch (filter) {
2214
- case "new": return ctx.isNew;
2215
- case "lost": return ctx.isLost;
2216
- case "improving": return ctx.clicksChange > 0;
2217
- case "declining": return ctx.clicksChange < 0;
2218
- default: return true;
2219
- }
2220
- }
2221
- function buildDataDetailRows(params) {
2222
- const state = requireBuilderState(params.q, "data-detail");
2223
- if (!state.dimensions.includes("date")) throw new Error("data-detail: `date` dimension is required");
2224
- const queries = {
2225
- main: state,
2226
- totals: totalsState(state)
2227
- };
2228
- const prev = optionalBuilderState(params.qc, "data-detail", "qc");
2229
- if (prev) queries.prevTotals = totalsState(prev);
2230
- return queries;
2231
- }
2232
- function shapeDataDetailRowResults(rowMap, params) {
2233
- const { startDate, endDate } = extractDateRange(requireBuilderState(params.q, "data-detail").filter);
2234
- const coerced = (rowMap.main ?? []).map((r) => coerceNumericCols(r));
2235
- const daily = startDate && endDate ? padTimeseries(coerced, {
2236
- startDate,
2237
- endDate
2238
- }) : coerced;
2239
- const meta = { totals: readTotals(rowMap.totals?.[0]) };
2240
- if (rowMap.prevTotals) meta.previousTotals = readTotals(rowMap.prevTotals[0]);
2241
- return {
2242
- results: daily,
2243
- meta
2244
- };
2245
- }
2246
- function shapeDataQueryRows(rows, params, extras) {
2247
- return shapeDataQuery(rows, extras, { hasPrev: params.qc != null });
2248
- }
2249
- function buildDataDetailPlan(params, options) {
2250
- const state = requireBuilderState(params.q, "data-detail");
2251
- if (!state.dimensions.includes("date")) throw new Error("data-detail: `date` dimension is required");
2252
- const main = resolveToSQLOptimized(state, options);
2253
- const prev = optionalBuilderState(params.qc, "data-detail", "qc");
2254
- const extraQueries = [];
2255
- if (prev) {
2256
- const previousTotals = buildTotalsSql(prev, options);
2257
- extraQueries.push({
2258
- name: "prevTotals",
2259
- sql: previousTotals.sql,
2260
- params: previousTotals.params
2261
- });
2262
- }
2263
- return {
2264
- tableKey: options.adapter.inferTable(state.dimensions),
2265
- sql: main.sql,
2266
- params: main.params,
2267
- extraQueries
2268
- };
2269
- }
2270
- function shapeDataDetailRows(rows, params, extras) {
2271
- const { startDate: rangeStart, endDate: rangeEnd } = extractDateRange(requireBuilderState(params.q, "data-detail").filter);
2272
- const first = rows[0];
2273
- const totals = {
2274
- clicks: Number(first?.totalClicks ?? 0),
2275
- impressions: Number(first?.totalImpressions ?? 0),
2276
- ctr: Number(first?.totalCtr ?? 0),
2277
- position: Number(first?.totalPosition ?? 0)
2278
- };
2279
- const coerced = rows.map((raw) => {
2280
- const { totalCount: _tc, totalClicks: _tclk, totalImpressions: _timp, totalCtr: _tctr, totalPosition: _tpos, sum_position: _sp, ...rest } = raw;
2281
- return coerceNumericCols(rest);
2282
- });
2283
- const daily = rangeStart && rangeEnd ? padTimeseries(coerced, {
2284
- startDate: rangeStart,
2285
- endDate: rangeEnd
2286
- }) : coerced;
2287
- const meta = { totals };
2288
- if (extras?.prevTotals) {
2289
- const previousTotalsRow = extras.prevTotals[0] ?? {};
2290
- meta.previousTotals = {
2291
- clicks: Number(previousTotalsRow.clicks ?? 0),
2292
- impressions: Number(previousTotalsRow.impressions ?? 0),
2293
- ctr: Number(previousTotalsRow.ctr ?? 0),
2294
- position: Number(previousTotalsRow.position ?? 0)
2295
- };
2296
- }
2297
- return {
2298
- results: daily,
2299
- meta
2300
- };
2301
- }
2302
- const dataDetailAnalyzer = defineAnalyzer({
2303
- id: "data-detail",
2304
- sqlRequires: [
2305
- "executeSql",
2306
- "attachedTables",
2307
- "adapter"
2308
- ],
2309
- buildSql(params, ctx) {
2310
- const plan = buildDataDetailPlan(params, {
2311
- adapter: requireAdapter(ctx, "data-detail"),
2312
- siteId: ctx.siteId
2313
- });
2314
- return {
2315
- sql: plan.sql,
2316
- params: plan.params,
2317
- current: {
2318
- table: plan.tableKey,
2319
- partitions: []
2320
- },
2321
- extraQueries: plan.extraQueries
2322
- };
2323
- },
2324
- reduceSql(rows, params, ctx) {
2325
- const { results, meta } = shapeDataDetailRows(Array.isArray(rows) ? rows : [], params, ctx.extras);
2326
- return {
2327
- results,
2328
- meta
2329
- };
2330
- },
2331
- buildRows(params) {
2332
- return buildDataDetailRows(params);
2333
- },
2334
- reduceRows(rows, params) {
2335
- const { results, meta } = shapeDataDetailRowResults(Array.isArray(rows) ? {} : rows, params);
2336
- return {
2337
- results,
2338
- meta
2339
- };
2340
- }
2341
- });
2342
- const dataQueryAnalyzer = defineAnalyzer({
2343
- id: "data-query",
2344
- sqlRequires: [
2345
- "executeSql",
2346
- "attachedTables",
2347
- "adapter"
2348
- ],
2349
- buildSql(params, ctx) {
2350
- const plan = buildDataQueryPlan(params, {
2351
- adapter: requireAdapter(ctx, "data-query"),
2352
- siteId: ctx.siteId
2353
- });
2354
- return {
2355
- sql: plan.sql,
2356
- params: plan.params,
2357
- current: {
2358
- table: plan.tableKey,
2359
- partitions: []
2360
- },
2361
- extraQueries: plan.extraQueries
2362
- };
2363
- },
2364
- reduceSql(rows, params, ctx) {
2365
- const { results, meta } = shapeDataQueryRows(Array.isArray(rows) ? rows : [], params, ctx.extras);
2366
- return {
2367
- results,
2368
- meta
2369
- };
2370
- },
2371
- buildRows(params) {
2372
- return buildDataQueryRows(params);
2373
- },
2374
- reduceRows(rows, params) {
2375
- const { results, meta } = shapeDataQueryRowResults(Array.isArray(rows) ? {} : rows, params);
2376
- return {
2377
- results,
2378
- meta
2379
- };
2380
- }
2381
- });
2382
- const DEFAULT_LIMIT = 1e3;
2383
- const MAX_LIMIT = 5e4;
2384
- function clampLimit(limit, fallback = DEFAULT_LIMIT) {
2385
- const n = Number(limit ?? fallback);
2386
- if (!Number.isFinite(n) || n <= 0) return fallback;
2387
- return Math.min(n, MAX_LIMIT);
2388
- }
2389
- function clampOffset(offset) {
2390
- const n = Number(offset ?? 0);
2391
- if (!Number.isFinite(n) || n < 0) return 0;
2392
- return Math.floor(n);
2393
- }
2394
- function paginateClause(input) {
2395
- const l = clampLimit(input.limit);
2396
- const o = clampOffset(input.offset);
2397
- return o > 0 ? `LIMIT ${l} OFFSET ${o}` : `LIMIT ${l}`;
2398
- }
2399
- function paginateInMemory(rows, input) {
2400
- const l = clampLimit(input.limit, rows.length);
2401
- const o = clampOffset(input.offset);
2402
- return rows.slice(o, o + l);
2403
- }
2404
- function paginateSortedInMemory(rows, input, compare) {
2405
- const limit = clampLimit(input.limit, rows.length);
2406
- const offset = clampOffset(input.offset);
2407
- const selectedCount = Math.min(rows.length, offset + limit);
2408
- if (limit === 0 || offset >= rows.length || selectedCount === 0) return [];
2409
- const fullSort = () => [...rows].sort(compare).slice(offset, offset + limit);
2410
- if (selectedCount >= rows.length / 2) return fullSort();
2411
- let invalidComparison = false;
2412
- const compareStable = (left, right) => {
2413
- const order = compare(left.value, right.value);
2414
- if (Number.isNaN(order)) {
2415
- invalidComparison = true;
2416
- return left.index - right.index;
2417
- }
2418
- return order || left.index - right.index;
2419
- };
2420
- const isWorse = (left, right) => compareStable(left, right) > 0;
2421
- const heap = [];
2422
- const siftUp = (start) => {
2423
- let index = start;
2424
- while (index > 0) {
2425
- const parent = index - 1 >>> 1;
2426
- if (!isWorse(heap[index], heap[parent])) break;
2427
- const swap = heap[parent];
2428
- heap[parent] = heap[index];
2429
- heap[index] = swap;
2430
- index = parent;
2431
- }
2432
- };
2433
- const siftDown = () => {
2434
- let index = 0;
2435
- for (;;) {
2436
- const left = index * 2 + 1;
2437
- if (left >= heap.length) return;
2438
- const right = left + 1;
2439
- let worse = left;
2440
- if (right < heap.length && isWorse(heap[right], heap[left])) worse = right;
2441
- if (!isWorse(heap[worse], heap[index])) return;
2442
- const swap = heap[index];
2443
- heap[index] = heap[worse];
2444
- heap[worse] = swap;
2445
- index = worse;
2446
- }
2447
- };
2448
- for (let index = 0; index < rows.length; index++) {
2449
- const candidate = {
2450
- value: rows[index],
2451
- index
2452
- };
2453
- if (heap.length < selectedCount) {
2454
- heap.push(candidate);
2455
- siftUp(heap.length - 1);
2456
- } else if (compareStable(candidate, heap[0]) < 0) {
2457
- heap[0] = candidate;
2458
- siftDown();
2459
- }
2460
- }
2461
- if (invalidComparison) return fullSort();
2462
- heap.sort(compareStable);
2463
- if (invalidComparison) return fullSort();
2464
- return heap.slice(offset, offset + limit).map((entry) => entry.value);
2465
- }
2466
- const sortResults = createMetricSorter("lostClicks", {
2467
- lostClicks: "desc",
2468
- declinePercent: "desc",
2469
- currentClicks: "asc"
2470
- });
2471
- function analyzeDecay(input, options = {}) {
2472
- const { minPreviousClicks = 50, threshold = .2, sortBy = "lostClicks" } = options;
2473
- const currentMap = buildPeriodMap(input.current, (r) => r.page, (r) => ({
2474
- clicks: num(r.clicks),
2475
- position: num(r.position)
2476
- }));
2477
- const previousMap = buildPeriodMap(input.previous, (r) => r.page, (r) => ({
2478
- clicks: num(r.clicks),
2479
- position: num(r.position)
2480
- }), (r) => num(r.clicks) >= minPreviousClicks);
2481
- const results = [];
2482
- for (const [page, prev] of previousMap) {
2483
- const curr = currentMap.get(page);
2484
- const currentClicks = curr?.clicks ?? 0;
2485
- const currentPosition = curr?.position ?? null;
2486
- const lostClicks = prev.clicks - currentClicks;
2487
- const declinePercent = prev.clicks > 0 ? lostClicks / prev.clicks : 0;
2488
- if (declinePercent >= threshold && lostClicks > 0) results.push({
2489
- page,
2490
- currentClicks,
2491
- previousClicks: prev.clicks,
2492
- lostClicks,
2493
- declinePercent,
2494
- currentPosition,
2495
- previousPosition: prev.position,
2496
- positionDrop: currentPosition != null ? currentPosition - prev.position : null
2497
- });
2498
- }
2499
- return sortResults(results, sortBy);
2500
- }
2501
- const decayAnalyzer = defineAnalyzer({
2502
- id: "decay",
2503
- buildSql(params) {
2504
- const { current: cur, previous: prev } = comparisonOf(params);
2505
- const minPreviousClicks = params.minPreviousClicks ?? 50;
2506
- const threshold = params.threshold ?? .2;
2507
- return {
2508
- sql: `
2509
- WITH cur AS (
2510
- SELECT
2511
- url,
2512
- ${METRIC_EXPR.clicks} AS clicks,
2513
- ${METRIC_EXPR.position} AS position
2514
- FROM read_parquet({{FILES}}, union_by_name = true)
2515
- WHERE date >= ? AND date <= ?
2516
- GROUP BY url
2517
- ),
2518
- prev AS (
2519
- SELECT
2520
- url,
2521
- ${METRIC_EXPR.clicks} AS clicks,
2522
- ${METRIC_EXPR.position} AS position
2523
- FROM read_parquet({{FILES_PREV}}, union_by_name = true)
2524
- WHERE date >= ? AND date <= ?
2525
- GROUP BY url
2526
- HAVING SUM(clicks) >= ?
2527
- ),
2528
- weekly AS (
2529
- SELECT url, date_trunc('week', CAST(date AS DATE)) AS week,
2530
- ${METRIC_EXPR.clicks} AS clicks,
2531
- ${METRIC_EXPR.impressions} AS impressions
2532
- FROM (
2533
- SELECT url, date, clicks, impressions
2534
- FROM read_parquet({{FILES}}, union_by_name = true)
2535
- WHERE date >= ? AND date <= ?
2536
- UNION ALL
2537
- SELECT url, date, clicks, impressions
2538
- FROM read_parquet({{FILES_PREV}}, union_by_name = true)
2539
- WHERE date >= ? AND date <= ?
2540
- )
2541
- GROUP BY url, week
2542
- ),
2543
- series_by_url AS (
2544
- SELECT url, to_json(list({
2545
- 'week': strftime(week, '%Y-%m-%d'),
2546
- 'clicks': clicks,
2547
- 'impressions': impressions
2548
- } ORDER BY week)) AS seriesJson
2549
- FROM weekly GROUP BY url
2550
- ),
2551
- joined AS (
2552
- SELECT
2553
- p.url AS page,
2554
- COALESCE(c.clicks, 0.0) AS currentClicks,
2555
- p.clicks AS previousClicks,
2556
- (p.clicks - COALESCE(c.clicks, 0.0)) AS lostClicks,
2557
- (p.clicks - COALESCE(c.clicks, 0.0)) / NULLIF(p.clicks, 0) AS declinePercent,
2558
- c.position AS currentPosition,
2559
- p.position AS previousPosition,
2560
- CASE WHEN c.position IS NOT NULL THEN (c.position - p.position) ELSE NULL END AS positionDrop,
2561
- s.seriesJson
2562
- FROM prev p
2563
- LEFT JOIN cur c ON p.url = c.url
2564
- LEFT JOIN series_by_url s ON p.url = s.url
2565
- )
2566
- SELECT *
2567
- FROM joined
2568
- WHERE declinePercent >= ? AND lostClicks > 0
2569
- ORDER BY lostClicks DESC
2570
- LIMIT 2000
2571
- `,
2572
- params: [
2573
- cur.startDate,
2574
- cur.endDate,
2575
- prev.startDate,
2576
- prev.endDate,
2577
- minPreviousClicks,
2578
- cur.startDate,
2579
- cur.endDate,
2580
- prev.startDate,
2581
- prev.endDate,
2582
- threshold
2583
- ],
2584
- current: {
2585
- table: "pages",
2586
- partitions: enumeratePartitions(cur.startDate, cur.endDate)
2587
- },
2588
- previous: {
2589
- table: "pages",
2590
- partitions: enumeratePartitions(prev.startDate, prev.endDate)
2591
- }
2592
- };
2593
- },
2594
- reduceSql(rows, params) {
2595
- const mapped = (Array.isArray(rows) ? rows : []).map((r) => ({
2596
- page: rowString(r.page),
2597
- currentClicks: num(r.currentClicks),
2598
- previousClicks: num(r.previousClicks),
2599
- lostClicks: num(r.lostClicks),
2600
- declinePercent: num(r.declinePercent),
2601
- currentPosition: r.currentPosition == null ? null : num(r.currentPosition),
2602
- previousPosition: num(r.previousPosition),
2603
- positionDrop: r.positionDrop == null ? null : num(r.positionDrop),
2604
- series: parseJsonRows(r.seriesJson).map((s) => ({
2605
- week: rowString(s.week),
2606
- clicks: num(s.clicks),
2607
- impressions: num(s.impressions)
2608
- }))
2609
- }));
2610
- return {
2611
- results: paginateInMemory(mapped, {
2612
- limit: params.limit ?? 2e3,
2613
- offset: params.offset
2614
- }),
2615
- meta: { total: mapped.length }
2616
- };
2617
- },
2618
- buildRows(params) {
2619
- const { current, previous } = comparisonOf(params);
2620
- return {
2621
- current: pagesQueryState(current, params.limit),
2622
- previous: pagesQueryState(previous, params.limit)
2623
- };
2624
- },
2625
- reduceRows(rows, params) {
2626
- const map = rows && !Array.isArray(rows) ? rows : {
2627
- current: [],
2628
- previous: []
2629
- };
2630
- const results = analyzeDecay({
2631
- current: map.current ?? [],
2632
- previous: map.previous ?? []
2633
- }, {
2634
- minPreviousClicks: params.minPreviousClicks,
2635
- threshold: params.threshold
2636
- });
2637
- return {
2638
- results,
2639
- meta: { total: results.length }
2640
- };
2641
- }
2642
- });
2643
- const deviceGapAnalyzer = defineAnalyzer({
2644
- id: "device-gap",
2645
- buildSql(params) {
2646
- const { startDate, endDate } = periodOf(params);
2647
- return {
2648
- sql: `
2649
- WITH raw AS (
2650
- SELECT
2651
- date,
2652
- COALESCE(clicks_desktop, 0) AS c_desktop,
2653
- COALESCE(clicks_mobile, 0) AS c_mobile,
2654
- COALESCE(clicks_tablet, 0) AS c_tablet,
2655
- COALESCE(impressions_desktop, 0) AS i_desktop,
2656
- COALESCE(impressions_mobile, 0) AS i_mobile,
2657
- COALESCE(impressions_tablet, 0) AS i_tablet,
2658
- COALESCE(sum_position_desktop, 0) AS p_desktop,
2659
- COALESCE(sum_position_mobile, 0) AS p_mobile,
2660
- COALESCE(sum_position_tablet, 0) AS p_tablet
2661
- FROM read_parquet({{FILES}}, union_by_name = true)
2662
- WHERE date >= ? AND date <= ?
2663
- ),
2664
- device_long AS (
2665
- SELECT date, 'DESKTOP' AS device, c_desktop AS clicks, i_desktop AS impressions, p_desktop AS sum_position FROM raw
2666
- UNION ALL
2667
- SELECT date, 'MOBILE', c_mobile, i_mobile, p_mobile FROM raw
2668
- UNION ALL
2669
- SELECT date, 'TABLET', c_tablet, i_tablet, p_tablet FROM raw
2670
- )
2671
- SELECT
2672
- date,
2673
- device,
2674
- CAST(SUM(clicks) AS DOUBLE) AS clicks,
2675
- CAST(SUM(impressions) AS DOUBLE) AS impressions,
2676
- CAST(SUM(clicks) AS DOUBLE) / NULLIF(SUM(impressions), 0) AS ctr,
2677
- SUM(sum_position) / NULLIF(SUM(impressions), 0) + 1 AS position
2678
- FROM device_long
2679
- GROUP BY date, device
2680
- ORDER BY date ASC
2681
- `,
2682
- params: [startDate, endDate],
2683
- current: {
2684
- table: "dates",
2685
- partitions: enumeratePartitions(startDate, endDate)
2686
- }
2687
- };
2688
- },
2689
- reduceSql(rows, params) {
2690
- const arr = Array.isArray(rows) ? rows : [];
2691
- const { startDate, endDate } = periodOf(params);
2692
- const typed = arr.map((r) => ({
2693
- date: rowString(r.date),
2694
- device: rowString(r.device).toUpperCase(),
2695
- clicks: num(r.clicks),
2696
- impressions: num(r.impressions),
2697
- ctr: num(r.ctr),
2698
- position: num(r.position)
2699
- }));
2700
- const byDate = /* @__PURE__ */ new Map();
2701
- for (const r of typed) {
2702
- const entry = byDate.get(r.date) ?? {};
2703
- const metrics = {
2704
- clicks: r.clicks,
2705
- impressions: r.impressions,
2706
- ctr: r.ctr,
2707
- position: r.position
2708
- };
2709
- if (r.device === "DESKTOP") entry.desktop = metrics;
2710
- else if (r.device === "MOBILE") entry.mobile = metrics;
2711
- byDate.set(r.date, entry);
2712
- }
2713
- const zero = {
2714
- clicks: 0,
2715
- impressions: 0,
2716
- ctr: 0,
2717
- position: 0
2718
- };
2719
- const daily = [...byDate.entries()].sort(([a], [b]) => a.localeCompare(b)).map(([date, sides]) => {
2720
- const d = sides.desktop ?? zero;
2721
- const m = sides.mobile ?? zero;
2722
- return {
2723
- date,
2724
- desktop: d,
2725
- mobile: m,
2726
- gaps: {
2727
- ctrGap: d.ctr - m.ctr,
2728
- positionGap: m.position - d.position
2729
- }
2730
- };
2731
- });
2732
- const weekly = (start, end) => {
2733
- const slice = daily.slice(start, end);
2734
- if (slice.length === 0) return {
2735
- ctr: 0,
2736
- pos: 0
2737
- };
2738
- const sum = slice.reduce((acc, d) => ({
2739
- ctr: acc.ctr + d.gaps.ctrGap,
2740
- pos: acc.pos + d.gaps.positionGap
2741
- }), {
2742
- ctr: 0,
2743
- pos: 0
2744
- });
2745
- return {
2746
- ctr: sum.ctr / slice.length,
2747
- pos: sum.pos / slice.length
2748
- };
2749
- };
2750
- const first = weekly(0, 7);
2751
- const last = weekly(Math.max(0, daily.length - 7), daily.length);
2752
- const classify = (firstVal, lastVal) => {
2753
- const diff = Math.abs(lastVal) - Math.abs(firstVal);
2754
- if (Math.abs(diff) < .005) return "stable";
2755
- return diff < 0 ? "improving" : "worsening";
2756
- };
2757
- return {
2758
- results: daily,
2759
- meta: {
2760
- summary: {
2761
- avgCtrGap: daily.reduce((s, d) => s + d.gaps.ctrGap, 0) / Math.max(1, daily.length),
2762
- avgPositionGap: daily.reduce((s, d) => s + d.gaps.positionGap, 0) / Math.max(1, daily.length),
2763
- ctrGapTrend: classify(first.ctr, last.ctr),
2764
- positionGapTrend: classify(first.pos, last.pos)
2765
- },
2766
- startDate,
2767
- endDate
2768
- }
2769
- };
2770
- }
2771
- });
2772
- const INTENT_ATLAS_STOP_WORDS = [
2773
- "the",
2774
- "a",
2775
- "an",
2776
- "is",
2777
- "are",
2778
- "was",
2779
- "were",
2780
- "be",
2781
- "been",
2782
- "of",
2783
- "to",
2784
- "in",
2785
- "for",
2786
- "on",
2787
- "and",
2788
- "or",
2789
- "with",
2790
- "at",
2791
- "by",
2792
- "from",
2793
- "into",
2794
- "about",
2795
- "as",
2796
- "so",
2797
- "than",
2798
- "then",
2799
- "that",
2800
- "this",
2801
- "my",
2802
- "your",
2803
- "our",
2804
- "their",
2805
- "his",
2806
- "her",
2807
- "its",
2808
- "me",
2809
- "you",
2810
- "what",
2811
- "how",
2812
- "why",
2813
- "when",
2814
- "where",
2815
- "who",
2816
- "which",
2817
- "do",
2818
- "does"
2819
- ];
2820
- const intentAtlasAnalyzer = defineAnalyzer({
2821
- id: "intent-atlas",
2822
- buildSql(params) {
2823
- const endDate = params.endDate ?? defaultEndDate();
2824
- const startDate = params.startDate ?? daysAgoUtc(90);
2825
- const minQueryImpressions = params.minImpressions ?? 20;
2826
- const minClusterSize = params.minClusterSize ?? 3;
2827
- const minTokenImpressions = 50;
2828
- const limit = params.limit ?? 200;
2829
- const stopList = INTENT_ATLAS_STOP_WORDS.map((w) => `'${w}'`).join(", ");
2830
- return {
2831
- sql: `
2832
- WITH queries AS (
2833
- SELECT
2834
- query,
2835
- ${METRIC_EXPR.impressions} AS impressions,
2836
- ${METRIC_EXPR.clicks} AS clicks,
2837
- ${METRIC_EXPR.position} AS position
2838
- FROM read_parquet({{FILES}}, union_by_name = true)
2839
- WHERE date >= ? AND date <= ?
2840
- AND query IS NOT NULL AND query <> ''
2841
- GROUP BY query
2842
- HAVING SUM(impressions) >= ?
2843
- ),
2844
- tokens AS (
2845
- SELECT q.query, q.impressions, q.clicks, q.position,
2846
- LOWER(t.token) AS token
2847
- FROM queries q,
2848
- unnest(regexp_split_to_array(LOWER(q.query), '\\s+')) AS t(token)
2849
- WHERE LENGTH(t.token) >= 3
2850
- AND LOWER(t.token) NOT IN (${stopList})
2851
- ),
2852
- token_weights AS (
2853
- SELECT token,
2854
- SUM(impressions) AS token_impressions,
2855
- COUNT(DISTINCT query) AS query_count
2856
- FROM tokens
2857
- GROUP BY token
2858
- HAVING SUM(impressions) >= ${Number(minTokenImpressions)}
2859
- ),
2860
- ranked_tokens AS (
2861
- SELECT t.query, t.token, tw.token_impressions,
2862
- ROW_NUMBER() OVER (
2863
- PARTITION BY t.query
2864
- ORDER BY tw.token_impressions DESC, t.token ASC
2865
- ) AS rnk
2866
- FROM tokens t
2867
- JOIN token_weights tw USING (token)
2868
- ),
2869
- cluster_keys AS (
2870
- SELECT query,
2871
- array_to_string(list(token ORDER BY token), ' + ') AS cluster_key
2872
- FROM ranked_tokens
2873
- WHERE rnk <= 2
2874
- GROUP BY query
2875
- HAVING COUNT(*) >= 2
2876
- ),
2877
- clustered AS (
2878
- SELECT q.query, q.impressions, q.clicks, q.position, ck.cluster_key
2879
- FROM queries q
2880
- JOIN cluster_keys ck USING (query)
2881
- )
2882
- SELECT
2883
- cluster_key AS clusterKey,
2884
- COUNT(*) AS keywordCount,
2885
- SUM(impressions) AS totalImpressions,
2886
- SUM(clicks) AS totalClicks,
2887
- SUM(clicks) / NULLIF(SUM(impressions), 0) AS ctr,
2888
- SUM((position - 1) * impressions) / NULLIF(SUM(impressions), 0) + 1 AS avgPosition,
2889
- to_json(list({
2890
- 'query': query,
2891
- 'impressions': impressions,
2892
- 'clicks': clicks,
2893
- 'position': position
2894
- } ORDER BY impressions DESC)) AS keywords
2895
- FROM clustered
2896
- GROUP BY cluster_key
2897
- HAVING COUNT(*) >= ${Number(minClusterSize)}
2898
- ORDER BY totalImpressions DESC
2899
- LIMIT ${Number(limit)}
2900
- `,
2901
- params: [
2902
- startDate,
2903
- endDate,
2904
- minQueryImpressions
2905
- ],
2906
- current: {
2907
- table: "queries",
2908
- partitions: enumeratePartitions(startDate, endDate)
2909
- }
2910
- };
2911
- },
2912
- reduceSql(rows) {
2913
- const clusters = (Array.isArray(rows) ? rows : []).map((r) => ({
2914
- clusterKey: rowString(r.clusterKey),
2915
- keywordCount: num(r.keywordCount),
2916
- totalImpressions: num(r.totalImpressions),
2917
- totalClicks: num(r.totalClicks),
2918
- ctr: num(r.ctr),
2919
- avgPosition: num(r.avgPosition),
2920
- keywords: parseJsonRows(r.keywords).slice(0, 25).map((k) => ({
2921
- query: rowString(k.query),
2922
- impressions: num(k.impressions),
2923
- clicks: num(k.clicks),
2924
- position: num(k.position)
2925
- }))
2926
- }));
2927
- const totalImpressions = clusters.reduce((s, c) => s + c.totalImpressions, 0);
2928
- const totalKeywords = clusters.reduce((s, c) => s + c.keywordCount, 0);
2929
- return {
2930
- results: clusters,
2931
- meta: {
2932
- total: clusters.length,
2933
- totalImpressions,
2934
- totalKeywords
2935
- }
2936
- };
2937
- }
2938
- });
2939
- const keywordBreadthAnalyzer = defineAnalyzer({
2940
- id: "keyword-breadth",
2941
- buildSql(params) {
2942
- const { startDate, endDate } = periodOf(params);
2943
- return {
2944
- sql: `
2945
- WITH per_page AS (
2946
- SELECT
2947
- url,
2948
- CAST(COUNT(DISTINCT query) AS DOUBLE) AS keywordCount,
2949
- ${METRIC_EXPR.clicks} AS clicks,
2950
- ${METRIC_EXPR.impressions} AS impressions
2951
- FROM read_parquet({{FILES}}, union_by_name = true)
2952
- WHERE date >= ? AND date <= ? AND impressions > 0
2953
- GROUP BY url
2954
- ),
2955
- bucketed AS (
2956
- SELECT
2957
- CASE
2958
- WHEN keywordCount = 1 THEN '1'
2959
- WHEN keywordCount BETWEEN 2 AND 5 THEN '2-5'
2960
- WHEN keywordCount BETWEEN 6 AND 15 THEN '6-15'
2961
- WHEN keywordCount BETWEEN 16 AND 50 THEN '16-50'
2962
- ELSE '50+'
2963
- END AS bucket,
2964
- MIN(keywordCount) AS sort_key,
2965
- CAST(COUNT(*) AS DOUBLE) AS pageCount
2966
- FROM per_page
2967
- GROUP BY bucket
2968
- ),
2969
- fragile AS (
2970
- SELECT url, keywordCount, clicks, impressions
2971
- FROM per_page
2972
- WHERE keywordCount <= 2 AND clicks >= 5
2973
- ORDER BY clicks DESC
2974
- LIMIT 20
2975
- ),
2976
- authority AS (
2977
- SELECT url, keywordCount, clicks, impressions
2978
- FROM per_page
2979
- WHERE keywordCount >= 20
2980
- ORDER BY keywordCount DESC
2981
- LIMIT 20
2982
- ),
2983
- stats AS (
2984
- SELECT
2985
- CAST(COUNT(*) AS DOUBLE) AS totalPages,
2986
- CAST(AVG(keywordCount) AS DOUBLE) AS avgKeywordsPerPage,
2987
- CAST(SUM(CASE WHEN keywordCount <= 2 THEN 1 ELSE 0 END) AS DOUBLE) AS fragileCount,
2988
- CAST(SUM(CASE WHEN keywordCount >= 20 THEN 1 ELSE 0 END) AS DOUBLE) AS authorityCount
2989
- FROM per_page
2990
- )
2991
- SELECT
2992
- (SELECT to_json(list({ 'bucket': bucket, 'pageCount': pageCount, 'sortKey': sort_key })) FROM bucketed) AS distribution_json,
2993
- (SELECT to_json(list({ 'url': url, 'keywordCount': keywordCount, 'clicks': clicks, 'impressions': impressions })) FROM fragile) AS fragile_json,
2994
- (SELECT to_json(list({ 'url': url, 'keywordCount': keywordCount, 'clicks': clicks, 'impressions': impressions })) FROM authority) AS authority_json,
2995
- (SELECT to_json({
2996
- 'totalPages': totalPages,
2997
- 'avgKeywordsPerPage': avgKeywordsPerPage,
2998
- 'fragileCount': fragileCount,
2999
- 'authorityCount': authorityCount
3000
- }) FROM stats) AS stats_json
3001
- `,
3002
- params: [startDate, endDate],
3003
- current: {
3004
- table: "page_queries",
3005
- partitions: enumeratePartitions(startDate, endDate)
3006
- }
3007
- };
3008
- },
3009
- reduceSql(rows, params) {
3010
- const arr = Array.isArray(rows) ? rows : [];
3011
- const { startDate, endDate } = periodOf(params);
3012
- const row = arr[0] ?? {};
3013
- const distribution = parseJsonRows(row.distribution_json).sort((a, b) => num(a.sortKey) - num(b.sortKey)).map((r) => ({
3014
- bucket: rowString(r.bucket),
3015
- pageCount: num(r.pageCount)
3016
- }));
3017
- const fragile = parseJsonRows(row.fragile_json).map((r) => ({
3018
- url: rowString(r.url),
3019
- keywordCount: num(r.keywordCount),
3020
- clicks: num(r.clicks),
3021
- impressions: num(r.impressions)
3022
- }));
3023
- const authority = parseJsonRows(row.authority_json).map((r) => ({
3024
- url: rowString(r.url),
3025
- keywordCount: num(r.keywordCount),
3026
- clicks: num(r.clicks),
3027
- impressions: num(r.impressions)
3028
- }));
3029
- const stats = typeof row.stats_json === "string" ? JSON.parse(row.stats_json) : row.stats_json ?? {};
3030
- return {
3031
- results: distribution,
3032
- meta: {
3033
- fragilePages: fragile,
3034
- authorityPages: authority,
3035
- summary: {
3036
- totalPages: num(stats.totalPages),
3037
- avgKeywordsPerPage: num(stats.avgKeywordsPerPage),
3038
- fragileCount: num(stats.fragileCount),
3039
- authorityCount: num(stats.authorityCount)
3040
- },
3041
- startDate,
3042
- endDate
3043
- }
3044
- };
3045
- }
3046
- });
3047
- function downsampleLogRank(points) {
3048
- const toPoint = (p) => ({
3049
- rank: num(p.rank),
3050
- impressions: num(p.impressions),
3051
- clicks: num(p.clicks),
3052
- query: rowString(p.query)
3053
- });
3054
- if (points.length <= 80) return points.map(toPoint);
3055
- const sampled = points.slice(0, 10).map(toPoint);
3056
- let nextThreshold = 1.15;
3057
- for (let i = 10; i < points.length; i++) {
3058
- const point = points[i];
3059
- if (num(point.rank) >= nextThreshold) {
3060
- sampled.push(toPoint(point));
3061
- nextThreshold *= 1.15;
3062
- }
3063
- }
3064
- return sampled;
3065
- }
3066
- const longTailAnalyzer = defineAnalyzer({
3067
- id: "long-tail",
3068
- buildSql(params) {
3069
- const { startDate, endDate } = periodOf(params);
3070
- const minQueries = 10;
3071
- const minQueryImpressions = params.minImpressions ?? 5;
3072
- const limit = params.limit ?? 100;
3073
- return {
3074
- sql: `
3075
- WITH pq AS (
3076
- SELECT
3077
- url AS page,
3078
- query,
3079
- ${METRIC_EXPR.impressions} AS impressions,
3080
- ${METRIC_EXPR.clicks} AS clicks
3081
- FROM read_parquet({{FILES}}, union_by_name = true)
3082
- WHERE date >= ? AND date <= ?
3083
- AND query IS NOT NULL AND query <> ''
3084
- AND url IS NOT NULL AND url <> ''
3085
- GROUP BY url, query
3086
- HAVING SUM(impressions) >= ?
3087
- ),
3088
- ranked AS (
3089
- SELECT
3090
- page, query, impressions, clicks,
3091
- ROW_NUMBER() OVER (PARTITION BY page ORDER BY impressions DESC, query ASC) AS rnk
3092
- FROM pq
3093
- ),
3094
- log_space AS (
3095
- SELECT *,
3096
- LN(rnk) AS log_rank,
3097
- LN(impressions) AS log_impr
3098
- FROM ranked
3099
- ),
3100
- fit AS (
3101
- SELECT
3102
- page,
3103
- COUNT(*) AS query_count,
3104
- SUM(impressions) AS total_impressions,
3105
- SUM(clicks) AS total_clicks,
3106
- REGR_SLOPE(log_impr, log_rank) AS slope,
3107
- REGR_INTERCEPT(log_impr, log_rank) AS intercept,
3108
- REGR_R2(log_impr, log_rank) AS r2,
3109
- MAX(impressions) AS head_impressions,
3110
- MAX(CASE WHEN rnk = 1 THEN impressions END) / NULLIF(SUM(impressions), 0) AS head_share
3111
- FROM log_space
3112
- GROUP BY page
3113
- HAVING COUNT(*) >= ${Number(minQueries)}
3114
- ),
3115
- scatter AS (
3116
- SELECT
3117
- l.page,
3118
- to_json(list({
3119
- 'rank': l.rnk,
3120
- 'impressions': l.impressions,
3121
- 'clicks': l.clicks,
3122
- 'query': l.query
3123
- } ORDER BY l.rnk)) AS pointsJson
3124
- FROM log_space l
3125
- JOIN fit f USING (page)
3126
- GROUP BY l.page
3127
- )
3128
- SELECT
3129
- f.page,
3130
- f.query_count AS queryCount,
3131
- f.total_impressions AS totalImpressions,
3132
- f.total_clicks AS totalClicks,
3133
- f.slope AS slope,
3134
- f.intercept AS intercept,
3135
- f.r2 AS r2,
3136
- f.head_impressions AS headImpressions,
3137
- f.head_share AS headShare,
3138
- s.pointsJson AS pointsJson,
3139
- CASE
3140
- WHEN f.slope > -0.6 THEN 'flat-tail'
3141
- WHEN f.slope > -1.2 THEN 'balanced'
3142
- ELSE 'head-heavy'
3143
- END AS fingerprint
3144
- FROM fit f
3145
- LEFT JOIN scatter s USING (page)
3146
- ORDER BY f.total_impressions DESC
3147
- LIMIT ${Number(limit)}
3148
- `,
3149
- params: [
3150
- startDate,
3151
- endDate,
3152
- minQueryImpressions
3153
- ],
3154
- current: {
3155
- table: "page_queries",
3156
- partitions: enumeratePartitions(startDate, endDate)
3157
- }
3158
- };
3159
- },
3160
- reduceSql(rows) {
3161
- const results = (Array.isArray(rows) ? rows : []).map((r) => ({
3162
- page: rowString(r.page),
3163
- queryCount: num(r.queryCount),
3164
- totalImpressions: num(r.totalImpressions),
3165
- totalClicks: num(r.totalClicks),
3166
- slope: num(r.slope),
3167
- intercept: num(r.intercept),
3168
- r2: num(r.r2),
3169
- headImpressions: num(r.headImpressions),
3170
- headShare: num(r.headShare),
3171
- fingerprint: rowString(r.fingerprint),
3172
- points: downsampleLogRank(parseJsonRows(r.pointsJson))
3173
- }));
3174
- const counts = {
3175
- "flat-tail": 0,
3176
- "balanced": 0,
3177
- "head-heavy": 0
3178
- };
3179
- for (const r of results) counts[r.fingerprint]++;
3180
- return {
3181
- results,
3182
- meta: {
3183
- total: results.length,
3184
- fingerprints: counts,
3185
- avgSlope: results.length > 0 ? results.reduce((s, r) => s + r.slope, 0) / results.length : 0
3186
- }
3187
- };
3188
- }
3189
- });
3190
- function percentDifference(current, previous) {
3191
- if (previous === 0) return current > 0 ? 100 : 0;
3192
- return (current - previous) / previous * 100;
3193
- }
3194
- function withCanonicalFields(r) {
3195
- return {
3196
- ...r,
3197
- clicks: r.recentClicks,
3198
- impressions: r.recentImpressions,
3199
- ctr: r.recentImpressions > 0 ? r.recentClicks / r.recentImpressions : 0,
3200
- position: r.recentPosition,
3201
- prevClicks: r.baselineClicks,
3202
- prevImpressions: r.baselineImpressions,
3203
- prevPosition: r.baselinePosition
3204
- };
3205
- }
3206
- function selectMoversDirection(rising, declining, stable, params) {
3207
- if (params.direction) {
3208
- const selected = params.direction === "rising" ? rising : declining;
3209
- const total = selected.length;
3210
- const offset = params.offset ?? 0;
3211
- const limit = params.limit ?? total;
3212
- return {
3213
- results: selected.slice(offset, offset + limit),
3214
- meta: {
3215
- total,
3216
- rising: rising.length,
3217
- declining: declining.length,
3218
- stable: stable.length
3219
- }
3220
- };
3221
- }
3222
- const combined = [...rising, ...declining];
3223
- return {
3224
- results: combined,
3225
- meta: {
3226
- total: combined.length,
3227
- rising: rising.length,
3228
- declining: declining.length,
3229
- stable: stable.length
3230
- }
3231
- };
3232
- }
3233
- function analyzeMovers(input, options = {}) {
3234
- const { changeThreshold = .2, minImpressions = 50, sortBy = "clicksChange" } = options;
3235
- const normFactor = input.normalizationFactor ?? 1;
3236
- const baselineMap = buildPeriodMap(input.previous, (r) => r.query, (r) => ({
3237
- clicks: num(r.clicks) / normFactor,
3238
- impressions: num(r.impressions) / normFactor,
3239
- position: num(r.position),
3240
- page: r.page ?? null
3241
- }));
3242
- const pageMap = /* @__PURE__ */ new Map();
3243
- for (const row of input.current) if (!pageMap.has(row.query) && row.page) pageMap.set(row.query, row.page);
3244
- for (const row of input.previous) if (!pageMap.has(row.query) && row.page) pageMap.set(row.query, row.page);
3245
- const rising = [];
3246
- const declining = [];
3247
- const stable = [];
3248
- for (const row of input.current) {
3249
- const impressions = num(row.impressions);
3250
- const clicks = num(row.clicks);
3251
- const position = num(row.position);
3252
- if (impressions < minImpressions) continue;
3253
- const baseline = baselineMap.get(row.query);
3254
- const baselineClicksRaw = baseline?.clicks ?? 0;
3255
- const baselineImpressionsRaw = baseline?.impressions ?? 0;
3256
- const baselinePosition = baseline?.position ?? null;
3257
- const clicksChangePercent = percentDifference(clicks, baselineClicksRaw);
3258
- const impressionsChangePercent = percentDifference(impressions, baselineImpressionsRaw);
3259
- const data = {
3260
- keyword: row.query,
3261
- page: pageMap.get(row.query) ?? null,
3262
- recentClicks: clicks,
3263
- recentImpressions: impressions,
3264
- recentPosition: position,
3265
- baselineClicks: Math.round(baselineClicksRaw),
3266
- baselineImpressions: Math.round(baselineImpressionsRaw),
3267
- baselinePosition,
3268
- clicksChange: clicks - Math.round(baselineClicksRaw),
3269
- clicksChangePercent,
3270
- impressionsChangePercent,
3271
- positionChange: baselinePosition != null ? position - baselinePosition : null
3272
- };
3273
- const absChange = Math.abs(clicksChangePercent / 100);
3274
- if (clicksChangePercent > 0 && absChange >= changeThreshold) rising.push(data);
3275
- else if (clicksChangePercent < 0 && absChange >= changeThreshold) declining.push(data);
3276
- else stable.push(data);
3277
- }
3278
- const sortFn = (a, b) => {
3279
- switch (sortBy) {
3280
- case "clicks": return b.recentClicks - a.recentClicks;
3281
- case "impressions": return b.recentImpressions - a.recentImpressions;
3282
- case "clicksChange": return Math.abs(b.clicksChangePercent) - Math.abs(a.clicksChangePercent);
3283
- case "impressionsChange": return Math.abs(b.impressionsChangePercent) - Math.abs(a.impressionsChangePercent);
3284
- case "positionChange": return Math.abs(b.positionChange ?? 0) - Math.abs(a.positionChange ?? 0);
3285
- default: return Math.abs(b.clicksChangePercent) - Math.abs(a.clicksChangePercent);
3286
- }
3287
- };
3288
- rising.sort(sortFn);
3289
- declining.sort(sortFn);
3290
- stable.sort((a, b) => b.recentClicks - a.recentClicks);
3291
- return {
3292
- rising,
3293
- declining,
3294
- stable
3295
- };
3296
- }
3297
- const moversAnalyzer = defineAnalyzer({
3298
- id: "movers",
3299
- buildSql(params) {
3300
- const { current: cur, previous: prev } = comparisonOf(params);
3301
- const minImpressions = params.minImpressions ?? 50;
3302
- const changeThreshold = params.changeThreshold ?? .2;
3303
- const limit = params.direction ? 5e3 : params.limit ?? 2e3;
3304
- return {
3305
- sql: `
3306
- WITH cur AS (
3307
- SELECT
3308
- query, url,
3309
- ${METRIC_EXPR.clicks} AS clicks,
3310
- ${METRIC_EXPR.impressions} AS impressions,
3311
- ${METRIC_EXPR.position} AS position
3312
- FROM read_parquet({{FILES}}, union_by_name = true)
3313
- WHERE date >= ? AND date <= ?
3314
- GROUP BY query, url
3315
- ),
3316
- prev AS (
3317
- SELECT
3318
- query, url,
3319
- ${METRIC_EXPR.clicks} AS clicks,
3320
- ${METRIC_EXPR.impressions} AS impressions,
3321
- ${METRIC_EXPR.position} AS position
3322
- FROM read_parquet({{FILES_PREV}}, union_by_name = true)
3323
- WHERE date >= ? AND date <= ?
3324
- GROUP BY query, url
3325
- ),
3326
- weekly AS (
3327
- SELECT query, url, date_trunc('week', CAST(date AS DATE)) AS week,
3328
- ${METRIC_EXPR.clicks} AS clicks,
3329
- ${METRIC_EXPR.impressions} AS impressions
3330
- FROM (
3331
- SELECT query, url, date, clicks, impressions
3332
- FROM read_parquet({{FILES}}, union_by_name = true)
3333
- WHERE date >= ? AND date <= ?
3334
- UNION ALL
3335
- SELECT query, url, date, clicks, impressions
3336
- FROM read_parquet({{FILES_PREV}}, union_by_name = true)
3337
- WHERE date >= ? AND date <= ?
3338
- )
3339
- GROUP BY query, url, week
3340
- ),
3341
- series_by_entity AS (
3342
- SELECT query, url, to_json(list({
3343
- 'week': strftime(week, '%Y-%m-%d'),
3344
- 'clicks': clicks,
3345
- 'impressions': impressions
3346
- } ORDER BY week)) AS seriesJson
3347
- FROM weekly GROUP BY query, url
3348
- ),
3349
- joined AS (
3350
- SELECT
3351
- c.query AS keyword,
3352
- c.url AS page,
3353
- c.clicks AS recentClicks,
3354
- c.impressions AS recentImpressions,
3355
- c.position AS recentPosition,
3356
- COALESCE(p.clicks, 0.0) AS baselineClicks,
3357
- COALESCE(p.impressions, 0.0) AS baselineImpressions,
3358
- p.position AS baselinePosition,
3359
- (c.clicks - COALESCE(p.clicks, 0.0)) AS clicksChange,
3360
- CASE
3361
- WHEN COALESCE(p.clicks, 0.0) = 0 THEN CASE WHEN c.clicks > 0 THEN 100.0 ELSE 0.0 END
3362
- ELSE (c.clicks - p.clicks) * 100.0 / p.clicks
3363
- END AS clicksChangePercent,
3364
- CASE
3365
- WHEN COALESCE(p.impressions, 0.0) = 0 THEN CASE WHEN c.impressions > 0 THEN 100.0 ELSE 0.0 END
3366
- ELSE (c.impressions - p.impressions) * 100.0 / p.impressions
3367
- END AS impressionsChangePercent,
3368
- CASE WHEN p.position IS NOT NULL THEN (c.position - p.position) ELSE NULL END AS positionChange,
3369
- s.seriesJson
3370
- FROM cur c
3371
- LEFT JOIN prev p ON c.query = p.query AND c.url = p.url
3372
- LEFT JOIN series_by_entity s ON c.query = s.query AND c.url = s.url
3373
- WHERE c.impressions >= ?
3374
- )
3375
- SELECT *,
3376
- CASE
3377
- WHEN clicksChangePercent > 0 AND ABS(clicksChangePercent) / 100.0 >= ? THEN 'rising'
3378
- WHEN clicksChangePercent < 0 AND ABS(clicksChangePercent) / 100.0 >= ? THEN 'declining'
3379
- ELSE 'stable'
3380
- END AS direction
3381
- FROM joined
3382
- ORDER BY ABS(clicksChangePercent) DESC
3383
- LIMIT ${Number(limit)}
3384
- `,
3385
- params: [
3386
- cur.startDate,
3387
- cur.endDate,
3388
- prev.startDate,
3389
- prev.endDate,
3390
- cur.startDate,
3391
- cur.endDate,
3392
- prev.startDate,
3393
- prev.endDate,
3394
- minImpressions,
3395
- changeThreshold,
3396
- changeThreshold
3397
- ],
3398
- current: {
3399
- table: "page_queries",
3400
- partitions: enumeratePartitions(cur.startDate, cur.endDate)
3401
- },
3402
- previous: {
3403
- table: "page_queries",
3404
- partitions: enumeratePartitions(prev.startDate, prev.endDate)
3405
- }
3406
- };
3407
- },
3408
- reduceSql(rows, params) {
3409
- const normalized = (Array.isArray(rows) ? rows : []).map((r) => {
3410
- const recentClicks = num(r.recentClicks);
3411
- const recentImpressions = num(r.recentImpressions);
3412
- const recentPosition = num(r.recentPosition);
3413
- const baselineClicks = Math.round(num(r.baselineClicks));
3414
- const baselineImpressions = Math.round(num(r.baselineImpressions));
3415
- const baselinePosition = r.baselinePosition == null ? null : num(r.baselinePosition);
3416
- return {
3417
- keyword: rowString(r.keyword),
3418
- page: r.page == null ? null : rowString(r.page),
3419
- recentClicks,
3420
- recentImpressions,
3421
- recentPosition,
3422
- baselineClicks,
3423
- baselineImpressions,
3424
- baselinePosition,
3425
- clicksChange: num(r.clicksChange),
3426
- clicksChangePercent: num(r.clicksChangePercent),
3427
- impressionsChangePercent: num(r.impressionsChangePercent),
3428
- positionChange: r.positionChange == null ? null : num(r.positionChange),
3429
- direction: rowString(r.direction),
3430
- series: parseJsonRows(r.seriesJson).map((s) => ({
3431
- week: rowString(s.week),
3432
- clicks: num(s.clicks),
3433
- impressions: num(s.impressions)
3434
- }))
3435
- };
3436
- }).map(withCanonicalFields);
3437
- return selectMoversDirection(normalized.filter((r) => r.direction === "rising"), normalized.filter((r) => r.direction === "declining"), normalized.filter((r) => r.direction === "stable"), params);
3438
- },
3439
- buildRows(params) {
3440
- const { current, previous } = comparisonOf(params);
3441
- return {
3442
- current: queriesQueryState(current, params.limit),
3443
- previous: queriesQueryState(previous, params.limit)
3444
- };
3445
- },
3446
- reduceRows(rows, params) {
3447
- const map = rows && !Array.isArray(rows) ? rows : {
3448
- current: [],
3449
- previous: []
3450
- };
3451
- const result = analyzeMovers({
3452
- current: map.current ?? [],
3453
- previous: map.previous ?? []
3454
- }, {
3455
- changeThreshold: params.changeThreshold,
3456
- minImpressions: params.minImpressions
3457
- });
3458
- return selectMoversDirection(result.rising.map((r) => withCanonicalFields({
3459
- ...r,
3460
- direction: "rising"
3461
- })), result.declining.map((r) => withCanonicalFields({
3462
- ...r,
3463
- direction: "declining"
3464
- })), [], params);
3465
- }
3466
- });
3467
- const EXPECTED_CTR_BY_POSITION = {
3468
- 1: .3,
3469
- 2: .15,
3470
- 3: .1,
3471
- 4: .07,
3472
- 5: .05,
3473
- 6: .04,
3474
- 7: .03,
3475
- 8: .025,
3476
- 9: .02,
3477
- 10: .015
3478
- };
3479
- function getExpectedCtr(position) {
3480
- return EXPECTED_CTR_BY_POSITION[Math.round(Math.max(1, Math.min(position, 10)))] || .01;
3481
- }
3482
- function calculatePositionScore(position) {
3483
- if (position <= 3) return .2;
3484
- if (position > 50) return .1;
3485
- const distance = Math.abs(position - 11);
3486
- return Math.max(0, 1 - distance / 15);
3487
- }
3488
- function calculateImpressionScore(impressions) {
3489
- if (impressions <= 0) return 0;
3490
- return Math.min(Math.log10(impressions) / 5, 1);
3491
- }
3492
- function calculateCtrGapScore(actualCtr, position) {
3493
- const expectedCtr = getExpectedCtr(position);
3494
- if (actualCtr >= expectedCtr) return 0;
3495
- const gap = expectedCtr - actualCtr;
3496
- return Math.min(gap / expectedCtr, 1);
3497
- }
3498
- const SORT_DIR = {
3499
- opportunityScore: "desc",
3500
- potentialClicks: "desc",
3501
- impressions: "desc",
3502
- position: "asc"
3503
- };
3504
- function analyzeOpportunity(keywords, options = {}) {
3505
- const minImpressions = options.minImpressions ?? 100;
3506
- const positionWeight = options.weights?.position ?? 1;
3507
- const impressionsWeight = options.weights?.impressions ?? 1;
3508
- const ctrGapWeight = options.weights?.ctrGap ?? 1;
3509
- const totalWeight = positionWeight + impressionsWeight + ctrGapWeight;
3510
- const sortBy = options.sortBy ?? "opportunityScore";
3511
- const results = [];
3512
- for (const row of keywords) {
3513
- const impressions = num(row.impressions);
3514
- if (impressions < minImpressions) continue;
3515
- const position = num(row.position);
3516
- const ctr = num(row.ctr);
3517
- const clicks = num(row.clicks);
3518
- const positionScore = calculatePositionScore(position);
3519
- const impressionScore = calculateImpressionScore(impressions);
3520
- const ctrGapScore = calculateCtrGapScore(ctr, position);
3521
- const weightedProduct = positionScore ** positionWeight * impressionScore ** impressionsWeight * ctrGapScore ** ctrGapWeight;
3522
- const opportunityScore = Math.round(weightedProduct ** (1 / totalWeight) * 100);
3523
- const potentialClicks = Math.round(impressions * getExpectedCtr(Math.min(3, position)));
3524
- results.push({
3525
- keyword: row.query,
3526
- page: row.page ?? null,
3527
- clicks,
3528
- impressions,
3529
- ctr,
3530
- position,
3531
- opportunityScore,
3532
- potentialClicks,
3533
- factors: {
3534
- positionScore,
3535
- impressionScore,
3536
- ctrGapScore
3537
- }
3538
- });
3539
- }
3540
- const direction = SORT_DIR[sortBy];
3541
- results.sort((left, right) => direction === "asc" ? left[sortBy] - right[sortBy] : right[sortBy] - left[sortBy]);
3542
- return options.limit ? results.slice(0, options.limit) : results;
3543
- }
3544
- const opportunityAnalyzer = defineAnalyzer({
3545
- id: "opportunity",
3546
- buildSql(params) {
3547
- const { startDate, endDate } = periodOf(params);
3548
- const minImpressions = params.minImpressions ?? 100;
3549
- const w1 = 1;
3550
- const w2 = 1;
3551
- const w3 = 1;
3552
- const totalW = 3;
3553
- const limit = params.limit ?? 1e3;
3554
- return {
3555
- sql: `
3556
- WITH agg AS (
3557
- SELECT
3558
- query AS keyword,
3559
- url AS page,
3560
- ${METRIC_EXPR.clicks} AS clicks,
3561
- ${METRIC_EXPR.impressions} AS impressions,
3562
- ${METRIC_EXPR.ctr} AS ctr,
3563
- ${METRIC_EXPR.position} AS position
3564
- FROM read_parquet({{FILES}}, union_by_name = true)
3565
- WHERE date >= ? AND date <= ?
3566
- GROUP BY query, url
3567
- HAVING SUM(impressions) >= ?
3568
- ),
3569
- scored AS (
3570
- SELECT
3571
- keyword, page, clicks, impressions, ctr, position,
3572
- CASE
3573
- WHEN position <= 3 THEN 0.2
3574
- WHEN position > 50 THEN 0.1
3575
- ELSE GREATEST(0.0, 1.0 - ABS(position - 11.0) / 15.0)
3576
- END AS positionScore,
3577
- CASE WHEN impressions <= 0 THEN 0.0 ELSE LEAST(LOG10(impressions) / 5.0, 1.0) END AS impressionScore,
3578
- CASE CAST(ROUND(GREATEST(LEAST(position, 10.0), 1.0)) AS INTEGER)
3579
- WHEN 1 THEN 0.30
3580
- WHEN 2 THEN 0.15
3581
- WHEN 3 THEN 0.10
3582
- WHEN 4 THEN 0.07
3583
- WHEN 5 THEN 0.05
3584
- WHEN 6 THEN 0.04
3585
- WHEN 7 THEN 0.03
3586
- WHEN 8 THEN 0.025
3587
- WHEN 9 THEN 0.02
3588
- WHEN 10 THEN 0.015
3589
- ELSE 0.01
3590
- END AS expectedCtr
3591
- FROM agg
3592
- ),
3593
- gapped AS (
3594
- SELECT
3595
- *,
3596
- CASE WHEN ctr >= expectedCtr THEN 0.0 ELSE LEAST((expectedCtr - ctr) / expectedCtr, 1.0) END AS ctrGapScore
3597
- FROM scored
3598
- )
3599
- SELECT
3600
- keyword, page, clicks, impressions, ctr, position,
3601
- CAST(ROUND(POWER(
3602
- POWER(positionScore, ${w1}) * POWER(impressionScore, ${w2}) * POWER(ctrGapScore, ${w3}),
3603
- 1.0 / ${totalW}
3604
- ) * 100) AS DOUBLE) AS opportunityScore,
3605
- CAST(ROUND(impressions * (
3606
- CASE CAST(ROUND(GREATEST(LEAST(position, 3.0), 1.0)) AS INTEGER)
3607
- WHEN 1 THEN 0.30
3608
- WHEN 2 THEN 0.15
3609
- WHEN 3 THEN 0.10
3610
- ELSE 0.10
3611
- END
3612
- )) AS DOUBLE) AS potentialClicks,
3613
- positionScore, impressionScore, ctrGapScore
3614
- FROM gapped
3615
- ORDER BY opportunityScore DESC
3616
- ${paginateClause({
3617
- limit,
3618
- offset: params.offset
3619
- })}
3620
- `,
3621
- params: [
3622
- startDate,
3623
- endDate,
3624
- minImpressions
3625
- ],
3626
- current: {
3627
- table: "page_queries",
3628
- partitions: enumeratePartitions(startDate, endDate)
3629
- }
3630
- };
3631
- },
3632
- reduceSql(rows) {
3633
- const arr = Array.isArray(rows) ? rows : [];
3634
- return {
3635
- results: arr.map((r) => ({
3636
- keyword: r.keyword == null ? "" : String(r.keyword),
3637
- page: r.page == null ? null : String(r.page),
3638
- clicks: num(r.clicks),
3639
- impressions: num(r.impressions),
3640
- ctr: num(r.ctr),
3641
- position: num(r.position),
3642
- opportunityScore: num(r.opportunityScore),
3643
- potentialClicks: num(r.potentialClicks),
3644
- factors: {
3645
- positionScore: num(r.positionScore),
3646
- impressionScore: num(r.impressionScore),
3647
- ctrGapScore: num(r.ctrGapScore)
3648
- }
3649
- })),
3650
- meta: { total: arr.length }
3651
- };
3652
- },
3653
- buildRows(params) {
3654
- return { queries: queriesQueryState(periodOf(params), params.limit) };
3655
- },
3656
- reduceRows(rows, params) {
3657
- const results = analyzeOpportunity((Array.isArray(rows) ? rows : []) ?? [], { minImpressions: params.minImpressions });
3658
- const paged = paginateSortedInMemory(results, {
3659
- limit: params.limit,
3660
- offset: params.offset
3661
- }, (left, right) => right.opportunityScore - left.opportunityScore);
3662
- return {
3663
- results: paged,
3664
- meta: {
3665
- total: results.length,
3666
- returned: paged.length
3667
- }
3668
- };
3669
- }
3670
- });
3671
- const positionDistributionAnalyzer = defineAnalyzer({
3672
- id: "position-distribution",
3673
- buildSql(params) {
3674
- const { startDate, endDate } = periodOf(params);
3675
- return {
3676
- sql: `
3677
- WITH pos AS (
3678
- SELECT
3679
- date,
3680
- (sum_position / NULLIF(impressions, 0) + 1) AS avg_pos
3681
- FROM read_parquet({{FILES}}, union_by_name = true)
3682
- WHERE date >= ? AND date <= ? AND impressions > 0
3683
- )
3684
- SELECT
3685
- date,
3686
- CAST(SUM(CASE WHEN avg_pos <= 3 THEN 1 ELSE 0 END) AS DOUBLE) AS pos_1_3,
3687
- CAST(SUM(CASE WHEN avg_pos > 3 AND avg_pos <= 10 THEN 1 ELSE 0 END) AS DOUBLE) AS pos_4_10,
3688
- CAST(SUM(CASE WHEN avg_pos > 10 AND avg_pos <= 20 THEN 1 ELSE 0 END) AS DOUBLE) AS pos_11_20,
3689
- CAST(SUM(CASE WHEN avg_pos > 20 THEN 1 ELSE 0 END) AS DOUBLE) AS pos_20_plus,
3690
- CAST(COUNT(*) AS DOUBLE) AS total
3691
- FROM pos
3692
- GROUP BY date
3693
- ORDER BY date ASC
3694
- `,
3695
- params: [startDate, endDate],
3696
- current: {
3697
- table: "queries",
3698
- partitions: enumeratePartitions(startDate, endDate)
3699
- }
3700
- };
3701
- },
3702
- reduceSql(rows, params) {
3703
- const arr = Array.isArray(rows) ? rows : [];
3704
- const { startDate, endDate } = periodOf(params);
3705
- return {
3706
- results: arr.map((r) => ({
3707
- date: rowString(r.date),
3708
- pos_1_3: num(r.pos_1_3),
3709
- pos_4_10: num(r.pos_4_10),
3710
- pos_11_20: num(r.pos_11_20),
3711
- pos_20_plus: num(r.pos_20_plus),
3712
- total: num(r.total)
3713
- })),
3714
- meta: {
3715
- total: arr.length,
3716
- startDate,
3717
- endDate
3718
- }
3719
- };
3720
- }
3721
- });
3722
- const positionVolatilityAnalyzer = defineAnalyzer({
3723
- id: "position-volatility",
3724
- buildSql(params) {
3725
- const { startDate, endDate } = periodOf(params);
3726
- const topN = params.topN ?? 30;
3727
- const minDayImpressions = params.minImpressions ?? 10;
3728
- const minDays = params.minWeeksWithData ?? 7;
3729
- return {
3730
- sql: `
3731
- WITH query_day AS (
3732
- SELECT
3733
- url AS page,
3734
- query,
3735
- -- Normalize at the source CTE: union_by_name=true can coerce date to
3736
- -- VARCHAR across parquets with mixed schemas, which makes downstream
3737
- -- strftime(date, ...) binder-error.
3738
- CAST(date AS DATE) AS date,
3739
- ${METRIC_EXPR.impressions} AS q_impressions,
3740
- ${METRIC_EXPR.position} AS q_position
3741
- FROM read_parquet({{FILES}}, union_by_name = true)
3742
- WHERE date >= ? AND date <= ?
3743
- AND query IS NOT NULL AND query <> ''
3744
- AND url IS NOT NULL AND url <> ''
3745
- GROUP BY url, query, date
3746
- HAVING SUM(impressions) >= 1
3747
- ),
3748
- daily AS (
3749
- SELECT
3750
- page, date,
3751
- COUNT(*) AS query_count,
3752
- SUM(q_impressions) AS day_impressions,
3753
- SUM(q_position * q_impressions) / NULLIF(SUM(q_impressions), 0) AS avg_position,
3754
- COALESCE(STDDEV_POP(q_position), 0.0) AS pos_stddev,
3755
- MIN(q_position) AS best_position,
3756
- MAX(q_position) AS worst_position
3757
- FROM query_day
3758
- GROUP BY page, date
3759
- HAVING SUM(q_impressions) >= ?
3760
- ),
3761
- with_shift AS (
3762
- SELECT *,
3763
- LAG(avg_position) OVER (PARTITION BY page ORDER BY date) AS prev_position,
3764
- COALESCE(
3765
- ABS(avg_position - LAG(avg_position) OVER (PARTITION BY page ORDER BY date)),
3766
- 0.0
3767
- ) AS dod_shift
3768
- FROM daily
3769
- ),
3770
- scored AS (
3771
- SELECT *,
3772
- pos_stddev + dod_shift AS volatility
3773
- FROM with_shift
3774
- ),
3775
- top_pages AS (
3776
- SELECT page,
3777
- SUM(day_impressions) AS total_impressions,
3778
- AVG(volatility) AS avg_volatility,
3779
- MAX(volatility) AS peak_volatility,
3780
- COUNT(*) AS days_with_data
3781
- FROM scored
3782
- GROUP BY page
3783
- HAVING COUNT(*) >= ?
3784
- ORDER BY avg_volatility DESC
3785
- LIMIT ${Number(topN)}
3786
- )
3787
- SELECT
3788
- s.page,
3789
- strftime(s.date, '%Y-%m-%d') AS date,
3790
- s.query_count AS queryCount,
3791
- s.day_impressions AS dayImpressions,
3792
- s.avg_position AS avgPosition,
3793
- s.pos_stddev AS posStddev,
3794
- s.best_position AS bestPosition,
3795
- s.worst_position AS worstPosition,
3796
- s.dod_shift AS dodShift,
3797
- s.volatility AS volatility,
3798
- t.avg_volatility AS pageAvgVolatility,
3799
- t.peak_volatility AS pagePeakVolatility,
3800
- t.total_impressions AS pageTotalImpressions
3801
- FROM scored s
3802
- JOIN top_pages t USING (page)
3803
- ORDER BY t.avg_volatility DESC, s.date ASC
3804
- `,
3805
- params: [
3806
- startDate,
3807
- endDate,
3808
- minDayImpressions,
3809
- minDays
3810
- ],
3811
- current: {
3812
- table: "page_queries",
3813
- partitions: enumeratePartitions(startDate, endDate)
3814
- }
3815
- };
3816
- },
3817
- reduceSql(rows) {
3818
- const arr = Array.isArray(rows) ? rows : [];
3819
- const byPage = /* @__PURE__ */ new Map();
3820
- const allDates = /* @__PURE__ */ new Set();
3821
- for (const r of arr) {
3822
- const page = rowString(r.page);
3823
- const date = rowString(r.date);
3824
- allDates.add(date);
3825
- const entry = byPage.get(page) ?? {
3826
- page,
3827
- avgVolatility: num(r.pageAvgVolatility),
3828
- peakVolatility: num(r.pagePeakVolatility),
3829
- totalImpressions: num(r.pageTotalImpressions),
3830
- days: []
3831
- };
3832
- entry.days.push({
3833
- date,
3834
- queryCount: num(r.queryCount),
3835
- dayImpressions: num(r.dayImpressions),
3836
- avgPosition: num(r.avgPosition),
3837
- posStddev: num(r.posStddev),
3838
- bestPosition: num(r.bestPosition),
3839
- worstPosition: num(r.worstPosition),
3840
- dodShift: num(r.dodShift),
3841
- volatility: num(r.volatility)
3842
- });
3843
- byPage.set(page, entry);
3844
- }
3845
- const pages = [...byPage.values()].sort((a, b) => b.avgVolatility - a.avgVolatility);
3846
- const dates = [...allDates].sort();
3847
- const maxVolatility = pages.reduce((m, p) => Math.max(m, p.peakVolatility), 0);
3848
- return {
3849
- results: pages,
3850
- meta: {
3851
- total: pages.length,
3852
- dates,
3853
- maxVolatility
3854
- }
3855
- };
3856
- }
3857
- });
3858
- const queryMigrationAnalyzer = defineAnalyzer({
3859
- id: "query-migration",
3860
- buildSql(params) {
3861
- const cur = periodOf(params);
3862
- let prevStart = params.prevStartDate;
3863
- let prevEnd = params.prevEndDate;
3864
- if (prevStart == null || prevEnd == null) {
3865
- const curStartMs = new Date(cur.startDate).getTime();
3866
- const span = new Date(cur.endDate).getTime() - curStartMs;
3867
- prevEnd = toIsoDate(new Date(curStartMs - MS_PER_DAY));
3868
- prevStart = toIsoDate(new Date(curStartMs - MS_PER_DAY - span));
3869
- }
3870
- const minImpressions = params.minImpressions ?? 20;
3871
- const limit = params.limit ?? 200;
3872
- const maxLevenshtein = 2;
3873
- return {
3874
- sql: `
3875
- WITH cur AS (
3876
- SELECT query, url AS page,
3877
- ${METRIC_EXPR.impressions} AS impressions,
3878
- ${METRIC_EXPR.clicks} AS clicks,
3879
- ${METRIC_EXPR.position} AS position
3880
- FROM read_parquet({{FILES}}, union_by_name = true)
3881
- WHERE date >= ? AND date <= ?
3882
- AND query IS NOT NULL AND query <> ''
3883
- AND url IS NOT NULL AND url <> ''
3884
- GROUP BY query, url
3885
- HAVING SUM(impressions) >= ?
3886
- ),
3887
- prev AS (
3888
- SELECT query, url AS page,
3889
- ${METRIC_EXPR.impressions} AS impressions,
3890
- ${METRIC_EXPR.clicks} AS clicks,
3891
- ${METRIC_EXPR.position} AS position
3892
- FROM read_parquet({{FILES_PREV}}, union_by_name = true)
3893
- WHERE date >= ? AND date <= ?
3894
- AND query IS NOT NULL AND query <> ''
3895
- AND url IS NOT NULL AND url <> ''
3896
- GROUP BY query, url
3897
- HAVING SUM(impressions) >= ?
3898
- ),
3899
- lost AS (
3900
- SELECT p.page AS source_page, p.query AS source_query, p.impressions AS source_impressions
3901
- FROM prev p
3902
- LEFT JOIN cur c ON p.page = c.page AND p.query = c.query
3903
- WHERE c.query IS NULL
3904
- ),
3905
- gained AS (
3906
- SELECT c.page AS target_page, c.query AS target_query, c.impressions AS target_impressions
3907
- FROM cur c
3908
- LEFT JOIN prev p ON p.page = c.page AND p.query = c.query
3909
- WHERE p.query IS NULL
3910
- ),
3911
- matched AS (
3912
- SELECT
3913
- l.source_page, l.source_query, l.source_impressions,
3914
- g.target_page, g.target_query, g.target_impressions,
3915
- CASE
3916
- WHEN l.source_query = g.target_query THEN 'exact'
3917
- ELSE 'fuzzy'
3918
- END AS match_type,
3919
- LEAST(l.source_impressions, g.target_impressions) AS absorbed_impressions
3920
- FROM lost l
3921
- JOIN gained g
3922
- ON l.source_page <> g.target_page
3923
- AND ABS(LENGTH(l.source_query) - LENGTH(g.target_query)) <= ${maxLevenshtein}
3924
- AND (
3925
- l.source_query = g.target_query
3926
- OR levenshtein(l.source_query, g.target_query) <= ${maxLevenshtein}
3927
- )
3928
- ),
3929
- edges AS (
3930
- SELECT
3931
- source_page, target_page,
3932
- SUM(absorbed_impressions) AS weight,
3933
- COUNT(*) AS query_count,
3934
- SUM(CASE WHEN match_type = 'exact' THEN 1 ELSE 0 END) AS exact_count,
3935
- to_json(list({
3936
- 'sourceQuery': source_query,
3937
- 'targetQuery': target_query,
3938
- 'absorbed': absorbed_impressions,
3939
- 'matchType': match_type
3940
- } ORDER BY absorbed_impressions DESC)) AS examplesJson
3941
- FROM matched
3942
- GROUP BY source_page, target_page
3943
- )
3944
- SELECT *
3945
- FROM edges
3946
- ORDER BY weight DESC
3947
- LIMIT ${Number(limit)}
3948
- `,
3949
- params: [
3950
- cur.startDate,
3951
- cur.endDate,
3952
- minImpressions,
3953
- prevStart,
3954
- prevEnd,
3955
- minImpressions
3956
- ],
3957
- current: {
3958
- table: "page_queries",
3959
- partitions: enumeratePartitions(cur.startDate, cur.endDate)
3960
- },
3961
- previous: {
3962
- table: "page_queries",
3963
- partitions: enumeratePartitions(prevStart, prevEnd)
3964
- }
3965
- };
3966
- },
3967
- reduceSql(rows, params) {
3968
- const arr = Array.isArray(rows) ? rows : [];
3969
- const cur = periodOf(params);
3970
- let prevStart = params.prevStartDate;
3971
- let prevEnd = params.prevEndDate;
3972
- if (prevStart == null || prevEnd == null) {
3973
- const curStartMs = new Date(cur.startDate).getTime();
3974
- const span = new Date(cur.endDate).getTime() - curStartMs;
3975
- prevEnd = toIsoDate(new Date(curStartMs - MS_PER_DAY));
3976
- prevStart = toIsoDate(new Date(curStartMs - MS_PER_DAY - span));
3977
- }
3978
- const edges = arr.map((r) => ({
3979
- sourcePage: rowString(r.source_page),
3980
- targetPage: rowString(r.target_page),
3981
- weight: num(r.weight),
3982
- queryCount: num(r.query_count),
3983
- exactCount: num(r.exact_count),
3984
- fuzzyCount: num(r.query_count) - num(r.exact_count),
3985
- examples: parseJsonRows(r.examplesJson).slice(0, 8).map((e) => ({
3986
- sourceQuery: rowString(e.sourceQuery),
3987
- targetQuery: rowString(e.targetQuery),
3988
- absorbed: num(e.absorbed),
3989
- matchType: rowString(e.matchType)
3990
- }))
3991
- }));
3992
- const nodeAgg = /* @__PURE__ */ new Map();
3993
- for (const e of edges) {
3994
- const src = nodeAgg.get(e.sourcePage) ?? {
3995
- url: e.sourcePage,
3996
- outgoing: 0,
3997
- incoming: 0
3998
- };
3999
- src.outgoing += e.weight;
4000
- nodeAgg.set(e.sourcePage, src);
4001
- const tgt = nodeAgg.get(e.targetPage) ?? {
4002
- url: e.targetPage,
4003
- outgoing: 0,
4004
- incoming: 0
4005
- };
4006
- tgt.incoming += e.weight;
4007
- nodeAgg.set(e.targetPage, tgt);
4008
- }
4009
- const nodes = [...nodeAgg.values()];
4010
- const totalAbsorbed = edges.reduce((s, e) => s + e.weight, 0);
4011
- return {
4012
- results: edges,
4013
- meta: {
4014
- total: edges.length,
4015
- totalAbsorbed,
4016
- period: {
4017
- current: cur,
4018
- previous: {
4019
- startDate: prevStart,
4020
- endDate: prevEnd
4021
- }
4022
- },
4023
- nodes
4024
- }
4025
- };
4026
- }
4027
- });
4028
- function calculateCV(values) {
4029
- if (values.length === 0) return 0;
4030
- const mean = values.reduce((a, b) => a + b, 0) / values.length;
4031
- if (mean === 0) return 0;
4032
- const variance = values.reduce((sum, v) => sum + (v - mean) ** 2, 0) / values.length;
4033
- return Math.min(Math.sqrt(variance) / mean, 1);
4034
- }
4035
- function analyzeSeasonality(dates, options = {}) {
4036
- const { metric = "clicks" } = options;
4037
- if (dates.length === 0) return {
4038
- hasSeasonality: false,
4039
- strength: 0,
4040
- peakMonths: [],
4041
- troughMonths: [],
4042
- monthlyBreakdown: [],
4043
- insufficientData: true
4044
- };
4045
- const monthlyMap = /* @__PURE__ */ new Map();
4046
- for (const row of dates) {
4047
- const month = row.date.substring(0, 7);
4048
- const value = metric === "clicks" ? row.clicks : row.impressions;
4049
- monthlyMap.set(month, (monthlyMap.get(month) || 0) + value);
4050
- }
4051
- const months = Array.from(monthlyMap.keys()).sort();
4052
- const values = months.map((m) => monthlyMap.get(m) || 0);
4053
- const insufficientData = months.length < 12;
4054
- const totalValue = values.reduce((a, b) => a + b, 0);
4055
- const avgValue = values.length > 0 ? totalValue / values.length : 0;
4056
- const monthlyBreakdown = months.map((month, i) => {
4057
- const value = values[i] ?? 0;
4058
- const vsAverage = avgValue > 0 ? value / avgValue : 0;
4059
- return {
4060
- month,
4061
- value,
4062
- vsAverage,
4063
- isPeak: vsAverage > 1.5,
4064
- isTrough: vsAverage < .5
4065
- };
4066
- });
4067
- const peakMonths = [...new Set(monthlyBreakdown.filter((m) => m.isPeak).map((m) => m.month.substring(5, 7)))];
4068
- const troughMonths = [...new Set(monthlyBreakdown.filter((m) => m.isTrough).map((m) => m.month.substring(5, 7)))];
4069
- const strength = calculateCV(values);
4070
- return {
4071
- hasSeasonality: peakMonths.length > 0 || troughMonths.length > 0 || strength > .3,
4072
- strength,
4073
- peakMonths,
4074
- troughMonths,
4075
- monthlyBreakdown,
4076
- insufficientData
4077
- };
4078
- }
4079
- const seasonalityAnalyzer = defineAnalyzer({
4080
- id: "seasonality",
4081
- buildSql(params) {
4082
- const { startDate, endDate } = periodOf(params);
4083
- return {
4084
- sql: `
4085
- WITH monthly AS (
4086
- SELECT
4087
- strftime(CAST(date AS DATE), '%Y-%m') AS month,
4088
- CAST(SUM(${params.metric === "impressions" ? "impressions" : "clicks"}) AS DOUBLE) AS value
4089
- FROM read_parquet({{FILES}}, union_by_name = true)
4090
- WHERE date >= ? AND date <= ?
4091
- GROUP BY month
4092
- ),
4093
- stats AS (
4094
- SELECT
4095
- AVG(value) AS avg_val,
4096
- COALESCE(STDDEV_POP(value), 0.0) AS std_val,
4097
- CAST(COUNT(*) AS DOUBLE) AS month_count
4098
- FROM monthly
4099
- )
4100
- SELECT
4101
- m.month AS month,
4102
- m.value AS value,
4103
- CASE WHEN s.avg_val > 0 THEN m.value / s.avg_val ELSE 0.0 END AS vsAverage,
4104
- (s.avg_val > 0 AND m.value / s.avg_val > 1.5) AS isPeak,
4105
- (s.avg_val > 0 AND m.value / s.avg_val < 0.5) AS isTrough,
4106
- CASE WHEN s.avg_val > 0 THEN LEAST(s.std_val / s.avg_val, 1.0) ELSE 0.0 END AS strength,
4107
- s.month_count AS monthCount
4108
- FROM monthly m, stats s
4109
- ORDER BY m.month
4110
- `,
4111
- params: [startDate, endDate],
4112
- current: {
4113
- table: "pages",
4114
- partitions: enumeratePartitions(startDate, endDate)
4115
- }
4116
- };
4117
- },
4118
- reduceSql(rows) {
4119
- const arr = Array.isArray(rows) ? rows : [];
4120
- const breakdown = arr.map((r) => ({
4121
- month: rowString(r.month),
4122
- value: num(r.value),
4123
- vsAverage: num(r.vsAverage),
4124
- isPeak: rowBoolean(r.isPeak),
4125
- isTrough: rowBoolean(r.isTrough)
4126
- }));
4127
- const first = arr[0];
4128
- const strength = first ? num(first.strength) : 0;
4129
- const monthCount = first ? num(first.monthCount) : 0;
4130
- const peakMonths = [...new Set(breakdown.filter((m) => m.isPeak).map((m) => m.month.substring(5, 7)))];
4131
- const troughMonths = [...new Set(breakdown.filter((m) => m.isTrough).map((m) => m.month.substring(5, 7)))];
4132
- const hasSeasonality = peakMonths.length > 0 || troughMonths.length > 0 || strength > .3;
4133
- const insufficientData = monthCount < 12;
4134
- return {
4135
- results: breakdown,
4136
- meta: {
4137
- total: breakdown.length,
4138
- hasSeasonality,
4139
- strength,
4140
- peakMonths,
4141
- troughMonths,
4142
- insufficientData
4143
- }
4144
- };
4145
- },
4146
- buildRows(params) {
4147
- return { dates: datesQueryState(periodOf(params), params.limit) };
4148
- },
4149
- reduceRows(rows, params) {
4150
- const result = analyzeSeasonality(Array.isArray(rows) ? rows : [], { metric: params.metric });
4151
- return {
4152
- results: result.monthlyBreakdown,
4153
- meta: { strength: result.strength }
4154
- };
4155
- }
4156
- });
4157
- const stlDecomposeAnalyzer = defineAnalyzer({
4158
- id: "stl-decompose",
4159
- buildSql(params) {
4160
- const endDate = params.endDate ?? defaultEndDate();
4161
- const startDate = params.startDate ?? daysAgoUtc(93);
4162
- const minImpressions = params.minImpressions ?? 100;
4163
- const minDays = 21;
4164
- const metric = params.metric === "clicks" ? "clicks" : "impressions";
4165
- const limit = params.limit ?? 100;
4166
- return {
4167
- sql: `
4168
- WITH daily AS (
4169
- SELECT
4170
- query,
4171
- url AS page,
4172
- -- Normalize at the source CTE: union_by_name=true can coerce date to
4173
- -- VARCHAR across parquets with mixed schemas, which makes downstream
4174
- -- strftime(date, ...) binder-error.
4175
- CAST(date AS DATE) AS date,
4176
- ${METRIC_EXPR.clicks} AS clicks,
4177
- ${METRIC_EXPR.impressions} AS impressions,
4178
- CAST(SUM(${metric}) AS DOUBLE) AS observed
4179
- FROM read_parquet({{FILES}}, union_by_name = true)
4180
- WHERE date >= ? AND date <= ?
4181
- AND query IS NOT NULL AND query <> ''
4182
- AND url IS NOT NULL AND url <> ''
4183
- GROUP BY query, url, date
4184
- ),
4185
- entity_stats AS (
4186
- SELECT query, page,
4187
- COUNT(*) AS days,
4188
- SUM(impressions) AS total_impressions
4189
- FROM daily
4190
- GROUP BY query, page
4191
- HAVING COUNT(*) >= ${Number(minDays)}
4192
- AND SUM(impressions) >= ?
4193
- ),
4194
- filtered AS (
4195
- SELECT d.*
4196
- FROM daily d
4197
- JOIN entity_stats e USING (query, page)
4198
- ),
4199
- trended AS (
4200
- SELECT *,
4201
- CASE
4202
- WHEN COUNT(*) OVER w = 7
4203
- THEN AVG(observed) OVER w
4204
- ELSE NULL
4205
- END AS trend
4206
- FROM filtered
4207
- WINDOW w AS (
4208
- PARTITION BY query, page
4209
- ORDER BY date
4210
- ROWS BETWEEN 3 PRECEDING AND 3 FOLLOWING
4211
- )
4212
- ),
4213
- detrended AS (
4214
- SELECT *,
4215
- observed - trend AS detrended,
4216
- dayofweek(date) AS dow
4217
- FROM trended
4218
- ),
4219
- seasonal_raw AS (
4220
- SELECT *,
4221
- AVG(detrended) OVER (PARTITION BY query, page, dow) AS seasonal_dow
4222
- FROM detrended
4223
- ),
4224
- seasonal_centered AS (
4225
- SELECT *,
4226
- seasonal_dow - AVG(seasonal_dow) OVER (PARTITION BY query, page) AS seasonal
4227
- FROM seasonal_raw
4228
- ),
4229
- residualed AS (
4230
- SELECT *,
4231
- CASE
4232
- WHEN trend IS NULL OR seasonal IS NULL THEN NULL
4233
- ELSE observed - trend - seasonal
4234
- END AS residual
4235
- FROM seasonal_centered
4236
- ),
4237
- scored AS (
4238
- SELECT *,
4239
- STDDEV_POP(residual) OVER (PARTITION BY query, page) AS resid_std,
4240
- CASE
4241
- WHEN residual IS NOT NULL
4242
- AND STDDEV_POP(residual) OVER (PARTITION BY query, page) > 0
4243
- AND ABS(residual) > 2.0 * STDDEV_POP(residual) OVER (PARTITION BY query, page)
4244
- THEN true ELSE false
4245
- END AS anomaly
4246
- FROM residualed
4247
- ),
4248
- per_entity AS (
4249
- SELECT query, page,
4250
- COUNT(*) AS days,
4251
- SUM(impressions) AS total_impressions,
4252
- VAR_POP(detrended) AS var_detrended,
4253
- VAR_POP(seasonal) AS var_seasonal,
4254
- VAR_POP(residual) AS var_residual,
4255
- COUNT(*) FILTER (WHERE anomaly) AS residual_anomalies,
4256
- REGR_SLOPE(observed, epoch(date) / 86400.0) AS trend_slope
4257
- FROM scored
4258
- GROUP BY query, page
4259
- ),
4260
- series AS (
4261
- SELECT query, page,
4262
- to_json(list({
4263
- 'date': strftime(date, '%Y-%m-%d'),
4264
- 'observed': observed,
4265
- 'trend': trend,
4266
- 'seasonal': seasonal,
4267
- 'residual': residual,
4268
- 'anomaly': anomaly
4269
- } ORDER BY date)) AS seriesJson
4270
- FROM scored
4271
- GROUP BY query, page
4272
- )
4273
- SELECT
4274
- e.query AS keyword,
4275
- e.page,
4276
- CAST(e.total_impressions AS DOUBLE) AS totalImpressions,
4277
- CAST(e.days AS DOUBLE) AS days,
4278
- CASE
4279
- WHEN e.var_detrended IS NULL OR e.var_detrended = 0 THEN 0.0
4280
- ELSE LEAST(e.var_seasonal / NULLIF(e.var_detrended, 0), 1.0)
4281
- END AS seasonalStrength,
4282
- CASE
4283
- WHEN e.var_detrended IS NULL OR e.var_detrended = 0 THEN 0.0
4284
- ELSE GREATEST(0.0, 1.0 - e.var_residual / NULLIF(e.var_detrended, 0))
4285
- END AS trendStrength,
4286
- CAST(e.residual_anomalies AS DOUBLE) AS residualAnomalies,
4287
- COALESCE(e.trend_slope, 0.0) AS trendSlope,
4288
- s.seriesJson
4289
- FROM per_entity e
4290
- LEFT JOIN series s USING (query, page)
4291
- ORDER BY seasonalStrength DESC, ABS(COALESCE(e.trend_slope, 0.0)) DESC
4292
- LIMIT ${Number(limit)}
4293
- `,
4294
- params: [
4295
- startDate,
4296
- endDate,
4297
- minImpressions
4298
- ],
4299
- current: {
4300
- table: "page_queries",
4301
- partitions: enumeratePartitions(startDate, endDate)
4302
- }
4303
- };
4304
- },
4305
- reduceSql(rows, params) {
4306
- const arr = Array.isArray(rows) ? rows : [];
4307
- const metric = params.metric === "clicks" ? "clicks" : "impressions";
4308
- const results = arr.map((r) => ({
4309
- keyword: rowString(r.keyword),
4310
- page: rowString(r.page),
4311
- totalImpressions: num(r.totalImpressions),
4312
- days: num(r.days),
4313
- seasonalStrength: num(r.seasonalStrength),
4314
- trendStrength: num(r.trendStrength),
4315
- residualAnomalies: num(r.residualAnomalies),
4316
- trendSlope: num(r.trendSlope),
4317
- series: parseJsonRows(r.seriesJson).map((s) => ({
4318
- date: rowString(s.date),
4319
- observed: num(s.observed),
4320
- trend: s.trend == null ? null : num(s.trend),
4321
- seasonal: s.seasonal == null ? null : num(s.seasonal),
4322
- residual: s.residual == null ? null : num(s.residual),
4323
- anomaly: rowBoolean(s.anomaly)
4324
- }))
4325
- }));
4326
- return {
4327
- results,
4328
- meta: {
4329
- total: results.length,
4330
- metric,
4331
- avgSeasonalStrength: results.length > 0 ? results.reduce((a, r) => a + r.seasonalStrength, 0) / results.length : 0
4332
- }
4333
- };
4334
- }
4335
- });
4336
- const DEFAULT_ROW_LIMIT$1 = 25e3;
4337
- function paginateStrikingDistance(results, options) {
4338
- return paginateSortedInMemory(results, {
4339
- limit: options.limit ?? 1e3,
4340
- offset: options.offset
4341
- }, (left, right) => right.potentialClicks - left.potentialClicks);
4342
- }
4343
- function filterStrikingDistance(rows, options = {}) {
4344
- const minPosition = options.minPosition ?? 4;
4345
- const maxPosition = options.maxPosition ?? 20;
4346
- const minImpressions = options.minImpressions ?? 100;
4347
- const maxCtr = options.maxCtr ?? .05;
4348
- const results = [];
4349
- for (const row of rows) {
4350
- const position = num(row.position);
4351
- const impressions = num(row.impressions);
4352
- const ctr = num(row.ctr);
4353
- const clicks = num(row.clicks);
4354
- if (position < minPosition || position > maxPosition) continue;
4355
- if (impressions < minImpressions) continue;
4356
- if (ctr > maxCtr) continue;
4357
- results.push({
4358
- keyword: String(row.query ?? ""),
4359
- page: row.page == null ? null : String(row.page),
4360
- clicks,
4361
- impressions,
4362
- ctr,
4363
- position,
4364
- potentialClicks: Math.round(impressions * .15)
4365
- });
4366
- }
4367
- return results;
4368
- }
4369
- const strikingDistanceAnalyzer = defineAnalyzer({
4370
- id: "striking-distance",
4371
- reduce(rows, params) {
4372
- const results = filterStrikingDistance(Array.isArray(rows) ? rows : [], params);
4373
- const paged = paginateStrikingDistance(results, params);
4374
- return {
4375
- results: paged,
4376
- meta: {
4377
- total: results.length,
4378
- returned: paged.length
4379
- }
4380
- };
4381
- },
4382
- buildSql(params) {
4383
- const { startDate, endDate } = periodOf(params);
4384
- return {
4385
- sql: `
4386
- SELECT
4387
- query,
4388
- url AS page,
4389
- CAST(SUM(clicks) AS DOUBLE) AS clicks,
4390
- CAST(SUM(impressions) AS DOUBLE) AS impressions,
4391
- CAST(SUM(clicks) AS DOUBLE) / NULLIF(SUM(impressions), 0) AS ctr,
4392
- SUM(sum_position) / NULLIF(SUM(impressions), 0) + 1 AS position
4393
- FROM read_parquet({{FILES}}, union_by_name = true)
4394
- WHERE date >= ? AND date <= ?
4395
- GROUP BY query, url
4396
- `,
4397
- params: [startDate, endDate],
4398
- current: {
4399
- table: "page_queries",
4400
- partitions: enumeratePartitions(startDate, endDate)
4401
- }
4402
- };
4403
- },
4404
- buildRows(params) {
4405
- return { queries: queriesQueryState(periodOf(params), params.limit ?? DEFAULT_ROW_LIMIT$1) };
4406
- }
4407
- });
4408
- const survivalAnalyzer = defineAnalyzer({
4409
- id: "survival",
4410
- buildSql(params) {
4411
- const endDate = params.endDate ?? defaultEndDate();
4412
- const startDate = params.startDate ?? daysAgoUtc(183);
4413
- const minImpressions = params.minImpressions ?? 5;
4414
- return {
4415
- sql: `
4416
- WITH daily AS (
4417
- SELECT
4418
- query,
4419
- url,
4420
- date,
4421
- ${METRIC_EXPR.clicks} AS day_clicks,
4422
- ${METRIC_EXPR.impressions} AS day_impressions,
4423
- ${METRIC_EXPR.position} AS day_position
4424
- FROM read_parquet({{FILES}}, union_by_name = true)
4425
- WHERE date >= ? AND date <= ?
4426
- AND query IS NOT NULL AND query <> ''
4427
- AND url IS NOT NULL AND url <> ''
4428
- GROUP BY query, url, date
4429
- HAVING SUM(impressions) >= ?
4430
- ),
4431
- classified AS (
4432
- SELECT *,
4433
- (day_position <= 10) AS in_top10
4434
- FROM daily
4435
- ),
4436
- transitions AS (
4437
- SELECT *,
4438
- CASE
4439
- WHEN in_top10 AND (LAG(in_top10) OVER w IS NULL OR NOT LAG(in_top10) OVER w)
4440
- THEN 1 ELSE 0
4441
- END AS is_entry
4442
- FROM classified
4443
- WINDOW w AS (PARTITION BY query, url ORDER BY date)
4444
- ),
4445
- run_ids AS (
4446
- SELECT *,
4447
- SUM(is_entry) OVER (PARTITION BY query, url ORDER BY date) AS run_id
4448
- FROM transitions
4449
- WHERE in_top10
4450
- ),
4451
- window_bounds AS (
4452
- SELECT MIN(date) AS window_start, MAX(date) AS window_end FROM daily
4453
- ),
4454
- episodes_raw AS (
4455
- SELECT
4456
- query, url, run_id,
4457
- MIN(date) AS entry_date,
4458
- MAX(date) AS exit_date,
4459
- DATEDIFF('day', MIN(date), MAX(date)) + 1 AS tenure
4460
- FROM run_ids
4461
- GROUP BY query, url, run_id
4462
- ),
4463
- episodes AS (
4464
- SELECT
4465
- e.query, e.url, e.run_id, e.entry_date, e.exit_date, e.tenure,
4466
- (e.exit_date >= wb.window_end - INTERVAL 2 DAY) AS censored,
4467
- CASE
4468
- WHEN regexp_extract(e.url, '^(?:https?://[^/]+)?(/[^/?#]*)', 1) = '/' OR e.url = '/'
4469
- THEN 'home'
4470
- WHEN regexp_extract(e.url, '^(?:https?://[^/]+)?/([^/?#]+)', 1) = ''
4471
- THEN 'home'
4472
- ELSE regexp_extract(e.url, '^(?:https?://[^/]+)?/([^/?#]+)', 1)
4473
- END AS cohort
4474
- FROM episodes_raw e
4475
- CROSS JOIN window_bounds wb
4476
- ),
4477
- episodes_all AS (
4478
- SELECT query, url, tenure, censored, cohort FROM episodes
4479
- UNION ALL
4480
- SELECT query, url, tenure, censored, '__all__' AS cohort FROM episodes
4481
- ),
4482
- cohort_totals AS (
4483
- SELECT cohort, COUNT(*) AS n_total
4484
- FROM episodes_all
4485
- GROUP BY cohort
4486
- ),
4487
- events AS (
4488
- SELECT
4489
- cohort,
4490
- tenure,
4491
- COUNT(*) FILTER (WHERE NOT censored) AS d_t,
4492
- COUNT(*) AS n_ending_at_t
4493
- FROM episodes_all
4494
- GROUP BY cohort, tenure
4495
- ),
4496
- km AS (
4497
- SELECT
4498
- e.cohort,
4499
- e.tenure,
4500
- e.d_t,
4501
- e.n_ending_at_t,
4502
- SUM(e.n_ending_at_t) OVER (PARTITION BY e.cohort ORDER BY e.tenure DESC
4503
- ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS at_risk
4504
- FROM events e
4505
- ),
4506
- km_surv AS (
4507
- SELECT
4508
- cohort, tenure, d_t, at_risk,
4509
- EXP(SUM(LN(GREATEST(1.0 - CAST(d_t AS DOUBLE) / NULLIF(at_risk, 0), 1e-9)))
4510
- OVER (PARTITION BY cohort ORDER BY tenure
4511
- ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)) AS survival
4512
- FROM km
4513
- ),
4514
- curve_agg AS (
4515
- SELECT
4516
- cohort,
4517
- to_json(list({
4518
- 'tenure': tenure,
4519
- 'survival': survival,
4520
- 'atRisk': at_risk,
4521
- 'events': d_t
4522
- } ORDER BY tenure)) AS curveJson
4523
- FROM km_surv
4524
- GROUP BY cohort
4525
- ),
4526
- cohort_stats AS (
4527
- SELECT
4528
- ea.cohort,
4529
- COUNT(*) AS episode_count,
4530
- AVG(CASE WHEN ea.censored THEN 1.0 ELSE 0.0 END) AS censoring_rate
4531
- FROM episodes_all ea
4532
- GROUP BY ea.cohort
4533
- )
4534
- SELECT
4535
- cs.cohort,
4536
- cs.episode_count AS episodeCount,
4537
- cs.censoring_rate AS censoringRate,
4538
- ca.curveJson
4539
- FROM cohort_stats cs
4540
- LEFT JOIN curve_agg ca USING (cohort)
4541
- ORDER BY cs.cohort
4542
- `,
4543
- params: [
4544
- startDate,
4545
- endDate,
4546
- minImpressions
4547
- ],
4548
- current: {
4549
- table: "page_queries",
4550
- partitions: enumeratePartitions(startDate, endDate)
4551
- }
4552
- };
4553
- },
4554
- reduceSql(rows, params) {
4555
- const arr = Array.isArray(rows) ? rows : [];
4556
- const endDate = params.endDate ?? defaultEndDate();
4557
- const startDate = params.startDate ?? daysAgoUtc(183);
4558
- const windowDays = Math.round((new Date(endDate).getTime() - new Date(startDate).getTime()) / MS_PER_DAY) + 1;
4559
- const results = arr.map((r) => {
4560
- const curve = parseJsonRows(r.curveJson).map((p) => ({
4561
- tenure: num(p.tenure),
4562
- survival: num(p.survival),
4563
- atRisk: num(p.atRisk),
4564
- events: num(p.events)
4565
- }));
4566
- let medianTenure = 0;
4567
- for (let i = 0; i < curve.length; i++) {
4568
- const cur = curve[i];
4569
- if (cur.survival <= .5) {
4570
- if (i === 0) medianTenure = cur.tenure;
4571
- else {
4572
- const prev = curve[i - 1];
4573
- const span = prev.survival - cur.survival;
4574
- const frac = span > 0 ? (prev.survival - .5) / span : 0;
4575
- medianTenure = prev.tenure + frac * (cur.tenure - prev.tenure);
4576
- }
4577
- break;
4578
- }
4579
- }
4580
- const last = curve[curve.length - 1];
4581
- if (medianTenure === 0 && last && last.survival > .5) medianTenure = last.tenure;
4582
- return {
4583
- cohort: rowString(r.cohort),
4584
- episodeCount: num(r.episodeCount),
4585
- censoringRate: num(r.censoringRate),
4586
- medianTenure,
4587
- curve
4588
- };
4589
- });
4590
- return {
4591
- results,
4592
- meta: {
4593
- totalEpisodes: results.find((r) => r.cohort === "__all__")?.episodeCount ?? 0,
4594
- cohortCount: results.filter((r) => r.cohort !== "__all__").length,
4595
- windowDays
4596
- }
4597
- };
4598
- }
4599
- });
4600
- const trendsAnalyzer = defineAnalyzer({
4601
- id: "trends",
4602
- buildSql(params) {
4603
- const weeks = params.weeks ?? 28;
4604
- const endDate = params.endDate || defaultEndDate();
4605
- const startDate = params.startDate || toIsoDate(/* @__PURE__ */ new Date(Date.parse(endDate) - (weeks * 7 - 1) * MS_PER_DAY));
4606
- const minImpressions = params.minImpressions ?? 100;
4607
- const minWeeksWithData = params.minWeeksWithData ?? Math.max(2, Math.floor(weeks / 4));
4608
- const limit = params.limit ?? 500;
4609
- const dim = params.dimension === "keywords" ? "keywords" : "pages";
4610
- const table = dim === "keywords" ? "queries" : "pages";
4611
- return {
4612
- sql: `
4613
- WITH bucketed AS (
4614
- SELECT
4615
- ${dim === "keywords" ? "query" : "url"} AS entity,
4616
- date_trunc('week', CAST(date AS DATE)) AS week,
4617
- ${METRIC_EXPR.clicks} AS clicks,
4618
- ${METRIC_EXPR.impressions} AS impressions,
4619
- SUM(sum_position) AS sum_position_sum
4620
- FROM read_parquet({{FILES}}, union_by_name = true)
4621
- WHERE date >= ? AND date <= ?
4622
- GROUP BY entity, week
4623
- ),
4624
- with_meta AS (
4625
- SELECT
4626
- entity, week, clicks, impressions, sum_position_sum,
4627
- ROW_NUMBER() OVER (PARTITION BY entity ORDER BY week) - 1 AS week_idx,
4628
- COUNT(*) OVER (PARTITION BY entity) AS n_weeks,
4629
- (ROW_NUMBER() OVER (PARTITION BY entity ORDER BY week) - 1)
4630
- < (COUNT(*) OVER (PARTITION BY entity) / 2) AS is_first_half
4631
- FROM bucketed
4632
- ),
4633
- agg AS (
4634
- SELECT
4635
- entity,
4636
- SUM(clicks) AS totalClicks,
4637
- SUM(impressions) AS totalImpressions,
4638
- any_value(n_weeks) AS weeksWithData,
4639
- COALESCE(regr_slope(clicks, CAST(week_idx AS DOUBLE)), 0.0) AS slope,
4640
- SUM(CASE WHEN is_first_half THEN clicks ELSE 0 END) AS firstHalfClicks,
4641
- SUM(CASE WHEN NOT is_first_half THEN clicks ELSE 0 END) AS secondHalfClicks,
4642
- SUM(sum_position_sum) / NULLIF(SUM(impressions), 0) + 1 AS avgPosition,
4643
- to_json(list({
4644
- 'week': strftime(week, '%Y-%m-%d'),
4645
- 'clicks': clicks,
4646
- 'impressions': impressions
4647
- } ORDER BY week)) AS seriesJson
4648
- FROM with_meta
4649
- GROUP BY entity
4650
- HAVING SUM(impressions) >= ? AND any_value(n_weeks) >= ?
4651
- ),
4652
- classified AS (
4653
- SELECT
4654
- *,
4655
- CASE
4656
- WHEN firstHalfClicks = 0 AND secondHalfClicks > 0 THEN 10.0
4657
- WHEN firstHalfClicks = 0 THEN 1.0
4658
- ELSE secondHalfClicks / firstHalfClicks
4659
- END AS growthRatio
4660
- FROM agg
4661
- )
4662
- SELECT
4663
- entity,
4664
- totalClicks,
4665
- totalImpressions,
4666
- weeksWithData,
4667
- slope,
4668
- growthRatio,
4669
- avgPosition,
4670
- CASE
4671
- WHEN growthRatio >= 1.5 AND slope > 0 THEN 'accelerating'
4672
- WHEN growthRatio >= 1.1 AND slope >= 0 THEN 'growing'
4673
- WHEN growthRatio < 0.5 THEN 'cratering'
4674
- WHEN growthRatio < 0.9 AND slope < 0 THEN 'declining'
4675
- ELSE 'steady'
4676
- END AS trend,
4677
- seriesJson
4678
- FROM classified
4679
- ORDER BY
4680
- CASE
4681
- WHEN growthRatio >= 1.5 AND slope > 0 THEN 0
4682
- WHEN growthRatio < 0.5 THEN 1
4683
- WHEN growthRatio >= 1.1 AND slope >= 0 THEN 2
4684
- WHEN growthRatio < 0.9 AND slope < 0 THEN 3
4685
- ELSE 4
4686
- END,
4687
- ABS(growthRatio - 1) DESC,
4688
- totalClicks DESC
4689
- LIMIT ${Number(limit)}
4690
- `,
4691
- params: [
4692
- startDate,
4693
- endDate,
4694
- minImpressions,
4695
- minWeeksWithData
4696
- ],
4697
- current: {
4698
- table,
4699
- partitions: enumeratePartitions(startDate, endDate)
4700
- }
4701
- };
4702
- },
4703
- reduceSql(rows, params) {
4704
- const arr = Array.isArray(rows) ? rows : [];
4705
- const weeks = params.weeks ?? 28;
4706
- const endDate = params.endDate || defaultEndDate();
4707
- const startDate = params.startDate || toIsoDate(/* @__PURE__ */ new Date(Date.parse(endDate) - (weeks * 7 - 1) * MS_PER_DAY));
4708
- const dim = params.dimension === "keywords" ? "keywords" : "pages";
4709
- const results = arr.map((r) => {
4710
- const series = parseJsonRows(r.seriesJson).map((s) => ({
4711
- week: rowString(s.week),
4712
- clicks: num(s.clicks),
4713
- impressions: num(s.impressions)
4714
- }));
4715
- return {
4716
- [dim === "keywords" ? "query" : "page"]: rowString(r.entity),
4717
- totalClicks: num(r.totalClicks),
4718
- totalImpressions: num(r.totalImpressions),
4719
- weeksWithData: num(r.weeksWithData),
4720
- slope: num(r.slope),
4721
- growthRatio: num(r.growthRatio),
4722
- avgPosition: num(r.avgPosition),
4723
- trend: rowString(r.trend),
4724
- series
4725
- };
4726
- });
4727
- const counts = {
4728
- accelerating: 0,
4729
- growing: 0,
4730
- steady: 0,
4731
- declining: 0,
4732
- cratering: 0
4733
- };
4734
- for (const r of results) counts[r.trend] = (counts[r.trend] ?? 0) + 1;
4735
- return {
4736
- results,
4737
- meta: {
4738
- total: results.length,
4739
- dimension: dim,
4740
- weeks: Number(weeks),
4741
- startDate,
4742
- endDate,
4743
- counts
4744
- }
4745
- };
4746
- }
4747
- });
4748
- const DEFAULT_ROW_LIMIT = 25e3;
4749
- function analyzeZeroClick(rows, options = {}) {
4750
- const minImpressions = options.minImpressions ?? 1e3;
4751
- const maxCtr = options.maxCtr ?? .03;
4752
- const maxPosition = options.maxPosition ?? 10;
4753
- const queryMap = /* @__PURE__ */ new Map();
4754
- for (const row of rows) {
4755
- if (row.impressions < minImpressions || row.position > maxPosition || row.ctr > maxCtr) continue;
4756
- const existing = queryMap.get(row.query);
4757
- if (!existing || row.position < existing.position) queryMap.set(row.query, { ...row });
4758
- }
4759
- return [...queryMap.values()].sort((left, right) => right.impressions - left.impressions);
4760
- }
4761
- const defaultAnalyzerRegistry = createAnalyzerRegistry({ defined: [
4762
- bayesianCtrAnalyzer,
4763
- bipartitePagerankAnalyzer,
4764
- brandAnalyzer,
4765
- cannibalizationAnalyzer,
4766
- changePointAnalyzer,
4767
- clusteringAnalyzer,
4768
- concentrationAnalyzer,
4769
- contentVelocityAnalyzer,
4770
- ctrAnomalyAnalyzer,
4771
- ctrCurveAnalyzer,
4772
- darkTrafficAnalyzer,
4773
- dataDetailAnalyzer,
4774
- dataQueryAnalyzer,
4775
- decayAnalyzer,
4776
- deviceGapAnalyzer,
4777
- intentAtlasAnalyzer,
4778
- keywordBreadthAnalyzer,
4779
- longTailAnalyzer,
4780
- moversAnalyzer,
4781
- opportunityAnalyzer,
4782
- positionDistributionAnalyzer,
4783
- positionVolatilityAnalyzer,
4784
- queryMigrationAnalyzer,
4785
- seasonalityAnalyzer,
4786
- stlDecomposeAnalyzer,
4787
- strikingDistanceAnalyzer,
4788
- survivalAnalyzer,
4789
- trendsAnalyzer,
4790
- defineAnalyzer({
4791
- id: "zero-click",
4792
- buildSql(params) {
4793
- const { startDate, endDate } = periodOf(params);
4794
- const minImpressions = params.minImpressions ?? 1e3;
4795
- const maxCtr = params.maxCtr ?? .03;
4796
- const maxPosition = params.maxPosition ?? 10;
4797
- const limit = params.limit ?? 1e3;
4798
- return {
4799
- sql: `
4800
- WITH agg AS (
4801
- SELECT
4802
- query,
4803
- url AS page,
4804
- ${METRIC_EXPR.clicks} AS clicks,
4805
- ${METRIC_EXPR.impressions} AS impressions,
4806
- ${METRIC_EXPR.ctr} AS ctr,
4807
- ${METRIC_EXPR.position} AS position
4808
- FROM read_parquet({{FILES}}, union_by_name = true)
4809
- WHERE date >= ? AND date <= ?
4810
- GROUP BY query, url
4811
- HAVING SUM(impressions) >= ?
4812
- )
4813
- SELECT
4814
- query, page, clicks, impressions, ctr, position,
4815
- CAST(GREATEST(0, ROUND(impressions * (
4816
- CASE
4817
- WHEN position <= 1 THEN 0.30
4818
- WHEN position <= 3 THEN 0.15
4819
- WHEN position <= 5 THEN 0.08
4820
- ELSE 0.04
4821
- END
4822
- )) - clicks) AS DOUBLE) AS missedClicks
4823
- FROM agg
4824
- WHERE position <= ? AND ctr < ?
4825
- ORDER BY impressions DESC
4826
- ${paginateClause({
4827
- limit,
4828
- offset: params.offset
4829
- })}
4830
- `,
4831
- params: [
4832
- startDate,
4833
- endDate,
4834
- minImpressions,
4835
- maxPosition,
4836
- maxCtr
4837
- ],
4838
- current: {
4839
- table: "page_queries",
4840
- partitions: enumeratePartitions(startDate, endDate)
4841
- }
4842
- };
4843
- },
4844
- reduceSql(rows, params) {
4845
- const arr = Array.isArray(rows) ? rows : [];
4846
- const minImpressions = params.minImpressions ?? 1e3;
4847
- const maxCtr = params.maxCtr ?? .03;
4848
- const maxPosition = params.maxPosition ?? 10;
4849
- return {
4850
- results: arr.map((r) => ({
4851
- query: r.query == null ? "" : String(r.query),
4852
- page: r.page == null ? "" : String(r.page),
4853
- clicks: num(r.clicks),
4854
- impressions: num(r.impressions),
4855
- ctr: num(r.ctr),
4856
- position: num(r.position),
4857
- missedClicks: num(r.missedClicks)
4858
- })),
4859
- meta: {
4860
- total: arr.length,
4861
- minImpressions,
4862
- maxCtr,
4863
- maxPosition
4864
- }
4865
- };
4866
- },
4867
- buildRows(params) {
4868
- const period = periodOf(params);
4869
- const limit = params.limit ?? DEFAULT_ROW_LIMIT;
4870
- return { rows: gsc.select(query, page).where(between(date, period.startDate, period.endDate)).limit(limit).getState() };
4871
- },
4872
- reduceRows(rows, params) {
4873
- const results = analyzeZeroClick(Array.isArray(rows) ? rows : [], {
4874
- minImpressions: params.minImpressions ?? 1e3,
4875
- maxCtr: params.maxCtr ?? .03,
4876
- maxPosition: params.maxPosition ?? 10
4877
- });
4878
- const paged = paginateSortedInMemory(results, {
4879
- limit: params.limit,
4880
- offset: params.offset
4881
- }, (left, right) => right.impressions - left.impressions);
4882
- return {
4883
- results: paged,
4884
- meta: {
4885
- total: results.length,
4886
- returned: paged.length
4887
- }
4888
- };
4889
- }
4890
- })
4891
- ] });
4892
- export { defaultAnalyzerRegistry };
1
+ import { ROW_ANALYZERS, SQL_ANALYZERS, defaultAnalyzerRegistry } from "./_chunks/default-registry.mjs";
2
+ export { ROW_ANALYZERS, SQL_ANALYZERS, defaultAnalyzerRegistry };