@gscdump/analysis 0.40.1 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,4868 +0,0 @@
1
- import { defineAnalyzer, requireAdapter } from "@gscdump/engine/analyzer";
2
- import { comparisonOf, defaultEndDate, padTimeseries, periodOf } from "@gscdump/engine/period";
3
- import { enumeratePartitions } from "@gscdump/engine/planner";
4
- import { METRIC_EXPR } from "@gscdump/engine/sql-fragments";
5
- import { num } from "@gscdump/engine/analysis-types";
6
- import { err, ok, unwrapResult } from "gscdump/result";
7
- import { between, date, extractDateRange, gsc, page, query } from "gscdump/query";
8
- import { MS_PER_DAY, daysAgoUtc, toIsoDate } from "gscdump/dates";
9
- import { buildExtrasQueries, buildTotalsSql, mergeExtras, resolveComparisonSQL, resolveToSQLOptimized } from "@gscdump/engine/resolver";
10
- function rowNumber(value) {
11
- if (typeof value === "number") return Number.isFinite(value) ? value : 0;
12
- if (typeof value === "bigint") return Number(value);
13
- if (value == null) return 0;
14
- const n = Number(value);
15
- return Number.isFinite(n) ? n : 0;
16
- }
17
- function rowString(value) {
18
- return value == null ? "" : String(value);
19
- }
20
- function rowBoolean(value) {
21
- return value === true || value === 1 || value === "true";
22
- }
23
- function parseJsonRows(value) {
24
- if (Array.isArray(value)) return value;
25
- if (typeof value === "string" && value.length > 0) {
26
- const parsed = JSON.parse(value);
27
- return Array.isArray(parsed) ? parsed : [];
28
- }
29
- return [];
30
- }
31
- const bayesianCtrAnalyzer = defineAnalyzer({
32
- id: "bayesian-ctr",
33
- buildSql(params) {
34
- const { startDate, endDate } = periodOf(params);
35
- const minImpressions = params.minImpressions ?? 50;
36
- const limit = params.limit ?? 300;
37
- const priorMinEntities = 5;
38
- return {
39
- sql: `
40
- WITH entity AS (
41
- SELECT
42
- query,
43
- url,
44
- ${METRIC_EXPR.clicks} AS clicks,
45
- ${METRIC_EXPR.impressions} AS impressions,
46
- ${METRIC_EXPR.ctr} AS observed_ctr,
47
- ${METRIC_EXPR.position} AS position,
48
- CAST(ROUND(LEAST(${METRIC_EXPR.position}, 30)) AS INTEGER) AS bucket
49
- FROM read_parquet({{FILES}}, union_by_name = true)
50
- WHERE date >= ? AND date <= ?
51
- AND query IS NOT NULL AND query <> ''
52
- AND url IS NOT NULL AND url <> ''
53
- GROUP BY query, url
54
- HAVING SUM(impressions) >= ?
55
- AND ${METRIC_EXPR.position} <= 30
56
- ),
57
- bucket_mu AS (
58
- SELECT
59
- bucket,
60
- COUNT(*) AS n_entities,
61
- SUM(observed_ctr * impressions) / NULLIF(SUM(impressions), 0) AS mu,
62
- SUM(impressions) AS total_impressions
63
- FROM entity
64
- GROUP BY bucket
65
- ),
66
- bucket_var AS (
67
- SELECT
68
- e.bucket,
69
- GREATEST(
70
- SUM(e.impressions * POWER(e.observed_ctr - b.mu, 2))
71
- / NULLIF(SUM(e.impressions), 0),
72
- 1e-9
73
- ) AS v
74
- FROM entity e
75
- JOIN bucket_mu b USING (bucket)
76
- GROUP BY e.bucket
77
- ),
78
- priors AS (
79
- SELECT
80
- m.bucket,
81
- m.n_entities,
82
- m.mu,
83
- v.v,
84
- CASE
85
- WHEN m.n_entities >= ${Number(priorMinEntities)}
86
- AND v.v > 0
87
- AND m.mu > 0 AND m.mu < 1
88
- AND (m.mu * (1.0 - m.mu) / v.v - 1.0) > 0
89
- THEN GREATEST(0.5, m.mu * (m.mu * (1.0 - m.mu) / v.v - 1.0))
90
- ELSE 2.0
91
- END AS alpha,
92
- CASE
93
- WHEN m.n_entities >= ${Number(priorMinEntities)}
94
- AND v.v > 0
95
- AND m.mu > 0 AND m.mu < 1
96
- AND (m.mu * (1.0 - m.mu) / v.v - 1.0) > 0
97
- THEN GREATEST(0.5, (1.0 - m.mu) * (m.mu * (1.0 - m.mu) / v.v - 1.0))
98
- ELSE 48.0
99
- END AS beta
100
- FROM bucket_mu m
101
- JOIN bucket_var v USING (bucket)
102
- ),
103
- posterior AS (
104
- SELECT
105
- e.query,
106
- e.url,
107
- e.clicks,
108
- e.impressions,
109
- e.observed_ctr,
110
- e.position,
111
- e.bucket,
112
- p.alpha AS prior_alpha,
113
- p.beta AS prior_beta,
114
- p.mu AS bucket_prior_mean,
115
- p.alpha + e.clicks AS alpha_post,
116
- p.beta + (e.impressions - e.clicks) AS beta_post
117
- FROM entity e
118
- JOIN priors p USING (bucket)
119
- ),
120
- scored AS (
121
- SELECT *,
122
- alpha_post / (alpha_post + beta_post) AS posterior_mean,
123
- SQRT((alpha_post * beta_post)
124
- / (POWER(alpha_post + beta_post, 2) * (alpha_post + beta_post + 1))) AS posterior_sd
125
- FROM posterior
126
- )
127
- SELECT
128
- query AS keyword,
129
- url AS page,
130
- clicks,
131
- impressions,
132
- observed_ctr AS observedCtr,
133
- position,
134
- bucket,
135
- prior_alpha AS priorAlpha,
136
- prior_beta AS priorBeta,
137
- bucket_prior_mean AS bucketPriorMean,
138
- posterior_mean AS posteriorMean,
139
- posterior_sd AS posteriorSd,
140
- GREATEST(0.0, posterior_mean - 1.96 * posterior_sd) AS ciLow,
141
- LEAST(1.0, posterior_mean + 1.96 * posterior_sd) AS ciHigh,
142
- posterior_mean - observed_ctr AS shrinkageDelta,
143
- (posterior_mean - observed_ctr) * impressions AS expectedClicksDelta,
144
- ABS(observed_ctr - posterior_mean) / NULLIF(posterior_sd, 0) AS significance,
145
- CASE
146
- WHEN observed_ctr > LEAST(1.0, posterior_mean + 1.96 * posterior_sd) THEN 'overperforming'
147
- WHEN observed_ctr < GREATEST(0.0, posterior_mean - 1.96 * posterior_sd) THEN 'underperforming'
148
- ELSE 'expected'
149
- END AS classification
150
- FROM scored
151
- ORDER BY significance DESC NULLS LAST
152
- LIMIT ${Number(limit)}
153
- `,
154
- params: [
155
- startDate,
156
- endDate,
157
- minImpressions
158
- ],
159
- current: {
160
- table: "page_queries",
161
- partitions: enumeratePartitions(startDate, endDate)
162
- }
163
- };
164
- },
165
- reduceSql(rows, params) {
166
- const arr = Array.isArray(rows) ? rows : [];
167
- const minImpressions = params.minImpressions ?? 50;
168
- const results = arr.map((r) => ({
169
- keyword: rowString(r.keyword),
170
- page: rowString(r.page),
171
- clicks: rowNumber(r.clicks),
172
- impressions: rowNumber(r.impressions),
173
- observedCtr: rowNumber(r.observedCtr),
174
- position: rowNumber(r.position),
175
- bucket: rowNumber(r.bucket),
176
- priorAlpha: rowNumber(r.priorAlpha),
177
- priorBeta: rowNumber(r.priorBeta),
178
- bucketPriorMean: rowNumber(r.bucketPriorMean),
179
- posteriorMean: rowNumber(r.posteriorMean),
180
- posteriorSd: rowNumber(r.posteriorSd),
181
- ciLow: rowNumber(r.ciLow),
182
- ciHigh: rowNumber(r.ciHigh),
183
- shrinkageDelta: rowNumber(r.shrinkageDelta),
184
- expectedClicksDelta: rowNumber(r.expectedClicksDelta),
185
- significance: rowNumber(r.significance),
186
- classification: rowString(r.classification)
187
- }));
188
- const under = results.filter((r) => r.classification === "underperforming").length;
189
- const over = results.filter((r) => r.classification === "overperforming").length;
190
- return {
191
- results,
192
- meta: {
193
- total: results.length,
194
- underperforming: under,
195
- overperforming: over,
196
- expected: results.length - under - over,
197
- minImpressions
198
- }
199
- };
200
- }
201
- });
202
- const BIPARTITE_PAGERANK_ITERATIONS = 25;
203
- const BIPARTITE_PAGERANK_DAMPING = .85;
204
- const bipartitePagerankAnalyzer = defineAnalyzer({
205
- id: "bipartite-pagerank",
206
- buildSql(params) {
207
- const { startDate, endDate } = periodOf(params);
208
- const minImpressions = params.minImpressions ?? 50;
209
- const topQueries = 1e3;
210
- const topUrls = 500;
211
- const limit = params.limit ?? 50;
212
- const bridgingEdgeThreshold = .05;
213
- const anchoringEdgeThreshold = .05;
214
- const iterations = BIPARTITE_PAGERANK_ITERATIONS;
215
- const d = BIPARTITE_PAGERANK_DAMPING;
216
- const iterCtes = [];
217
- for (let i = 1; i <= iterations; i++) iterCtes.push(`
218
- ranks_${i} AS (
219
- SELECT
220
- 'q' AS kind,
221
- e.qid AS id,
222
- (1.0 - ${d}) / (SELECT n FROM query_count)
223
- + ${d} * SUM(e.w_u_to_q * r.rank) AS rank
224
- FROM u_to_q_weights e
225
- JOIN ranks_${i - 1} r ON r.kind = 'u' AND r.id = e.uid
226
- GROUP BY e.qid
227
- UNION ALL
228
- SELECT
229
- 'u' AS kind,
230
- e.uid AS id,
231
- (1.0 - ${d}) / (SELECT n FROM url_count)
232
- + ${d} * SUM(e.w_q_to_u * r.rank) AS rank
233
- FROM q_to_u_weights e
234
- JOIN ranks_${i - 1} r ON r.kind = 'q' AND r.id = e.qid
235
- GROUP BY e.uid
236
- )`);
237
- const deltaParts = [];
238
- for (let i = 1; i <= iterations; i++) deltaParts.push(`
239
- SELECT ${i} AS step,
240
- (SELECT COALESCE(SUM(ABS(a.rank - b.rank)), 0.0)
241
- FROM ranks_${i} a
242
- JOIN ranks_${i - 1} b USING (kind, id)) AS l1`);
243
- return {
244
- sql: `
245
- WITH edges0 AS (
246
- SELECT
247
- query AS qid,
248
- url AS uid,
249
- CAST(SUM(impressions) AS DOUBLE) AS impressions
250
- FROM read_parquet({{FILES}}, union_by_name = true)
251
- WHERE date >= ? AND date <= ?
252
- AND query IS NOT NULL AND query <> ''
253
- AND url IS NOT NULL AND url <> ''
254
- GROUP BY query, url
255
- HAVING SUM(impressions) >= ?
256
- ),
257
- -- Top-N caps per side keep the iteration tractable.
258
- query_totals AS (
259
- SELECT qid, SUM(impressions) AS tot
260
- FROM edges0 GROUP BY qid
261
- ),
262
- url_totals AS (
263
- SELECT uid, SUM(impressions) AS tot
264
- FROM edges0 GROUP BY uid
265
- ),
266
- top_queries AS (
267
- SELECT qid FROM query_totals
268
- ORDER BY tot DESC, qid ASC LIMIT ${Number(topQueries)}
269
- ),
270
- top_urls AS (
271
- SELECT uid FROM url_totals
272
- ORDER BY tot DESC, uid ASC LIMIT ${Number(topUrls)}
273
- ),
274
- edges AS (
275
- SELECT e.qid, e.uid, e.impressions
276
- FROM edges0 e
277
- JOIN top_queries tq USING (qid)
278
- JOIN top_urls tu USING (uid)
279
- ),
280
- query_nodes AS (SELECT DISTINCT qid FROM edges),
281
- url_nodes AS (SELECT DISTINCT uid FROM edges),
282
- query_count AS (SELECT GREATEST(COUNT(*), 1) AS n FROM query_nodes),
283
- url_count AS (SELECT GREATEST(COUNT(*), 1) AS n FROM url_nodes),
284
- -- Row-stochastic transition weights in each direction. For q->u the
285
- -- weights out of a query sum to 1; symmetric for u->q.
286
- q_out AS (SELECT qid, SUM(impressions) AS s FROM edges GROUP BY qid),
287
- u_out AS (SELECT uid, SUM(impressions) AS s FROM edges GROUP BY uid),
288
- q_to_u_weights AS (
289
- SELECT e.qid, e.uid,
290
- e.impressions / NULLIF(q.s, 0) AS w_q_to_u
291
- FROM edges e JOIN q_out q USING (qid)
292
- ),
293
- u_to_q_weights AS (
294
- SELECT e.qid, e.uid,
295
- e.impressions / NULLIF(u.s, 0) AS w_u_to_q
296
- FROM edges e JOIN u_out u USING (uid)
297
- ),
298
- -- Seed: uniform distribution per side. Total mass = 2 (one unit per side).
299
- ranks_0 AS (
300
- SELECT 'q' AS kind, q.qid AS id, 1.0 / (SELECT n FROM query_count) AS rank
301
- FROM query_nodes q
302
- UNION ALL
303
- SELECT 'u' AS kind, u.uid AS id, 1.0 / (SELECT n FROM url_count) AS rank
304
- FROM url_nodes u
305
- ),
306
- ${iterCtes.join(",\n")},
307
- final_ranks AS (SELECT * FROM ranks_${iterations}),
308
- -- Hub/anchor diagnostics computed from raw edge mass (not rank). A
309
- -- query "bridges" URLs it sends >= ${bridgingEdgeThreshold} of its mass
310
- -- to; a URL "anchors" queries that contribute >= ${anchoringEdgeThreshold}
311
- -- of its incoming mass.
312
- q_bridging AS (
313
- SELECT qid, COUNT(*) AS bridging
314
- FROM q_to_u_weights
315
- WHERE w_q_to_u >= ${bridgingEdgeThreshold}
316
- GROUP BY qid
317
- ),
318
- u_anchoring AS (
319
- SELECT uid, COUNT(*) AS anchoring
320
- FROM u_to_q_weights
321
- WHERE w_u_to_q >= ${anchoringEdgeThreshold}
322
- GROUP BY uid
323
- ),
324
- q_degree AS (
325
- SELECT qid, COUNT(*) AS degree, SUM(impressions) AS impressions
326
- FROM edges GROUP BY qid
327
- ),
328
- u_degree AS (
329
- SELECT uid, COUNT(*) AS degree, SUM(impressions) AS impressions
330
- FROM edges GROUP BY uid
331
- ),
332
- deltas AS (
333
- ${deltaParts.join("\n UNION ALL\n")}
334
- ),
335
- query_rows AS (
336
- SELECT
337
- 'query' AS kind, f.id, f.rank,
338
- COALESCE(b.bridging, 0) AS bridging,
339
- 0 AS anchoring,
340
- COALESCE(qd.degree, 0) AS degree,
341
- COALESCE(qd.impressions, 0) AS impressions
342
- FROM final_ranks f
343
- LEFT JOIN q_bridging b ON b.qid = f.id
344
- LEFT JOIN q_degree qd ON qd.qid = f.id
345
- WHERE f.kind = 'q'
346
- ORDER BY f.rank DESC
347
- LIMIT ${Number(limit)}
348
- ),
349
- url_rows AS (
350
- SELECT
351
- 'url' AS kind, f.id, f.rank,
352
- 0 AS bridging,
353
- COALESCE(a.anchoring, 0) AS anchoring,
354
- COALESCE(ud.degree, 0) AS degree,
355
- COALESCE(ud.impressions, 0) AS impressions
356
- FROM final_ranks f
357
- LEFT JOIN u_anchoring a ON a.uid = f.id
358
- LEFT JOIN u_degree ud ON ud.uid = f.id
359
- WHERE f.kind = 'u'
360
- ORDER BY f.rank DESC
361
- LIMIT ${Number(limit)}
362
- ),
363
- nodes AS (
364
- SELECT * FROM query_rows
365
- UNION ALL
366
- SELECT * FROM url_rows
367
- ),
368
- counts AS (
369
- SELECT
370
- (SELECT n FROM query_count) AS q_count,
371
- (SELECT n FROM url_count) AS u_count
372
- ),
373
- deltas_json AS (
374
- SELECT to_json(list({ 'step': step, 'l1': l1 } ORDER BY step)) AS dj
375
- FROM deltas
376
- )
377
- SELECT
378
- n.kind,
379
- n.id,
380
- n.rank,
381
- n.bridging,
382
- n.anchoring,
383
- n.degree,
384
- n.impressions,
385
- c.q_count AS queryCount,
386
- c.u_count AS urlCount,
387
- dj.dj AS deltasJson
388
- FROM nodes n
389
- CROSS JOIN counts c
390
- CROSS JOIN deltas_json dj
391
- ORDER BY n.kind, n.rank DESC
392
- `,
393
- params: [
394
- startDate,
395
- endDate,
396
- minImpressions
397
- ],
398
- current: {
399
- table: "page_queries",
400
- partitions: enumeratePartitions(startDate, endDate)
401
- }
402
- };
403
- },
404
- reduceSql(rows) {
405
- const arr = Array.isArray(rows) ? rows : [];
406
- const iterations = BIPARTITE_PAGERANK_ITERATIONS;
407
- const d = BIPARTITE_PAGERANK_DAMPING;
408
- const results = arr.map((r) => ({
409
- kind: rowString(r.kind),
410
- id: rowString(r.id),
411
- rank: num(r.rank),
412
- bridging: num(r.bridging),
413
- anchoring: num(r.anchoring),
414
- degree: num(r.degree),
415
- impressions: num(r.impressions)
416
- }));
417
- const first = arr[0] ?? {};
418
- const queryCount = num(first.queryCount);
419
- const urlCount = num(first.urlCount);
420
- const deltas = parseJsonRows(first.deltasJson).map((e) => ({
421
- step: num(e.step),
422
- l1: num(e.l1)
423
- }));
424
- const convergenceDelta = deltas.length > 0 ? deltas[deltas.length - 1].l1 : 0;
425
- return {
426
- results,
427
- meta: {
428
- total: results.length,
429
- convergenceDelta,
430
- iterations,
431
- damping: d,
432
- queryCount,
433
- urlCount,
434
- deltas
435
- }
436
- };
437
- }
438
- });
439
- const DEFAULT_LIMIT$1 = 25e3;
440
- function queriesQueryState(period, limit = DEFAULT_LIMIT$1) {
441
- return gsc.select(query, page).where(between(date, period.startDate, period.endDate)).limit(limit).getState();
442
- }
443
- function pagesQueryState(period, limit = DEFAULT_LIMIT$1) {
444
- return gsc.select(page).where(between(date, period.startDate, period.endDate)).limit(limit).getState();
445
- }
446
- function datesQueryState(period, limit = DEFAULT_LIMIT$1) {
447
- return gsc.select(date).where(between(date, period.startDate, period.endDate)).limit(limit).getState();
448
- }
449
- const analysisErrors = {
450
- missingReportParam(report, param, message) {
451
- return {
452
- kind: "missing-report-param",
453
- report,
454
- param,
455
- message
456
- };
457
- },
458
- missingComparisonWindow(report, message) {
459
- return {
460
- kind: "missing-comparison-window",
461
- report,
462
- message
463
- };
464
- },
465
- missingBrandTerms() {
466
- return {
467
- kind: "missing-brand-terms",
468
- message: "Brand analysis requires brandTerms"
469
- };
470
- },
471
- unknownReport(report, available) {
472
- return {
473
- kind: "unknown-report",
474
- report,
475
- available,
476
- message: `unknown report "${report}"; available: ${available.join(", ")}`
477
- };
478
- },
479
- unknownAnalyzer(analyzer) {
480
- return {
481
- kind: "unknown-analyzer",
482
- analyzer,
483
- message: `unknown analyzer "${analyzer}"`
484
- };
485
- },
486
- requiredStepFailed(report, stepKey, stepError, cause) {
487
- return {
488
- kind: "required-step-failed",
489
- report,
490
- stepKey,
491
- stepError,
492
- cause,
493
- message: `runReport(${report}): required step "${stepKey}" failed: ${stepError}`
494
- };
495
- }
496
- };
497
- function analysisErrorToException(error) {
498
- const exception = new Error(error.message);
499
- if ("cause" in error && error.cause !== void 0) exception.cause = error.cause;
500
- exception.analysisError = error;
501
- return exception;
502
- }
503
- function escapeRegexAlt(s) {
504
- return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
505
- }
506
- function requireBrandTermsResult(brandTerms) {
507
- return brandTerms?.length ? ok(brandTerms) : err(analysisErrors.missingBrandTerms());
508
- }
509
- function requireBrandTerms(brandTerms) {
510
- return unwrapResult(requireBrandTermsResult(brandTerms), analysisErrorToException);
511
- }
512
- function analyzeBrandSegmentation(keywords, options) {
513
- const { brandTerms, minImpressions = 10 } = options;
514
- const lowerBrandTerms = brandTerms.map((t) => t.toLowerCase());
515
- const brand = [];
516
- const nonBrand = [];
517
- let brandClicks = 0;
518
- let nonBrandClicks = 0;
519
- let brandImpressions = 0;
520
- let nonBrandImpressions = 0;
521
- for (const row of keywords) {
522
- const impressions = num(row.impressions);
523
- if (impressions < minImpressions) continue;
524
- const clicks = num(row.clicks);
525
- if ((row.variants?.length ? row.variants : [row.query ?? row.keyword ?? ""]).some((query) => lowerBrandTerms.some((term) => query.toLowerCase().includes(term)))) {
526
- brand.push(row);
527
- brandClicks += clicks;
528
- brandImpressions += impressions;
529
- } else {
530
- nonBrand.push(row);
531
- nonBrandClicks += clicks;
532
- nonBrandImpressions += impressions;
533
- }
534
- }
535
- const totalClicks = brandClicks + nonBrandClicks;
536
- return {
537
- brand,
538
- nonBrand,
539
- summary: {
540
- brandClicks,
541
- nonBrandClicks,
542
- brandShare: totalClicks > 0 ? brandClicks / totalClicks : 0,
543
- brandImpressions,
544
- nonBrandImpressions
545
- }
546
- };
547
- }
548
- const brandAnalyzer = defineAnalyzer({
549
- id: "brand",
550
- buildSql(params) {
551
- const brandTerms = requireBrandTerms(params.brandTerms);
552
- const { startDate, endDate } = periodOf(params);
553
- const minImpressions = params.minImpressions ?? 10;
554
- const limit = params.limit ?? 1e4;
555
- const regex = `(${brandTerms.map((t) => escapeRegexAlt(t.toLowerCase())).join("|")})`;
556
- return {
557
- sql: `
558
- WITH agg AS (
559
- SELECT
560
- query,
561
- url AS page,
562
- ${METRIC_EXPR.clicks} AS clicks,
563
- ${METRIC_EXPR.impressions} AS impressions,
564
- ${METRIC_EXPR.ctr} AS ctr,
565
- ${METRIC_EXPR.position} AS position
566
- FROM read_parquet({{FILES}}, union_by_name = true)
567
- WHERE date >= ? AND date <= ?
568
- GROUP BY query, url
569
- HAVING SUM(impressions) >= ?
570
- )
571
- SELECT
572
- query, page, clicks, impressions, ctr, position,
573
- CASE WHEN regexp_matches(LOWER(query), ?) THEN 'brand' ELSE 'non-brand' END AS segment
574
- FROM agg
575
- ORDER BY clicks DESC
576
- LIMIT ${Number(limit)}
577
- `,
578
- params: [
579
- startDate,
580
- endDate,
581
- minImpressions,
582
- regex
583
- ],
584
- current: {
585
- table: "page_queries",
586
- partitions: enumeratePartitions(startDate, endDate)
587
- }
588
- };
589
- },
590
- reduceSql(rows) {
591
- const normalized = (Array.isArray(rows) ? rows : []).map((r) => ({
592
- query: rowString(r.query),
593
- page: r.page == null ? void 0 : rowString(r.page),
594
- clicks: num(r.clicks),
595
- impressions: num(r.impressions),
596
- ctr: num(r.ctr),
597
- position: num(r.position),
598
- segment: rowString(r.segment)
599
- }));
600
- let brandClicks = 0;
601
- let nonBrandClicks = 0;
602
- let brandImpressions = 0;
603
- let nonBrandImpressions = 0;
604
- for (const r of normalized) if (r.segment === "brand") {
605
- brandClicks += r.clicks;
606
- brandImpressions += r.impressions;
607
- } else {
608
- nonBrandClicks += r.clicks;
609
- nonBrandImpressions += r.impressions;
610
- }
611
- const totalClicks = brandClicks + nonBrandClicks;
612
- return {
613
- results: normalized,
614
- meta: {
615
- total: normalized.length,
616
- summary: {
617
- brandClicks,
618
- nonBrandClicks,
619
- brandShare: totalClicks > 0 ? brandClicks / totalClicks : 0,
620
- brandImpressions,
621
- nonBrandImpressions
622
- }
623
- }
624
- };
625
- },
626
- buildRows(params) {
627
- return { queries: queriesQueryState(periodOf(params), params.limit) };
628
- },
629
- reduceRows(rows, params) {
630
- const brandTerms = requireBrandTerms(params.brandTerms);
631
- const result = analyzeBrandSegmentation(Array.isArray(rows) ? rows : [], {
632
- brandTerms,
633
- minImpressions: params.minImpressions
634
- });
635
- return {
636
- results: [...result.brand.map((r) => ({
637
- ...r,
638
- segment: "brand"
639
- })), ...result.nonBrand.map((r) => ({
640
- ...r,
641
- segment: "non-brand"
642
- }))],
643
- meta: { summary: result.summary }
644
- };
645
- }
646
- });
647
- function buildPeriodMap(rows, key, value, filter) {
648
- const out = /* @__PURE__ */ new Map();
649
- for (const row of rows) {
650
- if (filter && !filter(row)) continue;
651
- out.set(key(row), value(row));
652
- }
653
- return out;
654
- }
655
- function createSorter(getValue, defaultMetric, defaultOrder = "desc") {
656
- return (items, sortBy = defaultMetric, sortOrder = defaultOrder) => {
657
- const mult = sortOrder === "desc" ? -1 : 1;
658
- return [...items].sort((a, b) => (getValue(a, sortBy) - getValue(b, sortBy)) * mult);
659
- };
660
- }
661
- function createMetricSorter(defaultMetric, orderByMetric) {
662
- return (items, sortBy = defaultMetric) => {
663
- const mult = orderByMetric[sortBy] === "desc" ? -1 : 1;
664
- return [...items].sort((a, b) => (a[sortBy] - b[sortBy]) * mult);
665
- };
666
- }
667
- const sortRowResults = createSorter((item, metric) => {
668
- switch (metric) {
669
- case "clicks": return item.totalClicks;
670
- case "impressions": return item.totalImpressions;
671
- case "positionSpread": return item.positionSpread;
672
- case "pageCount": return item.pages.length;
673
- }
674
- }, "clicks");
675
- function analyzeCannibalization(rows, options = {}) {
676
- const { minImpressions = 10, maxPositionSpread = 10, minPages = 2, sortBy = "clicks", sortOrder = "desc" } = options;
677
- const queryMap = /* @__PURE__ */ new Map();
678
- for (const row of rows) {
679
- if (row.impressions < minImpressions) continue;
680
- const pages = queryMap.get(row.query) || [];
681
- pages.push({
682
- page: row.page,
683
- clicks: row.clicks,
684
- impressions: row.impressions,
685
- ctr: row.ctr,
686
- position: row.position
687
- });
688
- queryMap.set(row.query, pages);
689
- }
690
- const results = [];
691
- for (const [query, pages] of queryMap) {
692
- if (pages.length < minPages) continue;
693
- let minPosition = Number.POSITIVE_INFINITY;
694
- let maxPosition = Number.NEGATIVE_INFINITY;
695
- let totalClicks = 0;
696
- let totalImpressions = 0;
697
- for (const page of pages) {
698
- minPosition = Math.min(minPosition, page.position);
699
- maxPosition = Math.max(maxPosition, page.position);
700
- totalClicks += page.clicks;
701
- totalImpressions += page.impressions;
702
- }
703
- const positionSpread = maxPosition - minPosition;
704
- if (positionSpread > maxPositionSpread) continue;
705
- pages.sort((a, b) => b.clicks - a.clicks);
706
- results.push({
707
- query,
708
- pages,
709
- totalClicks,
710
- totalImpressions,
711
- positionSpread
712
- });
713
- }
714
- return sortRowResults(results, sortBy, sortOrder);
715
- }
716
- const cannibalizationAnalyzer = defineAnalyzer({
717
- id: "cannibalization",
718
- buildSql(params) {
719
- const { startDate, endDate } = periodOf(params);
720
- const minImpressions = params.minImpressions ?? 50;
721
- const minCompetitors = 2;
722
- const minQueryImpressions = (params.minImpressions ?? 50) * 2;
723
- const limit = params.limit ?? 200;
724
- return {
725
- sql: `
726
- WITH agg AS (
727
- SELECT
728
- query,
729
- url,
730
- ${METRIC_EXPR.clicks} AS clicks,
731
- ${METRIC_EXPR.impressions} AS impressions,
732
- ${METRIC_EXPR.ctr} AS ctr,
733
- ${METRIC_EXPR.position} AS position
734
- FROM read_parquet({{FILES}}, union_by_name = true)
735
- WHERE date >= ? AND date <= ?
736
- AND query IS NOT NULL AND query <> ''
737
- AND url IS NOT NULL AND url <> ''
738
- GROUP BY query, url
739
- HAVING SUM(impressions) >= ?
740
- ),
741
- query_totals AS (
742
- SELECT
743
- query,
744
- SUM(impressions) AS total_impressions,
745
- SUM(clicks) AS total_clicks,
746
- COUNT(*) AS competitor_count
747
- FROM agg
748
- GROUP BY query
749
- HAVING COUNT(*) >= ? AND SUM(impressions) >= ?
750
- ),
751
- ranked AS (
752
- SELECT
753
- a.query,
754
- a.url,
755
- a.clicks,
756
- a.impressions,
757
- a.ctr,
758
- a.position,
759
- a.impressions / NULLIF(t.total_impressions, 0) AS share,
760
- ROW_NUMBER() OVER (
761
- PARTITION BY a.query
762
- ORDER BY a.impressions DESC, a.clicks DESC, a.url ASC
763
- ) AS rnk
764
- FROM agg a
765
- JOIN query_totals t USING (query)
766
- ),
767
- leader AS (
768
- SELECT query, url AS leader_url, ctr AS leader_ctr, position AS leader_position
769
- FROM ranked WHERE rnk = 1
770
- ),
771
- events AS (
772
- SELECT
773
- r.query,
774
- any_value(l.leader_url) AS leader_url,
775
- any_value(l.leader_ctr) AS leader_ctr,
776
- any_value(l.leader_position) AS leader_position,
777
- SUM(POWER(r.share * 100.0, 2)) AS hhi,
778
- SUM(CASE
779
- WHEN r.rnk > 1 AND l.leader_ctr > r.ctr
780
- THEN (l.leader_ctr - r.ctr) * r.impressions
781
- ELSE 0.0
782
- END) AS stolen_clicks,
783
- to_json(list({
784
- 'url': r.url,
785
- 'clicks': r.clicks,
786
- 'impressions': r.impressions,
787
- 'ctr': r.ctr,
788
- 'position': r.position,
789
- 'share': r.share,
790
- 'rank': r.rnk
791
- } ORDER BY r.rnk)) AS competitors
792
- FROM ranked r
793
- JOIN leader l USING (query)
794
- GROUP BY r.query
795
- )
796
- SELECT
797
- e.query AS keyword,
798
- t.total_impressions AS totalImpressions,
799
- t.total_clicks AS totalClicks,
800
- t.competitor_count AS competitorCount,
801
- e.leader_url AS leaderUrl,
802
- e.leader_ctr AS leaderCtr,
803
- e.leader_position AS leaderPosition,
804
- e.hhi AS hhi,
805
- GREATEST(0.0, 1.0 - e.hhi / 10000.0) AS fragmentation,
806
- e.stolen_clicks AS stolenClicks,
807
- e.competitors AS competitors,
808
- CAST(ROUND(LEAST(100.0,
809
- 100.0 * POWER(
810
- GREATEST(1.0 - e.hhi / 10000.0, 0.0)
811
- * LEAST(e.stolen_clicks / GREATEST(t.total_clicks + e.stolen_clicks, 1.0), 1.0)
812
- * LEAST(LOG10(GREATEST(t.total_impressions, 10.0)) / 5.0, 1.0),
813
- 1.0 / 3.0
814
- )
815
- )) AS DOUBLE) AS severity
816
- FROM events e
817
- JOIN query_totals t USING (query)
818
- ORDER BY severity DESC, stolenClicks DESC
819
- LIMIT ${Number(limit)}
820
- `,
821
- params: [
822
- startDate,
823
- endDate,
824
- minImpressions,
825
- minCompetitors,
826
- minQueryImpressions
827
- ],
828
- current: {
829
- table: "page_queries",
830
- partitions: enumeratePartitions(startDate, endDate)
831
- }
832
- };
833
- },
834
- reduceSql(rows) {
835
- const events = (Array.isArray(rows) ? rows : []).map((r) => ({
836
- keyword: rowString(r.keyword),
837
- totalImpressions: num(r.totalImpressions),
838
- totalClicks: num(r.totalClicks),
839
- competitorCount: num(r.competitorCount),
840
- leaderUrl: rowString(r.leaderUrl),
841
- leaderCtr: num(r.leaderCtr),
842
- leaderPosition: num(r.leaderPosition),
843
- hhi: num(r.hhi),
844
- fragmentation: num(r.fragmentation),
845
- stolenClicks: num(r.stolenClicks),
846
- severity: num(r.severity),
847
- competitors: parseJsonRows(r.competitors).map((c) => ({
848
- url: rowString(c.url),
849
- clicks: num(c.clicks),
850
- impressions: num(c.impressions),
851
- ctr: num(c.ctr),
852
- position: num(c.position),
853
- share: num(c.share),
854
- rank: num(c.rank)
855
- }))
856
- }));
857
- const nodeAgg = /* @__PURE__ */ new Map();
858
- const edgeAgg = /* @__PURE__ */ new Map();
859
- for (const ev of events) {
860
- for (const c of ev.competitors) {
861
- const n = nodeAgg.get(c.url) ?? {
862
- impressions: 0,
863
- clicks: 0,
864
- queries: /* @__PURE__ */ new Set()
865
- };
866
- n.impressions += c.impressions;
867
- n.clicks += c.clicks;
868
- n.queries.add(ev.keyword);
869
- nodeAgg.set(c.url, n);
870
- }
871
- for (let i = 0; i < ev.competitors.length; i++) for (let j = i + 1; j < ev.competitors.length; j++) {
872
- const a = ev.competitors[i];
873
- const b = ev.competitors[j];
874
- const [src, tgt] = a.url < b.url ? [a.url, b.url] : [b.url, a.url];
875
- const key = `${src}${tgt}`;
876
- const weight = Math.min(a.impressions, b.impressions);
877
- const edge = edgeAgg.get(key) ?? {
878
- source: src,
879
- target: tgt,
880
- weight: 0,
881
- queries: 0
882
- };
883
- edge.weight += weight;
884
- edge.queries += 1;
885
- edgeAgg.set(key, edge);
886
- }
887
- }
888
- const nodes = [...nodeAgg.entries()].map(([url, n]) => ({
889
- url,
890
- impressions: n.impressions,
891
- clicks: n.clicks,
892
- queryCount: n.queries.size
893
- }));
894
- const edges = [...edgeAgg.values()];
895
- const avgFragmentation = events.length > 0 ? events.reduce((s, e) => s + e.fragmentation, 0) / events.length : 0;
896
- const totalStolenClicks = events.reduce((s, e) => s + e.stolenClicks, 0);
897
- return {
898
- results: events,
899
- meta: {
900
- total: events.length,
901
- totalStolenClicks,
902
- avgFragmentation,
903
- graph: {
904
- nodes,
905
- edges
906
- }
907
- }
908
- };
909
- },
910
- buildRows(params) {
911
- return { rows: queriesQueryState(periodOf(params), params.limit) };
912
- },
913
- reduceRows(rows, params) {
914
- const results = analyzeCannibalization(Array.isArray(rows) ? rows : [], {
915
- minImpressions: params.minImpressions,
916
- maxPositionSpread: params.maxPositionSpread,
917
- minPages: params.minPages
918
- });
919
- return {
920
- results,
921
- meta: { total: results.length }
922
- };
923
- }
924
- });
925
- const changePointAnalyzer = defineAnalyzer({
926
- id: "change-point",
927
- buildSql(params) {
928
- const endDate = params.endDate ?? defaultEndDate();
929
- const startDate = params.startDate ?? daysAgoUtc(93);
930
- const minDays = 21;
931
- const minSide = 7;
932
- const threshold = params.threshold ?? 10;
933
- const minImpressions = params.minImpressions ?? 50;
934
- const metric = params.metric === "clicks" || params.metric === "impressions" ? params.metric : "position";
935
- const limit = params.limit ?? 100;
936
- const valueExpr = metric === "position" ? METRIC_EXPR.position : `CAST(SUM(${metric}) AS DOUBLE)`;
937
- const weightExpr = metric === "position" ? METRIC_EXPR.impressions : "1.0";
938
- return {
939
- sql: `
940
- WITH daily AS (
941
- SELECT
942
- query,
943
- url AS page,
944
- -- Normalize at the source CTE: union_by_name=true can coerce date to
945
- -- VARCHAR across parquets with mixed schemas, which makes downstream
946
- -- strftime(date, ...) binder-error.
947
- CAST(date AS DATE) AS date,
948
- ${METRIC_EXPR.clicks} AS clicks,
949
- ${METRIC_EXPR.impressions} AS impressions,
950
- ${valueExpr} AS value,
951
- ${weightExpr} AS weight
952
- FROM read_parquet({{FILES}}, union_by_name = true)
953
- WHERE date >= ? AND date <= ?
954
- AND query IS NOT NULL AND query <> ''
955
- AND url IS NOT NULL AND url <> ''
956
- GROUP BY query, url, date
957
- HAVING SUM(impressions) >= 1
958
- ),
959
- entity_stats AS (
960
- SELECT query, page,
961
- COUNT(*) AS n_total,
962
- SUM(impressions) AS total_impressions,
963
- SUM(weight) AS w_total,
964
- SUM(value * weight) AS sum_total,
965
- SUM(value * value * weight) AS sumsq_total
966
- FROM daily
967
- GROUP BY query, page
968
- HAVING COUNT(*) >= ${Number(minDays)}
969
- AND SUM(impressions) >= ?
970
- ),
971
- filtered AS (
972
- SELECT d.*,
973
- e.n_total, e.w_total, e.sum_total, e.sumsq_total, e.total_impressions
974
- FROM daily d
975
- JOIN entity_stats e USING (query, page)
976
- ),
977
- cumulated AS (
978
- SELECT *,
979
- COUNT(*) OVER w AS n_left,
980
- SUM(weight) OVER w AS w_left,
981
- SUM(value * weight) OVER w AS sum_left,
982
- SUM(value * value * weight) OVER w AS sumsq_left
983
- FROM filtered
984
- WINDOW w AS (
985
- PARTITION BY query, page
986
- ORDER BY date
987
- ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
988
- )
989
- ),
990
- llr_scored AS (
991
- SELECT *,
992
- (n_total - n_left) AS n_right,
993
- (w_total - w_left) AS w_right,
994
- (sum_total - sum_left) AS sum_right,
995
- (sumsq_total - sumsq_left) AS sumsq_right,
996
- GREATEST(
997
- (sumsq_left / NULLIF(w_left, 0))
998
- - (sum_left / NULLIF(w_left, 0)) * (sum_left / NULLIF(w_left, 0)),
999
- 1e-9
1000
- ) AS var_left,
1001
- GREATEST(
1002
- ((sumsq_total - sumsq_left) / NULLIF(w_total - w_left, 0))
1003
- - ((sum_total - sum_left) / NULLIF(w_total - w_left, 0))
1004
- * ((sum_total - sum_left) / NULLIF(w_total - w_left, 0)),
1005
- 1e-9
1006
- ) AS var_right,
1007
- GREATEST(
1008
- (sumsq_total / NULLIF(w_total, 0))
1009
- - (sum_total / NULLIF(w_total, 0)) * (sum_total / NULLIF(w_total, 0)),
1010
- 1e-9
1011
- ) AS var_single
1012
- FROM cumulated
1013
- ),
1014
- llr AS (
1015
- SELECT *,
1016
- CASE
1017
- WHEN n_left >= ${Number(minSide)} AND (n_total - n_left) >= ${Number(minSide)}
1018
- THEN n_total * LN(var_single)
1019
- - n_left * LN(var_left)
1020
- - (n_total - n_left) * LN(var_right)
1021
- ELSE NULL
1022
- END AS llr
1023
- FROM llr_scored
1024
- ),
1025
- best AS (
1026
- SELECT query, page, n_total, total_impressions,
1027
- arg_max(date, llr) AS change_date,
1028
- MAX(llr) AS best_llr,
1029
- arg_max(sum_left / NULLIF(w_left, 0), llr) AS left_mean,
1030
- arg_max((sum_total - sum_left) / NULLIF(w_total - w_left, 0), llr) AS right_mean,
1031
- arg_max(sqrt(var_left), llr) AS left_std,
1032
- arg_max(sqrt(var_right), llr) AS right_std
1033
- FROM llr
1034
- WHERE llr IS NOT NULL
1035
- GROUP BY query, page, n_total, total_impressions
1036
- HAVING MAX(llr) > ${Number(threshold)}
1037
- ),
1038
- series AS (
1039
- SELECT query, page,
1040
- to_json(list({
1041
- 'date': strftime(date, '%Y-%m-%d'),
1042
- 'value': value
1043
- } ORDER BY date)) AS seriesJson
1044
- FROM daily
1045
- GROUP BY query, page
1046
- )
1047
- SELECT
1048
- b.query AS keyword,
1049
- b.page,
1050
- CAST(b.n_total AS DOUBLE) AS totalDays,
1051
- CAST(b.total_impressions AS DOUBLE) AS totalImpressions,
1052
- strftime(b.change_date, '%Y-%m-%d') AS changeDate,
1053
- b.best_llr AS llr,
1054
- b.left_mean AS leftMean,
1055
- b.right_mean AS rightMean,
1056
- (b.right_mean - b.left_mean) AS delta,
1057
- b.left_std AS leftStddev,
1058
- b.right_std AS rightStddev,
1059
- s.seriesJson
1060
- FROM best b
1061
- LEFT JOIN series s USING (query, page)
1062
- ORDER BY b.best_llr DESC
1063
- LIMIT ${Number(limit)}
1064
- `,
1065
- params: [
1066
- startDate,
1067
- endDate,
1068
- minImpressions
1069
- ],
1070
- current: {
1071
- table: "page_queries",
1072
- partitions: enumeratePartitions(startDate, endDate)
1073
- }
1074
- };
1075
- },
1076
- reduceSql(rows, params) {
1077
- const arr = Array.isArray(rows) ? rows : [];
1078
- const threshold = params.threshold ?? 10;
1079
- const metric = params.metric === "clicks" || params.metric === "impressions" ? params.metric : "position";
1080
- const lowerIsBetter = metric === "position";
1081
- const results = arr.map((r) => {
1082
- const delta = rowNumber(r.delta);
1083
- const improved = lowerIsBetter ? delta < 0 : delta > 0;
1084
- return {
1085
- keyword: rowString(r.keyword),
1086
- page: rowString(r.page),
1087
- totalDays: rowNumber(r.totalDays),
1088
- totalImpressions: rowNumber(r.totalImpressions),
1089
- changeDate: rowString(r.changeDate),
1090
- llr: rowNumber(r.llr),
1091
- leftMean: rowNumber(r.leftMean),
1092
- rightMean: rowNumber(r.rightMean),
1093
- delta,
1094
- leftStddev: rowNumber(r.leftStddev),
1095
- rightStddev: rowNumber(r.rightStddev),
1096
- direction: improved ? "improved" : "worsened",
1097
- series: parseJsonRows(r.seriesJson).map((s) => ({
1098
- date: rowString(s.date),
1099
- value: rowNumber(s.value)
1100
- }))
1101
- };
1102
- });
1103
- return {
1104
- results,
1105
- meta: {
1106
- total: results.length,
1107
- metric,
1108
- threshold,
1109
- improved: results.filter((r) => r.direction === "improved").length,
1110
- worsened: results.filter((r) => r.direction === "worsened").length
1111
- }
1112
- };
1113
- }
1114
- });
1115
- const INTENT_PREFIXES_REGEX = "^(how to|what is|what are|why is|why do|where to|when to|best|top|vs|versus|compare|review|buy|cheap|free|near me)(\\s|$)";
1116
- const INTENT_PREFIXES = [
1117
- "how to",
1118
- "what is",
1119
- "what are",
1120
- "why is",
1121
- "why do",
1122
- "where to",
1123
- "when to",
1124
- "best",
1125
- "top",
1126
- "vs",
1127
- "versus",
1128
- "compare",
1129
- "review",
1130
- "buy",
1131
- "cheap",
1132
- "free",
1133
- "near me"
1134
- ];
1135
- const WHITESPACE_RE = /\s+/;
1136
- function extractIntentPrefix(keyword) {
1137
- const lower = keyword.toLowerCase();
1138
- for (const prefix of INTENT_PREFIXES) if (lower.startsWith(`${prefix} `) || lower.startsWith(prefix)) return prefix;
1139
- return null;
1140
- }
1141
- function extractWordPrefix(keyword, wordCount = 2) {
1142
- const words = keyword.toLowerCase().split(WHITESPACE_RE).filter(Boolean);
1143
- if (words.length < wordCount + 1) return null;
1144
- return words.slice(0, wordCount).join(" ");
1145
- }
1146
- function analyzeClustering(keywords, options = {}) {
1147
- const { minClusterSize = 2, minImpressions = 10, clusterBy = "both" } = options;
1148
- const filtered = keywords.filter((k) => num(k.impressions) >= minImpressions);
1149
- const clusterMap = /* @__PURE__ */ new Map();
1150
- const clusteredKeywords = /* @__PURE__ */ new Set();
1151
- if (clusterBy === "intent" || clusterBy === "both") for (const kw of filtered) {
1152
- const intent = extractIntentPrefix(kw.query);
1153
- if (intent) {
1154
- const existing = clusterMap.get(intent);
1155
- if (existing) existing.keywords.push(kw);
1156
- else clusterMap.set(intent, {
1157
- type: "intent",
1158
- keywords: [kw]
1159
- });
1160
- clusteredKeywords.add(kw.query);
1161
- }
1162
- }
1163
- if (clusterBy === "prefix" || clusterBy === "both") {
1164
- const prefixMap = /* @__PURE__ */ new Map();
1165
- for (const kw of filtered) {
1166
- if (clusteredKeywords.has(kw.query)) continue;
1167
- const prefix = extractWordPrefix(kw.query);
1168
- if (prefix) {
1169
- const existing = prefixMap.get(prefix);
1170
- if (existing) existing.push(kw);
1171
- else prefixMap.set(prefix, [kw]);
1172
- }
1173
- }
1174
- for (const [prefix, kws] of prefixMap) if (kws.length >= minClusterSize) {
1175
- clusterMap.set(prefix, {
1176
- type: "prefix",
1177
- keywords: kws
1178
- });
1179
- for (const kw of kws) clusteredKeywords.add(kw.query);
1180
- }
1181
- }
1182
- const clusters = [];
1183
- for (const [name, data] of clusterMap) {
1184
- if (data.keywords.length < minClusterSize) continue;
1185
- let totalClicks = 0;
1186
- let totalImpressions = 0;
1187
- let weightedPositionSum = 0;
1188
- let positionWeight = 0;
1189
- let positionSum = 0;
1190
- for (const k of data.keywords) {
1191
- const impressions = num(k.impressions);
1192
- const position = num(k.position);
1193
- totalClicks += num(k.clicks);
1194
- totalImpressions += impressions;
1195
- positionSum += position;
1196
- if (impressions > 0) {
1197
- weightedPositionSum += (position - 1) * impressions;
1198
- positionWeight += impressions;
1199
- }
1200
- }
1201
- const avgPosition = positionWeight > 0 ? weightedPositionSum / positionWeight + 1 : positionSum / data.keywords.length;
1202
- clusters.push({
1203
- clusterName: name,
1204
- clusterType: data.type,
1205
- keywords: data.keywords,
1206
- totalClicks,
1207
- totalImpressions,
1208
- avgPosition,
1209
- keywordCount: data.keywords.length
1210
- });
1211
- }
1212
- clusters.sort((a, b) => b.totalClicks - a.totalClicks);
1213
- return {
1214
- clusters,
1215
- unclustered: filtered.filter((kw) => !clusteredKeywords.has(kw.query))
1216
- };
1217
- }
1218
- const clusteringAnalyzer = defineAnalyzer({
1219
- id: "clustering",
1220
- buildSql(params) {
1221
- const { startDate, endDate } = periodOf(params);
1222
- const minImpressions = params.minImpressions ?? 10;
1223
- const minClusterSize = params.minClusterSize ?? 2;
1224
- const clusterBy = params.clusterBy ?? "both";
1225
- const doIntent = clusterBy === "intent" || clusterBy === "both";
1226
- const doPrefix = clusterBy === "prefix" || clusterBy === "both";
1227
- const intentExpr = doIntent ? `NULLIF(regexp_extract(LOWER(query), '${INTENT_PREFIXES_REGEX}', 1), '')` : `CAST(NULL AS VARCHAR)`;
1228
- const prefixExpr = doPrefix ? `CASE WHEN len(regexp_split_to_array(LOWER(query), '\\s+')) >= 3
1229
- THEN array_to_string(list_slice(regexp_split_to_array(LOWER(query), '\\s+'), 1, 2), ' ')
1230
- ELSE CAST(NULL AS VARCHAR) END` : `CAST(NULL AS VARCHAR)`;
1231
- return {
1232
- sql: `
1233
- WITH agg AS (
1234
- SELECT
1235
- query,
1236
- ${METRIC_EXPR.clicks} AS clicks,
1237
- ${METRIC_EXPR.impressions} AS impressions,
1238
- ${METRIC_EXPR.ctr} AS ctr,
1239
- ${METRIC_EXPR.position} AS position
1240
- FROM read_parquet({{FILES}}, union_by_name = true)
1241
- WHERE date >= ? AND date <= ?
1242
- GROUP BY query
1243
- HAVING SUM(impressions) >= ?
1244
- ),
1245
- classified AS (
1246
- SELECT
1247
- query, clicks, impressions, ctr, position,
1248
- ${intentExpr} AS intent_prefix,
1249
- ${prefixExpr} AS word_prefix
1250
- FROM agg
1251
- ),
1252
- keyed AS (
1253
- SELECT
1254
- query, clicks, impressions, ctr, position,
1255
- COALESCE(intent_prefix, word_prefix) AS cluster_name,
1256
- CASE WHEN intent_prefix IS NOT NULL THEN 'intent' ELSE 'prefix' END AS cluster_type
1257
- FROM classified
1258
- WHERE COALESCE(intent_prefix, word_prefix) IS NOT NULL
1259
- )
1260
- SELECT
1261
- cluster_name AS clusterName,
1262
- any_value(cluster_type) AS clusterType,
1263
- CAST(COUNT(*) AS DOUBLE) AS keywordCount,
1264
- ${METRIC_EXPR.clicks} AS totalClicks,
1265
- ${METRIC_EXPR.impressions} AS totalImpressions,
1266
- SUM((position - 1) * impressions) / NULLIF(SUM(impressions), 0) + 1 AS avgPosition,
1267
- to_json(list({ 'query': query, 'clicks': clicks, 'impressions': impressions, 'ctr': ctr, 'position': position })) AS keywords
1268
- FROM keyed
1269
- GROUP BY cluster_name
1270
- HAVING COUNT(*) >= ?
1271
- ORDER BY totalClicks DESC
1272
- `,
1273
- params: [
1274
- startDate,
1275
- endDate,
1276
- minImpressions,
1277
- minClusterSize
1278
- ],
1279
- current: {
1280
- table: "queries",
1281
- partitions: enumeratePartitions(startDate, endDate)
1282
- }
1283
- };
1284
- },
1285
- reduceSql(rows) {
1286
- const clusters = (Array.isArray(rows) ? rows : []).map((r) => ({
1287
- clusterName: rowString(r.clusterName),
1288
- clusterType: rowString(r.clusterType),
1289
- keywordCount: num(r.keywordCount),
1290
- totalClicks: num(r.totalClicks),
1291
- totalImpressions: num(r.totalImpressions),
1292
- avgPosition: num(r.avgPosition),
1293
- keywords: parseJsonRows(r.keywords).map((k) => ({
1294
- query: rowString(k.query),
1295
- clicks: num(k.clicks),
1296
- impressions: num(k.impressions),
1297
- ctr: num(k.ctr),
1298
- position: num(k.position)
1299
- }))
1300
- }));
1301
- return {
1302
- results: clusters,
1303
- meta: {
1304
- total: clusters.length,
1305
- totalClusters: clusters.length
1306
- }
1307
- };
1308
- },
1309
- buildRows(params) {
1310
- return { queries: queriesQueryState(periodOf(params), params.limit) };
1311
- },
1312
- reduceRows(rows, params) {
1313
- const result = analyzeClustering(Array.isArray(rows) ? rows : [], {
1314
- clusterBy: params.clusterBy,
1315
- minClusterSize: params.minClusterSize,
1316
- minImpressions: params.minImpressions
1317
- });
1318
- return {
1319
- results: result.clusters,
1320
- meta: { totalClusters: result.clusters.length }
1321
- };
1322
- }
1323
- });
1324
- function analyzeConcentration(items, options = {}) {
1325
- const { topN = 10 } = options;
1326
- if (items.length === 0) return {
1327
- giniCoefficient: 0,
1328
- hhi: 0,
1329
- topNConcentration: 0,
1330
- topNItems: [],
1331
- totalItems: 0,
1332
- totalClicks: 0,
1333
- riskLevel: "low"
1334
- };
1335
- const sorted = [...items].sort((a, b) => b.clicks - a.clicks);
1336
- const n = sorted.length;
1337
- let totalClicks = 0;
1338
- let squaredClicks = 0;
1339
- let weightedSum = 0;
1340
- for (let i = 0; i < n; i++) {
1341
- const clicks = sorted[i].clicks;
1342
- totalClicks += clicks;
1343
- squaredClicks += clicks * clicks;
1344
- weightedSum += (n - 2 * i - 1) * clicks;
1345
- }
1346
- const giniCoefficient = totalClicks === 0 ? 0 : weightedSum / (n * totalClicks);
1347
- const hhi = totalClicks > 0 ? squaredClicks / (totalClicks * totalClicks) * 1e4 : 0;
1348
- const topNItems = sorted.slice(0, topN).map((item) => ({
1349
- key: item.key,
1350
- clicks: item.clicks,
1351
- share: totalClicks > 0 ? item.clicks / totalClicks : 0
1352
- }));
1353
- const topNClicks = topNItems.reduce((sum, item) => sum + item.clicks, 0);
1354
- const topNConcentration = totalClicks > 0 ? topNClicks / totalClicks : 0;
1355
- let riskLevel = "low";
1356
- if (hhi > 2500) riskLevel = "high";
1357
- else if (hhi > 1500) riskLevel = "medium";
1358
- return {
1359
- giniCoefficient,
1360
- hhi,
1361
- topNConcentration,
1362
- topNItems,
1363
- totalItems: items.length,
1364
- totalClicks,
1365
- riskLevel
1366
- };
1367
- }
1368
- function analyzePageConcentration(pages, options) {
1369
- return analyzeConcentration(pages.map((p) => ({
1370
- key: p.page,
1371
- clicks: num(p.clicks)
1372
- })), options);
1373
- }
1374
- function analyzeKeywordConcentration(keywords, options) {
1375
- return analyzeConcentration(keywords.map((k) => ({
1376
- key: k.query,
1377
- clicks: num(k.clicks)
1378
- })), options);
1379
- }
1380
- const concentrationAnalyzer = defineAnalyzer({
1381
- id: "concentration",
1382
- buildSql(params) {
1383
- const { startDate, endDate } = periodOf(params);
1384
- const dim = params.dimension || "pages";
1385
- const topN = params.topN ?? 10;
1386
- const table = dim === "keywords" ? "queries" : "pages";
1387
- const keyCol = dim === "keywords" ? "query" : "url";
1388
- return {
1389
- sql: `
1390
- WITH items AS (
1391
- SELECT
1392
- ${keyCol} AS key,
1393
- ${METRIC_EXPR.clicks} AS clicks
1394
- FROM read_parquet({{FILES}}, union_by_name = true)
1395
- WHERE date >= ? AND date <= ?
1396
- GROUP BY ${keyCol}
1397
- HAVING SUM(clicks) > 0
1398
- ),
1399
- totals AS (
1400
- SELECT SUM(clicks) AS total_clicks, COUNT(*) AS total_items FROM items
1401
- ),
1402
- ranked AS (
1403
- SELECT
1404
- i.key, i.clicks,
1405
- i.clicks / NULLIF(t.total_clicks, 0) AS share,
1406
- ROW_NUMBER() OVER (ORDER BY i.clicks DESC, i.key ASC) AS rnk_desc,
1407
- ROW_NUMBER() OVER (ORDER BY i.clicks ASC, i.key ASC) AS rnk_asc,
1408
- t.total_clicks AS tclicks,
1409
- t.total_items AS titems
1410
- FROM items i, totals t
1411
- ),
1412
- gini_num AS (
1413
- SELECT SUM((2.0 * rnk_asc - titems - 1) * clicks) AS weighted_sum FROM ranked
1414
- ),
1415
- hhi_calc AS (
1416
- SELECT SUM(POWER(share * 100, 2)) AS hhi FROM ranked
1417
- ),
1418
- top_list AS (
1419
- SELECT
1420
- list({ 'key': key, 'clicks': clicks, 'share': share } ORDER BY clicks DESC, key ASC) AS items,
1421
- SUM(clicks) AS top_clicks
1422
- FROM ranked WHERE rnk_desc <= ?
1423
- )
1424
- SELECT
1425
- COALESCE(
1426
- (SELECT weighted_sum FROM gini_num)
1427
- / NULLIF((SELECT total_items FROM totals) * (SELECT total_clicks FROM totals), 0),
1428
- 0.0
1429
- ) AS giniCoefficient,
1430
- COALESCE((SELECT hhi FROM hhi_calc), 0.0) AS hhi,
1431
- COALESCE(
1432
- CAST((SELECT top_clicks FROM top_list) AS DOUBLE)
1433
- / NULLIF((SELECT total_clicks FROM totals), 0),
1434
- 0.0
1435
- ) AS topNConcentration,
1436
- COALESCE((SELECT to_json(items) FROM top_list), '[]') AS topNItems,
1437
- COALESCE((SELECT total_items FROM totals), 0) AS totalItems,
1438
- COALESCE((SELECT total_clicks FROM totals), 0.0) AS totalClicks,
1439
- CASE
1440
- WHEN COALESCE((SELECT hhi FROM hhi_calc), 0.0) > 2500 THEN 'high'
1441
- WHEN COALESCE((SELECT hhi FROM hhi_calc), 0.0) > 1500 THEN 'medium'
1442
- ELSE 'low'
1443
- END AS riskLevel
1444
- `,
1445
- params: [
1446
- startDate,
1447
- endDate,
1448
- topN
1449
- ],
1450
- current: {
1451
- table,
1452
- partitions: enumeratePartitions(startDate, endDate)
1453
- }
1454
- };
1455
- },
1456
- reduceSql(rows, params) {
1457
- const r = (Array.isArray(rows) ? rows : [])[0] ?? {};
1458
- const topRaw = parseJsonRows(r.topNItems);
1459
- return {
1460
- results: [{
1461
- giniCoefficient: num(r.giniCoefficient),
1462
- hhi: num(r.hhi),
1463
- topNConcentration: num(r.topNConcentration),
1464
- topNItems: topRaw.map((t) => ({
1465
- key: rowString(t.key),
1466
- clicks: num(t.clicks),
1467
- share: num(t.share)
1468
- })),
1469
- totalItems: num(r.totalItems),
1470
- totalClicks: num(r.totalClicks),
1471
- riskLevel: rowString(r.riskLevel)
1472
- }],
1473
- meta: {
1474
- total: 1,
1475
- dimension: params.dimension || "pages"
1476
- }
1477
- };
1478
- },
1479
- buildRows(params) {
1480
- const dim = params.dimension || "pages";
1481
- const period = periodOf(params);
1482
- const out = {};
1483
- if (dim === "pages") out.pages = pagesQueryState(period, params.limit);
1484
- else out.queries = queriesQueryState(period, params.limit);
1485
- return out;
1486
- },
1487
- reduceRows(rows, params) {
1488
- const dim = params.dimension || "pages";
1489
- const arr = Array.isArray(rows) ? rows : rows[dim === "pages" ? "pages" : "queries"] ?? [];
1490
- return {
1491
- results: [dim === "pages" ? analyzePageConcentration(arr, { topN: params.topN }) : analyzeKeywordConcentration(arr, { topN: params.topN })],
1492
- meta: { dimension: dim }
1493
- };
1494
- }
1495
- });
1496
- const contentVelocityAnalyzer = defineAnalyzer({
1497
- id: "content-velocity",
1498
- buildSql(params) {
1499
- const days = Math.min(Math.max(Number(params.days ?? 90), 7), 365);
1500
- const { endDate } = periodOf(params);
1501
- const start = new Date(endDate);
1502
- start.setUTCDate(start.getUTCDate() - days);
1503
- const startDate = toIsoDate(start);
1504
- return {
1505
- sql: `
1506
- WITH src AS (
1507
- SELECT query, date
1508
- FROM read_parquet({{FILES}}, union_by_name = true)
1509
- WHERE date >= ? AND date <= ? AND impressions > 0
1510
- ),
1511
- first_seen AS (
1512
- SELECT query, MIN(date) AS first_date FROM src GROUP BY query
1513
- ),
1514
- per_week AS (
1515
- SELECT
1516
- strftime(CAST(date AS DATE), '%G-W%V') AS week,
1517
- MIN(date) AS week_start,
1518
- CAST(COUNT(DISTINCT query) AS DOUBLE) AS totalKeywords
1519
- FROM src
1520
- GROUP BY week
1521
- ),
1522
- new_per_week AS (
1523
- SELECT
1524
- strftime(CAST(first_date AS DATE), '%G-W%V') AS week,
1525
- CAST(COUNT(*) AS DOUBLE) AS newKeywords
1526
- FROM first_seen
1527
- GROUP BY week
1528
- )
1529
- SELECT
1530
- pw.week AS week,
1531
- COALESCE(npw.newKeywords, 0) AS newKeywords,
1532
- pw.totalKeywords AS totalKeywords
1533
- FROM per_week pw
1534
- LEFT JOIN new_per_week npw ON pw.week = npw.week
1535
- ORDER BY pw.week ASC
1536
- `,
1537
- params: [startDate, endDate],
1538
- current: {
1539
- table: "queries",
1540
- partitions: enumeratePartitions(startDate, endDate)
1541
- }
1542
- };
1543
- },
1544
- reduceSql(rows, params) {
1545
- const arr = Array.isArray(rows) ? rows : [];
1546
- const days = Math.min(Math.max(Number(params.days ?? 90), 7), 365);
1547
- const { endDate } = periodOf(params);
1548
- const startDateD = new Date(endDate);
1549
- startDateD.setUTCDate(startDateD.getUTCDate() - days);
1550
- const startDate = toIsoDate(startDateD);
1551
- const weekly = arr.map((r) => ({
1552
- week: rowString(r.week),
1553
- newKeywords: rowNumber(r.newKeywords),
1554
- totalKeywords: rowNumber(r.totalKeywords)
1555
- }));
1556
- const total = weekly.reduce((s, w) => s + w.newKeywords, 0);
1557
- const avg = weekly.length > 0 ? total / weekly.length : 0;
1558
- const mid = Math.floor(weekly.length / 2);
1559
- const firstAvg = mid > 0 ? weekly.slice(0, mid).reduce((s, w) => s + w.newKeywords, 0) / mid : 0;
1560
- const diff = (weekly.length - mid > 0 ? weekly.slice(mid).reduce((s, w) => s + w.newKeywords, 0) / (weekly.length - mid) : 0) - firstAvg;
1561
- const threshold = Math.max(1, avg * .15);
1562
- return {
1563
- results: weekly,
1564
- meta: {
1565
- summary: {
1566
- totalNewKeywords: total,
1567
- avgPerWeek: avg,
1568
- trend: diff > threshold ? "accelerating" : diff < -threshold ? "decelerating" : "stable"
1569
- },
1570
- days,
1571
- startDate,
1572
- endDate
1573
- }
1574
- };
1575
- }
1576
- });
1577
- const ctrAnomalyAnalyzer = defineAnalyzer({
1578
- id: "ctr-anomaly",
1579
- buildSql(params) {
1580
- const endDate = params.endDate ?? defaultEndDate();
1581
- const startDate = params.startDate ?? daysAgoUtc(93);
1582
- const minDailyImpressions = params.minImpressions ?? 5;
1583
- const minRollingN = 14;
1584
- const zThreshold = params.threshold ?? 2;
1585
- const maxPositionDelta = 1.5;
1586
- const minBreachDays = 2;
1587
- const limit = params.limit ?? 200;
1588
- return {
1589
- sql: `
1590
- WITH daily AS (
1591
- SELECT
1592
- query,
1593
- url AS page,
1594
- -- Normalize at the source CTE: union_by_name=true can coerce date to
1595
- -- VARCHAR across parquets with mixed schemas, which makes downstream
1596
- -- strftime(date, ...) binder-error.
1597
- CAST(date AS DATE) AS date,
1598
- ${METRIC_EXPR.clicks} AS day_clicks,
1599
- ${METRIC_EXPR.impressions} AS day_impressions,
1600
- ${METRIC_EXPR.ctr} AS day_ctr,
1601
- ${METRIC_EXPR.position} AS day_position
1602
- FROM read_parquet({{FILES}}, union_by_name = true)
1603
- WHERE date >= ? AND date <= ?
1604
- AND query IS NOT NULL AND query <> ''
1605
- AND url IS NOT NULL AND url <> ''
1606
- GROUP BY query, url, date
1607
- HAVING SUM(impressions) >= ?
1608
- ),
1609
- rolled AS (
1610
- SELECT *,
1611
- AVG(day_ctr) OVER w AS rolling_ctr,
1612
- STDDEV_POP(day_ctr) OVER w AS rolling_stddev,
1613
- -- Impression-weighted rolling mean: day_position is already a recovered
1614
- -- 1-based daily weighted mean, so weighting by day impressions directly
1615
- -- is algebraically exact (no -1/+1 dance needed).
1616
- SUM(day_position * day_impressions) OVER w
1617
- / NULLIF(SUM(day_impressions) OVER w, 0) AS rolling_position,
1618
- COUNT(*) OVER w AS rolling_n
1619
- FROM daily
1620
- WINDOW w AS (
1621
- PARTITION BY query, page
1622
- ORDER BY date
1623
- ROWS BETWEEN 28 PRECEDING AND 1 PRECEDING
1624
- )
1625
- ),
1626
- flagged AS (
1627
- SELECT *,
1628
- CASE
1629
- WHEN rolling_n >= ${Number(minRollingN)} AND rolling_stddev > 0
1630
- THEN (day_ctr - rolling_ctr) / rolling_stddev
1631
- ELSE 0.0
1632
- END AS z_score,
1633
- CASE
1634
- WHEN rolling_position IS NULL THEN 0.0
1635
- ELSE ABS(day_position - rolling_position)
1636
- END AS position_delta
1637
- FROM rolled
1638
- ),
1639
- breaches AS (
1640
- SELECT *,
1641
- CASE
1642
- WHEN ABS(z_score) >= ${zThreshold}
1643
- AND position_delta <= ${maxPositionDelta}
1644
- AND rolling_n >= ${Number(minRollingN)}
1645
- THEN true ELSE false
1646
- END AS is_breach
1647
- FROM flagged
1648
- ),
1649
- per_entity AS (
1650
- SELECT
1651
- query, page,
1652
- COUNT(*) FILTER (WHERE is_breach AND z_score < 0) AS breach_days_down,
1653
- COUNT(*) FILTER (WHERE is_breach AND z_score > 0) AS breach_days_up,
1654
- SUM(CASE
1655
- WHEN is_breach AND z_score < 0
1656
- THEN (rolling_ctr - day_ctr) * day_impressions
1657
- ELSE 0.0
1658
- END) AS clicks_lost,
1659
- SUM(CASE
1660
- WHEN is_breach AND z_score < 0
1661
- THEN ABS(z_score) * day_impressions
1662
- ELSE 0.0
1663
- END) AS severity_raw,
1664
- MAX(CASE WHEN is_breach THEN ABS(z_score) ELSE 0.0 END) AS max_z,
1665
- AVG(rolling_ctr) FILTER (WHERE rolling_n >= ${Number(minRollingN)}) AS baseline_ctr,
1666
- SUM(rolling_position * day_impressions) FILTER (WHERE rolling_n >= ${Number(minRollingN)})
1667
- / NULLIF(SUM(day_impressions) FILTER (WHERE rolling_n >= ${Number(minRollingN)}), 0) AS baseline_position,
1668
- SUM(day_impressions) AS total_impressions,
1669
- SUM(day_clicks) AS total_clicks
1670
- FROM breaches
1671
- GROUP BY query, page
1672
- HAVING COUNT(*) FILTER (WHERE is_breach AND z_score < 0) >= ${Number(minBreachDays)}
1673
- ),
1674
- series AS (
1675
- SELECT query, page,
1676
- to_json(list({
1677
- 'date': strftime(date, '%Y-%m-%d'),
1678
- 'ctr': day_ctr,
1679
- 'position': day_position,
1680
- 'impressions': day_impressions,
1681
- 'rollingCtr': rolling_ctr,
1682
- 'rollingStddev': rolling_stddev,
1683
- 'z': z_score,
1684
- 'breach': is_breach AND z_score < 0
1685
- } ORDER BY date)) AS seriesJson
1686
- FROM breaches
1687
- GROUP BY query, page
1688
- )
1689
- SELECT
1690
- e.query AS keyword,
1691
- e.page,
1692
- CAST(e.breach_days_down AS DOUBLE) AS breachDaysDown,
1693
- CAST(e.breach_days_up AS DOUBLE) AS breachDaysUp,
1694
- CAST(ROUND(e.clicks_lost) AS DOUBLE) AS clicksLost,
1695
- e.severity_raw AS severityRaw,
1696
- e.max_z AS maxZ,
1697
- e.baseline_ctr AS baselineCtr,
1698
- e.baseline_position AS baselinePosition,
1699
- e.total_impressions AS totalImpressions,
1700
- e.total_clicks AS totalClicks,
1701
- s.seriesJson
1702
- FROM per_entity e
1703
- LEFT JOIN series s USING (query, page)
1704
- ORDER BY clicksLost DESC
1705
- LIMIT ${Number(limit)}
1706
- `,
1707
- params: [
1708
- startDate,
1709
- endDate,
1710
- minDailyImpressions
1711
- ],
1712
- current: {
1713
- table: "page_queries",
1714
- partitions: enumeratePartitions(startDate, endDate)
1715
- }
1716
- };
1717
- },
1718
- reduceSql(rows, params) {
1719
- const arr = Array.isArray(rows) ? rows : [];
1720
- const minRollingN = 14;
1721
- const zThreshold = params.threshold ?? 2;
1722
- const anomalies = arr.map((r) => ({
1723
- keyword: rowString(r.keyword),
1724
- page: rowString(r.page),
1725
- breachDaysDown: rowNumber(r.breachDaysDown),
1726
- breachDaysUp: rowNumber(r.breachDaysUp),
1727
- clicksLost: rowNumber(r.clicksLost),
1728
- severity: rowNumber(r.severityRaw),
1729
- maxZ: rowNumber(r.maxZ),
1730
- baselineCtr: rowNumber(r.baselineCtr),
1731
- baselinePosition: rowNumber(r.baselinePosition),
1732
- totalImpressions: rowNumber(r.totalImpressions),
1733
- totalClicks: rowNumber(r.totalClicks),
1734
- series: parseJsonRows(r.seriesJson).map((s) => ({
1735
- date: rowString(s.date),
1736
- ctr: rowNumber(s.ctr),
1737
- position: rowNumber(s.position),
1738
- impressions: rowNumber(s.impressions),
1739
- rollingCtr: s.rollingCtr == null ? null : rowNumber(s.rollingCtr),
1740
- rollingStddev: s.rollingStddev == null ? null : rowNumber(s.rollingStddev),
1741
- z: rowNumber(s.z),
1742
- breach: rowBoolean(s.breach)
1743
- }))
1744
- }));
1745
- const totalClicksLost = anomalies.reduce((s, a) => s + a.clicksLost, 0);
1746
- const totalBreachDays = anomalies.reduce((s, a) => s + a.breachDaysDown, 0);
1747
- return {
1748
- results: anomalies,
1749
- meta: {
1750
- total: anomalies.length,
1751
- totalClicksLost,
1752
- totalBreachDays,
1753
- zThreshold,
1754
- minRollingN
1755
- }
1756
- };
1757
- }
1758
- });
1759
- const ctrCurveAnalyzer = defineAnalyzer({
1760
- id: "ctr-curve",
1761
- buildSql(params) {
1762
- const { startDate, endDate } = periodOf(params);
1763
- return {
1764
- sql: `
1765
- WITH src AS (
1766
- SELECT
1767
- query,
1768
- clicks,
1769
- impressions,
1770
- sum_position,
1771
- (sum_position / NULLIF(impressions, 0) + 1) AS avg_pos
1772
- FROM read_parquet({{FILES}}, union_by_name = true)
1773
- WHERE date >= ? AND date <= ? AND impressions > 0
1774
- ),
1775
- curve AS (
1776
- SELECT
1777
- CASE
1778
- WHEN avg_pos <= 1.5 THEN '1'
1779
- WHEN avg_pos <= 2.5 THEN '2'
1780
- WHEN avg_pos <= 3.5 THEN '3'
1781
- WHEN avg_pos <= 5.5 THEN '4-5'
1782
- WHEN avg_pos <= 10.5 THEN '6-10'
1783
- WHEN avg_pos <= 20.5 THEN '11-20'
1784
- ELSE '20+'
1785
- END AS bucket,
1786
- AVG(CAST(clicks AS DOUBLE) / NULLIF(impressions, 0)) AS avgCtr,
1787
- (SUM(sum_position) / NULLIF(SUM(impressions), 0) + 1) AS medianPosition,
1788
- CAST(COUNT(DISTINCT query) AS DOUBLE) AS keywordCount,
1789
- ${METRIC_EXPR.clicks} AS totalClicks,
1790
- ${METRIC_EXPR.impressions} AS totalImpressions
1791
- FROM src
1792
- GROUP BY bucket
1793
- ),
1794
- ks AS (
1795
- SELECT
1796
- query,
1797
- ${METRIC_EXPR.clicks} AS clicks,
1798
- ${METRIC_EXPR.impressions} AS impressions,
1799
- ${METRIC_EXPR.ctr} AS ctr,
1800
- ${METRIC_EXPR.position} AS position,
1801
- CASE
1802
- WHEN ${METRIC_EXPR.position} <= 3.5 THEN 'top3'
1803
- WHEN ${METRIC_EXPR.position} <= 10.5 THEN 'page1'
1804
- WHEN ${METRIC_EXPR.position} <= 20.5 THEN 'page2'
1805
- ELSE 'deep'
1806
- END AS band
1807
- FROM src
1808
- GROUP BY query
1809
- HAVING SUM(impressions) >= 20
1810
- ),
1811
- band_avg AS (
1812
- SELECT band, AVG(ctr) AS band_avg_ctr FROM ks GROUP BY band
1813
- ),
1814
- outliers AS (
1815
- SELECT
1816
- ks.query, ks.clicks, ks.impressions, ks.ctr, ks.position,
1817
- ba.band_avg_ctr AS expectedCtr,
1818
- ks.ctr - ba.band_avg_ctr AS ctrDiff
1819
- FROM ks JOIN band_avg ba ON ks.band = ba.band
1820
- ORDER BY ABS(ks.ctr - ba.band_avg_ctr) DESC
1821
- LIMIT 50
1822
- )
1823
- SELECT
1824
- (SELECT to_json(list({
1825
- 'bucket': bucket,
1826
- 'avgCtr': avgCtr,
1827
- 'medianPosition': medianPosition,
1828
- 'keywordCount': keywordCount,
1829
- 'totalClicks': totalClicks,
1830
- 'totalImpressions': totalImpressions
1831
- })) FROM curve) AS curve_json,
1832
- (SELECT to_json(list({
1833
- 'query': query,
1834
- 'clicks': clicks,
1835
- 'impressions': impressions,
1836
- 'ctr': ctr,
1837
- 'position': position,
1838
- 'expectedCtr': expectedCtr,
1839
- 'ctrDiff': ctrDiff
1840
- })) FROM outliers) AS outliers_json
1841
- `,
1842
- params: [startDate, endDate],
1843
- current: {
1844
- table: "queries",
1845
- partitions: enumeratePartitions(startDate, endDate)
1846
- }
1847
- };
1848
- },
1849
- reduceSql(rows, params) {
1850
- const arr = Array.isArray(rows) ? rows : [];
1851
- const { startDate, endDate } = periodOf(params);
1852
- const row = arr[0] ?? {};
1853
- const curve = parseJsonRows(row.curve_json).map((r) => ({
1854
- bucket: rowString(r.bucket),
1855
- avgCtr: rowNumber(r.avgCtr),
1856
- medianPosition: rowNumber(r.medianPosition),
1857
- keywordCount: rowNumber(r.keywordCount),
1858
- totalClicks: rowNumber(r.totalClicks),
1859
- totalImpressions: rowNumber(r.totalImpressions)
1860
- }));
1861
- const outliers = parseJsonRows(row.outliers_json).map((r) => ({
1862
- query: rowString(r.query),
1863
- clicks: rowNumber(r.clicks),
1864
- impressions: rowNumber(r.impressions),
1865
- ctr: rowNumber(r.ctr),
1866
- position: rowNumber(r.position),
1867
- expectedCtr: rowNumber(r.expectedCtr),
1868
- ctrDiff: rowNumber(r.ctrDiff)
1869
- }));
1870
- return {
1871
- results: curve,
1872
- meta: {
1873
- overperforming: outliers.filter((o) => o.ctrDiff > 0).slice(0, 25),
1874
- underperforming: outliers.filter((o) => o.ctrDiff < 0).slice(0, 25),
1875
- startDate,
1876
- endDate
1877
- }
1878
- };
1879
- }
1880
- });
1881
- const darkTrafficAnalyzer = defineAnalyzer({
1882
- id: "dark-traffic",
1883
- buildSql(params) {
1884
- const { startDate, endDate } = periodOf(params);
1885
- return {
1886
- sql: `
1887
- WITH page_totals AS (
1888
- SELECT SUM(clicks) AS total_clicks, SUM(impressions) AS total_impressions
1889
- FROM read_parquet({{FILES}}, union_by_name = true)
1890
- WHERE date >= ? AND date <= ?
1891
- ),
1892
- kw_totals AS (
1893
- SELECT SUM(clicks) AS total_clicks, SUM(impressions) AS total_impressions
1894
- FROM read_parquet({{FILES_KEYWORDS}}, union_by_name = true)
1895
- WHERE date >= ? AND date <= ?
1896
- ),
1897
- per_page AS (
1898
- SELECT url, SUM(clicks) AS page_clicks
1899
- FROM read_parquet({{FILES}}, union_by_name = true)
1900
- WHERE date >= ? AND date <= ?
1901
- GROUP BY url
1902
- HAVING SUM(clicks) > 0
1903
- ),
1904
- per_page_kw AS (
1905
- SELECT url, SUM(clicks) AS attributed_clicks, COUNT(DISTINCT query) AS kw_count
1906
- FROM read_parquet({{FILES_PAGE_KEYWORDS}}, union_by_name = true)
1907
- WHERE date >= ? AND date <= ?
1908
- GROUP BY url
1909
- ),
1910
- page_rows AS (
1911
- SELECT
1912
- p.url AS url,
1913
- CAST(p.page_clicks AS DOUBLE) AS totalClicks,
1914
- CAST(COALESCE(k.attributed_clicks, 0) AS DOUBLE) AS attributedClicks,
1915
- CAST(p.page_clicks - COALESCE(k.attributed_clicks, 0) AS DOUBLE) AS darkClicks,
1916
- CAST(p.page_clicks - COALESCE(k.attributed_clicks, 0) AS DOUBLE)
1917
- / NULLIF(p.page_clicks, 0) AS darkPercent,
1918
- CAST(COALESCE(k.kw_count, 0) AS DOUBLE) AS keywordCount
1919
- FROM per_page p
1920
- LEFT JOIN per_page_kw k ON p.url = k.url
1921
- WHERE p.page_clicks - COALESCE(k.attributed_clicks, 0) > 0
1922
- ORDER BY darkClicks DESC
1923
- LIMIT 50
1924
- )
1925
- SELECT
1926
- (SELECT to_json({
1927
- 'totalClicks': CAST(total_clicks AS DOUBLE),
1928
- 'totalImpressions': CAST(total_impressions AS DOUBLE)
1929
- }) FROM page_totals) AS page_totals_json,
1930
- (SELECT to_json({
1931
- 'attributedClicks': CAST(total_clicks AS DOUBLE),
1932
- 'attributedImpressions': CAST(total_impressions AS DOUBLE)
1933
- }) FROM kw_totals) AS kw_totals_json,
1934
- (SELECT to_json(list({
1935
- 'url': url,
1936
- 'totalClicks': totalClicks,
1937
- 'attributedClicks': attributedClicks,
1938
- 'darkClicks': darkClicks,
1939
- 'darkPercent': darkPercent,
1940
- 'keywordCount': keywordCount
1941
- })) FROM page_rows) AS pages_json
1942
- `,
1943
- params: [
1944
- startDate,
1945
- endDate,
1946
- startDate,
1947
- endDate,
1948
- startDate,
1949
- endDate,
1950
- startDate,
1951
- endDate
1952
- ],
1953
- current: {
1954
- table: "pages",
1955
- partitions: enumeratePartitions(startDate, endDate)
1956
- },
1957
- extraFiles: {
1958
- KEYWORDS: {
1959
- table: "queries",
1960
- partitions: enumeratePartitions(startDate, endDate)
1961
- },
1962
- PAGE_KEYWORDS: {
1963
- table: "page_queries",
1964
- partitions: enumeratePartitions(startDate, endDate)
1965
- }
1966
- }
1967
- };
1968
- },
1969
- reduceSql(rows, params) {
1970
- const arr = Array.isArray(rows) ? rows : [];
1971
- const { startDate, endDate } = periodOf(params);
1972
- const row = arr[0] ?? {};
1973
- const pageTotals = typeof row.page_totals_json === "string" ? JSON.parse(row.page_totals_json) : row.page_totals_json ?? {};
1974
- const kwTotals = typeof row.kw_totals_json === "string" ? JSON.parse(row.kw_totals_json) : row.kw_totals_json ?? {};
1975
- const totalClicks = num(pageTotals.totalClicks);
1976
- const totalImpressions = num(pageTotals.totalImpressions);
1977
- const attributedClicks = num(kwTotals.attributedClicks);
1978
- const attributedImpressions = num(kwTotals.attributedImpressions);
1979
- const darkClicks = Math.max(0, totalClicks - attributedClicks);
1980
- const darkPercent = totalClicks > 0 ? darkClicks / totalClicks : 0;
1981
- return {
1982
- results: parseJsonRows(row.pages_json).map((r) => ({
1983
- url: rowString(r.url),
1984
- totalClicks: num(r.totalClicks),
1985
- attributedClicks: num(r.attributedClicks),
1986
- darkClicks: num(r.darkClicks),
1987
- darkPercent: num(r.darkPercent),
1988
- keywordCount: num(r.keywordCount)
1989
- })),
1990
- meta: {
1991
- summary: {
1992
- totalClicks,
1993
- attributedClicks,
1994
- darkClicks,
1995
- darkPercent,
1996
- totalImpressions,
1997
- attributedImpressions
1998
- },
1999
- startDate,
2000
- endDate
2001
- }
2002
- };
2003
- }
2004
- });
2005
- function requireBuilderState(input, tool) {
2006
- if (!input || typeof input !== "object" || !("dimensions" in input) || !Array.isArray(input.dimensions)) throw new Error(`${tool}: params.q is required (BuilderState)`);
2007
- return input;
2008
- }
2009
- function optionalBuilderState(input, tool, key) {
2010
- if (input == null) return null;
2011
- if (typeof input !== "object" || !("dimensions" in input) || !Array.isArray(input.dimensions)) throw new Error(`${tool}: params.${key} must be a BuilderState`);
2012
- return input;
2013
- }
2014
- const NUMERIC_METRIC_COLS = [
2015
- "clicks",
2016
- "impressions",
2017
- "ctr",
2018
- "position",
2019
- "prevClicks",
2020
- "prevImpressions",
2021
- "prevCtr",
2022
- "prevPosition",
2023
- "variantCount",
2024
- "totalCount"
2025
- ];
2026
- function coerceNumericCols(row) {
2027
- const out = { ...row };
2028
- for (const col of NUMERIC_METRIC_COLS) if (col in out && out[col] != null) out[col] = Number(out[col]);
2029
- return out;
2030
- }
2031
- function shapeDataQuery(rows, extras, opts) {
2032
- let totalCount;
2033
- let cleaned;
2034
- let totals;
2035
- if (opts.hasPrev) {
2036
- cleaned = rows.map(coerceNumericCols);
2037
- totalCount = Number((extras?.count?.[0])?.total ?? cleaned.length);
2038
- const totalsRow = extras?.totals?.[0] ?? {};
2039
- totals = {
2040
- clicks: Number(totalsRow.clicks ?? 0),
2041
- impressions: Number(totalsRow.impressions ?? 0),
2042
- ctr: Number(totalsRow.ctr ?? 0),
2043
- position: Number(totalsRow.position ?? 0)
2044
- };
2045
- } else {
2046
- const first = rows[0];
2047
- totalCount = Number(first?.totalCount ?? 0);
2048
- totals = {
2049
- clicks: Number(first?.totalClicks ?? 0),
2050
- impressions: Number(first?.totalImpressions ?? 0),
2051
- ctr: Number(first?.totalCtr ?? 0),
2052
- position: Number(first?.totalPosition ?? 0)
2053
- };
2054
- cleaned = rows.map((raw) => {
2055
- const { totalCount: _tc, totalClicks: _tclk, totalImpressions: _timp, totalCtr: _tctr, totalPosition: _tpos, sum_position: _sp, ...rest } = raw;
2056
- return coerceNumericCols(rest);
2057
- });
2058
- }
2059
- const extrasResults = [];
2060
- if (extras?.canonicalExtras) extrasResults.push({
2061
- key: "canonicalExtras",
2062
- results: extras.canonicalExtras
2063
- });
2064
- return {
2065
- results: mergeExtras(cleaned, extrasResults),
2066
- meta: {
2067
- totalCount,
2068
- totals
2069
- }
2070
- };
2071
- }
2072
- function buildDataQueryPlan(params, options) {
2073
- const state = requireBuilderState(params.q, "data-query");
2074
- if (state.dimensions.includes("date")) throw new Error("data-query: date dimension not supported; use data-detail");
2075
- const prev = optionalBuilderState(params.qc, "data-query", "qc");
2076
- const extraQueries = [...buildExtrasQueries(state, options).map((extra) => ({
2077
- name: extra.key,
2078
- sql: extra.sql,
2079
- params: extra.params
2080
- }))];
2081
- const tableKey = options.adapter.inferTable(state.dimensions);
2082
- if (prev) {
2083
- const totals = buildTotalsSql(state, options);
2084
- const comparison = resolveComparisonSQL(state, prev, options, params.comparisonFilter);
2085
- extraQueries.unshift({
2086
- name: "totals",
2087
- sql: totals.sql,
2088
- params: totals.params
2089
- });
2090
- extraQueries.push({
2091
- name: "count",
2092
- sql: comparison.countSql,
2093
- params: comparison.countParams
2094
- });
2095
- return {
2096
- tableKey,
2097
- sql: comparison.sql,
2098
- params: comparison.params,
2099
- extraQueries
2100
- };
2101
- }
2102
- const optimized = resolveToSQLOptimized(state, options);
2103
- return {
2104
- tableKey,
2105
- sql: optimized.sql,
2106
- params: optimized.params,
2107
- extraQueries
2108
- };
2109
- }
2110
- function totalsState(state) {
2111
- return {
2112
- ...state,
2113
- dimensions: [],
2114
- orderBy: void 0,
2115
- rowLimit: 1
2116
- };
2117
- }
2118
- function readTotals(row) {
2119
- return {
2120
- clicks: Number(row?.clicks ?? 0),
2121
- impressions: Number(row?.impressions ?? 0),
2122
- ctr: Number(row?.ctr ?? 0),
2123
- position: Number(row?.position ?? 0)
2124
- };
2125
- }
2126
- function buildDataQueryRows(params) {
2127
- const state = requireBuilderState(params.q, "data-query");
2128
- if (state.dimensions.includes("date")) throw new Error("data-query: date dimension not supported; use data-detail");
2129
- const queries = {
2130
- main: state,
2131
- totals: totalsState(state)
2132
- };
2133
- const prev = optionalBuilderState(params.qc, "data-query", "qc");
2134
- if (prev) {
2135
- queries.prevMain = prev;
2136
- queries.prevTotals = totalsState(prev);
2137
- }
2138
- return queries;
2139
- }
2140
- function shapeDataQueryRowResults(rowMap, params) {
2141
- const main = (rowMap.main ?? []).map((r) => coerceNumericCols(r));
2142
- const totals = readTotals(rowMap.totals?.[0]);
2143
- if (params.qc == null) return {
2144
- results: main,
2145
- meta: {
2146
- totalCount: main.length,
2147
- totals
2148
- }
2149
- };
2150
- const state = requireBuilderState(params.q, "data-query");
2151
- const dims = state.dimensions;
2152
- const keyOf = (row) => dims.map((d) => String(row[d] ?? "")).join("");
2153
- const prevByKey = /* @__PURE__ */ new Map();
2154
- for (const r of rowMap.prevMain ?? []) prevByKey.set(keyOf(r), r);
2155
- const filter = params.comparisonFilter;
2156
- const merged = [];
2157
- const seen = /* @__PURE__ */ new Set();
2158
- for (const cur of main) {
2159
- const key = keyOf(cur);
2160
- seen.add(key);
2161
- const prev = prevByKey.get(key);
2162
- const row = {
2163
- ...cur,
2164
- prevClicks: Number(prev?.clicks ?? 0),
2165
- prevImpressions: Number(prev?.impressions ?? 0),
2166
- prevCtr: Number(prev?.ctr ?? 0),
2167
- prevPosition: Number(prev?.position ?? 0)
2168
- };
2169
- if (passesComparisonFilter(filter, {
2170
- isNew: !prev,
2171
- isLost: false,
2172
- clicksChange: Number(cur.clicks ?? 0) - Number(prev?.clicks ?? 0)
2173
- })) merged.push(row);
2174
- }
2175
- for (const prev of rowMap.prevMain ?? []) {
2176
- const p = prev;
2177
- const key = keyOf(p);
2178
- if (seen.has(key)) continue;
2179
- const row = {
2180
- ...p,
2181
- clicks: 0,
2182
- impressions: 0,
2183
- ctr: 0,
2184
- position: 0,
2185
- prevClicks: Number(p.clicks ?? 0),
2186
- prevImpressions: Number(p.impressions ?? 0),
2187
- prevCtr: Number(p.ctr ?? 0),
2188
- prevPosition: Number(p.position ?? 0)
2189
- };
2190
- if (passesComparisonFilter(filter, {
2191
- isNew: false,
2192
- isLost: true,
2193
- clicksChange: -Number(p.clicks ?? 0)
2194
- })) merged.push(row);
2195
- }
2196
- if (state.orderBy) {
2197
- const { column, dir } = state.orderBy;
2198
- merged.sort((a, b) => {
2199
- const av = Number(a[column]) || 0;
2200
- const bv = Number(b[column]) || 0;
2201
- return dir === "asc" ? av - bv : bv - av;
2202
- });
2203
- }
2204
- return {
2205
- results: merged.map((r) => coerceNumericCols(r)),
2206
- meta: {
2207
- totalCount: merged.length,
2208
- totals
2209
- }
2210
- };
2211
- }
2212
- function passesComparisonFilter(filter, ctx) {
2213
- switch (filter) {
2214
- case "new": return ctx.isNew;
2215
- case "lost": return ctx.isLost;
2216
- case "improving": return ctx.clicksChange > 0;
2217
- case "declining": return ctx.clicksChange < 0;
2218
- default: return true;
2219
- }
2220
- }
2221
- function buildDataDetailRows(params) {
2222
- const state = requireBuilderState(params.q, "data-detail");
2223
- if (!state.dimensions.includes("date")) throw new Error("data-detail: `date` dimension is required");
2224
- const queries = {
2225
- main: state,
2226
- totals: totalsState(state)
2227
- };
2228
- const prev = optionalBuilderState(params.qc, "data-detail", "qc");
2229
- if (prev) queries.prevTotals = totalsState(prev);
2230
- return queries;
2231
- }
2232
- function shapeDataDetailRowResults(rowMap, params) {
2233
- const { startDate, endDate } = extractDateRange(requireBuilderState(params.q, "data-detail").filter);
2234
- const coerced = (rowMap.main ?? []).map((r) => coerceNumericCols(r));
2235
- const daily = startDate && endDate ? padTimeseries(coerced, {
2236
- startDate,
2237
- endDate
2238
- }) : coerced;
2239
- const meta = { totals: readTotals(rowMap.totals?.[0]) };
2240
- if (rowMap.prevTotals) meta.previousTotals = readTotals(rowMap.prevTotals[0]);
2241
- return {
2242
- results: daily,
2243
- meta
2244
- };
2245
- }
2246
- function shapeDataQueryRows(rows, params, extras) {
2247
- return shapeDataQuery(rows, extras, { hasPrev: params.qc != null });
2248
- }
2249
- function buildDataDetailPlan(params, options) {
2250
- const state = requireBuilderState(params.q, "data-detail");
2251
- if (!state.dimensions.includes("date")) throw new Error("data-detail: `date` dimension is required");
2252
- const main = resolveToSQLOptimized(state, options);
2253
- const prev = optionalBuilderState(params.qc, "data-detail", "qc");
2254
- const extraQueries = [];
2255
- if (prev) {
2256
- const previousTotals = buildTotalsSql(prev, options);
2257
- extraQueries.push({
2258
- name: "prevTotals",
2259
- sql: previousTotals.sql,
2260
- params: previousTotals.params
2261
- });
2262
- }
2263
- return {
2264
- tableKey: options.adapter.inferTable(state.dimensions),
2265
- sql: main.sql,
2266
- params: main.params,
2267
- extraQueries
2268
- };
2269
- }
2270
- function shapeDataDetailRows(rows, params, extras) {
2271
- const { startDate: rangeStart, endDate: rangeEnd } = extractDateRange(requireBuilderState(params.q, "data-detail").filter);
2272
- const first = rows[0];
2273
- const totals = {
2274
- clicks: Number(first?.totalClicks ?? 0),
2275
- impressions: Number(first?.totalImpressions ?? 0),
2276
- ctr: Number(first?.totalCtr ?? 0),
2277
- position: Number(first?.totalPosition ?? 0)
2278
- };
2279
- const coerced = rows.map((raw) => {
2280
- const { totalCount: _tc, totalClicks: _tclk, totalImpressions: _timp, totalCtr: _tctr, totalPosition: _tpos, sum_position: _sp, ...rest } = raw;
2281
- return coerceNumericCols(rest);
2282
- });
2283
- const daily = rangeStart && rangeEnd ? padTimeseries(coerced, {
2284
- startDate: rangeStart,
2285
- endDate: rangeEnd
2286
- }) : coerced;
2287
- const meta = { totals };
2288
- if (extras?.prevTotals) {
2289
- const previousTotalsRow = extras.prevTotals[0] ?? {};
2290
- meta.previousTotals = {
2291
- clicks: Number(previousTotalsRow.clicks ?? 0),
2292
- impressions: Number(previousTotalsRow.impressions ?? 0),
2293
- ctr: Number(previousTotalsRow.ctr ?? 0),
2294
- position: Number(previousTotalsRow.position ?? 0)
2295
- };
2296
- }
2297
- return {
2298
- results: daily,
2299
- meta
2300
- };
2301
- }
2302
- const dataDetailAnalyzer = defineAnalyzer({
2303
- id: "data-detail",
2304
- sqlRequires: [
2305
- "executeSql",
2306
- "attachedTables",
2307
- "adapter"
2308
- ],
2309
- buildSql(params, ctx) {
2310
- const plan = buildDataDetailPlan(params, {
2311
- adapter: requireAdapter(ctx, "data-detail"),
2312
- siteId: ctx.siteId
2313
- });
2314
- return {
2315
- sql: plan.sql,
2316
- params: plan.params,
2317
- current: {
2318
- table: plan.tableKey,
2319
- partitions: []
2320
- },
2321
- extraQueries: plan.extraQueries
2322
- };
2323
- },
2324
- reduceSql(rows, params, ctx) {
2325
- const { results, meta } = shapeDataDetailRows(Array.isArray(rows) ? rows : [], params, ctx.extras);
2326
- return {
2327
- results,
2328
- meta
2329
- };
2330
- },
2331
- buildRows(params) {
2332
- return buildDataDetailRows(params);
2333
- },
2334
- reduceRows(rows, params) {
2335
- const { results, meta } = shapeDataDetailRowResults(Array.isArray(rows) ? {} : rows, params);
2336
- return {
2337
- results,
2338
- meta
2339
- };
2340
- }
2341
- });
2342
- const dataQueryAnalyzer = defineAnalyzer({
2343
- id: "data-query",
2344
- sqlRequires: [
2345
- "executeSql",
2346
- "attachedTables",
2347
- "adapter"
2348
- ],
2349
- buildSql(params, ctx) {
2350
- const plan = buildDataQueryPlan(params, {
2351
- adapter: requireAdapter(ctx, "data-query"),
2352
- siteId: ctx.siteId
2353
- });
2354
- return {
2355
- sql: plan.sql,
2356
- params: plan.params,
2357
- current: {
2358
- table: plan.tableKey,
2359
- partitions: []
2360
- },
2361
- extraQueries: plan.extraQueries
2362
- };
2363
- },
2364
- reduceSql(rows, params, ctx) {
2365
- const { results, meta } = shapeDataQueryRows(Array.isArray(rows) ? rows : [], params, ctx.extras);
2366
- return {
2367
- results,
2368
- meta
2369
- };
2370
- },
2371
- buildRows(params) {
2372
- return buildDataQueryRows(params);
2373
- },
2374
- reduceRows(rows, params) {
2375
- const { results, meta } = shapeDataQueryRowResults(Array.isArray(rows) ? {} : rows, params);
2376
- return {
2377
- results,
2378
- meta
2379
- };
2380
- }
2381
- });
2382
- const DEFAULT_LIMIT = 1e3;
2383
- const MAX_LIMIT = 5e4;
2384
- function clampLimit(limit, fallback = DEFAULT_LIMIT) {
2385
- const n = Number(limit ?? fallback);
2386
- if (!Number.isFinite(n) || n <= 0) return fallback;
2387
- return Math.min(n, MAX_LIMIT);
2388
- }
2389
- function clampOffset(offset) {
2390
- const n = Number(offset ?? 0);
2391
- if (!Number.isFinite(n) || n < 0) return 0;
2392
- return Math.floor(n);
2393
- }
2394
- function paginateClause(input) {
2395
- const l = clampLimit(input.limit);
2396
- const o = clampOffset(input.offset);
2397
- return o > 0 ? `LIMIT ${l} OFFSET ${o}` : `LIMIT ${l}`;
2398
- }
2399
- function paginateInMemory(rows, input) {
2400
- const l = clampLimit(input.limit, rows.length);
2401
- const o = clampOffset(input.offset);
2402
- return rows.slice(o, o + l);
2403
- }
2404
- function paginateSortedInMemory(rows, input, compare) {
2405
- const limit = clampLimit(input.limit, rows.length);
2406
- const offset = clampOffset(input.offset);
2407
- const selectedCount = Math.min(rows.length, offset + limit);
2408
- if (limit === 0 || offset >= rows.length || selectedCount === 0) return [];
2409
- const fullSort = () => [...rows].sort(compare).slice(offset, offset + limit);
2410
- if (selectedCount >= rows.length / 2) return fullSort();
2411
- let invalidComparison = false;
2412
- const compareStable = (left, right) => {
2413
- const order = compare(left.value, right.value);
2414
- if (Number.isNaN(order)) {
2415
- invalidComparison = true;
2416
- return left.index - right.index;
2417
- }
2418
- return order || left.index - right.index;
2419
- };
2420
- const isWorse = (left, right) => compareStable(left, right) > 0;
2421
- const heap = [];
2422
- const siftUp = (start) => {
2423
- let index = start;
2424
- while (index > 0) {
2425
- const parent = index - 1 >>> 1;
2426
- if (!isWorse(heap[index], heap[parent])) break;
2427
- const swap = heap[parent];
2428
- heap[parent] = heap[index];
2429
- heap[index] = swap;
2430
- index = parent;
2431
- }
2432
- };
2433
- const siftDown = () => {
2434
- let index = 0;
2435
- for (;;) {
2436
- const left = index * 2 + 1;
2437
- if (left >= heap.length) return;
2438
- const right = left + 1;
2439
- let worse = left;
2440
- if (right < heap.length && isWorse(heap[right], heap[left])) worse = right;
2441
- if (!isWorse(heap[worse], heap[index])) return;
2442
- const swap = heap[index];
2443
- heap[index] = heap[worse];
2444
- heap[worse] = swap;
2445
- index = worse;
2446
- }
2447
- };
2448
- for (let index = 0; index < rows.length; index++) {
2449
- const candidate = {
2450
- value: rows[index],
2451
- index
2452
- };
2453
- if (heap.length < selectedCount) {
2454
- heap.push(candidate);
2455
- siftUp(heap.length - 1);
2456
- } else if (compareStable(candidate, heap[0]) < 0) {
2457
- heap[0] = candidate;
2458
- siftDown();
2459
- }
2460
- }
2461
- if (invalidComparison) return fullSort();
2462
- heap.sort(compareStable);
2463
- if (invalidComparison) return fullSort();
2464
- return heap.slice(offset, offset + limit).map((entry) => entry.value);
2465
- }
2466
- function resolveSort(input, allowed, defaults) {
2467
- return {
2468
- sortBy: input.sortBy && allowed.includes(input.sortBy) ? input.sortBy : defaults.sortBy,
2469
- sortDir: input.sortDir === "asc" || input.sortDir === "desc" ? input.sortDir : defaults.sortDir
2470
- };
2471
- }
2472
- const sortResults = createMetricSorter("lostClicks", {
2473
- lostClicks: "desc",
2474
- declinePercent: "desc",
2475
- currentClicks: "asc"
2476
- });
2477
- function analyzeDecay(input, options = {}) {
2478
- const { minPreviousClicks = 50, threshold = .2, sortBy = "lostClicks" } = options;
2479
- const currentMap = buildPeriodMap(input.current, (r) => r.page, (r) => ({
2480
- clicks: num(r.clicks),
2481
- position: num(r.position)
2482
- }));
2483
- const previousMap = buildPeriodMap(input.previous, (r) => r.page, (r) => ({
2484
- clicks: num(r.clicks),
2485
- position: num(r.position)
2486
- }), (r) => num(r.clicks) >= minPreviousClicks);
2487
- const results = [];
2488
- for (const [page, prev] of previousMap) {
2489
- const curr = currentMap.get(page);
2490
- const currentClicks = curr?.clicks ?? 0;
2491
- const currentPosition = curr?.position ?? null;
2492
- const lostClicks = prev.clicks - currentClicks;
2493
- const declinePercent = prev.clicks > 0 ? lostClicks / prev.clicks : 0;
2494
- if (declinePercent >= threshold && lostClicks > 0) results.push({
2495
- page,
2496
- currentClicks,
2497
- previousClicks: prev.clicks,
2498
- lostClicks,
2499
- declinePercent,
2500
- currentPosition,
2501
- previousPosition: prev.position,
2502
- positionDrop: currentPosition != null ? currentPosition - prev.position : null
2503
- });
2504
- }
2505
- return sortResults(results, sortBy);
2506
- }
2507
- const decayAnalyzer = defineAnalyzer({
2508
- id: "decay",
2509
- buildSql(params) {
2510
- const { current: cur, previous: prev } = comparisonOf(params);
2511
- const minPreviousClicks = params.minPreviousClicks ?? 50;
2512
- const threshold = params.threshold ?? .2;
2513
- return {
2514
- sql: `
2515
- WITH cur AS (
2516
- SELECT
2517
- url,
2518
- ${METRIC_EXPR.clicks} AS clicks,
2519
- ${METRIC_EXPR.position} AS position
2520
- FROM read_parquet({{FILES}}, union_by_name = true)
2521
- WHERE date >= ? AND date <= ?
2522
- GROUP BY url
2523
- ),
2524
- prev AS (
2525
- SELECT
2526
- url,
2527
- ${METRIC_EXPR.clicks} AS clicks,
2528
- ${METRIC_EXPR.position} AS position
2529
- FROM read_parquet({{FILES_PREV}}, union_by_name = true)
2530
- WHERE date >= ? AND date <= ?
2531
- GROUP BY url
2532
- HAVING SUM(clicks) >= ?
2533
- ),
2534
- weekly AS (
2535
- SELECT url, date_trunc('week', CAST(date AS DATE)) AS week,
2536
- ${METRIC_EXPR.clicks} AS clicks,
2537
- ${METRIC_EXPR.impressions} AS impressions
2538
- FROM (
2539
- SELECT url, date, clicks, impressions
2540
- FROM read_parquet({{FILES}}, union_by_name = true)
2541
- WHERE date >= ? AND date <= ?
2542
- UNION ALL
2543
- SELECT url, date, clicks, impressions
2544
- FROM read_parquet({{FILES_PREV}}, union_by_name = true)
2545
- WHERE date >= ? AND date <= ?
2546
- )
2547
- GROUP BY url, week
2548
- ),
2549
- series_by_url AS (
2550
- SELECT url, to_json(list({
2551
- 'week': strftime(week, '%Y-%m-%d'),
2552
- 'clicks': clicks,
2553
- 'impressions': impressions
2554
- } ORDER BY week)) AS seriesJson
2555
- FROM weekly GROUP BY url
2556
- ),
2557
- joined AS (
2558
- SELECT
2559
- p.url AS page,
2560
- COALESCE(c.clicks, 0.0) AS currentClicks,
2561
- p.clicks AS previousClicks,
2562
- (p.clicks - COALESCE(c.clicks, 0.0)) AS lostClicks,
2563
- (p.clicks - COALESCE(c.clicks, 0.0)) / NULLIF(p.clicks, 0) AS declinePercent,
2564
- c.position AS currentPosition,
2565
- p.position AS previousPosition,
2566
- CASE WHEN c.position IS NOT NULL THEN (c.position - p.position) ELSE NULL END AS positionDrop,
2567
- s.seriesJson
2568
- FROM prev p
2569
- LEFT JOIN cur c ON p.url = c.url
2570
- LEFT JOIN series_by_url s ON p.url = s.url
2571
- )
2572
- SELECT *
2573
- FROM joined
2574
- WHERE declinePercent >= ? AND lostClicks > 0
2575
- ORDER BY lostClicks DESC
2576
- LIMIT 2000
2577
- `,
2578
- params: [
2579
- cur.startDate,
2580
- cur.endDate,
2581
- prev.startDate,
2582
- prev.endDate,
2583
- minPreviousClicks,
2584
- cur.startDate,
2585
- cur.endDate,
2586
- prev.startDate,
2587
- prev.endDate,
2588
- threshold
2589
- ],
2590
- current: {
2591
- table: "pages",
2592
- partitions: enumeratePartitions(cur.startDate, cur.endDate)
2593
- },
2594
- previous: {
2595
- table: "pages",
2596
- partitions: enumeratePartitions(prev.startDate, prev.endDate)
2597
- }
2598
- };
2599
- },
2600
- reduceSql(rows, params) {
2601
- const mapped = (Array.isArray(rows) ? rows : []).map((r) => ({
2602
- page: rowString(r.page),
2603
- currentClicks: num(r.currentClicks),
2604
- previousClicks: num(r.previousClicks),
2605
- lostClicks: num(r.lostClicks),
2606
- declinePercent: num(r.declinePercent),
2607
- currentPosition: r.currentPosition == null ? null : num(r.currentPosition),
2608
- previousPosition: num(r.previousPosition),
2609
- positionDrop: r.positionDrop == null ? null : num(r.positionDrop),
2610
- series: parseJsonRows(r.seriesJson).map((s) => ({
2611
- week: rowString(s.week),
2612
- clicks: num(s.clicks),
2613
- impressions: num(s.impressions)
2614
- }))
2615
- }));
2616
- return {
2617
- results: paginateInMemory(mapped, {
2618
- limit: params.limit ?? 2e3,
2619
- offset: params.offset
2620
- }),
2621
- meta: { total: mapped.length }
2622
- };
2623
- },
2624
- buildRows(params) {
2625
- const { current, previous } = comparisonOf(params);
2626
- return {
2627
- current: pagesQueryState(current, params.limit),
2628
- previous: pagesQueryState(previous, params.limit)
2629
- };
2630
- },
2631
- reduceRows(rows, params) {
2632
- const map = rows && !Array.isArray(rows) ? rows : {
2633
- current: [],
2634
- previous: []
2635
- };
2636
- const results = analyzeDecay({
2637
- current: map.current ?? [],
2638
- previous: map.previous ?? []
2639
- }, {
2640
- minPreviousClicks: params.minPreviousClicks,
2641
- threshold: params.threshold
2642
- });
2643
- return {
2644
- results,
2645
- meta: { total: results.length }
2646
- };
2647
- }
2648
- });
2649
- const deviceGapAnalyzer = defineAnalyzer({
2650
- id: "device-gap",
2651
- buildSql(params) {
2652
- const { startDate, endDate } = periodOf(params);
2653
- return {
2654
- sql: `
2655
- WITH raw AS (
2656
- SELECT
2657
- date,
2658
- COALESCE(clicks_desktop, 0) AS c_desktop,
2659
- COALESCE(clicks_mobile, 0) AS c_mobile,
2660
- COALESCE(clicks_tablet, 0) AS c_tablet,
2661
- COALESCE(impressions_desktop, 0) AS i_desktop,
2662
- COALESCE(impressions_mobile, 0) AS i_mobile,
2663
- COALESCE(impressions_tablet, 0) AS i_tablet,
2664
- COALESCE(sum_position_desktop, 0) AS p_desktop,
2665
- COALESCE(sum_position_mobile, 0) AS p_mobile,
2666
- COALESCE(sum_position_tablet, 0) AS p_tablet
2667
- FROM read_parquet({{FILES}}, union_by_name = true)
2668
- WHERE date >= ? AND date <= ?
2669
- ),
2670
- device_long AS (
2671
- SELECT date, 'DESKTOP' AS device, c_desktop AS clicks, i_desktop AS impressions, p_desktop AS sum_position FROM raw
2672
- UNION ALL
2673
- SELECT date, 'MOBILE', c_mobile, i_mobile, p_mobile FROM raw
2674
- UNION ALL
2675
- SELECT date, 'TABLET', c_tablet, i_tablet, p_tablet FROM raw
2676
- )
2677
- SELECT
2678
- date,
2679
- device,
2680
- CAST(SUM(clicks) AS DOUBLE) AS clicks,
2681
- CAST(SUM(impressions) AS DOUBLE) AS impressions,
2682
- CAST(SUM(clicks) AS DOUBLE) / NULLIF(SUM(impressions), 0) AS ctr,
2683
- SUM(sum_position) / NULLIF(SUM(impressions), 0) + 1 AS position
2684
- FROM device_long
2685
- GROUP BY date, device
2686
- ORDER BY date ASC
2687
- `,
2688
- params: [startDate, endDate],
2689
- current: {
2690
- table: "dates",
2691
- partitions: enumeratePartitions(startDate, endDate)
2692
- }
2693
- };
2694
- },
2695
- reduceSql(rows, params) {
2696
- const arr = Array.isArray(rows) ? rows : [];
2697
- const { startDate, endDate } = periodOf(params);
2698
- const typed = arr.map((r) => ({
2699
- date: rowString(r.date),
2700
- device: rowString(r.device).toUpperCase(),
2701
- clicks: num(r.clicks),
2702
- impressions: num(r.impressions),
2703
- ctr: num(r.ctr),
2704
- position: num(r.position)
2705
- }));
2706
- const byDate = /* @__PURE__ */ new Map();
2707
- for (const r of typed) {
2708
- const entry = byDate.get(r.date) ?? {};
2709
- const metrics = {
2710
- clicks: r.clicks,
2711
- impressions: r.impressions,
2712
- ctr: r.ctr,
2713
- position: r.position
2714
- };
2715
- if (r.device === "DESKTOP") entry.desktop = metrics;
2716
- else if (r.device === "MOBILE") entry.mobile = metrics;
2717
- byDate.set(r.date, entry);
2718
- }
2719
- const zero = {
2720
- clicks: 0,
2721
- impressions: 0,
2722
- ctr: 0,
2723
- position: 0
2724
- };
2725
- const daily = [...byDate.entries()].sort(([a], [b]) => a.localeCompare(b)).map(([date, sides]) => {
2726
- const d = sides.desktop ?? zero;
2727
- const m = sides.mobile ?? zero;
2728
- return {
2729
- date,
2730
- desktop: d,
2731
- mobile: m,
2732
- gaps: {
2733
- ctrGap: d.ctr - m.ctr,
2734
- positionGap: m.position - d.position
2735
- }
2736
- };
2737
- });
2738
- const weekly = (start, end) => {
2739
- const slice = daily.slice(start, end);
2740
- if (slice.length === 0) return {
2741
- ctr: 0,
2742
- pos: 0
2743
- };
2744
- const sum = slice.reduce((acc, d) => ({
2745
- ctr: acc.ctr + d.gaps.ctrGap,
2746
- pos: acc.pos + d.gaps.positionGap
2747
- }), {
2748
- ctr: 0,
2749
- pos: 0
2750
- });
2751
- return {
2752
- ctr: sum.ctr / slice.length,
2753
- pos: sum.pos / slice.length
2754
- };
2755
- };
2756
- const first = weekly(0, 7);
2757
- const last = weekly(Math.max(0, daily.length - 7), daily.length);
2758
- const classify = (firstVal, lastVal) => {
2759
- const diff = Math.abs(lastVal) - Math.abs(firstVal);
2760
- if (Math.abs(diff) < .005) return "stable";
2761
- return diff < 0 ? "improving" : "worsening";
2762
- };
2763
- return {
2764
- results: daily,
2765
- meta: {
2766
- summary: {
2767
- avgCtrGap: daily.reduce((s, d) => s + d.gaps.ctrGap, 0) / Math.max(1, daily.length),
2768
- avgPositionGap: daily.reduce((s, d) => s + d.gaps.positionGap, 0) / Math.max(1, daily.length),
2769
- ctrGapTrend: classify(first.ctr, last.ctr),
2770
- positionGapTrend: classify(first.pos, last.pos)
2771
- },
2772
- startDate,
2773
- endDate
2774
- }
2775
- };
2776
- }
2777
- });
2778
- const INTENT_ATLAS_STOP_WORDS = [
2779
- "the",
2780
- "a",
2781
- "an",
2782
- "is",
2783
- "are",
2784
- "was",
2785
- "were",
2786
- "be",
2787
- "been",
2788
- "of",
2789
- "to",
2790
- "in",
2791
- "for",
2792
- "on",
2793
- "and",
2794
- "or",
2795
- "with",
2796
- "at",
2797
- "by",
2798
- "from",
2799
- "into",
2800
- "about",
2801
- "as",
2802
- "so",
2803
- "than",
2804
- "then",
2805
- "that",
2806
- "this",
2807
- "my",
2808
- "your",
2809
- "our",
2810
- "their",
2811
- "his",
2812
- "her",
2813
- "its",
2814
- "me",
2815
- "you",
2816
- "what",
2817
- "how",
2818
- "why",
2819
- "when",
2820
- "where",
2821
- "who",
2822
- "which",
2823
- "do",
2824
- "does"
2825
- ];
2826
- const intentAtlasAnalyzer = defineAnalyzer({
2827
- id: "intent-atlas",
2828
- buildSql(params) {
2829
- const endDate = params.endDate ?? defaultEndDate();
2830
- const startDate = params.startDate ?? daysAgoUtc(90);
2831
- const minQueryImpressions = params.minImpressions ?? 20;
2832
- const minClusterSize = params.minClusterSize ?? 3;
2833
- const minTokenImpressions = 50;
2834
- const limit = params.limit ?? 200;
2835
- const stopList = INTENT_ATLAS_STOP_WORDS.map((w) => `'${w}'`).join(", ");
2836
- return {
2837
- sql: `
2838
- WITH queries AS (
2839
- SELECT
2840
- query,
2841
- ${METRIC_EXPR.impressions} AS impressions,
2842
- ${METRIC_EXPR.clicks} AS clicks,
2843
- ${METRIC_EXPR.position} AS position
2844
- FROM read_parquet({{FILES}}, union_by_name = true)
2845
- WHERE date >= ? AND date <= ?
2846
- AND query IS NOT NULL AND query <> ''
2847
- GROUP BY query
2848
- HAVING SUM(impressions) >= ?
2849
- ),
2850
- tokens AS (
2851
- SELECT q.query, q.impressions, q.clicks, q.position,
2852
- LOWER(t.token) AS token
2853
- FROM queries q,
2854
- unnest(regexp_split_to_array(LOWER(q.query), '\\s+')) AS t(token)
2855
- WHERE LENGTH(t.token) >= 3
2856
- AND LOWER(t.token) NOT IN (${stopList})
2857
- ),
2858
- token_weights AS (
2859
- SELECT token,
2860
- SUM(impressions) AS token_impressions,
2861
- COUNT(DISTINCT query) AS query_count
2862
- FROM tokens
2863
- GROUP BY token
2864
- HAVING SUM(impressions) >= ${Number(minTokenImpressions)}
2865
- ),
2866
- ranked_tokens AS (
2867
- SELECT t.query, t.token, tw.token_impressions,
2868
- ROW_NUMBER() OVER (
2869
- PARTITION BY t.query
2870
- ORDER BY tw.token_impressions DESC, t.token ASC
2871
- ) AS rnk
2872
- FROM tokens t
2873
- JOIN token_weights tw USING (token)
2874
- ),
2875
- cluster_keys AS (
2876
- SELECT query,
2877
- array_to_string(list(token ORDER BY token), ' + ') AS cluster_key
2878
- FROM ranked_tokens
2879
- WHERE rnk <= 2
2880
- GROUP BY query
2881
- HAVING COUNT(*) >= 2
2882
- ),
2883
- clustered AS (
2884
- SELECT q.query, q.impressions, q.clicks, q.position, ck.cluster_key
2885
- FROM queries q
2886
- JOIN cluster_keys ck USING (query)
2887
- )
2888
- SELECT
2889
- cluster_key AS clusterKey,
2890
- COUNT(*) AS keywordCount,
2891
- SUM(impressions) AS totalImpressions,
2892
- SUM(clicks) AS totalClicks,
2893
- SUM(clicks) / NULLIF(SUM(impressions), 0) AS ctr,
2894
- SUM((position - 1) * impressions) / NULLIF(SUM(impressions), 0) + 1 AS avgPosition,
2895
- to_json(list({
2896
- 'query': query,
2897
- 'impressions': impressions,
2898
- 'clicks': clicks,
2899
- 'position': position
2900
- } ORDER BY impressions DESC)) AS keywords
2901
- FROM clustered
2902
- GROUP BY cluster_key
2903
- HAVING COUNT(*) >= ${Number(minClusterSize)}
2904
- ORDER BY totalImpressions DESC
2905
- LIMIT ${Number(limit)}
2906
- `,
2907
- params: [
2908
- startDate,
2909
- endDate,
2910
- minQueryImpressions
2911
- ],
2912
- current: {
2913
- table: "queries",
2914
- partitions: enumeratePartitions(startDate, endDate)
2915
- }
2916
- };
2917
- },
2918
- reduceSql(rows) {
2919
- const clusters = (Array.isArray(rows) ? rows : []).map((r) => ({
2920
- clusterKey: rowString(r.clusterKey),
2921
- keywordCount: num(r.keywordCount),
2922
- totalImpressions: num(r.totalImpressions),
2923
- totalClicks: num(r.totalClicks),
2924
- ctr: num(r.ctr),
2925
- avgPosition: num(r.avgPosition),
2926
- keywords: parseJsonRows(r.keywords).slice(0, 25).map((k) => ({
2927
- query: rowString(k.query),
2928
- impressions: num(k.impressions),
2929
- clicks: num(k.clicks),
2930
- position: num(k.position)
2931
- }))
2932
- }));
2933
- const totalImpressions = clusters.reduce((s, c) => s + c.totalImpressions, 0);
2934
- const totalKeywords = clusters.reduce((s, c) => s + c.keywordCount, 0);
2935
- return {
2936
- results: clusters,
2937
- meta: {
2938
- total: clusters.length,
2939
- totalImpressions,
2940
- totalKeywords
2941
- }
2942
- };
2943
- }
2944
- });
2945
- const keywordBreadthAnalyzer = defineAnalyzer({
2946
- id: "keyword-breadth",
2947
- buildSql(params) {
2948
- const { startDate, endDate } = periodOf(params);
2949
- return {
2950
- sql: `
2951
- WITH per_page AS (
2952
- SELECT
2953
- url,
2954
- CAST(COUNT(DISTINCT query) AS DOUBLE) AS keywordCount,
2955
- ${METRIC_EXPR.clicks} AS clicks,
2956
- ${METRIC_EXPR.impressions} AS impressions
2957
- FROM read_parquet({{FILES}}, union_by_name = true)
2958
- WHERE date >= ? AND date <= ? AND impressions > 0
2959
- GROUP BY url
2960
- ),
2961
- bucketed AS (
2962
- SELECT
2963
- CASE
2964
- WHEN keywordCount = 1 THEN '1'
2965
- WHEN keywordCount BETWEEN 2 AND 5 THEN '2-5'
2966
- WHEN keywordCount BETWEEN 6 AND 15 THEN '6-15'
2967
- WHEN keywordCount BETWEEN 16 AND 50 THEN '16-50'
2968
- ELSE '50+'
2969
- END AS bucket,
2970
- MIN(keywordCount) AS sort_key,
2971
- CAST(COUNT(*) AS DOUBLE) AS pageCount
2972
- FROM per_page
2973
- GROUP BY bucket
2974
- ),
2975
- fragile AS (
2976
- SELECT url, keywordCount, clicks, impressions
2977
- FROM per_page
2978
- WHERE keywordCount <= 2 AND clicks >= 5
2979
- ORDER BY clicks DESC
2980
- LIMIT 20
2981
- ),
2982
- authority AS (
2983
- SELECT url, keywordCount, clicks, impressions
2984
- FROM per_page
2985
- WHERE keywordCount >= 20
2986
- ORDER BY keywordCount DESC
2987
- LIMIT 20
2988
- ),
2989
- stats AS (
2990
- SELECT
2991
- CAST(COUNT(*) AS DOUBLE) AS totalPages,
2992
- CAST(AVG(keywordCount) AS DOUBLE) AS avgKeywordsPerPage,
2993
- CAST(SUM(CASE WHEN keywordCount <= 2 THEN 1 ELSE 0 END) AS DOUBLE) AS fragileCount,
2994
- CAST(SUM(CASE WHEN keywordCount >= 20 THEN 1 ELSE 0 END) AS DOUBLE) AS authorityCount
2995
- FROM per_page
2996
- )
2997
- SELECT
2998
- (SELECT to_json(list({ 'bucket': bucket, 'pageCount': pageCount, 'sortKey': sort_key })) FROM bucketed) AS distribution_json,
2999
- (SELECT to_json(list({ 'url': url, 'keywordCount': keywordCount, 'clicks': clicks, 'impressions': impressions })) FROM fragile) AS fragile_json,
3000
- (SELECT to_json(list({ 'url': url, 'keywordCount': keywordCount, 'clicks': clicks, 'impressions': impressions })) FROM authority) AS authority_json,
3001
- (SELECT to_json({
3002
- 'totalPages': totalPages,
3003
- 'avgKeywordsPerPage': avgKeywordsPerPage,
3004
- 'fragileCount': fragileCount,
3005
- 'authorityCount': authorityCount
3006
- }) FROM stats) AS stats_json
3007
- `,
3008
- params: [startDate, endDate],
3009
- current: {
3010
- table: "page_queries",
3011
- partitions: enumeratePartitions(startDate, endDate)
3012
- }
3013
- };
3014
- },
3015
- reduceSql(rows, params) {
3016
- const arr = Array.isArray(rows) ? rows : [];
3017
- const { startDate, endDate } = periodOf(params);
3018
- const row = arr[0] ?? {};
3019
- const distribution = parseJsonRows(row.distribution_json).sort((a, b) => num(a.sortKey) - num(b.sortKey)).map((r) => ({
3020
- bucket: rowString(r.bucket),
3021
- pageCount: num(r.pageCount)
3022
- }));
3023
- const fragile = parseJsonRows(row.fragile_json).map((r) => ({
3024
- url: rowString(r.url),
3025
- keywordCount: num(r.keywordCount),
3026
- clicks: num(r.clicks),
3027
- impressions: num(r.impressions)
3028
- }));
3029
- const authority = parseJsonRows(row.authority_json).map((r) => ({
3030
- url: rowString(r.url),
3031
- keywordCount: num(r.keywordCount),
3032
- clicks: num(r.clicks),
3033
- impressions: num(r.impressions)
3034
- }));
3035
- const stats = typeof row.stats_json === "string" ? JSON.parse(row.stats_json) : row.stats_json ?? {};
3036
- return {
3037
- results: distribution,
3038
- meta: {
3039
- fragilePages: fragile,
3040
- authorityPages: authority,
3041
- summary: {
3042
- totalPages: num(stats.totalPages),
3043
- avgKeywordsPerPage: num(stats.avgKeywordsPerPage),
3044
- fragileCount: num(stats.fragileCount),
3045
- authorityCount: num(stats.authorityCount)
3046
- },
3047
- startDate,
3048
- endDate
3049
- }
3050
- };
3051
- }
3052
- });
3053
- function downsampleLogRank(points) {
3054
- const toPoint = (p) => ({
3055
- rank: num(p.rank),
3056
- impressions: num(p.impressions),
3057
- clicks: num(p.clicks),
3058
- query: rowString(p.query)
3059
- });
3060
- if (points.length <= 80) return points.map(toPoint);
3061
- const sampled = points.slice(0, 10).map(toPoint);
3062
- let nextThreshold = 1.15;
3063
- for (let i = 10; i < points.length; i++) {
3064
- const point = points[i];
3065
- if (num(point.rank) >= nextThreshold) {
3066
- sampled.push(toPoint(point));
3067
- nextThreshold *= 1.15;
3068
- }
3069
- }
3070
- return sampled;
3071
- }
3072
- const longTailAnalyzer = defineAnalyzer({
3073
- id: "long-tail",
3074
- buildSql(params) {
3075
- const { startDate, endDate } = periodOf(params);
3076
- const minQueries = 10;
3077
- const minQueryImpressions = params.minImpressions ?? 5;
3078
- const limit = params.limit ?? 100;
3079
- return {
3080
- sql: `
3081
- WITH pq AS (
3082
- SELECT
3083
- url AS page,
3084
- query,
3085
- ${METRIC_EXPR.impressions} AS impressions,
3086
- ${METRIC_EXPR.clicks} AS clicks
3087
- FROM read_parquet({{FILES}}, union_by_name = true)
3088
- WHERE date >= ? AND date <= ?
3089
- AND query IS NOT NULL AND query <> ''
3090
- AND url IS NOT NULL AND url <> ''
3091
- GROUP BY url, query
3092
- HAVING SUM(impressions) >= ?
3093
- ),
3094
- ranked AS (
3095
- SELECT
3096
- page, query, impressions, clicks,
3097
- ROW_NUMBER() OVER (PARTITION BY page ORDER BY impressions DESC, query ASC) AS rnk
3098
- FROM pq
3099
- ),
3100
- log_space AS (
3101
- SELECT *,
3102
- LN(rnk) AS log_rank,
3103
- LN(impressions) AS log_impr
3104
- FROM ranked
3105
- ),
3106
- fit AS (
3107
- SELECT
3108
- page,
3109
- COUNT(*) AS query_count,
3110
- SUM(impressions) AS total_impressions,
3111
- SUM(clicks) AS total_clicks,
3112
- REGR_SLOPE(log_impr, log_rank) AS slope,
3113
- REGR_INTERCEPT(log_impr, log_rank) AS intercept,
3114
- REGR_R2(log_impr, log_rank) AS r2,
3115
- MAX(impressions) AS head_impressions,
3116
- MAX(CASE WHEN rnk = 1 THEN impressions END) / NULLIF(SUM(impressions), 0) AS head_share
3117
- FROM log_space
3118
- GROUP BY page
3119
- HAVING COUNT(*) >= ${Number(minQueries)}
3120
- ),
3121
- scatter AS (
3122
- SELECT
3123
- l.page,
3124
- to_json(list({
3125
- 'rank': l.rnk,
3126
- 'impressions': l.impressions,
3127
- 'clicks': l.clicks,
3128
- 'query': l.query
3129
- } ORDER BY l.rnk)) AS pointsJson
3130
- FROM log_space l
3131
- JOIN fit f USING (page)
3132
- GROUP BY l.page
3133
- )
3134
- SELECT
3135
- f.page,
3136
- f.query_count AS queryCount,
3137
- f.total_impressions AS totalImpressions,
3138
- f.total_clicks AS totalClicks,
3139
- f.slope AS slope,
3140
- f.intercept AS intercept,
3141
- f.r2 AS r2,
3142
- f.head_impressions AS headImpressions,
3143
- f.head_share AS headShare,
3144
- s.pointsJson AS pointsJson,
3145
- CASE
3146
- WHEN f.slope > -0.6 THEN 'flat-tail'
3147
- WHEN f.slope > -1.2 THEN 'balanced'
3148
- ELSE 'head-heavy'
3149
- END AS fingerprint
3150
- FROM fit f
3151
- LEFT JOIN scatter s USING (page)
3152
- ORDER BY f.total_impressions DESC
3153
- LIMIT ${Number(limit)}
3154
- `,
3155
- params: [
3156
- startDate,
3157
- endDate,
3158
- minQueryImpressions
3159
- ],
3160
- current: {
3161
- table: "page_queries",
3162
- partitions: enumeratePartitions(startDate, endDate)
3163
- }
3164
- };
3165
- },
3166
- reduceSql(rows) {
3167
- const results = (Array.isArray(rows) ? rows : []).map((r) => ({
3168
- page: rowString(r.page),
3169
- queryCount: num(r.queryCount),
3170
- totalImpressions: num(r.totalImpressions),
3171
- totalClicks: num(r.totalClicks),
3172
- slope: num(r.slope),
3173
- intercept: num(r.intercept),
3174
- r2: num(r.r2),
3175
- headImpressions: num(r.headImpressions),
3176
- headShare: num(r.headShare),
3177
- fingerprint: rowString(r.fingerprint),
3178
- points: downsampleLogRank(parseJsonRows(r.pointsJson))
3179
- }));
3180
- const counts = {
3181
- "flat-tail": 0,
3182
- "balanced": 0,
3183
- "head-heavy": 0
3184
- };
3185
- for (const r of results) counts[r.fingerprint]++;
3186
- return {
3187
- results,
3188
- meta: {
3189
- total: results.length,
3190
- fingerprints: counts,
3191
- avgSlope: results.length > 0 ? results.reduce((s, r) => s + r.slope, 0) / results.length : 0
3192
- }
3193
- };
3194
- }
3195
- });
3196
- function percentDifference(current, previous) {
3197
- if (previous === 0) return current > 0 ? 100 : 0;
3198
- return (current - previous) / previous * 100;
3199
- }
3200
- function withCanonicalFields(r) {
3201
- return {
3202
- ...r,
3203
- clicks: r.recentClicks,
3204
- impressions: r.recentImpressions,
3205
- ctr: r.recentImpressions > 0 ? r.recentClicks / r.recentImpressions : 0,
3206
- position: r.recentPosition,
3207
- prevClicks: r.baselineClicks,
3208
- prevImpressions: r.baselineImpressions,
3209
- prevPosition: r.baselinePosition
3210
- };
3211
- }
3212
- function selectMoversDirection(rising, declining, stable, params) {
3213
- if (params.direction) {
3214
- const selected = params.direction === "rising" ? rising : declining;
3215
- const total = selected.length;
3216
- const offset = params.offset ?? 0;
3217
- const limit = params.limit ?? total;
3218
- return {
3219
- results: selected.slice(offset, offset + limit),
3220
- meta: {
3221
- total,
3222
- rising: rising.length,
3223
- declining: declining.length,
3224
- stable: stable.length
3225
- }
3226
- };
3227
- }
3228
- const combined = [...rising, ...declining];
3229
- return {
3230
- results: combined,
3231
- meta: {
3232
- total: combined.length,
3233
- rising: rising.length,
3234
- declining: declining.length,
3235
- stable: stable.length
3236
- }
3237
- };
3238
- }
3239
- function analyzeMovers(input, options = {}) {
3240
- const { changeThreshold = .2, minImpressions = 50, sortBy = "clicksChange" } = options;
3241
- const normFactor = input.normalizationFactor ?? 1;
3242
- const baselineMap = buildPeriodMap(input.previous, (r) => r.query, (r) => ({
3243
- clicks: num(r.clicks) / normFactor,
3244
- impressions: num(r.impressions) / normFactor,
3245
- position: num(r.position),
3246
- page: r.page ?? null
3247
- }));
3248
- const pageMap = /* @__PURE__ */ new Map();
3249
- for (const row of input.current) if (!pageMap.has(row.query) && row.page) pageMap.set(row.query, row.page);
3250
- for (const row of input.previous) if (!pageMap.has(row.query) && row.page) pageMap.set(row.query, row.page);
3251
- const rising = [];
3252
- const declining = [];
3253
- const stable = [];
3254
- for (const row of input.current) {
3255
- const impressions = num(row.impressions);
3256
- const clicks = num(row.clicks);
3257
- const position = num(row.position);
3258
- if (impressions < minImpressions) continue;
3259
- const baseline = baselineMap.get(row.query);
3260
- const baselineClicksRaw = baseline?.clicks ?? 0;
3261
- const baselineImpressionsRaw = baseline?.impressions ?? 0;
3262
- const baselinePosition = baseline?.position ?? null;
3263
- const clicksChangePercent = percentDifference(clicks, baselineClicksRaw);
3264
- const impressionsChangePercent = percentDifference(impressions, baselineImpressionsRaw);
3265
- const data = {
3266
- keyword: row.query,
3267
- page: pageMap.get(row.query) ?? null,
3268
- recentClicks: clicks,
3269
- recentImpressions: impressions,
3270
- recentPosition: position,
3271
- baselineClicks: Math.round(baselineClicksRaw),
3272
- baselineImpressions: Math.round(baselineImpressionsRaw),
3273
- baselinePosition,
3274
- clicksChange: clicks - Math.round(baselineClicksRaw),
3275
- clicksChangePercent,
3276
- impressionsChangePercent,
3277
- positionChange: baselinePosition != null ? position - baselinePosition : null
3278
- };
3279
- const absChange = Math.abs(clicksChangePercent / 100);
3280
- if (clicksChangePercent > 0 && absChange >= changeThreshold) rising.push(data);
3281
- else if (clicksChangePercent < 0 && absChange >= changeThreshold) declining.push(data);
3282
- else stable.push(data);
3283
- }
3284
- const sortFn = (a, b) => {
3285
- switch (sortBy) {
3286
- case "clicks": return b.recentClicks - a.recentClicks;
3287
- case "impressions": return b.recentImpressions - a.recentImpressions;
3288
- case "clicksChange": return Math.abs(b.clicksChangePercent) - Math.abs(a.clicksChangePercent);
3289
- case "impressionsChange": return Math.abs(b.impressionsChangePercent) - Math.abs(a.impressionsChangePercent);
3290
- case "positionChange": return Math.abs(b.positionChange ?? 0) - Math.abs(a.positionChange ?? 0);
3291
- default: return Math.abs(b.clicksChangePercent) - Math.abs(a.clicksChangePercent);
3292
- }
3293
- };
3294
- rising.sort(sortFn);
3295
- declining.sort(sortFn);
3296
- stable.sort((a, b) => b.recentClicks - a.recentClicks);
3297
- return {
3298
- rising,
3299
- declining,
3300
- stable
3301
- };
3302
- }
3303
- const moversAnalyzer = defineAnalyzer({
3304
- id: "movers",
3305
- buildSql(params) {
3306
- const { current: cur, previous: prev } = comparisonOf(params);
3307
- const minImpressions = params.minImpressions ?? 50;
3308
- const changeThreshold = params.changeThreshold ?? .2;
3309
- const limit = params.direction ? 5e3 : params.limit ?? 2e3;
3310
- return {
3311
- sql: `
3312
- WITH cur AS (
3313
- SELECT
3314
- query, url,
3315
- ${METRIC_EXPR.clicks} AS clicks,
3316
- ${METRIC_EXPR.impressions} AS impressions,
3317
- ${METRIC_EXPR.position} AS position
3318
- FROM read_parquet({{FILES}}, union_by_name = true)
3319
- WHERE date >= ? AND date <= ?
3320
- GROUP BY query, url
3321
- ),
3322
- prev AS (
3323
- SELECT
3324
- query, url,
3325
- ${METRIC_EXPR.clicks} AS clicks,
3326
- ${METRIC_EXPR.impressions} AS impressions,
3327
- ${METRIC_EXPR.position} AS position
3328
- FROM read_parquet({{FILES_PREV}}, union_by_name = true)
3329
- WHERE date >= ? AND date <= ?
3330
- GROUP BY query, url
3331
- ),
3332
- weekly AS (
3333
- SELECT query, url, date_trunc('week', CAST(date AS DATE)) AS week,
3334
- ${METRIC_EXPR.clicks} AS clicks,
3335
- ${METRIC_EXPR.impressions} AS impressions
3336
- FROM (
3337
- SELECT query, url, date, clicks, impressions
3338
- FROM read_parquet({{FILES}}, union_by_name = true)
3339
- WHERE date >= ? AND date <= ?
3340
- UNION ALL
3341
- SELECT query, url, date, clicks, impressions
3342
- FROM read_parquet({{FILES_PREV}}, union_by_name = true)
3343
- WHERE date >= ? AND date <= ?
3344
- )
3345
- GROUP BY query, url, week
3346
- ),
3347
- series_by_entity AS (
3348
- SELECT query, url, to_json(list({
3349
- 'week': strftime(week, '%Y-%m-%d'),
3350
- 'clicks': clicks,
3351
- 'impressions': impressions
3352
- } ORDER BY week)) AS seriesJson
3353
- FROM weekly GROUP BY query, url
3354
- ),
3355
- joined AS (
3356
- SELECT
3357
- c.query AS keyword,
3358
- c.url AS page,
3359
- c.clicks AS recentClicks,
3360
- c.impressions AS recentImpressions,
3361
- c.position AS recentPosition,
3362
- COALESCE(p.clicks, 0.0) AS baselineClicks,
3363
- COALESCE(p.impressions, 0.0) AS baselineImpressions,
3364
- p.position AS baselinePosition,
3365
- (c.clicks - COALESCE(p.clicks, 0.0)) AS clicksChange,
3366
- CASE
3367
- WHEN COALESCE(p.clicks, 0.0) = 0 THEN CASE WHEN c.clicks > 0 THEN 100.0 ELSE 0.0 END
3368
- ELSE (c.clicks - p.clicks) * 100.0 / p.clicks
3369
- END AS clicksChangePercent,
3370
- CASE
3371
- WHEN COALESCE(p.impressions, 0.0) = 0 THEN CASE WHEN c.impressions > 0 THEN 100.0 ELSE 0.0 END
3372
- ELSE (c.impressions - p.impressions) * 100.0 / p.impressions
3373
- END AS impressionsChangePercent,
3374
- CASE WHEN p.position IS NOT NULL THEN (c.position - p.position) ELSE NULL END AS positionChange,
3375
- s.seriesJson
3376
- FROM cur c
3377
- LEFT JOIN prev p ON c.query = p.query AND c.url = p.url
3378
- LEFT JOIN series_by_entity s ON c.query = s.query AND c.url = s.url
3379
- WHERE c.impressions >= ?
3380
- )
3381
- SELECT *,
3382
- CASE
3383
- WHEN clicksChangePercent > 0 AND ABS(clicksChangePercent) / 100.0 >= ? THEN 'rising'
3384
- WHEN clicksChangePercent < 0 AND ABS(clicksChangePercent) / 100.0 >= ? THEN 'declining'
3385
- ELSE 'stable'
3386
- END AS direction
3387
- FROM joined
3388
- ORDER BY ABS(clicksChangePercent) DESC
3389
- LIMIT ${Number(limit)}
3390
- `,
3391
- params: [
3392
- cur.startDate,
3393
- cur.endDate,
3394
- prev.startDate,
3395
- prev.endDate,
3396
- cur.startDate,
3397
- cur.endDate,
3398
- prev.startDate,
3399
- prev.endDate,
3400
- minImpressions,
3401
- changeThreshold,
3402
- changeThreshold
3403
- ],
3404
- current: {
3405
- table: "page_queries",
3406
- partitions: enumeratePartitions(cur.startDate, cur.endDate)
3407
- },
3408
- previous: {
3409
- table: "page_queries",
3410
- partitions: enumeratePartitions(prev.startDate, prev.endDate)
3411
- }
3412
- };
3413
- },
3414
- reduceSql(rows, params) {
3415
- const normalized = (Array.isArray(rows) ? rows : []).map((r) => {
3416
- const recentClicks = num(r.recentClicks);
3417
- const recentImpressions = num(r.recentImpressions);
3418
- const recentPosition = num(r.recentPosition);
3419
- const baselineClicks = Math.round(num(r.baselineClicks));
3420
- const baselineImpressions = Math.round(num(r.baselineImpressions));
3421
- const baselinePosition = r.baselinePosition == null ? null : num(r.baselinePosition);
3422
- return {
3423
- keyword: rowString(r.keyword),
3424
- page: r.page == null ? null : rowString(r.page),
3425
- recentClicks,
3426
- recentImpressions,
3427
- recentPosition,
3428
- baselineClicks,
3429
- baselineImpressions,
3430
- baselinePosition,
3431
- clicksChange: num(r.clicksChange),
3432
- clicksChangePercent: num(r.clicksChangePercent),
3433
- impressionsChangePercent: num(r.impressionsChangePercent),
3434
- positionChange: r.positionChange == null ? null : num(r.positionChange),
3435
- direction: rowString(r.direction),
3436
- series: parseJsonRows(r.seriesJson).map((s) => ({
3437
- week: rowString(s.week),
3438
- clicks: num(s.clicks),
3439
- impressions: num(s.impressions)
3440
- }))
3441
- };
3442
- }).map(withCanonicalFields);
3443
- return selectMoversDirection(normalized.filter((r) => r.direction === "rising"), normalized.filter((r) => r.direction === "declining"), normalized.filter((r) => r.direction === "stable"), params);
3444
- },
3445
- buildRows(params) {
3446
- const { current, previous } = comparisonOf(params);
3447
- return {
3448
- current: queriesQueryState(current, params.limit),
3449
- previous: queriesQueryState(previous, params.limit)
3450
- };
3451
- },
3452
- reduceRows(rows, params) {
3453
- const map = rows && !Array.isArray(rows) ? rows : {
3454
- current: [],
3455
- previous: []
3456
- };
3457
- const result = analyzeMovers({
3458
- current: map.current ?? [],
3459
- previous: map.previous ?? []
3460
- }, {
3461
- changeThreshold: params.changeThreshold,
3462
- minImpressions: params.minImpressions
3463
- });
3464
- return selectMoversDirection(result.rising.map((r) => withCanonicalFields({
3465
- ...r,
3466
- direction: "rising"
3467
- })), result.declining.map((r) => withCanonicalFields({
3468
- ...r,
3469
- direction: "declining"
3470
- })), [], params);
3471
- }
3472
- });
3473
- const EXPECTED_CTR_BY_POSITION = {
3474
- 1: .3,
3475
- 2: .15,
3476
- 3: .1,
3477
- 4: .07,
3478
- 5: .05,
3479
- 6: .04,
3480
- 7: .03,
3481
- 8: .025,
3482
- 9: .02,
3483
- 10: .015
3484
- };
3485
- function getExpectedCtr(position) {
3486
- return EXPECTED_CTR_BY_POSITION[Math.round(Math.max(1, Math.min(position, 10)))] || .01;
3487
- }
3488
- function calculatePositionScore(position) {
3489
- if (position <= 3) return .2;
3490
- if (position > 50) return .1;
3491
- const distance = Math.abs(position - 11);
3492
- return Math.max(0, 1 - distance / 15);
3493
- }
3494
- function calculateImpressionScore(impressions) {
3495
- if (impressions <= 0) return 0;
3496
- return Math.min(Math.log10(impressions) / 5, 1);
3497
- }
3498
- function calculateCtrGapScore(actualCtr, position) {
3499
- const expectedCtr = getExpectedCtr(position);
3500
- if (actualCtr >= expectedCtr) return 0;
3501
- const gap = expectedCtr - actualCtr;
3502
- return Math.min(gap / expectedCtr, 1);
3503
- }
3504
- const SORT_DIR = {
3505
- opportunityScore: "desc",
3506
- potentialClicks: "desc",
3507
- impressions: "desc",
3508
- position: "asc"
3509
- };
3510
- function analyzeOpportunity(keywords, options = {}) {
3511
- const minImpressions = options.minImpressions ?? 100;
3512
- const positionWeight = options.weights?.position ?? 1;
3513
- const impressionsWeight = options.weights?.impressions ?? 1;
3514
- const ctrGapWeight = options.weights?.ctrGap ?? 1;
3515
- const totalWeight = positionWeight + impressionsWeight + ctrGapWeight;
3516
- const sortBy = options.sortBy ?? "opportunityScore";
3517
- const results = [];
3518
- for (const row of keywords) {
3519
- const impressions = num(row.impressions);
3520
- if (impressions < minImpressions) continue;
3521
- const position = num(row.position);
3522
- const ctr = num(row.ctr);
3523
- const clicks = num(row.clicks);
3524
- const positionScore = calculatePositionScore(position);
3525
- const impressionScore = calculateImpressionScore(impressions);
3526
- const ctrGapScore = calculateCtrGapScore(ctr, position);
3527
- const weightedProduct = positionScore ** positionWeight * impressionScore ** impressionsWeight * ctrGapScore ** ctrGapWeight;
3528
- const opportunityScore = Math.round(weightedProduct ** (1 / totalWeight) * 100);
3529
- const potentialClicks = Math.round(impressions * getExpectedCtr(Math.min(3, position)));
3530
- results.push({
3531
- keyword: row.query,
3532
- page: row.page ?? null,
3533
- clicks,
3534
- impressions,
3535
- ctr,
3536
- position,
3537
- opportunityScore,
3538
- potentialClicks,
3539
- factors: {
3540
- positionScore,
3541
- impressionScore,
3542
- ctrGapScore
3543
- }
3544
- });
3545
- }
3546
- const direction = SORT_DIR[sortBy];
3547
- results.sort((left, right) => direction === "asc" ? left[sortBy] - right[sortBy] : right[sortBy] - left[sortBy]);
3548
- return options.limit ? results.slice(0, options.limit) : results;
3549
- }
3550
- const opportunityAnalyzer = defineAnalyzer({
3551
- id: "opportunity",
3552
- buildSql(params) {
3553
- const { startDate, endDate } = periodOf(params);
3554
- const minImpressions = params.minImpressions ?? 100;
3555
- const w1 = 1;
3556
- const w2 = 1;
3557
- const w3 = 1;
3558
- const totalW = 3;
3559
- const limit = params.limit ?? 1e3;
3560
- return {
3561
- sql: `
3562
- WITH agg AS (
3563
- SELECT
3564
- query AS keyword,
3565
- url AS page,
3566
- ${METRIC_EXPR.clicks} AS clicks,
3567
- ${METRIC_EXPR.impressions} AS impressions,
3568
- ${METRIC_EXPR.ctr} AS ctr,
3569
- ${METRIC_EXPR.position} AS position
3570
- FROM read_parquet({{FILES}}, union_by_name = true)
3571
- WHERE date >= ? AND date <= ?
3572
- GROUP BY query, url
3573
- HAVING SUM(impressions) >= ?
3574
- ),
3575
- scored AS (
3576
- SELECT
3577
- keyword, page, clicks, impressions, ctr, position,
3578
- CASE
3579
- WHEN position <= 3 THEN 0.2
3580
- WHEN position > 50 THEN 0.1
3581
- ELSE GREATEST(0.0, 1.0 - ABS(position - 11.0) / 15.0)
3582
- END AS positionScore,
3583
- CASE WHEN impressions <= 0 THEN 0.0 ELSE LEAST(LOG10(impressions) / 5.0, 1.0) END AS impressionScore,
3584
- CASE CAST(ROUND(GREATEST(LEAST(position, 10.0), 1.0)) AS INTEGER)
3585
- WHEN 1 THEN 0.30
3586
- WHEN 2 THEN 0.15
3587
- WHEN 3 THEN 0.10
3588
- WHEN 4 THEN 0.07
3589
- WHEN 5 THEN 0.05
3590
- WHEN 6 THEN 0.04
3591
- WHEN 7 THEN 0.03
3592
- WHEN 8 THEN 0.025
3593
- WHEN 9 THEN 0.02
3594
- WHEN 10 THEN 0.015
3595
- ELSE 0.01
3596
- END AS expectedCtr
3597
- FROM agg
3598
- ),
3599
- gapped AS (
3600
- SELECT
3601
- *,
3602
- CASE WHEN ctr >= expectedCtr THEN 0.0 ELSE LEAST((expectedCtr - ctr) / expectedCtr, 1.0) END AS ctrGapScore
3603
- FROM scored
3604
- )
3605
- SELECT
3606
- keyword, page, clicks, impressions, ctr, position,
3607
- CAST(ROUND(POWER(
3608
- POWER(positionScore, ${w1}) * POWER(impressionScore, ${w2}) * POWER(ctrGapScore, ${w3}),
3609
- 1.0 / ${totalW}
3610
- ) * 100) AS DOUBLE) AS opportunityScore,
3611
- CAST(ROUND(impressions * (
3612
- CASE CAST(ROUND(GREATEST(LEAST(position, 3.0), 1.0)) AS INTEGER)
3613
- WHEN 1 THEN 0.30
3614
- WHEN 2 THEN 0.15
3615
- WHEN 3 THEN 0.10
3616
- ELSE 0.10
3617
- END
3618
- )) AS DOUBLE) AS potentialClicks,
3619
- positionScore, impressionScore, ctrGapScore
3620
- FROM gapped
3621
- ORDER BY opportunityScore DESC
3622
- ${paginateClause({
3623
- limit,
3624
- offset: params.offset
3625
- })}
3626
- `,
3627
- params: [
3628
- startDate,
3629
- endDate,
3630
- minImpressions
3631
- ],
3632
- current: {
3633
- table: "page_queries",
3634
- partitions: enumeratePartitions(startDate, endDate)
3635
- }
3636
- };
3637
- },
3638
- reduceSql(rows) {
3639
- const arr = Array.isArray(rows) ? rows : [];
3640
- return {
3641
- results: arr.map((r) => ({
3642
- keyword: r.keyword == null ? "" : String(r.keyword),
3643
- page: r.page == null ? null : String(r.page),
3644
- clicks: num(r.clicks),
3645
- impressions: num(r.impressions),
3646
- ctr: num(r.ctr),
3647
- position: num(r.position),
3648
- opportunityScore: num(r.opportunityScore),
3649
- potentialClicks: num(r.potentialClicks),
3650
- factors: {
3651
- positionScore: num(r.positionScore),
3652
- impressionScore: num(r.impressionScore),
3653
- ctrGapScore: num(r.ctrGapScore)
3654
- }
3655
- })),
3656
- meta: { total: arr.length }
3657
- };
3658
- },
3659
- buildRows(params) {
3660
- return { queries: queriesQueryState(periodOf(params), params.limit) };
3661
- },
3662
- reduceRows(rows, params) {
3663
- const results = analyzeOpportunity((Array.isArray(rows) ? rows : []) ?? [], { minImpressions: params.minImpressions });
3664
- const paged = paginateSortedInMemory(results, {
3665
- limit: params.limit,
3666
- offset: params.offset
3667
- }, (left, right) => right.opportunityScore - left.opportunityScore);
3668
- return {
3669
- results: paged,
3670
- meta: {
3671
- total: results.length,
3672
- returned: paged.length
3673
- }
3674
- };
3675
- }
3676
- });
3677
- const positionDistributionAnalyzer = defineAnalyzer({
3678
- id: "position-distribution",
3679
- buildSql(params) {
3680
- const { startDate, endDate } = periodOf(params);
3681
- return {
3682
- sql: `
3683
- WITH pos AS (
3684
- SELECT
3685
- date,
3686
- (sum_position / NULLIF(impressions, 0) + 1) AS avg_pos
3687
- FROM read_parquet({{FILES}}, union_by_name = true)
3688
- WHERE date >= ? AND date <= ? AND impressions > 0
3689
- )
3690
- SELECT
3691
- date,
3692
- CAST(SUM(CASE WHEN avg_pos <= 3 THEN 1 ELSE 0 END) AS DOUBLE) AS pos_1_3,
3693
- CAST(SUM(CASE WHEN avg_pos > 3 AND avg_pos <= 10 THEN 1 ELSE 0 END) AS DOUBLE) AS pos_4_10,
3694
- CAST(SUM(CASE WHEN avg_pos > 10 AND avg_pos <= 20 THEN 1 ELSE 0 END) AS DOUBLE) AS pos_11_20,
3695
- CAST(SUM(CASE WHEN avg_pos > 20 THEN 1 ELSE 0 END) AS DOUBLE) AS pos_20_plus,
3696
- CAST(COUNT(*) AS DOUBLE) AS total
3697
- FROM pos
3698
- GROUP BY date
3699
- ORDER BY date ASC
3700
- `,
3701
- params: [startDate, endDate],
3702
- current: {
3703
- table: "queries",
3704
- partitions: enumeratePartitions(startDate, endDate)
3705
- }
3706
- };
3707
- },
3708
- reduceSql(rows, params) {
3709
- const arr = Array.isArray(rows) ? rows : [];
3710
- const { startDate, endDate } = periodOf(params);
3711
- return {
3712
- results: arr.map((r) => ({
3713
- date: rowString(r.date),
3714
- pos_1_3: num(r.pos_1_3),
3715
- pos_4_10: num(r.pos_4_10),
3716
- pos_11_20: num(r.pos_11_20),
3717
- pos_20_plus: num(r.pos_20_plus),
3718
- total: num(r.total)
3719
- })),
3720
- meta: {
3721
- total: arr.length,
3722
- startDate,
3723
- endDate
3724
- }
3725
- };
3726
- }
3727
- });
3728
- const positionVolatilityAnalyzer = defineAnalyzer({
3729
- id: "position-volatility",
3730
- buildSql(params) {
3731
- const { startDate, endDate } = periodOf(params);
3732
- const topN = params.topN ?? 30;
3733
- const minDayImpressions = params.minImpressions ?? 10;
3734
- const minDays = params.minWeeksWithData ?? 7;
3735
- return {
3736
- sql: `
3737
- WITH query_day AS (
3738
- SELECT
3739
- url AS page,
3740
- query,
3741
- -- Normalize at the source CTE: union_by_name=true can coerce date to
3742
- -- VARCHAR across parquets with mixed schemas, which makes downstream
3743
- -- strftime(date, ...) binder-error.
3744
- CAST(date AS DATE) AS date,
3745
- ${METRIC_EXPR.impressions} AS q_impressions,
3746
- ${METRIC_EXPR.position} AS q_position
3747
- FROM read_parquet({{FILES}}, union_by_name = true)
3748
- WHERE date >= ? AND date <= ?
3749
- AND query IS NOT NULL AND query <> ''
3750
- AND url IS NOT NULL AND url <> ''
3751
- GROUP BY url, query, date
3752
- HAVING SUM(impressions) >= 1
3753
- ),
3754
- daily AS (
3755
- SELECT
3756
- page, date,
3757
- COUNT(*) AS query_count,
3758
- SUM(q_impressions) AS day_impressions,
3759
- SUM(q_position * q_impressions) / NULLIF(SUM(q_impressions), 0) AS avg_position,
3760
- COALESCE(STDDEV_POP(q_position), 0.0) AS pos_stddev,
3761
- MIN(q_position) AS best_position,
3762
- MAX(q_position) AS worst_position
3763
- FROM query_day
3764
- GROUP BY page, date
3765
- HAVING SUM(q_impressions) >= ?
3766
- ),
3767
- with_shift AS (
3768
- SELECT *,
3769
- LAG(avg_position) OVER (PARTITION BY page ORDER BY date) AS prev_position,
3770
- COALESCE(
3771
- ABS(avg_position - LAG(avg_position) OVER (PARTITION BY page ORDER BY date)),
3772
- 0.0
3773
- ) AS dod_shift
3774
- FROM daily
3775
- ),
3776
- scored AS (
3777
- SELECT *,
3778
- pos_stddev + dod_shift AS volatility
3779
- FROM with_shift
3780
- ),
3781
- top_pages AS (
3782
- SELECT page,
3783
- SUM(day_impressions) AS total_impressions,
3784
- AVG(volatility) AS avg_volatility,
3785
- MAX(volatility) AS peak_volatility,
3786
- COUNT(*) AS days_with_data
3787
- FROM scored
3788
- GROUP BY page
3789
- HAVING COUNT(*) >= ?
3790
- ORDER BY avg_volatility DESC
3791
- LIMIT ${Number(topN)}
3792
- )
3793
- SELECT
3794
- s.page,
3795
- strftime(s.date, '%Y-%m-%d') AS date,
3796
- s.query_count AS queryCount,
3797
- s.day_impressions AS dayImpressions,
3798
- s.avg_position AS avgPosition,
3799
- s.pos_stddev AS posStddev,
3800
- s.best_position AS bestPosition,
3801
- s.worst_position AS worstPosition,
3802
- s.dod_shift AS dodShift,
3803
- s.volatility AS volatility,
3804
- t.avg_volatility AS pageAvgVolatility,
3805
- t.peak_volatility AS pagePeakVolatility,
3806
- t.total_impressions AS pageTotalImpressions
3807
- FROM scored s
3808
- JOIN top_pages t USING (page)
3809
- ORDER BY t.avg_volatility DESC, s.date ASC
3810
- `,
3811
- params: [
3812
- startDate,
3813
- endDate,
3814
- minDayImpressions,
3815
- minDays
3816
- ],
3817
- current: {
3818
- table: "page_queries",
3819
- partitions: enumeratePartitions(startDate, endDate)
3820
- }
3821
- };
3822
- },
3823
- reduceSql(rows) {
3824
- const arr = Array.isArray(rows) ? rows : [];
3825
- const byPage = /* @__PURE__ */ new Map();
3826
- const allDates = /* @__PURE__ */ new Set();
3827
- for (const r of arr) {
3828
- const page = rowString(r.page);
3829
- const date = rowString(r.date);
3830
- allDates.add(date);
3831
- const entry = byPage.get(page) ?? {
3832
- page,
3833
- avgVolatility: num(r.pageAvgVolatility),
3834
- peakVolatility: num(r.pagePeakVolatility),
3835
- totalImpressions: num(r.pageTotalImpressions),
3836
- days: []
3837
- };
3838
- entry.days.push({
3839
- date,
3840
- queryCount: num(r.queryCount),
3841
- dayImpressions: num(r.dayImpressions),
3842
- avgPosition: num(r.avgPosition),
3843
- posStddev: num(r.posStddev),
3844
- bestPosition: num(r.bestPosition),
3845
- worstPosition: num(r.worstPosition),
3846
- dodShift: num(r.dodShift),
3847
- volatility: num(r.volatility)
3848
- });
3849
- byPage.set(page, entry);
3850
- }
3851
- const pages = [...byPage.values()].sort((a, b) => b.avgVolatility - a.avgVolatility);
3852
- const dates = [...allDates].sort();
3853
- const maxVolatility = pages.reduce((m, p) => Math.max(m, p.peakVolatility), 0);
3854
- return {
3855
- results: pages,
3856
- meta: {
3857
- total: pages.length,
3858
- dates,
3859
- maxVolatility
3860
- }
3861
- };
3862
- }
3863
- });
3864
- const queryMigrationAnalyzer = defineAnalyzer({
3865
- id: "query-migration",
3866
- buildSql(params) {
3867
- const cur = periodOf(params);
3868
- let prevStart = params.prevStartDate;
3869
- let prevEnd = params.prevEndDate;
3870
- if (prevStart == null || prevEnd == null) {
3871
- const curStartMs = new Date(cur.startDate).getTime();
3872
- const span = new Date(cur.endDate).getTime() - curStartMs;
3873
- prevEnd = toIsoDate(new Date(curStartMs - MS_PER_DAY));
3874
- prevStart = toIsoDate(new Date(curStartMs - MS_PER_DAY - span));
3875
- }
3876
- const minImpressions = params.minImpressions ?? 20;
3877
- const limit = params.limit ?? 200;
3878
- const maxLevenshtein = 2;
3879
- return {
3880
- sql: `
3881
- WITH cur AS (
3882
- SELECT query, url AS page,
3883
- ${METRIC_EXPR.impressions} AS impressions,
3884
- ${METRIC_EXPR.clicks} AS clicks,
3885
- ${METRIC_EXPR.position} AS position
3886
- FROM read_parquet({{FILES}}, union_by_name = true)
3887
- WHERE date >= ? AND date <= ?
3888
- AND query IS NOT NULL AND query <> ''
3889
- AND url IS NOT NULL AND url <> ''
3890
- GROUP BY query, url
3891
- HAVING SUM(impressions) >= ?
3892
- ),
3893
- prev AS (
3894
- SELECT query, url AS page,
3895
- ${METRIC_EXPR.impressions} AS impressions,
3896
- ${METRIC_EXPR.clicks} AS clicks,
3897
- ${METRIC_EXPR.position} AS position
3898
- FROM read_parquet({{FILES_PREV}}, union_by_name = true)
3899
- WHERE date >= ? AND date <= ?
3900
- AND query IS NOT NULL AND query <> ''
3901
- AND url IS NOT NULL AND url <> ''
3902
- GROUP BY query, url
3903
- HAVING SUM(impressions) >= ?
3904
- ),
3905
- lost AS (
3906
- SELECT p.page AS source_page, p.query AS source_query, p.impressions AS source_impressions
3907
- FROM prev p
3908
- LEFT JOIN cur c ON p.page = c.page AND p.query = c.query
3909
- WHERE c.query IS NULL
3910
- ),
3911
- gained AS (
3912
- SELECT c.page AS target_page, c.query AS target_query, c.impressions AS target_impressions
3913
- FROM cur c
3914
- LEFT JOIN prev p ON p.page = c.page AND p.query = c.query
3915
- WHERE p.query IS NULL
3916
- ),
3917
- matched AS (
3918
- SELECT
3919
- l.source_page, l.source_query, l.source_impressions,
3920
- g.target_page, g.target_query, g.target_impressions,
3921
- CASE
3922
- WHEN l.source_query = g.target_query THEN 'exact'
3923
- ELSE 'fuzzy'
3924
- END AS match_type,
3925
- LEAST(l.source_impressions, g.target_impressions) AS absorbed_impressions
3926
- FROM lost l
3927
- JOIN gained g
3928
- ON l.source_page <> g.target_page
3929
- AND ABS(LENGTH(l.source_query) - LENGTH(g.target_query)) <= ${maxLevenshtein}
3930
- AND (
3931
- l.source_query = g.target_query
3932
- OR levenshtein(l.source_query, g.target_query) <= ${maxLevenshtein}
3933
- )
3934
- ),
3935
- edges AS (
3936
- SELECT
3937
- source_page, target_page,
3938
- SUM(absorbed_impressions) AS weight,
3939
- COUNT(*) AS query_count,
3940
- SUM(CASE WHEN match_type = 'exact' THEN 1 ELSE 0 END) AS exact_count,
3941
- to_json(list({
3942
- 'sourceQuery': source_query,
3943
- 'targetQuery': target_query,
3944
- 'absorbed': absorbed_impressions,
3945
- 'matchType': match_type
3946
- } ORDER BY absorbed_impressions DESC)) AS examplesJson
3947
- FROM matched
3948
- GROUP BY source_page, target_page
3949
- )
3950
- SELECT *
3951
- FROM edges
3952
- ORDER BY weight DESC
3953
- LIMIT ${Number(limit)}
3954
- `,
3955
- params: [
3956
- cur.startDate,
3957
- cur.endDate,
3958
- minImpressions,
3959
- prevStart,
3960
- prevEnd,
3961
- minImpressions
3962
- ],
3963
- current: {
3964
- table: "page_queries",
3965
- partitions: enumeratePartitions(cur.startDate, cur.endDate)
3966
- },
3967
- previous: {
3968
- table: "page_queries",
3969
- partitions: enumeratePartitions(prevStart, prevEnd)
3970
- }
3971
- };
3972
- },
3973
- reduceSql(rows, params) {
3974
- const arr = Array.isArray(rows) ? rows : [];
3975
- const cur = periodOf(params);
3976
- let prevStart = params.prevStartDate;
3977
- let prevEnd = params.prevEndDate;
3978
- if (prevStart == null || prevEnd == null) {
3979
- const curStartMs = new Date(cur.startDate).getTime();
3980
- const span = new Date(cur.endDate).getTime() - curStartMs;
3981
- prevEnd = toIsoDate(new Date(curStartMs - MS_PER_DAY));
3982
- prevStart = toIsoDate(new Date(curStartMs - MS_PER_DAY - span));
3983
- }
3984
- const edges = arr.map((r) => ({
3985
- sourcePage: rowString(r.source_page),
3986
- targetPage: rowString(r.target_page),
3987
- weight: num(r.weight),
3988
- queryCount: num(r.query_count),
3989
- exactCount: num(r.exact_count),
3990
- fuzzyCount: num(r.query_count) - num(r.exact_count),
3991
- examples: parseJsonRows(r.examplesJson).slice(0, 8).map((e) => ({
3992
- sourceQuery: rowString(e.sourceQuery),
3993
- targetQuery: rowString(e.targetQuery),
3994
- absorbed: num(e.absorbed),
3995
- matchType: rowString(e.matchType)
3996
- }))
3997
- }));
3998
- const nodeAgg = /* @__PURE__ */ new Map();
3999
- for (const e of edges) {
4000
- const src = nodeAgg.get(e.sourcePage) ?? {
4001
- url: e.sourcePage,
4002
- outgoing: 0,
4003
- incoming: 0
4004
- };
4005
- src.outgoing += e.weight;
4006
- nodeAgg.set(e.sourcePage, src);
4007
- const tgt = nodeAgg.get(e.targetPage) ?? {
4008
- url: e.targetPage,
4009
- outgoing: 0,
4010
- incoming: 0
4011
- };
4012
- tgt.incoming += e.weight;
4013
- nodeAgg.set(e.targetPage, tgt);
4014
- }
4015
- const nodes = [...nodeAgg.values()];
4016
- const totalAbsorbed = edges.reduce((s, e) => s + e.weight, 0);
4017
- return {
4018
- results: edges,
4019
- meta: {
4020
- total: edges.length,
4021
- totalAbsorbed,
4022
- period: {
4023
- current: cur,
4024
- previous: {
4025
- startDate: prevStart,
4026
- endDate: prevEnd
4027
- }
4028
- },
4029
- nodes
4030
- }
4031
- };
4032
- }
4033
- });
4034
- function calculateCV(values) {
4035
- if (values.length === 0) return 0;
4036
- const mean = values.reduce((a, b) => a + b, 0) / values.length;
4037
- if (mean === 0) return 0;
4038
- const variance = values.reduce((sum, v) => sum + (v - mean) ** 2, 0) / values.length;
4039
- return Math.min(Math.sqrt(variance) / mean, 1);
4040
- }
4041
- function analyzeSeasonality(dates, options = {}) {
4042
- const { metric = "clicks" } = options;
4043
- if (dates.length === 0) return {
4044
- hasSeasonality: false,
4045
- strength: 0,
4046
- peakMonths: [],
4047
- troughMonths: [],
4048
- monthlyBreakdown: [],
4049
- insufficientData: true
4050
- };
4051
- const monthlyMap = /* @__PURE__ */ new Map();
4052
- for (const row of dates) {
4053
- const month = row.date.substring(0, 7);
4054
- const value = metric === "clicks" ? row.clicks : row.impressions;
4055
- monthlyMap.set(month, (monthlyMap.get(month) || 0) + value);
4056
- }
4057
- const months = Array.from(monthlyMap.keys()).sort();
4058
- const values = months.map((m) => monthlyMap.get(m) || 0);
4059
- const insufficientData = months.length < 12;
4060
- const totalValue = values.reduce((a, b) => a + b, 0);
4061
- const avgValue = values.length > 0 ? totalValue / values.length : 0;
4062
- const monthlyBreakdown = months.map((month, i) => {
4063
- const value = values[i] ?? 0;
4064
- const vsAverage = avgValue > 0 ? value / avgValue : 0;
4065
- return {
4066
- month,
4067
- value,
4068
- vsAverage,
4069
- isPeak: vsAverage > 1.5,
4070
- isTrough: vsAverage < .5
4071
- };
4072
- });
4073
- const peakMonths = [...new Set(monthlyBreakdown.filter((m) => m.isPeak).map((m) => m.month.substring(5, 7)))];
4074
- const troughMonths = [...new Set(monthlyBreakdown.filter((m) => m.isTrough).map((m) => m.month.substring(5, 7)))];
4075
- const strength = calculateCV(values);
4076
- return {
4077
- hasSeasonality: peakMonths.length > 0 || troughMonths.length > 0 || strength > .3,
4078
- strength,
4079
- peakMonths,
4080
- troughMonths,
4081
- monthlyBreakdown,
4082
- insufficientData
4083
- };
4084
- }
4085
- const seasonalityAnalyzer = defineAnalyzer({
4086
- id: "seasonality",
4087
- buildSql(params) {
4088
- const { startDate, endDate } = periodOf(params);
4089
- return {
4090
- sql: `
4091
- WITH monthly AS (
4092
- SELECT
4093
- strftime(CAST(date AS DATE), '%Y-%m') AS month,
4094
- CAST(SUM(${params.metric === "impressions" ? "impressions" : "clicks"}) AS DOUBLE) AS value
4095
- FROM read_parquet({{FILES}}, union_by_name = true)
4096
- WHERE date >= ? AND date <= ?
4097
- GROUP BY month
4098
- ),
4099
- stats AS (
4100
- SELECT
4101
- AVG(value) AS avg_val,
4102
- COALESCE(STDDEV_POP(value), 0.0) AS std_val,
4103
- CAST(COUNT(*) AS DOUBLE) AS month_count
4104
- FROM monthly
4105
- )
4106
- SELECT
4107
- m.month AS month,
4108
- m.value AS value,
4109
- CASE WHEN s.avg_val > 0 THEN m.value / s.avg_val ELSE 0.0 END AS vsAverage,
4110
- (s.avg_val > 0 AND m.value / s.avg_val > 1.5) AS isPeak,
4111
- (s.avg_val > 0 AND m.value / s.avg_val < 0.5) AS isTrough,
4112
- CASE WHEN s.avg_val > 0 THEN LEAST(s.std_val / s.avg_val, 1.0) ELSE 0.0 END AS strength,
4113
- s.month_count AS monthCount
4114
- FROM monthly m, stats s
4115
- ORDER BY m.month
4116
- `,
4117
- params: [startDate, endDate],
4118
- current: {
4119
- table: "pages",
4120
- partitions: enumeratePartitions(startDate, endDate)
4121
- }
4122
- };
4123
- },
4124
- reduceSql(rows) {
4125
- const arr = Array.isArray(rows) ? rows : [];
4126
- const breakdown = arr.map((r) => ({
4127
- month: rowString(r.month),
4128
- value: num(r.value),
4129
- vsAverage: num(r.vsAverage),
4130
- isPeak: rowBoolean(r.isPeak),
4131
- isTrough: rowBoolean(r.isTrough)
4132
- }));
4133
- const first = arr[0];
4134
- const strength = first ? num(first.strength) : 0;
4135
- const monthCount = first ? num(first.monthCount) : 0;
4136
- const peakMonths = [...new Set(breakdown.filter((m) => m.isPeak).map((m) => m.month.substring(5, 7)))];
4137
- const troughMonths = [...new Set(breakdown.filter((m) => m.isTrough).map((m) => m.month.substring(5, 7)))];
4138
- const hasSeasonality = peakMonths.length > 0 || troughMonths.length > 0 || strength > .3;
4139
- const insufficientData = monthCount < 12;
4140
- return {
4141
- results: breakdown,
4142
- meta: {
4143
- total: breakdown.length,
4144
- hasSeasonality,
4145
- strength,
4146
- peakMonths,
4147
- troughMonths,
4148
- insufficientData
4149
- }
4150
- };
4151
- },
4152
- buildRows(params) {
4153
- return { dates: datesQueryState(periodOf(params), params.limit) };
4154
- },
4155
- reduceRows(rows, params) {
4156
- const result = analyzeSeasonality(Array.isArray(rows) ? rows : [], { metric: params.metric });
4157
- return {
4158
- results: result.monthlyBreakdown,
4159
- meta: { strength: result.strength }
4160
- };
4161
- }
4162
- });
4163
- const stlDecomposeAnalyzer = defineAnalyzer({
4164
- id: "stl-decompose",
4165
- buildSql(params) {
4166
- const endDate = params.endDate ?? defaultEndDate();
4167
- const startDate = params.startDate ?? daysAgoUtc(93);
4168
- const minImpressions = params.minImpressions ?? 100;
4169
- const minDays = 21;
4170
- const metric = params.metric === "clicks" ? "clicks" : "impressions";
4171
- const limit = params.limit ?? 100;
4172
- return {
4173
- sql: `
4174
- WITH daily AS (
4175
- SELECT
4176
- query,
4177
- url AS page,
4178
- -- Normalize at the source CTE: union_by_name=true can coerce date to
4179
- -- VARCHAR across parquets with mixed schemas, which makes downstream
4180
- -- strftime(date, ...) binder-error.
4181
- CAST(date AS DATE) AS date,
4182
- ${METRIC_EXPR.clicks} AS clicks,
4183
- ${METRIC_EXPR.impressions} AS impressions,
4184
- CAST(SUM(${metric}) AS DOUBLE) AS observed
4185
- FROM read_parquet({{FILES}}, union_by_name = true)
4186
- WHERE date >= ? AND date <= ?
4187
- AND query IS NOT NULL AND query <> ''
4188
- AND url IS NOT NULL AND url <> ''
4189
- GROUP BY query, url, date
4190
- ),
4191
- entity_stats AS (
4192
- SELECT query, page,
4193
- COUNT(*) AS days,
4194
- SUM(impressions) AS total_impressions
4195
- FROM daily
4196
- GROUP BY query, page
4197
- HAVING COUNT(*) >= ${Number(minDays)}
4198
- AND SUM(impressions) >= ?
4199
- ),
4200
- filtered AS (
4201
- SELECT d.*
4202
- FROM daily d
4203
- JOIN entity_stats e USING (query, page)
4204
- ),
4205
- trended AS (
4206
- SELECT *,
4207
- CASE
4208
- WHEN COUNT(*) OVER w = 7
4209
- THEN AVG(observed) OVER w
4210
- ELSE NULL
4211
- END AS trend
4212
- FROM filtered
4213
- WINDOW w AS (
4214
- PARTITION BY query, page
4215
- ORDER BY date
4216
- ROWS BETWEEN 3 PRECEDING AND 3 FOLLOWING
4217
- )
4218
- ),
4219
- detrended AS (
4220
- SELECT *,
4221
- observed - trend AS detrended,
4222
- dayofweek(date) AS dow
4223
- FROM trended
4224
- ),
4225
- seasonal_raw AS (
4226
- SELECT *,
4227
- AVG(detrended) OVER (PARTITION BY query, page, dow) AS seasonal_dow
4228
- FROM detrended
4229
- ),
4230
- seasonal_centered AS (
4231
- SELECT *,
4232
- seasonal_dow - AVG(seasonal_dow) OVER (PARTITION BY query, page) AS seasonal
4233
- FROM seasonal_raw
4234
- ),
4235
- residualed AS (
4236
- SELECT *,
4237
- CASE
4238
- WHEN trend IS NULL OR seasonal IS NULL THEN NULL
4239
- ELSE observed - trend - seasonal
4240
- END AS residual
4241
- FROM seasonal_centered
4242
- ),
4243
- scored AS (
4244
- SELECT *,
4245
- STDDEV_POP(residual) OVER (PARTITION BY query, page) AS resid_std,
4246
- CASE
4247
- WHEN residual IS NOT NULL
4248
- AND STDDEV_POP(residual) OVER (PARTITION BY query, page) > 0
4249
- AND ABS(residual) > 2.0 * STDDEV_POP(residual) OVER (PARTITION BY query, page)
4250
- THEN true ELSE false
4251
- END AS anomaly
4252
- FROM residualed
4253
- ),
4254
- per_entity AS (
4255
- SELECT query, page,
4256
- COUNT(*) AS days,
4257
- SUM(impressions) AS total_impressions,
4258
- VAR_POP(detrended) AS var_detrended,
4259
- VAR_POP(seasonal) AS var_seasonal,
4260
- VAR_POP(residual) AS var_residual,
4261
- COUNT(*) FILTER (WHERE anomaly) AS residual_anomalies,
4262
- REGR_SLOPE(observed, epoch(date) / 86400.0) AS trend_slope
4263
- FROM scored
4264
- GROUP BY query, page
4265
- ),
4266
- series AS (
4267
- SELECT query, page,
4268
- to_json(list({
4269
- 'date': strftime(date, '%Y-%m-%d'),
4270
- 'observed': observed,
4271
- 'trend': trend,
4272
- 'seasonal': seasonal,
4273
- 'residual': residual,
4274
- 'anomaly': anomaly
4275
- } ORDER BY date)) AS seriesJson
4276
- FROM scored
4277
- GROUP BY query, page
4278
- )
4279
- SELECT
4280
- e.query AS keyword,
4281
- e.page,
4282
- CAST(e.total_impressions AS DOUBLE) AS totalImpressions,
4283
- CAST(e.days AS DOUBLE) AS days,
4284
- CASE
4285
- WHEN e.var_detrended IS NULL OR e.var_detrended = 0 THEN 0.0
4286
- ELSE LEAST(e.var_seasonal / NULLIF(e.var_detrended, 0), 1.0)
4287
- END AS seasonalStrength,
4288
- CASE
4289
- WHEN e.var_detrended IS NULL OR e.var_detrended = 0 THEN 0.0
4290
- ELSE GREATEST(0.0, 1.0 - e.var_residual / NULLIF(e.var_detrended, 0))
4291
- END AS trendStrength,
4292
- CAST(e.residual_anomalies AS DOUBLE) AS residualAnomalies,
4293
- COALESCE(e.trend_slope, 0.0) AS trendSlope,
4294
- s.seriesJson
4295
- FROM per_entity e
4296
- LEFT JOIN series s USING (query, page)
4297
- ORDER BY seasonalStrength DESC, ABS(COALESCE(e.trend_slope, 0.0)) DESC
4298
- LIMIT ${Number(limit)}
4299
- `,
4300
- params: [
4301
- startDate,
4302
- endDate,
4303
- minImpressions
4304
- ],
4305
- current: {
4306
- table: "page_queries",
4307
- partitions: enumeratePartitions(startDate, endDate)
4308
- }
4309
- };
4310
- },
4311
- reduceSql(rows, params) {
4312
- const arr = Array.isArray(rows) ? rows : [];
4313
- const metric = params.metric === "clicks" ? "clicks" : "impressions";
4314
- const results = arr.map((r) => ({
4315
- keyword: rowString(r.keyword),
4316
- page: rowString(r.page),
4317
- totalImpressions: num(r.totalImpressions),
4318
- days: num(r.days),
4319
- seasonalStrength: num(r.seasonalStrength),
4320
- trendStrength: num(r.trendStrength),
4321
- residualAnomalies: num(r.residualAnomalies),
4322
- trendSlope: num(r.trendSlope),
4323
- series: parseJsonRows(r.seriesJson).map((s) => ({
4324
- date: rowString(s.date),
4325
- observed: num(s.observed),
4326
- trend: s.trend == null ? null : num(s.trend),
4327
- seasonal: s.seasonal == null ? null : num(s.seasonal),
4328
- residual: s.residual == null ? null : num(s.residual),
4329
- anomaly: rowBoolean(s.anomaly)
4330
- }))
4331
- }));
4332
- return {
4333
- results,
4334
- meta: {
4335
- total: results.length,
4336
- metric,
4337
- avgSeasonalStrength: results.length > 0 ? results.reduce((a, r) => a + r.seasonalStrength, 0) / results.length : 0
4338
- }
4339
- };
4340
- }
4341
- });
4342
- const DEFAULT_ROW_LIMIT$1 = 25e3;
4343
- function paginateStrikingDistance(results, options) {
4344
- return paginateSortedInMemory(results, {
4345
- limit: options.limit ?? 1e3,
4346
- offset: options.offset
4347
- }, (left, right) => right.potentialClicks - left.potentialClicks);
4348
- }
4349
- function filterStrikingDistance(rows, options = {}) {
4350
- const minPosition = options.minPosition ?? 4;
4351
- const maxPosition = options.maxPosition ?? 20;
4352
- const minImpressions = options.minImpressions ?? 100;
4353
- const maxCtr = options.maxCtr ?? .05;
4354
- const results = [];
4355
- for (const row of rows) {
4356
- const position = num(row.position);
4357
- const impressions = num(row.impressions);
4358
- const ctr = num(row.ctr);
4359
- const clicks = num(row.clicks);
4360
- if (position < minPosition || position > maxPosition) continue;
4361
- if (impressions < minImpressions) continue;
4362
- if (ctr > maxCtr) continue;
4363
- results.push({
4364
- keyword: String(row.query ?? ""),
4365
- page: row.page == null ? null : String(row.page),
4366
- clicks,
4367
- impressions,
4368
- ctr,
4369
- position,
4370
- potentialClicks: Math.round(impressions * .15)
4371
- });
4372
- }
4373
- return results;
4374
- }
4375
- const strikingDistanceAnalyzer = defineAnalyzer({
4376
- id: "striking-distance",
4377
- reduce(rows, params) {
4378
- const results = filterStrikingDistance(Array.isArray(rows) ? rows : [], params);
4379
- const paged = paginateStrikingDistance(results, params);
4380
- return {
4381
- results: paged,
4382
- meta: {
4383
- total: results.length,
4384
- returned: paged.length
4385
- }
4386
- };
4387
- },
4388
- buildSql(params) {
4389
- const { startDate, endDate } = periodOf(params);
4390
- return {
4391
- sql: `
4392
- SELECT
4393
- query,
4394
- url AS page,
4395
- CAST(SUM(clicks) AS DOUBLE) AS clicks,
4396
- CAST(SUM(impressions) AS DOUBLE) AS impressions,
4397
- CAST(SUM(clicks) AS DOUBLE) / NULLIF(SUM(impressions), 0) AS ctr,
4398
- SUM(sum_position) / NULLIF(SUM(impressions), 0) + 1 AS position
4399
- FROM read_parquet({{FILES}}, union_by_name = true)
4400
- WHERE date >= ? AND date <= ?
4401
- GROUP BY query, url
4402
- `,
4403
- params: [startDate, endDate],
4404
- current: {
4405
- table: "page_queries",
4406
- partitions: enumeratePartitions(startDate, endDate)
4407
- }
4408
- };
4409
- },
4410
- buildRows(params) {
4411
- return { queries: queriesQueryState(periodOf(params), params.limit ?? DEFAULT_ROW_LIMIT$1) };
4412
- }
4413
- });
4414
- const survivalAnalyzer = defineAnalyzer({
4415
- id: "survival",
4416
- buildSql(params) {
4417
- const endDate = params.endDate ?? defaultEndDate();
4418
- const startDate = params.startDate ?? daysAgoUtc(183);
4419
- const minImpressions = params.minImpressions ?? 5;
4420
- return {
4421
- sql: `
4422
- WITH daily AS (
4423
- SELECT
4424
- query,
4425
- url,
4426
- date,
4427
- ${METRIC_EXPR.clicks} AS day_clicks,
4428
- ${METRIC_EXPR.impressions} AS day_impressions,
4429
- ${METRIC_EXPR.position} AS day_position
4430
- FROM read_parquet({{FILES}}, union_by_name = true)
4431
- WHERE date >= ? AND date <= ?
4432
- AND query IS NOT NULL AND query <> ''
4433
- AND url IS NOT NULL AND url <> ''
4434
- GROUP BY query, url, date
4435
- HAVING SUM(impressions) >= ?
4436
- ),
4437
- classified AS (
4438
- SELECT *,
4439
- (day_position <= 10) AS in_top10
4440
- FROM daily
4441
- ),
4442
- transitions AS (
4443
- SELECT *,
4444
- CASE
4445
- WHEN in_top10 AND (LAG(in_top10) OVER w IS NULL OR NOT LAG(in_top10) OVER w)
4446
- THEN 1 ELSE 0
4447
- END AS is_entry
4448
- FROM classified
4449
- WINDOW w AS (PARTITION BY query, url ORDER BY date)
4450
- ),
4451
- run_ids AS (
4452
- SELECT *,
4453
- SUM(is_entry) OVER (PARTITION BY query, url ORDER BY date) AS run_id
4454
- FROM transitions
4455
- WHERE in_top10
4456
- ),
4457
- window_bounds AS (
4458
- SELECT MIN(date) AS window_start, MAX(date) AS window_end FROM daily
4459
- ),
4460
- episodes_raw AS (
4461
- SELECT
4462
- query, url, run_id,
4463
- MIN(date) AS entry_date,
4464
- MAX(date) AS exit_date,
4465
- DATEDIFF('day', MIN(date), MAX(date)) + 1 AS tenure
4466
- FROM run_ids
4467
- GROUP BY query, url, run_id
4468
- ),
4469
- episodes AS (
4470
- SELECT
4471
- e.query, e.url, e.run_id, e.entry_date, e.exit_date, e.tenure,
4472
- (e.exit_date >= wb.window_end - INTERVAL 2 DAY) AS censored,
4473
- CASE
4474
- WHEN regexp_extract(e.url, '^(?:https?://[^/]+)?(/[^/?#]*)', 1) = '/' OR e.url = '/'
4475
- THEN 'home'
4476
- WHEN regexp_extract(e.url, '^(?:https?://[^/]+)?/([^/?#]+)', 1) = ''
4477
- THEN 'home'
4478
- ELSE regexp_extract(e.url, '^(?:https?://[^/]+)?/([^/?#]+)', 1)
4479
- END AS cohort
4480
- FROM episodes_raw e
4481
- CROSS JOIN window_bounds wb
4482
- ),
4483
- episodes_all AS (
4484
- SELECT query, url, tenure, censored, cohort FROM episodes
4485
- UNION ALL
4486
- SELECT query, url, tenure, censored, '__all__' AS cohort FROM episodes
4487
- ),
4488
- cohort_totals AS (
4489
- SELECT cohort, COUNT(*) AS n_total
4490
- FROM episodes_all
4491
- GROUP BY cohort
4492
- ),
4493
- events AS (
4494
- SELECT
4495
- cohort,
4496
- tenure,
4497
- COUNT(*) FILTER (WHERE NOT censored) AS d_t,
4498
- COUNT(*) AS n_ending_at_t
4499
- FROM episodes_all
4500
- GROUP BY cohort, tenure
4501
- ),
4502
- km AS (
4503
- SELECT
4504
- e.cohort,
4505
- e.tenure,
4506
- e.d_t,
4507
- e.n_ending_at_t,
4508
- SUM(e.n_ending_at_t) OVER (PARTITION BY e.cohort ORDER BY e.tenure DESC
4509
- ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS at_risk
4510
- FROM events e
4511
- ),
4512
- km_surv AS (
4513
- SELECT
4514
- cohort, tenure, d_t, at_risk,
4515
- EXP(SUM(LN(GREATEST(1.0 - CAST(d_t AS DOUBLE) / NULLIF(at_risk, 0), 1e-9)))
4516
- OVER (PARTITION BY cohort ORDER BY tenure
4517
- ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)) AS survival
4518
- FROM km
4519
- ),
4520
- curve_agg AS (
4521
- SELECT
4522
- cohort,
4523
- to_json(list({
4524
- 'tenure': tenure,
4525
- 'survival': survival,
4526
- 'atRisk': at_risk,
4527
- 'events': d_t
4528
- } ORDER BY tenure)) AS curveJson
4529
- FROM km_surv
4530
- GROUP BY cohort
4531
- ),
4532
- cohort_stats AS (
4533
- SELECT
4534
- ea.cohort,
4535
- COUNT(*) AS episode_count,
4536
- AVG(CASE WHEN ea.censored THEN 1.0 ELSE 0.0 END) AS censoring_rate
4537
- FROM episodes_all ea
4538
- GROUP BY ea.cohort
4539
- )
4540
- SELECT
4541
- cs.cohort,
4542
- cs.episode_count AS episodeCount,
4543
- cs.censoring_rate AS censoringRate,
4544
- ca.curveJson
4545
- FROM cohort_stats cs
4546
- LEFT JOIN curve_agg ca USING (cohort)
4547
- ORDER BY cs.cohort
4548
- `,
4549
- params: [
4550
- startDate,
4551
- endDate,
4552
- minImpressions
4553
- ],
4554
- current: {
4555
- table: "page_queries",
4556
- partitions: enumeratePartitions(startDate, endDate)
4557
- }
4558
- };
4559
- },
4560
- reduceSql(rows, params) {
4561
- const arr = Array.isArray(rows) ? rows : [];
4562
- const endDate = params.endDate ?? defaultEndDate();
4563
- const startDate = params.startDate ?? daysAgoUtc(183);
4564
- const windowDays = Math.round((new Date(endDate).getTime() - new Date(startDate).getTime()) / MS_PER_DAY) + 1;
4565
- const results = arr.map((r) => {
4566
- const curve = parseJsonRows(r.curveJson).map((p) => ({
4567
- tenure: num(p.tenure),
4568
- survival: num(p.survival),
4569
- atRisk: num(p.atRisk),
4570
- events: num(p.events)
4571
- }));
4572
- let medianTenure = 0;
4573
- for (let i = 0; i < curve.length; i++) {
4574
- const cur = curve[i];
4575
- if (cur.survival <= .5) {
4576
- if (i === 0) medianTenure = cur.tenure;
4577
- else {
4578
- const prev = curve[i - 1];
4579
- const span = prev.survival - cur.survival;
4580
- const frac = span > 0 ? (prev.survival - .5) / span : 0;
4581
- medianTenure = prev.tenure + frac * (cur.tenure - prev.tenure);
4582
- }
4583
- break;
4584
- }
4585
- }
4586
- const last = curve[curve.length - 1];
4587
- if (medianTenure === 0 && last && last.survival > .5) medianTenure = last.tenure;
4588
- return {
4589
- cohort: rowString(r.cohort),
4590
- episodeCount: num(r.episodeCount),
4591
- censoringRate: num(r.censoringRate),
4592
- medianTenure,
4593
- curve
4594
- };
4595
- });
4596
- return {
4597
- results,
4598
- meta: {
4599
- totalEpisodes: results.find((r) => r.cohort === "__all__")?.episodeCount ?? 0,
4600
- cohortCount: results.filter((r) => r.cohort !== "__all__").length,
4601
- windowDays
4602
- }
4603
- };
4604
- }
4605
- });
4606
- const trendsAnalyzer = defineAnalyzer({
4607
- id: "trends",
4608
- buildSql(params) {
4609
- const weeks = params.weeks ?? 28;
4610
- const endDate = params.endDate || defaultEndDate();
4611
- const startDate = params.startDate || toIsoDate(/* @__PURE__ */ new Date(Date.parse(endDate) - (weeks * 7 - 1) * MS_PER_DAY));
4612
- const minImpressions = params.minImpressions ?? 100;
4613
- const minWeeksWithData = params.minWeeksWithData ?? Math.max(2, Math.floor(weeks / 4));
4614
- const limit = params.limit ?? 500;
4615
- const dim = params.dimension === "keywords" ? "keywords" : "pages";
4616
- const table = dim === "keywords" ? "queries" : "pages";
4617
- return {
4618
- sql: `
4619
- WITH bucketed AS (
4620
- SELECT
4621
- ${dim === "keywords" ? "query" : "url"} AS entity,
4622
- date_trunc('week', CAST(date AS DATE)) AS week,
4623
- ${METRIC_EXPR.clicks} AS clicks,
4624
- ${METRIC_EXPR.impressions} AS impressions,
4625
- SUM(sum_position) AS sum_position_sum
4626
- FROM read_parquet({{FILES}}, union_by_name = true)
4627
- WHERE date >= ? AND date <= ?
4628
- GROUP BY entity, week
4629
- ),
4630
- with_meta AS (
4631
- SELECT
4632
- entity, week, clicks, impressions, sum_position_sum,
4633
- ROW_NUMBER() OVER (PARTITION BY entity ORDER BY week) - 1 AS week_idx,
4634
- COUNT(*) OVER (PARTITION BY entity) AS n_weeks,
4635
- (ROW_NUMBER() OVER (PARTITION BY entity ORDER BY week) - 1)
4636
- < (COUNT(*) OVER (PARTITION BY entity) / 2) AS is_first_half
4637
- FROM bucketed
4638
- ),
4639
- agg AS (
4640
- SELECT
4641
- entity,
4642
- SUM(clicks) AS totalClicks,
4643
- SUM(impressions) AS totalImpressions,
4644
- any_value(n_weeks) AS weeksWithData,
4645
- COALESCE(regr_slope(clicks, CAST(week_idx AS DOUBLE)), 0.0) AS slope,
4646
- SUM(CASE WHEN is_first_half THEN clicks ELSE 0 END) AS firstHalfClicks,
4647
- SUM(CASE WHEN NOT is_first_half THEN clicks ELSE 0 END) AS secondHalfClicks,
4648
- SUM(sum_position_sum) / NULLIF(SUM(impressions), 0) + 1 AS avgPosition,
4649
- to_json(list({
4650
- 'week': strftime(week, '%Y-%m-%d'),
4651
- 'clicks': clicks,
4652
- 'impressions': impressions
4653
- } ORDER BY week)) AS seriesJson
4654
- FROM with_meta
4655
- GROUP BY entity
4656
- HAVING SUM(impressions) >= ? AND any_value(n_weeks) >= ?
4657
- ),
4658
- classified AS (
4659
- SELECT
4660
- *,
4661
- CASE
4662
- WHEN firstHalfClicks = 0 AND secondHalfClicks > 0 THEN 10.0
4663
- WHEN firstHalfClicks = 0 THEN 1.0
4664
- ELSE secondHalfClicks / firstHalfClicks
4665
- END AS growthRatio
4666
- FROM agg
4667
- )
4668
- SELECT
4669
- entity,
4670
- totalClicks,
4671
- totalImpressions,
4672
- weeksWithData,
4673
- slope,
4674
- growthRatio,
4675
- avgPosition,
4676
- CASE
4677
- WHEN growthRatio >= 1.5 AND slope > 0 THEN 'accelerating'
4678
- WHEN growthRatio >= 1.1 AND slope >= 0 THEN 'growing'
4679
- WHEN growthRatio < 0.5 THEN 'cratering'
4680
- WHEN growthRatio < 0.9 AND slope < 0 THEN 'declining'
4681
- ELSE 'steady'
4682
- END AS trend,
4683
- seriesJson
4684
- FROM classified
4685
- ORDER BY
4686
- CASE
4687
- WHEN growthRatio >= 1.5 AND slope > 0 THEN 0
4688
- WHEN growthRatio < 0.5 THEN 1
4689
- WHEN growthRatio >= 1.1 AND slope >= 0 THEN 2
4690
- WHEN growthRatio < 0.9 AND slope < 0 THEN 3
4691
- ELSE 4
4692
- END,
4693
- ABS(growthRatio - 1) DESC,
4694
- totalClicks DESC
4695
- LIMIT ${Number(limit)}
4696
- `,
4697
- params: [
4698
- startDate,
4699
- endDate,
4700
- minImpressions,
4701
- minWeeksWithData
4702
- ],
4703
- current: {
4704
- table,
4705
- partitions: enumeratePartitions(startDate, endDate)
4706
- }
4707
- };
4708
- },
4709
- reduceSql(rows, params) {
4710
- const arr = Array.isArray(rows) ? rows : [];
4711
- const weeks = params.weeks ?? 28;
4712
- const endDate = params.endDate || defaultEndDate();
4713
- const startDate = params.startDate || toIsoDate(/* @__PURE__ */ new Date(Date.parse(endDate) - (weeks * 7 - 1) * MS_PER_DAY));
4714
- const dim = params.dimension === "keywords" ? "keywords" : "pages";
4715
- const results = arr.map((r) => {
4716
- const series = parseJsonRows(r.seriesJson).map((s) => ({
4717
- week: rowString(s.week),
4718
- clicks: num(s.clicks),
4719
- impressions: num(s.impressions)
4720
- }));
4721
- return {
4722
- [dim === "keywords" ? "query" : "page"]: rowString(r.entity),
4723
- totalClicks: num(r.totalClicks),
4724
- totalImpressions: num(r.totalImpressions),
4725
- weeksWithData: num(r.weeksWithData),
4726
- slope: num(r.slope),
4727
- growthRatio: num(r.growthRatio),
4728
- avgPosition: num(r.avgPosition),
4729
- trend: rowString(r.trend),
4730
- series
4731
- };
4732
- });
4733
- const counts = {
4734
- accelerating: 0,
4735
- growing: 0,
4736
- steady: 0,
4737
- declining: 0,
4738
- cratering: 0
4739
- };
4740
- for (const r of results) counts[r.trend] = (counts[r.trend] ?? 0) + 1;
4741
- return {
4742
- results,
4743
- meta: {
4744
- total: results.length,
4745
- dimension: dim,
4746
- weeks: Number(weeks),
4747
- startDate,
4748
- endDate,
4749
- counts
4750
- }
4751
- };
4752
- }
4753
- });
4754
- const DEFAULT_ROW_LIMIT = 25e3;
4755
- function analyzeZeroClick(rows, options = {}) {
4756
- const minImpressions = options.minImpressions ?? 1e3;
4757
- const maxCtr = options.maxCtr ?? .03;
4758
- const maxPosition = options.maxPosition ?? 10;
4759
- const queryMap = /* @__PURE__ */ new Map();
4760
- for (const row of rows) {
4761
- if (row.impressions < minImpressions || row.position > maxPosition || row.ctr > maxCtr) continue;
4762
- const existing = queryMap.get(row.query);
4763
- if (!existing || row.position < existing.position) queryMap.set(row.query, { ...row });
4764
- }
4765
- return [...queryMap.values()].sort((left, right) => right.impressions - left.impressions);
4766
- }
4767
- const zeroClickAnalyzer = defineAnalyzer({
4768
- id: "zero-click",
4769
- buildSql(params) {
4770
- const { startDate, endDate } = periodOf(params);
4771
- const minImpressions = params.minImpressions ?? 1e3;
4772
- const maxCtr = params.maxCtr ?? .03;
4773
- const maxPosition = params.maxPosition ?? 10;
4774
- const limit = params.limit ?? 1e3;
4775
- return {
4776
- sql: `
4777
- WITH agg AS (
4778
- SELECT
4779
- query,
4780
- url AS page,
4781
- ${METRIC_EXPR.clicks} AS clicks,
4782
- ${METRIC_EXPR.impressions} AS impressions,
4783
- ${METRIC_EXPR.ctr} AS ctr,
4784
- ${METRIC_EXPR.position} AS position
4785
- FROM read_parquet({{FILES}}, union_by_name = true)
4786
- WHERE date >= ? AND date <= ?
4787
- GROUP BY query, url
4788
- HAVING SUM(impressions) >= ?
4789
- )
4790
- SELECT
4791
- query, page, clicks, impressions, ctr, position,
4792
- CAST(GREATEST(0, ROUND(impressions * (
4793
- CASE
4794
- WHEN position <= 1 THEN 0.30
4795
- WHEN position <= 3 THEN 0.15
4796
- WHEN position <= 5 THEN 0.08
4797
- ELSE 0.04
4798
- END
4799
- )) - clicks) AS DOUBLE) AS missedClicks
4800
- FROM agg
4801
- WHERE position <= ? AND ctr < ?
4802
- ORDER BY impressions DESC
4803
- ${paginateClause({
4804
- limit,
4805
- offset: params.offset
4806
- })}
4807
- `,
4808
- params: [
4809
- startDate,
4810
- endDate,
4811
- minImpressions,
4812
- maxPosition,
4813
- maxCtr
4814
- ],
4815
- current: {
4816
- table: "page_queries",
4817
- partitions: enumeratePartitions(startDate, endDate)
4818
- }
4819
- };
4820
- },
4821
- reduceSql(rows, params) {
4822
- const arr = Array.isArray(rows) ? rows : [];
4823
- const minImpressions = params.minImpressions ?? 1e3;
4824
- const maxCtr = params.maxCtr ?? .03;
4825
- const maxPosition = params.maxPosition ?? 10;
4826
- return {
4827
- results: arr.map((r) => ({
4828
- query: r.query == null ? "" : String(r.query),
4829
- page: r.page == null ? "" : String(r.page),
4830
- clicks: num(r.clicks),
4831
- impressions: num(r.impressions),
4832
- ctr: num(r.ctr),
4833
- position: num(r.position),
4834
- missedClicks: num(r.missedClicks)
4835
- })),
4836
- meta: {
4837
- total: arr.length,
4838
- minImpressions,
4839
- maxCtr,
4840
- maxPosition
4841
- }
4842
- };
4843
- },
4844
- buildRows(params) {
4845
- const period = periodOf(params);
4846
- const limit = params.limit ?? DEFAULT_ROW_LIMIT;
4847
- return { rows: gsc.select(query, page).where(between(date, period.startDate, period.endDate)).limit(limit).getState() };
4848
- },
4849
- reduceRows(rows, params) {
4850
- const results = analyzeZeroClick(Array.isArray(rows) ? rows : [], {
4851
- minImpressions: params.minImpressions ?? 1e3,
4852
- maxCtr: params.maxCtr ?? .03,
4853
- maxPosition: params.maxPosition ?? 10
4854
- });
4855
- const paged = paginateSortedInMemory(results, {
4856
- limit: params.limit,
4857
- offset: params.offset
4858
- }, (left, right) => right.impressions - left.impressions);
4859
- return {
4860
- results: paged,
4861
- meta: {
4862
- total: results.length,
4863
- returned: paged.length
4864
- }
4865
- };
4866
- }
4867
- });
4868
- export { bayesianCtrAnalyzer, bipartitePagerankAnalyzer, brandAnalyzer, cannibalizationAnalyzer, changePointAnalyzer, clampLimit, clampOffset, clusteringAnalyzer, concentrationAnalyzer, contentVelocityAnalyzer, ctrAnomalyAnalyzer, ctrCurveAnalyzer, darkTrafficAnalyzer, dataDetailAnalyzer, dataQueryAnalyzer, datesQueryState, decayAnalyzer, deviceGapAnalyzer, intentAtlasAnalyzer, keywordBreadthAnalyzer, longTailAnalyzer, moversAnalyzer, opportunityAnalyzer, pagesQueryState, paginateClause, paginateInMemory, positionDistributionAnalyzer, positionVolatilityAnalyzer, queriesQueryState, queryMigrationAnalyzer, resolveSort, seasonalityAnalyzer, stlDecomposeAnalyzer, strikingDistanceAnalyzer, survivalAnalyzer, trendsAnalyzer, zeroClickAnalyzer };