@gscdump/analysis 1.3.2 → 1.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/action-priority.d.mts +34 -0
- package/dist/action-priority.mjs +201 -0
- package/dist/analyzer/adapt-rows.mjs +12 -0
- package/dist/analyzer/all.mjs +61 -0
- package/dist/analyzer/paginate.mjs +85 -0
- package/dist/analyzer/row-analyzers.d.mts +3 -0
- package/dist/analyzer/row-analyzers.mjs +3 -0
- package/dist/analyzer/row-values.mjs +22 -0
- package/dist/analyzers/bayesian-ctr.mjs +177 -0
- package/dist/analyzers/bipartite-pagerank.mjs +243 -0
- package/dist/analyzers/brand.d.mts +33 -0
- package/dist/analyzers/brand.mjs +154 -0
- package/dist/analyzers/cannibalization.d.mts +58 -0
- package/dist/analyzers/cannibalization.mjs +267 -0
- package/dist/analyzers/change-point.mjs +197 -0
- package/dist/analyzers/clustering.d.mts +29 -0
- package/dist/analyzers/clustering.mjs +217 -0
- package/dist/analyzers/concentration.d.mts +42 -0
- package/dist/analyzers/concentration.mjs +180 -0
- package/dist/analyzers/content-velocity.mjs +87 -0
- package/dist/analyzers/ctr-anomaly.mjs +189 -0
- package/dist/analyzers/ctr-curve.mjs +128 -0
- package/dist/analyzers/dark-traffic.mjs +130 -0
- package/dist/analyzers/data-detail.mjs +44 -0
- package/dist/analyzers/data-query.mjs +44 -0
- package/dist/analyzers/decay.d.mts +38 -0
- package/dist/analyzers/decay.mjs +187 -0
- package/dist/analyzers/device-gap.mjs +135 -0
- package/dist/analyzers/intent-atlas.mjs +175 -0
- package/dist/analyzers/keyword-breadth.mjs +115 -0
- package/dist/analyzers/long-tail.mjs +150 -0
- package/dist/analyzers/movers.d.mts +43 -0
- package/dist/analyzers/movers.mjs +283 -0
- package/dist/analyzers/opportunity.d.mts +32 -0
- package/dist/analyzers/opportunity.mjs +212 -0
- package/dist/analyzers/position-distribution.mjs +57 -0
- package/dist/analyzers/position-volatility.mjs +143 -0
- package/dist/analyzers/query-migration.mjs +178 -0
- package/dist/analyzers/seasonality.d.mts +28 -0
- package/dist/analyzers/seasonality.mjs +136 -0
- package/dist/analyzers/stl-decompose.mjs +187 -0
- package/dist/analyzers/striking-distance.d.mts +54 -0
- package/dist/analyzers/striking-distance.mjs +82 -0
- package/dist/analyzers/survival.mjs +200 -0
- package/dist/analyzers/trends.mjs +156 -0
- package/dist/analyzers/zero-click.d.mts +17 -0
- package/dist/analyzers/zero-click.mjs +122 -0
- package/dist/browser.d.mts +5 -0
- package/dist/browser.mjs +11 -0
- package/dist/default-registry.d.mts +3 -1
- package/dist/default-registry.mjs +5 -1
- package/dist/errors.d.mts +51 -1
- package/dist/index.d.mts +30 -494
- package/dist/index.mjs +28 -49
- package/dist/query/analyzers.mjs +301 -0
- package/dist/query/index.mjs +4 -0
- package/dist/query/intent.d.mts +20 -0
- package/dist/query/intent.mjs +139 -0
- package/dist/query/normalize.d.mts +15 -0
- package/dist/query/normalize.mjs +204 -0
- package/dist/report/format.d.mts +7 -0
- package/dist/report/format.mjs +45 -0
- package/dist/report/index.d.mts +4 -1
- package/dist/report/index.mjs +4 -1
- package/dist/report/registry.d.mts +4 -0
- package/dist/report/registry.mjs +26 -0
- package/dist/report/reports/brand.mjs +104 -0
- package/dist/report/reports/growth.mjs +144 -0
- package/dist/report/reports/health.mjs +141 -0
- package/dist/report/reports/movers.mjs +203 -0
- package/dist/report/reports/opportunities.mjs +184 -0
- package/dist/report/reports/pre-publish.mjs +111 -0
- package/dist/report/reports/priority.mjs +97 -0
- package/dist/report/reports/risks.mjs +201 -0
- package/dist/report/reports/triage.mjs +155 -0
- package/dist/report/require.mjs +15 -0
- package/dist/report/resolve-target.d.mts +29 -0
- package/dist/report/resolve-target.mjs +32 -0
- package/dist/{_chunks/index.d.mts → report/runtime.d.mts} +2 -37
- package/dist/report/runtime.mjs +80 -0
- package/dist/report/sections.mjs +22 -0
- package/dist/sitemap-health.d.mts +34 -0
- package/dist/sitemap-health.mjs +37 -0
- package/dist/{_chunks/index2.d.mts → source/composite.d.mts} +2 -13
- package/dist/{_chunks/source.mjs → source/composite.mjs} +1 -17
- package/dist/source/in-memory.d.mts +14 -0
- package/dist/source/in-memory.mjs +17 -0
- package/dist/source/index.d.mts +2 -1
- package/dist/source/index.mjs +2 -1
- package/dist/sql-analyzers.d.mts +3 -0
- package/dist/sql-analyzers.mjs +3 -0
- package/dist/types.d.mts +34 -0
- package/dist/types.mjs +21 -0
- package/package.json +4 -4
- package/dist/_chunks/default-registry.d.mts +0 -5
- package/dist/_chunks/default-registry.mjs +0 -5183
- package/dist/_chunks/errors.d.mts +0 -52
- package/dist/_chunks/report.mjs +0 -1710
- /package/dist/{_chunks/scoring.mjs → scoring.mjs} +0 -0
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { AnalysisResult } from "@gscdump/engine/analysis-types";
|
|
2
|
+
type ActionSource = 'cannibalization' | 'striking-distance' | 'ctr-anomaly' | 'change-point' | 'opportunity';
|
|
3
|
+
type Effort = 'low' | 'medium' | 'high';
|
|
4
|
+
interface PriorityAction {
|
|
5
|
+
id: string;
|
|
6
|
+
title: string;
|
|
7
|
+
keyword: string;
|
|
8
|
+
page: string;
|
|
9
|
+
sources: ActionSource[];
|
|
10
|
+
severity: number;
|
|
11
|
+
impressions: number;
|
|
12
|
+
impact: number;
|
|
13
|
+
why: string;
|
|
14
|
+
effort: Effort;
|
|
15
|
+
priorityScore: number;
|
|
16
|
+
data: Partial<Record<ActionSource, Record<string, unknown>>>;
|
|
17
|
+
}
|
|
18
|
+
type ActionPrioritySourceStatus = 'pending' | 'running' | 'done' | 'skipped' | 'error';
|
|
19
|
+
interface ActionPrioritySourceState {
|
|
20
|
+
source: ActionSource;
|
|
21
|
+
status: ActionPrioritySourceStatus;
|
|
22
|
+
count: number;
|
|
23
|
+
error?: string;
|
|
24
|
+
}
|
|
25
|
+
interface ActionPriorityResult {
|
|
26
|
+
actions: PriorityAction[];
|
|
27
|
+
totalSignals: number;
|
|
28
|
+
sources: ActionPrioritySourceState[];
|
|
29
|
+
}
|
|
30
|
+
declare const DEFAULT_PRIORITY_SOURCES: ActionSource[];
|
|
31
|
+
declare function normalizePriorityActions(source: ActionSource, result: AnalysisResult): PriorityAction[];
|
|
32
|
+
declare function mergePriorityActions(all: PriorityAction[]): PriorityAction[];
|
|
33
|
+
declare function scorePriorityActions(actions: PriorityAction[]): PriorityAction[];
|
|
34
|
+
export { ActionPriorityResult, ActionPrioritySourceState, ActionPrioritySourceStatus, ActionSource, DEFAULT_PRIORITY_SOURCES, Effort, PriorityAction, mergePriorityActions, normalizePriorityActions, scorePriorityActions };
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
import { clamp, clamp01 } from "./scoring.mjs";
|
|
2
|
+
const DEFAULT_PRIORITY_SOURCES = [
|
|
3
|
+
"striking-distance",
|
|
4
|
+
"opportunity",
|
|
5
|
+
"cannibalization",
|
|
6
|
+
"ctr-anomaly",
|
|
7
|
+
"change-point"
|
|
8
|
+
];
|
|
9
|
+
const EFFORT_BY_SOURCE = {
|
|
10
|
+
"striking-distance": "low",
|
|
11
|
+
"opportunity": "low",
|
|
12
|
+
"cannibalization": "medium",
|
|
13
|
+
"ctr-anomaly": "high",
|
|
14
|
+
"change-point": "high"
|
|
15
|
+
};
|
|
16
|
+
const EFFORT_MULTIPLIER = {
|
|
17
|
+
low: 1.3,
|
|
18
|
+
medium: 1,
|
|
19
|
+
high: .7
|
|
20
|
+
};
|
|
21
|
+
const EFFORT_RANK = {
|
|
22
|
+
low: 0,
|
|
23
|
+
medium: 1,
|
|
24
|
+
high: 2
|
|
25
|
+
};
|
|
26
|
+
function idKey(keyword, page) {
|
|
27
|
+
return `${keyword.toLowerCase()}|${page.toLowerCase()}`;
|
|
28
|
+
}
|
|
29
|
+
function truncate(s, n) {
|
|
30
|
+
return s.length <= n ? s : `${s.slice(0, n - 1)}...`;
|
|
31
|
+
}
|
|
32
|
+
function buildAction(spec) {
|
|
33
|
+
return {
|
|
34
|
+
id: idKey(spec.keyword, spec.page),
|
|
35
|
+
title: spec.title,
|
|
36
|
+
keyword: spec.keyword,
|
|
37
|
+
page: spec.page,
|
|
38
|
+
sources: [spec.source],
|
|
39
|
+
severity: spec.severity,
|
|
40
|
+
impressions: spec.impressions,
|
|
41
|
+
impact: spec.impact,
|
|
42
|
+
effort: EFFORT_BY_SOURCE[spec.source],
|
|
43
|
+
why: spec.why,
|
|
44
|
+
priorityScore: 0,
|
|
45
|
+
data: { [spec.source]: spec.data }
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
function fromStrikingDistance(rows) {
|
|
49
|
+
const out = [];
|
|
50
|
+
for (const r of rows) {
|
|
51
|
+
if (r.page == null) continue;
|
|
52
|
+
const impact = Math.max(0, r.potentialClicks);
|
|
53
|
+
if (impact <= 0) continue;
|
|
54
|
+
const posScore = clamp01((20 - r.position) / 16);
|
|
55
|
+
const imprScore = Math.min(1, r.impressions / 5e3);
|
|
56
|
+
out.push(buildAction({
|
|
57
|
+
source: "striking-distance",
|
|
58
|
+
keyword: r.keyword,
|
|
59
|
+
page: r.page,
|
|
60
|
+
title: `Push "${truncate(r.keyword, 40)}" onto page 1`,
|
|
61
|
+
why: `Ranks #${r.position.toFixed(1)} with ${Math.round(r.impressions)} impressions; small gains unlock page-1 clicks.`,
|
|
62
|
+
severity: Math.round(100 * Math.sqrt(posScore * imprScore)),
|
|
63
|
+
impressions: r.impressions,
|
|
64
|
+
impact,
|
|
65
|
+
data: r
|
|
66
|
+
}));
|
|
67
|
+
}
|
|
68
|
+
return out;
|
|
69
|
+
}
|
|
70
|
+
function fromOpportunity(rows) {
|
|
71
|
+
const out = [];
|
|
72
|
+
for (const r of rows) {
|
|
73
|
+
if (r.page == null) continue;
|
|
74
|
+
const impact = Math.max(0, r.potentialClicks);
|
|
75
|
+
if (impact <= 0) continue;
|
|
76
|
+
out.push(buildAction({
|
|
77
|
+
source: "opportunity",
|
|
78
|
+
keyword: r.keyword,
|
|
79
|
+
page: r.page,
|
|
80
|
+
title: `Improve on-page for "${truncate(r.keyword, 40)}"`,
|
|
81
|
+
why: `Opportunity score ${Math.round(r.opportunityScore)}; CTR ${(r.ctr * 100).toFixed(1)}% vs expected at pos ${r.position.toFixed(1)}.`,
|
|
82
|
+
severity: Math.round(r.opportunityScore),
|
|
83
|
+
impressions: r.impressions,
|
|
84
|
+
impact,
|
|
85
|
+
data: r
|
|
86
|
+
}));
|
|
87
|
+
}
|
|
88
|
+
return out;
|
|
89
|
+
}
|
|
90
|
+
function fromCannibalization(events) {
|
|
91
|
+
const out = [];
|
|
92
|
+
for (const ev of events) {
|
|
93
|
+
if (ev.severity < 30) continue;
|
|
94
|
+
out.push(buildAction({
|
|
95
|
+
source: "cannibalization",
|
|
96
|
+
keyword: ev.keyword,
|
|
97
|
+
page: ev.leaderUrl,
|
|
98
|
+
title: `Consolidate cannibalization on "${truncate(ev.keyword, 36)}"`,
|
|
99
|
+
why: `${ev.competitorCount} URLs split ${Math.round(ev.totalImpressions)} impressions; leader loses ~${Math.round(ev.stolenClicks)} clicks to siblings.`,
|
|
100
|
+
severity: Math.round(ev.severity),
|
|
101
|
+
impressions: ev.totalImpressions,
|
|
102
|
+
impact: Math.max(0, ev.stolenClicks),
|
|
103
|
+
data: ev
|
|
104
|
+
}));
|
|
105
|
+
}
|
|
106
|
+
return out;
|
|
107
|
+
}
|
|
108
|
+
function fromCtrAnomaly(rows) {
|
|
109
|
+
const out = [];
|
|
110
|
+
let maxRaw = 0;
|
|
111
|
+
for (const r of rows) if (r.severity > maxRaw) maxRaw = r.severity;
|
|
112
|
+
for (const r of rows) {
|
|
113
|
+
const impact = Math.max(0, r.clicksLost);
|
|
114
|
+
if (impact <= 0) continue;
|
|
115
|
+
out.push(buildAction({
|
|
116
|
+
source: "ctr-anomaly",
|
|
117
|
+
keyword: r.keyword,
|
|
118
|
+
page: r.page,
|
|
119
|
+
title: `Lift CTR on "${truncate(r.keyword, 36)}"`,
|
|
120
|
+
why: `CTR collapsed ${r.breachDaysDown} days at flat position; ~${Math.round(r.clicksLost)} clicks lost vs baseline ${(r.baselineCtr * 100).toFixed(1)}%.`,
|
|
121
|
+
severity: maxRaw > 0 ? Math.round(r.severity / maxRaw * 100) : 0,
|
|
122
|
+
impressions: r.totalImpressions,
|
|
123
|
+
impact,
|
|
124
|
+
data: r
|
|
125
|
+
}));
|
|
126
|
+
}
|
|
127
|
+
return out;
|
|
128
|
+
}
|
|
129
|
+
function fromChangePoint(rows) {
|
|
130
|
+
const out = [];
|
|
131
|
+
for (const r of rows) {
|
|
132
|
+
if (r.direction !== "worsened") continue;
|
|
133
|
+
const days = Math.max(1, r.totalDays / 2);
|
|
134
|
+
const impact = Math.abs(r.leftMean - r.rightMean) * days;
|
|
135
|
+
if (impact <= 0) continue;
|
|
136
|
+
out.push(buildAction({
|
|
137
|
+
source: "change-point",
|
|
138
|
+
keyword: r.keyword,
|
|
139
|
+
page: r.page,
|
|
140
|
+
title: `Diagnose drop on "${truncate(r.keyword, 34)}"`,
|
|
141
|
+
why: `Significant regression around ${r.changeDate} (${r.leftMean.toFixed(1)} -> ${r.rightMean.toFixed(1)}, LLR ${r.llr.toFixed(0)}).`,
|
|
142
|
+
severity: clamp(Math.round(Math.log10(Math.max(10, r.llr)) / 3 * 100), 0, 100),
|
|
143
|
+
impressions: r.totalImpressions,
|
|
144
|
+
impact,
|
|
145
|
+
data: r
|
|
146
|
+
}));
|
|
147
|
+
}
|
|
148
|
+
return out;
|
|
149
|
+
}
|
|
150
|
+
function normalizePriorityActions(source, result) {
|
|
151
|
+
const rows = result.results;
|
|
152
|
+
if (source === "striking-distance") return fromStrikingDistance(rows);
|
|
153
|
+
if (source === "opportunity") return fromOpportunity(rows);
|
|
154
|
+
if (source === "cannibalization") return fromCannibalization(rows);
|
|
155
|
+
if (source === "ctr-anomaly") return fromCtrAnomaly(rows);
|
|
156
|
+
return fromChangePoint(rows);
|
|
157
|
+
}
|
|
158
|
+
function mergePriorityActions(all) {
|
|
159
|
+
const byId = /* @__PURE__ */ new Map();
|
|
160
|
+
for (const a of all) {
|
|
161
|
+
const existing = byId.get(a.id);
|
|
162
|
+
if (existing == null) {
|
|
163
|
+
byId.set(a.id, {
|
|
164
|
+
...a,
|
|
165
|
+
sources: [...a.sources],
|
|
166
|
+
data: { ...a.data }
|
|
167
|
+
});
|
|
168
|
+
continue;
|
|
169
|
+
}
|
|
170
|
+
const mergedSources = [.../* @__PURE__ */ new Set([...existing.sources, ...a.sources])];
|
|
171
|
+
const preferNew = a.severity > existing.severity;
|
|
172
|
+
const mergedEffort = EFFORT_RANK[a.effort] < EFFORT_RANK[existing.effort] ? a.effort : existing.effort;
|
|
173
|
+
byId.set(a.id, {
|
|
174
|
+
id: existing.id,
|
|
175
|
+
title: preferNew ? a.title : existing.title,
|
|
176
|
+
keyword: existing.keyword,
|
|
177
|
+
page: existing.page,
|
|
178
|
+
sources: mergedSources,
|
|
179
|
+
severity: Math.max(existing.severity, a.severity),
|
|
180
|
+
impressions: Math.max(existing.impressions, a.impressions),
|
|
181
|
+
impact: existing.impact + a.impact,
|
|
182
|
+
why: preferNew ? a.why : existing.why,
|
|
183
|
+
effort: mergedEffort,
|
|
184
|
+
priorityScore: 0,
|
|
185
|
+
data: {
|
|
186
|
+
...existing.data,
|
|
187
|
+
...a.data
|
|
188
|
+
}
|
|
189
|
+
});
|
|
190
|
+
}
|
|
191
|
+
return [...byId.values()];
|
|
192
|
+
}
|
|
193
|
+
function scorePriorityActions(actions) {
|
|
194
|
+
for (const a of actions) {
|
|
195
|
+
const mult = EFFORT_MULTIPLIER[a.effort];
|
|
196
|
+
a.priorityScore = a.impact * (1 + a.severity / 100) * mult;
|
|
197
|
+
}
|
|
198
|
+
actions.sort((a, b) => b.priorityScore - a.priorityScore);
|
|
199
|
+
return actions;
|
|
200
|
+
}
|
|
201
|
+
export { DEFAULT_PRIORITY_SOURCES, mergePriorityActions, normalizePriorityActions, scorePriorityActions };
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { between, date, gsc, page, query } from "gscdump/query";
|
|
2
|
+
const DEFAULT_LIMIT = 25e3;
|
|
3
|
+
function queriesQueryState(period, limit = DEFAULT_LIMIT) {
|
|
4
|
+
return gsc.select(query, page).where(between(date, period.startDate, period.endDate)).limit(limit).getState();
|
|
5
|
+
}
|
|
6
|
+
function pagesQueryState(period, limit = DEFAULT_LIMIT) {
|
|
7
|
+
return gsc.select(page).where(between(date, period.startDate, period.endDate)).limit(limit).getState();
|
|
8
|
+
}
|
|
9
|
+
function datesQueryState(period, limit = DEFAULT_LIMIT) {
|
|
10
|
+
return gsc.select(date).where(between(date, period.startDate, period.endDate)).limit(limit).getState();
|
|
11
|
+
}
|
|
12
|
+
export { datesQueryState, pagesQueryState, queriesQueryState };
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import { bayesianCtrAnalyzer } from "../analyzers/bayesian-ctr.mjs";
|
|
2
|
+
import { bipartitePagerankAnalyzer } from "../analyzers/bipartite-pagerank.mjs";
|
|
3
|
+
import { brandAnalyzer } from "../analyzers/brand.mjs";
|
|
4
|
+
import { cannibalizationAnalyzer } from "../analyzers/cannibalization.mjs";
|
|
5
|
+
import { changePointAnalyzer } from "../analyzers/change-point.mjs";
|
|
6
|
+
import { clusteringAnalyzer } from "../analyzers/clustering.mjs";
|
|
7
|
+
import { concentrationAnalyzer } from "../analyzers/concentration.mjs";
|
|
8
|
+
import { contentVelocityAnalyzer } from "../analyzers/content-velocity.mjs";
|
|
9
|
+
import { ctrAnomalyAnalyzer } from "../analyzers/ctr-anomaly.mjs";
|
|
10
|
+
import { ctrCurveAnalyzer } from "../analyzers/ctr-curve.mjs";
|
|
11
|
+
import { darkTrafficAnalyzer } from "../analyzers/dark-traffic.mjs";
|
|
12
|
+
import { dataDetailAnalyzer } from "../analyzers/data-detail.mjs";
|
|
13
|
+
import { dataQueryAnalyzer } from "../analyzers/data-query.mjs";
|
|
14
|
+
import { decayAnalyzer } from "../analyzers/decay.mjs";
|
|
15
|
+
import { deviceGapAnalyzer } from "../analyzers/device-gap.mjs";
|
|
16
|
+
import { intentAtlasAnalyzer } from "../analyzers/intent-atlas.mjs";
|
|
17
|
+
import { keywordBreadthAnalyzer } from "../analyzers/keyword-breadth.mjs";
|
|
18
|
+
import { longTailAnalyzer } from "../analyzers/long-tail.mjs";
|
|
19
|
+
import { moversAnalyzer } from "../analyzers/movers.mjs";
|
|
20
|
+
import { opportunityAnalyzer } from "../analyzers/opportunity.mjs";
|
|
21
|
+
import { positionDistributionAnalyzer } from "../analyzers/position-distribution.mjs";
|
|
22
|
+
import { positionVolatilityAnalyzer } from "../analyzers/position-volatility.mjs";
|
|
23
|
+
import { queryMigrationAnalyzer } from "../analyzers/query-migration.mjs";
|
|
24
|
+
import { seasonalityAnalyzer } from "../analyzers/seasonality.mjs";
|
|
25
|
+
import { stlDecomposeAnalyzer } from "../analyzers/stl-decompose.mjs";
|
|
26
|
+
import { strikingDistanceAnalyzer } from "../analyzers/striking-distance.mjs";
|
|
27
|
+
import { survivalAnalyzer } from "../analyzers/survival.mjs";
|
|
28
|
+
import { trendsAnalyzer } from "../analyzers/trends.mjs";
|
|
29
|
+
import { zeroClickAnalyzer } from "../analyzers/zero-click.mjs";
|
|
30
|
+
const ALL_ANALYZERS = [
|
|
31
|
+
bayesianCtrAnalyzer,
|
|
32
|
+
bipartitePagerankAnalyzer,
|
|
33
|
+
brandAnalyzer,
|
|
34
|
+
cannibalizationAnalyzer,
|
|
35
|
+
changePointAnalyzer,
|
|
36
|
+
clusteringAnalyzer,
|
|
37
|
+
concentrationAnalyzer,
|
|
38
|
+
contentVelocityAnalyzer,
|
|
39
|
+
ctrAnomalyAnalyzer,
|
|
40
|
+
ctrCurveAnalyzer,
|
|
41
|
+
darkTrafficAnalyzer,
|
|
42
|
+
dataDetailAnalyzer,
|
|
43
|
+
dataQueryAnalyzer,
|
|
44
|
+
decayAnalyzer,
|
|
45
|
+
deviceGapAnalyzer,
|
|
46
|
+
intentAtlasAnalyzer,
|
|
47
|
+
keywordBreadthAnalyzer,
|
|
48
|
+
longTailAnalyzer,
|
|
49
|
+
moversAnalyzer,
|
|
50
|
+
opportunityAnalyzer,
|
|
51
|
+
positionDistributionAnalyzer,
|
|
52
|
+
positionVolatilityAnalyzer,
|
|
53
|
+
queryMigrationAnalyzer,
|
|
54
|
+
seasonalityAnalyzer,
|
|
55
|
+
stlDecomposeAnalyzer,
|
|
56
|
+
strikingDistanceAnalyzer,
|
|
57
|
+
survivalAnalyzer,
|
|
58
|
+
trendsAnalyzer,
|
|
59
|
+
zeroClickAnalyzer
|
|
60
|
+
];
|
|
61
|
+
export { ALL_ANALYZERS };
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
const DEFAULT_LIMIT = 1e3;
|
|
2
|
+
const MAX_LIMIT = 5e4;
|
|
3
|
+
function clampLimit(limit, fallback = DEFAULT_LIMIT) {
|
|
4
|
+
const n = Number(limit ?? fallback);
|
|
5
|
+
if (!Number.isFinite(n) || n <= 0) return fallback;
|
|
6
|
+
return Math.min(n, MAX_LIMIT);
|
|
7
|
+
}
|
|
8
|
+
function clampOffset(offset) {
|
|
9
|
+
const n = Number(offset ?? 0);
|
|
10
|
+
if (!Number.isFinite(n) || n < 0) return 0;
|
|
11
|
+
return Math.floor(n);
|
|
12
|
+
}
|
|
13
|
+
function paginateClause(input) {
|
|
14
|
+
const l = clampLimit(input.limit);
|
|
15
|
+
const o = clampOffset(input.offset);
|
|
16
|
+
return o > 0 ? `LIMIT ${l} OFFSET ${o}` : `LIMIT ${l}`;
|
|
17
|
+
}
|
|
18
|
+
function paginateInMemory(rows, input) {
|
|
19
|
+
const l = clampLimit(input.limit, rows.length);
|
|
20
|
+
const o = clampOffset(input.offset);
|
|
21
|
+
return rows.slice(o, o + l);
|
|
22
|
+
}
|
|
23
|
+
function paginateSortedInMemory(rows, input, compare) {
|
|
24
|
+
const limit = clampLimit(input.limit, rows.length);
|
|
25
|
+
const offset = clampOffset(input.offset);
|
|
26
|
+
const selectedCount = Math.min(rows.length, offset + limit);
|
|
27
|
+
if (limit === 0 || offset >= rows.length || selectedCount === 0) return [];
|
|
28
|
+
const fullSort = () => [...rows].sort(compare).slice(offset, offset + limit);
|
|
29
|
+
if (selectedCount >= rows.length / 2) return fullSort();
|
|
30
|
+
let invalidComparison = false;
|
|
31
|
+
const compareStable = (left, right) => {
|
|
32
|
+
const order = compare(left.value, right.value);
|
|
33
|
+
if (Number.isNaN(order)) {
|
|
34
|
+
invalidComparison = true;
|
|
35
|
+
return left.index - right.index;
|
|
36
|
+
}
|
|
37
|
+
return order || left.index - right.index;
|
|
38
|
+
};
|
|
39
|
+
const isWorse = (left, right) => compareStable(left, right) > 0;
|
|
40
|
+
const heap = [];
|
|
41
|
+
const siftUp = (start) => {
|
|
42
|
+
let index = start;
|
|
43
|
+
while (index > 0) {
|
|
44
|
+
const parent = index - 1 >>> 1;
|
|
45
|
+
if (!isWorse(heap[index], heap[parent])) break;
|
|
46
|
+
const swap = heap[parent];
|
|
47
|
+
heap[parent] = heap[index];
|
|
48
|
+
heap[index] = swap;
|
|
49
|
+
index = parent;
|
|
50
|
+
}
|
|
51
|
+
};
|
|
52
|
+
const siftDown = () => {
|
|
53
|
+
let index = 0;
|
|
54
|
+
for (;;) {
|
|
55
|
+
const left = index * 2 + 1;
|
|
56
|
+
if (left >= heap.length) return;
|
|
57
|
+
const right = left + 1;
|
|
58
|
+
let worse = left;
|
|
59
|
+
if (right < heap.length && isWorse(heap[right], heap[left])) worse = right;
|
|
60
|
+
if (!isWorse(heap[worse], heap[index])) return;
|
|
61
|
+
const swap = heap[index];
|
|
62
|
+
heap[index] = heap[worse];
|
|
63
|
+
heap[worse] = swap;
|
|
64
|
+
index = worse;
|
|
65
|
+
}
|
|
66
|
+
};
|
|
67
|
+
for (let index = 0; index < rows.length; index++) {
|
|
68
|
+
const candidate = {
|
|
69
|
+
value: rows[index],
|
|
70
|
+
index
|
|
71
|
+
};
|
|
72
|
+
if (heap.length < selectedCount) {
|
|
73
|
+
heap.push(candidate);
|
|
74
|
+
siftUp(heap.length - 1);
|
|
75
|
+
} else if (compareStable(candidate, heap[0]) < 0) {
|
|
76
|
+
heap[0] = candidate;
|
|
77
|
+
siftDown();
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
if (invalidComparison) return fullSort();
|
|
81
|
+
heap.sort(compareStable);
|
|
82
|
+
if (invalidComparison) return fullSort();
|
|
83
|
+
return heap.slice(offset, offset + limit).map((entry) => entry.value);
|
|
84
|
+
}
|
|
85
|
+
export { clampLimit, clampOffset, paginateClause, paginateInMemory, paginateSortedInMemory };
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
function rowNumber(value) {
|
|
2
|
+
if (typeof value === "number") return Number.isFinite(value) ? value : 0;
|
|
3
|
+
if (typeof value === "bigint") return Number(value);
|
|
4
|
+
if (value == null) return 0;
|
|
5
|
+
const n = Number(value);
|
|
6
|
+
return Number.isFinite(n) ? n : 0;
|
|
7
|
+
}
|
|
8
|
+
function rowString(value) {
|
|
9
|
+
return value == null ? "" : String(value);
|
|
10
|
+
}
|
|
11
|
+
function rowBoolean(value) {
|
|
12
|
+
return value === true || value === 1 || value === "true";
|
|
13
|
+
}
|
|
14
|
+
function parseJsonRows(value) {
|
|
15
|
+
if (Array.isArray(value)) return value;
|
|
16
|
+
if (typeof value === "string" && value.length > 0) {
|
|
17
|
+
const parsed = JSON.parse(value);
|
|
18
|
+
return Array.isArray(parsed) ? parsed : [];
|
|
19
|
+
}
|
|
20
|
+
return [];
|
|
21
|
+
}
|
|
22
|
+
export { parseJsonRows, rowBoolean, rowNumber, rowString };
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
import { rowNumber, rowString } from "../analyzer/row-values.mjs";
|
|
2
|
+
import { defineAnalyzer } from "@gscdump/engine/analyzer";
|
|
3
|
+
import { periodOf } from "@gscdump/engine/period";
|
|
4
|
+
import { enumeratePartitions } from "@gscdump/engine/planner";
|
|
5
|
+
import { METRIC_EXPR } from "@gscdump/engine/sql-fragments";
|
|
6
|
+
const bayesianCtrAnalyzer = defineAnalyzer({
|
|
7
|
+
id: "bayesian-ctr",
|
|
8
|
+
buildSql(params) {
|
|
9
|
+
const { startDate, endDate } = periodOf(params);
|
|
10
|
+
const minImpressions = params.minImpressions ?? 50;
|
|
11
|
+
const limit = params.limit ?? 300;
|
|
12
|
+
const priorMinEntities = 5;
|
|
13
|
+
return {
|
|
14
|
+
sql: `
|
|
15
|
+
WITH entity AS (
|
|
16
|
+
SELECT
|
|
17
|
+
query,
|
|
18
|
+
url,
|
|
19
|
+
${METRIC_EXPR.clicks} AS clicks,
|
|
20
|
+
${METRIC_EXPR.impressions} AS impressions,
|
|
21
|
+
${METRIC_EXPR.ctr} AS observed_ctr,
|
|
22
|
+
${METRIC_EXPR.position} AS position,
|
|
23
|
+
CAST(ROUND(LEAST(${METRIC_EXPR.position}, 30)) AS INTEGER) AS bucket
|
|
24
|
+
FROM read_parquet({{FILES}}, union_by_name = true)
|
|
25
|
+
WHERE date >= ? AND date <= ?
|
|
26
|
+
AND query IS NOT NULL AND query <> ''
|
|
27
|
+
AND url IS NOT NULL AND url <> ''
|
|
28
|
+
GROUP BY query, url
|
|
29
|
+
HAVING SUM(impressions) >= ?
|
|
30
|
+
AND ${METRIC_EXPR.position} <= 30
|
|
31
|
+
),
|
|
32
|
+
bucket_mu AS (
|
|
33
|
+
SELECT
|
|
34
|
+
bucket,
|
|
35
|
+
COUNT(*) AS n_entities,
|
|
36
|
+
SUM(observed_ctr * impressions) / NULLIF(SUM(impressions), 0) AS mu,
|
|
37
|
+
SUM(impressions) AS total_impressions
|
|
38
|
+
FROM entity
|
|
39
|
+
GROUP BY bucket
|
|
40
|
+
),
|
|
41
|
+
bucket_var AS (
|
|
42
|
+
SELECT
|
|
43
|
+
e.bucket,
|
|
44
|
+
GREATEST(
|
|
45
|
+
SUM(e.impressions * POWER(e.observed_ctr - b.mu, 2))
|
|
46
|
+
/ NULLIF(SUM(e.impressions), 0),
|
|
47
|
+
1e-9
|
|
48
|
+
) AS v
|
|
49
|
+
FROM entity e
|
|
50
|
+
JOIN bucket_mu b USING (bucket)
|
|
51
|
+
GROUP BY e.bucket
|
|
52
|
+
),
|
|
53
|
+
priors AS (
|
|
54
|
+
SELECT
|
|
55
|
+
m.bucket,
|
|
56
|
+
m.n_entities,
|
|
57
|
+
m.mu,
|
|
58
|
+
v.v,
|
|
59
|
+
CASE
|
|
60
|
+
WHEN m.n_entities >= ${Number(priorMinEntities)}
|
|
61
|
+
AND v.v > 0
|
|
62
|
+
AND m.mu > 0 AND m.mu < 1
|
|
63
|
+
AND (m.mu * (1.0 - m.mu) / v.v - 1.0) > 0
|
|
64
|
+
THEN GREATEST(0.5, m.mu * (m.mu * (1.0 - m.mu) / v.v - 1.0))
|
|
65
|
+
ELSE 2.0
|
|
66
|
+
END AS alpha,
|
|
67
|
+
CASE
|
|
68
|
+
WHEN m.n_entities >= ${Number(priorMinEntities)}
|
|
69
|
+
AND v.v > 0
|
|
70
|
+
AND m.mu > 0 AND m.mu < 1
|
|
71
|
+
AND (m.mu * (1.0 - m.mu) / v.v - 1.0) > 0
|
|
72
|
+
THEN GREATEST(0.5, (1.0 - m.mu) * (m.mu * (1.0 - m.mu) / v.v - 1.0))
|
|
73
|
+
ELSE 48.0
|
|
74
|
+
END AS beta
|
|
75
|
+
FROM bucket_mu m
|
|
76
|
+
JOIN bucket_var v USING (bucket)
|
|
77
|
+
),
|
|
78
|
+
posterior AS (
|
|
79
|
+
SELECT
|
|
80
|
+
e.query,
|
|
81
|
+
e.url,
|
|
82
|
+
e.clicks,
|
|
83
|
+
e.impressions,
|
|
84
|
+
e.observed_ctr,
|
|
85
|
+
e.position,
|
|
86
|
+
e.bucket,
|
|
87
|
+
p.alpha AS prior_alpha,
|
|
88
|
+
p.beta AS prior_beta,
|
|
89
|
+
p.mu AS bucket_prior_mean,
|
|
90
|
+
p.alpha + e.clicks AS alpha_post,
|
|
91
|
+
p.beta + (e.impressions - e.clicks) AS beta_post
|
|
92
|
+
FROM entity e
|
|
93
|
+
JOIN priors p USING (bucket)
|
|
94
|
+
),
|
|
95
|
+
scored AS (
|
|
96
|
+
SELECT *,
|
|
97
|
+
alpha_post / (alpha_post + beta_post) AS posterior_mean,
|
|
98
|
+
SQRT((alpha_post * beta_post)
|
|
99
|
+
/ (POWER(alpha_post + beta_post, 2) * (alpha_post + beta_post + 1))) AS posterior_sd
|
|
100
|
+
FROM posterior
|
|
101
|
+
)
|
|
102
|
+
SELECT
|
|
103
|
+
query AS keyword,
|
|
104
|
+
url AS page,
|
|
105
|
+
clicks,
|
|
106
|
+
impressions,
|
|
107
|
+
observed_ctr AS observedCtr,
|
|
108
|
+
position,
|
|
109
|
+
bucket,
|
|
110
|
+
prior_alpha AS priorAlpha,
|
|
111
|
+
prior_beta AS priorBeta,
|
|
112
|
+
bucket_prior_mean AS bucketPriorMean,
|
|
113
|
+
posterior_mean AS posteriorMean,
|
|
114
|
+
posterior_sd AS posteriorSd,
|
|
115
|
+
GREATEST(0.0, posterior_mean - 1.96 * posterior_sd) AS ciLow,
|
|
116
|
+
LEAST(1.0, posterior_mean + 1.96 * posterior_sd) AS ciHigh,
|
|
117
|
+
posterior_mean - observed_ctr AS shrinkageDelta,
|
|
118
|
+
(posterior_mean - observed_ctr) * impressions AS expectedClicksDelta,
|
|
119
|
+
ABS(observed_ctr - posterior_mean) / NULLIF(posterior_sd, 0) AS significance,
|
|
120
|
+
CASE
|
|
121
|
+
WHEN observed_ctr > LEAST(1.0, posterior_mean + 1.96 * posterior_sd) THEN 'overperforming'
|
|
122
|
+
WHEN observed_ctr < GREATEST(0.0, posterior_mean - 1.96 * posterior_sd) THEN 'underperforming'
|
|
123
|
+
ELSE 'expected'
|
|
124
|
+
END AS classification
|
|
125
|
+
FROM scored
|
|
126
|
+
ORDER BY significance DESC NULLS LAST
|
|
127
|
+
LIMIT ${Number(limit)}
|
|
128
|
+
`,
|
|
129
|
+
params: [
|
|
130
|
+
startDate,
|
|
131
|
+
endDate,
|
|
132
|
+
minImpressions
|
|
133
|
+
],
|
|
134
|
+
current: {
|
|
135
|
+
table: "page_queries",
|
|
136
|
+
partitions: enumeratePartitions(startDate, endDate)
|
|
137
|
+
}
|
|
138
|
+
};
|
|
139
|
+
},
|
|
140
|
+
reduceSql(rows, params) {
|
|
141
|
+
const arr = Array.isArray(rows) ? rows : [];
|
|
142
|
+
const minImpressions = params.minImpressions ?? 50;
|
|
143
|
+
const results = arr.map((r) => ({
|
|
144
|
+
keyword: rowString(r.keyword),
|
|
145
|
+
page: rowString(r.page),
|
|
146
|
+
clicks: rowNumber(r.clicks),
|
|
147
|
+
impressions: rowNumber(r.impressions),
|
|
148
|
+
observedCtr: rowNumber(r.observedCtr),
|
|
149
|
+
position: rowNumber(r.position),
|
|
150
|
+
bucket: rowNumber(r.bucket),
|
|
151
|
+
priorAlpha: rowNumber(r.priorAlpha),
|
|
152
|
+
priorBeta: rowNumber(r.priorBeta),
|
|
153
|
+
bucketPriorMean: rowNumber(r.bucketPriorMean),
|
|
154
|
+
posteriorMean: rowNumber(r.posteriorMean),
|
|
155
|
+
posteriorSd: rowNumber(r.posteriorSd),
|
|
156
|
+
ciLow: rowNumber(r.ciLow),
|
|
157
|
+
ciHigh: rowNumber(r.ciHigh),
|
|
158
|
+
shrinkageDelta: rowNumber(r.shrinkageDelta),
|
|
159
|
+
expectedClicksDelta: rowNumber(r.expectedClicksDelta),
|
|
160
|
+
significance: rowNumber(r.significance),
|
|
161
|
+
classification: rowString(r.classification)
|
|
162
|
+
}));
|
|
163
|
+
const under = results.filter((r) => r.classification === "underperforming").length;
|
|
164
|
+
const over = results.filter((r) => r.classification === "overperforming").length;
|
|
165
|
+
return {
|
|
166
|
+
results,
|
|
167
|
+
meta: {
|
|
168
|
+
total: results.length,
|
|
169
|
+
underperforming: under,
|
|
170
|
+
overperforming: over,
|
|
171
|
+
expected: results.length - under - over,
|
|
172
|
+
minImpressions
|
|
173
|
+
}
|
|
174
|
+
};
|
|
175
|
+
}
|
|
176
|
+
});
|
|
177
|
+
export { bayesianCtrAnalyzer };
|