@gscdump/analysis 1.4.0 → 1.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/action-priority.d.mts +34 -0
- package/dist/action-priority.mjs +201 -0
- package/dist/analyzer/adapt-rows.mjs +12 -0
- package/dist/analyzer/all.mjs +61 -0
- package/dist/analyzer/paginate.mjs +85 -0
- package/dist/analyzer/row-analyzers.d.mts +3 -0
- package/dist/analyzer/row-analyzers.mjs +3 -0
- package/dist/analyzer/row-values.mjs +22 -0
- package/dist/analyzers/bayesian-ctr.mjs +177 -0
- package/dist/analyzers/bipartite-pagerank.mjs +243 -0
- package/dist/analyzers/brand.d.mts +33 -0
- package/dist/analyzers/brand.mjs +154 -0
- package/dist/analyzers/cannibalization.d.mts +58 -0
- package/dist/analyzers/cannibalization.mjs +267 -0
- package/dist/analyzers/change-point.mjs +197 -0
- package/dist/analyzers/clustering.d.mts +29 -0
- package/dist/analyzers/clustering.mjs +217 -0
- package/dist/analyzers/concentration.d.mts +42 -0
- package/dist/analyzers/concentration.mjs +180 -0
- package/dist/analyzers/content-velocity.mjs +87 -0
- package/dist/analyzers/ctr-anomaly.mjs +189 -0
- package/dist/analyzers/ctr-curve.mjs +128 -0
- package/dist/analyzers/dark-traffic.mjs +130 -0
- package/dist/analyzers/data-detail.mjs +44 -0
- package/dist/analyzers/data-query.mjs +44 -0
- package/dist/analyzers/decay.d.mts +38 -0
- package/dist/analyzers/decay.mjs +187 -0
- package/dist/analyzers/device-gap.mjs +135 -0
- package/dist/analyzers/intent-atlas.mjs +175 -0
- package/dist/analyzers/keyword-breadth.mjs +115 -0
- package/dist/analyzers/long-tail.mjs +150 -0
- package/dist/analyzers/movers.d.mts +43 -0
- package/dist/analyzers/movers.mjs +283 -0
- package/dist/analyzers/opportunity.d.mts +32 -0
- package/dist/analyzers/opportunity.mjs +212 -0
- package/dist/analyzers/position-distribution.mjs +57 -0
- package/dist/analyzers/position-volatility.mjs +143 -0
- package/dist/analyzers/query-migration.mjs +178 -0
- package/dist/analyzers/seasonality.d.mts +28 -0
- package/dist/analyzers/seasonality.mjs +136 -0
- package/dist/analyzers/stl-decompose.mjs +187 -0
- package/dist/analyzers/striking-distance.d.mts +54 -0
- package/dist/analyzers/striking-distance.mjs +82 -0
- package/dist/analyzers/survival.mjs +200 -0
- package/dist/analyzers/trends.mjs +156 -0
- package/dist/analyzers/zero-click.d.mts +17 -0
- package/dist/analyzers/zero-click.mjs +122 -0
- package/dist/browser.d.mts +5 -0
- package/dist/browser.mjs +11 -0
- package/dist/default-registry.d.mts +3 -1
- package/dist/default-registry.mjs +5 -1
- package/dist/errors.d.mts +51 -1
- package/dist/index.d.mts +30 -494
- package/dist/index.mjs +28 -49
- package/dist/query/analyzers.mjs +301 -0
- package/dist/query/index.mjs +4 -0
- package/dist/query/intent.d.mts +20 -0
- package/dist/query/intent.mjs +139 -0
- package/dist/query/normalize.d.mts +15 -0
- package/dist/query/normalize.mjs +204 -0
- package/dist/report/format.d.mts +7 -0
- package/dist/report/format.mjs +45 -0
- package/dist/report/index.d.mts +4 -1
- package/dist/report/index.mjs +4 -1
- package/dist/report/registry.d.mts +4 -0
- package/dist/report/registry.mjs +26 -0
- package/dist/report/reports/brand.mjs +104 -0
- package/dist/report/reports/growth.mjs +144 -0
- package/dist/report/reports/health.mjs +141 -0
- package/dist/report/reports/movers.mjs +203 -0
- package/dist/report/reports/opportunities.mjs +184 -0
- package/dist/report/reports/pre-publish.mjs +111 -0
- package/dist/report/reports/priority.mjs +97 -0
- package/dist/report/reports/risks.mjs +201 -0
- package/dist/report/reports/triage.mjs +155 -0
- package/dist/report/require.mjs +15 -0
- package/dist/report/resolve-target.d.mts +29 -0
- package/dist/report/resolve-target.mjs +32 -0
- package/dist/{_chunks/index.d.mts → report/runtime.d.mts} +2 -37
- package/dist/report/runtime.mjs +80 -0
- package/dist/report/sections.mjs +22 -0
- package/dist/sitemap-health.d.mts +34 -0
- package/dist/sitemap-health.mjs +37 -0
- package/dist/{_chunks/index2.d.mts → source/composite.d.mts} +2 -13
- package/dist/{_chunks/source.mjs → source/composite.mjs} +1 -17
- package/dist/source/in-memory.d.mts +14 -0
- package/dist/source/in-memory.mjs +17 -0
- package/dist/source/index.d.mts +2 -1
- package/dist/source/index.mjs +2 -1
- package/dist/sql-analyzers.d.mts +3 -0
- package/dist/sql-analyzers.mjs +3 -0
- package/dist/types.d.mts +34 -0
- package/dist/types.mjs +21 -0
- package/package.json +4 -4
- package/dist/_chunks/default-registry.d.mts +0 -5
- package/dist/_chunks/default-registry.mjs +0 -5183
- package/dist/_chunks/errors.d.mts +0 -52
- package/dist/_chunks/report.mjs +0 -1710
- /package/dist/{_chunks/scoring.mjs → scoring.mjs} +0 -0
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { PageRow, QueriesRow } from "../types.mjs";
|
|
2
|
+
type ConcentrationRiskLevel = 'low' | 'medium' | 'high';
|
|
3
|
+
interface ConcentrationOptions {
|
|
4
|
+
/** Number of top items to report. Default: 10 */
|
|
5
|
+
topN?: number;
|
|
6
|
+
}
|
|
7
|
+
interface ConcentrationItem {
|
|
8
|
+
key: string;
|
|
9
|
+
clicks: number;
|
|
10
|
+
share: number;
|
|
11
|
+
}
|
|
12
|
+
interface ConcentrationInput {
|
|
13
|
+
key: string;
|
|
14
|
+
clicks: number;
|
|
15
|
+
}
|
|
16
|
+
interface ConcentrationResult {
|
|
17
|
+
/** Gini coefficient: 0 = equal distribution, 1 = fully concentrated */
|
|
18
|
+
giniCoefficient: number;
|
|
19
|
+
/** Herfindahl-Hirschman Index: 0-10000, >2500 = highly concentrated */
|
|
20
|
+
hhi: number;
|
|
21
|
+
/** Percentage of total clicks from top N items */
|
|
22
|
+
topNConcentration: number;
|
|
23
|
+
topNItems: ConcentrationItem[];
|
|
24
|
+
totalItems: number;
|
|
25
|
+
totalClicks: number;
|
|
26
|
+
/** Risk level derived from HHI: <1500 low, 1500-2500 medium, >2500 high */
|
|
27
|
+
riskLevel: ConcentrationRiskLevel;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Pure helper: analyze traffic concentration across items (pages or keywords).
|
|
31
|
+
* Re-exported from `@gscdump/analysis` for portable callers.
|
|
32
|
+
*/
|
|
33
|
+
declare function analyzeConcentration(items: ConcentrationInput[], options?: ConcentrationOptions): ConcentrationResult;
|
|
34
|
+
/**
|
|
35
|
+
* Page concentration analysis.
|
|
36
|
+
*/
|
|
37
|
+
declare function analyzePageConcentration(pages: PageRow[], options?: ConcentrationOptions): ConcentrationResult;
|
|
38
|
+
/**
|
|
39
|
+
* Keyword concentration analysis.
|
|
40
|
+
*/
|
|
41
|
+
declare function analyzeKeywordConcentration(keywords: QueriesRow[], options?: ConcentrationOptions): ConcentrationResult;
|
|
42
|
+
export { ConcentrationInput, ConcentrationItem, ConcentrationOptions, ConcentrationResult, ConcentrationRiskLevel, analyzeConcentration, analyzeKeywordConcentration, analyzePageConcentration };
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
import { parseJsonRows, rowString } from "../analyzer/row-values.mjs";
|
|
2
|
+
import { pagesQueryState, queriesQueryState } from "../analyzer/adapt-rows.mjs";
|
|
3
|
+
import { defineAnalyzer } from "@gscdump/engine/analyzer";
|
|
4
|
+
import { periodOf } from "@gscdump/engine/period";
|
|
5
|
+
import { enumeratePartitions } from "@gscdump/engine/planner";
|
|
6
|
+
import { METRIC_EXPR } from "@gscdump/engine/sql-fragments";
|
|
7
|
+
import { num } from "@gscdump/engine/analysis-types";
|
|
8
|
+
function analyzeConcentration(items, options = {}) {
|
|
9
|
+
const { topN = 10 } = options;
|
|
10
|
+
if (items.length === 0) return {
|
|
11
|
+
giniCoefficient: 0,
|
|
12
|
+
hhi: 0,
|
|
13
|
+
topNConcentration: 0,
|
|
14
|
+
topNItems: [],
|
|
15
|
+
totalItems: 0,
|
|
16
|
+
totalClicks: 0,
|
|
17
|
+
riskLevel: "low"
|
|
18
|
+
};
|
|
19
|
+
const sorted = [...items].sort((a, b) => b.clicks - a.clicks);
|
|
20
|
+
const n = sorted.length;
|
|
21
|
+
let totalClicks = 0;
|
|
22
|
+
let squaredClicks = 0;
|
|
23
|
+
let weightedSum = 0;
|
|
24
|
+
for (let i = 0; i < n; i++) {
|
|
25
|
+
const clicks = sorted[i].clicks;
|
|
26
|
+
totalClicks += clicks;
|
|
27
|
+
squaredClicks += clicks * clicks;
|
|
28
|
+
weightedSum += (n - 2 * i - 1) * clicks;
|
|
29
|
+
}
|
|
30
|
+
const giniCoefficient = totalClicks === 0 ? 0 : weightedSum / (n * totalClicks);
|
|
31
|
+
const hhi = totalClicks > 0 ? squaredClicks / (totalClicks * totalClicks) * 1e4 : 0;
|
|
32
|
+
const topNItems = sorted.slice(0, topN).map((item) => ({
|
|
33
|
+
key: item.key,
|
|
34
|
+
clicks: item.clicks,
|
|
35
|
+
share: totalClicks > 0 ? item.clicks / totalClicks : 0
|
|
36
|
+
}));
|
|
37
|
+
const topNClicks = topNItems.reduce((sum, item) => sum + item.clicks, 0);
|
|
38
|
+
const topNConcentration = totalClicks > 0 ? topNClicks / totalClicks : 0;
|
|
39
|
+
let riskLevel = "low";
|
|
40
|
+
if (hhi > 2500) riskLevel = "high";
|
|
41
|
+
else if (hhi > 1500) riskLevel = "medium";
|
|
42
|
+
return {
|
|
43
|
+
giniCoefficient,
|
|
44
|
+
hhi,
|
|
45
|
+
topNConcentration,
|
|
46
|
+
topNItems,
|
|
47
|
+
totalItems: items.length,
|
|
48
|
+
totalClicks,
|
|
49
|
+
riskLevel
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
function analyzePageConcentration(pages, options) {
|
|
53
|
+
return analyzeConcentration(pages.map((p) => ({
|
|
54
|
+
key: p.page,
|
|
55
|
+
clicks: num(p.clicks)
|
|
56
|
+
})), options);
|
|
57
|
+
}
|
|
58
|
+
function analyzeKeywordConcentration(keywords, options) {
|
|
59
|
+
return analyzeConcentration(keywords.map((k) => ({
|
|
60
|
+
key: k.query,
|
|
61
|
+
clicks: num(k.clicks)
|
|
62
|
+
})), options);
|
|
63
|
+
}
|
|
64
|
+
const concentrationAnalyzer = defineAnalyzer({
|
|
65
|
+
id: "concentration",
|
|
66
|
+
buildSql(params) {
|
|
67
|
+
const { startDate, endDate } = periodOf(params);
|
|
68
|
+
const dim = params.dimension || "pages";
|
|
69
|
+
const topN = params.topN ?? 10;
|
|
70
|
+
const table = dim === "keywords" ? "queries" : "pages";
|
|
71
|
+
const keyCol = dim === "keywords" ? "query" : "url";
|
|
72
|
+
return {
|
|
73
|
+
sql: `
|
|
74
|
+
WITH items AS (
|
|
75
|
+
SELECT
|
|
76
|
+
${keyCol} AS key,
|
|
77
|
+
${METRIC_EXPR.clicks} AS clicks
|
|
78
|
+
FROM read_parquet({{FILES}}, union_by_name = true)
|
|
79
|
+
WHERE date >= ? AND date <= ?
|
|
80
|
+
GROUP BY ${keyCol}
|
|
81
|
+
HAVING SUM(clicks) > 0
|
|
82
|
+
),
|
|
83
|
+
totals AS (
|
|
84
|
+
SELECT SUM(clicks) AS total_clicks, COUNT(*) AS total_items FROM items
|
|
85
|
+
),
|
|
86
|
+
ranked AS (
|
|
87
|
+
SELECT
|
|
88
|
+
i.key, i.clicks,
|
|
89
|
+
i.clicks / NULLIF(t.total_clicks, 0) AS share,
|
|
90
|
+
ROW_NUMBER() OVER (ORDER BY i.clicks DESC, i.key ASC) AS rnk_desc,
|
|
91
|
+
ROW_NUMBER() OVER (ORDER BY i.clicks ASC, i.key ASC) AS rnk_asc,
|
|
92
|
+
t.total_clicks AS tclicks,
|
|
93
|
+
t.total_items AS titems
|
|
94
|
+
FROM items i, totals t
|
|
95
|
+
),
|
|
96
|
+
gini_num AS (
|
|
97
|
+
SELECT SUM((2.0 * rnk_asc - titems - 1) * clicks) AS weighted_sum FROM ranked
|
|
98
|
+
),
|
|
99
|
+
hhi_calc AS (
|
|
100
|
+
SELECT SUM(POWER(share * 100, 2)) AS hhi FROM ranked
|
|
101
|
+
),
|
|
102
|
+
top_list AS (
|
|
103
|
+
SELECT
|
|
104
|
+
list({ 'key': key, 'clicks': clicks, 'share': share } ORDER BY clicks DESC, key ASC) AS items,
|
|
105
|
+
SUM(clicks) AS top_clicks
|
|
106
|
+
FROM ranked WHERE rnk_desc <= ?
|
|
107
|
+
)
|
|
108
|
+
SELECT
|
|
109
|
+
COALESCE(
|
|
110
|
+
(SELECT weighted_sum FROM gini_num)
|
|
111
|
+
/ NULLIF((SELECT total_items FROM totals) * (SELECT total_clicks FROM totals), 0),
|
|
112
|
+
0.0
|
|
113
|
+
) AS giniCoefficient,
|
|
114
|
+
COALESCE((SELECT hhi FROM hhi_calc), 0.0) AS hhi,
|
|
115
|
+
COALESCE(
|
|
116
|
+
CAST((SELECT top_clicks FROM top_list) AS DOUBLE)
|
|
117
|
+
/ NULLIF((SELECT total_clicks FROM totals), 0),
|
|
118
|
+
0.0
|
|
119
|
+
) AS topNConcentration,
|
|
120
|
+
COALESCE((SELECT to_json(items) FROM top_list), '[]') AS topNItems,
|
|
121
|
+
COALESCE((SELECT total_items FROM totals), 0) AS totalItems,
|
|
122
|
+
COALESCE((SELECT total_clicks FROM totals), 0.0) AS totalClicks,
|
|
123
|
+
CASE
|
|
124
|
+
WHEN COALESCE((SELECT hhi FROM hhi_calc), 0.0) > 2500 THEN 'high'
|
|
125
|
+
WHEN COALESCE((SELECT hhi FROM hhi_calc), 0.0) > 1500 THEN 'medium'
|
|
126
|
+
ELSE 'low'
|
|
127
|
+
END AS riskLevel
|
|
128
|
+
`,
|
|
129
|
+
params: [
|
|
130
|
+
startDate,
|
|
131
|
+
endDate,
|
|
132
|
+
topN
|
|
133
|
+
],
|
|
134
|
+
current: {
|
|
135
|
+
table,
|
|
136
|
+
partitions: enumeratePartitions(startDate, endDate)
|
|
137
|
+
}
|
|
138
|
+
};
|
|
139
|
+
},
|
|
140
|
+
reduceSql(rows, params) {
|
|
141
|
+
const r = (Array.isArray(rows) ? rows : [])[0] ?? {};
|
|
142
|
+
const topRaw = parseJsonRows(r.topNItems);
|
|
143
|
+
return {
|
|
144
|
+
results: [{
|
|
145
|
+
giniCoefficient: num(r.giniCoefficient),
|
|
146
|
+
hhi: num(r.hhi),
|
|
147
|
+
topNConcentration: num(r.topNConcentration),
|
|
148
|
+
topNItems: topRaw.map((t) => ({
|
|
149
|
+
key: rowString(t.key),
|
|
150
|
+
clicks: num(t.clicks),
|
|
151
|
+
share: num(t.share)
|
|
152
|
+
})),
|
|
153
|
+
totalItems: num(r.totalItems),
|
|
154
|
+
totalClicks: num(r.totalClicks),
|
|
155
|
+
riskLevel: rowString(r.riskLevel)
|
|
156
|
+
}],
|
|
157
|
+
meta: {
|
|
158
|
+
total: 1,
|
|
159
|
+
dimension: params.dimension || "pages"
|
|
160
|
+
}
|
|
161
|
+
};
|
|
162
|
+
},
|
|
163
|
+
buildRows(params) {
|
|
164
|
+
const dim = params.dimension || "pages";
|
|
165
|
+
const period = periodOf(params);
|
|
166
|
+
const out = {};
|
|
167
|
+
if (dim === "pages") out.pages = pagesQueryState(period, params.limit);
|
|
168
|
+
else out.queries = queriesQueryState(period, params.limit);
|
|
169
|
+
return out;
|
|
170
|
+
},
|
|
171
|
+
reduceRows(rows, params) {
|
|
172
|
+
const dim = params.dimension || "pages";
|
|
173
|
+
const arr = Array.isArray(rows) ? rows : rows[dim === "pages" ? "pages" : "queries"] ?? [];
|
|
174
|
+
return {
|
|
175
|
+
results: [dim === "pages" ? analyzePageConcentration(arr, { topN: params.topN }) : analyzeKeywordConcentration(arr, { topN: params.topN })],
|
|
176
|
+
meta: { dimension: dim }
|
|
177
|
+
};
|
|
178
|
+
}
|
|
179
|
+
});
|
|
180
|
+
export { analyzeConcentration, analyzeKeywordConcentration, analyzePageConcentration, concentrationAnalyzer };
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import { rowNumber, rowString } from "../analyzer/row-values.mjs";
|
|
2
|
+
import { defineAnalyzer } from "@gscdump/engine/analyzer";
|
|
3
|
+
import { periodOf } from "@gscdump/engine/period";
|
|
4
|
+
import { enumeratePartitions } from "@gscdump/engine/planner";
|
|
5
|
+
import { toIsoDate } from "gscdump/dates";
|
|
6
|
+
const contentVelocityAnalyzer = defineAnalyzer({
|
|
7
|
+
id: "content-velocity",
|
|
8
|
+
buildSql(params) {
|
|
9
|
+
const days = Math.min(Math.max(Number(params.days ?? 90), 7), 365);
|
|
10
|
+
const { endDate } = periodOf(params);
|
|
11
|
+
const start = new Date(endDate);
|
|
12
|
+
start.setUTCDate(start.getUTCDate() - days);
|
|
13
|
+
const startDate = toIsoDate(start);
|
|
14
|
+
return {
|
|
15
|
+
sql: `
|
|
16
|
+
WITH src AS (
|
|
17
|
+
SELECT query, date
|
|
18
|
+
FROM read_parquet({{FILES}}, union_by_name = true)
|
|
19
|
+
WHERE date >= ? AND date <= ? AND impressions > 0
|
|
20
|
+
),
|
|
21
|
+
first_seen AS (
|
|
22
|
+
SELECT query, MIN(date) AS first_date FROM src GROUP BY query
|
|
23
|
+
),
|
|
24
|
+
per_week AS (
|
|
25
|
+
SELECT
|
|
26
|
+
strftime(CAST(date AS DATE), '%G-W%V') AS week,
|
|
27
|
+
MIN(date) AS week_start,
|
|
28
|
+
CAST(COUNT(DISTINCT query) AS DOUBLE) AS totalKeywords
|
|
29
|
+
FROM src
|
|
30
|
+
GROUP BY week
|
|
31
|
+
),
|
|
32
|
+
new_per_week AS (
|
|
33
|
+
SELECT
|
|
34
|
+
strftime(CAST(first_date AS DATE), '%G-W%V') AS week,
|
|
35
|
+
CAST(COUNT(*) AS DOUBLE) AS newKeywords
|
|
36
|
+
FROM first_seen
|
|
37
|
+
GROUP BY week
|
|
38
|
+
)
|
|
39
|
+
SELECT
|
|
40
|
+
pw.week AS week,
|
|
41
|
+
COALESCE(npw.newKeywords, 0) AS newKeywords,
|
|
42
|
+
pw.totalKeywords AS totalKeywords
|
|
43
|
+
FROM per_week pw
|
|
44
|
+
LEFT JOIN new_per_week npw ON pw.week = npw.week
|
|
45
|
+
ORDER BY pw.week ASC
|
|
46
|
+
`,
|
|
47
|
+
params: [startDate, endDate],
|
|
48
|
+
current: {
|
|
49
|
+
table: "queries",
|
|
50
|
+
partitions: enumeratePartitions(startDate, endDate)
|
|
51
|
+
}
|
|
52
|
+
};
|
|
53
|
+
},
|
|
54
|
+
reduceSql(rows, params) {
|
|
55
|
+
const arr = Array.isArray(rows) ? rows : [];
|
|
56
|
+
const days = Math.min(Math.max(Number(params.days ?? 90), 7), 365);
|
|
57
|
+
const { endDate } = periodOf(params);
|
|
58
|
+
const startDateD = new Date(endDate);
|
|
59
|
+
startDateD.setUTCDate(startDateD.getUTCDate() - days);
|
|
60
|
+
const startDate = toIsoDate(startDateD);
|
|
61
|
+
const weekly = arr.map((r) => ({
|
|
62
|
+
week: rowString(r.week),
|
|
63
|
+
newKeywords: rowNumber(r.newKeywords),
|
|
64
|
+
totalKeywords: rowNumber(r.totalKeywords)
|
|
65
|
+
}));
|
|
66
|
+
const total = weekly.reduce((s, w) => s + w.newKeywords, 0);
|
|
67
|
+
const avg = weekly.length > 0 ? total / weekly.length : 0;
|
|
68
|
+
const mid = Math.floor(weekly.length / 2);
|
|
69
|
+
const firstAvg = mid > 0 ? weekly.slice(0, mid).reduce((s, w) => s + w.newKeywords, 0) / mid : 0;
|
|
70
|
+
const diff = (weekly.length - mid > 0 ? weekly.slice(mid).reduce((s, w) => s + w.newKeywords, 0) / (weekly.length - mid) : 0) - firstAvg;
|
|
71
|
+
const threshold = Math.max(1, avg * .15);
|
|
72
|
+
return {
|
|
73
|
+
results: weekly,
|
|
74
|
+
meta: {
|
|
75
|
+
summary: {
|
|
76
|
+
totalNewKeywords: total,
|
|
77
|
+
avgPerWeek: avg,
|
|
78
|
+
trend: diff > threshold ? "accelerating" : diff < -threshold ? "decelerating" : "stable"
|
|
79
|
+
},
|
|
80
|
+
days,
|
|
81
|
+
startDate,
|
|
82
|
+
endDate
|
|
83
|
+
}
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
});
|
|
87
|
+
export { contentVelocityAnalyzer };
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
import { parseJsonRows, rowBoolean, rowNumber, rowString } from "../analyzer/row-values.mjs";
|
|
2
|
+
import { defineAnalyzer } from "@gscdump/engine/analyzer";
|
|
3
|
+
import { defaultEndDate } from "@gscdump/engine/period";
|
|
4
|
+
import { enumeratePartitions } from "@gscdump/engine/planner";
|
|
5
|
+
import { METRIC_EXPR } from "@gscdump/engine/sql-fragments";
|
|
6
|
+
import { daysAgoUtc } from "gscdump/dates";
|
|
7
|
+
const ctrAnomalyAnalyzer = defineAnalyzer({
|
|
8
|
+
id: "ctr-anomaly",
|
|
9
|
+
buildSql(params) {
|
|
10
|
+
const endDate = params.endDate ?? defaultEndDate();
|
|
11
|
+
const startDate = params.startDate ?? daysAgoUtc(93);
|
|
12
|
+
const minDailyImpressions = params.minImpressions ?? 5;
|
|
13
|
+
const minRollingN = 14;
|
|
14
|
+
const zThreshold = params.threshold ?? 2;
|
|
15
|
+
const maxPositionDelta = 1.5;
|
|
16
|
+
const minBreachDays = 2;
|
|
17
|
+
const limit = params.limit ?? 200;
|
|
18
|
+
return {
|
|
19
|
+
sql: `
|
|
20
|
+
WITH daily AS (
|
|
21
|
+
SELECT
|
|
22
|
+
query,
|
|
23
|
+
url AS page,
|
|
24
|
+
-- Normalize at the source CTE: union_by_name=true can coerce date to
|
|
25
|
+
-- VARCHAR across parquets with mixed schemas, which makes downstream
|
|
26
|
+
-- strftime(date, ...) binder-error.
|
|
27
|
+
CAST(date AS DATE) AS date,
|
|
28
|
+
${METRIC_EXPR.clicks} AS day_clicks,
|
|
29
|
+
${METRIC_EXPR.impressions} AS day_impressions,
|
|
30
|
+
${METRIC_EXPR.ctr} AS day_ctr,
|
|
31
|
+
${METRIC_EXPR.position} AS day_position
|
|
32
|
+
FROM read_parquet({{FILES}}, union_by_name = true)
|
|
33
|
+
WHERE date >= ? AND date <= ?
|
|
34
|
+
AND query IS NOT NULL AND query <> ''
|
|
35
|
+
AND url IS NOT NULL AND url <> ''
|
|
36
|
+
GROUP BY query, url, date
|
|
37
|
+
HAVING SUM(impressions) >= ?
|
|
38
|
+
),
|
|
39
|
+
rolled AS (
|
|
40
|
+
SELECT *,
|
|
41
|
+
AVG(day_ctr) OVER w AS rolling_ctr,
|
|
42
|
+
STDDEV_POP(day_ctr) OVER w AS rolling_stddev,
|
|
43
|
+
-- Impression-weighted rolling mean: day_position is already a recovered
|
|
44
|
+
-- 1-based daily weighted mean, so weighting by day impressions directly
|
|
45
|
+
-- is algebraically exact (no -1/+1 dance needed).
|
|
46
|
+
SUM(day_position * day_impressions) OVER w
|
|
47
|
+
/ NULLIF(SUM(day_impressions) OVER w, 0) AS rolling_position,
|
|
48
|
+
COUNT(*) OVER w AS rolling_n
|
|
49
|
+
FROM daily
|
|
50
|
+
WINDOW w AS (
|
|
51
|
+
PARTITION BY query, page
|
|
52
|
+
ORDER BY date
|
|
53
|
+
ROWS BETWEEN 28 PRECEDING AND 1 PRECEDING
|
|
54
|
+
)
|
|
55
|
+
),
|
|
56
|
+
flagged AS (
|
|
57
|
+
SELECT *,
|
|
58
|
+
CASE
|
|
59
|
+
WHEN rolling_n >= ${Number(minRollingN)} AND rolling_stddev > 0
|
|
60
|
+
THEN (day_ctr - rolling_ctr) / rolling_stddev
|
|
61
|
+
ELSE 0.0
|
|
62
|
+
END AS z_score,
|
|
63
|
+
CASE
|
|
64
|
+
WHEN rolling_position IS NULL THEN 0.0
|
|
65
|
+
ELSE ABS(day_position - rolling_position)
|
|
66
|
+
END AS position_delta
|
|
67
|
+
FROM rolled
|
|
68
|
+
),
|
|
69
|
+
breaches AS (
|
|
70
|
+
SELECT *,
|
|
71
|
+
CASE
|
|
72
|
+
WHEN ABS(z_score) >= ${zThreshold}
|
|
73
|
+
AND position_delta <= ${maxPositionDelta}
|
|
74
|
+
AND rolling_n >= ${Number(minRollingN)}
|
|
75
|
+
THEN true ELSE false
|
|
76
|
+
END AS is_breach
|
|
77
|
+
FROM flagged
|
|
78
|
+
),
|
|
79
|
+
per_entity AS (
|
|
80
|
+
SELECT
|
|
81
|
+
query, page,
|
|
82
|
+
COUNT(*) FILTER (WHERE is_breach AND z_score < 0) AS breach_days_down,
|
|
83
|
+
COUNT(*) FILTER (WHERE is_breach AND z_score > 0) AS breach_days_up,
|
|
84
|
+
SUM(CASE
|
|
85
|
+
WHEN is_breach AND z_score < 0
|
|
86
|
+
THEN (rolling_ctr - day_ctr) * day_impressions
|
|
87
|
+
ELSE 0.0
|
|
88
|
+
END) AS clicks_lost,
|
|
89
|
+
SUM(CASE
|
|
90
|
+
WHEN is_breach AND z_score < 0
|
|
91
|
+
THEN ABS(z_score) * day_impressions
|
|
92
|
+
ELSE 0.0
|
|
93
|
+
END) AS severity_raw,
|
|
94
|
+
MAX(CASE WHEN is_breach THEN ABS(z_score) ELSE 0.0 END) AS max_z,
|
|
95
|
+
AVG(rolling_ctr) FILTER (WHERE rolling_n >= ${Number(minRollingN)}) AS baseline_ctr,
|
|
96
|
+
SUM(rolling_position * day_impressions) FILTER (WHERE rolling_n >= ${Number(minRollingN)})
|
|
97
|
+
/ NULLIF(SUM(day_impressions) FILTER (WHERE rolling_n >= ${Number(minRollingN)}), 0) AS baseline_position,
|
|
98
|
+
SUM(day_impressions) AS total_impressions,
|
|
99
|
+
SUM(day_clicks) AS total_clicks
|
|
100
|
+
FROM breaches
|
|
101
|
+
GROUP BY query, page
|
|
102
|
+
HAVING COUNT(*) FILTER (WHERE is_breach AND z_score < 0) >= ${Number(minBreachDays)}
|
|
103
|
+
),
|
|
104
|
+
series AS (
|
|
105
|
+
SELECT query, page,
|
|
106
|
+
to_json(list({
|
|
107
|
+
'date': strftime(date, '%Y-%m-%d'),
|
|
108
|
+
'ctr': day_ctr,
|
|
109
|
+
'position': day_position,
|
|
110
|
+
'impressions': day_impressions,
|
|
111
|
+
'rollingCtr': rolling_ctr,
|
|
112
|
+
'rollingStddev': rolling_stddev,
|
|
113
|
+
'z': z_score,
|
|
114
|
+
'breach': is_breach AND z_score < 0
|
|
115
|
+
} ORDER BY date)) AS seriesJson
|
|
116
|
+
FROM breaches
|
|
117
|
+
GROUP BY query, page
|
|
118
|
+
)
|
|
119
|
+
SELECT
|
|
120
|
+
e.query AS keyword,
|
|
121
|
+
e.page,
|
|
122
|
+
CAST(e.breach_days_down AS DOUBLE) AS breachDaysDown,
|
|
123
|
+
CAST(e.breach_days_up AS DOUBLE) AS breachDaysUp,
|
|
124
|
+
CAST(ROUND(e.clicks_lost) AS DOUBLE) AS clicksLost,
|
|
125
|
+
e.severity_raw AS severityRaw,
|
|
126
|
+
e.max_z AS maxZ,
|
|
127
|
+
e.baseline_ctr AS baselineCtr,
|
|
128
|
+
e.baseline_position AS baselinePosition,
|
|
129
|
+
e.total_impressions AS totalImpressions,
|
|
130
|
+
e.total_clicks AS totalClicks,
|
|
131
|
+
s.seriesJson
|
|
132
|
+
FROM per_entity e
|
|
133
|
+
LEFT JOIN series s USING (query, page)
|
|
134
|
+
ORDER BY clicksLost DESC
|
|
135
|
+
LIMIT ${Number(limit)}
|
|
136
|
+
`,
|
|
137
|
+
params: [
|
|
138
|
+
startDate,
|
|
139
|
+
endDate,
|
|
140
|
+
minDailyImpressions
|
|
141
|
+
],
|
|
142
|
+
current: {
|
|
143
|
+
table: "page_queries",
|
|
144
|
+
partitions: enumeratePartitions(startDate, endDate)
|
|
145
|
+
}
|
|
146
|
+
};
|
|
147
|
+
},
|
|
148
|
+
reduceSql(rows, params) {
|
|
149
|
+
const arr = Array.isArray(rows) ? rows : [];
|
|
150
|
+
const minRollingN = 14;
|
|
151
|
+
const zThreshold = params.threshold ?? 2;
|
|
152
|
+
const anomalies = arr.map((r) => ({
|
|
153
|
+
keyword: rowString(r.keyword),
|
|
154
|
+
page: rowString(r.page),
|
|
155
|
+
breachDaysDown: rowNumber(r.breachDaysDown),
|
|
156
|
+
breachDaysUp: rowNumber(r.breachDaysUp),
|
|
157
|
+
clicksLost: rowNumber(r.clicksLost),
|
|
158
|
+
severity: rowNumber(r.severityRaw),
|
|
159
|
+
maxZ: rowNumber(r.maxZ),
|
|
160
|
+
baselineCtr: rowNumber(r.baselineCtr),
|
|
161
|
+
baselinePosition: rowNumber(r.baselinePosition),
|
|
162
|
+
totalImpressions: rowNumber(r.totalImpressions),
|
|
163
|
+
totalClicks: rowNumber(r.totalClicks),
|
|
164
|
+
series: parseJsonRows(r.seriesJson).map((s) => ({
|
|
165
|
+
date: rowString(s.date),
|
|
166
|
+
ctr: rowNumber(s.ctr),
|
|
167
|
+
position: rowNumber(s.position),
|
|
168
|
+
impressions: rowNumber(s.impressions),
|
|
169
|
+
rollingCtr: s.rollingCtr == null ? null : rowNumber(s.rollingCtr),
|
|
170
|
+
rollingStddev: s.rollingStddev == null ? null : rowNumber(s.rollingStddev),
|
|
171
|
+
z: rowNumber(s.z),
|
|
172
|
+
breach: rowBoolean(s.breach)
|
|
173
|
+
}))
|
|
174
|
+
}));
|
|
175
|
+
const totalClicksLost = anomalies.reduce((s, a) => s + a.clicksLost, 0);
|
|
176
|
+
const totalBreachDays = anomalies.reduce((s, a) => s + a.breachDaysDown, 0);
|
|
177
|
+
return {
|
|
178
|
+
results: anomalies,
|
|
179
|
+
meta: {
|
|
180
|
+
total: anomalies.length,
|
|
181
|
+
totalClicksLost,
|
|
182
|
+
totalBreachDays,
|
|
183
|
+
zThreshold,
|
|
184
|
+
minRollingN
|
|
185
|
+
}
|
|
186
|
+
};
|
|
187
|
+
}
|
|
188
|
+
});
|
|
189
|
+
export { ctrAnomalyAnalyzer };
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
import { parseJsonRows, rowNumber, rowString } from "../analyzer/row-values.mjs";
|
|
2
|
+
import { defineAnalyzer } from "@gscdump/engine/analyzer";
|
|
3
|
+
import { periodOf } from "@gscdump/engine/period";
|
|
4
|
+
import { enumeratePartitions } from "@gscdump/engine/planner";
|
|
5
|
+
import { METRIC_EXPR } from "@gscdump/engine/sql-fragments";
|
|
6
|
+
const ctrCurveAnalyzer = defineAnalyzer({
|
|
7
|
+
id: "ctr-curve",
|
|
8
|
+
buildSql(params) {
|
|
9
|
+
const { startDate, endDate } = periodOf(params);
|
|
10
|
+
return {
|
|
11
|
+
sql: `
|
|
12
|
+
WITH src AS (
|
|
13
|
+
SELECT
|
|
14
|
+
query,
|
|
15
|
+
clicks,
|
|
16
|
+
impressions,
|
|
17
|
+
sum_position,
|
|
18
|
+
(sum_position / NULLIF(impressions, 0) + 1) AS avg_pos
|
|
19
|
+
FROM read_parquet({{FILES}}, union_by_name = true)
|
|
20
|
+
WHERE date >= ? AND date <= ? AND impressions > 0
|
|
21
|
+
),
|
|
22
|
+
curve AS (
|
|
23
|
+
SELECT
|
|
24
|
+
CASE
|
|
25
|
+
WHEN avg_pos <= 1.5 THEN '1'
|
|
26
|
+
WHEN avg_pos <= 2.5 THEN '2'
|
|
27
|
+
WHEN avg_pos <= 3.5 THEN '3'
|
|
28
|
+
WHEN avg_pos <= 5.5 THEN '4-5'
|
|
29
|
+
WHEN avg_pos <= 10.5 THEN '6-10'
|
|
30
|
+
WHEN avg_pos <= 20.5 THEN '11-20'
|
|
31
|
+
ELSE '20+'
|
|
32
|
+
END AS bucket,
|
|
33
|
+
AVG(CAST(clicks AS DOUBLE) / NULLIF(impressions, 0)) AS avgCtr,
|
|
34
|
+
(SUM(sum_position) / NULLIF(SUM(impressions), 0) + 1) AS medianPosition,
|
|
35
|
+
CAST(COUNT(DISTINCT query) AS DOUBLE) AS keywordCount,
|
|
36
|
+
${METRIC_EXPR.clicks} AS totalClicks,
|
|
37
|
+
${METRIC_EXPR.impressions} AS totalImpressions
|
|
38
|
+
FROM src
|
|
39
|
+
GROUP BY bucket
|
|
40
|
+
),
|
|
41
|
+
ks AS (
|
|
42
|
+
SELECT
|
|
43
|
+
query,
|
|
44
|
+
${METRIC_EXPR.clicks} AS clicks,
|
|
45
|
+
${METRIC_EXPR.impressions} AS impressions,
|
|
46
|
+
${METRIC_EXPR.ctr} AS ctr,
|
|
47
|
+
${METRIC_EXPR.position} AS position,
|
|
48
|
+
CASE
|
|
49
|
+
WHEN ${METRIC_EXPR.position} <= 3.5 THEN 'top3'
|
|
50
|
+
WHEN ${METRIC_EXPR.position} <= 10.5 THEN 'page1'
|
|
51
|
+
WHEN ${METRIC_EXPR.position} <= 20.5 THEN 'page2'
|
|
52
|
+
ELSE 'deep'
|
|
53
|
+
END AS band
|
|
54
|
+
FROM src
|
|
55
|
+
GROUP BY query
|
|
56
|
+
HAVING SUM(impressions) >= 20
|
|
57
|
+
),
|
|
58
|
+
band_avg AS (
|
|
59
|
+
SELECT band, AVG(ctr) AS band_avg_ctr FROM ks GROUP BY band
|
|
60
|
+
),
|
|
61
|
+
outliers AS (
|
|
62
|
+
SELECT
|
|
63
|
+
ks.query, ks.clicks, ks.impressions, ks.ctr, ks.position,
|
|
64
|
+
ba.band_avg_ctr AS expectedCtr,
|
|
65
|
+
ks.ctr - ba.band_avg_ctr AS ctrDiff
|
|
66
|
+
FROM ks JOIN band_avg ba ON ks.band = ba.band
|
|
67
|
+
ORDER BY ABS(ks.ctr - ba.band_avg_ctr) DESC
|
|
68
|
+
LIMIT 50
|
|
69
|
+
)
|
|
70
|
+
SELECT
|
|
71
|
+
(SELECT to_json(list({
|
|
72
|
+
'bucket': bucket,
|
|
73
|
+
'avgCtr': avgCtr,
|
|
74
|
+
'medianPosition': medianPosition,
|
|
75
|
+
'keywordCount': keywordCount,
|
|
76
|
+
'totalClicks': totalClicks,
|
|
77
|
+
'totalImpressions': totalImpressions
|
|
78
|
+
})) FROM curve) AS curve_json,
|
|
79
|
+
(SELECT to_json(list({
|
|
80
|
+
'query': query,
|
|
81
|
+
'clicks': clicks,
|
|
82
|
+
'impressions': impressions,
|
|
83
|
+
'ctr': ctr,
|
|
84
|
+
'position': position,
|
|
85
|
+
'expectedCtr': expectedCtr,
|
|
86
|
+
'ctrDiff': ctrDiff
|
|
87
|
+
})) FROM outliers) AS outliers_json
|
|
88
|
+
`,
|
|
89
|
+
params: [startDate, endDate],
|
|
90
|
+
current: {
|
|
91
|
+
table: "queries",
|
|
92
|
+
partitions: enumeratePartitions(startDate, endDate)
|
|
93
|
+
}
|
|
94
|
+
};
|
|
95
|
+
},
|
|
96
|
+
reduceSql(rows, params) {
|
|
97
|
+
const arr = Array.isArray(rows) ? rows : [];
|
|
98
|
+
const { startDate, endDate } = periodOf(params);
|
|
99
|
+
const row = arr[0] ?? {};
|
|
100
|
+
const curve = parseJsonRows(row.curve_json).map((r) => ({
|
|
101
|
+
bucket: rowString(r.bucket),
|
|
102
|
+
avgCtr: rowNumber(r.avgCtr),
|
|
103
|
+
medianPosition: rowNumber(r.medianPosition),
|
|
104
|
+
keywordCount: rowNumber(r.keywordCount),
|
|
105
|
+
totalClicks: rowNumber(r.totalClicks),
|
|
106
|
+
totalImpressions: rowNumber(r.totalImpressions)
|
|
107
|
+
}));
|
|
108
|
+
const outliers = parseJsonRows(row.outliers_json).map((r) => ({
|
|
109
|
+
query: rowString(r.query),
|
|
110
|
+
clicks: rowNumber(r.clicks),
|
|
111
|
+
impressions: rowNumber(r.impressions),
|
|
112
|
+
ctr: rowNumber(r.ctr),
|
|
113
|
+
position: rowNumber(r.position),
|
|
114
|
+
expectedCtr: rowNumber(r.expectedCtr),
|
|
115
|
+
ctrDiff: rowNumber(r.ctrDiff)
|
|
116
|
+
}));
|
|
117
|
+
return {
|
|
118
|
+
results: curve,
|
|
119
|
+
meta: {
|
|
120
|
+
overperforming: outliers.filter((o) => o.ctrDiff > 0).slice(0, 25),
|
|
121
|
+
underperforming: outliers.filter((o) => o.ctrDiff < 0).slice(0, 25),
|
|
122
|
+
startDate,
|
|
123
|
+
endDate
|
|
124
|
+
}
|
|
125
|
+
};
|
|
126
|
+
}
|
|
127
|
+
});
|
|
128
|
+
export { ctrCurveAnalyzer };
|