@gscdump/analysis 1.4.0 → 1.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/action-priority.d.mts +34 -0
- package/dist/action-priority.mjs +201 -0
- package/dist/analyzer/adapt-rows.mjs +12 -0
- package/dist/analyzer/all.mjs +61 -0
- package/dist/analyzer/paginate.mjs +85 -0
- package/dist/analyzer/row-analyzers.d.mts +3 -0
- package/dist/analyzer/row-analyzers.mjs +3 -0
- package/dist/analyzer/row-values.mjs +22 -0
- package/dist/analyzers/bayesian-ctr.mjs +177 -0
- package/dist/analyzers/bipartite-pagerank.mjs +243 -0
- package/dist/analyzers/brand.d.mts +33 -0
- package/dist/analyzers/brand.mjs +154 -0
- package/dist/analyzers/cannibalization.d.mts +58 -0
- package/dist/analyzers/cannibalization.mjs +267 -0
- package/dist/analyzers/change-point.mjs +197 -0
- package/dist/analyzers/clustering.d.mts +29 -0
- package/dist/analyzers/clustering.mjs +217 -0
- package/dist/analyzers/concentration.d.mts +42 -0
- package/dist/analyzers/concentration.mjs +180 -0
- package/dist/analyzers/content-velocity.mjs +87 -0
- package/dist/analyzers/ctr-anomaly.mjs +189 -0
- package/dist/analyzers/ctr-curve.mjs +128 -0
- package/dist/analyzers/dark-traffic.mjs +130 -0
- package/dist/analyzers/data-detail.mjs +44 -0
- package/dist/analyzers/data-query.mjs +44 -0
- package/dist/analyzers/decay.d.mts +38 -0
- package/dist/analyzers/decay.mjs +187 -0
- package/dist/analyzers/device-gap.mjs +135 -0
- package/dist/analyzers/intent-atlas.mjs +175 -0
- package/dist/analyzers/keyword-breadth.mjs +115 -0
- package/dist/analyzers/long-tail.mjs +150 -0
- package/dist/analyzers/movers.d.mts +43 -0
- package/dist/analyzers/movers.mjs +283 -0
- package/dist/analyzers/opportunity.d.mts +32 -0
- package/dist/analyzers/opportunity.mjs +212 -0
- package/dist/analyzers/position-distribution.mjs +57 -0
- package/dist/analyzers/position-volatility.mjs +143 -0
- package/dist/analyzers/query-migration.mjs +178 -0
- package/dist/analyzers/seasonality.d.mts +28 -0
- package/dist/analyzers/seasonality.mjs +136 -0
- package/dist/analyzers/stl-decompose.mjs +187 -0
- package/dist/analyzers/striking-distance.d.mts +54 -0
- package/dist/analyzers/striking-distance.mjs +82 -0
- package/dist/analyzers/survival.mjs +200 -0
- package/dist/analyzers/trends.mjs +156 -0
- package/dist/analyzers/zero-click.d.mts +17 -0
- package/dist/analyzers/zero-click.mjs +122 -0
- package/dist/browser.d.mts +5 -0
- package/dist/browser.mjs +11 -0
- package/dist/default-registry.d.mts +3 -1
- package/dist/default-registry.mjs +5 -1
- package/dist/errors.d.mts +51 -1
- package/dist/index.d.mts +30 -494
- package/dist/index.mjs +28 -49
- package/dist/query/analyzers.mjs +301 -0
- package/dist/query/index.mjs +4 -0
- package/dist/query/intent.d.mts +20 -0
- package/dist/query/intent.mjs +139 -0
- package/dist/query/normalize.d.mts +15 -0
- package/dist/query/normalize.mjs +204 -0
- package/dist/report/format.d.mts +7 -0
- package/dist/report/format.mjs +45 -0
- package/dist/report/index.d.mts +4 -1
- package/dist/report/index.mjs +4 -1
- package/dist/report/registry.d.mts +4 -0
- package/dist/report/registry.mjs +26 -0
- package/dist/report/reports/brand.mjs +104 -0
- package/dist/report/reports/growth.mjs +144 -0
- package/dist/report/reports/health.mjs +141 -0
- package/dist/report/reports/movers.mjs +203 -0
- package/dist/report/reports/opportunities.mjs +184 -0
- package/dist/report/reports/pre-publish.mjs +111 -0
- package/dist/report/reports/priority.mjs +97 -0
- package/dist/report/reports/risks.mjs +201 -0
- package/dist/report/reports/triage.mjs +155 -0
- package/dist/report/require.mjs +15 -0
- package/dist/report/resolve-target.d.mts +29 -0
- package/dist/report/resolve-target.mjs +32 -0
- package/dist/{_chunks/index.d.mts → report/runtime.d.mts} +2 -37
- package/dist/report/runtime.mjs +80 -0
- package/dist/report/sections.mjs +22 -0
- package/dist/sitemap-health.d.mts +34 -0
- package/dist/sitemap-health.mjs +37 -0
- package/dist/{_chunks/index2.d.mts → source/composite.d.mts} +2 -13
- package/dist/{_chunks/source.mjs → source/composite.mjs} +1 -17
- package/dist/source/in-memory.d.mts +14 -0
- package/dist/source/in-memory.mjs +17 -0
- package/dist/source/index.d.mts +2 -1
- package/dist/source/index.mjs +2 -1
- package/dist/sql-analyzers.d.mts +3 -0
- package/dist/sql-analyzers.mjs +3 -0
- package/dist/types.d.mts +34 -0
- package/dist/types.mjs +21 -0
- package/package.json +4 -4
- package/dist/_chunks/default-registry.d.mts +0 -5
- package/dist/_chunks/default-registry.mjs +0 -5183
- package/dist/_chunks/errors.d.mts +0 -52
- package/dist/_chunks/report.mjs +0 -1710
- /package/dist/{_chunks/scoring.mjs → scoring.mjs} +0 -0
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
import { rowString } from "../analyzer/row-values.mjs";
|
|
2
|
+
import { defineAnalyzer } from "@gscdump/engine/analyzer";
|
|
3
|
+
import { periodOf } from "@gscdump/engine/period";
|
|
4
|
+
import { enumeratePartitions } from "@gscdump/engine/planner";
|
|
5
|
+
import { METRIC_EXPR } from "@gscdump/engine/sql-fragments";
|
|
6
|
+
import { num } from "@gscdump/engine/analysis-types";
|
|
7
|
+
const positionVolatilityAnalyzer = defineAnalyzer({
|
|
8
|
+
id: "position-volatility",
|
|
9
|
+
buildSql(params) {
|
|
10
|
+
const { startDate, endDate } = periodOf(params);
|
|
11
|
+
const topN = params.topN ?? 30;
|
|
12
|
+
const minDayImpressions = params.minImpressions ?? 10;
|
|
13
|
+
const minDays = params.minWeeksWithData ?? 7;
|
|
14
|
+
return {
|
|
15
|
+
sql: `
|
|
16
|
+
WITH query_day AS (
|
|
17
|
+
SELECT
|
|
18
|
+
url AS page,
|
|
19
|
+
query,
|
|
20
|
+
-- Normalize at the source CTE: union_by_name=true can coerce date to
|
|
21
|
+
-- VARCHAR across parquets with mixed schemas, which makes downstream
|
|
22
|
+
-- strftime(date, ...) binder-error.
|
|
23
|
+
CAST(date AS DATE) AS date,
|
|
24
|
+
${METRIC_EXPR.impressions} AS q_impressions,
|
|
25
|
+
${METRIC_EXPR.position} AS q_position
|
|
26
|
+
FROM read_parquet({{FILES}}, union_by_name = true)
|
|
27
|
+
WHERE date >= ? AND date <= ?
|
|
28
|
+
AND query IS NOT NULL AND query <> ''
|
|
29
|
+
AND url IS NOT NULL AND url <> ''
|
|
30
|
+
GROUP BY url, query, date
|
|
31
|
+
HAVING SUM(impressions) >= 1
|
|
32
|
+
),
|
|
33
|
+
daily AS (
|
|
34
|
+
SELECT
|
|
35
|
+
page, date,
|
|
36
|
+
COUNT(*) AS query_count,
|
|
37
|
+
SUM(q_impressions) AS day_impressions,
|
|
38
|
+
SUM(q_position * q_impressions) / NULLIF(SUM(q_impressions), 0) AS avg_position,
|
|
39
|
+
COALESCE(STDDEV_POP(q_position), 0.0) AS pos_stddev,
|
|
40
|
+
MIN(q_position) AS best_position,
|
|
41
|
+
MAX(q_position) AS worst_position
|
|
42
|
+
FROM query_day
|
|
43
|
+
GROUP BY page, date
|
|
44
|
+
HAVING SUM(q_impressions) >= ?
|
|
45
|
+
),
|
|
46
|
+
with_shift AS (
|
|
47
|
+
SELECT *,
|
|
48
|
+
LAG(avg_position) OVER (PARTITION BY page ORDER BY date) AS prev_position,
|
|
49
|
+
COALESCE(
|
|
50
|
+
ABS(avg_position - LAG(avg_position) OVER (PARTITION BY page ORDER BY date)),
|
|
51
|
+
0.0
|
|
52
|
+
) AS dod_shift
|
|
53
|
+
FROM daily
|
|
54
|
+
),
|
|
55
|
+
scored AS (
|
|
56
|
+
SELECT *,
|
|
57
|
+
pos_stddev + dod_shift AS volatility
|
|
58
|
+
FROM with_shift
|
|
59
|
+
),
|
|
60
|
+
top_pages AS (
|
|
61
|
+
SELECT page,
|
|
62
|
+
SUM(day_impressions) AS total_impressions,
|
|
63
|
+
AVG(volatility) AS avg_volatility,
|
|
64
|
+
MAX(volatility) AS peak_volatility,
|
|
65
|
+
COUNT(*) AS days_with_data
|
|
66
|
+
FROM scored
|
|
67
|
+
GROUP BY page
|
|
68
|
+
HAVING COUNT(*) >= ?
|
|
69
|
+
ORDER BY avg_volatility DESC
|
|
70
|
+
LIMIT ${Number(topN)}
|
|
71
|
+
)
|
|
72
|
+
SELECT
|
|
73
|
+
s.page,
|
|
74
|
+
strftime(s.date, '%Y-%m-%d') AS date,
|
|
75
|
+
s.query_count AS queryCount,
|
|
76
|
+
s.day_impressions AS dayImpressions,
|
|
77
|
+
s.avg_position AS avgPosition,
|
|
78
|
+
s.pos_stddev AS posStddev,
|
|
79
|
+
s.best_position AS bestPosition,
|
|
80
|
+
s.worst_position AS worstPosition,
|
|
81
|
+
s.dod_shift AS dodShift,
|
|
82
|
+
s.volatility AS volatility,
|
|
83
|
+
t.avg_volatility AS pageAvgVolatility,
|
|
84
|
+
t.peak_volatility AS pagePeakVolatility,
|
|
85
|
+
t.total_impressions AS pageTotalImpressions
|
|
86
|
+
FROM scored s
|
|
87
|
+
JOIN top_pages t USING (page)
|
|
88
|
+
ORDER BY t.avg_volatility DESC, s.date ASC
|
|
89
|
+
`,
|
|
90
|
+
params: [
|
|
91
|
+
startDate,
|
|
92
|
+
endDate,
|
|
93
|
+
minDayImpressions,
|
|
94
|
+
minDays
|
|
95
|
+
],
|
|
96
|
+
current: {
|
|
97
|
+
table: "page_queries",
|
|
98
|
+
partitions: enumeratePartitions(startDate, endDate)
|
|
99
|
+
}
|
|
100
|
+
};
|
|
101
|
+
},
|
|
102
|
+
reduceSql(rows) {
|
|
103
|
+
const arr = Array.isArray(rows) ? rows : [];
|
|
104
|
+
const byPage = /* @__PURE__ */ new Map();
|
|
105
|
+
const allDates = /* @__PURE__ */ new Set();
|
|
106
|
+
for (const r of arr) {
|
|
107
|
+
const page = rowString(r.page);
|
|
108
|
+
const date = rowString(r.date);
|
|
109
|
+
allDates.add(date);
|
|
110
|
+
const entry = byPage.get(page) ?? {
|
|
111
|
+
page,
|
|
112
|
+
avgVolatility: num(r.pageAvgVolatility),
|
|
113
|
+
peakVolatility: num(r.pagePeakVolatility),
|
|
114
|
+
totalImpressions: num(r.pageTotalImpressions),
|
|
115
|
+
days: []
|
|
116
|
+
};
|
|
117
|
+
entry.days.push({
|
|
118
|
+
date,
|
|
119
|
+
queryCount: num(r.queryCount),
|
|
120
|
+
dayImpressions: num(r.dayImpressions),
|
|
121
|
+
avgPosition: num(r.avgPosition),
|
|
122
|
+
posStddev: num(r.posStddev),
|
|
123
|
+
bestPosition: num(r.bestPosition),
|
|
124
|
+
worstPosition: num(r.worstPosition),
|
|
125
|
+
dodShift: num(r.dodShift),
|
|
126
|
+
volatility: num(r.volatility)
|
|
127
|
+
});
|
|
128
|
+
byPage.set(page, entry);
|
|
129
|
+
}
|
|
130
|
+
const pages = [...byPage.values()].sort((a, b) => b.avgVolatility - a.avgVolatility);
|
|
131
|
+
const dates = [...allDates].sort();
|
|
132
|
+
const maxVolatility = pages.reduce((m, p) => Math.max(m, p.peakVolatility), 0);
|
|
133
|
+
return {
|
|
134
|
+
results: pages,
|
|
135
|
+
meta: {
|
|
136
|
+
total: pages.length,
|
|
137
|
+
dates,
|
|
138
|
+
maxVolatility
|
|
139
|
+
}
|
|
140
|
+
};
|
|
141
|
+
}
|
|
142
|
+
});
|
|
143
|
+
export { positionVolatilityAnalyzer };
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
import { parseJsonRows, rowString } from "../analyzer/row-values.mjs";
|
|
2
|
+
import { defineAnalyzer } from "@gscdump/engine/analyzer";
|
|
3
|
+
import { periodOf } from "@gscdump/engine/period";
|
|
4
|
+
import { enumeratePartitions } from "@gscdump/engine/planner";
|
|
5
|
+
import { METRIC_EXPR } from "@gscdump/engine/sql-fragments";
|
|
6
|
+
import { num } from "@gscdump/engine/analysis-types";
|
|
7
|
+
import { MS_PER_DAY, toIsoDate } from "gscdump/dates";
|
|
8
|
+
const queryMigrationAnalyzer = defineAnalyzer({
|
|
9
|
+
id: "query-migration",
|
|
10
|
+
buildSql(params) {
|
|
11
|
+
const cur = periodOf(params);
|
|
12
|
+
let prevStart = params.prevStartDate;
|
|
13
|
+
let prevEnd = params.prevEndDate;
|
|
14
|
+
if (prevStart == null || prevEnd == null) {
|
|
15
|
+
const curStartMs = new Date(cur.startDate).getTime();
|
|
16
|
+
const span = new Date(cur.endDate).getTime() - curStartMs;
|
|
17
|
+
prevEnd = toIsoDate(new Date(curStartMs - MS_PER_DAY));
|
|
18
|
+
prevStart = toIsoDate(new Date(curStartMs - MS_PER_DAY - span));
|
|
19
|
+
}
|
|
20
|
+
const minImpressions = params.minImpressions ?? 20;
|
|
21
|
+
const limit = params.limit ?? 200;
|
|
22
|
+
const maxLevenshtein = 2;
|
|
23
|
+
return {
|
|
24
|
+
sql: `
|
|
25
|
+
WITH cur AS (
|
|
26
|
+
SELECT query, url AS page,
|
|
27
|
+
${METRIC_EXPR.impressions} AS impressions,
|
|
28
|
+
${METRIC_EXPR.clicks} AS clicks,
|
|
29
|
+
${METRIC_EXPR.position} AS position
|
|
30
|
+
FROM read_parquet({{FILES}}, union_by_name = true)
|
|
31
|
+
WHERE date >= ? AND date <= ?
|
|
32
|
+
AND query IS NOT NULL AND query <> ''
|
|
33
|
+
AND url IS NOT NULL AND url <> ''
|
|
34
|
+
GROUP BY query, url
|
|
35
|
+
HAVING SUM(impressions) >= ?
|
|
36
|
+
),
|
|
37
|
+
prev AS (
|
|
38
|
+
SELECT query, url AS page,
|
|
39
|
+
${METRIC_EXPR.impressions} AS impressions,
|
|
40
|
+
${METRIC_EXPR.clicks} AS clicks,
|
|
41
|
+
${METRIC_EXPR.position} AS position
|
|
42
|
+
FROM read_parquet({{FILES_PREV}}, union_by_name = true)
|
|
43
|
+
WHERE date >= ? AND date <= ?
|
|
44
|
+
AND query IS NOT NULL AND query <> ''
|
|
45
|
+
AND url IS NOT NULL AND url <> ''
|
|
46
|
+
GROUP BY query, url
|
|
47
|
+
HAVING SUM(impressions) >= ?
|
|
48
|
+
),
|
|
49
|
+
lost AS (
|
|
50
|
+
SELECT p.page AS source_page, p.query AS source_query, p.impressions AS source_impressions
|
|
51
|
+
FROM prev p
|
|
52
|
+
LEFT JOIN cur c ON p.page = c.page AND p.query = c.query
|
|
53
|
+
WHERE c.query IS NULL
|
|
54
|
+
),
|
|
55
|
+
gained AS (
|
|
56
|
+
SELECT c.page AS target_page, c.query AS target_query, c.impressions AS target_impressions
|
|
57
|
+
FROM cur c
|
|
58
|
+
LEFT JOIN prev p ON p.page = c.page AND p.query = c.query
|
|
59
|
+
WHERE p.query IS NULL
|
|
60
|
+
),
|
|
61
|
+
matched AS (
|
|
62
|
+
SELECT
|
|
63
|
+
l.source_page, l.source_query, l.source_impressions,
|
|
64
|
+
g.target_page, g.target_query, g.target_impressions,
|
|
65
|
+
CASE
|
|
66
|
+
WHEN l.source_query = g.target_query THEN 'exact'
|
|
67
|
+
ELSE 'fuzzy'
|
|
68
|
+
END AS match_type,
|
|
69
|
+
LEAST(l.source_impressions, g.target_impressions) AS absorbed_impressions
|
|
70
|
+
FROM lost l
|
|
71
|
+
JOIN gained g
|
|
72
|
+
ON l.source_page <> g.target_page
|
|
73
|
+
AND ABS(LENGTH(l.source_query) - LENGTH(g.target_query)) <= ${maxLevenshtein}
|
|
74
|
+
AND (
|
|
75
|
+
l.source_query = g.target_query
|
|
76
|
+
OR levenshtein(l.source_query, g.target_query) <= ${maxLevenshtein}
|
|
77
|
+
)
|
|
78
|
+
),
|
|
79
|
+
edges AS (
|
|
80
|
+
SELECT
|
|
81
|
+
source_page, target_page,
|
|
82
|
+
SUM(absorbed_impressions) AS weight,
|
|
83
|
+
COUNT(*) AS query_count,
|
|
84
|
+
SUM(CASE WHEN match_type = 'exact' THEN 1 ELSE 0 END) AS exact_count,
|
|
85
|
+
to_json(list({
|
|
86
|
+
'sourceQuery': source_query,
|
|
87
|
+
'targetQuery': target_query,
|
|
88
|
+
'absorbed': absorbed_impressions,
|
|
89
|
+
'matchType': match_type
|
|
90
|
+
} ORDER BY absorbed_impressions DESC)) AS examplesJson
|
|
91
|
+
FROM matched
|
|
92
|
+
GROUP BY source_page, target_page
|
|
93
|
+
)
|
|
94
|
+
SELECT *
|
|
95
|
+
FROM edges
|
|
96
|
+
ORDER BY weight DESC
|
|
97
|
+
LIMIT ${Number(limit)}
|
|
98
|
+
`,
|
|
99
|
+
params: [
|
|
100
|
+
cur.startDate,
|
|
101
|
+
cur.endDate,
|
|
102
|
+
minImpressions,
|
|
103
|
+
prevStart,
|
|
104
|
+
prevEnd,
|
|
105
|
+
minImpressions
|
|
106
|
+
],
|
|
107
|
+
current: {
|
|
108
|
+
table: "page_queries",
|
|
109
|
+
partitions: enumeratePartitions(cur.startDate, cur.endDate)
|
|
110
|
+
},
|
|
111
|
+
previous: {
|
|
112
|
+
table: "page_queries",
|
|
113
|
+
partitions: enumeratePartitions(prevStart, prevEnd)
|
|
114
|
+
}
|
|
115
|
+
};
|
|
116
|
+
},
|
|
117
|
+
reduceSql(rows, params) {
|
|
118
|
+
const arr = Array.isArray(rows) ? rows : [];
|
|
119
|
+
const cur = periodOf(params);
|
|
120
|
+
let prevStart = params.prevStartDate;
|
|
121
|
+
let prevEnd = params.prevEndDate;
|
|
122
|
+
if (prevStart == null || prevEnd == null) {
|
|
123
|
+
const curStartMs = new Date(cur.startDate).getTime();
|
|
124
|
+
const span = new Date(cur.endDate).getTime() - curStartMs;
|
|
125
|
+
prevEnd = toIsoDate(new Date(curStartMs - MS_PER_DAY));
|
|
126
|
+
prevStart = toIsoDate(new Date(curStartMs - MS_PER_DAY - span));
|
|
127
|
+
}
|
|
128
|
+
const edges = arr.map((r) => ({
|
|
129
|
+
sourcePage: rowString(r.source_page),
|
|
130
|
+
targetPage: rowString(r.target_page),
|
|
131
|
+
weight: num(r.weight),
|
|
132
|
+
queryCount: num(r.query_count),
|
|
133
|
+
exactCount: num(r.exact_count),
|
|
134
|
+
fuzzyCount: num(r.query_count) - num(r.exact_count),
|
|
135
|
+
examples: parseJsonRows(r.examplesJson).slice(0, 8).map((e) => ({
|
|
136
|
+
sourceQuery: rowString(e.sourceQuery),
|
|
137
|
+
targetQuery: rowString(e.targetQuery),
|
|
138
|
+
absorbed: num(e.absorbed),
|
|
139
|
+
matchType: rowString(e.matchType)
|
|
140
|
+
}))
|
|
141
|
+
}));
|
|
142
|
+
const nodeAgg = /* @__PURE__ */ new Map();
|
|
143
|
+
for (const e of edges) {
|
|
144
|
+
const src = nodeAgg.get(e.sourcePage) ?? {
|
|
145
|
+
url: e.sourcePage,
|
|
146
|
+
outgoing: 0,
|
|
147
|
+
incoming: 0
|
|
148
|
+
};
|
|
149
|
+
src.outgoing += e.weight;
|
|
150
|
+
nodeAgg.set(e.sourcePage, src);
|
|
151
|
+
const tgt = nodeAgg.get(e.targetPage) ?? {
|
|
152
|
+
url: e.targetPage,
|
|
153
|
+
outgoing: 0,
|
|
154
|
+
incoming: 0
|
|
155
|
+
};
|
|
156
|
+
tgt.incoming += e.weight;
|
|
157
|
+
nodeAgg.set(e.targetPage, tgt);
|
|
158
|
+
}
|
|
159
|
+
const nodes = [...nodeAgg.values()];
|
|
160
|
+
const totalAbsorbed = edges.reduce((s, e) => s + e.weight, 0);
|
|
161
|
+
return {
|
|
162
|
+
results: edges,
|
|
163
|
+
meta: {
|
|
164
|
+
total: edges.length,
|
|
165
|
+
totalAbsorbed,
|
|
166
|
+
period: {
|
|
167
|
+
current: cur,
|
|
168
|
+
previous: {
|
|
169
|
+
startDate: prevStart,
|
|
170
|
+
endDate: prevEnd
|
|
171
|
+
}
|
|
172
|
+
},
|
|
173
|
+
nodes
|
|
174
|
+
}
|
|
175
|
+
};
|
|
176
|
+
}
|
|
177
|
+
});
|
|
178
|
+
export { queryMigrationAnalyzer };
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { DateRow } from "../types.mjs";
|
|
2
|
+
type SeasonalityMetric = 'clicks' | 'impressions';
|
|
3
|
+
interface SeasonalityOptions {
|
|
4
|
+
/** Metric to analyze for seasonality. Default: clicks */
|
|
5
|
+
metric?: SeasonalityMetric;
|
|
6
|
+
}
|
|
7
|
+
interface MonthlyData {
|
|
8
|
+
month: string;
|
|
9
|
+
value: number;
|
|
10
|
+
vsAverage: number;
|
|
11
|
+
isPeak: boolean;
|
|
12
|
+
isTrough: boolean;
|
|
13
|
+
}
|
|
14
|
+
interface SeasonalityResult {
|
|
15
|
+
hasSeasonality: boolean;
|
|
16
|
+
/** Coefficient of variation: std dev / mean. Higher = more seasonal. */
|
|
17
|
+
strength: number;
|
|
18
|
+
peakMonths: string[];
|
|
19
|
+
troughMonths: string[];
|
|
20
|
+
monthlyBreakdown: MonthlyData[];
|
|
21
|
+
insufficientData: boolean;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Pure helper: detects seasonality patterns by analyzing monthly traffic
|
|
25
|
+
* variation. Re-exported from `@gscdump/analysis` for portable callers.
|
|
26
|
+
*/
|
|
27
|
+
declare function analyzeSeasonality(dates: DateRow[], options?: SeasonalityOptions): SeasonalityResult;
|
|
28
|
+
export { MonthlyData, SeasonalityMetric, SeasonalityOptions, SeasonalityResult, analyzeSeasonality };
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
import { rowBoolean, rowString } from "../analyzer/row-values.mjs";
|
|
2
|
+
import { datesQueryState } from "../analyzer/adapt-rows.mjs";
|
|
3
|
+
import { defineAnalyzer } from "@gscdump/engine/analyzer";
|
|
4
|
+
import { periodOf } from "@gscdump/engine/period";
|
|
5
|
+
import { enumeratePartitions } from "@gscdump/engine/planner";
|
|
6
|
+
import { num } from "@gscdump/engine/analysis-types";
|
|
7
|
+
function calculateCV(values) {
|
|
8
|
+
if (values.length === 0) return 0;
|
|
9
|
+
const mean = values.reduce((a, b) => a + b, 0) / values.length;
|
|
10
|
+
if (mean === 0) return 0;
|
|
11
|
+
const variance = values.reduce((sum, v) => sum + (v - mean) ** 2, 0) / values.length;
|
|
12
|
+
return Math.min(Math.sqrt(variance) / mean, 1);
|
|
13
|
+
}
|
|
14
|
+
function analyzeSeasonality(dates, options = {}) {
|
|
15
|
+
const { metric = "clicks" } = options;
|
|
16
|
+
if (dates.length === 0) return {
|
|
17
|
+
hasSeasonality: false,
|
|
18
|
+
strength: 0,
|
|
19
|
+
peakMonths: [],
|
|
20
|
+
troughMonths: [],
|
|
21
|
+
monthlyBreakdown: [],
|
|
22
|
+
insufficientData: true
|
|
23
|
+
};
|
|
24
|
+
const monthlyMap = /* @__PURE__ */ new Map();
|
|
25
|
+
for (const row of dates) {
|
|
26
|
+
const month = row.date.substring(0, 7);
|
|
27
|
+
const value = metric === "clicks" ? row.clicks : row.impressions;
|
|
28
|
+
monthlyMap.set(month, (monthlyMap.get(month) || 0) + value);
|
|
29
|
+
}
|
|
30
|
+
const months = Array.from(monthlyMap.keys()).sort();
|
|
31
|
+
const values = months.map((m) => monthlyMap.get(m) || 0);
|
|
32
|
+
const insufficientData = months.length < 12;
|
|
33
|
+
const totalValue = values.reduce((a, b) => a + b, 0);
|
|
34
|
+
const avgValue = values.length > 0 ? totalValue / values.length : 0;
|
|
35
|
+
const monthlyBreakdown = months.map((month, i) => {
|
|
36
|
+
const value = values[i] ?? 0;
|
|
37
|
+
const vsAverage = avgValue > 0 ? value / avgValue : 0;
|
|
38
|
+
return {
|
|
39
|
+
month,
|
|
40
|
+
value,
|
|
41
|
+
vsAverage,
|
|
42
|
+
isPeak: vsAverage > 1.5,
|
|
43
|
+
isTrough: vsAverage < .5
|
|
44
|
+
};
|
|
45
|
+
});
|
|
46
|
+
const peakMonths = [...new Set(monthlyBreakdown.filter((m) => m.isPeak).map((m) => m.month.substring(5, 7)))];
|
|
47
|
+
const troughMonths = [...new Set(monthlyBreakdown.filter((m) => m.isTrough).map((m) => m.month.substring(5, 7)))];
|
|
48
|
+
const strength = calculateCV(values);
|
|
49
|
+
return {
|
|
50
|
+
hasSeasonality: peakMonths.length > 0 || troughMonths.length > 0 || strength > .3,
|
|
51
|
+
strength,
|
|
52
|
+
peakMonths,
|
|
53
|
+
troughMonths,
|
|
54
|
+
monthlyBreakdown,
|
|
55
|
+
insufficientData
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
const seasonalityAnalyzer = defineAnalyzer({
|
|
59
|
+
id: "seasonality",
|
|
60
|
+
buildSql(params) {
|
|
61
|
+
const { startDate, endDate } = periodOf(params);
|
|
62
|
+
return {
|
|
63
|
+
sql: `
|
|
64
|
+
WITH monthly AS (
|
|
65
|
+
SELECT
|
|
66
|
+
strftime(CAST(date AS DATE), '%Y-%m') AS month,
|
|
67
|
+
CAST(SUM(${params.metric === "impressions" ? "impressions" : "clicks"}) AS DOUBLE) AS value
|
|
68
|
+
FROM read_parquet({{FILES}}, union_by_name = true)
|
|
69
|
+
WHERE date >= ? AND date <= ?
|
|
70
|
+
GROUP BY month
|
|
71
|
+
),
|
|
72
|
+
stats AS (
|
|
73
|
+
SELECT
|
|
74
|
+
AVG(value) AS avg_val,
|
|
75
|
+
COALESCE(STDDEV_POP(value), 0.0) AS std_val,
|
|
76
|
+
CAST(COUNT(*) AS DOUBLE) AS month_count
|
|
77
|
+
FROM monthly
|
|
78
|
+
)
|
|
79
|
+
SELECT
|
|
80
|
+
m.month AS month,
|
|
81
|
+
m.value AS value,
|
|
82
|
+
CASE WHEN s.avg_val > 0 THEN m.value / s.avg_val ELSE 0.0 END AS vsAverage,
|
|
83
|
+
(s.avg_val > 0 AND m.value / s.avg_val > 1.5) AS isPeak,
|
|
84
|
+
(s.avg_val > 0 AND m.value / s.avg_val < 0.5) AS isTrough,
|
|
85
|
+
CASE WHEN s.avg_val > 0 THEN LEAST(s.std_val / s.avg_val, 1.0) ELSE 0.0 END AS strength,
|
|
86
|
+
s.month_count AS monthCount
|
|
87
|
+
FROM monthly m, stats s
|
|
88
|
+
ORDER BY m.month
|
|
89
|
+
`,
|
|
90
|
+
params: [startDate, endDate],
|
|
91
|
+
current: {
|
|
92
|
+
table: "pages",
|
|
93
|
+
partitions: enumeratePartitions(startDate, endDate)
|
|
94
|
+
}
|
|
95
|
+
};
|
|
96
|
+
},
|
|
97
|
+
reduceSql(rows) {
|
|
98
|
+
const arr = Array.isArray(rows) ? rows : [];
|
|
99
|
+
const breakdown = arr.map((r) => ({
|
|
100
|
+
month: rowString(r.month),
|
|
101
|
+
value: num(r.value),
|
|
102
|
+
vsAverage: num(r.vsAverage),
|
|
103
|
+
isPeak: rowBoolean(r.isPeak),
|
|
104
|
+
isTrough: rowBoolean(r.isTrough)
|
|
105
|
+
}));
|
|
106
|
+
const first = arr[0];
|
|
107
|
+
const strength = first ? num(first.strength) : 0;
|
|
108
|
+
const monthCount = first ? num(first.monthCount) : 0;
|
|
109
|
+
const peakMonths = [...new Set(breakdown.filter((m) => m.isPeak).map((m) => m.month.substring(5, 7)))];
|
|
110
|
+
const troughMonths = [...new Set(breakdown.filter((m) => m.isTrough).map((m) => m.month.substring(5, 7)))];
|
|
111
|
+
const hasSeasonality = peakMonths.length > 0 || troughMonths.length > 0 || strength > .3;
|
|
112
|
+
const insufficientData = monthCount < 12;
|
|
113
|
+
return {
|
|
114
|
+
results: breakdown,
|
|
115
|
+
meta: {
|
|
116
|
+
total: breakdown.length,
|
|
117
|
+
hasSeasonality,
|
|
118
|
+
strength,
|
|
119
|
+
peakMonths,
|
|
120
|
+
troughMonths,
|
|
121
|
+
insufficientData
|
|
122
|
+
}
|
|
123
|
+
};
|
|
124
|
+
},
|
|
125
|
+
buildRows(params) {
|
|
126
|
+
return { dates: datesQueryState(periodOf(params), params.limit) };
|
|
127
|
+
},
|
|
128
|
+
reduceRows(rows, params) {
|
|
129
|
+
const result = analyzeSeasonality(Array.isArray(rows) ? rows : [], { metric: params.metric });
|
|
130
|
+
return {
|
|
131
|
+
results: result.monthlyBreakdown,
|
|
132
|
+
meta: { strength: result.strength }
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
});
|
|
136
|
+
export { analyzeSeasonality, seasonalityAnalyzer };
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
import { parseJsonRows, rowBoolean, rowString } from "../analyzer/row-values.mjs";
|
|
2
|
+
import { defineAnalyzer } from "@gscdump/engine/analyzer";
|
|
3
|
+
import { defaultEndDate } from "@gscdump/engine/period";
|
|
4
|
+
import { enumeratePartitions } from "@gscdump/engine/planner";
|
|
5
|
+
import { METRIC_EXPR } from "@gscdump/engine/sql-fragments";
|
|
6
|
+
import { num } from "@gscdump/engine/analysis-types";
|
|
7
|
+
import { daysAgoUtc } from "gscdump/dates";
|
|
8
|
+
const stlDecomposeAnalyzer = defineAnalyzer({
|
|
9
|
+
id: "stl-decompose",
|
|
10
|
+
buildSql(params) {
|
|
11
|
+
const endDate = params.endDate ?? defaultEndDate();
|
|
12
|
+
const startDate = params.startDate ?? daysAgoUtc(93);
|
|
13
|
+
const minImpressions = params.minImpressions ?? 100;
|
|
14
|
+
const minDays = 21;
|
|
15
|
+
const metric = params.metric === "clicks" ? "clicks" : "impressions";
|
|
16
|
+
const limit = params.limit ?? 100;
|
|
17
|
+
return {
|
|
18
|
+
sql: `
|
|
19
|
+
WITH daily AS (
|
|
20
|
+
SELECT
|
|
21
|
+
query,
|
|
22
|
+
url AS page,
|
|
23
|
+
-- Normalize at the source CTE: union_by_name=true can coerce date to
|
|
24
|
+
-- VARCHAR across parquets with mixed schemas, which makes downstream
|
|
25
|
+
-- strftime(date, ...) binder-error.
|
|
26
|
+
CAST(date AS DATE) AS date,
|
|
27
|
+
${METRIC_EXPR.clicks} AS clicks,
|
|
28
|
+
${METRIC_EXPR.impressions} AS impressions,
|
|
29
|
+
CAST(SUM(${metric}) AS DOUBLE) AS observed
|
|
30
|
+
FROM read_parquet({{FILES}}, union_by_name = true)
|
|
31
|
+
WHERE date >= ? AND date <= ?
|
|
32
|
+
AND query IS NOT NULL AND query <> ''
|
|
33
|
+
AND url IS NOT NULL AND url <> ''
|
|
34
|
+
GROUP BY query, url, date
|
|
35
|
+
),
|
|
36
|
+
entity_stats AS (
|
|
37
|
+
SELECT query, page,
|
|
38
|
+
COUNT(*) AS days,
|
|
39
|
+
SUM(impressions) AS total_impressions
|
|
40
|
+
FROM daily
|
|
41
|
+
GROUP BY query, page
|
|
42
|
+
HAVING COUNT(*) >= ${Number(minDays)}
|
|
43
|
+
AND SUM(impressions) >= ?
|
|
44
|
+
),
|
|
45
|
+
filtered AS (
|
|
46
|
+
SELECT d.*
|
|
47
|
+
FROM daily d
|
|
48
|
+
JOIN entity_stats e USING (query, page)
|
|
49
|
+
),
|
|
50
|
+
trended AS (
|
|
51
|
+
SELECT *,
|
|
52
|
+
CASE
|
|
53
|
+
WHEN COUNT(*) OVER w = 7
|
|
54
|
+
THEN AVG(observed) OVER w
|
|
55
|
+
ELSE NULL
|
|
56
|
+
END AS trend
|
|
57
|
+
FROM filtered
|
|
58
|
+
WINDOW w AS (
|
|
59
|
+
PARTITION BY query, page
|
|
60
|
+
ORDER BY date
|
|
61
|
+
ROWS BETWEEN 3 PRECEDING AND 3 FOLLOWING
|
|
62
|
+
)
|
|
63
|
+
),
|
|
64
|
+
detrended AS (
|
|
65
|
+
SELECT *,
|
|
66
|
+
observed - trend AS detrended,
|
|
67
|
+
dayofweek(date) AS dow
|
|
68
|
+
FROM trended
|
|
69
|
+
),
|
|
70
|
+
seasonal_raw AS (
|
|
71
|
+
SELECT *,
|
|
72
|
+
AVG(detrended) OVER (PARTITION BY query, page, dow) AS seasonal_dow
|
|
73
|
+
FROM detrended
|
|
74
|
+
),
|
|
75
|
+
seasonal_centered AS (
|
|
76
|
+
SELECT *,
|
|
77
|
+
seasonal_dow - AVG(seasonal_dow) OVER (PARTITION BY query, page) AS seasonal
|
|
78
|
+
FROM seasonal_raw
|
|
79
|
+
),
|
|
80
|
+
residualed AS (
|
|
81
|
+
SELECT *,
|
|
82
|
+
CASE
|
|
83
|
+
WHEN trend IS NULL OR seasonal IS NULL THEN NULL
|
|
84
|
+
ELSE observed - trend - seasonal
|
|
85
|
+
END AS residual
|
|
86
|
+
FROM seasonal_centered
|
|
87
|
+
),
|
|
88
|
+
scored AS (
|
|
89
|
+
SELECT *,
|
|
90
|
+
STDDEV_POP(residual) OVER (PARTITION BY query, page) AS resid_std,
|
|
91
|
+
CASE
|
|
92
|
+
WHEN residual IS NOT NULL
|
|
93
|
+
AND STDDEV_POP(residual) OVER (PARTITION BY query, page) > 0
|
|
94
|
+
AND ABS(residual) > 2.0 * STDDEV_POP(residual) OVER (PARTITION BY query, page)
|
|
95
|
+
THEN true ELSE false
|
|
96
|
+
END AS anomaly
|
|
97
|
+
FROM residualed
|
|
98
|
+
),
|
|
99
|
+
per_entity AS (
|
|
100
|
+
SELECT query, page,
|
|
101
|
+
COUNT(*) AS days,
|
|
102
|
+
SUM(impressions) AS total_impressions,
|
|
103
|
+
VAR_POP(detrended) AS var_detrended,
|
|
104
|
+
VAR_POP(seasonal) AS var_seasonal,
|
|
105
|
+
VAR_POP(residual) AS var_residual,
|
|
106
|
+
COUNT(*) FILTER (WHERE anomaly) AS residual_anomalies,
|
|
107
|
+
REGR_SLOPE(observed, epoch(date) / 86400.0) AS trend_slope
|
|
108
|
+
FROM scored
|
|
109
|
+
GROUP BY query, page
|
|
110
|
+
),
|
|
111
|
+
series AS (
|
|
112
|
+
SELECT query, page,
|
|
113
|
+
to_json(list({
|
|
114
|
+
'date': strftime(date, '%Y-%m-%d'),
|
|
115
|
+
'observed': observed,
|
|
116
|
+
'trend': trend,
|
|
117
|
+
'seasonal': seasonal,
|
|
118
|
+
'residual': residual,
|
|
119
|
+
'anomaly': anomaly
|
|
120
|
+
} ORDER BY date)) AS seriesJson
|
|
121
|
+
FROM scored
|
|
122
|
+
GROUP BY query, page
|
|
123
|
+
)
|
|
124
|
+
SELECT
|
|
125
|
+
e.query AS keyword,
|
|
126
|
+
e.page,
|
|
127
|
+
CAST(e.total_impressions AS DOUBLE) AS totalImpressions,
|
|
128
|
+
CAST(e.days AS DOUBLE) AS days,
|
|
129
|
+
CASE
|
|
130
|
+
WHEN e.var_detrended IS NULL OR e.var_detrended = 0 THEN 0.0
|
|
131
|
+
ELSE LEAST(e.var_seasonal / NULLIF(e.var_detrended, 0), 1.0)
|
|
132
|
+
END AS seasonalStrength,
|
|
133
|
+
CASE
|
|
134
|
+
WHEN e.var_detrended IS NULL OR e.var_detrended = 0 THEN 0.0
|
|
135
|
+
ELSE GREATEST(0.0, 1.0 - e.var_residual / NULLIF(e.var_detrended, 0))
|
|
136
|
+
END AS trendStrength,
|
|
137
|
+
CAST(e.residual_anomalies AS DOUBLE) AS residualAnomalies,
|
|
138
|
+
COALESCE(e.trend_slope, 0.0) AS trendSlope,
|
|
139
|
+
s.seriesJson
|
|
140
|
+
FROM per_entity e
|
|
141
|
+
LEFT JOIN series s USING (query, page)
|
|
142
|
+
ORDER BY seasonalStrength DESC, ABS(COALESCE(e.trend_slope, 0.0)) DESC
|
|
143
|
+
LIMIT ${Number(limit)}
|
|
144
|
+
`,
|
|
145
|
+
params: [
|
|
146
|
+
startDate,
|
|
147
|
+
endDate,
|
|
148
|
+
minImpressions
|
|
149
|
+
],
|
|
150
|
+
current: {
|
|
151
|
+
table: "page_queries",
|
|
152
|
+
partitions: enumeratePartitions(startDate, endDate)
|
|
153
|
+
}
|
|
154
|
+
};
|
|
155
|
+
},
|
|
156
|
+
reduceSql(rows, params) {
|
|
157
|
+
const arr = Array.isArray(rows) ? rows : [];
|
|
158
|
+
const metric = params.metric === "clicks" ? "clicks" : "impressions";
|
|
159
|
+
const results = arr.map((r) => ({
|
|
160
|
+
keyword: rowString(r.keyword),
|
|
161
|
+
page: rowString(r.page),
|
|
162
|
+
totalImpressions: num(r.totalImpressions),
|
|
163
|
+
days: num(r.days),
|
|
164
|
+
seasonalStrength: num(r.seasonalStrength),
|
|
165
|
+
trendStrength: num(r.trendStrength),
|
|
166
|
+
residualAnomalies: num(r.residualAnomalies),
|
|
167
|
+
trendSlope: num(r.trendSlope),
|
|
168
|
+
series: parseJsonRows(r.seriesJson).map((s) => ({
|
|
169
|
+
date: rowString(s.date),
|
|
170
|
+
observed: num(s.observed),
|
|
171
|
+
trend: s.trend == null ? null : num(s.trend),
|
|
172
|
+
seasonal: s.seasonal == null ? null : num(s.seasonal),
|
|
173
|
+
residual: s.residual == null ? null : num(s.residual),
|
|
174
|
+
anomaly: rowBoolean(s.anomaly)
|
|
175
|
+
}))
|
|
176
|
+
}));
|
|
177
|
+
return {
|
|
178
|
+
results,
|
|
179
|
+
meta: {
|
|
180
|
+
total: results.length,
|
|
181
|
+
metric,
|
|
182
|
+
avgSeasonalStrength: results.length > 0 ? results.reduce((a, r) => a + r.seasonalStrength, 0) / results.length : 0
|
|
183
|
+
}
|
|
184
|
+
};
|
|
185
|
+
}
|
|
186
|
+
});
|
|
187
|
+
export { stlDecomposeAnalyzer };
|