@gscdump/sdk 7.12.1 → 7.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -0
- package/dist/indexing/cohorts.d.mts +201 -0
- package/dist/indexing/cohorts.mjs +343 -0
- package/dist/indexing/overview.d.mts +158 -0
- package/dist/indexing/overview.mjs +310 -0
- package/dist/indexing/portfolio-markdown.d.mts +60 -0
- package/dist/indexing/portfolio-markdown.mjs +127 -0
- package/dist/indexing/portfolio.d.mts +99 -0
- package/dist/indexing/portfolio.mjs +206 -0
- package/dist/indexing/readings.d.mts +42 -0
- package/dist/indexing/readings.mjs +47 -0
- package/dist/indexing/regression-lead.d.mts +22 -0
- package/dist/indexing/regression-lead.mjs +48 -0
- package/dist/indexing/site-issues.d.mts +15 -0
- package/dist/indexing/site-issues.mjs +85 -0
- package/dist/indexing/trust-gate.d.mts +63 -0
- package/dist/indexing/trust-gate.mjs +74 -0
- package/dist/indexing/window.d.mts +40 -0
- package/dist/indexing/window.mjs +45 -0
- package/dist/indexing.d.mts +10 -0
- package/dist/indexing.mjs +10 -0
- package/dist/package.mjs +1 -1
- package/package.json +9 -3
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import { IndexingSummaryReading, IndexingTrendReading } from "./readings.mjs";
|
|
2
|
+
import { SiteIndexingIssue } from "./site-issues.mjs";
|
|
3
|
+
import { IndexingReadingWindow, IndexingWindowDays } from "./window.mjs";
|
|
4
|
+
export interface IndexingPortfolioSite {
|
|
5
|
+
id: string;
|
|
6
|
+
label: string;
|
|
7
|
+
trend: readonly IndexingTrendReading[];
|
|
8
|
+
}
|
|
9
|
+
export interface IndexingStackBucket {
|
|
10
|
+
/** First day folded into this bucket. */
|
|
11
|
+
start: string;
|
|
12
|
+
/** Last day folded into this bucket. */
|
|
13
|
+
end: string;
|
|
14
|
+
}
|
|
15
|
+
export interface IndexingStackSeries {
|
|
16
|
+
key: string;
|
|
17
|
+
label: string;
|
|
18
|
+
/** One value per bucket, aligned to `buckets`. */
|
|
19
|
+
values: number[];
|
|
20
|
+
isOther: boolean;
|
|
21
|
+
}
|
|
22
|
+
export interface IndexingStack {
|
|
23
|
+
buckets: IndexingStackBucket[];
|
|
24
|
+
series: IndexingStackSeries[];
|
|
25
|
+
}
|
|
26
|
+
export interface IndexingStackOptions {
|
|
27
|
+
/** Series kept by name before the rest fold into All Others. Default 5. */
|
|
28
|
+
topN?: number;
|
|
29
|
+
/** Most buckets on the axis. Default 30. */
|
|
30
|
+
maxBuckets?: number;
|
|
31
|
+
}
|
|
32
|
+
export interface PortfolioIssueSiteCount {
|
|
33
|
+
siteId: string;
|
|
34
|
+
label: string;
|
|
35
|
+
count: number;
|
|
36
|
+
}
|
|
37
|
+
/** One issue across every Site in the Portfolio, at the window end. */
|
|
38
|
+
export interface PortfolioIssueRow {
|
|
39
|
+
id: string;
|
|
40
|
+
label: string;
|
|
41
|
+
count: number;
|
|
42
|
+
siteCount: number;
|
|
43
|
+
filter: SiteIndexingIssue['filter'];
|
|
44
|
+
/** Largest count first. */
|
|
45
|
+
sites: PortfolioIssueSiteCount[];
|
|
46
|
+
}
|
|
47
|
+
/** A Site's reading at the window end, and its indexed rate changes before it. */
|
|
48
|
+
export interface IndexingWindowReading {
|
|
49
|
+
/** The trend reading its issues come from. Null when the Site has a summary and no trend. */
|
|
50
|
+
point: IndexingTrendReading | null;
|
|
51
|
+
indexed: number;
|
|
52
|
+
totalUrls: number;
|
|
53
|
+
indexedPercent: number;
|
|
54
|
+
/** Points, against the last reading on or before 7 days earlier. Null without one. */
|
|
55
|
+
change7d: number | null;
|
|
56
|
+
/** Points, against the last reading on or before 28 days earlier. Null without one. */
|
|
57
|
+
change28d: number | null;
|
|
58
|
+
}
|
|
59
|
+
/** One day of the Portfolio trend: every Site's held level summed. */
|
|
60
|
+
export interface PortfolioIndexingTrendPoint {
|
|
61
|
+
date: string;
|
|
62
|
+
indexedPercent: number;
|
|
63
|
+
indexedCount: number;
|
|
64
|
+
/** Crawl errors: not found, soft 404, and server errors. */
|
|
65
|
+
errors: number;
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* The readings in the window, led by the held reading when one exists. A Site
|
|
69
|
+
* with no reading in the window keeps its held reading, so it holds its level
|
|
70
|
+
* across the window instead of leaving the Portfolio total.
|
|
71
|
+
*/
|
|
72
|
+
export declare function windowReadings<T extends Pick<IndexingTrendReading, 'date' | 'totalUrls'>>(trend: readonly T[], window: IndexingWindowDays): T[];
|
|
73
|
+
/**
|
|
74
|
+
* The one reading every Site value comes from: tiles, rows, cards, popovers,
|
|
75
|
+
* export, and sort. A current window shows the Site summary, which carries the
|
|
76
|
+
* latest level and its 7 and 28 day changes. A past window shows the trend
|
|
77
|
+
* reading at its end, so it never shows today's level.
|
|
78
|
+
*/
|
|
79
|
+
export declare function siteIndexingReading(summary: IndexingSummaryReading | null | undefined, trend: readonly IndexingTrendReading[], window: IndexingReadingWindow): IndexingWindowReading | null;
|
|
80
|
+
/** Not-indexed URLs per day, stacked by Site: the largest Sites plus All Others. */
|
|
81
|
+
export declare function notIndexedStack(sites: readonly IndexingPortfolioSite[], window: IndexingWindowDays, options?: IndexingStackOptions): IndexingStack;
|
|
82
|
+
/** Reported URLs per day, stacked by issue across every Site. */
|
|
83
|
+
export declare function issueStack(sites: readonly IndexingPortfolioSite[], window: IndexingWindowDays, options?: IndexingStackOptions): IndexingStack;
|
|
84
|
+
/** The reported issues across every Site at the window end, largest count first. */
|
|
85
|
+
export declare function portfolioIssueCounts(sites: readonly IndexingPortfolioSite[], window: IndexingReadingWindow): PortfolioIssueRow[];
|
|
86
|
+
/**
|
|
87
|
+
* The Portfolio trend, one point per reported day in the window. Each Site
|
|
88
|
+
* carries its last reading forward, or skipping a day would read as its URLs
|
|
89
|
+
* leaving the total. It starts from its held reading, or else holds its first
|
|
90
|
+
* reading, so joining the window does not read as a rate change. The rate is
|
|
91
|
+
* weighted by inspected URLs.
|
|
92
|
+
*/
|
|
93
|
+
export declare function portfolioIndexingTrend(series: readonly (readonly IndexingTrendReading[])[], window: IndexingWindowDays): PortfolioIndexingTrendPoint[];
|
|
94
|
+
/** The Portfolio rate change between the latest day and the day `days` before it, in points. */
|
|
95
|
+
export declare function portfolioRateChangeOver(trend: readonly PortfolioIndexingTrendPoint[], days: number): number | null;
|
|
96
|
+
/** The rate change from the first to the last point, in points. Null below two points. */
|
|
97
|
+
export declare function indexedRateChange(trend: ReadonlyArray<{
|
|
98
|
+
indexedPercent: number;
|
|
99
|
+
}>): number | null;
|
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
import { clampIndexedPercent, crawlErrorCount, notIndexedCount } from "./readings.mjs";
|
|
2
|
+
import { getSiteIndexingIssues } from "./site-issues.mjs";
|
|
3
|
+
import { readingDatesInWindow } from "./window.mjs";
|
|
4
|
+
const DEFAULT_TOP_N = 5;
|
|
5
|
+
const DEFAULT_MAX_BUCKETS = 30;
|
|
6
|
+
const OTHER_KEY = "__other";
|
|
7
|
+
const OTHER_LABEL = "All Others";
|
|
8
|
+
const DAY_MS = 864e5;
|
|
9
|
+
function byDate(a, b) {
|
|
10
|
+
return a.date.localeCompare(b.date);
|
|
11
|
+
}
|
|
12
|
+
function daysBefore(date, days) {
|
|
13
|
+
return (/* @__PURE__ */ new Date(Date.parse(`${date}T00:00:00Z`) - days * DAY_MS)).toISOString().slice(0, 10);
|
|
14
|
+
}
|
|
15
|
+
function windowReadings(trend, window) {
|
|
16
|
+
const readings = trend.filter((point) => point.totalUrls > 0 && point.date <= window.end).toSorted(byDate);
|
|
17
|
+
const start = readings.findIndex((point) => point.date >= window.start);
|
|
18
|
+
if (start < 0) return readings.slice(-1);
|
|
19
|
+
return readings.slice(Math.max(0, start - 1));
|
|
20
|
+
}
|
|
21
|
+
function readingAtWindowEnd(trend, window) {
|
|
22
|
+
const point = windowReadings(trend, window).at(-1);
|
|
23
|
+
if (!point) return null;
|
|
24
|
+
const readings = trend.filter((reading) => reading.totalUrls > 0);
|
|
25
|
+
function changeOver(days) {
|
|
26
|
+
const target = daysBefore(point.date, days);
|
|
27
|
+
const base = readings.filter((reading) => reading.date <= target).toSorted(byDate).at(-1);
|
|
28
|
+
return base ? Math.round((point.indexedPercent - base.indexedPercent) * 10) / 10 : null;
|
|
29
|
+
}
|
|
30
|
+
return {
|
|
31
|
+
point,
|
|
32
|
+
indexed: point.indexedCount,
|
|
33
|
+
totalUrls: point.totalUrls,
|
|
34
|
+
indexedPercent: point.indexedPercent,
|
|
35
|
+
change7d: changeOver(7),
|
|
36
|
+
change28d: changeOver(28)
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
function windowEndPoint(trend, window) {
|
|
40
|
+
if (!window.current) return readingAtWindowEnd(trend, window)?.point ?? null;
|
|
41
|
+
return trend.toSorted(byDate).at(-1) ?? null;
|
|
42
|
+
}
|
|
43
|
+
function siteIndexingReading(summary, trend, window) {
|
|
44
|
+
if (!window.current) return readingAtWindowEnd(trend, window);
|
|
45
|
+
if (!summary || summary.totalUrls <= 0) return null;
|
|
46
|
+
return {
|
|
47
|
+
point: windowEndPoint(trend, window),
|
|
48
|
+
indexed: summary.indexed,
|
|
49
|
+
totalUrls: summary.totalUrls,
|
|
50
|
+
indexedPercent: summary.indexedPercent,
|
|
51
|
+
change7d: summary.change7d ?? null,
|
|
52
|
+
change28d: summary.change28d ?? null
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
function bucketAxis(dates, maxBuckets) {
|
|
56
|
+
const size = Math.max(1, Math.ceil(dates.length / maxBuckets));
|
|
57
|
+
const out = [];
|
|
58
|
+
for (let start = 0; start < dates.length; start += size) {
|
|
59
|
+
const last = Math.min(start + size, dates.length) - 1;
|
|
60
|
+
out.push({
|
|
61
|
+
bucket: {
|
|
62
|
+
start: dates[start],
|
|
63
|
+
end: dates[last]
|
|
64
|
+
},
|
|
65
|
+
last
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
return out;
|
|
69
|
+
}
|
|
70
|
+
function stackLevels(dates, levels, options) {
|
|
71
|
+
const topN = options.topN ?? DEFAULT_TOP_N;
|
|
72
|
+
const present = levels.filter((series) => series.levels.some((level) => level > 0));
|
|
73
|
+
if (!dates.length || !present.length) return {
|
|
74
|
+
buckets: [],
|
|
75
|
+
series: []
|
|
76
|
+
};
|
|
77
|
+
const ranked = [...present].sort((a, b) => (b.levels.at(-1) ?? 0) - (a.levels.at(-1) ?? 0));
|
|
78
|
+
const axis = bucketAxis(dates, options.maxBuckets ?? DEFAULT_MAX_BUCKETS);
|
|
79
|
+
const pick = (series) => axis.map(({ last }) => series.levels[last] ?? 0);
|
|
80
|
+
const kept = ranked.length === topN + 1 ? ranked.length : topN;
|
|
81
|
+
const series = ranked.slice(0, kept).map((entry) => ({
|
|
82
|
+
key: entry.key,
|
|
83
|
+
label: entry.label,
|
|
84
|
+
values: pick(entry),
|
|
85
|
+
isOther: false
|
|
86
|
+
}));
|
|
87
|
+
const rest = ranked.slice(kept);
|
|
88
|
+
if (rest.length) series.push({
|
|
89
|
+
key: OTHER_KEY,
|
|
90
|
+
label: OTHER_LABEL,
|
|
91
|
+
values: axis.map(({ last }) => rest.reduce((sum, entry) => sum + (entry.levels[last] ?? 0), 0)),
|
|
92
|
+
isOther: true
|
|
93
|
+
});
|
|
94
|
+
return {
|
|
95
|
+
buckets: axis.map(({ bucket }) => bucket),
|
|
96
|
+
series
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
function carryForward(points, dates, read) {
|
|
100
|
+
let next = 0;
|
|
101
|
+
let level = points[0] ? read(points[0]) : 0;
|
|
102
|
+
return dates.map((date) => {
|
|
103
|
+
while (next < points.length && points[next].date <= date) {
|
|
104
|
+
level = read(points[next]);
|
|
105
|
+
next++;
|
|
106
|
+
}
|
|
107
|
+
return level;
|
|
108
|
+
});
|
|
109
|
+
}
|
|
110
|
+
function notIndexedStack(sites, window, options = {}) {
|
|
111
|
+
const inWindow = sites.map((site) => ({
|
|
112
|
+
site,
|
|
113
|
+
points: windowReadings(site.trend, window)
|
|
114
|
+
}));
|
|
115
|
+
const dates = readingDatesInWindow(inWindow.map((entry) => entry.points), window);
|
|
116
|
+
return stackLevels(dates, inWindow.map(({ site, points }) => ({
|
|
117
|
+
key: site.id,
|
|
118
|
+
label: site.label,
|
|
119
|
+
levels: carryForward(points, dates, notIndexedCount)
|
|
120
|
+
})), options);
|
|
121
|
+
}
|
|
122
|
+
function issueStack(sites, window, options = {}) {
|
|
123
|
+
const inWindow = sites.map((site) => windowReadings(site.trend, window));
|
|
124
|
+
const dates = readingDatesInWindow(inWindow, window);
|
|
125
|
+
const byIssue = /* @__PURE__ */ new Map();
|
|
126
|
+
for (const points of inWindow) {
|
|
127
|
+
const issuesByDate = new Map(points.map((point) => [point.date, getSiteIndexingIssues(point)]));
|
|
128
|
+
const issues = [...issuesByDate.values()].flat();
|
|
129
|
+
for (const id of new Set(issues.map((issue) => issue.id))) {
|
|
130
|
+
const levels = carryForward(points, dates, (point) => issuesByDate.get(point.date)?.find((issue) => issue.id === id)?.count ?? 0);
|
|
131
|
+
const label = issues.find((issue) => issue.id === id).label;
|
|
132
|
+
const entry = byIssue.get(id);
|
|
133
|
+
if (entry) entry.levels = entry.levels.map((level, index) => level + (levels[index] ?? 0));
|
|
134
|
+
else byIssue.set(id, {
|
|
135
|
+
key: id,
|
|
136
|
+
label,
|
|
137
|
+
levels
|
|
138
|
+
});
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
return stackLevels(dates, [...byIssue.values()], options);
|
|
142
|
+
}
|
|
143
|
+
function portfolioIssueCounts(sites, window) {
|
|
144
|
+
const rows = /* @__PURE__ */ new Map();
|
|
145
|
+
for (const site of sites) for (const issue of getSiteIndexingIssues(windowEndPoint(site.trend, window))) {
|
|
146
|
+
const row = rows.get(issue.id) ?? {
|
|
147
|
+
id: issue.id,
|
|
148
|
+
label: issue.label,
|
|
149
|
+
count: 0,
|
|
150
|
+
siteCount: 0,
|
|
151
|
+
filter: issue.filter,
|
|
152
|
+
sites: []
|
|
153
|
+
};
|
|
154
|
+
row.count += issue.count;
|
|
155
|
+
row.siteCount += 1;
|
|
156
|
+
row.sites.push({
|
|
157
|
+
siteId: site.id,
|
|
158
|
+
label: site.label,
|
|
159
|
+
count: issue.count
|
|
160
|
+
});
|
|
161
|
+
rows.set(issue.id, row);
|
|
162
|
+
}
|
|
163
|
+
return Array.from(rows.values(), (row) => ({
|
|
164
|
+
...row,
|
|
165
|
+
sites: row.sites.toSorted((a, b) => b.count - a.count)
|
|
166
|
+
})).sort((a, b) => b.count - a.count);
|
|
167
|
+
}
|
|
168
|
+
function portfolioIndexingTrend(series, window) {
|
|
169
|
+
const inWindow = series.map((trend) => windowReadings(trend, window));
|
|
170
|
+
const dates = readingDatesInWindow(inWindow, window);
|
|
171
|
+
const days = new Map(dates.map((date) => [date, {
|
|
172
|
+
totalUrls: 0,
|
|
173
|
+
indexedCount: 0,
|
|
174
|
+
errors: 0
|
|
175
|
+
}]));
|
|
176
|
+
for (const trend of inWindow) {
|
|
177
|
+
let next = 0;
|
|
178
|
+
let latest = trend[0];
|
|
179
|
+
for (const date of dates) {
|
|
180
|
+
while (next < trend.length && trend[next].date <= date) latest = trend[next++];
|
|
181
|
+
if (!latest) continue;
|
|
182
|
+
const day = days.get(date);
|
|
183
|
+
day.totalUrls += latest.totalUrls;
|
|
184
|
+
day.indexedCount += latest.indexedCount ?? 0;
|
|
185
|
+
day.errors += crawlErrorCount(latest);
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
return Array.from(days, ([date, day]) => ({
|
|
189
|
+
date,
|
|
190
|
+
indexedPercent: day.totalUrls > 0 ? clampIndexedPercent(day.indexedCount / day.totalUrls * 100) : 0,
|
|
191
|
+
indexedCount: day.indexedCount,
|
|
192
|
+
errors: day.errors
|
|
193
|
+
})).sort(byDate);
|
|
194
|
+
}
|
|
195
|
+
function portfolioRateChangeOver(trend, days) {
|
|
196
|
+
const last = trend.at(-1);
|
|
197
|
+
if (!last) return null;
|
|
198
|
+
const target = daysBefore(last.date, days);
|
|
199
|
+
const base = trend.findLast((point) => point.date <= target);
|
|
200
|
+
return base ? last.indexedPercent - base.indexedPercent : null;
|
|
201
|
+
}
|
|
202
|
+
function indexedRateChange(trend) {
|
|
203
|
+
if (trend.length < 2) return null;
|
|
204
|
+
return trend.at(-1).indexedPercent - trend[0].indexedPercent;
|
|
205
|
+
}
|
|
206
|
+
export { indexedRateChange, issueStack, notIndexedStack, portfolioIndexingTrend, portfolioIssueCounts, portfolioRateChangeOver, siteIndexingReading, windowReadings };
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { GscdumpIndexingResponse, GscdumpIndexingTrendPoint } from "@gscdump/contracts";
|
|
2
|
+
type IssueKey = 'notFound' | 'soft404' | 'serverError' | 'blockedByRobots';
|
|
3
|
+
type CoverageKey = 'crawledNotIndexed' | 'discoveredNotCrawled';
|
|
4
|
+
type SignalKey = 'richResultsPass' | 'richResultsFail';
|
|
5
|
+
/**
|
|
6
|
+
* One day of a Site's indexing trend, with no null counts and a rate within
|
|
7
|
+
* 0 to 100. The field names are those of `GscdumpIndexingTrendPoint`.
|
|
8
|
+
*/
|
|
9
|
+
export interface IndexingTrendReading {
|
|
10
|
+
/** `YYYY-MM-DD`. */
|
|
11
|
+
date: GscdumpIndexingTrendPoint['date'];
|
|
12
|
+
totalUrls: number;
|
|
13
|
+
indexedCount: number;
|
|
14
|
+
/** 0 to 100. */
|
|
15
|
+
indexedPercent: number;
|
|
16
|
+
issues?: { [Key in IssueKey]?: number; };
|
|
17
|
+
/** Google's "knows about it and chose not to index it" counts. */
|
|
18
|
+
coverage?: { [Key in CoverageKey]?: number; };
|
|
19
|
+
signals?: { [Key in SignalKey]?: number; };
|
|
20
|
+
}
|
|
21
|
+
/** A Site's latest indexing summary, with its rate changes in points. */
|
|
22
|
+
export interface IndexingSummaryReading {
|
|
23
|
+
indexed: number;
|
|
24
|
+
totalUrls: number;
|
|
25
|
+
/** 0 to 100. */
|
|
26
|
+
indexedPercent: number;
|
|
27
|
+
change7d?: number | null;
|
|
28
|
+
change28d?: number | null;
|
|
29
|
+
}
|
|
30
|
+
export interface SiteIndexingReadings {
|
|
31
|
+
summary: IndexingSummaryReading;
|
|
32
|
+
/** Oldest first, as the hosted read returns it. */
|
|
33
|
+
trend: IndexingTrendReading[];
|
|
34
|
+
}
|
|
35
|
+
/** Clamp an index rate to 0 to 100. */
|
|
36
|
+
export declare function clampIndexedPercent(value: number): number;
|
|
37
|
+
/** Parse one hosted Site indexing read into the readings every portfolio function takes. */
|
|
38
|
+
export declare function readSiteIndexing(response: Pick<GscdumpIndexingResponse, 'summary' | 'trend'>): SiteIndexingReadings;
|
|
39
|
+
/** Inspected URLs that are not indexed on this day. Never negative. */
|
|
40
|
+
export declare function notIndexedCount(point: Pick<IndexingTrendReading, 'indexedCount' | 'totalUrls'>): number;
|
|
41
|
+
/** Crawl errors on one reading: not found, soft 404, and server errors. */
|
|
42
|
+
export declare function crawlErrorCount(point: Pick<IndexingTrendReading, 'issues'>): number;
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
function clampIndexedPercent(value) {
|
|
2
|
+
return Math.max(0, Math.min(100, value));
|
|
3
|
+
}
|
|
4
|
+
function count(value) {
|
|
5
|
+
return value ?? void 0;
|
|
6
|
+
}
|
|
7
|
+
function readSiteIndexing(response) {
|
|
8
|
+
return {
|
|
9
|
+
summary: {
|
|
10
|
+
indexed: response.summary.indexed,
|
|
11
|
+
totalUrls: response.summary.totalUrls,
|
|
12
|
+
indexedPercent: clampIndexedPercent(response.summary.indexedPercent),
|
|
13
|
+
change7d: response.summary.change7d,
|
|
14
|
+
change28d: response.summary.change28d
|
|
15
|
+
},
|
|
16
|
+
trend: response.trend.map((point) => {
|
|
17
|
+
const indexedPercent = clampIndexedPercent(point.indexedPercent);
|
|
18
|
+
return {
|
|
19
|
+
date: point.date,
|
|
20
|
+
indexedPercent,
|
|
21
|
+
indexedCount: point.indexedCount ?? Math.round(point.totalUrls * indexedPercent / 100),
|
|
22
|
+
totalUrls: point.totalUrls,
|
|
23
|
+
issues: {
|
|
24
|
+
notFound: count(point.issues.notFound),
|
|
25
|
+
soft404: count(point.issues.soft404),
|
|
26
|
+
serverError: count(point.issues.serverError),
|
|
27
|
+
blockedByRobots: count(point.issues.blockedByRobots)
|
|
28
|
+
},
|
|
29
|
+
coverage: {
|
|
30
|
+
crawledNotIndexed: count(point.coverage?.crawledNotIndexed),
|
|
31
|
+
discoveredNotCrawled: count(point.coverage?.discoveredNotCrawled)
|
|
32
|
+
},
|
|
33
|
+
signals: {
|
|
34
|
+
richResultsPass: count(point.signals.richResultsPass),
|
|
35
|
+
richResultsFail: count(point.signals.richResultsFail)
|
|
36
|
+
}
|
|
37
|
+
};
|
|
38
|
+
})
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
function notIndexedCount(point) {
|
|
42
|
+
return Math.max(0, point.totalUrls - (point.indexedCount ?? 0));
|
|
43
|
+
}
|
|
44
|
+
function crawlErrorCount(point) {
|
|
45
|
+
return (point.issues?.notFound ?? 0) + (point.issues?.soft404 ?? 0) + (point.issues?.serverError ?? 0);
|
|
46
|
+
}
|
|
47
|
+
export { clampIndexedPercent, crawlErrorCount, notIndexedCount, readSiteIndexing };
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { IndexCohortCell } from "./cohorts.mjs";
|
|
2
|
+
import { PartnerIndexingTransitionsV1Response } from "@gscdump/contracts/v1/http";
|
|
3
|
+
/** One recorded transition, as the hosted transitions read returns it. */
|
|
4
|
+
export type IndexingTransitionRow = Pick<PartnerIndexingTransitionsV1Response['data']['transitions'][number], 'url' | 'fromValue' | 'toValue' | 'changedAfter' | 'changedBefore'> & {
|
|
5
|
+
/** One of `GSCDUMP_INDEXING_TRANSITION_FIELDS`. Only `indexStatus` rows count. */
|
|
6
|
+
field: string;
|
|
7
|
+
};
|
|
8
|
+
export type IndexingRegressionLead = {
|
|
9
|
+
_tag: 'lead';
|
|
10
|
+
/** Distinct URLs whose verdict changed away from PASS. */
|
|
11
|
+
count: number;
|
|
12
|
+
title: string;
|
|
13
|
+
detail: string;
|
|
14
|
+
/** The earliest `changedAfter`, as ISO. */
|
|
15
|
+
start: string;
|
|
16
|
+
/** The latest `changedBefore`, as ISO. */
|
|
17
|
+
end: string;
|
|
18
|
+
} | {
|
|
19
|
+
_tag: 'none';
|
|
20
|
+
reason: 'cohort-has-no-path' | 'no-index-regression';
|
|
21
|
+
};
|
|
22
|
+
export declare function selectIndexingRegressionLead(cohort: Pick<IndexCohortCell, 'label' | 'pathPrefix'>, transitions: readonly IndexingTransitionRow[]): IndexingRegressionLead;
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
function pathBelongsTo(path, prefix) {
|
|
2
|
+
return path === prefix || path.startsWith(`${prefix.replace(/\/$/, "")}/`);
|
|
3
|
+
}
|
|
4
|
+
function transitionTime(value) {
|
|
5
|
+
const time = Date.parse(value);
|
|
6
|
+
return Number.isFinite(time) ? time : null;
|
|
7
|
+
}
|
|
8
|
+
const transitionDateFormat = new Intl.DateTimeFormat("en-AU", {
|
|
9
|
+
day: "numeric",
|
|
10
|
+
month: "short",
|
|
11
|
+
timeZone: "UTC",
|
|
12
|
+
year: "numeric"
|
|
13
|
+
});
|
|
14
|
+
function selectIndexingRegressionLead(cohort, transitions) {
|
|
15
|
+
const prefix = cohort.pathPrefix;
|
|
16
|
+
if (!prefix) return {
|
|
17
|
+
_tag: "none",
|
|
18
|
+
reason: "cohort-has-no-path"
|
|
19
|
+
};
|
|
20
|
+
const regressions = transitions.flatMap((transition) => {
|
|
21
|
+
if (transition.field !== "indexStatus" || transition.fromValue !== "PASS" || transition.toValue === "PASS" || !URL.canParse(transition.url)) return [];
|
|
22
|
+
const start = transitionTime(transition.changedAfter);
|
|
23
|
+
const end = transitionTime(transition.changedBefore);
|
|
24
|
+
if (start === null || end === null || end < start) return [];
|
|
25
|
+
if (!pathBelongsTo(new URL(transition.url).pathname, prefix)) return [];
|
|
26
|
+
return [{
|
|
27
|
+
...transition,
|
|
28
|
+
end,
|
|
29
|
+
start
|
|
30
|
+
}];
|
|
31
|
+
});
|
|
32
|
+
if (!regressions.length) return {
|
|
33
|
+
_tag: "none",
|
|
34
|
+
reason: "no-index-regression"
|
|
35
|
+
};
|
|
36
|
+
const count = new Set(regressions.map((transition) => transition.url)).size;
|
|
37
|
+
const first = regressions.reduce((earliest, transition) => transition.start < earliest.start ? transition : earliest);
|
|
38
|
+
const last = regressions.reduce((latest, transition) => transition.end > latest.end ? transition : latest);
|
|
39
|
+
return {
|
|
40
|
+
_tag: "lead",
|
|
41
|
+
count,
|
|
42
|
+
detail: "URL Inspection samples bound the change window; they do not identify the exact drop time.",
|
|
43
|
+
end: last.changedBefore,
|
|
44
|
+
start: first.changedAfter,
|
|
45
|
+
title: `Google dropped ${count.toLocaleString("en-US")} ${count === 1 ? "page" : "pages"} in ${cohort.label} between ${transitionDateFormat.format(new Date(first.changedAfter))} and ${transitionDateFormat.format(new Date(last.changedBefore))}.`
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
export { selectIndexingRegressionLead };
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { IndexingTrendReading } from "./readings.mjs";
|
|
2
|
+
export type SiteIndexingIssuePoint = Pick<IndexingTrendReading, 'issues' | 'coverage' | 'signals'>;
|
|
3
|
+
export interface SiteIndexingIssue {
|
|
4
|
+
/** Kebab-case row id, such as `crawled-not-indexed`. */
|
|
5
|
+
id: string;
|
|
6
|
+
label: string;
|
|
7
|
+
count: number;
|
|
8
|
+
/** The URLs list filter that shows this row's URLs. */
|
|
9
|
+
filter: {
|
|
10
|
+
key: 'issue' | 'facet';
|
|
11
|
+
value: string;
|
|
12
|
+
};
|
|
13
|
+
}
|
|
14
|
+
/** The positive-count issue rows of one reading: failures first, then larger counts. */
|
|
15
|
+
export declare function getSiteIndexingIssues(point: SiteIndexingIssuePoint | null | undefined): SiteIndexingIssue[];
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
const ISSUE_SPECS = [
|
|
2
|
+
{
|
|
3
|
+
id: "server-error",
|
|
4
|
+
label: "Server error",
|
|
5
|
+
rank: 0,
|
|
6
|
+
pick: (p) => p.issues?.serverError,
|
|
7
|
+
filter: {
|
|
8
|
+
key: "issue",
|
|
9
|
+
value: "server_error"
|
|
10
|
+
}
|
|
11
|
+
},
|
|
12
|
+
{
|
|
13
|
+
id: "not-found",
|
|
14
|
+
label: "Not found",
|
|
15
|
+
rank: 0,
|
|
16
|
+
pick: (p) => p.issues?.notFound,
|
|
17
|
+
filter: {
|
|
18
|
+
key: "issue",
|
|
19
|
+
value: "not_found"
|
|
20
|
+
}
|
|
21
|
+
},
|
|
22
|
+
{
|
|
23
|
+
id: "soft-404",
|
|
24
|
+
label: "Soft 404",
|
|
25
|
+
rank: 0,
|
|
26
|
+
pick: (p) => p.issues?.soft404,
|
|
27
|
+
filter: {
|
|
28
|
+
key: "issue",
|
|
29
|
+
value: "soft_404"
|
|
30
|
+
}
|
|
31
|
+
},
|
|
32
|
+
{
|
|
33
|
+
id: "crawled-not-indexed",
|
|
34
|
+
label: "Crawled, not indexed",
|
|
35
|
+
rank: 0,
|
|
36
|
+
pick: (p) => p.coverage?.crawledNotIndexed,
|
|
37
|
+
filter: {
|
|
38
|
+
key: "issue",
|
|
39
|
+
value: "crawled_not_indexed"
|
|
40
|
+
}
|
|
41
|
+
},
|
|
42
|
+
{
|
|
43
|
+
id: "discovered-not-indexed",
|
|
44
|
+
label: "Discovered, not crawled",
|
|
45
|
+
rank: 1,
|
|
46
|
+
pick: (p) => p.coverage?.discoveredNotCrawled,
|
|
47
|
+
filter: {
|
|
48
|
+
key: "issue",
|
|
49
|
+
value: "discovered_not_indexed"
|
|
50
|
+
}
|
|
51
|
+
},
|
|
52
|
+
{
|
|
53
|
+
id: "blocked-robots",
|
|
54
|
+
label: "Blocked by robots.txt",
|
|
55
|
+
rank: 1,
|
|
56
|
+
pick: (p) => p.issues?.blockedByRobots,
|
|
57
|
+
filter: {
|
|
58
|
+
key: "issue",
|
|
59
|
+
value: "blocked_robots"
|
|
60
|
+
}
|
|
61
|
+
},
|
|
62
|
+
{
|
|
63
|
+
id: "rich-results",
|
|
64
|
+
label: "Rich result errors",
|
|
65
|
+
rank: 1,
|
|
66
|
+
pick: (p) => p.signals?.richResultsFail,
|
|
67
|
+
filter: {
|
|
68
|
+
key: "facet",
|
|
69
|
+
value: "rich_results"
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
];
|
|
73
|
+
function getSiteIndexingIssues(point) {
|
|
74
|
+
if (!point) return [];
|
|
75
|
+
return ISSUE_SPECS.map((spec) => ({
|
|
76
|
+
...spec,
|
|
77
|
+
count: spec.pick(point) ?? 0
|
|
78
|
+
})).filter((issue) => issue.count > 0).sort((a, b) => a.rank - b.rank || b.count - a.count).map(({ id, label, count, filter }) => ({
|
|
79
|
+
id,
|
|
80
|
+
label,
|
|
81
|
+
count,
|
|
82
|
+
filter
|
|
83
|
+
}));
|
|
84
|
+
}
|
|
85
|
+
export { getSiteIndexingIssues };
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import { CurrentSitemapScope, SitemapCollapseState } from "@gscdump/analysis/sitemap-health";
|
|
2
|
+
export type TrustGateState = 'ok' | 'unknown' | 'blip' | 'broken' | 'no_sitemap' | 'not_connected' | 'pending';
|
|
3
|
+
export interface TrustGate {
|
|
4
|
+
state: TrustGateState;
|
|
5
|
+
/** One plain sentence naming the state. */
|
|
6
|
+
reason: string;
|
|
7
|
+
}
|
|
8
|
+
/** States where the figures are degraded, so a panel shows its diagnosis variant. */
|
|
9
|
+
export declare const TRUST_DEGRADED_STATES: ReadonlySet<TrustGateState>;
|
|
10
|
+
export declare function isTrustDegraded(state: TrustGateState): boolean;
|
|
11
|
+
/** Search Console downloads a healthy sitemap well within a week. Older than this is stale. */
|
|
12
|
+
export declare const STALE_DOWNLOAD_DAYS = 7;
|
|
13
|
+
/** One live sitemap probe result. */
|
|
14
|
+
export interface SitemapLivenessCheck {
|
|
15
|
+
/** `timeout`: the probe aborted. `error`: a non-2xx response or a network failure. */
|
|
16
|
+
status: 'reachable' | 'timeout' | 'error';
|
|
17
|
+
/** The HTTP status when a response came back. */
|
|
18
|
+
statusCode?: number | null;
|
|
19
|
+
}
|
|
20
|
+
/** One day of a Site's sitemap URL count. */
|
|
21
|
+
export interface SitemapCountPoint {
|
|
22
|
+
/** `YYYY-MM-DD` or an ISO timestamp. Only the order matters. */
|
|
23
|
+
date: string;
|
|
24
|
+
urlCount: number | null | undefined;
|
|
25
|
+
}
|
|
26
|
+
/** Did the sitemap URL count ever drop below 20% of an earlier peak? */
|
|
27
|
+
export declare function sitemapHistoryCollapsed(history: readonly SitemapCountPoint[]): boolean;
|
|
28
|
+
/**
|
|
29
|
+
* Compare the latest sitemap count with the earlier high-water mark. The calm
|
|
30
|
+
* state guard of the Indexing overview, distinct from the history check above,
|
|
31
|
+
* which corroborates probe failures.
|
|
32
|
+
*/
|
|
33
|
+
declare function compareCurrentSitemapScope$1(history: readonly SitemapCountPoint[]): CurrentSitemapScope | null;
|
|
34
|
+
/**
|
|
35
|
+
* Classify only the latest large sitemap drop. A new drop waits for another
|
|
36
|
+
* count, two low counts in a row persist, and an immediate recovery is a blip.
|
|
37
|
+
*/
|
|
38
|
+
export declare function classifyCurrentSitemapDrop(history: readonly SitemapCountPoint[]): SitemapCollapseState;
|
|
39
|
+
export interface TrustGateInput {
|
|
40
|
+
/** Search Console is connected for this Site. */
|
|
41
|
+
connected: boolean;
|
|
42
|
+
/**
|
|
43
|
+
* The URL Inspection sample size, never the sitemap total. 0 means Google has
|
|
44
|
+
* inspected nothing yet. `totalUrls` is the fallback when this is absent.
|
|
45
|
+
*/
|
|
46
|
+
inspectedCount: number | null;
|
|
47
|
+
totalUrls: number | null;
|
|
48
|
+
/** The hosted indexing meta status: `pending`, `partial`, or `complete`. */
|
|
49
|
+
indexingStatus: string | null;
|
|
50
|
+
/** Search Console reports a submitted sitemap it has not parsed yet. */
|
|
51
|
+
sitemapsPending: boolean;
|
|
52
|
+
/** Search Console reports no submitted sitemap. */
|
|
53
|
+
noSitemapsSubmitted: boolean;
|
|
54
|
+
/** The live probe. Null when it has not run or is unavailable. */
|
|
55
|
+
liveness: SitemapLivenessCheck | null;
|
|
56
|
+
/** The latest Search Console sitemap download, as ISO. Null if never downloaded. */
|
|
57
|
+
lastDownloadedAt: string | null;
|
|
58
|
+
/** A sitemap collapse in the window. Corroboration for `broken`. */
|
|
59
|
+
sitemapCollapsed: boolean;
|
|
60
|
+
now: Date;
|
|
61
|
+
}
|
|
62
|
+
export declare function computeTrustGate(input: TrustGateInput): TrustGate;
|
|
63
|
+
export { type CurrentSitemapScope, type SitemapCollapseState, compareCurrentSitemapScope$1 as compareCurrentSitemapScope };
|