@gscdump/sdk 7.12.0 → 7.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -0
- package/dist/indexing/cohorts.d.mts +201 -0
- package/dist/indexing/cohorts.mjs +343 -0
- package/dist/indexing/overview.d.mts +158 -0
- package/dist/indexing/overview.mjs +310 -0
- package/dist/indexing/portfolio-markdown.d.mts +60 -0
- package/dist/indexing/portfolio-markdown.mjs +127 -0
- package/dist/indexing/portfolio.d.mts +99 -0
- package/dist/indexing/portfolio.mjs +206 -0
- package/dist/indexing/readings.d.mts +42 -0
- package/dist/indexing/readings.mjs +47 -0
- package/dist/indexing/regression-lead.d.mts +22 -0
- package/dist/indexing/regression-lead.mjs +48 -0
- package/dist/indexing/site-issues.d.mts +15 -0
- package/dist/indexing/site-issues.mjs +85 -0
- package/dist/indexing/trust-gate.d.mts +63 -0
- package/dist/indexing/trust-gate.mjs +74 -0
- package/dist/indexing/window.d.mts +40 -0
- package/dist/indexing/window.mjs +45 -0
- package/dist/indexing.d.mts +10 -0
- package/dist/indexing.mjs +10 -0
- package/dist/package.mjs +1 -1
- package/package.json +9 -3
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
import { SearchConsoleStageIssue } from "../search-console-stage.mjs";
|
|
2
|
+
import { TrustGate } from "./trust-gate.mjs";
|
|
3
|
+
import { GscdumpIndexingUrl } from "@gscdump/contracts";
|
|
4
|
+
export type IndexingDiagnosisReason = 'none' | 'crawl_blocked' | 'canonical_or_indexability_blocked' | 'unknown_to_google_discovery_gap' | 'discovered_not_crawled' | 'crawled_not_indexed_quality_selection' | 'not_indexed_unspecified' | 'mixed_indexing_failure' | 'data_incomplete';
|
|
5
|
+
export type IndexingDiagnosisActionId = 'complete-indexing-data' | 'fix-crawl-access' | 'align-indexability-signals' | 'improve-discovery' | 'strengthen-crawl-priority' | 'improve-or-consolidate-content' | 'run-url-level-triage';
|
|
6
|
+
/** The part of a diagnosis the Indexing overview reads. */
|
|
7
|
+
export interface IndexingOverviewDiagnosis {
|
|
8
|
+
reason: IndexingDiagnosisReason;
|
|
9
|
+
recommendedActions: ReadonlyArray<{
|
|
10
|
+
id: IndexingDiagnosisActionId;
|
|
11
|
+
}>;
|
|
12
|
+
}
|
|
13
|
+
/** What the caller's diagnosis function receives. */
|
|
14
|
+
export interface IndexingDiagnoseInput<Row> {
|
|
15
|
+
totalUrls: number;
|
|
16
|
+
indexed: number;
|
|
17
|
+
issues: SearchConsoleStageIssue[];
|
|
18
|
+
sampleUrls?: Row[];
|
|
19
|
+
}
|
|
20
|
+
/** One inspected URL, as far as the Inspection stages are concerned. */
|
|
21
|
+
export type InspectionStageRow = Partial<Pick<GscdumpIndexingUrl, 'verdict' | 'indexingState' | 'robotsTxtState' | 'pageFetchState' | 'lastCrawlTime'>>;
|
|
22
|
+
export interface InspectionStageSource {
|
|
23
|
+
urls: ReadonlyArray<InspectionStageRow>;
|
|
24
|
+
pagination: {
|
|
25
|
+
total: number;
|
|
26
|
+
hasMore: boolean;
|
|
27
|
+
};
|
|
28
|
+
inspectedCount: number;
|
|
29
|
+
}
|
|
30
|
+
export interface AvailableInspectionStages {
|
|
31
|
+
_tag: 'available';
|
|
32
|
+
inspected: number;
|
|
33
|
+
crawled: number;
|
|
34
|
+
indexable: number;
|
|
35
|
+
indexed: number;
|
|
36
|
+
}
|
|
37
|
+
export interface UnavailableInspectionStages {
|
|
38
|
+
_tag: 'unavailable';
|
|
39
|
+
reason: 'partial_rows' | 'scope_mismatch';
|
|
40
|
+
}
|
|
41
|
+
export type InspectionStageCounts = AvailableInspectionStages | UnavailableInspectionStages;
|
|
42
|
+
export interface IndexingOverviewInput<Row = unknown> {
|
|
43
|
+
totalUrls: number;
|
|
44
|
+
indexed: number;
|
|
45
|
+
issues: SearchConsoleStageIssue[];
|
|
46
|
+
indexingStatus: 'pending' | 'partial' | 'complete' | null;
|
|
47
|
+
inspectedCount: number;
|
|
48
|
+
sitemapTotal: number;
|
|
49
|
+
/** Sitemap URL counts by day, in any order. */
|
|
50
|
+
sitemapHistory: ReadonlyArray<{
|
|
51
|
+
date: string;
|
|
52
|
+
urlCount: number;
|
|
53
|
+
}>;
|
|
54
|
+
sampleUrls?: Row[];
|
|
55
|
+
stages: InspectionStageCounts;
|
|
56
|
+
trust: TrustGate;
|
|
57
|
+
}
|
|
58
|
+
export interface IndexingOverviewCount {
|
|
59
|
+
id: string;
|
|
60
|
+
value: number;
|
|
61
|
+
label: string;
|
|
62
|
+
}
|
|
63
|
+
export interface IndexingOverviewReason {
|
|
64
|
+
issueType: string;
|
|
65
|
+
label: string;
|
|
66
|
+
count: number;
|
|
67
|
+
countLabel: string;
|
|
68
|
+
severity: NonNullable<SearchConsoleStageIssue['severity']>;
|
|
69
|
+
}
|
|
70
|
+
export type IndexingReasonGroupId = 'crawl_fetch' | 'discovery' | 'indexability' | 'selection' | 'unknown' | 'other';
|
|
71
|
+
export interface IndexingOverviewReasonGroup {
|
|
72
|
+
id: IndexingReasonGroupId;
|
|
73
|
+
label: string;
|
|
74
|
+
countLabel: string;
|
|
75
|
+
reasons: IndexingOverviewReason[];
|
|
76
|
+
}
|
|
77
|
+
export interface IndexingOverviewAction {
|
|
78
|
+
engineActionId: IndexingDiagnosisActionId;
|
|
79
|
+
issueType: string | null;
|
|
80
|
+
label: string;
|
|
81
|
+
detail: string;
|
|
82
|
+
buttonLabel: string;
|
|
83
|
+
}
|
|
84
|
+
export interface IndexingOverviewScope {
|
|
85
|
+
inspected: number;
|
|
86
|
+
indexed: number;
|
|
87
|
+
indexedLabel: string;
|
|
88
|
+
sitemapTotal: number;
|
|
89
|
+
complete: boolean;
|
|
90
|
+
}
|
|
91
|
+
export interface InspectionStageStep {
|
|
92
|
+
id: 'inspected' | 'crawled' | 'indexable' | 'indexed';
|
|
93
|
+
label: 'Inspected' | 'Crawled' | 'Indexable' | 'Indexed';
|
|
94
|
+
value: number;
|
|
95
|
+
countLabel: string;
|
|
96
|
+
}
|
|
97
|
+
export interface InspectionStageSteps {
|
|
98
|
+
_tag: 'available';
|
|
99
|
+
steps: [InspectionStageStep, InspectionStageStep, InspectionStageStep, InspectionStageStep];
|
|
100
|
+
largestLoss: {
|
|
101
|
+
from: InspectionStageStep['label'];
|
|
102
|
+
to: InspectionStageStep['label'];
|
|
103
|
+
count: number;
|
|
104
|
+
} | null;
|
|
105
|
+
}
|
|
106
|
+
export interface IndexingOverviewWaiting {
|
|
107
|
+
_tag: 'waiting';
|
|
108
|
+
/** `not_connected` needs a connect action; `pending` needs patience. */
|
|
109
|
+
state: 'not_connected' | 'pending';
|
|
110
|
+
reason: string;
|
|
111
|
+
progress: {
|
|
112
|
+
inspected: number;
|
|
113
|
+
total: number;
|
|
114
|
+
} | null;
|
|
115
|
+
}
|
|
116
|
+
export interface IndexingOverviewTrustFailure {
|
|
117
|
+
_tag: 'trust_failure';
|
|
118
|
+
reason: string;
|
|
119
|
+
state: 'unknown' | 'broken' | 'no_sitemap';
|
|
120
|
+
}
|
|
121
|
+
export interface IndexingOverviewVerifyScope {
|
|
122
|
+
_tag: 'verify_scope';
|
|
123
|
+
current: number;
|
|
124
|
+
highWater: number;
|
|
125
|
+
reason: string;
|
|
126
|
+
}
|
|
127
|
+
export interface IndexingOverviewPartialScope {
|
|
128
|
+
_tag: 'partial_scope';
|
|
129
|
+
scope: IndexingOverviewScope;
|
|
130
|
+
reason: string;
|
|
131
|
+
}
|
|
132
|
+
export interface IndexingOverviewDiagnosisState<Diagnosis extends IndexingOverviewDiagnosis = IndexingOverviewDiagnosis> {
|
|
133
|
+
_tag: 'diagnosis';
|
|
134
|
+
diagnosis: Diagnosis;
|
|
135
|
+
scope: IndexingOverviewScope;
|
|
136
|
+
counts: IndexingOverviewCount[];
|
|
137
|
+
reasonGroups: IndexingOverviewReasonGroup[];
|
|
138
|
+
stages: InspectionStageSteps | UnavailableInspectionStages;
|
|
139
|
+
primaryAction: IndexingOverviewAction | null;
|
|
140
|
+
trustNote: string | null;
|
|
141
|
+
}
|
|
142
|
+
export type IndexingOverviewModel<Diagnosis extends IndexingOverviewDiagnosis = IndexingOverviewDiagnosis> = IndexingOverviewWaiting | IndexingOverviewTrustFailure | IndexingOverviewVerifyScope | IndexingOverviewPartialScope | IndexingOverviewDiagnosisState<Diagnosis>;
|
|
143
|
+
/**
|
|
144
|
+
* Count the Inspection stages from the inspected rows. Refuses a partial page:
|
|
145
|
+
* stages drawn from 500 of 4,000 rows read as a 4,000-URL answer.
|
|
146
|
+
*/
|
|
147
|
+
export declare function countInspectionStages(input: InspectionStageSource): InspectionStageCounts;
|
|
148
|
+
/** The recommended action that addresses one issue type, if the diagnosis has one. */
|
|
149
|
+
export declare function indexingActionForIssue<Action extends {
|
|
150
|
+
id: IndexingDiagnosisActionId;
|
|
151
|
+
}>(diagnosis: {
|
|
152
|
+
recommendedActions: ReadonlyArray<Action>;
|
|
153
|
+
}, issueType: string): Action | undefined;
|
|
154
|
+
/**
|
|
155
|
+
* The Indexing overview for one Site. `diagnose` is called only once the Trust
|
|
156
|
+
* gate and the sitemap scope allow a diagnosis.
|
|
157
|
+
*/
|
|
158
|
+
export declare function buildIndexingOverviewModel<Row, Diagnosis extends IndexingOverviewDiagnosis>(input: IndexingOverviewInput<Row>, diagnose: (input: IndexingDiagnoseInput<Row>) => Diagnosis): IndexingOverviewModel<Diagnosis>;
|
|
@@ -0,0 +1,310 @@
|
|
|
1
|
+
import { compareCurrentSitemapScope } from "./trust-gate.mjs";
|
|
2
|
+
const hardCrawlIssueTypes = /* @__PURE__ */ new Set([
|
|
3
|
+
"blocked_robots",
|
|
4
|
+
"server_error",
|
|
5
|
+
"access_denied",
|
|
6
|
+
"access_forbidden",
|
|
7
|
+
"blocked_4xx",
|
|
8
|
+
"redirect_error",
|
|
9
|
+
"crawl_error",
|
|
10
|
+
"not_found",
|
|
11
|
+
"soft_404"
|
|
12
|
+
]);
|
|
13
|
+
const issueGroupByType = {
|
|
14
|
+
blocked_robots: "crawl_fetch",
|
|
15
|
+
server_error: "crawl_fetch",
|
|
16
|
+
access_denied: "crawl_fetch",
|
|
17
|
+
access_forbidden: "crawl_fetch",
|
|
18
|
+
blocked_4xx: "crawl_fetch",
|
|
19
|
+
redirect_error: "crawl_fetch",
|
|
20
|
+
crawl_error: "crawl_fetch",
|
|
21
|
+
not_found: "crawl_fetch",
|
|
22
|
+
soft_404: "crawl_fetch",
|
|
23
|
+
discovered_not_indexed: "discovery",
|
|
24
|
+
canonical_mismatch: "indexability",
|
|
25
|
+
canonical_cross_domain: "indexability",
|
|
26
|
+
noindex: "indexability",
|
|
27
|
+
duplicate_no_canonical: "indexability",
|
|
28
|
+
crawled_not_indexed: "selection",
|
|
29
|
+
unknown_to_google: "unknown"
|
|
30
|
+
};
|
|
31
|
+
const reasonGroupLabels = {
|
|
32
|
+
crawl_fetch: "Crawl or fetch faults",
|
|
33
|
+
discovery: "Discovered, not crawled",
|
|
34
|
+
indexability: "Canonical or indexability faults",
|
|
35
|
+
selection: "Crawled, not indexed",
|
|
36
|
+
unknown: "Unknown to Google",
|
|
37
|
+
other: "Other indexing reasons"
|
|
38
|
+
};
|
|
39
|
+
function formatNumber(value) {
|
|
40
|
+
return new Intl.NumberFormat("en").format(Math.max(0, Math.round(value)));
|
|
41
|
+
}
|
|
42
|
+
function denominatorLabel(value, inspected) {
|
|
43
|
+
return `${formatNumber(value)} of ${formatNumber(inspected)} inspected`;
|
|
44
|
+
}
|
|
45
|
+
function countInspectionStages(input) {
|
|
46
|
+
const inspected = Math.max(0, Math.round(input.inspectedCount));
|
|
47
|
+
const total = Math.max(0, Math.round(input.pagination.total));
|
|
48
|
+
if (input.pagination.hasMore || input.urls.length !== total) return {
|
|
49
|
+
_tag: "unavailable",
|
|
50
|
+
reason: "partial_rows"
|
|
51
|
+
};
|
|
52
|
+
if (total !== inspected) return {
|
|
53
|
+
_tag: "unavailable",
|
|
54
|
+
reason: "scope_mismatch"
|
|
55
|
+
};
|
|
56
|
+
let crawled = 0;
|
|
57
|
+
let indexable = 0;
|
|
58
|
+
let indexed = 0;
|
|
59
|
+
for (const row of input.urls) {
|
|
60
|
+
const isIndexed = row.verdict === "PASS";
|
|
61
|
+
const isCrawled = isIndexed || Boolean(row.lastCrawlTime) || row.pageFetchState === "SUCCESSFUL";
|
|
62
|
+
const isIndexable = isIndexed || isCrawled && row.pageFetchState === "SUCCESSFUL" && row.indexingState === "INDEXING_ALLOWED" && row.robotsTxtState !== "DISALLOWED";
|
|
63
|
+
if (isCrawled) crawled++;
|
|
64
|
+
if (isIndexable) indexable++;
|
|
65
|
+
if (isIndexed) indexed++;
|
|
66
|
+
}
|
|
67
|
+
return {
|
|
68
|
+
_tag: "available",
|
|
69
|
+
inspected,
|
|
70
|
+
crawled,
|
|
71
|
+
indexable,
|
|
72
|
+
indexed
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
function inspectionStageSteps(stages, inspected) {
|
|
76
|
+
if (stages._tag === "unavailable") return stages;
|
|
77
|
+
if (stages.inspected !== inspected) return {
|
|
78
|
+
_tag: "unavailable",
|
|
79
|
+
reason: "scope_mismatch"
|
|
80
|
+
};
|
|
81
|
+
const values = [
|
|
82
|
+
stages.inspected,
|
|
83
|
+
Math.min(stages.inspected, stages.crawled),
|
|
84
|
+
Math.min(stages.crawled, stages.indexable),
|
|
85
|
+
Math.min(stages.indexable, stages.indexed)
|
|
86
|
+
];
|
|
87
|
+
const labels = [
|
|
88
|
+
"Inspected",
|
|
89
|
+
"Crawled",
|
|
90
|
+
"Indexable",
|
|
91
|
+
"Indexed"
|
|
92
|
+
];
|
|
93
|
+
const ids = [
|
|
94
|
+
"inspected",
|
|
95
|
+
"crawled",
|
|
96
|
+
"indexable",
|
|
97
|
+
"indexed"
|
|
98
|
+
];
|
|
99
|
+
const steps = values.map((value, index) => ({
|
|
100
|
+
id: ids[index],
|
|
101
|
+
label: labels[index],
|
|
102
|
+
value,
|
|
103
|
+
countLabel: denominatorLabel(value, inspected)
|
|
104
|
+
}));
|
|
105
|
+
let largestLoss = null;
|
|
106
|
+
for (let index = 1; index < steps.length; index++) {
|
|
107
|
+
const previous = steps[index - 1];
|
|
108
|
+
const current = steps[index];
|
|
109
|
+
const count = previous.value - current.value;
|
|
110
|
+
if (count > 0 && (!largestLoss || count > largestLoss.count)) largestLoss = {
|
|
111
|
+
from: previous.label,
|
|
112
|
+
to: current.label,
|
|
113
|
+
count
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
return {
|
|
117
|
+
_tag: "available",
|
|
118
|
+
steps,
|
|
119
|
+
largestLoss
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
function issueTypeForAction(actionId, issues) {
|
|
123
|
+
if (actionId === "fix-crawl-access") return issues.filter((issue) => hardCrawlIssueTypes.has(issue.type) && issue.count > 0).sort((a, b) => b.count - a.count).at(0)?.type ?? null;
|
|
124
|
+
return (() => {
|
|
125
|
+
switch (actionId) {
|
|
126
|
+
case "align-indexability-signals": return [
|
|
127
|
+
"canonical_cross_domain",
|
|
128
|
+
"canonical_mismatch",
|
|
129
|
+
"noindex",
|
|
130
|
+
"duplicate_no_canonical"
|
|
131
|
+
];
|
|
132
|
+
case "improve-discovery": return ["unknown_to_google"];
|
|
133
|
+
case "strengthen-crawl-priority": return ["discovered_not_indexed"];
|
|
134
|
+
case "improve-or-consolidate-content":
|
|
135
|
+
case "run-url-level-triage": return ["crawled_not_indexed", "not_indexed"];
|
|
136
|
+
case "complete-indexing-data": return [];
|
|
137
|
+
}
|
|
138
|
+
})().map((type) => issues.find((issue) => issue.type === type && issue.count > 0)).find(Boolean)?.type ?? null;
|
|
139
|
+
}
|
|
140
|
+
const actionIdsByReason = {
|
|
141
|
+
none: [],
|
|
142
|
+
crawl_blocked: ["fix-crawl-access"],
|
|
143
|
+
canonical_or_indexability_blocked: ["align-indexability-signals"],
|
|
144
|
+
unknown_to_google_discovery_gap: ["improve-discovery"],
|
|
145
|
+
discovered_not_crawled: ["strengthen-crawl-priority"],
|
|
146
|
+
crawled_not_indexed_quality_selection: ["improve-or-consolidate-content", "run-url-level-triage"],
|
|
147
|
+
not_indexed_unspecified: ["run-url-level-triage"],
|
|
148
|
+
mixed_indexing_failure: ["run-url-level-triage"],
|
|
149
|
+
data_incomplete: ["complete-indexing-data"]
|
|
150
|
+
};
|
|
151
|
+
function actionForDiagnosis(diagnosis) {
|
|
152
|
+
const alignedIds = actionIdsByReason[diagnosis.reason];
|
|
153
|
+
return diagnosis.recommendedActions.find((action) => alignedIds.includes(action.id)) ?? diagnosis.recommendedActions[0];
|
|
154
|
+
}
|
|
155
|
+
function indexingActionForIssue(diagnosis, issueType) {
|
|
156
|
+
const alignedIds = (() => {
|
|
157
|
+
if (hardCrawlIssueTypes.has(issueType)) return ["fix-crawl-access"];
|
|
158
|
+
if ([
|
|
159
|
+
"noindex",
|
|
160
|
+
"canonical_mismatch",
|
|
161
|
+
"canonical_cross_domain",
|
|
162
|
+
"duplicate_no_canonical",
|
|
163
|
+
"alternate_canonical"
|
|
164
|
+
].includes(issueType)) return ["align-indexability-signals"];
|
|
165
|
+
if (issueType === "unknown_to_google") return ["improve-discovery"];
|
|
166
|
+
if (issueType === "discovered_not_indexed") return ["strengthen-crawl-priority"];
|
|
167
|
+
if (["crawled_not_indexed", "not_indexed"].includes(issueType)) return ["improve-or-consolidate-content", "run-url-level-triage"];
|
|
168
|
+
return [];
|
|
169
|
+
})();
|
|
170
|
+
return diagnosis.recommendedActions.find((action) => alignedIds.includes(action.id));
|
|
171
|
+
}
|
|
172
|
+
const actionCopy = {
|
|
173
|
+
"fix-crawl-access": {
|
|
174
|
+
label: "Fix crawl or fetch blockers",
|
|
175
|
+
detail: "Remove unintended robots blocks, response errors, authentication, or redirect failures.",
|
|
176
|
+
buttonLabel: "Review blocked URLs"
|
|
177
|
+
},
|
|
178
|
+
"align-indexability-signals": {
|
|
179
|
+
label: "Align indexability signals",
|
|
180
|
+
detail: "Remove unintended noindex directives and align canonicals, redirects, sitemap URLs, and internal links.",
|
|
181
|
+
buttonLabel: "Review conflicting URLs"
|
|
182
|
+
},
|
|
183
|
+
"improve-discovery": {
|
|
184
|
+
label: "Make these URLs discoverable",
|
|
185
|
+
detail: "Put canonical URLs in the sitemap and link to them from indexed pages.",
|
|
186
|
+
buttonLabel: "Review unknown URLs"
|
|
187
|
+
},
|
|
188
|
+
"strengthen-crawl-priority": {
|
|
189
|
+
label: "Strengthen crawl priority",
|
|
190
|
+
detail: "Reduce low-value URL inventory and link to important canonical pages from already-indexed pages.",
|
|
191
|
+
buttonLabel: "Review uncrawled URLs"
|
|
192
|
+
},
|
|
193
|
+
"improve-or-consolidate-content": {
|
|
194
|
+
label: "Improve or consolidate these pages",
|
|
195
|
+
detail: "Compare rendered content, duplicate intent, canonicals, and internal links before requesting indexing again.",
|
|
196
|
+
buttonLabel: "Review crawled URLs"
|
|
197
|
+
},
|
|
198
|
+
"run-url-level-triage": {
|
|
199
|
+
label: "Inspect crawled URLs before changing content",
|
|
200
|
+
detail: "Check fetch, robots, noindex, canonicals, rendered content, and duplication before changing pages in bulk.",
|
|
201
|
+
buttonLabel: "Review crawled URLs"
|
|
202
|
+
},
|
|
203
|
+
"complete-indexing-data": {
|
|
204
|
+
label: "Complete URL Inspection data",
|
|
205
|
+
detail: "Finish URL Inspection collection before assigning a cause.",
|
|
206
|
+
buttonLabel: "View inspected URLs"
|
|
207
|
+
}
|
|
208
|
+
};
|
|
209
|
+
function groupReasons(issues, inspected, primaryIssueType) {
|
|
210
|
+
const groups = /* @__PURE__ */ new Map();
|
|
211
|
+
for (const issue of issues) {
|
|
212
|
+
if (issue.count <= 0 || issue.type === "not_indexed") continue;
|
|
213
|
+
const groupId = issueGroupByType[issue.type];
|
|
214
|
+
if (!groupId) continue;
|
|
215
|
+
const group = groups.get(groupId) ?? [];
|
|
216
|
+
group.push({
|
|
217
|
+
issueType: issue.type,
|
|
218
|
+
label: issue.label,
|
|
219
|
+
count: issue.count,
|
|
220
|
+
countLabel: denominatorLabel(issue.count, inspected),
|
|
221
|
+
severity: issue.severity ?? "info"
|
|
222
|
+
});
|
|
223
|
+
groups.set(groupId, group);
|
|
224
|
+
}
|
|
225
|
+
const primaryGroup = primaryIssueType ? issueGroupByType[primaryIssueType] ?? "other" : null;
|
|
226
|
+
return Array.from(groups.entries(), ([id, reasons]) => ({
|
|
227
|
+
id,
|
|
228
|
+
label: reasonGroupLabels[id],
|
|
229
|
+
countLabel: denominatorLabel(reasons.reduce((total, reason) => total + reason.count, 0), inspected),
|
|
230
|
+
reasons: reasons.sort((left, right) => Number(right.issueType === primaryIssueType) - Number(left.issueType === primaryIssueType) || right.count - left.count).slice(0, 3)
|
|
231
|
+
})).sort((left, right) => Number(right.id === primaryGroup) - Number(left.id === primaryGroup) || Math.max(...right.reasons.map((reason) => reason.count)) - Math.max(...left.reasons.map((reason) => reason.count))).slice(0, 3);
|
|
232
|
+
}
|
|
233
|
+
function buildIndexingOverviewModel(input, diagnose) {
|
|
234
|
+
if (input.trust.state === "not_connected" || input.trust.state === "pending") return {
|
|
235
|
+
_tag: "waiting",
|
|
236
|
+
state: input.trust.state,
|
|
237
|
+
reason: input.trust.reason,
|
|
238
|
+
progress: input.sitemapTotal > 0 ? {
|
|
239
|
+
inspected: Math.max(0, input.inspectedCount),
|
|
240
|
+
total: Math.max(0, input.sitemapTotal)
|
|
241
|
+
} : null
|
|
242
|
+
};
|
|
243
|
+
if (input.trust.state === "unknown" || input.trust.state === "broken" || input.trust.state === "no_sitemap") return {
|
|
244
|
+
_tag: "trust_failure",
|
|
245
|
+
state: input.trust.state,
|
|
246
|
+
reason: input.trust.reason
|
|
247
|
+
};
|
|
248
|
+
const sitemapScope = compareCurrentSitemapScope(input.sitemapHistory);
|
|
249
|
+
if (sitemapScope?.collapsed) return {
|
|
250
|
+
_tag: "verify_scope",
|
|
251
|
+
current: sitemapScope.current,
|
|
252
|
+
highWater: sitemapScope.highWater,
|
|
253
|
+
reason: `Today's sitemap scope is ${formatNumber(sitemapScope.current)} URLs, down from a recent high of ${formatNumber(sitemapScope.highWater)}.`
|
|
254
|
+
};
|
|
255
|
+
const inspected = Math.max(0, input.inspectedCount || input.totalUrls);
|
|
256
|
+
const complete = input.indexingStatus === "complete" && (input.sitemapTotal <= 0 || inspected >= input.sitemapTotal);
|
|
257
|
+
const scope = {
|
|
258
|
+
inspected,
|
|
259
|
+
indexed: Math.max(0, input.indexed),
|
|
260
|
+
indexedLabel: denominatorLabel(input.indexed, inspected),
|
|
261
|
+
sitemapTotal: Math.max(0, input.sitemapTotal),
|
|
262
|
+
complete
|
|
263
|
+
};
|
|
264
|
+
const diagnosis = diagnose({
|
|
265
|
+
totalUrls: input.totalUrls,
|
|
266
|
+
indexed: input.indexed,
|
|
267
|
+
issues: input.issues,
|
|
268
|
+
sampleUrls: input.sampleUrls
|
|
269
|
+
});
|
|
270
|
+
if (diagnosis.reason === "none" && !complete) return {
|
|
271
|
+
_tag: "partial_scope",
|
|
272
|
+
scope,
|
|
273
|
+
reason: `${formatNumber(inspected)} of ${formatNumber(input.sitemapTotal)} sitemap URLs are inspected. Keep collecting evidence before calling coverage complete.`
|
|
274
|
+
};
|
|
275
|
+
const action = actionForDiagnosis(diagnosis);
|
|
276
|
+
const primaryAction = action ? {
|
|
277
|
+
engineActionId: action.id,
|
|
278
|
+
issueType: issueTypeForAction(action.id, input.issues),
|
|
279
|
+
...actionCopy[action.id]
|
|
280
|
+
} : null;
|
|
281
|
+
const reasonGroups = groupReasons(input.issues, inspected, primaryAction?.issueType ?? null);
|
|
282
|
+
const notIndexed = Math.max(0, input.totalUrls - input.indexed);
|
|
283
|
+
return {
|
|
284
|
+
_tag: "diagnosis",
|
|
285
|
+
diagnosis,
|
|
286
|
+
scope,
|
|
287
|
+
counts: [
|
|
288
|
+
{
|
|
289
|
+
id: "indexed",
|
|
290
|
+
value: input.indexed,
|
|
291
|
+
label: denominatorLabel(input.indexed, inspected)
|
|
292
|
+
},
|
|
293
|
+
{
|
|
294
|
+
id: "not-indexed",
|
|
295
|
+
value: notIndexed,
|
|
296
|
+
label: denominatorLabel(notIndexed, inspected)
|
|
297
|
+
},
|
|
298
|
+
...reasonGroups.flatMap((group) => group.reasons).map((reason) => ({
|
|
299
|
+
id: reason.issueType,
|
|
300
|
+
value: reason.count,
|
|
301
|
+
label: reason.countLabel
|
|
302
|
+
}))
|
|
303
|
+
],
|
|
304
|
+
reasonGroups,
|
|
305
|
+
stages: inspectionStageSteps(input.stages, inspected),
|
|
306
|
+
primaryAction,
|
|
307
|
+
trustNote: input.trust.state === "blip" ? input.trust.reason : null
|
|
308
|
+
};
|
|
309
|
+
}
|
|
310
|
+
export { buildIndexingOverviewModel, countInspectionStages, indexingActionForIssue };
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import { Period } from "../period.mjs";
|
|
2
|
+
/** One day of a Site's exported trend. */
|
|
3
|
+
export interface IndexingMarkdownTrendPoint {
|
|
4
|
+
date: string;
|
|
5
|
+
/** Indexed share at this point, 0 to 100. */
|
|
6
|
+
indexedPercent: number;
|
|
7
|
+
/** Crawl errors at this point: not found, soft 404, and server errors. */
|
|
8
|
+
errors: number;
|
|
9
|
+
}
|
|
10
|
+
export interface IndexingMarkdownIssues {
|
|
11
|
+
notFound: number;
|
|
12
|
+
soft404: number;
|
|
13
|
+
serverError: number;
|
|
14
|
+
}
|
|
15
|
+
export interface IndexingMarkdownSite {
|
|
16
|
+
name: string;
|
|
17
|
+
hostname: string;
|
|
18
|
+
hasGsc: boolean;
|
|
19
|
+
/** The Site reading at the window end. */
|
|
20
|
+
indexed: number | null;
|
|
21
|
+
totalUrls: number | null;
|
|
22
|
+
indexedPercent: number | null;
|
|
23
|
+
change7d: number | null;
|
|
24
|
+
change28d: number | null;
|
|
25
|
+
/** The window's daily series. */
|
|
26
|
+
trend: IndexingMarkdownTrendPoint[] | null;
|
|
27
|
+
/** Crawl errors from the reading at the window end. */
|
|
28
|
+
latestIssues: IndexingMarkdownIssues | null;
|
|
29
|
+
}
|
|
30
|
+
export interface IndexingMarkdownInput {
|
|
31
|
+
title: string;
|
|
32
|
+
periodLabel: string;
|
|
33
|
+
range: {
|
|
34
|
+
start: string;
|
|
35
|
+
end: string;
|
|
36
|
+
} | null;
|
|
37
|
+
sites: IndexingMarkdownSite[];
|
|
38
|
+
}
|
|
39
|
+
export interface IndexingMarkdownBucket {
|
|
40
|
+
start: string;
|
|
41
|
+
end: string;
|
|
42
|
+
/** Mean indexed share across the bucket's days. */
|
|
43
|
+
indexedPercent: number;
|
|
44
|
+
/** Summed errors across the bucket's days. */
|
|
45
|
+
errors: number;
|
|
46
|
+
}
|
|
47
|
+
/** Days per exported bucket for a Period. */
|
|
48
|
+
export declare function indexingMarkdownBucketDays(period: Period | string): number;
|
|
49
|
+
/**
|
|
50
|
+
* Bucket a daily series into rolling equal-length windows anchored to the most
|
|
51
|
+
* recent date, so the latest bucket is always full. The indexed share is the
|
|
52
|
+
* bucket mean; errors are summed. Oldest first.
|
|
53
|
+
*/
|
|
54
|
+
export declare function bucketIndexingSeries(trend: readonly IndexingMarkdownTrendPoint[], bucketDays: number): IndexingMarkdownBucket[];
|
|
55
|
+
/**
|
|
56
|
+
* The Markdown indexing payload for a set of Sites. With one Site (a row copy)
|
|
57
|
+
* the Portfolio table is skipped; with many (a Portfolio copy) it leads. Sites
|
|
58
|
+
* without indexing data are dropped, and the Markdown says how many.
|
|
59
|
+
*/
|
|
60
|
+
export declare function buildIndexingPortfolioMarkdown(input: IndexingMarkdownInput, period: Period | string): string;
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
const BUCKET_DAYS = {
|
|
2
|
+
"7d": 1,
|
|
3
|
+
"28d": 7,
|
|
4
|
+
"3m": 7,
|
|
5
|
+
"6m": 14,
|
|
6
|
+
"12m": 30
|
|
7
|
+
};
|
|
8
|
+
function indexingMarkdownBucketDays(period) {
|
|
9
|
+
return Object.hasOwn(BUCKET_DAYS, period) ? BUCKET_DAYS[period] : 7;
|
|
10
|
+
}
|
|
11
|
+
function dayNumber(date) {
|
|
12
|
+
return Math.floor(Date.parse(`${date}T00:00:00Z`) / 864e5);
|
|
13
|
+
}
|
|
14
|
+
function bucketIndexingSeries(trend, bucketDays) {
|
|
15
|
+
if (!trend.length) return [];
|
|
16
|
+
const sorted = trend.toSorted((a, b) => a.date.localeCompare(b.date));
|
|
17
|
+
const maxDay = dayNumber(sorted.at(-1).date);
|
|
18
|
+
const groups = /* @__PURE__ */ new Map();
|
|
19
|
+
for (const point of sorted) {
|
|
20
|
+
const index = Math.floor((maxDay - dayNumber(point.date)) / bucketDays);
|
|
21
|
+
const group = groups.get(index) ?? [];
|
|
22
|
+
group.push(point);
|
|
23
|
+
groups.set(index, group);
|
|
24
|
+
}
|
|
25
|
+
return [...groups.keys()].sort((a, b) => b - a).map((index) => {
|
|
26
|
+
const points = groups.get(index);
|
|
27
|
+
return {
|
|
28
|
+
start: points[0].date,
|
|
29
|
+
end: points.at(-1).date,
|
|
30
|
+
indexedPercent: points.reduce((sum, point) => sum + point.indexedPercent, 0) / points.length,
|
|
31
|
+
errors: points.reduce((sum, point) => sum + point.errors, 0)
|
|
32
|
+
};
|
|
33
|
+
});
|
|
34
|
+
}
|
|
35
|
+
function mergeTrend(sites) {
|
|
36
|
+
const days = /* @__PURE__ */ new Map();
|
|
37
|
+
for (const site of sites) for (const point of site.trend ?? []) {
|
|
38
|
+
const day = days.get(point.date) ?? {
|
|
39
|
+
percentSum: 0,
|
|
40
|
+
percentCount: 0,
|
|
41
|
+
errors: 0
|
|
42
|
+
};
|
|
43
|
+
day.percentSum += point.indexedPercent;
|
|
44
|
+
day.percentCount += 1;
|
|
45
|
+
day.errors += point.errors;
|
|
46
|
+
days.set(point.date, day);
|
|
47
|
+
}
|
|
48
|
+
return [...days.entries()].sort((a, b) => a[0].localeCompare(b[0])).map(([date, day]) => ({
|
|
49
|
+
date,
|
|
50
|
+
indexedPercent: day.percentSum / day.percentCount,
|
|
51
|
+
errors: day.errors
|
|
52
|
+
}));
|
|
53
|
+
}
|
|
54
|
+
function formatCount(value) {
|
|
55
|
+
return Math.round(value).toLocaleString("en-US");
|
|
56
|
+
}
|
|
57
|
+
function signed(value, digits, suffix = "") {
|
|
58
|
+
return `${value >= 0 ? "+" : ""}${value.toFixed(digits)}${suffix}`;
|
|
59
|
+
}
|
|
60
|
+
function issuesLabel(issues) {
|
|
61
|
+
if (!issues) return "n/a";
|
|
62
|
+
const total = issues.notFound + issues.soft404 + issues.serverError;
|
|
63
|
+
if (total === 0) return "0";
|
|
64
|
+
const parts = [];
|
|
65
|
+
if (issues.notFound) parts.push(`not-found ${formatCount(issues.notFound)}`);
|
|
66
|
+
if (issues.soft404) parts.push(`soft-404 ${formatCount(issues.soft404)}`);
|
|
67
|
+
if (issues.serverError) parts.push(`server-error ${formatCount(issues.serverError)}`);
|
|
68
|
+
return `${formatCount(total)} (${parts.join(", ")})`;
|
|
69
|
+
}
|
|
70
|
+
function trendTable(buckets) {
|
|
71
|
+
const lines = ["| Period | Indexed % | Errors |", "| --- | ---: | ---: |"];
|
|
72
|
+
for (const bucket of buckets) lines.push(`| ${bucket.start} → ${bucket.end} | ${bucket.indexedPercent.toFixed(1)}% | ${formatCount(bucket.errors)} |`);
|
|
73
|
+
return lines.join("\n");
|
|
74
|
+
}
|
|
75
|
+
function siteSummary(site) {
|
|
76
|
+
const lines = [];
|
|
77
|
+
if (site.indexed != null && site.totalUrls != null) {
|
|
78
|
+
const percent = site.indexedPercent != null ? ` (${site.indexedPercent.toFixed(1)}%)` : "";
|
|
79
|
+
lines.push(`Indexed: ${formatCount(site.indexed)} / ${formatCount(site.totalUrls)}${percent}`);
|
|
80
|
+
}
|
|
81
|
+
const deltas = [];
|
|
82
|
+
if (site.change7d != null) deltas.push(`Δ 7d ${signed(site.change7d, 1, "%")}`);
|
|
83
|
+
if (site.change28d != null) deltas.push(`Δ 28d ${signed(site.change28d, 1, "%")}`);
|
|
84
|
+
if (deltas.length) lines.push(deltas.join(" "));
|
|
85
|
+
lines.push(`Errors: ${issuesLabel(site.latestIssues)}`);
|
|
86
|
+
return lines.join("\n");
|
|
87
|
+
}
|
|
88
|
+
const SCOPE_NOTE = "Scope: registered host only, subdomains excluded. Source: Google Search Console URL Inspection / coverage.";
|
|
89
|
+
const METHOD_NOTE = "Buckets are rolling and equal-length, oldest first. Indexed % is the mean across the bucket's days; errors are summed.";
|
|
90
|
+
function buildIndexingPortfolioMarkdown(input, period) {
|
|
91
|
+
const sites = input.sites.filter((site) => site.hasGsc && site.trend && site.trend.length);
|
|
92
|
+
const bucketDays = indexingMarkdownBucketDays(period);
|
|
93
|
+
const out = [`# Indexing coverage: ${input.title}`];
|
|
94
|
+
const range = input.range ? ` (${input.range.start} → ${input.range.end})` : "";
|
|
95
|
+
out.push(`Period: ${input.periodLabel}${range}.`);
|
|
96
|
+
out.push(SCOPE_NOTE);
|
|
97
|
+
out.push(METHOD_NOTE);
|
|
98
|
+
if (sites.length < input.sites.length) out.push(`Coverage: ${sites.length} of ${input.sites.length} sites have indexing data; the rest are omitted.`);
|
|
99
|
+
out.push("");
|
|
100
|
+
if (!sites.length) {
|
|
101
|
+
out.push("_No indexing data available for the selected period._");
|
|
102
|
+
return `${out.join("\n").trim()}\n`;
|
|
103
|
+
}
|
|
104
|
+
if (sites.length > 1) {
|
|
105
|
+
let indexed = 0;
|
|
106
|
+
let total = 0;
|
|
107
|
+
for (const site of sites) {
|
|
108
|
+
indexed += site.indexed ?? 0;
|
|
109
|
+
total += site.totalUrls ?? 0;
|
|
110
|
+
}
|
|
111
|
+
out.push(`## Portfolio (${sites.length} sites)`);
|
|
112
|
+
out.push(`Indexed: ${formatCount(indexed)} / ${formatCount(total)}${total > 0 ? ` (${(indexed / total * 100).toFixed(1)}%)` : ""}`);
|
|
113
|
+
out.push("");
|
|
114
|
+
out.push(trendTable(bucketIndexingSeries(mergeTrend(sites), bucketDays)));
|
|
115
|
+
out.push("");
|
|
116
|
+
}
|
|
117
|
+
for (const site of sites) {
|
|
118
|
+
const label = site.hostname && site.hostname !== site.name ? `${site.name} (${site.hostname})` : site.name;
|
|
119
|
+
out.push(`## ${label}`);
|
|
120
|
+
out.push(siteSummary(site));
|
|
121
|
+
out.push("");
|
|
122
|
+
out.push(trendTable(bucketIndexingSeries(site.trend, bucketDays)));
|
|
123
|
+
out.push("");
|
|
124
|
+
}
|
|
125
|
+
return `${out.join("\n").trim()}\n`;
|
|
126
|
+
}
|
|
127
|
+
export { bucketIndexingSeries, buildIndexingPortfolioMarkdown, indexingMarkdownBucketDays };
|