pagesight 0.16.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -83
- package/docs/credentials.md +54 -0
- package/docs/diagnostics.md +85 -0
- package/docs/snapshots.md +123 -0
- package/docs/usage.md +130 -0
- package/package.json +26 -29
- package/src/api/bing.ts +83 -0
- package/src/api/compare-snapshots.ts +377 -0
- package/src/api/discover.ts +39 -0
- package/src/api/doctor.ts +35 -0
- package/src/api/evidence-schema.ts +78 -0
- package/src/api/evidence.ts +99 -0
- package/src/api/execute.ts +147 -0
- package/src/api/index.ts +5 -0
- package/src/api/reports.ts +90 -0
- package/src/api/schema.ts +207 -0
- package/src/api/snapshot.ts +183 -0
- package/src/api/ui-findings.ts +68 -0
- package/src/cli.ts +182 -0
- package/src/http.ts +39 -0
- package/src/index.ts +8 -25
- package/src/mcp-server.ts +27 -0
- package/src/mcp.ts +5 -0
- package/src/providers/bing.ts +48 -0
- package/src/{lib → providers}/crux.ts +4 -9
- package/src/providers/ga.ts +114 -0
- package/src/providers/google-tokens.ts +86 -0
- package/src/providers/gsc-auth.ts +93 -0
- package/src/{lib → providers}/gsc.ts +27 -24
- package/src/{lib/psi.ts → providers/pagespeed.ts} +3 -14
- package/src/shared/dates.ts +32 -0
- package/src/shared/http.ts +63 -0
- package/src/tools/ai.ts +19 -27
- package/src/tools/audit.ts +26 -97
- package/src/tools/observe.ts +28 -0
- package/src/tools/page/analyze.ts +194 -0
- package/src/tools/page/batch.ts +163 -0
- package/src/tools/page/contrast.ts +128 -0
- package/src/tools/page/links.ts +200 -0
- package/src/tools/page/metadata.ts +225 -0
- package/src/tools/page/structured-data.ts +288 -0
- package/src/tools/page/tool.ts +48 -0
- package/src/tools/search/actions.ts +56 -0
- package/src/tools/search/analytics.ts +266 -0
- package/src/tools/search/coverage.ts +247 -0
- package/src/tools/search/gaps.ts +129 -0
- package/src/tools/search/inspection.ts +160 -0
- package/src/tools/search/result.ts +3 -0
- package/src/tools/search/sample.ts +110 -0
- package/src/tools/search/schema.ts +62 -0
- package/src/{lib/sitemap.ts → tools/search/sitemap-sampling.ts} +1 -45
- package/src/tools/search/sites.ts +86 -0
- package/src/tools/search/tool.ts +11 -0
- package/src/tools/setup.ts +59 -20
- package/src/tools/speed/analyze.ts +191 -0
- package/src/tools/speed/batch.ts +265 -0
- package/src/tools/speed/crux.ts +176 -0
- package/src/tools/speed/pagespeed.ts +273 -0
- package/src/tools/speed/schema.ts +39 -0
- package/src/tools/speed/tool.ts +11 -0
- package/src/web/fetch.ts +31 -0
- package/src/web/images.ts +62 -0
- package/src/web/page-observation.ts +69 -0
- package/src/{lib → web}/robots.ts +20 -12
- package/src/web/sitemap-inventory.ts +64 -0
- package/src/web/sitemap-parser.ts +59 -0
- package/src/lib/auth.ts +0 -187
- package/src/tools/page.ts +0 -1241
- package/src/tools/search.ts +0 -852
- package/src/tools/speed.ts +0 -956
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
import { listSitemaps } from "../../providers/gsc.js";
|
|
2
|
+
import { type SearchOptions } from "./schema.js";
|
|
3
|
+
import { humanizeState, type InspectionSummary, inspectSingle } from "./inspection.js";
|
|
4
|
+
import { fetchSitemap, sampleUrls } from "./sitemap-sampling.js";
|
|
5
|
+
import { textResult } from "./result.js";
|
|
6
|
+
const COVERAGE_FILTERS = [
|
|
7
|
+
"not_indexed",
|
|
8
|
+
"server_error",
|
|
9
|
+
"redirect",
|
|
10
|
+
"soft_404",
|
|
11
|
+
"blocked",
|
|
12
|
+
"duplicate",
|
|
13
|
+
"discovered",
|
|
14
|
+
"crawled_not_indexed",
|
|
15
|
+
] as const;
|
|
16
|
+
|
|
17
|
+
type CoverageFilter = (typeof COVERAGE_FILTERS)[number];
|
|
18
|
+
|
|
19
|
+
export function matchesCoverageFilter(result: InspectionSummary, filter: CoverageFilter): boolean {
|
|
20
|
+
if (result.error) return false;
|
|
21
|
+
const cs = result.coverageState.toLowerCase();
|
|
22
|
+
const pf = result.pageFetchState.toLowerCase();
|
|
23
|
+
switch (filter) {
|
|
24
|
+
case "not_indexed":
|
|
25
|
+
return result.verdict !== "PASS";
|
|
26
|
+
case "server_error":
|
|
27
|
+
return cs.includes("server error") || pf === "server_error";
|
|
28
|
+
case "redirect":
|
|
29
|
+
return cs.includes("redirect") || pf.includes("redirect");
|
|
30
|
+
case "soft_404":
|
|
31
|
+
return cs.includes("soft 404") || pf === "soft_404";
|
|
32
|
+
case "blocked":
|
|
33
|
+
return cs.includes("blocked");
|
|
34
|
+
case "duplicate":
|
|
35
|
+
return cs.includes("duplicate");
|
|
36
|
+
case "discovered":
|
|
37
|
+
return cs.includes("discovered");
|
|
38
|
+
case "crawled_not_indexed":
|
|
39
|
+
return cs.includes("crawled - currently not indexed");
|
|
40
|
+
default:
|
|
41
|
+
return false;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function filterNameForState(state: string): string | null {
|
|
46
|
+
const s = state.toLowerCase();
|
|
47
|
+
if (s.includes("server error")) return "server_error";
|
|
48
|
+
if (s.includes("blocked")) return "blocked";
|
|
49
|
+
if (s.includes("redirect")) return "redirect";
|
|
50
|
+
if (s.includes("soft 404")) return "soft_404";
|
|
51
|
+
if (s.includes("duplicate")) return "duplicate";
|
|
52
|
+
if (s.includes("discovered - currently not indexed")) return "discovered";
|
|
53
|
+
if (s.includes("crawled - currently not indexed")) return "crawled_not_indexed";
|
|
54
|
+
return null;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export function formatSampleResults(
|
|
58
|
+
siteUrl: string,
|
|
59
|
+
sitemapUrl: string,
|
|
60
|
+
totalUrls: number,
|
|
61
|
+
results: InspectionSummary[],
|
|
62
|
+
): string {
|
|
63
|
+
const lines: string[] = [
|
|
64
|
+
`=== Sample Inspection: ${siteUrl} ===`,
|
|
65
|
+
`Sitemap: ${sitemapUrl} (${totalUrls.toLocaleString()} URLs)`,
|
|
66
|
+
`Sampled: ${results.length}`,
|
|
67
|
+
"",
|
|
68
|
+
];
|
|
69
|
+
|
|
70
|
+
// Summary
|
|
71
|
+
const verdictCounts: Record<string, number> = {};
|
|
72
|
+
const coverageCounts: Record<string, number> = {};
|
|
73
|
+
const fetchCounts: Record<string, number> = {};
|
|
74
|
+
let indexed = 0;
|
|
75
|
+
let errors = 0;
|
|
76
|
+
|
|
77
|
+
for (const r of results) {
|
|
78
|
+
if (r.error) {
|
|
79
|
+
errors++;
|
|
80
|
+
continue;
|
|
81
|
+
}
|
|
82
|
+
verdictCounts[r.verdict] = (verdictCounts[r.verdict] ?? 0) + 1;
|
|
83
|
+
coverageCounts[r.coverageState] = (coverageCounts[r.coverageState] ?? 0) + 1;
|
|
84
|
+
fetchCounts[r.pageFetchState] = (fetchCounts[r.pageFetchState] ?? 0) + 1;
|
|
85
|
+
if (r.verdict === "PASS") indexed++;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
const inspected = results.length - errors;
|
|
89
|
+
lines.push("--- Inspected sample ---", "");
|
|
90
|
+
lines.push(`Indexed: ${indexed}/${inspected}`);
|
|
91
|
+
|
|
92
|
+
if (indexed < inspected) {
|
|
93
|
+
lines.push(`Not indexed: ${inspected - indexed}/${inspected}`);
|
|
94
|
+
for (const [state, count] of Object.entries(coverageCounts)) {
|
|
95
|
+
if (state !== "Submitted and indexed" && state !== "Indexing allowed") {
|
|
96
|
+
lines.push(` ${state}: ${count}`);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
// Page fetch issues
|
|
102
|
+
const fetchIssues = Object.entries(fetchCounts).filter(([s]) => s !== "SUCCESSFUL");
|
|
103
|
+
if (fetchIssues.length > 0) {
|
|
104
|
+
lines.push("");
|
|
105
|
+
lines.push("Page fetch issues:");
|
|
106
|
+
for (const [state, count] of fetchIssues) {
|
|
107
|
+
lines.push(` ${humanizeState(state)}: ${count}`);
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
if (errors > 0) {
|
|
112
|
+
lines.push(`\nInspection errors: ${errors}`);
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
// Individual results
|
|
116
|
+
lines.push("", "--- Details ---", "");
|
|
117
|
+
for (let i = 0; i < results.length; i++) {
|
|
118
|
+
const r = results[i];
|
|
119
|
+
lines.push(`${i + 1}. ${r.url}`);
|
|
120
|
+
if (r.error) {
|
|
121
|
+
lines.push(` Error: ${r.error}`);
|
|
122
|
+
} else {
|
|
123
|
+
lines.push(` Verdict: ${r.verdict}`);
|
|
124
|
+
lines.push(` Coverage: ${r.coverageState}`);
|
|
125
|
+
lines.push(` Page fetch: ${humanizeState(r.pageFetchState)}`);
|
|
126
|
+
if (r.robotsTxtState !== "ALLOWED") lines.push(` Robots.txt: ${humanizeState(r.robotsTxtState)}`);
|
|
127
|
+
if (r.indexingState !== "INDEXING_ALLOWED") lines.push(` Indexing: ${humanizeState(r.indexingState)}`);
|
|
128
|
+
if (r.lastCrawlTime) lines.push(` Last crawled: ${r.lastCrawlTime}`);
|
|
129
|
+
if (r.googleCanonical && r.googleCanonical !== r.url) {
|
|
130
|
+
lines.push(` Google canonical: ${r.googleCanonical}`);
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
lines.push("");
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
return lines.join("\n");
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
function formatCoverage(siteUrl: string, sitemapUrl: string, totalUrls: number, results: InspectionSummary[]): string {
|
|
140
|
+
const inspected = results.filter((r) => !r.error);
|
|
141
|
+
const errors = results.filter((r) => r.error);
|
|
142
|
+
const indexed = inspected.filter((r) => r.verdict === "PASS");
|
|
143
|
+
const notIndexed = inspected.filter((r) => r.verdict !== "PASS");
|
|
144
|
+
|
|
145
|
+
const lines: string[] = [
|
|
146
|
+
`=== Index Coverage: ${siteUrl} ===`,
|
|
147
|
+
`Sitemap: ${sitemapUrl} (${totalUrls.toLocaleString()} URLs)`,
|
|
148
|
+
`Inspected: ${results.length}`,
|
|
149
|
+
"",
|
|
150
|
+
];
|
|
151
|
+
|
|
152
|
+
const total = inspected.length;
|
|
153
|
+
if (total === 0) {
|
|
154
|
+
lines.push("No URLs could be inspected.");
|
|
155
|
+
if (errors.length > 0) {
|
|
156
|
+
lines.push("");
|
|
157
|
+
for (const e of errors) lines.push(` ${e.url}: ${e.error}`);
|
|
158
|
+
}
|
|
159
|
+
return lines.join("\n");
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
// Group not-indexed by coverageState
|
|
163
|
+
const issueGroups: Record<string, InspectionSummary[]> = {};
|
|
164
|
+
for (const r of notIndexed) {
|
|
165
|
+
const key = r.coverageState;
|
|
166
|
+
if (!issueGroups[key]) issueGroups[key] = [];
|
|
167
|
+
issueGroups[key].push(r);
|
|
168
|
+
}
|
|
169
|
+
const sortedIssues = Object.entries(issueGroups).sort((a, b) => b[1].length - a[1].length);
|
|
170
|
+
|
|
171
|
+
// Breakdown
|
|
172
|
+
lines.push("--- Breakdown ---", "");
|
|
173
|
+
lines.push(`Indexed: ${indexed.length}/${total} (${Math.round((indexed.length / total) * 100)}%)`);
|
|
174
|
+
lines.push(`Not indexed: ${notIndexed.length}/${total} (${Math.round((notIndexed.length / total) * 100)}%)`);
|
|
175
|
+
for (const [state, urls] of sortedIssues) {
|
|
176
|
+
lines.push(` ${state}: ${urls.length}`);
|
|
177
|
+
}
|
|
178
|
+
if (errors.length > 0) {
|
|
179
|
+
lines.push(`Inspection errors: ${errors.length}`);
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
lines.push("", "These verdicts describe only the inspected sample, not site-wide coverage.");
|
|
183
|
+
|
|
184
|
+
// Example URLs per issue
|
|
185
|
+
if (sortedIssues.length > 0) {
|
|
186
|
+
lines.push("", "--- URLs by Issue ---", "");
|
|
187
|
+
for (const [state, urls] of sortedIssues) {
|
|
188
|
+
lines.push(`${state}:`);
|
|
189
|
+
for (const u of urls.slice(0, 3)) {
|
|
190
|
+
lines.push(` ${u.url}`);
|
|
191
|
+
if (u.lastCrawlTime) lines.push(` Last crawled: ${u.lastCrawlTime}`);
|
|
192
|
+
if (u.pageFetchState !== "SUCCESSFUL") lines.push(` Fetch: ${humanizeState(u.pageFetchState)}`);
|
|
193
|
+
}
|
|
194
|
+
if (urls.length > 3) lines.push(` ... and ${urls.length - 3} more in sample`);
|
|
195
|
+
lines.push("");
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
// Suggest drill-down
|
|
200
|
+
const filterSuggestions = sortedIssues.map(([state]) => filterNameForState(state)).filter(Boolean);
|
|
201
|
+
if (filterSuggestions.length > 0) {
|
|
202
|
+
lines.push(`Drill deeper: search(action='sample', filter='...') to investigate specific issues.`);
|
|
203
|
+
lines.push(` Available filters: ${filterSuggestions.join(", ")}`);
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
return lines.join("\n");
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
export async function runCoverage(options: SearchOptions) {
|
|
210
|
+
const { site_url, sitemap_url, sample_size, sample_strategy } = options;
|
|
211
|
+
if (!site_url) return textResult("Error: site_url is required for coverage.");
|
|
212
|
+
|
|
213
|
+
const count = sample_size ?? 20;
|
|
214
|
+
const strategy = sample_strategy ?? "spread";
|
|
215
|
+
|
|
216
|
+
// Discover sitemap URL if not provided
|
|
217
|
+
let resolvedSitemapUrl = sitemap_url;
|
|
218
|
+
if (!resolvedSitemapUrl) {
|
|
219
|
+
const sitemaps = await listSitemaps(site_url);
|
|
220
|
+
if (sitemaps.length === 0) {
|
|
221
|
+
return textResult(`No sitemaps found for ${site_url} in GSC. Provide a sitemap_url directly.`);
|
|
222
|
+
}
|
|
223
|
+
const nonIndex = sitemaps.find((s) => !s.isSitemapsIndex);
|
|
224
|
+
resolvedSitemapUrl = (nonIndex ?? sitemaps[0]).path;
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
let parsed = await fetchSitemap(resolvedSitemapUrl);
|
|
228
|
+
|
|
229
|
+
if (parsed.isSitemapIndex && parsed.childSitemaps.length > 0) {
|
|
230
|
+
const childUrl = parsed.childSitemaps[0];
|
|
231
|
+
parsed = await fetchSitemap(childUrl);
|
|
232
|
+
resolvedSitemapUrl = `${resolvedSitemapUrl} → ${childUrl}`;
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
if (parsed.urls.length === 0) {
|
|
236
|
+
return textResult(`Sitemap ${resolvedSitemapUrl} contains no URLs.`);
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
const sampled = sampleUrls(parsed.urls, count, strategy);
|
|
240
|
+
|
|
241
|
+
const results: InspectionSummary[] = [];
|
|
242
|
+
for (const u of sampled) {
|
|
243
|
+
results.push(await inspectSingle(u, site_url));
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
return textResult(formatCoverage(site_url, resolvedSitemapUrl, parsed.urls.length, results));
|
|
247
|
+
}
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
import { querySearchAnalytics, type SearchAnalyticsFilter } from "../../providers/gsc.js";
|
|
2
|
+
import { pacificDaysAgo } from "../../shared/dates.js";
|
|
3
|
+
import { type SearchOptions } from "./schema.js";
|
|
4
|
+
import { textResult } from "./result.js";
|
|
5
|
+
function extractPageText(html: string): string {
|
|
6
|
+
// Extract text from HTML; this does not check rendered visibility.
|
|
7
|
+
let text = html;
|
|
8
|
+
text = text.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, " ");
|
|
9
|
+
text = text.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, " ");
|
|
10
|
+
text = text.replace(/<[^>]+>/g, " ");
|
|
11
|
+
// Decode common HTML entities
|
|
12
|
+
text = text
|
|
13
|
+
.replace(/&/g, "&")
|
|
14
|
+
.replace(/</g, "<")
|
|
15
|
+
.replace(/>/g, ">")
|
|
16
|
+
.replace(/"/g, '"')
|
|
17
|
+
.replace(/'/g, "'")
|
|
18
|
+
.replace(/ /g, " ");
|
|
19
|
+
// Normalize whitespace
|
|
20
|
+
text = text.replace(/\s+/g, " ").trim().toLowerCase();
|
|
21
|
+
return text;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function formatGapAnalysis(
|
|
25
|
+
pageUrl: string,
|
|
26
|
+
siteUrl: string,
|
|
27
|
+
queries: Array<{ query: string; clicks: number; impressions: number; ctr: number; position: number }>,
|
|
28
|
+
pageText: string,
|
|
29
|
+
): string {
|
|
30
|
+
const lines: string[] = [`=== Keyword Gap Analysis: ${pageUrl} ===`, `Property: ${siteUrl}`, ""];
|
|
31
|
+
|
|
32
|
+
if (queries.length === 0) {
|
|
33
|
+
lines.push("No search queries found for this page.");
|
|
34
|
+
return lines.join("\n");
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
const onPage: typeof queries = [];
|
|
38
|
+
const gaps: typeof queries = [];
|
|
39
|
+
|
|
40
|
+
for (const q of queries) {
|
|
41
|
+
const words = q.query.toLowerCase().split(/\s+/);
|
|
42
|
+
const allPresent = words.every((w) => pageText.includes(w));
|
|
43
|
+
if (allPresent) {
|
|
44
|
+
onPage.push(q);
|
|
45
|
+
} else {
|
|
46
|
+
gaps.push(q);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
lines.push(`Queries analyzed: ${queries.length}`);
|
|
51
|
+
lines.push(`On page: ${onPage.length}`);
|
|
52
|
+
lines.push(`Gaps (not on page): ${gaps.length}`);
|
|
53
|
+
lines.push("");
|
|
54
|
+
|
|
55
|
+
if (gaps.length > 0) {
|
|
56
|
+
lines.push("--- Keyword Gaps (queries with impressions but missing from page) ---", "");
|
|
57
|
+
gaps.sort((a, b) => b.impressions - a.impressions);
|
|
58
|
+
for (const q of gaps.slice(0, 20)) {
|
|
59
|
+
const ctr = (q.ctr * 100).toFixed(1);
|
|
60
|
+
lines.push(
|
|
61
|
+
` "${q.query}" — ${q.impressions} imp, ${q.clicks} clicks, CTR ${ctr}%, pos ${q.position.toFixed(1)}`,
|
|
62
|
+
);
|
|
63
|
+
}
|
|
64
|
+
if (gaps.length > 20) lines.push(` ... and ${gaps.length - 20} more`);
|
|
65
|
+
lines.push("");
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
if (onPage.length > 0) {
|
|
69
|
+
lines.push("--- Already Targeted (query found on page) ---", "");
|
|
70
|
+
onPage.sort((a, b) => b.impressions - a.impressions);
|
|
71
|
+
for (const q of onPage.slice(0, 10)) {
|
|
72
|
+
const ctr = (q.ctr * 100).toFixed(1);
|
|
73
|
+
lines.push(
|
|
74
|
+
` "${q.query}" — ${q.impressions} imp, ${q.clicks} clicks, CTR ${ctr}%, pos ${q.position.toFixed(1)}`,
|
|
75
|
+
);
|
|
76
|
+
}
|
|
77
|
+
if (onPage.length > 10) lines.push(` ... and ${onPage.length - 10} more`);
|
|
78
|
+
lines.push("");
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
const totalGapImpressions = gaps.reduce((sum, q) => sum + q.impressions, 0);
|
|
82
|
+
const totalImpressions = queries.reduce((sum, q) => sum + q.impressions, 0);
|
|
83
|
+
if (totalImpressions > 0) {
|
|
84
|
+
const gapPct = Math.round((totalGapImpressions / totalImpressions) * 100);
|
|
85
|
+
lines.push(`${gapPct}% of impressions come from queries not on the page — potential content opportunity.`);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
return lines.join("\n");
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
export async function runGaps(options: SearchOptions) {
|
|
92
|
+
const { site_url, url, start_date, end_date, row_limit } = options;
|
|
93
|
+
if (!url) return textResult("Error: url is required for gap analysis.");
|
|
94
|
+
if (!site_url) return textResult("Error: site_url is required for gap analysis.");
|
|
95
|
+
|
|
96
|
+
// Fetch page content and GSC queries in parallel
|
|
97
|
+
const [pageRes, analyticsRes] = await Promise.all([
|
|
98
|
+
fetch(url, {
|
|
99
|
+
headers: { "User-Agent": "Mozilla/5.0 (compatible; Googlebot/2.1)", Accept: "text/html" },
|
|
100
|
+
redirect: "follow",
|
|
101
|
+
}),
|
|
102
|
+
querySearchAnalytics(site_url, {
|
|
103
|
+
startDate: start_date ?? pacificDaysAgo(28),
|
|
104
|
+
endDate: end_date ?? pacificDaysAgo(3),
|
|
105
|
+
dimensions: ["query"],
|
|
106
|
+
rowLimit: row_limit ?? 500,
|
|
107
|
+
dimensionFilterGroups: [
|
|
108
|
+
{
|
|
109
|
+
groupType: "and" as const,
|
|
110
|
+
filters: [{ dimension: "page", operator: "equals", expression: url } as SearchAnalyticsFilter],
|
|
111
|
+
},
|
|
112
|
+
],
|
|
113
|
+
}),
|
|
114
|
+
]);
|
|
115
|
+
|
|
116
|
+
if (!pageRes.ok) return textResult(`Error fetching ${url}: HTTP ${pageRes.status}`);
|
|
117
|
+
const html = await pageRes.text();
|
|
118
|
+
const pageText = extractPageText(html);
|
|
119
|
+
|
|
120
|
+
const queries = (analyticsRes.rows ?? []).map((row) => ({
|
|
121
|
+
query: row.keys[0],
|
|
122
|
+
clicks: row.clicks,
|
|
123
|
+
impressions: row.impressions,
|
|
124
|
+
ctr: row.ctr,
|
|
125
|
+
position: row.position,
|
|
126
|
+
}));
|
|
127
|
+
|
|
128
|
+
return textResult(formatGapAnalysis(url, site_url, queries, pageText));
|
|
129
|
+
}
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
import { type InspectionResult, inspectUrl } from "../../providers/gsc.js";
|
|
2
|
+
import { type SearchOptions } from "./schema.js";
|
|
3
|
+
import { textResult } from "./result.js";
|
|
4
|
+
function formatInspection(url: string, siteUrl: string, r: InspectionResult): string {
|
|
5
|
+
const idx = r.indexStatusResult;
|
|
6
|
+
const lines: string[] = [
|
|
7
|
+
`=== URL Inspection: ${url} ===`,
|
|
8
|
+
`Property: ${siteUrl}`,
|
|
9
|
+
"",
|
|
10
|
+
"--- Index Status ---",
|
|
11
|
+
"",
|
|
12
|
+
`Verdict: ${idx.verdict}`,
|
|
13
|
+
`Coverage: ${idx.coverageState}`,
|
|
14
|
+
`Page fetch: ${idx.pageFetchState}`,
|
|
15
|
+
`Robots.txt: ${idx.robotsTxtState}`,
|
|
16
|
+
`Indexing: ${idx.indexingState}`,
|
|
17
|
+
`Crawled as: ${idx.crawledAs ?? "unknown"}`,
|
|
18
|
+
];
|
|
19
|
+
|
|
20
|
+
if (idx.lastCrawlTime) lines.push(`Last crawled: ${idx.lastCrawlTime}`);
|
|
21
|
+
if (idx.userCanonical) lines.push(`Your canonical: ${idx.userCanonical}`);
|
|
22
|
+
if (idx.googleCanonical) lines.push(`Google's canonical: ${idx.googleCanonical}`);
|
|
23
|
+
|
|
24
|
+
if (idx.userCanonical && idx.googleCanonical && idx.userCanonical !== idx.googleCanonical) {
|
|
25
|
+
lines.push(`\n⚠ CANONICAL MISMATCH: You declared "${idx.userCanonical}" but Google chose "${idx.googleCanonical}"`);
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
if (idx.sitemap && idx.sitemap.length > 0) {
|
|
29
|
+
lines.push(`\nSitemaps: ${idx.sitemap.join(", ")}`);
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
if (idx.referringUrls && idx.referringUrls.length > 0) {
|
|
33
|
+
lines.push(`\nReferring URLs: ${idx.referringUrls.join(", ")}`);
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
if (idx.verdict !== "PASS") {
|
|
37
|
+
const gscUrl = `https://search.google.com/search-console/inspect?resource_id=${encodeURIComponent(siteUrl)}&id=${encodeURIComponent(url)}`;
|
|
38
|
+
lines.push("", "→ This page is not indexed. Request indexing manually in Google Search Console:", ` ${gscUrl}`);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
// Rich Results
|
|
42
|
+
if (r.richResultsResult) {
|
|
43
|
+
lines.push("", "--- Rich Results ---", "");
|
|
44
|
+
lines.push(`Verdict: ${r.richResultsResult.verdict}`);
|
|
45
|
+
|
|
46
|
+
for (const item of r.richResultsResult.detectedItems ?? []) {
|
|
47
|
+
lines.push(`\nType: ${item.richResultType}`);
|
|
48
|
+
for (const instance of item.items ?? []) {
|
|
49
|
+
if (instance.name) lines.push(` Name: ${instance.name}`);
|
|
50
|
+
const issues = instance.issues ?? [];
|
|
51
|
+
if (issues.length === 0) {
|
|
52
|
+
lines.push(" Status: PASS");
|
|
53
|
+
} else {
|
|
54
|
+
for (const issue of issues) {
|
|
55
|
+
lines.push(` ${issue.severity}: ${issue.issueMessage}`);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
// Display deprecated mobile-usability evidence when supplied.
|
|
63
|
+
if (r.mobileUsabilityResult) {
|
|
64
|
+
lines.push("", "--- Mobile Usability (deprecated) ---", "");
|
|
65
|
+
lines.push(`Verdict: ${r.mobileUsabilityResult.verdict}`);
|
|
66
|
+
if (r.mobileUsabilityResult.issues) {
|
|
67
|
+
for (const issue of r.mobileUsabilityResult.issues) {
|
|
68
|
+
lines.push(` ${issue.issueType}${issue.message ? `: ${issue.message}` : ""}`);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
// Inspection link
|
|
74
|
+
if (r.inspectionResultLink) {
|
|
75
|
+
lines.push("", `Full report: ${r.inspectionResultLink}`);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
return lines.join("\n");
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
export function humanizeState(state: string): string {
|
|
82
|
+
const map: Record<string, string> = {
|
|
83
|
+
PAGE_FETCH_STATE_UNSPECIFIED: "not yet crawled",
|
|
84
|
+
ROBOTS_TXT_STATE_UNSPECIFIED: "not yet checked",
|
|
85
|
+
INDEXING_STATE_UNSPECIFIED: "not yet determined",
|
|
86
|
+
};
|
|
87
|
+
return map[state] ?? state;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
export interface InspectionSummary {
|
|
91
|
+
url: string;
|
|
92
|
+
verdict: string;
|
|
93
|
+
coverageState: string;
|
|
94
|
+
pageFetchState: string;
|
|
95
|
+
robotsTxtState: string;
|
|
96
|
+
indexingState: string;
|
|
97
|
+
lastCrawlTime: string | null;
|
|
98
|
+
googleCanonical: string | null;
|
|
99
|
+
error: string | null;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
export async function inspectSingle(url: string, siteUrl: string): Promise<InspectionSummary> {
|
|
103
|
+
try {
|
|
104
|
+
const r = await inspectUrl(url, siteUrl);
|
|
105
|
+
const idx = r.indexStatusResult;
|
|
106
|
+
return {
|
|
107
|
+
url,
|
|
108
|
+
verdict: idx.verdict,
|
|
109
|
+
coverageState: idx.coverageState,
|
|
110
|
+
pageFetchState: idx.pageFetchState,
|
|
111
|
+
robotsTxtState: idx.robotsTxtState,
|
|
112
|
+
indexingState: idx.indexingState,
|
|
113
|
+
lastCrawlTime: idx.lastCrawlTime ?? null,
|
|
114
|
+
googleCanonical: idx.googleCanonical ?? null,
|
|
115
|
+
error: null,
|
|
116
|
+
};
|
|
117
|
+
} catch (err) {
|
|
118
|
+
return {
|
|
119
|
+
url,
|
|
120
|
+
verdict: "ERROR",
|
|
121
|
+
coverageState: "ERROR",
|
|
122
|
+
pageFetchState: "ERROR",
|
|
123
|
+
robotsTxtState: "ERROR",
|
|
124
|
+
indexingState: "ERROR",
|
|
125
|
+
lastCrawlTime: null,
|
|
126
|
+
googleCanonical: null,
|
|
127
|
+
error: err instanceof Error ? err.message : String(err),
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
export async function runInspect(options: SearchOptions) {
|
|
133
|
+
const { site_url, url } = options;
|
|
134
|
+
if (!url) return textResult("Error: url is required for inspect.");
|
|
135
|
+
if (!site_url) return textResult("Error: site_url is required for inspect.");
|
|
136
|
+
|
|
137
|
+
try {
|
|
138
|
+
const result = await inspectUrl(url, site_url);
|
|
139
|
+
return textResult(formatInspection(url, site_url, result));
|
|
140
|
+
} catch (err) {
|
|
141
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
142
|
+
if (msg.includes("PERMISSION_DENIED")) {
|
|
143
|
+
const domain = new URL(url).hostname;
|
|
144
|
+
return textResult(
|
|
145
|
+
[
|
|
146
|
+
`Error: Cannot inspect "${url}" — permission denied for property "${site_url}".`,
|
|
147
|
+
"",
|
|
148
|
+
"Possible causes:",
|
|
149
|
+
` 1. ${domain} is not a verified property in Google Search Console`,
|
|
150
|
+
` 2. The authenticated account does not have access to "${site_url}"`,
|
|
151
|
+
` 3. The property format is wrong — try "sc-domain:${domain}" or "https://${domain}/"`,
|
|
152
|
+
"",
|
|
153
|
+
"To verify a property: https://search.google.com/search-console",
|
|
154
|
+
"To check your auth: use the setup tool to verify credentials",
|
|
155
|
+
].join("\n"),
|
|
156
|
+
);
|
|
157
|
+
}
|
|
158
|
+
return textResult(`Error inspecting URL: ${msg}`);
|
|
159
|
+
}
|
|
160
|
+
}
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
import { listSitemaps } from "../../providers/gsc.js";
|
|
2
|
+
import { type SearchOptions } from "./schema.js";
|
|
3
|
+
import { humanizeState, type InspectionSummary, inspectSingle } from "./inspection.js";
|
|
4
|
+
import { matchesCoverageFilter, formatSampleResults } from "./coverage.js";
|
|
5
|
+
import { fetchSitemap, sampleUrls } from "./sitemap-sampling.js";
|
|
6
|
+
import { textResult } from "./result.js";
|
|
7
|
+
|
|
8
|
+
export async function runSample(options: SearchOptions) {
|
|
9
|
+
const { site_url, sitemap_url, sample_size, sample_strategy, filter } = options;
|
|
10
|
+
if (!site_url) return textResult("Error: site_url is required for sample.");
|
|
11
|
+
|
|
12
|
+
const count = sample_size ?? 5;
|
|
13
|
+
const strategy = sample_strategy ?? "random";
|
|
14
|
+
|
|
15
|
+
// Discover sitemap URL if not provided
|
|
16
|
+
let resolvedSitemapUrl = sitemap_url;
|
|
17
|
+
if (!resolvedSitemapUrl) {
|
|
18
|
+
const sitemaps = await listSitemaps(site_url);
|
|
19
|
+
if (sitemaps.length === 0) {
|
|
20
|
+
return textResult(`No sitemaps found for ${site_url} in GSC. Provide a sitemap_url directly.`);
|
|
21
|
+
}
|
|
22
|
+
const nonIndex = sitemaps.find((s) => !s.isSitemapsIndex);
|
|
23
|
+
resolvedSitemapUrl = (nonIndex ?? sitemaps[0]).path;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
// Fetch and parse sitemap
|
|
27
|
+
let parsed = await fetchSitemap(resolvedSitemapUrl);
|
|
28
|
+
|
|
29
|
+
if (parsed.isSitemapIndex && parsed.childSitemaps.length > 0) {
|
|
30
|
+
const childUrl = parsed.childSitemaps[0];
|
|
31
|
+
parsed = await fetchSitemap(childUrl);
|
|
32
|
+
resolvedSitemapUrl = `${resolvedSitemapUrl} → ${childUrl}`;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
if (parsed.urls.length === 0) {
|
|
36
|
+
return textResult(`Sitemap ${resolvedSitemapUrl} contains no URLs.`);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
if (filter) {
|
|
40
|
+
// Filtered sample: inspect more URLs, return only matches
|
|
41
|
+
const scanLimit = Math.min(count * 5, 50, parsed.urls.length);
|
|
42
|
+
const pool = sampleUrls(parsed.urls, scanLimit, strategy);
|
|
43
|
+
const matched: InspectionSummary[] = [];
|
|
44
|
+
let scanned = 0;
|
|
45
|
+
|
|
46
|
+
for (const u of pool) {
|
|
47
|
+
const result = await inspectSingle(u, site_url);
|
|
48
|
+
scanned++;
|
|
49
|
+
if (matchesCoverageFilter(result, filter)) {
|
|
50
|
+
matched.push(result);
|
|
51
|
+
if (matched.length >= count) break;
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
if (matched.length === 0) {
|
|
56
|
+
return textResult(
|
|
57
|
+
[
|
|
58
|
+
`=== Filtered Sample: ${site_url} ===`,
|
|
59
|
+
`Sitemap: ${resolvedSitemapUrl} (${parsed.urls.length.toLocaleString()} URLs)`,
|
|
60
|
+
`Filter: ${filter}`,
|
|
61
|
+
`Scanned: ${scanned} | Matched: 0`,
|
|
62
|
+
"",
|
|
63
|
+
`No URLs matching "${filter}" found in ${scanned} inspected URLs.`,
|
|
64
|
+
"Try a larger sample_size or a different filter.",
|
|
65
|
+
].join("\n"),
|
|
66
|
+
);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
const lines: string[] = [
|
|
70
|
+
`=== Filtered Sample: ${site_url} ===`,
|
|
71
|
+
`Sitemap: ${resolvedSitemapUrl} (${parsed.urls.length.toLocaleString()} URLs)`,
|
|
72
|
+
`Filter: ${filter}`,
|
|
73
|
+
`Scanned: ${scanned} | Matched: ${matched.length}`,
|
|
74
|
+
"",
|
|
75
|
+
"--- Matching URLs ---",
|
|
76
|
+
"",
|
|
77
|
+
];
|
|
78
|
+
|
|
79
|
+
for (let i = 0; i < matched.length; i++) {
|
|
80
|
+
const r = matched[i];
|
|
81
|
+
lines.push(`${i + 1}. ${r.url}`);
|
|
82
|
+
lines.push(` Coverage: ${r.coverageState}`);
|
|
83
|
+
lines.push(` Page fetch: ${humanizeState(r.pageFetchState)}`);
|
|
84
|
+
if (r.robotsTxtState !== "ALLOWED") lines.push(` Robots.txt: ${humanizeState(r.robotsTxtState)}`);
|
|
85
|
+
if (r.indexingState !== "INDEXING_ALLOWED") lines.push(` Indexing: ${humanizeState(r.indexingState)}`);
|
|
86
|
+
if (r.lastCrawlTime) lines.push(` Last crawled: ${r.lastCrawlTime}`);
|
|
87
|
+
if (r.googleCanonical && r.googleCanonical !== r.url) {
|
|
88
|
+
lines.push(` Google canonical: ${r.googleCanonical}`);
|
|
89
|
+
}
|
|
90
|
+
lines.push("");
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
if (matched.length < count && scanned >= scanLimit) {
|
|
94
|
+
lines.push(
|
|
95
|
+
`Found ${matched.length}/${count} requested — scanned ${scanned} URLs (limit ${scanLimit}). Increase sample_size for a wider scan.`,
|
|
96
|
+
);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
return textResult(lines.join("\n"));
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
const sampled = sampleUrls(parsed.urls, count, strategy);
|
|
103
|
+
|
|
104
|
+
const results: InspectionSummary[] = [];
|
|
105
|
+
for (const u of sampled) {
|
|
106
|
+
results.push(await inspectSingle(u, site_url));
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
return textResult(formatSampleResults(site_url, resolvedSitemapUrl, parsed.urls.length, results));
|
|
110
|
+
}
|