mcp-scraper 0.3.14 → 0.3.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/api-server.cjs +2261 -511
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +3 -3
- package/dist/bin/browser-agent-stdio-server.cjs +6 -6
- package/dist/bin/browser-agent-stdio-server.cjs.map +1 -1
- package/dist/bin/browser-agent-stdio-server.js +2 -2
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs +663 -48
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.js +6 -5
- package/dist/bin/mcp-scraper-combined-stdio-server.js.map +1 -1
- package/dist/bin/mcp-scraper-install.cjs +4 -4
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +2 -2
- package/dist/bin/mcp-stdio-server.cjs +655 -40
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +2 -2
- package/dist/{chunk-DBQDG7EH.js → chunk-5H22TOXQ.js} +28 -1
- package/dist/chunk-5H22TOXQ.js.map +1 -0
- package/dist/{chunk-HM6FHV5U.js → chunk-7XARJHS6.js} +662 -41
- package/dist/chunk-7XARJHS6.js.map +1 -0
- package/dist/{chunk-KPXMPAJ3.js → chunk-ACIUCZ27.js} +4 -4
- package/dist/chunk-ACIUCZ27.js.map +1 -0
- package/dist/{chunk-AX7UBYLG.js → chunk-ISZWGIWL.js} +7 -7
- package/dist/chunk-ISZWGIWL.js.map +1 -0
- package/dist/chunk-SOMBZK3V.js +7 -0
- package/dist/chunk-SOMBZK3V.js.map +1 -0
- package/dist/{chunk-AZ5PKU4F.js → chunk-ZAUMSBV3.js} +2 -2
- package/dist/{db-BE4JVB3V.js → db-P76GVIFN.js} +2 -2
- package/dist/{server-ILIVSPNY.js → server-LPVBSE2E.js} +1133 -93
- package/dist/server-LPVBSE2E.js.map +1 -0
- package/dist/{worker-FG7ZWEGA.js → worker-2WVKKCC7.js} +3 -3
- package/docs/final-tooling-spec.md +206 -0
- package/docs/mcp-tool-design-guide.md +225 -0
- package/docs/mcp-tool-manifest.generated.json +57 -22
- package/docs/seo-crawl-report-spec.md +287 -0
- package/docs/tool-catalog-spec.md +388 -0
- package/package.json +2 -1
- package/dist/chunk-3QHZPR4U.js +0 -7
- package/dist/chunk-3QHZPR4U.js.map +0 -1
- package/dist/chunk-AX7UBYLG.js.map +0 -1
- package/dist/chunk-DBQDG7EH.js.map +0 -1
- package/dist/chunk-HM6FHV5U.js.map +0 -1
- package/dist/chunk-KPXMPAJ3.js.map +0 -1
- package/dist/server-ILIVSPNY.js.map +0 -1
- /package/dist/{chunk-AZ5PKU4F.js.map → chunk-ZAUMSBV3.js.map} +0 -0
- /package/dist/{db-BE4JVB3V.js.map → db-P76GVIFN.js.map} +0 -0
- /package/dist/{worker-FG7ZWEGA.js.map → worker-2WVKKCC7.js.map} +0 -0
|
@@ -11,7 +11,7 @@ import {
|
|
|
11
11
|
} from "./chunk-LFATOGDF.js";
|
|
12
12
|
import {
|
|
13
13
|
PACKAGE_VERSION
|
|
14
|
-
} from "./chunk-
|
|
14
|
+
} from "./chunk-SOMBZK3V.js";
|
|
15
15
|
|
|
16
16
|
// src/harvest-timeout.ts
|
|
17
17
|
var VERCEL_FUNCTION_MAX_MS = 3e5;
|
|
@@ -27,6 +27,33 @@ function harvestTimeoutBudget(maxQuestions, serpOnly = false) {
|
|
|
27
27
|
return { serverMs, clientMs };
|
|
28
28
|
}
|
|
29
29
|
|
|
30
|
+
// src/mcp/server-instructions.ts
|
|
31
|
+
var SERVER_INSTRUCTIONS = `
|
|
32
|
+
This server scrapes and analyzes web, social, search, maps, and site data. Pick a tool by NAME using
|
|
33
|
+
this routing map, then load its schema before calling.
|
|
34
|
+
|
|
35
|
+
ROUTING
|
|
36
|
+
- One web page -> extract_url. Whole site (crawl + SEO report) -> extract_site. Just the URL list -> map_site_urls.
|
|
37
|
+
- Web search (organic SERP) -> search_serp. Full SERP + People-Also-Ask / AI-Overview detail -> harvest_paa.
|
|
38
|
+
- Google Maps: find places -> maps_search; one place deep-dive + reviews -> maps_place_intel.
|
|
39
|
+
- YouTube: find or list videos -> youtube_harvest; transcribe one video -> youtube_transcribe.
|
|
40
|
+
- Facebook: find ads -> facebook_ad_search; transcribe an ad -> facebook_ad_transcribe; transcribe a
|
|
41
|
+
video -> facebook_video_transcribe; page profile/intel -> facebook_page_intel.
|
|
42
|
+
- Instagram: profile inventory -> instagram_profile_content; one post or reel -> instagram_media_download.
|
|
43
|
+
- Workflows (multi-step): local directory build -> directory_workflow; rank blueprint -> rank_tracker_workflow;
|
|
44
|
+
AI-answer citation fan-out (AEO) -> query_fanout_workflow; deep research -> deep_research_workflow.
|
|
45
|
+
- Anything without a dedicated tool (e.g. Reddit, arbitrary logged-in sites) -> the browser_* agent
|
|
46
|
+
(browser_open, then navigate/read); save/reuse logins via browser_profile_*.
|
|
47
|
+
|
|
48
|
+
NOTES
|
|
49
|
+
- Bulk / full-site crawls: call extract_site with rotateProxies:true for blocked or rate-limited sites.
|
|
50
|
+
It discovers URLs and returns a saved folder/artifact plus a summary, not the full content inline.
|
|
51
|
+
- Tools that open a browser session or transcribe media cost credits and take longer. Prefer the
|
|
52
|
+
cheapest tool that answers the question (plain search/extract before browser agents).
|
|
53
|
+
- Large results are saved to disk or an artifact and returned as a summary plus a path; read the path
|
|
54
|
+
for full detail rather than expecting the whole payload inline.
|
|
55
|
+
`.trim();
|
|
56
|
+
|
|
30
57
|
// src/mcp/paa-mcp-server.ts
|
|
31
58
|
import { McpServer, ResourceTemplate } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
32
59
|
import { readdirSync, readFileSync, statSync } from "fs";
|
|
@@ -152,6 +179,322 @@ function suggestWorkflowRecipes(goal, limit = 3) {
|
|
|
152
179
|
return scored.sort((a, b) => b.score - a.score || a.recipe.title.localeCompare(b.recipe.title)).slice(0, Math.max(1, limit)).map((item) => item.recipe);
|
|
153
180
|
}
|
|
154
181
|
|
|
182
|
+
// src/api/seo-link-graph.ts
|
|
183
|
+
function normalize2(u) {
|
|
184
|
+
return u.split("#")[0].replace(/\/$/, "");
|
|
185
|
+
}
|
|
186
|
+
function buildLinkGraph(pages, startUrl) {
|
|
187
|
+
const statusByUrl = /* @__PURE__ */ new Map();
|
|
188
|
+
for (const p of pages) statusByUrl.set(normalize2(p.url), p.status);
|
|
189
|
+
const edges = [];
|
|
190
|
+
for (const p of pages) {
|
|
191
|
+
for (const l of p.outlinks ?? []) {
|
|
192
|
+
const nofollow = (l.rel ?? "").split(/\s+/).includes("nofollow");
|
|
193
|
+
edges.push({
|
|
194
|
+
from: p.url,
|
|
195
|
+
to: l.href,
|
|
196
|
+
anchor: l.anchor,
|
|
197
|
+
rel: l.rel,
|
|
198
|
+
internal: l.internal,
|
|
199
|
+
nofollow,
|
|
200
|
+
targetStatus: statusByUrl.get(normalize2(l.href)) ?? null
|
|
201
|
+
});
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
const inboundFrom = /* @__PURE__ */ new Map();
|
|
205
|
+
const inboundAnchors = /* @__PURE__ */ new Map();
|
|
206
|
+
const adjacency = /* @__PURE__ */ new Map();
|
|
207
|
+
for (const e of edges) {
|
|
208
|
+
if (!e.internal) continue;
|
|
209
|
+
const to = normalize2(e.to);
|
|
210
|
+
const from = normalize2(e.from);
|
|
211
|
+
if (!inboundFrom.has(to)) inboundFrom.set(to, /* @__PURE__ */ new Set());
|
|
212
|
+
inboundFrom.get(to).add(from);
|
|
213
|
+
if (e.anchor) {
|
|
214
|
+
if (!inboundAnchors.has(to)) inboundAnchors.set(to, []);
|
|
215
|
+
inboundAnchors.get(to).push(e.anchor);
|
|
216
|
+
}
|
|
217
|
+
if (!e.nofollow && (statusByUrl.get(to) === 200 || !statusByUrl.has(to))) {
|
|
218
|
+
if (!adjacency.has(from)) adjacency.set(from, /* @__PURE__ */ new Set());
|
|
219
|
+
adjacency.get(from).add(to);
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
const depth = /* @__PURE__ */ new Map();
|
|
223
|
+
const start = normalize2(startUrl);
|
|
224
|
+
const queue = [start];
|
|
225
|
+
depth.set(start, 0);
|
|
226
|
+
while (queue.length > 0) {
|
|
227
|
+
const cur = queue.shift();
|
|
228
|
+
const d = depth.get(cur);
|
|
229
|
+
for (const next of adjacency.get(cur) ?? []) {
|
|
230
|
+
if (!depth.has(next)) {
|
|
231
|
+
depth.set(next, d + 1);
|
|
232
|
+
queue.push(next);
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
const topAnchorsFor = (url) => {
|
|
237
|
+
const counts = /* @__PURE__ */ new Map();
|
|
238
|
+
for (const a of inboundAnchors.get(url) ?? []) counts.set(a, (counts.get(a) ?? 0) + 1);
|
|
239
|
+
return [...counts.entries()].sort((a, b) => b[1] - a[1]).slice(0, 3).map((e) => e[0]);
|
|
240
|
+
};
|
|
241
|
+
const metrics = /* @__PURE__ */ new Map();
|
|
242
|
+
for (const p of pages) {
|
|
243
|
+
const key = normalize2(p.url);
|
|
244
|
+
const inbound = inboundFrom.get(key) ?? /* @__PURE__ */ new Set();
|
|
245
|
+
metrics.set(p.url, {
|
|
246
|
+
url: p.url,
|
|
247
|
+
inlinks: inbound.size,
|
|
248
|
+
uniqueInlinks: inbound.size,
|
|
249
|
+
outlinksInternal: (p.outlinks ?? []).filter((l) => l.internal).length,
|
|
250
|
+
outlinksExternal: (p.outlinks ?? []).filter((l) => !l.internal).length,
|
|
251
|
+
crawlDepth: depth.has(key) ? depth.get(key) : null,
|
|
252
|
+
orphan: inbound.size === 0 && key !== start,
|
|
253
|
+
topAnchors: topAnchorsFor(key)
|
|
254
|
+
});
|
|
255
|
+
}
|
|
256
|
+
return { edges, metrics };
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
// src/api/seo-issues.ts
|
|
260
|
+
var THIN_WORDS = 200;
|
|
261
|
+
var TITLE_MAX = 60;
|
|
262
|
+
var TITLE_PX_MAX = 561;
|
|
263
|
+
var TITLE_MIN = 30;
|
|
264
|
+
var META_MAX = 155;
|
|
265
|
+
var META_MIN = 70;
|
|
266
|
+
var H1_MAX = 70;
|
|
267
|
+
var URL_MAX = 115;
|
|
268
|
+
function dupes(pages, key) {
|
|
269
|
+
const groups = /* @__PURE__ */ new Map();
|
|
270
|
+
for (const p of pages) {
|
|
271
|
+
const k = key(p);
|
|
272
|
+
if (k === null || k === void 0 || k === "") continue;
|
|
273
|
+
const ks = String(k);
|
|
274
|
+
if (!groups.has(ks)) groups.set(ks, []);
|
|
275
|
+
groups.get(ks).push(p.url);
|
|
276
|
+
}
|
|
277
|
+
return [...groups.values()].filter((g) => g.length > 1).flat();
|
|
278
|
+
}
|
|
279
|
+
function computeIssues(pages, metrics) {
|
|
280
|
+
const group = (urls) => ({ count: urls.length, urls: urls.slice(0, 200) });
|
|
281
|
+
const where = (fn) => pages.filter(fn).map((p) => p.url);
|
|
282
|
+
const ok = (p) => p.status === 200;
|
|
283
|
+
const pathname = (u) => {
|
|
284
|
+
try {
|
|
285
|
+
return new URL(u).pathname;
|
|
286
|
+
} catch {
|
|
287
|
+
return "";
|
|
288
|
+
}
|
|
289
|
+
};
|
|
290
|
+
const report = {
|
|
291
|
+
"title.missing": group(where((p) => ok(p) && !p.title)),
|
|
292
|
+
"title.duplicate": group(dupes(pages.filter(ok), (p) => p.title)),
|
|
293
|
+
"title.tooLong": group(where((p) => (p.titleLength ?? 0) > TITLE_MAX || (p.titlePixels ?? 0) > TITLE_PX_MAX)),
|
|
294
|
+
"title.tooShort": group(where((p) => p.title != null && (p.titleLength ?? 0) < TITLE_MIN)),
|
|
295
|
+
"title.sameAsH1": group(where((p) => !!p.title && !!p.h1 && p.title.trim() === p.h1.trim())),
|
|
296
|
+
"meta.missing": group(where((p) => ok(p) && !p.metaDescription)),
|
|
297
|
+
"meta.duplicate": group(dupes(pages.filter(ok), (p) => p.metaDescription)),
|
|
298
|
+
"meta.tooLong": group(where((p) => (p.metaDescLength ?? 0) > META_MAX)),
|
|
299
|
+
"meta.tooShort": group(where((p) => p.metaDescription != null && (p.metaDescLength ?? 0) < META_MIN)),
|
|
300
|
+
"h1.missing": group(where((p) => ok(p) && !p.h1)),
|
|
301
|
+
"h1.multiple": group(where((p) => !!p.h1_2)),
|
|
302
|
+
"h1.duplicate": group(dupes(pages.filter(ok), (p) => p.h1)),
|
|
303
|
+
"h1.tooLong": group(where((p) => (p.h1?.length ?? 0) > H1_MAX)),
|
|
304
|
+
"h2.missing": group(where((p) => ok(p) && p.h2Count === 0)),
|
|
305
|
+
"indexability.nonIndexable": group(where((p) => !p.indexable)),
|
|
306
|
+
"indexability.noindex": group(where((p) => p.indexabilityReason === "noindex")),
|
|
307
|
+
"canonical.missing": group(where((p) => ok(p) && !p.canonicalUrl)),
|
|
308
|
+
"canonical.canonicalised": group(where((p) => p.indexabilityReason === "canonicalised")),
|
|
309
|
+
"response.broken4xx": group(where((p) => (p.status ?? 0) >= 400 && (p.status ?? 0) < 500)),
|
|
310
|
+
"response.error5xx": group(where((p) => (p.status ?? 0) >= 500)),
|
|
311
|
+
"response.redirect3xx": group(where((p) => (p.status ?? 0) >= 300 && (p.status ?? 0) < 400)),
|
|
312
|
+
"response.noResponse": group(where((p) => p.status === null)),
|
|
313
|
+
"content.thin": group(where((p) => ok(p) && p.wordCount > 0 && p.wordCount < THIN_WORDS)),
|
|
314
|
+
"content.exactDuplicate": group(dupes(pages.filter((p) => ok(p) && p.contentHash), (p) => p.contentHash)),
|
|
315
|
+
"images.missingAlt": group(where((p) => p.imagesMissingAlt > 0)),
|
|
316
|
+
"schema.missing": group(where((p) => ok(p) && p.schemaTypes.length === 0)),
|
|
317
|
+
"url.tooLong": group(where((p) => p.url.length > URL_MAX)),
|
|
318
|
+
"url.uppercase": group(where((p) => /[A-Z]/.test(pathname(p.url)))),
|
|
319
|
+
"url.underscores": group(where((p) => pathname(p.url).includes("_"))),
|
|
320
|
+
"links.orphan": group([...metrics.values()].filter((m) => m.orphan).map((m) => m.url))
|
|
321
|
+
};
|
|
322
|
+
const normUrl = (u) => {
|
|
323
|
+
try {
|
|
324
|
+
const x = new URL(u);
|
|
325
|
+
x.hash = "";
|
|
326
|
+
return (x.origin + x.pathname).replace(/\/+$/, "") + x.search;
|
|
327
|
+
} catch {
|
|
328
|
+
return u.replace(/#.*$/, "").replace(/\/+$/, "");
|
|
329
|
+
}
|
|
330
|
+
};
|
|
331
|
+
const statusByUrl = new Map(pages.map((p) => [normUrl(p.url), p.status]));
|
|
332
|
+
const brokenLinkPages = /* @__PURE__ */ new Set();
|
|
333
|
+
for (const p of pages) {
|
|
334
|
+
for (const l of p.outlinks ?? []) {
|
|
335
|
+
if (!l.internal) continue;
|
|
336
|
+
const st = statusByUrl.get(normUrl(l.href));
|
|
337
|
+
if (st != null && st >= 400) brokenLinkPages.add(p.url);
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
report["links.brokenInternal"] = group([...brokenLinkPages]);
|
|
341
|
+
return report;
|
|
342
|
+
}
|
|
343
|
+
function renderIssueReport(siteUrl, pages, report, metrics) {
|
|
344
|
+
const total = pages.length;
|
|
345
|
+
const sev = (key) => key.startsWith("response.broken") || key.startsWith("response.error") || key.startsWith("links.broken") ? "\u{1F534}" : key.startsWith("title.missing") || key.startsWith("h1.missing") || key.startsWith("indexability") || key.startsWith("canonical.missing") ? "\u{1F7E0}" : "\u{1F7E1}";
|
|
346
|
+
const rows = Object.entries(report).filter(([, g]) => g.count > 0).sort((a, b) => b[1].count - a[1].count).map(([k, g]) => `| ${sev(k)} | \`${k}\` | ${g.count} | ${g.urls.slice(0, 3).join(" \xB7 ")}${g.urls.length > 3 ? " \u2026" : ""} |`);
|
|
347
|
+
const depths = [...metrics.values()].map((m) => m.crawlDepth).filter((d) => d != null);
|
|
348
|
+
const maxDepth = depths.length ? Math.max(...depths) : 0;
|
|
349
|
+
const orphans = [...metrics.values()].filter((m) => m.orphan).length;
|
|
350
|
+
const topLinked = [...metrics.values()].sort((a, b) => b.inlinks - a.inlinks).slice(0, 5);
|
|
351
|
+
return [
|
|
352
|
+
`# SEO Crawl Report: ${siteUrl}`,
|
|
353
|
+
`**${total} pages crawled** \xB7 max crawl depth ${maxDepth} \xB7 ${orphans} orphan page(s)`,
|
|
354
|
+
`
|
|
355
|
+
## Issues
|
|
356
|
+
| | Issue | Count | Examples |
|
|
357
|
+
|---|-------|-------|----------|
|
|
358
|
+
${rows.join("\n") || "| \u2705 | none | 0 | \u2014 |"}`,
|
|
359
|
+
`
|
|
360
|
+
## Most-linked pages
|
|
361
|
+
${topLinked.map((m) => `- ${m.inlinks} inlinks \xB7 depth ${m.crawlDepth ?? "\u2014"} \xB7 ${m.url}`).join("\n")}`,
|
|
362
|
+
`
|
|
363
|
+
_Thresholds: title ${TITLE_MAX}ch/${TITLE_PX_MAX}px, meta ${META_MAX}ch, H1 ${H1_MAX}ch, thin <${THIN_WORDS} words, URL ${URL_MAX}ch. Pixel widths are estimates._`
|
|
364
|
+
].join("\n");
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
// src/api/image-audit.ts
|
|
368
|
+
var OVER_BYTES = 100 * 1024;
|
|
369
|
+
var MODERN = /* @__PURE__ */ new Set(["webp", "avif", "svg", "svg+xml"]);
|
|
370
|
+
function formatBytes(n) {
|
|
371
|
+
if (n == null) return null;
|
|
372
|
+
if (n < 1024) return `${n} B`;
|
|
373
|
+
if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)} KB`;
|
|
374
|
+
return `${(n / (1024 * 1024)).toFixed(2)} MB`;
|
|
375
|
+
}
|
|
376
|
+
var decodeEntities = (u) => u.replace(/&/g, "&").replace(/�?38;/g, "&").replace(/&/gi, "&").trim();
|
|
377
|
+
var dedupKey = (u) => u.replace(/^https?:\/\//i, "//").replace(/\/$/, "");
|
|
378
|
+
var formatOf = (ct, url) => {
|
|
379
|
+
if (ct) {
|
|
380
|
+
const m = ct.split(";")[0].trim().toLowerCase();
|
|
381
|
+
if (m.startsWith("image/")) return m.replace("image/", "");
|
|
382
|
+
}
|
|
383
|
+
return (url.split("?")[0].match(/\.([a-z0-9]+)$/i)?.[1] || "unknown").toLowerCase();
|
|
384
|
+
};
|
|
385
|
+
function collectUrls(pages) {
|
|
386
|
+
const byKey = /* @__PURE__ */ new Map();
|
|
387
|
+
for (const p of pages) for (const raw of p.imageLinks || []) {
|
|
388
|
+
const u = decodeEntities(raw);
|
|
389
|
+
if (!/^https?:\/\//i.test(u)) continue;
|
|
390
|
+
if (/[{}]|%7[bd]/i.test(u)) continue;
|
|
391
|
+
const k = dedupKey(u);
|
|
392
|
+
const prev = byKey.get(k);
|
|
393
|
+
if (!prev || /^https:/i.test(u) && /^http:/i.test(prev)) byKey.set(k, u);
|
|
394
|
+
}
|
|
395
|
+
return [...byKey.values()];
|
|
396
|
+
}
|
|
397
|
+
async function sizeAndType(url, timeoutMs) {
|
|
398
|
+
const ctrl = new AbortController();
|
|
399
|
+
const t = setTimeout(() => ctrl.abort(), timeoutMs);
|
|
400
|
+
const read = (res) => ({
|
|
401
|
+
len: res.headers.get("content-length"),
|
|
402
|
+
cr: res.headers.get("content-range"),
|
|
403
|
+
ct: res.headers.get("content-type"),
|
|
404
|
+
status: res.status
|
|
405
|
+
});
|
|
406
|
+
try {
|
|
407
|
+
let bytes = null;
|
|
408
|
+
let ct = null;
|
|
409
|
+
let status = null;
|
|
410
|
+
try {
|
|
411
|
+
const h = read(await fetch(url, { method: "HEAD", redirect: "follow", signal: ctrl.signal }));
|
|
412
|
+
status = h.status;
|
|
413
|
+
ct = h.ct;
|
|
414
|
+
if (h.len) bytes = Number(h.len);
|
|
415
|
+
} catch {
|
|
416
|
+
}
|
|
417
|
+
if (bytes == null) {
|
|
418
|
+
const g = await fetch(url, { method: "GET", headers: { Range: "bytes=0-0" }, redirect: "follow", signal: ctrl.signal });
|
|
419
|
+
const r = read(g);
|
|
420
|
+
status = status ?? r.status;
|
|
421
|
+
ct = ct ?? r.ct;
|
|
422
|
+
if (r.cr && r.cr.includes("/")) {
|
|
423
|
+
const tail = r.cr.split("/")[1];
|
|
424
|
+
if (tail && tail !== "*") bytes = Number(tail);
|
|
425
|
+
} else if (r.len) bytes = Number(r.len);
|
|
426
|
+
try {
|
|
427
|
+
await g.body?.cancel();
|
|
428
|
+
} catch {
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
return { url, status, bytes: Number.isFinite(bytes) ? bytes : null, contentType: ct };
|
|
432
|
+
} catch (e) {
|
|
433
|
+
return { url, status: null, bytes: null, contentType: null, error: String(e.message || e) };
|
|
434
|
+
} finally {
|
|
435
|
+
clearTimeout(t);
|
|
436
|
+
}
|
|
437
|
+
}
|
|
438
|
+
async function pool(items, n, fn) {
|
|
439
|
+
const out = new Array(items.length);
|
|
440
|
+
let i = 0;
|
|
441
|
+
await Promise.all(Array.from({ length: Math.min(n, items.length) }, async () => {
|
|
442
|
+
while (i < items.length) {
|
|
443
|
+
const idx = i++;
|
|
444
|
+
out[idx] = await fn(items[idx]);
|
|
445
|
+
}
|
|
446
|
+
}));
|
|
447
|
+
return out;
|
|
448
|
+
}
|
|
449
|
+
async function auditImages(pages, opts = {}) {
|
|
450
|
+
const concurrency = opts.concurrency ?? 12;
|
|
451
|
+
const timeoutMs = opts.timeoutMs ?? 12e3;
|
|
452
|
+
const urls = collectUrls(pages).slice(0, opts.max ?? 5e3);
|
|
453
|
+
const heads = await pool(urls, concurrency, (u) => sizeAndType(u, timeoutMs));
|
|
454
|
+
const rows = heads.map((r) => {
|
|
455
|
+
const format = formatOf(r.contentType, r.url);
|
|
456
|
+
return {
|
|
457
|
+
...r,
|
|
458
|
+
size: formatBytes(r.bytes),
|
|
459
|
+
format,
|
|
460
|
+
over100kb: r.bytes != null && r.bytes > OVER_BYTES,
|
|
461
|
+
legacyFormat: format !== "unknown" && !MODERN.has(format)
|
|
462
|
+
};
|
|
463
|
+
});
|
|
464
|
+
const sized = rows.filter((r) => r.bytes != null);
|
|
465
|
+
const totalBytes = sized.reduce((a, r) => a + r.bytes, 0);
|
|
466
|
+
const formatCounts = {};
|
|
467
|
+
for (const r of rows) formatCounts[r.format] = (formatCounts[r.format] || 0) + 1;
|
|
468
|
+
return {
|
|
469
|
+
rows: rows.sort((a, b) => (b.bytes ?? 0) - (a.bytes ?? 0)),
|
|
470
|
+
summary: {
|
|
471
|
+
unique: rows.length,
|
|
472
|
+
sized: sized.length,
|
|
473
|
+
totalBytes,
|
|
474
|
+
totalSize: formatBytes(totalBytes) ?? "0 B",
|
|
475
|
+
avgSize: formatBytes(Math.round(totalBytes / (sized.length || 1))) ?? "0 B",
|
|
476
|
+
over100kb: rows.filter((r) => r.over100kb).length,
|
|
477
|
+
legacyFormat: rows.filter((r) => r.legacyFormat).length,
|
|
478
|
+
formatCounts
|
|
479
|
+
}
|
|
480
|
+
};
|
|
481
|
+
}
|
|
482
|
+
function renderImageSection(audit) {
|
|
483
|
+
const s = audit.summary;
|
|
484
|
+
const fmt = Object.entries(s.formatCounts).sort((a, b) => b[1] - a[1]).map(([k, v]) => `${k}:${v}`).join(" ");
|
|
485
|
+
const heaviest = audit.rows.filter((r) => r.over100kb).slice(0, 15);
|
|
486
|
+
const lines = [
|
|
487
|
+
`## Images`,
|
|
488
|
+
`**${s.unique} unique images** \xB7 ${s.totalSize} total \xB7 ${s.avgSize} avg \xB7 ${s.over100kb} over 100 KB \xB7 ${s.legacyFormat} legacy format`,
|
|
489
|
+
`Formats: ${fmt}`
|
|
490
|
+
];
|
|
491
|
+
if (heaviest.length) {
|
|
492
|
+
lines.push("", "| Size | Format | URL |", "|------|--------|-----|");
|
|
493
|
+
for (const r of heaviest) lines.push(`| ${r.size} | ${r.format} | ${r.url} |`);
|
|
494
|
+
}
|
|
495
|
+
return lines.join("\n");
|
|
496
|
+
}
|
|
497
|
+
|
|
155
498
|
// src/mcp/mcp-response-formatter.ts
|
|
156
499
|
var reportSavingEnabled = true;
|
|
157
500
|
function configureReportSaving(enabled) {
|
|
@@ -185,6 +528,111 @@ function saveFullReport(full) {
|
|
|
185
528
|
return null;
|
|
186
529
|
}
|
|
187
530
|
}
|
|
531
|
+
var BULK_PAGE_THRESHOLD = 25;
|
|
532
|
+
var BULK_URL_THRESHOLD = 500;
|
|
533
|
+
function reportSavingActive() {
|
|
534
|
+
return reportSavingEnabled && process.env.MCP_SCRAPER_SAVE_REPORTS !== "false";
|
|
535
|
+
}
|
|
536
|
+
function saveBulkSite(siteUrl, pages, seo, imageAudit) {
|
|
537
|
+
if (!reportSavingActive()) return null;
|
|
538
|
+
try {
|
|
539
|
+
const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
540
|
+
const dir = join(outputBaseDir(), `extract-${slugifyReportName(siteUrl)}-${stamp}`);
|
|
541
|
+
const pagesDir = join(dir, "pages");
|
|
542
|
+
mkdirSync(pagesDir, { recursive: true });
|
|
543
|
+
const indexRows = pages.map((p, i) => {
|
|
544
|
+
const num = String(i + 1).padStart(4, "0");
|
|
545
|
+
const slug = slugifyReportName(p.url.replace(/^https?:\/\//, "")).slice(0, 60) || "page";
|
|
546
|
+
const fname = `${num}-${slug}.md`;
|
|
547
|
+
const body = (p.bodyMarkdown ?? "").trim();
|
|
548
|
+
const content = [
|
|
549
|
+
`# ${p.title ?? "Untitled"}`,
|
|
550
|
+
`- **URL:** ${p.url}`,
|
|
551
|
+
p.metaDescription ? `- **Description:** ${p.metaDescription}` : "",
|
|
552
|
+
p.schemaTypes?.length ? `- **Schema:** ${p.schemaTypes.join(", ")}` : "",
|
|
553
|
+
"",
|
|
554
|
+
body || "_(no content extracted)_"
|
|
555
|
+
].filter(Boolean).join("\n");
|
|
556
|
+
writeFileSync(join(pagesDir, fname), content, "utf8");
|
|
557
|
+
return `| ${i + 1} | ${cell(p.title ?? "Untitled")} | ${p.url} | pages/${fname} |`;
|
|
558
|
+
});
|
|
559
|
+
const dataFilesSection = seo ? [
|
|
560
|
+
`
|
|
561
|
+
## Data files (in this folder)`,
|
|
562
|
+
`- \`report.md\` \u2014 SEO issues summary (counts + example URLs)`,
|
|
563
|
+
`- \`issues.json\` \u2014 every finding with the full list of offender URLs`,
|
|
564
|
+
`- \`pages.jsonl\` \u2014 per-page SEO fields, one JSON object per page`,
|
|
565
|
+
`- \`links.jsonl\` \u2014 internal/external link edges (anchor, rel, target status)`,
|
|
566
|
+
`- \`link-metrics.jsonl\` \u2014 inlinks, crawl depth, orphan flags per page`,
|
|
567
|
+
...imageAudit ? [
|
|
568
|
+
`- \`images.jsonl\` \u2014 every image with byte size, format, over-100KB and legacy-format flags`,
|
|
569
|
+
`- \`images-summary.json\` \u2014 image totals (count, total weight, over-100KB, legacy)`
|
|
570
|
+
] : [],
|
|
571
|
+
...seo.branding ? [`- \`branding.json\` \u2014 site logo, colors, fonts`] : []
|
|
572
|
+
].join("\n") : "";
|
|
573
|
+
const index = [
|
|
574
|
+
`# Site Extract: ${siteUrl}`,
|
|
575
|
+
`**${pages.length} pages** crawled. Everything is in this folder.`,
|
|
576
|
+
`- Page content: one Markdown file per page under \`pages/\` (listed in the table below).`,
|
|
577
|
+
seo ? `- SEO crawl data: the data files listed below.` : "",
|
|
578
|
+
dataFilesSection,
|
|
579
|
+
`
|
|
580
|
+
## Pages
|
|
581
|
+
| # | Title | URL | File |
|
|
582
|
+
|---|-------|-----|------|
|
|
583
|
+
${indexRows.join("\n")}`
|
|
584
|
+
].filter(Boolean).join("\n");
|
|
585
|
+
const indexFile = join(dir, "index.md");
|
|
586
|
+
writeFileSync(indexFile, index, "utf8");
|
|
587
|
+
let seoFiles;
|
|
588
|
+
if (seo) {
|
|
589
|
+
const toJsonl = (rows) => rows.map((r) => JSON.stringify(r)).join("\n");
|
|
590
|
+
const pagesJsonl = join(dir, "pages.jsonl");
|
|
591
|
+
const linksJsonl = join(dir, "links.jsonl");
|
|
592
|
+
const metricsJsonl = join(dir, "link-metrics.jsonl");
|
|
593
|
+
const issuesFile = join(dir, "issues.json");
|
|
594
|
+
const reportFile = join(dir, "report.md");
|
|
595
|
+
writeFileSync(pagesJsonl, toJsonl(seo.pageRows), "utf8");
|
|
596
|
+
writeFileSync(linksJsonl, toJsonl(seo.edges), "utf8");
|
|
597
|
+
writeFileSync(metricsJsonl, toJsonl(seo.metrics), "utf8");
|
|
598
|
+
writeFileSync(issuesFile, JSON.stringify(seo.issues, null, 2), "utf8");
|
|
599
|
+
writeFileSync(reportFile, seo.reportMd + (imageAudit ? `
|
|
600
|
+
|
|
601
|
+
${renderImageSection(imageAudit)}` : ""), "utf8");
|
|
602
|
+
seoFiles = [pagesJsonl, linksJsonl, metricsJsonl, issuesFile, reportFile];
|
|
603
|
+
if (imageAudit) {
|
|
604
|
+
const imagesJsonl = join(dir, "images.jsonl");
|
|
605
|
+
const imagesSummary = join(dir, "images-summary.json");
|
|
606
|
+
writeFileSync(imagesJsonl, imageAudit.rows.map((r) => JSON.stringify(r)).join("\n"), "utf8");
|
|
607
|
+
writeFileSync(imagesSummary, JSON.stringify(imageAudit.summary, null, 2), "utf8");
|
|
608
|
+
seoFiles.push(imagesJsonl, imagesSummary);
|
|
609
|
+
}
|
|
610
|
+
if (seo.branding) {
|
|
611
|
+
const brandingFile = join(dir, "branding.json");
|
|
612
|
+
writeFileSync(brandingFile, JSON.stringify(seo.branding, null, 2), "utf8");
|
|
613
|
+
seoFiles.push(brandingFile);
|
|
614
|
+
}
|
|
615
|
+
}
|
|
616
|
+
return { dir, indexFile, fileCount: pages.length, seoFiles };
|
|
617
|
+
} catch {
|
|
618
|
+
return null;
|
|
619
|
+
}
|
|
620
|
+
}
|
|
621
|
+
function saveUrlInventory(siteUrl, urls) {
|
|
622
|
+
if (!reportSavingActive()) return null;
|
|
623
|
+
try {
|
|
624
|
+
const outDir = outputBaseDir();
|
|
625
|
+
mkdirSync(outDir, { recursive: true });
|
|
626
|
+
const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
627
|
+
const file = join(outDir, `${stamp}-urlmap-${slugifyReportName(siteUrl.replace(/^https?:\/\//, ""))}.csv`);
|
|
628
|
+
const csv = (v) => /[",\n]/.test(v) ? `"${v.replace(/"/g, '""')}"` : v;
|
|
629
|
+
const rows = ["url,status", ...urls.map((u) => `${csv(u.url)},${u.status ?? ""}`)];
|
|
630
|
+
writeFileSync(file, rows.join("\n"), "utf8");
|
|
631
|
+
return file;
|
|
632
|
+
} catch {
|
|
633
|
+
return null;
|
|
634
|
+
}
|
|
635
|
+
}
|
|
188
636
|
function persistScreenshotLocally(base64, url) {
|
|
189
637
|
if (!reportSavingEnabled || process.env.MCP_SCRAPER_SAVE_REPORTS === "false") return null;
|
|
190
638
|
try {
|
|
@@ -199,8 +647,8 @@ function persistScreenshotLocally(base64, url) {
|
|
|
199
647
|
return null;
|
|
200
648
|
}
|
|
201
649
|
}
|
|
202
|
-
function oneBlock(content) {
|
|
203
|
-
const filePath = saveFullReport(content);
|
|
650
|
+
function oneBlock(content, diskContent) {
|
|
651
|
+
const filePath = saveFullReport(diskContent ?? content);
|
|
204
652
|
const text = filePath ? `${content}
|
|
205
653
|
|
|
206
654
|
\u{1F4C4} Saved: \`${filePath}\`` : content;
|
|
@@ -469,7 +917,12 @@ ${[h1Lines, h2Lines].filter(Boolean).join("\n")}` : "";
|
|
|
469
917
|
].filter(Boolean).join("\n") : "";
|
|
470
918
|
const bodySection = bodyMd ? `
|
|
471
919
|
## Page Content
|
|
472
|
-
${bodyMd.slice(0, 3e3)}${bodyMd.length > 3e3 ?
|
|
920
|
+
${bodyMd.slice(0, 3e3)}${bodyMd.length > 3e3 ? `
|
|
921
|
+
|
|
922
|
+
*(truncated to 3,000 of ${bodyMd.length.toLocaleString()} chars \u2014 full content in the saved report)*` : ""}` : "";
|
|
923
|
+
const bodySectionFull = bodyMd ? `
|
|
924
|
+
## Page Content
|
|
925
|
+
${bodyMd}` : "";
|
|
473
926
|
const screenshotSection = screenshotMeta ? `
|
|
474
927
|
## Screenshot
|
|
475
928
|
- **File:** ${screenshotPath ?? "(returned inline only \u2014 disk write unavailable in this environment)"}
|
|
@@ -500,7 +953,10 @@ ${bodyMd.slice(0, 3e3)}${bodyMd.length > 3e3 ? "\n\n*(truncated)*" : ""}` : "";
|
|
|
500
953
|
const full = `# URL Extract: ${url}
|
|
501
954
|
**${title}**
|
|
502
955
|
${headingSection}${kpoSection}${brandingSection}${bodySection}${screenshotSection}${mediaSection}${tips}`;
|
|
503
|
-
const
|
|
956
|
+
const diskReport = `# URL Extract: ${url}
|
|
957
|
+
**${title}**
|
|
958
|
+
${headingSection}${kpoSection}${brandingSection}${bodySectionFull}${screenshotSection}${mediaSection}${tips}`;
|
|
959
|
+
const textResult = oneBlock(full, diskReport);
|
|
504
960
|
const structuredContent = {
|
|
505
961
|
url,
|
|
506
962
|
title: d.title ?? null,
|
|
@@ -533,7 +989,16 @@ function formatMapSiteUrls(raw, input) {
|
|
|
533
989
|
const ok = urls.filter((u) => (u.status ?? 0) >= 200 && (u.status ?? 0) < 300);
|
|
534
990
|
const broken = urls.filter((u) => u.status !== null && u.status >= 400);
|
|
535
991
|
const redirects = urls.filter((u) => u.status !== null && u.status >= 300 && u.status < 400);
|
|
536
|
-
const
|
|
992
|
+
const isBulk = urls.length > BULK_URL_THRESHOLD;
|
|
993
|
+
const inventoryFile = isBulk ? saveUrlInventory(input.url, urls.map((u) => ({ url: u.url, status: u.status ?? null }))) : null;
|
|
994
|
+
const inlineCount = isBulk ? BULK_URL_THRESHOLD : 200;
|
|
995
|
+
const urlRows = urls.slice(0, inlineCount).map((u, i) => `| ${i + 1} | ${u.url} | ${u.status ?? "\u2014"} |`).join("\n");
|
|
996
|
+
const inventoryNote = isBulk ? inventoryFile ? `
|
|
997
|
+
## \u{1F4C1} Full inventory saved
|
|
998
|
+
- **File:** \`${inventoryFile}\` (CSV: url, status \u2014 all ${urls.length} URLs)
|
|
999
|
+
Showing the first ${inlineCount} below; read the file for the complete list.` : `
|
|
1000
|
+
## \u26A0\uFE0F Full inventory not saved
|
|
1001
|
+
Report saving is disabled, so only the first ${inlineCount} of ${urls.length} URLs are shown. Enable \`MCP_SCRAPER_SAVE_REPORTS\` to capture the full inventory.` : "";
|
|
537
1002
|
const full = [
|
|
538
1003
|
`# URL Map: ${input.url}`,
|
|
539
1004
|
`**${d.totalFound} URLs** \xB7 ${(d.durationMs / 1e3).toFixed(1)}s${d.truncated ? " \xB7 *truncated*" : ""}`,
|
|
@@ -542,12 +1007,13 @@ function formatMapSiteUrls(raw, input) {
|
|
|
542
1007
|
- \u2705 2xx: ${ok.length}
|
|
543
1008
|
- \u{1F500} 3xx: ${redirects.length}
|
|
544
1009
|
- \u274C 4xx+: ${broken.length}`,
|
|
1010
|
+
inventoryNote,
|
|
545
1011
|
`
|
|
546
|
-
## URL Inventory
|
|
1012
|
+
## URL Inventory${isBulk ? ` (first ${inlineCount} of ${urls.length})` : ""}
|
|
547
1013
|
| # | URL | Status |
|
|
548
1014
|
|---|-----|--------|
|
|
549
1015
|
${urlRows}`,
|
|
550
|
-
broken.length ? `
|
|
1016
|
+
!isBulk && broken.length ? `
|
|
551
1017
|
## Broken URLs
|
|
552
1018
|
${broken.map((u) => `- ${u.url} (${u.status})`).join("\n")}` : "",
|
|
553
1019
|
`
|
|
@@ -565,47 +1031,164 @@ ${broken.map((u) => `- ${u.url} (${u.status})`).join("\n")}` : "",
|
|
|
565
1031
|
okCount: ok.length,
|
|
566
1032
|
redirectCount: redirects.length,
|
|
567
1033
|
brokenCount: broken.length,
|
|
1034
|
+
inventoryFile: inventoryFile ?? null,
|
|
568
1035
|
urls: urls.map((u) => ({ url: u.url, status: u.status ?? null })),
|
|
569
1036
|
durationMs: d.durationMs ?? 0
|
|
570
1037
|
}
|
|
571
1038
|
};
|
|
572
1039
|
}
|
|
1040
|
+
function buildSeoExport(siteUrl, pages, branding) {
|
|
1041
|
+
const { edges, metrics } = buildLinkGraph(pages, siteUrl);
|
|
1042
|
+
const pageRows = pages.map((p) => {
|
|
1043
|
+
const { bodyMarkdown: _b, schema: _s, outlinks: _o, ...rest } = p;
|
|
1044
|
+
const m = metrics.get(p.url);
|
|
1045
|
+
return { ...rest, inlinks: m?.inlinks ?? 0, crawlDepth: m?.crawlDepth ?? null, orphan: m?.orphan ?? false };
|
|
1046
|
+
});
|
|
1047
|
+
const issues = computeIssues(pages, metrics);
|
|
1048
|
+
return {
|
|
1049
|
+
pageRows,
|
|
1050
|
+
edges,
|
|
1051
|
+
metrics: [...metrics.values()],
|
|
1052
|
+
issues,
|
|
1053
|
+
reportMd: renderIssueReport(siteUrl, pages, issues, metrics),
|
|
1054
|
+
branding: branding ?? void 0
|
|
1055
|
+
};
|
|
1056
|
+
}
|
|
573
1057
|
function formatExtractSite(raw, input) {
|
|
574
1058
|
const parsed = parseData(raw);
|
|
575
1059
|
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
576
1060
|
const d = parsed.data;
|
|
577
1061
|
const pages = d.pages ?? [];
|
|
578
|
-
const
|
|
579
|
-
|
|
1062
|
+
const schemaTypesOf = (p) => p.kpo?.type ?? (Array.isArray(p.schema) && p.schema.length ? [`${p.schema.length} block(s)`] : []);
|
|
1063
|
+
const pageRow = (p, i) => {
|
|
1064
|
+
const schemaInfo = schemaTypesOf(p).join(", ") || "\u2014";
|
|
580
1065
|
return `| ${i + 1} | ${cell(p.title ?? "Untitled")} | ${p.url} | ${schemaInfo} |`;
|
|
581
|
-
}
|
|
582
|
-
const
|
|
1066
|
+
};
|
|
1067
|
+
const tips = `
|
|
1068
|
+
---
|
|
1069
|
+
\u{1F4A1} **Tips**
|
|
1070
|
+
- Map URLs first: use \`map_site_urls\`
|
|
1071
|
+
- Inspect a single page: use \`extract_url\``;
|
|
1072
|
+
const durationLine = `**${pages.length} pages** \xB7 ${((d.durationMs ?? 0) / 1e3).toFixed(1)}s`;
|
|
1073
|
+
const structuredContent = {
|
|
1074
|
+
url: input.url,
|
|
1075
|
+
pageCount: pages.length,
|
|
1076
|
+
pages: pages.map((p) => ({
|
|
1077
|
+
url: String(p.url ?? ""),
|
|
1078
|
+
title: p.title ?? null,
|
|
1079
|
+
schemaTypes: schemaTypesOf(p)
|
|
1080
|
+
})),
|
|
1081
|
+
durationMs: d.durationMs ?? 0
|
|
1082
|
+
};
|
|
1083
|
+
if (pages.length > BULK_PAGE_THRESHOLD) {
|
|
1084
|
+
const bulk = saveBulkSite(input.url, pages.map((p) => ({
|
|
1085
|
+
url: String(p.url ?? ""),
|
|
1086
|
+
title: p.title ?? null,
|
|
1087
|
+
bodyMarkdown: p.bodyMarkdown ?? null,
|
|
1088
|
+
metaDescription: p.metaDescription ?? null,
|
|
1089
|
+
schemaTypes: schemaTypesOf(p)
|
|
1090
|
+
})));
|
|
1091
|
+
const preview = pages.slice(0, BULK_PAGE_THRESHOLD).map(pageRow).join("\n");
|
|
1092
|
+
const location = bulk ? `
|
|
1093
|
+
## \u{1F4C1} Bulk scrape saved
|
|
1094
|
+
- **Folder:** \`${bulk.dir}\` \u2190 all scraped page content is here
|
|
1095
|
+
- **Start here:** \`${bulk.indexFile}\` \u2014 lists every page and its file
|
|
1096
|
+
- **Page content:** ${bulk.fileCount} Markdown files under \`pages/\` (one per page)
|
|
1097
|
+
|
|
1098
|
+
The page content is NOT inlined here to keep the context window clear \u2014 it is saved to disk. Open \`index.md\`, then open the page file you need from \`pages/\`.
|
|
1099
|
+
|
|
1100
|
+
\u{1F4A1} Want the full technical SEO audit (headings, link graph, indexability, image sizes/formats, issues)? Run \`audit_site\` on this URL instead.` : `
|
|
1101
|
+
## \u26A0\uFE0F Bulk scrape not saved
|
|
1102
|
+
Report saving is disabled in this environment, so the ${pages.length} pages of content could not be written to disk and are omitted to protect the context window. Enable \`MCP_SCRAPER_SAVE_REPORTS\` to capture bulk crawls.`;
|
|
1103
|
+
const full2 = [
|
|
1104
|
+
`# Site Extract: ${input.url}`,
|
|
1105
|
+
durationLine,
|
|
1106
|
+
location,
|
|
1107
|
+
`
|
|
1108
|
+
## Pages (first ${Math.min(BULK_PAGE_THRESHOLD, pages.length)} of ${pages.length})
|
|
1109
|
+
| # | Title | URL | Schema |
|
|
1110
|
+
|---|-------|-----|--------|
|
|
1111
|
+
${preview}`,
|
|
1112
|
+
tips
|
|
1113
|
+
].join("\n");
|
|
1114
|
+
return { content: [{ type: "text", text: full2 }], structuredContent: { ...structuredContent, bulkFolder: bulk?.dir ?? null } };
|
|
1115
|
+
}
|
|
1116
|
+
const header = [
|
|
583
1117
|
`# Site Extract: ${input.url}`,
|
|
584
|
-
|
|
1118
|
+
durationLine,
|
|
585
1119
|
`
|
|
586
1120
|
## Pages
|
|
587
1121
|
| # | Title | URL | Schema |
|
|
588
1122
|
|---|-------|-----|--------|
|
|
589
|
-
${
|
|
590
|
-
`
|
|
591
|
-
---
|
|
592
|
-
\u{1F4A1} **Tips**
|
|
593
|
-
- Map URLs first: use \`map_site_urls\`
|
|
594
|
-
- Inspect a single page: use \`extract_url\``
|
|
1123
|
+
${pages.map(pageRow).join("\n")}`
|
|
595
1124
|
].join("\n");
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
1125
|
+
const full = `${header}${tips}`;
|
|
1126
|
+
const pageDetails = pages.map((p, i) => {
|
|
1127
|
+
const body = (p.bodyMarkdown ?? "").trim();
|
|
1128
|
+
return [
|
|
1129
|
+
`
|
|
1130
|
+
## ${i + 1}. ${p.title ?? "Untitled"}`,
|
|
1131
|
+
`- **URL:** ${p.url}`,
|
|
1132
|
+
p.metaDescription ? `- **Description:** ${p.metaDescription}` : "",
|
|
1133
|
+
body ? `
|
|
1134
|
+
${body}` : "_(no content extracted)_"
|
|
1135
|
+
].filter(Boolean).join("\n");
|
|
1136
|
+
}).join("\n");
|
|
1137
|
+
const diskReport = `${header}
|
|
1138
|
+
|
|
1139
|
+
---
|
|
1140
|
+
# Full Page Content
|
|
1141
|
+
${pageDetails}${tips}`;
|
|
1142
|
+
return { ...oneBlock(full, diskReport), structuredContent };
|
|
1143
|
+
}
|
|
1144
|
+
async function formatAuditSite(raw, input) {
|
|
1145
|
+
const parsed = parseData(raw);
|
|
1146
|
+
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
1147
|
+
const d = parsed.data;
|
|
1148
|
+
const pages = d.pages ?? [];
|
|
1149
|
+
const schemaTypesOf = (p) => p.kpo?.type ?? (Array.isArray(p.schema) && p.schema.length ? [`${p.schema.length} block(s)`] : []);
|
|
1150
|
+
const seo = buildSeoExport(input.url, pages, d.branding);
|
|
1151
|
+
const imageAudit = await auditImages(pages, { concurrency: 12 });
|
|
1152
|
+
const bulk = saveBulkSite(input.url, pages.map((p) => ({
|
|
1153
|
+
url: String(p.url ?? ""),
|
|
1154
|
+
title: p.title ?? null,
|
|
1155
|
+
bodyMarkdown: p.bodyMarkdown ?? null,
|
|
1156
|
+
metaDescription: p.metaDescription ?? null,
|
|
1157
|
+
schemaTypes: schemaTypesOf(p)
|
|
1158
|
+
})), seo, imageAudit);
|
|
1159
|
+
const topIssues = Object.entries(seo.issues).sort((a, b) => b[1].count - a[1].count).slice(0, 8).map(([k, v]) => `- \`${k}\` \u2014 ${v.count}`).join("\n");
|
|
1160
|
+
const img = imageAudit.summary;
|
|
1161
|
+
const imgLine = `${img.unique} images \xB7 ${img.totalSize} total \xB7 ${img.over100kb} over 100 KB \xB7 ${img.legacyFormat} legacy format`;
|
|
1162
|
+
const durationLine = `**${pages.length} pages** \xB7 ${((d.durationMs ?? 0) / 1e3).toFixed(1)}s`;
|
|
1163
|
+
const structuredContent = {
|
|
1164
|
+
url: input.url,
|
|
1165
|
+
pageCount: pages.length,
|
|
1166
|
+
durationMs: d.durationMs ?? 0,
|
|
1167
|
+
bulkFolder: bulk?.dir ?? null,
|
|
1168
|
+
issues: Object.fromEntries(Object.entries(seo.issues).map(([k, v]) => [k, v.count])),
|
|
1169
|
+
images: { unique: img.unique, totalBytes: img.totalBytes, over100kb: img.over100kb, legacyFormat: img.legacyFormat }
|
|
608
1170
|
};
|
|
1171
|
+
const location = bulk ? `
|
|
1172
|
+
## \u{1F4C1} Technical audit saved
|
|
1173
|
+
- **Folder:** \`${bulk.dir}\` \u2190 full audit + page content here
|
|
1174
|
+
- **Start here:** \`${bulk.indexFile}\` \u2014 maps every file in the folder
|
|
1175
|
+
- **Data:** \`report.md\` (issues + image audit), \`issues.json\`, \`pages.jsonl\` (per-page fields), \`links.jsonl\`, \`link-metrics.jsonl\`, \`images.jsonl\`
|
|
1176
|
+
|
|
1177
|
+
Detail is saved to disk, not inlined. Open \`report.md\` for the audit, \`pages.jsonl\` for per-page fields (headings, canonical, indexability), \`images.jsonl\` for image sizes/formats.` : `
|
|
1178
|
+
## \u26A0\uFE0F Audit not saved
|
|
1179
|
+
Report saving is disabled in this environment. Enable \`MCP_SCRAPER_SAVE_REPORTS\` to capture the technical audit.`;
|
|
1180
|
+
const full = [
|
|
1181
|
+
`# Technical SEO Audit: ${input.url}`,
|
|
1182
|
+
durationLine,
|
|
1183
|
+
location,
|
|
1184
|
+
`
|
|
1185
|
+
## Top issues
|
|
1186
|
+
${topIssues || "_none found_"}`,
|
|
1187
|
+
`
|
|
1188
|
+
## Images
|
|
1189
|
+
${imgLine}`
|
|
1190
|
+
].join("\n");
|
|
1191
|
+
return { content: [{ type: "text", text: full }], structuredContent };
|
|
609
1192
|
}
|
|
610
1193
|
function formatYoutubeHarvest(raw, input) {
|
|
611
1194
|
const parsed = parseData(raw);
|
|
@@ -1806,11 +2389,20 @@ var ExtractUrlInputSchema = {
|
|
|
1806
2389
|
};
|
|
1807
2390
|
var MapSiteUrlsInputSchema = {
|
|
1808
2391
|
url: z.string().url().describe("Public website URL or domain to crawl for internal URLs. Use before extract_site when the user asks to audit/map/crawl a site."),
|
|
1809
|
-
maxUrls: z.number().int().min(1).max(
|
|
2392
|
+
maxUrls: z.number().int().min(1).max(1e4).optional().describe("Maximum URLs to discover. Use 100 for normal maps, up to 10000 for a full inventory. Large maps (over 500 URLs) write the complete inventory to a local file and return only a summary plus the file path instead of the full list inline.")
|
|
1810
2393
|
};
|
|
1811
2394
|
var ExtractSiteInputSchema = {
|
|
1812
|
-
url: z.string().url().describe("Public website URL or domain to
|
|
1813
|
-
maxPages: z.number().int().min(1).max(
|
|
2395
|
+
url: z.string().url().describe("Public website URL or domain to crawl for page CONTENT across multiple pages (map + scrape). Use when the user wants the content/text/markdown of a site's pages. For a technical SEO audit (issues, link graph, indexability, headings, image weights) use audit_site instead \u2014 extract_site returns content only, not analysis."),
|
|
2396
|
+
maxPages: z.number().int().min(1).max(1e4).optional().describe("Maximum pages to extract. Use 50 for a normal crawl, up to 10000 for a full-site bulk scrape. Bulk crawls (over 25 pages) switch to folder mode: every page is saved as its own Markdown file in a local folder and the response returns only a summary plus the folder path, so the full content never floods the context window."),
|
|
2397
|
+
rotateProxies: z.boolean().optional().describe("Route page fetches through rotating residential proxies in headful browsers to defeat rate-limiting and bot blocks (403/429). Discovers URLs from the sitemap, fetches in batches with a fresh proxy per batch, retries failures on a new proxy, and automatically parallelizes across the account's concurrency slots. Slower and pricier than the default fetch path, so use only when a site blocks normal crawling."),
|
|
2398
|
+
rotateProxyEvery: z.number().int().min(1).max(100).optional().describe("When rotateProxies is on, how many pages to fetch per browser/proxy before discarding it and minting a fresh one. Default 30. Lower values rotate IPs more aggressively against strict rate limits."),
|
|
2399
|
+
formats: z.array(z.enum(["markdown", "links", "json", "images", "branding"])).optional().describe("Which data formats to include in the per-page output. markdown=page content as Markdown, links=internal/external link graph, json=structured data/schema, images=per-page image URLs (links only, no downloads), branding=site-level logo/colors/fonts captured once from the homepage (requires a browser, adds time). markdown, links, json, and images are always captured cheaply from the HTML; branding is the only opt-in capture. Defaults to markdown+links when omitted.")
|
|
2400
|
+
};
|
|
2401
|
+
var AuditSiteInputSchema = {
|
|
2402
|
+
url: z.string().url().describe("Public website URL or domain to run a full technical SEO audit on. Use when the user asks for a technical audit, SEO audit, site health check, or a Screaming-Frog-style crawl \u2014 i.e. they want ANALYSIS (issues, internal link graph, indexability, heading breakdown, image sizes/formats), not just page content. For plain content scraping use extract_site instead."),
|
|
2403
|
+
maxPages: z.number().int().min(1).max(1e4).optional().describe("Maximum pages to crawl and audit. Use 50 for a normal audit, up to 10000 for a full-site audit. The audit always writes a folder of analysis files (report.md, issues.json, pages.jsonl, links.jsonl, link-metrics.jsonl, images.jsonl) plus per-page content, and returns a headline summary plus the folder path."),
|
|
2404
|
+
rotateProxies: z.boolean().optional().describe("Route page fetches through rotating residential proxies to defeat rate-limiting and bot blocks (403/429). Slower and pricier than the default fetch path, so use only when a site blocks normal crawling."),
|
|
2405
|
+
rotateProxyEvery: z.number().int().min(1).max(100).optional().describe("When rotateProxies is on, how many pages to fetch per browser/proxy before discarding it and minting a fresh one. Default 30.")
|
|
1814
2406
|
};
|
|
1815
2407
|
var YoutubeHarvestInputSchema = {
|
|
1816
2408
|
mode: z.enum(["search", "channel"]).describe("Use search for topic/keyword requests. Use channel when the user provides @handle, channel ID, or channel URL."),
|
|
@@ -2112,6 +2704,19 @@ var ExtractSiteOutputSchema = {
|
|
|
2112
2704
|
})),
|
|
2113
2705
|
durationMs: z.number().min(0)
|
|
2114
2706
|
};
|
|
2707
|
+
var AuditSiteOutputSchema = {
|
|
2708
|
+
url: z.string(),
|
|
2709
|
+
pageCount: z.number().int().min(0),
|
|
2710
|
+
durationMs: z.number().min(0),
|
|
2711
|
+
bulkFolder: z.string().nullable(),
|
|
2712
|
+
issues: z.record(z.string(), z.number()),
|
|
2713
|
+
images: z.object({
|
|
2714
|
+
unique: z.number().int().min(0),
|
|
2715
|
+
totalBytes: z.number().min(0),
|
|
2716
|
+
over100kb: z.number().int().min(0),
|
|
2717
|
+
legacyFormat: z.number().int().min(0)
|
|
2718
|
+
})
|
|
2719
|
+
};
|
|
2115
2720
|
var MapsPlaceIntelOutputSchema = {
|
|
2116
2721
|
name: z.string(),
|
|
2117
2722
|
rating: NullableString,
|
|
@@ -2915,7 +3520,7 @@ function registerSavedReportResources(server) {
|
|
|
2915
3520
|
);
|
|
2916
3521
|
}
|
|
2917
3522
|
function buildPaaExtractorMcpServer(executor, options = {}) {
|
|
2918
|
-
const server = new McpServer({ name: "mcp-scraper", version: PACKAGE_VERSION });
|
|
3523
|
+
const server = new McpServer({ name: "mcp-scraper", version: PACKAGE_VERSION }, { instructions: SERVER_INSTRUCTIONS });
|
|
2919
3524
|
registerPaaExtractorMcpTools(server, executor, options);
|
|
2920
3525
|
return server;
|
|
2921
3526
|
}
|
|
@@ -2947,18 +3552,25 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
2947
3552
|
}, async (input) => formatExtractUrl(await executor.extractUrl(input), input));
|
|
2948
3553
|
server.registerTool("map_site_urls", {
|
|
2949
3554
|
title: "Site URL Map",
|
|
2950
|
-
description: withReportNote("Map/crawl a public website when the user asks for a sitemap, URL inventory, broken-link scan, redirect scan, or crawl planning. Returns internal URLs with HTTP status and truncation metadata. Use this before extract_site when choosing pages for an audit; use extract_url for one known page."),
|
|
3555
|
+
description: withReportNote("Map/crawl a public website when the user asks for a sitemap, URL inventory, broken-link scan, redirect scan, or crawl planning. Returns internal URLs with HTTP status and truncation metadata. Supports up to maxUrls=10000; maps over 500 URLs write the complete inventory to a local CSV file and return a summary plus the file path instead of the full list inline. Use this before extract_site when choosing pages for an audit; use extract_url for one known page."),
|
|
2951
3556
|
inputSchema: MapSiteUrlsInputSchema,
|
|
2952
3557
|
outputSchema: MapSiteUrlsOutputSchema,
|
|
2953
3558
|
annotations: liveWebToolAnnotations("Site URL Map")
|
|
2954
3559
|
}, async (input) => formatMapSiteUrls(await executor.mapSiteUrls(input), input));
|
|
2955
3560
|
server.registerTool("extract_site", {
|
|
2956
|
-
title: "Multi-Page Site
|
|
2957
|
-
description: withReportNote("
|
|
3561
|
+
title: "Multi-Page Site Content Crawl",
|
|
3562
|
+
description: withReportNote("Crawl a public website and return the page CONTENT across multiple pages (map + scrape). Returns per-page titles and Markdown content. Supports up to maxPages=10000; bulk crawls over 25 pages switch to folder mode: each page is saved as its own Markdown file in a local folder and the response returns a summary plus the folder path instead of inlining all content. This tool returns content only \u2014 for a technical SEO audit (issues, internal link graph, indexability, heading breakdown, image sizes/formats) use audit_site. Use map_site_urls first when URL selection matters; use extract_url for one page."),
|
|
2958
3563
|
inputSchema: ExtractSiteInputSchema,
|
|
2959
3564
|
outputSchema: ExtractSiteOutputSchema,
|
|
2960
|
-
annotations: liveWebToolAnnotations("Multi-Page Site
|
|
3565
|
+
annotations: liveWebToolAnnotations("Multi-Page Site Content Crawl")
|
|
2961
3566
|
}, async (input) => formatExtractSite(await executor.extractSite(input), input));
|
|
3567
|
+
server.registerTool("audit_site", {
|
|
3568
|
+
title: "Technical SEO Audit",
|
|
3569
|
+
description: withReportNote("Run a full technical SEO audit (Screaming-Frog-style) on a public website. Crawls up to maxPages and produces a complete on-page + crawl analysis: per-page title/meta lengths, heading breakdown, indexability and canonical status, schema, an internal link graph with inlinks/orphans/crawl-depth, an issues report, and an image audit (byte size, format, over-100KB and legacy-format flags). Writes everything to a local folder (report.md, issues.json, pages.jsonl, links.jsonl, link-metrics.jsonl, images.jsonl) plus per-page content, and returns a headline summary plus the folder path. Use this when the user wants an audit or analysis; use extract_site when they just want page content."),
|
|
3570
|
+
inputSchema: AuditSiteInputSchema,
|
|
3571
|
+
outputSchema: AuditSiteOutputSchema,
|
|
3572
|
+
annotations: liveWebToolAnnotations("Technical SEO Audit")
|
|
3573
|
+
}, async (input) => formatAuditSite(await executor.auditSite(input), input));
|
|
2962
3574
|
server.registerTool("youtube_harvest", {
|
|
2963
3575
|
title: "YouTube Video Harvest",
|
|
2964
3576
|
description: withReportNote('Harvest YouTube video metadata when the user wants to find videos by topic, inspect a channel library, compare video angles, or get videoIds for later transcription. Use mode "search" for keyword/topic requests and mode "channel" for @handles, channel IDs, or channel URLs. Returns titles, views, durations, URLs, and videoIds for follow-up transcription. Use youtube_transcribe after selecting one video.'),
|
|
@@ -3078,7 +3690,7 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
3078
3690
|
outputSchema: WorkflowArtifactReadOutputSchema,
|
|
3079
3691
|
annotations: liveWebToolAnnotations("Read Workflow Artifact")
|
|
3080
3692
|
}, async (input) => formatWorkflowArtifactRead(await executor.workflowArtifactRead(input), input));
|
|
3081
|
-
server.registerTool("
|
|
3693
|
+
server.registerTool("rank_tracker_workflow", {
|
|
3082
3694
|
title: "Rank Tracker Blueprint Builder",
|
|
3083
3695
|
description: "Generate a build-ready database, cron/heartbeat, ingestion, metrics, and AI implementation prompt for a rank tracker powered by MCP Scraper. Supports Maps rankings through directory_workflow/maps_search, organic rankings through search_serp, AI Overview citation tracking, and People Also Ask source presence tracking. This tool is local planning only; it does not call the web or spend credits.",
|
|
3084
3696
|
inputSchema: RankTrackerBlueprintInputSchema,
|
|
@@ -3275,6 +3887,9 @@ var HttpMcpToolExecutor = class {
|
|
|
3275
3887
|
extractSite(input) {
|
|
3276
3888
|
return this.call("/extract-site", input);
|
|
3277
3889
|
}
|
|
3890
|
+
auditSite(input) {
|
|
3891
|
+
return this.call("/extract-site", input);
|
|
3892
|
+
}
|
|
3278
3893
|
youtubeHarvest(input) {
|
|
3279
3894
|
return this.call("/youtube/harvest", input);
|
|
3280
3895
|
}
|
|
@@ -3362,10 +3977,16 @@ var HttpMcpToolExecutor = class {
|
|
|
3362
3977
|
|
|
3363
3978
|
export {
|
|
3364
3979
|
harvestTimeoutBudget,
|
|
3980
|
+
buildLinkGraph,
|
|
3981
|
+
computeIssues,
|
|
3982
|
+
renderIssueReport,
|
|
3983
|
+
auditImages,
|
|
3984
|
+
renderImageSection,
|
|
3365
3985
|
configureReportSaving,
|
|
3366
3986
|
outputBaseDir,
|
|
3367
3987
|
formatCaptureSerpSnapshot,
|
|
3368
3988
|
formatCaptureSerpPageSnapshots,
|
|
3989
|
+
SERVER_INSTRUCTIONS,
|
|
3369
3990
|
CaptureSerpSnapshotOutputSchema,
|
|
3370
3991
|
CaptureSerpPageSnapshotsOutputSchema,
|
|
3371
3992
|
CaptureSerpSnapshotInputSchema,
|
|
@@ -3375,4 +3996,4 @@ export {
|
|
|
3375
3996
|
registerPaaExtractorMcpTools,
|
|
3376
3997
|
HttpMcpToolExecutor
|
|
3377
3998
|
};
|
|
3378
|
-
//# sourceMappingURL=chunk-
|
|
3999
|
+
//# sourceMappingURL=chunk-7XARJHS6.js.map
|