mcp-scraper 0.3.14 → 0.3.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/dist/bin/api-server.cjs +2261 -511
  2. package/dist/bin/api-server.cjs.map +1 -1
  3. package/dist/bin/api-server.js +3 -3
  4. package/dist/bin/browser-agent-stdio-server.cjs +6 -6
  5. package/dist/bin/browser-agent-stdio-server.cjs.map +1 -1
  6. package/dist/bin/browser-agent-stdio-server.js +2 -2
  7. package/dist/bin/mcp-scraper-cli.cjs +1 -1
  8. package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
  9. package/dist/bin/mcp-scraper-cli.js +1 -1
  10. package/dist/bin/mcp-scraper-combined-stdio-server.cjs +663 -48
  11. package/dist/bin/mcp-scraper-combined-stdio-server.cjs.map +1 -1
  12. package/dist/bin/mcp-scraper-combined-stdio-server.js +6 -5
  13. package/dist/bin/mcp-scraper-combined-stdio-server.js.map +1 -1
  14. package/dist/bin/mcp-scraper-install.cjs +4 -4
  15. package/dist/bin/mcp-scraper-install.cjs.map +1 -1
  16. package/dist/bin/mcp-scraper-install.js +2 -2
  17. package/dist/bin/mcp-stdio-server.cjs +655 -40
  18. package/dist/bin/mcp-stdio-server.cjs.map +1 -1
  19. package/dist/bin/mcp-stdio-server.js +2 -2
  20. package/dist/{chunk-DBQDG7EH.js → chunk-5H22TOXQ.js} +28 -1
  21. package/dist/chunk-5H22TOXQ.js.map +1 -0
  22. package/dist/{chunk-HM6FHV5U.js → chunk-7XARJHS6.js} +662 -41
  23. package/dist/chunk-7XARJHS6.js.map +1 -0
  24. package/dist/{chunk-KPXMPAJ3.js → chunk-ACIUCZ27.js} +4 -4
  25. package/dist/chunk-ACIUCZ27.js.map +1 -0
  26. package/dist/{chunk-AX7UBYLG.js → chunk-ISZWGIWL.js} +7 -7
  27. package/dist/chunk-ISZWGIWL.js.map +1 -0
  28. package/dist/chunk-SOMBZK3V.js +7 -0
  29. package/dist/chunk-SOMBZK3V.js.map +1 -0
  30. package/dist/{chunk-AZ5PKU4F.js → chunk-ZAUMSBV3.js} +2 -2
  31. package/dist/{db-BE4JVB3V.js → db-P76GVIFN.js} +2 -2
  32. package/dist/{server-ILIVSPNY.js → server-LPVBSE2E.js} +1133 -93
  33. package/dist/server-LPVBSE2E.js.map +1 -0
  34. package/dist/{worker-FG7ZWEGA.js → worker-2WVKKCC7.js} +3 -3
  35. package/docs/final-tooling-spec.md +206 -0
  36. package/docs/mcp-tool-design-guide.md +225 -0
  37. package/docs/mcp-tool-manifest.generated.json +57 -22
  38. package/docs/seo-crawl-report-spec.md +287 -0
  39. package/docs/tool-catalog-spec.md +388 -0
  40. package/package.json +2 -1
  41. package/dist/chunk-3QHZPR4U.js +0 -7
  42. package/dist/chunk-3QHZPR4U.js.map +0 -1
  43. package/dist/chunk-AX7UBYLG.js.map +0 -1
  44. package/dist/chunk-DBQDG7EH.js.map +0 -1
  45. package/dist/chunk-HM6FHV5U.js.map +0 -1
  46. package/dist/chunk-KPXMPAJ3.js.map +0 -1
  47. package/dist/server-ILIVSPNY.js.map +0 -1
  48. /package/dist/{chunk-AZ5PKU4F.js.map → chunk-ZAUMSBV3.js.map} +0 -0
  49. /package/dist/{db-BE4JVB3V.js.map → db-P76GVIFN.js.map} +0 -0
  50. /package/dist/{worker-FG7ZWEGA.js.map → worker-2WVKKCC7.js.map} +0 -0
@@ -11,7 +11,7 @@ import {
11
11
  } from "./chunk-LFATOGDF.js";
12
12
  import {
13
13
  PACKAGE_VERSION
14
- } from "./chunk-3QHZPR4U.js";
14
+ } from "./chunk-SOMBZK3V.js";
15
15
 
16
16
  // src/harvest-timeout.ts
17
17
  var VERCEL_FUNCTION_MAX_MS = 3e5;
@@ -27,6 +27,33 @@ function harvestTimeoutBudget(maxQuestions, serpOnly = false) {
27
27
  return { serverMs, clientMs };
28
28
  }
29
29
 
30
+ // src/mcp/server-instructions.ts
31
+ var SERVER_INSTRUCTIONS = `
32
+ This server scrapes and analyzes web, social, search, maps, and site data. Pick a tool by NAME using
33
+ this routing map, then load its schema before calling.
34
+
35
+ ROUTING
36
+ - One web page -> extract_url. Whole site (crawl + SEO report) -> extract_site. Just the URL list -> map_site_urls.
37
+ - Web search (organic SERP) -> search_serp. Full SERP + People-Also-Ask / AI-Overview detail -> harvest_paa.
38
+ - Google Maps: find places -> maps_search; one place deep-dive + reviews -> maps_place_intel.
39
+ - YouTube: find or list videos -> youtube_harvest; transcribe one video -> youtube_transcribe.
40
+ - Facebook: find ads -> facebook_ad_search; transcribe an ad -> facebook_ad_transcribe; transcribe a
41
+ video -> facebook_video_transcribe; page profile/intel -> facebook_page_intel.
42
+ - Instagram: profile inventory -> instagram_profile_content; one post or reel -> instagram_media_download.
43
+ - Workflows (multi-step): local directory build -> directory_workflow; rank blueprint -> rank_tracker_workflow;
44
+ AI-answer citation fan-out (AEO) -> query_fanout_workflow; deep research -> deep_research_workflow.
45
+ - Anything without a dedicated tool (e.g. Reddit, arbitrary logged-in sites) -> the browser_* agent
46
+ (browser_open, then navigate/read); save/reuse logins via browser_profile_*.
47
+
48
+ NOTES
49
+ - Bulk / full-site crawls: call extract_site with rotateProxies:true for blocked or rate-limited sites.
50
+ It discovers URLs and returns a saved folder/artifact plus a summary, not the full content inline.
51
+ - Tools that open a browser session or transcribe media cost credits and take longer. Prefer the
52
+ cheapest tool that answers the question (plain search/extract before browser agents).
53
+ - Large results are saved to disk or an artifact and returned as a summary plus a path; read the path
54
+ for full detail rather than expecting the whole payload inline.
55
+ `.trim();
56
+
30
57
  // src/mcp/paa-mcp-server.ts
31
58
  import { McpServer, ResourceTemplate } from "@modelcontextprotocol/sdk/server/mcp.js";
32
59
  import { readdirSync, readFileSync, statSync } from "fs";
@@ -152,6 +179,322 @@ function suggestWorkflowRecipes(goal, limit = 3) {
152
179
  return scored.sort((a, b) => b.score - a.score || a.recipe.title.localeCompare(b.recipe.title)).slice(0, Math.max(1, limit)).map((item) => item.recipe);
153
180
  }
154
181
 
182
+ // src/api/seo-link-graph.ts
183
+ function normalize2(u) {
184
+ return u.split("#")[0].replace(/\/$/, "");
185
+ }
186
+ function buildLinkGraph(pages, startUrl) {
187
+ const statusByUrl = /* @__PURE__ */ new Map();
188
+ for (const p of pages) statusByUrl.set(normalize2(p.url), p.status);
189
+ const edges = [];
190
+ for (const p of pages) {
191
+ for (const l of p.outlinks ?? []) {
192
+ const nofollow = (l.rel ?? "").split(/\s+/).includes("nofollow");
193
+ edges.push({
194
+ from: p.url,
195
+ to: l.href,
196
+ anchor: l.anchor,
197
+ rel: l.rel,
198
+ internal: l.internal,
199
+ nofollow,
200
+ targetStatus: statusByUrl.get(normalize2(l.href)) ?? null
201
+ });
202
+ }
203
+ }
204
+ const inboundFrom = /* @__PURE__ */ new Map();
205
+ const inboundAnchors = /* @__PURE__ */ new Map();
206
+ const adjacency = /* @__PURE__ */ new Map();
207
+ for (const e of edges) {
208
+ if (!e.internal) continue;
209
+ const to = normalize2(e.to);
210
+ const from = normalize2(e.from);
211
+ if (!inboundFrom.has(to)) inboundFrom.set(to, /* @__PURE__ */ new Set());
212
+ inboundFrom.get(to).add(from);
213
+ if (e.anchor) {
214
+ if (!inboundAnchors.has(to)) inboundAnchors.set(to, []);
215
+ inboundAnchors.get(to).push(e.anchor);
216
+ }
217
+ if (!e.nofollow && (statusByUrl.get(to) === 200 || !statusByUrl.has(to))) {
218
+ if (!adjacency.has(from)) adjacency.set(from, /* @__PURE__ */ new Set());
219
+ adjacency.get(from).add(to);
220
+ }
221
+ }
222
+ const depth = /* @__PURE__ */ new Map();
223
+ const start = normalize2(startUrl);
224
+ const queue = [start];
225
+ depth.set(start, 0);
226
+ while (queue.length > 0) {
227
+ const cur = queue.shift();
228
+ const d = depth.get(cur);
229
+ for (const next of adjacency.get(cur) ?? []) {
230
+ if (!depth.has(next)) {
231
+ depth.set(next, d + 1);
232
+ queue.push(next);
233
+ }
234
+ }
235
+ }
236
+ const topAnchorsFor = (url) => {
237
+ const counts = /* @__PURE__ */ new Map();
238
+ for (const a of inboundAnchors.get(url) ?? []) counts.set(a, (counts.get(a) ?? 0) + 1);
239
+ return [...counts.entries()].sort((a, b) => b[1] - a[1]).slice(0, 3).map((e) => e[0]);
240
+ };
241
+ const metrics = /* @__PURE__ */ new Map();
242
+ for (const p of pages) {
243
+ const key = normalize2(p.url);
244
+ const inbound = inboundFrom.get(key) ?? /* @__PURE__ */ new Set();
245
+ metrics.set(p.url, {
246
+ url: p.url,
247
+ inlinks: inbound.size,
248
+ uniqueInlinks: inbound.size,
249
+ outlinksInternal: (p.outlinks ?? []).filter((l) => l.internal).length,
250
+ outlinksExternal: (p.outlinks ?? []).filter((l) => !l.internal).length,
251
+ crawlDepth: depth.has(key) ? depth.get(key) : null,
252
+ orphan: inbound.size === 0 && key !== start,
253
+ topAnchors: topAnchorsFor(key)
254
+ });
255
+ }
256
+ return { edges, metrics };
257
+ }
258
+
259
+ // src/api/seo-issues.ts
260
+ var THIN_WORDS = 200;
261
+ var TITLE_MAX = 60;
262
+ var TITLE_PX_MAX = 561;
263
+ var TITLE_MIN = 30;
264
+ var META_MAX = 155;
265
+ var META_MIN = 70;
266
+ var H1_MAX = 70;
267
+ var URL_MAX = 115;
268
+ function dupes(pages, key) {
269
+ const groups = /* @__PURE__ */ new Map();
270
+ for (const p of pages) {
271
+ const k = key(p);
272
+ if (k === null || k === void 0 || k === "") continue;
273
+ const ks = String(k);
274
+ if (!groups.has(ks)) groups.set(ks, []);
275
+ groups.get(ks).push(p.url);
276
+ }
277
+ return [...groups.values()].filter((g) => g.length > 1).flat();
278
+ }
279
+ function computeIssues(pages, metrics) {
280
+ const group = (urls) => ({ count: urls.length, urls: urls.slice(0, 200) });
281
+ const where = (fn) => pages.filter(fn).map((p) => p.url);
282
+ const ok = (p) => p.status === 200;
283
+ const pathname = (u) => {
284
+ try {
285
+ return new URL(u).pathname;
286
+ } catch {
287
+ return "";
288
+ }
289
+ };
290
+ const report = {
291
+ "title.missing": group(where((p) => ok(p) && !p.title)),
292
+ "title.duplicate": group(dupes(pages.filter(ok), (p) => p.title)),
293
+ "title.tooLong": group(where((p) => (p.titleLength ?? 0) > TITLE_MAX || (p.titlePixels ?? 0) > TITLE_PX_MAX)),
294
+ "title.tooShort": group(where((p) => p.title != null && (p.titleLength ?? 0) < TITLE_MIN)),
295
+ "title.sameAsH1": group(where((p) => !!p.title && !!p.h1 && p.title.trim() === p.h1.trim())),
296
+ "meta.missing": group(where((p) => ok(p) && !p.metaDescription)),
297
+ "meta.duplicate": group(dupes(pages.filter(ok), (p) => p.metaDescription)),
298
+ "meta.tooLong": group(where((p) => (p.metaDescLength ?? 0) > META_MAX)),
299
+ "meta.tooShort": group(where((p) => p.metaDescription != null && (p.metaDescLength ?? 0) < META_MIN)),
300
+ "h1.missing": group(where((p) => ok(p) && !p.h1)),
301
+ "h1.multiple": group(where((p) => !!p.h1_2)),
302
+ "h1.duplicate": group(dupes(pages.filter(ok), (p) => p.h1)),
303
+ "h1.tooLong": group(where((p) => (p.h1?.length ?? 0) > H1_MAX)),
304
+ "h2.missing": group(where((p) => ok(p) && p.h2Count === 0)),
305
+ "indexability.nonIndexable": group(where((p) => !p.indexable)),
306
+ "indexability.noindex": group(where((p) => p.indexabilityReason === "noindex")),
307
+ "canonical.missing": group(where((p) => ok(p) && !p.canonicalUrl)),
308
+ "canonical.canonicalised": group(where((p) => p.indexabilityReason === "canonicalised")),
309
+ "response.broken4xx": group(where((p) => (p.status ?? 0) >= 400 && (p.status ?? 0) < 500)),
310
+ "response.error5xx": group(where((p) => (p.status ?? 0) >= 500)),
311
+ "response.redirect3xx": group(where((p) => (p.status ?? 0) >= 300 && (p.status ?? 0) < 400)),
312
+ "response.noResponse": group(where((p) => p.status === null)),
313
+ "content.thin": group(where((p) => ok(p) && p.wordCount > 0 && p.wordCount < THIN_WORDS)),
314
+ "content.exactDuplicate": group(dupes(pages.filter((p) => ok(p) && p.contentHash), (p) => p.contentHash)),
315
+ "images.missingAlt": group(where((p) => p.imagesMissingAlt > 0)),
316
+ "schema.missing": group(where((p) => ok(p) && p.schemaTypes.length === 0)),
317
+ "url.tooLong": group(where((p) => p.url.length > URL_MAX)),
318
+ "url.uppercase": group(where((p) => /[A-Z]/.test(pathname(p.url)))),
319
+ "url.underscores": group(where((p) => pathname(p.url).includes("_"))),
320
+ "links.orphan": group([...metrics.values()].filter((m) => m.orphan).map((m) => m.url))
321
+ };
322
+ const normUrl = (u) => {
323
+ try {
324
+ const x = new URL(u);
325
+ x.hash = "";
326
+ return (x.origin + x.pathname).replace(/\/+$/, "") + x.search;
327
+ } catch {
328
+ return u.replace(/#.*$/, "").replace(/\/+$/, "");
329
+ }
330
+ };
331
+ const statusByUrl = new Map(pages.map((p) => [normUrl(p.url), p.status]));
332
+ const brokenLinkPages = /* @__PURE__ */ new Set();
333
+ for (const p of pages) {
334
+ for (const l of p.outlinks ?? []) {
335
+ if (!l.internal) continue;
336
+ const st = statusByUrl.get(normUrl(l.href));
337
+ if (st != null && st >= 400) brokenLinkPages.add(p.url);
338
+ }
339
+ }
340
+ report["links.brokenInternal"] = group([...brokenLinkPages]);
341
+ return report;
342
+ }
343
+ function renderIssueReport(siteUrl, pages, report, metrics) {
344
+ const total = pages.length;
345
+ const sev = (key) => key.startsWith("response.broken") || key.startsWith("response.error") || key.startsWith("links.broken") ? "\u{1F534}" : key.startsWith("title.missing") || key.startsWith("h1.missing") || key.startsWith("indexability") || key.startsWith("canonical.missing") ? "\u{1F7E0}" : "\u{1F7E1}";
346
+ const rows = Object.entries(report).filter(([, g]) => g.count > 0).sort((a, b) => b[1].count - a[1].count).map(([k, g]) => `| ${sev(k)} | \`${k}\` | ${g.count} | ${g.urls.slice(0, 3).join(" \xB7 ")}${g.urls.length > 3 ? " \u2026" : ""} |`);
347
+ const depths = [...metrics.values()].map((m) => m.crawlDepth).filter((d) => d != null);
348
+ const maxDepth = depths.length ? Math.max(...depths) : 0;
349
+ const orphans = [...metrics.values()].filter((m) => m.orphan).length;
350
+ const topLinked = [...metrics.values()].sort((a, b) => b.inlinks - a.inlinks).slice(0, 5);
351
+ return [
352
+ `# SEO Crawl Report: ${siteUrl}`,
353
+ `**${total} pages crawled** \xB7 max crawl depth ${maxDepth} \xB7 ${orphans} orphan page(s)`,
354
+ `
355
+ ## Issues
356
+ | | Issue | Count | Examples |
357
+ |---|-------|-------|----------|
358
+ ${rows.join("\n") || "| \u2705 | none | 0 | \u2014 |"}`,
359
+ `
360
+ ## Most-linked pages
361
+ ${topLinked.map((m) => `- ${m.inlinks} inlinks \xB7 depth ${m.crawlDepth ?? "\u2014"} \xB7 ${m.url}`).join("\n")}`,
362
+ `
363
+ _Thresholds: title ${TITLE_MAX}ch/${TITLE_PX_MAX}px, meta ${META_MAX}ch, H1 ${H1_MAX}ch, thin <${THIN_WORDS} words, URL ${URL_MAX}ch. Pixel widths are estimates._`
364
+ ].join("\n");
365
+ }
366
+
367
+ // src/api/image-audit.ts
368
+ var OVER_BYTES = 100 * 1024;
369
+ var MODERN = /* @__PURE__ */ new Set(["webp", "avif", "svg", "svg+xml"]);
370
+ function formatBytes(n) {
371
+ if (n == null) return null;
372
+ if (n < 1024) return `${n} B`;
373
+ if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)} KB`;
374
+ return `${(n / (1024 * 1024)).toFixed(2)} MB`;
375
+ }
376
+ var decodeEntities = (u) => u.replace(/&amp;/g, "&").replace(/&#0?38;/g, "&").replace(/&#x26;/gi, "&").trim();
377
+ var dedupKey = (u) => u.replace(/^https?:\/\//i, "//").replace(/\/$/, "");
378
+ var formatOf = (ct, url) => {
379
+ if (ct) {
380
+ const m = ct.split(";")[0].trim().toLowerCase();
381
+ if (m.startsWith("image/")) return m.replace("image/", "");
382
+ }
383
+ return (url.split("?")[0].match(/\.([a-z0-9]+)$/i)?.[1] || "unknown").toLowerCase();
384
+ };
385
+ function collectUrls(pages) {
386
+ const byKey = /* @__PURE__ */ new Map();
387
+ for (const p of pages) for (const raw of p.imageLinks || []) {
388
+ const u = decodeEntities(raw);
389
+ if (!/^https?:\/\//i.test(u)) continue;
390
+ if (/[{}]|%7[bd]/i.test(u)) continue;
391
+ const k = dedupKey(u);
392
+ const prev = byKey.get(k);
393
+ if (!prev || /^https:/i.test(u) && /^http:/i.test(prev)) byKey.set(k, u);
394
+ }
395
+ return [...byKey.values()];
396
+ }
397
+ async function sizeAndType(url, timeoutMs) {
398
+ const ctrl = new AbortController();
399
+ const t = setTimeout(() => ctrl.abort(), timeoutMs);
400
+ const read = (res) => ({
401
+ len: res.headers.get("content-length"),
402
+ cr: res.headers.get("content-range"),
403
+ ct: res.headers.get("content-type"),
404
+ status: res.status
405
+ });
406
+ try {
407
+ let bytes = null;
408
+ let ct = null;
409
+ let status = null;
410
+ try {
411
+ const h = read(await fetch(url, { method: "HEAD", redirect: "follow", signal: ctrl.signal }));
412
+ status = h.status;
413
+ ct = h.ct;
414
+ if (h.len) bytes = Number(h.len);
415
+ } catch {
416
+ }
417
+ if (bytes == null) {
418
+ const g = await fetch(url, { method: "GET", headers: { Range: "bytes=0-0" }, redirect: "follow", signal: ctrl.signal });
419
+ const r = read(g);
420
+ status = status ?? r.status;
421
+ ct = ct ?? r.ct;
422
+ if (r.cr && r.cr.includes("/")) {
423
+ const tail = r.cr.split("/")[1];
424
+ if (tail && tail !== "*") bytes = Number(tail);
425
+ } else if (r.len) bytes = Number(r.len);
426
+ try {
427
+ await g.body?.cancel();
428
+ } catch {
429
+ }
430
+ }
431
+ return { url, status, bytes: Number.isFinite(bytes) ? bytes : null, contentType: ct };
432
+ } catch (e) {
433
+ return { url, status: null, bytes: null, contentType: null, error: String(e.message || e) };
434
+ } finally {
435
+ clearTimeout(t);
436
+ }
437
+ }
438
+ async function pool(items, n, fn) {
439
+ const out = new Array(items.length);
440
+ let i = 0;
441
+ await Promise.all(Array.from({ length: Math.min(n, items.length) }, async () => {
442
+ while (i < items.length) {
443
+ const idx = i++;
444
+ out[idx] = await fn(items[idx]);
445
+ }
446
+ }));
447
+ return out;
448
+ }
449
+ async function auditImages(pages, opts = {}) {
450
+ const concurrency = opts.concurrency ?? 12;
451
+ const timeoutMs = opts.timeoutMs ?? 12e3;
452
+ const urls = collectUrls(pages).slice(0, opts.max ?? 5e3);
453
+ const heads = await pool(urls, concurrency, (u) => sizeAndType(u, timeoutMs));
454
+ const rows = heads.map((r) => {
455
+ const format = formatOf(r.contentType, r.url);
456
+ return {
457
+ ...r,
458
+ size: formatBytes(r.bytes),
459
+ format,
460
+ over100kb: r.bytes != null && r.bytes > OVER_BYTES,
461
+ legacyFormat: format !== "unknown" && !MODERN.has(format)
462
+ };
463
+ });
464
+ const sized = rows.filter((r) => r.bytes != null);
465
+ const totalBytes = sized.reduce((a, r) => a + r.bytes, 0);
466
+ const formatCounts = {};
467
+ for (const r of rows) formatCounts[r.format] = (formatCounts[r.format] || 0) + 1;
468
+ return {
469
+ rows: rows.sort((a, b) => (b.bytes ?? 0) - (a.bytes ?? 0)),
470
+ summary: {
471
+ unique: rows.length,
472
+ sized: sized.length,
473
+ totalBytes,
474
+ totalSize: formatBytes(totalBytes) ?? "0 B",
475
+ avgSize: formatBytes(Math.round(totalBytes / (sized.length || 1))) ?? "0 B",
476
+ over100kb: rows.filter((r) => r.over100kb).length,
477
+ legacyFormat: rows.filter((r) => r.legacyFormat).length,
478
+ formatCounts
479
+ }
480
+ };
481
+ }
482
+ function renderImageSection(audit) {
483
+ const s = audit.summary;
484
+ const fmt = Object.entries(s.formatCounts).sort((a, b) => b[1] - a[1]).map(([k, v]) => `${k}:${v}`).join(" ");
485
+ const heaviest = audit.rows.filter((r) => r.over100kb).slice(0, 15);
486
+ const lines = [
487
+ `## Images`,
488
+ `**${s.unique} unique images** \xB7 ${s.totalSize} total \xB7 ${s.avgSize} avg \xB7 ${s.over100kb} over 100 KB \xB7 ${s.legacyFormat} legacy format`,
489
+ `Formats: ${fmt}`
490
+ ];
491
+ if (heaviest.length) {
492
+ lines.push("", "| Size | Format | URL |", "|------|--------|-----|");
493
+ for (const r of heaviest) lines.push(`| ${r.size} | ${r.format} | ${r.url} |`);
494
+ }
495
+ return lines.join("\n");
496
+ }
497
+
155
498
  // src/mcp/mcp-response-formatter.ts
156
499
  var reportSavingEnabled = true;
157
500
  function configureReportSaving(enabled) {
@@ -185,6 +528,111 @@ function saveFullReport(full) {
185
528
  return null;
186
529
  }
187
530
  }
531
+ var BULK_PAGE_THRESHOLD = 25;
532
+ var BULK_URL_THRESHOLD = 500;
533
+ function reportSavingActive() {
534
+ return reportSavingEnabled && process.env.MCP_SCRAPER_SAVE_REPORTS !== "false";
535
+ }
536
+ function saveBulkSite(siteUrl, pages, seo, imageAudit) {
537
+ if (!reportSavingActive()) return null;
538
+ try {
539
+ const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
540
+ const dir = join(outputBaseDir(), `extract-${slugifyReportName(siteUrl)}-${stamp}`);
541
+ const pagesDir = join(dir, "pages");
542
+ mkdirSync(pagesDir, { recursive: true });
543
+ const indexRows = pages.map((p, i) => {
544
+ const num = String(i + 1).padStart(4, "0");
545
+ const slug = slugifyReportName(p.url.replace(/^https?:\/\//, "")).slice(0, 60) || "page";
546
+ const fname = `${num}-${slug}.md`;
547
+ const body = (p.bodyMarkdown ?? "").trim();
548
+ const content = [
549
+ `# ${p.title ?? "Untitled"}`,
550
+ `- **URL:** ${p.url}`,
551
+ p.metaDescription ? `- **Description:** ${p.metaDescription}` : "",
552
+ p.schemaTypes?.length ? `- **Schema:** ${p.schemaTypes.join(", ")}` : "",
553
+ "",
554
+ body || "_(no content extracted)_"
555
+ ].filter(Boolean).join("\n");
556
+ writeFileSync(join(pagesDir, fname), content, "utf8");
557
+ return `| ${i + 1} | ${cell(p.title ?? "Untitled")} | ${p.url} | pages/${fname} |`;
558
+ });
559
+ const dataFilesSection = seo ? [
560
+ `
561
+ ## Data files (in this folder)`,
562
+ `- \`report.md\` \u2014 SEO issues summary (counts + example URLs)`,
563
+ `- \`issues.json\` \u2014 every finding with the full list of offender URLs`,
564
+ `- \`pages.jsonl\` \u2014 per-page SEO fields, one JSON object per page`,
565
+ `- \`links.jsonl\` \u2014 internal/external link edges (anchor, rel, target status)`,
566
+ `- \`link-metrics.jsonl\` \u2014 inlinks, crawl depth, orphan flags per page`,
567
+ ...imageAudit ? [
568
+ `- \`images.jsonl\` \u2014 every image with byte size, format, over-100KB and legacy-format flags`,
569
+ `- \`images-summary.json\` \u2014 image totals (count, total weight, over-100KB, legacy)`
570
+ ] : [],
571
+ ...seo.branding ? [`- \`branding.json\` \u2014 site logo, colors, fonts`] : []
572
+ ].join("\n") : "";
573
+ const index = [
574
+ `# Site Extract: ${siteUrl}`,
575
+ `**${pages.length} pages** crawled. Everything is in this folder.`,
576
+ `- Page content: one Markdown file per page under \`pages/\` (listed in the table below).`,
577
+ seo ? `- SEO crawl data: the data files listed below.` : "",
578
+ dataFilesSection,
579
+ `
580
+ ## Pages
581
+ | # | Title | URL | File |
582
+ |---|-------|-----|------|
583
+ ${indexRows.join("\n")}`
584
+ ].filter(Boolean).join("\n");
585
+ const indexFile = join(dir, "index.md");
586
+ writeFileSync(indexFile, index, "utf8");
587
+ let seoFiles;
588
+ if (seo) {
589
+ const toJsonl = (rows) => rows.map((r) => JSON.stringify(r)).join("\n");
590
+ const pagesJsonl = join(dir, "pages.jsonl");
591
+ const linksJsonl = join(dir, "links.jsonl");
592
+ const metricsJsonl = join(dir, "link-metrics.jsonl");
593
+ const issuesFile = join(dir, "issues.json");
594
+ const reportFile = join(dir, "report.md");
595
+ writeFileSync(pagesJsonl, toJsonl(seo.pageRows), "utf8");
596
+ writeFileSync(linksJsonl, toJsonl(seo.edges), "utf8");
597
+ writeFileSync(metricsJsonl, toJsonl(seo.metrics), "utf8");
598
+ writeFileSync(issuesFile, JSON.stringify(seo.issues, null, 2), "utf8");
599
+ writeFileSync(reportFile, seo.reportMd + (imageAudit ? `
600
+
601
+ ${renderImageSection(imageAudit)}` : ""), "utf8");
602
+ seoFiles = [pagesJsonl, linksJsonl, metricsJsonl, issuesFile, reportFile];
603
+ if (imageAudit) {
604
+ const imagesJsonl = join(dir, "images.jsonl");
605
+ const imagesSummary = join(dir, "images-summary.json");
606
+ writeFileSync(imagesJsonl, imageAudit.rows.map((r) => JSON.stringify(r)).join("\n"), "utf8");
607
+ writeFileSync(imagesSummary, JSON.stringify(imageAudit.summary, null, 2), "utf8");
608
+ seoFiles.push(imagesJsonl, imagesSummary);
609
+ }
610
+ if (seo.branding) {
611
+ const brandingFile = join(dir, "branding.json");
612
+ writeFileSync(brandingFile, JSON.stringify(seo.branding, null, 2), "utf8");
613
+ seoFiles.push(brandingFile);
614
+ }
615
+ }
616
+ return { dir, indexFile, fileCount: pages.length, seoFiles };
617
+ } catch {
618
+ return null;
619
+ }
620
+ }
621
+ function saveUrlInventory(siteUrl, urls) {
622
+ if (!reportSavingActive()) return null;
623
+ try {
624
+ const outDir = outputBaseDir();
625
+ mkdirSync(outDir, { recursive: true });
626
+ const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
627
+ const file = join(outDir, `${stamp}-urlmap-${slugifyReportName(siteUrl.replace(/^https?:\/\//, ""))}.csv`);
628
+ const csv = (v) => /[",\n]/.test(v) ? `"${v.replace(/"/g, '""')}"` : v;
629
+ const rows = ["url,status", ...urls.map((u) => `${csv(u.url)},${u.status ?? ""}`)];
630
+ writeFileSync(file, rows.join("\n"), "utf8");
631
+ return file;
632
+ } catch {
633
+ return null;
634
+ }
635
+ }
188
636
  function persistScreenshotLocally(base64, url) {
189
637
  if (!reportSavingEnabled || process.env.MCP_SCRAPER_SAVE_REPORTS === "false") return null;
190
638
  try {
@@ -199,8 +647,8 @@ function persistScreenshotLocally(base64, url) {
199
647
  return null;
200
648
  }
201
649
  }
202
- function oneBlock(content) {
203
- const filePath = saveFullReport(content);
650
+ function oneBlock(content, diskContent) {
651
+ const filePath = saveFullReport(diskContent ?? content);
204
652
  const text = filePath ? `${content}
205
653
 
206
654
  \u{1F4C4} Saved: \`${filePath}\`` : content;
@@ -469,7 +917,12 @@ ${[h1Lines, h2Lines].filter(Boolean).join("\n")}` : "";
469
917
  ].filter(Boolean).join("\n") : "";
470
918
  const bodySection = bodyMd ? `
471
919
  ## Page Content
472
- ${bodyMd.slice(0, 3e3)}${bodyMd.length > 3e3 ? "\n\n*(truncated)*" : ""}` : "";
920
+ ${bodyMd.slice(0, 3e3)}${bodyMd.length > 3e3 ? `
921
+
922
+ *(truncated to 3,000 of ${bodyMd.length.toLocaleString()} chars \u2014 full content in the saved report)*` : ""}` : "";
923
+ const bodySectionFull = bodyMd ? `
924
+ ## Page Content
925
+ ${bodyMd}` : "";
473
926
  const screenshotSection = screenshotMeta ? `
474
927
  ## Screenshot
475
928
  - **File:** ${screenshotPath ?? "(returned inline only \u2014 disk write unavailable in this environment)"}
@@ -500,7 +953,10 @@ ${bodyMd.slice(0, 3e3)}${bodyMd.length > 3e3 ? "\n\n*(truncated)*" : ""}` : "";
500
953
  const full = `# URL Extract: ${url}
501
954
  **${title}**
502
955
  ${headingSection}${kpoSection}${brandingSection}${bodySection}${screenshotSection}${mediaSection}${tips}`;
503
- const textResult = oneBlock(full);
956
+ const diskReport = `# URL Extract: ${url}
957
+ **${title}**
958
+ ${headingSection}${kpoSection}${brandingSection}${bodySectionFull}${screenshotSection}${mediaSection}${tips}`;
959
+ const textResult = oneBlock(full, diskReport);
504
960
  const structuredContent = {
505
961
  url,
506
962
  title: d.title ?? null,
@@ -533,7 +989,16 @@ function formatMapSiteUrls(raw, input) {
533
989
  const ok = urls.filter((u) => (u.status ?? 0) >= 200 && (u.status ?? 0) < 300);
534
990
  const broken = urls.filter((u) => u.status !== null && u.status >= 400);
535
991
  const redirects = urls.filter((u) => u.status !== null && u.status >= 300 && u.status < 400);
536
- const urlRows = urls.slice(0, 200).map((u, i) => `| ${i + 1} | ${u.url} | ${u.status ?? "\u2014"} |`).join("\n");
992
+ const isBulk = urls.length > BULK_URL_THRESHOLD;
993
+ const inventoryFile = isBulk ? saveUrlInventory(input.url, urls.map((u) => ({ url: u.url, status: u.status ?? null }))) : null;
994
+ const inlineCount = isBulk ? BULK_URL_THRESHOLD : 200;
995
+ const urlRows = urls.slice(0, inlineCount).map((u, i) => `| ${i + 1} | ${u.url} | ${u.status ?? "\u2014"} |`).join("\n");
996
+ const inventoryNote = isBulk ? inventoryFile ? `
997
+ ## \u{1F4C1} Full inventory saved
998
+ - **File:** \`${inventoryFile}\` (CSV: url, status \u2014 all ${urls.length} URLs)
999
+ Showing the first ${inlineCount} below; read the file for the complete list.` : `
1000
+ ## \u26A0\uFE0F Full inventory not saved
1001
+ Report saving is disabled, so only the first ${inlineCount} of ${urls.length} URLs are shown. Enable \`MCP_SCRAPER_SAVE_REPORTS\` to capture the full inventory.` : "";
537
1002
  const full = [
538
1003
  `# URL Map: ${input.url}`,
539
1004
  `**${d.totalFound} URLs** \xB7 ${(d.durationMs / 1e3).toFixed(1)}s${d.truncated ? " \xB7 *truncated*" : ""}`,
@@ -542,12 +1007,13 @@ function formatMapSiteUrls(raw, input) {
542
1007
  - \u2705 2xx: ${ok.length}
543
1008
  - \u{1F500} 3xx: ${redirects.length}
544
1009
  - \u274C 4xx+: ${broken.length}`,
1010
+ inventoryNote,
545
1011
  `
546
- ## URL Inventory
1012
+ ## URL Inventory${isBulk ? ` (first ${inlineCount} of ${urls.length})` : ""}
547
1013
  | # | URL | Status |
548
1014
  |---|-----|--------|
549
1015
  ${urlRows}`,
550
- broken.length ? `
1016
+ !isBulk && broken.length ? `
551
1017
  ## Broken URLs
552
1018
  ${broken.map((u) => `- ${u.url} (${u.status})`).join("\n")}` : "",
553
1019
  `
@@ -565,47 +1031,164 @@ ${broken.map((u) => `- ${u.url} (${u.status})`).join("\n")}` : "",
565
1031
  okCount: ok.length,
566
1032
  redirectCount: redirects.length,
567
1033
  brokenCount: broken.length,
1034
+ inventoryFile: inventoryFile ?? null,
568
1035
  urls: urls.map((u) => ({ url: u.url, status: u.status ?? null })),
569
1036
  durationMs: d.durationMs ?? 0
570
1037
  }
571
1038
  };
572
1039
  }
1040
+ function buildSeoExport(siteUrl, pages, branding) {
1041
+ const { edges, metrics } = buildLinkGraph(pages, siteUrl);
1042
+ const pageRows = pages.map((p) => {
1043
+ const { bodyMarkdown: _b, schema: _s, outlinks: _o, ...rest } = p;
1044
+ const m = metrics.get(p.url);
1045
+ return { ...rest, inlinks: m?.inlinks ?? 0, crawlDepth: m?.crawlDepth ?? null, orphan: m?.orphan ?? false };
1046
+ });
1047
+ const issues = computeIssues(pages, metrics);
1048
+ return {
1049
+ pageRows,
1050
+ edges,
1051
+ metrics: [...metrics.values()],
1052
+ issues,
1053
+ reportMd: renderIssueReport(siteUrl, pages, issues, metrics),
1054
+ branding: branding ?? void 0
1055
+ };
1056
+ }
573
1057
  function formatExtractSite(raw, input) {
574
1058
  const parsed = parseData(raw);
575
1059
  if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
576
1060
  const d = parsed.data;
577
1061
  const pages = d.pages ?? [];
578
- const pageRows = pages.map((p, i) => {
579
- const schemaInfo = p.kpo?.type?.join(", ") ?? (Array.isArray(p.schema) && p.schema.length ? `${p.schema.length} block(s)` : "\u2014");
1062
+ const schemaTypesOf = (p) => p.kpo?.type ?? (Array.isArray(p.schema) && p.schema.length ? [`${p.schema.length} block(s)`] : []);
1063
+ const pageRow = (p, i) => {
1064
+ const schemaInfo = schemaTypesOf(p).join(", ") || "\u2014";
580
1065
  return `| ${i + 1} | ${cell(p.title ?? "Untitled")} | ${p.url} | ${schemaInfo} |`;
581
- }).join("\n");
582
- const full = [
1066
+ };
1067
+ const tips = `
1068
+ ---
1069
+ \u{1F4A1} **Tips**
1070
+ - Map URLs first: use \`map_site_urls\`
1071
+ - Inspect a single page: use \`extract_url\``;
1072
+ const durationLine = `**${pages.length} pages** \xB7 ${((d.durationMs ?? 0) / 1e3).toFixed(1)}s`;
1073
+ const structuredContent = {
1074
+ url: input.url,
1075
+ pageCount: pages.length,
1076
+ pages: pages.map((p) => ({
1077
+ url: String(p.url ?? ""),
1078
+ title: p.title ?? null,
1079
+ schemaTypes: schemaTypesOf(p)
1080
+ })),
1081
+ durationMs: d.durationMs ?? 0
1082
+ };
1083
+ if (pages.length > BULK_PAGE_THRESHOLD) {
1084
+ const bulk = saveBulkSite(input.url, pages.map((p) => ({
1085
+ url: String(p.url ?? ""),
1086
+ title: p.title ?? null,
1087
+ bodyMarkdown: p.bodyMarkdown ?? null,
1088
+ metaDescription: p.metaDescription ?? null,
1089
+ schemaTypes: schemaTypesOf(p)
1090
+ })));
1091
+ const preview = pages.slice(0, BULK_PAGE_THRESHOLD).map(pageRow).join("\n");
1092
+ const location = bulk ? `
1093
+ ## \u{1F4C1} Bulk scrape saved
1094
+ - **Folder:** \`${bulk.dir}\` \u2190 all scraped page content is here
1095
+ - **Start here:** \`${bulk.indexFile}\` \u2014 lists every page and its file
1096
+ - **Page content:** ${bulk.fileCount} Markdown files under \`pages/\` (one per page)
1097
+
1098
+ The page content is NOT inlined here to keep the context window clear \u2014 it is saved to disk. Open \`index.md\`, then open the page file you need from \`pages/\`.
1099
+
1100
+ \u{1F4A1} Want the full technical SEO audit (headings, link graph, indexability, image sizes/formats, issues)? Run \`audit_site\` on this URL instead.` : `
1101
+ ## \u26A0\uFE0F Bulk scrape not saved
1102
+ Report saving is disabled in this environment, so the ${pages.length} pages of content could not be written to disk and are omitted to protect the context window. Enable \`MCP_SCRAPER_SAVE_REPORTS\` to capture bulk crawls.`;
1103
+ const full2 = [
1104
+ `# Site Extract: ${input.url}`,
1105
+ durationLine,
1106
+ location,
1107
+ `
1108
+ ## Pages (first ${Math.min(BULK_PAGE_THRESHOLD, pages.length)} of ${pages.length})
1109
+ | # | Title | URL | Schema |
1110
+ |---|-------|-----|--------|
1111
+ ${preview}`,
1112
+ tips
1113
+ ].join("\n");
1114
+ return { content: [{ type: "text", text: full2 }], structuredContent: { ...structuredContent, bulkFolder: bulk?.dir ?? null } };
1115
+ }
1116
+ const header = [
583
1117
  `# Site Extract: ${input.url}`,
584
- `**${pages.length} pages** \xB7 ${((d.durationMs ?? 0) / 1e3).toFixed(1)}s`,
1118
+ durationLine,
585
1119
  `
586
1120
  ## Pages
587
1121
  | # | Title | URL | Schema |
588
1122
  |---|-------|-----|--------|
589
- ${pageRows}`,
590
- `
591
- ---
592
- \u{1F4A1} **Tips**
593
- - Map URLs first: use \`map_site_urls\`
594
- - Inspect a single page: use \`extract_url\``
1123
+ ${pages.map(pageRow).join("\n")}`
595
1124
  ].join("\n");
596
- return {
597
- ...oneBlock(full),
598
- structuredContent: {
599
- url: input.url,
600
- pageCount: pages.length,
601
- pages: pages.map((p) => ({
602
- url: String(p.url ?? ""),
603
- title: p.title ?? null,
604
- schemaTypes: p.kpo?.type ?? []
605
- })),
606
- durationMs: d.durationMs ?? 0
607
- }
1125
+ const full = `${header}${tips}`;
1126
+ const pageDetails = pages.map((p, i) => {
1127
+ const body = (p.bodyMarkdown ?? "").trim();
1128
+ return [
1129
+ `
1130
+ ## ${i + 1}. ${p.title ?? "Untitled"}`,
1131
+ `- **URL:** ${p.url}`,
1132
+ p.metaDescription ? `- **Description:** ${p.metaDescription}` : "",
1133
+ body ? `
1134
+ ${body}` : "_(no content extracted)_"
1135
+ ].filter(Boolean).join("\n");
1136
+ }).join("\n");
1137
+ const diskReport = `${header}
1138
+
1139
+ ---
1140
+ # Full Page Content
1141
+ ${pageDetails}${tips}`;
1142
+ return { ...oneBlock(full, diskReport), structuredContent };
1143
+ }
1144
+ async function formatAuditSite(raw, input) {
1145
+ const parsed = parseData(raw);
1146
+ if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
1147
+ const d = parsed.data;
1148
+ const pages = d.pages ?? [];
1149
+ const schemaTypesOf = (p) => p.kpo?.type ?? (Array.isArray(p.schema) && p.schema.length ? [`${p.schema.length} block(s)`] : []);
1150
+ const seo = buildSeoExport(input.url, pages, d.branding);
1151
+ const imageAudit = await auditImages(pages, { concurrency: 12 });
1152
+ const bulk = saveBulkSite(input.url, pages.map((p) => ({
1153
+ url: String(p.url ?? ""),
1154
+ title: p.title ?? null,
1155
+ bodyMarkdown: p.bodyMarkdown ?? null,
1156
+ metaDescription: p.metaDescription ?? null,
1157
+ schemaTypes: schemaTypesOf(p)
1158
+ })), seo, imageAudit);
1159
+ const topIssues = Object.entries(seo.issues).sort((a, b) => b[1].count - a[1].count).slice(0, 8).map(([k, v]) => `- \`${k}\` \u2014 ${v.count}`).join("\n");
1160
+ const img = imageAudit.summary;
1161
+ const imgLine = `${img.unique} images \xB7 ${img.totalSize} total \xB7 ${img.over100kb} over 100 KB \xB7 ${img.legacyFormat} legacy format`;
1162
+ const durationLine = `**${pages.length} pages** \xB7 ${((d.durationMs ?? 0) / 1e3).toFixed(1)}s`;
1163
+ const structuredContent = {
1164
+ url: input.url,
1165
+ pageCount: pages.length,
1166
+ durationMs: d.durationMs ?? 0,
1167
+ bulkFolder: bulk?.dir ?? null,
1168
+ issues: Object.fromEntries(Object.entries(seo.issues).map(([k, v]) => [k, v.count])),
1169
+ images: { unique: img.unique, totalBytes: img.totalBytes, over100kb: img.over100kb, legacyFormat: img.legacyFormat }
608
1170
  };
1171
+ const location = bulk ? `
1172
+ ## \u{1F4C1} Technical audit saved
1173
+ - **Folder:** \`${bulk.dir}\` \u2190 full audit + page content here
1174
+ - **Start here:** \`${bulk.indexFile}\` \u2014 maps every file in the folder
1175
+ - **Data:** \`report.md\` (issues + image audit), \`issues.json\`, \`pages.jsonl\` (per-page fields), \`links.jsonl\`, \`link-metrics.jsonl\`, \`images.jsonl\`
1176
+
1177
+ Detail is saved to disk, not inlined. Open \`report.md\` for the audit, \`pages.jsonl\` for per-page fields (headings, canonical, indexability), \`images.jsonl\` for image sizes/formats.` : `
1178
+ ## \u26A0\uFE0F Audit not saved
1179
+ Report saving is disabled in this environment. Enable \`MCP_SCRAPER_SAVE_REPORTS\` to capture the technical audit.`;
1180
+ const full = [
1181
+ `# Technical SEO Audit: ${input.url}`,
1182
+ durationLine,
1183
+ location,
1184
+ `
1185
+ ## Top issues
1186
+ ${topIssues || "_none found_"}`,
1187
+ `
1188
+ ## Images
1189
+ ${imgLine}`
1190
+ ].join("\n");
1191
+ return { content: [{ type: "text", text: full }], structuredContent };
609
1192
  }
610
1193
  function formatYoutubeHarvest(raw, input) {
611
1194
  const parsed = parseData(raw);
@@ -1806,11 +2389,20 @@ var ExtractUrlInputSchema = {
1806
2389
  };
1807
2390
  var MapSiteUrlsInputSchema = {
1808
2391
  url: z.string().url().describe("Public website URL or domain to crawl for internal URLs. Use before extract_site when the user asks to audit/map/crawl a site."),
1809
- maxUrls: z.number().int().min(1).max(500).optional().describe("Maximum URLs to discover. Use 100 for normal maps, higher when the user asks for a full inventory.")
2392
+ maxUrls: z.number().int().min(1).max(1e4).optional().describe("Maximum URLs to discover. Use 100 for normal maps, up to 10000 for a full inventory. Large maps (over 500 URLs) write the complete inventory to a local file and return only a summary plus the file path instead of the full list inline.")
1810
2393
  };
1811
2394
  var ExtractSiteInputSchema = {
1812
- url: z.string().url().describe("Public website URL or domain to extract across multiple pages. Use when the user asks for a site audit, website crawl, or full-site content/schema extraction."),
1813
- maxPages: z.number().int().min(1).max(50).optional().describe("Maximum pages to extract. Use 50 when the user asks for full results or a complete crawl within MCP limits.")
2395
+ url: z.string().url().describe("Public website URL or domain to crawl for page CONTENT across multiple pages (map + scrape). Use when the user wants the content/text/markdown of a site's pages. For a technical SEO audit (issues, link graph, indexability, headings, image weights) use audit_site instead \u2014 extract_site returns content only, not analysis."),
2396
+ maxPages: z.number().int().min(1).max(1e4).optional().describe("Maximum pages to extract. Use 50 for a normal crawl, up to 10000 for a full-site bulk scrape. Bulk crawls (over 25 pages) switch to folder mode: every page is saved as its own Markdown file in a local folder and the response returns only a summary plus the folder path, so the full content never floods the context window."),
2397
+ rotateProxies: z.boolean().optional().describe("Route page fetches through rotating residential proxies in headful browsers to defeat rate-limiting and bot blocks (403/429). Discovers URLs from the sitemap, fetches in batches with a fresh proxy per batch, retries failures on a new proxy, and automatically parallelizes across the account's concurrency slots. Slower and pricier than the default fetch path, so use only when a site blocks normal crawling."),
2398
+ rotateProxyEvery: z.number().int().min(1).max(100).optional().describe("When rotateProxies is on, how many pages to fetch per browser/proxy before discarding it and minting a fresh one. Default 30. Lower values rotate IPs more aggressively against strict rate limits."),
2399
+ formats: z.array(z.enum(["markdown", "links", "json", "images", "branding"])).optional().describe("Which data formats to include in the per-page output. markdown=page content as Markdown, links=internal/external link graph, json=structured data/schema, images=per-page image URLs (links only, no downloads), branding=site-level logo/colors/fonts captured once from the homepage (requires a browser, adds time). markdown, links, json, and images are always captured cheaply from the HTML; branding is the only opt-in capture. Defaults to markdown+links when omitted.")
2400
+ };
2401
+ var AuditSiteInputSchema = {
2402
+ url: z.string().url().describe("Public website URL or domain to run a full technical SEO audit on. Use when the user asks for a technical audit, SEO audit, site health check, or a Screaming-Frog-style crawl \u2014 i.e. they want ANALYSIS (issues, internal link graph, indexability, heading breakdown, image sizes/formats), not just page content. For plain content scraping use extract_site instead."),
2403
+ maxPages: z.number().int().min(1).max(1e4).optional().describe("Maximum pages to crawl and audit. Use 50 for a normal audit, up to 10000 for a full-site audit. The audit always writes a folder of analysis files (report.md, issues.json, pages.jsonl, links.jsonl, link-metrics.jsonl, images.jsonl) plus per-page content, and returns a headline summary plus the folder path."),
2404
+ rotateProxies: z.boolean().optional().describe("Route page fetches through rotating residential proxies to defeat rate-limiting and bot blocks (403/429). Slower and pricier than the default fetch path, so use only when a site blocks normal crawling."),
2405
+ rotateProxyEvery: z.number().int().min(1).max(100).optional().describe("When rotateProxies is on, how many pages to fetch per browser/proxy before discarding it and minting a fresh one. Default 30.")
1814
2406
  };
1815
2407
  var YoutubeHarvestInputSchema = {
1816
2408
  mode: z.enum(["search", "channel"]).describe("Use search for topic/keyword requests. Use channel when the user provides @handle, channel ID, or channel URL."),
@@ -2112,6 +2704,19 @@ var ExtractSiteOutputSchema = {
2112
2704
  })),
2113
2705
  durationMs: z.number().min(0)
2114
2706
  };
2707
+ var AuditSiteOutputSchema = {
2708
+ url: z.string(),
2709
+ pageCount: z.number().int().min(0),
2710
+ durationMs: z.number().min(0),
2711
+ bulkFolder: z.string().nullable(),
2712
+ issues: z.record(z.string(), z.number()),
2713
+ images: z.object({
2714
+ unique: z.number().int().min(0),
2715
+ totalBytes: z.number().min(0),
2716
+ over100kb: z.number().int().min(0),
2717
+ legacyFormat: z.number().int().min(0)
2718
+ })
2719
+ };
2115
2720
  var MapsPlaceIntelOutputSchema = {
2116
2721
  name: z.string(),
2117
2722
  rating: NullableString,
@@ -2915,7 +3520,7 @@ function registerSavedReportResources(server) {
2915
3520
  );
2916
3521
  }
2917
3522
  function buildPaaExtractorMcpServer(executor, options = {}) {
2918
- const server = new McpServer({ name: "mcp-scraper", version: PACKAGE_VERSION });
3523
+ const server = new McpServer({ name: "mcp-scraper", version: PACKAGE_VERSION }, { instructions: SERVER_INSTRUCTIONS });
2919
3524
  registerPaaExtractorMcpTools(server, executor, options);
2920
3525
  return server;
2921
3526
  }
@@ -2947,18 +3552,25 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
2947
3552
  }, async (input) => formatExtractUrl(await executor.extractUrl(input), input));
2948
3553
  server.registerTool("map_site_urls", {
2949
3554
  title: "Site URL Map",
2950
- description: withReportNote("Map/crawl a public website when the user asks for a sitemap, URL inventory, broken-link scan, redirect scan, or crawl planning. Returns internal URLs with HTTP status and truncation metadata. Use this before extract_site when choosing pages for an audit; use extract_url for one known page."),
3555
+ description: withReportNote("Map/crawl a public website when the user asks for a sitemap, URL inventory, broken-link scan, redirect scan, or crawl planning. Returns internal URLs with HTTP status and truncation metadata. Supports up to maxUrls=10000; maps over 500 URLs write the complete inventory to a local CSV file and return a summary plus the file path instead of the full list inline. Use this before extract_site when choosing pages for an audit; use extract_url for one known page."),
2951
3556
  inputSchema: MapSiteUrlsInputSchema,
2952
3557
  outputSchema: MapSiteUrlsOutputSchema,
2953
3558
  annotations: liveWebToolAnnotations("Site URL Map")
2954
3559
  }, async (input) => formatMapSiteUrls(await executor.mapSiteUrls(input), input));
2955
3560
  server.registerTool("extract_site", {
2956
- title: "Multi-Page Site Extract",
2957
- description: withReportNote("Run multi-page extraction across a public website when the user asks for a website audit, competitor audit, full-site content crawl, schema inventory, or page metadata review. Returns per-page titles, H1s, metadata, headings, schema/entity data, canonical URLs, and content. Use map_site_urls first when URL selection matters; use extract_url for one page."),
3561
+ title: "Multi-Page Site Content Crawl",
3562
+ description: withReportNote("Crawl a public website and return the page CONTENT across multiple pages (map + scrape). Returns per-page titles and Markdown content. Supports up to maxPages=10000; bulk crawls over 25 pages switch to folder mode: each page is saved as its own Markdown file in a local folder and the response returns a summary plus the folder path instead of inlining all content. This tool returns content only \u2014 for a technical SEO audit (issues, internal link graph, indexability, heading breakdown, image sizes/formats) use audit_site. Use map_site_urls first when URL selection matters; use extract_url for one page."),
2958
3563
  inputSchema: ExtractSiteInputSchema,
2959
3564
  outputSchema: ExtractSiteOutputSchema,
2960
- annotations: liveWebToolAnnotations("Multi-Page Site Extract")
3565
+ annotations: liveWebToolAnnotations("Multi-Page Site Content Crawl")
2961
3566
  }, async (input) => formatExtractSite(await executor.extractSite(input), input));
3567
+ server.registerTool("audit_site", {
3568
+ title: "Technical SEO Audit",
3569
+ description: withReportNote("Run a full technical SEO audit (Screaming-Frog-style) on a public website. Crawls up to maxPages and produces a complete on-page + crawl analysis: per-page title/meta lengths, heading breakdown, indexability and canonical status, schema, an internal link graph with inlinks/orphans/crawl-depth, an issues report, and an image audit (byte size, format, over-100KB and legacy-format flags). Writes everything to a local folder (report.md, issues.json, pages.jsonl, links.jsonl, link-metrics.jsonl, images.jsonl) plus per-page content, and returns a headline summary plus the folder path. Use this when the user wants an audit or analysis; use extract_site when they just want page content."),
3570
+ inputSchema: AuditSiteInputSchema,
3571
+ outputSchema: AuditSiteOutputSchema,
3572
+ annotations: liveWebToolAnnotations("Technical SEO Audit")
3573
+ }, async (input) => formatAuditSite(await executor.auditSite(input), input));
2962
3574
  server.registerTool("youtube_harvest", {
2963
3575
  title: "YouTube Video Harvest",
2964
3576
  description: withReportNote('Harvest YouTube video metadata when the user wants to find videos by topic, inspect a channel library, compare video angles, or get videoIds for later transcription. Use mode "search" for keyword/topic requests and mode "channel" for @handles, channel IDs, or channel URLs. Returns titles, views, durations, URLs, and videoIds for follow-up transcription. Use youtube_transcribe after selecting one video.'),
@@ -3078,7 +3690,7 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
3078
3690
  outputSchema: WorkflowArtifactReadOutputSchema,
3079
3691
  annotations: liveWebToolAnnotations("Read Workflow Artifact")
3080
3692
  }, async (input) => formatWorkflowArtifactRead(await executor.workflowArtifactRead(input), input));
3081
- server.registerTool("rank_tracker_blueprint", {
3693
+ server.registerTool("rank_tracker_workflow", {
3082
3694
  title: "Rank Tracker Blueprint Builder",
3083
3695
  description: "Generate a build-ready database, cron/heartbeat, ingestion, metrics, and AI implementation prompt for a rank tracker powered by MCP Scraper. Supports Maps rankings through directory_workflow/maps_search, organic rankings through search_serp, AI Overview citation tracking, and People Also Ask source presence tracking. This tool is local planning only; it does not call the web or spend credits.",
3084
3696
  inputSchema: RankTrackerBlueprintInputSchema,
@@ -3275,6 +3887,9 @@ var HttpMcpToolExecutor = class {
3275
3887
  extractSite(input) {
3276
3888
  return this.call("/extract-site", input);
3277
3889
  }
3890
+ auditSite(input) {
3891
+ return this.call("/extract-site", input);
3892
+ }
3278
3893
  youtubeHarvest(input) {
3279
3894
  return this.call("/youtube/harvest", input);
3280
3895
  }
@@ -3362,10 +3977,16 @@ var HttpMcpToolExecutor = class {
3362
3977
 
3363
3978
  export {
3364
3979
  harvestTimeoutBudget,
3980
+ buildLinkGraph,
3981
+ computeIssues,
3982
+ renderIssueReport,
3983
+ auditImages,
3984
+ renderImageSection,
3365
3985
  configureReportSaving,
3366
3986
  outputBaseDir,
3367
3987
  formatCaptureSerpSnapshot,
3368
3988
  formatCaptureSerpPageSnapshots,
3989
+ SERVER_INSTRUCTIONS,
3369
3990
  CaptureSerpSnapshotOutputSchema,
3370
3991
  CaptureSerpPageSnapshotsOutputSchema,
3371
3992
  CaptureSerpSnapshotInputSchema,
@@ -3375,4 +3996,4 @@ export {
3375
3996
  registerPaaExtractorMcpTools,
3376
3997
  HttpMcpToolExecutor
3377
3998
  };
3378
- //# sourceMappingURL=chunk-HM6FHV5U.js.map
3999
+ //# sourceMappingURL=chunk-7XARJHS6.js.map