mcp-scraper 0.3.45 → 0.3.46
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/api-server.cjs +378 -157
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +1 -1
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-install.cjs +1 -1
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +1 -1
- package/dist/bin/mcp-stdio-server.cjs +435 -180
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +2 -2
- package/dist/{chunk-E7BZFMWB.js → chunk-SS5YBZVI.js} +472 -212
- package/dist/chunk-SS5YBZVI.js.map +1 -0
- package/dist/chunk-WT6OT2T3.js +7 -0
- package/dist/chunk-WT6OT2T3.js.map +1 -0
- package/dist/{server-MKUN6NQZ.js → server-MGE3GYJW.js} +26 -76
- package/dist/server-MGE3GYJW.js.map +1 -0
- package/package.json +1 -1
- package/dist/chunk-E7BZFMWB.js.map +0 -1
- package/dist/chunk-NVTFLLEG.js +0 -7
- package/dist/chunk-NVTFLLEG.js.map +0 -1
- package/dist/server-MKUN6NQZ.js.map +0 -1
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
PACKAGE_VERSION
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-WT6OT2T3.js";
|
|
4
4
|
import {
|
|
5
5
|
browserServiceProfileName,
|
|
6
6
|
browserServiceProfileSaveChanges,
|
|
@@ -91,9 +91,13 @@ function sanitizeHarvestResult(result) {
|
|
|
91
91
|
}
|
|
92
92
|
|
|
93
93
|
// src/mcp/server-instructions.ts
|
|
94
|
-
|
|
94
|
+
function serverInstructions(savesReportsLocally) {
|
|
95
|
+
const reportLine = savesReportsLocally ? "All report-producing tools also save a full Markdown report to disk." : "On this hosted endpoint, small reports return inline; large reports are stored as artifacts \u2014 read them back with report_artifact_read.";
|
|
96
|
+
return `
|
|
95
97
|
# MCP Scraper
|
|
96
98
|
|
|
99
|
+
${reportLine}
|
|
100
|
+
|
|
97
101
|
Scrapes and analyzes web, search, maps, social, and site data. **Pick a tool by NAME from the map
|
|
98
102
|
below, then load its schema before calling.** Where a tool needs a value another tool produces, the
|
|
99
103
|
seam is noted so you can chain them.
|
|
@@ -177,19 +181,22 @@ Multi-step orchestrations \u2014 prefer these over hand-chaining primitives when
|
|
|
177
181
|
sites. It returns a saved folder/artifact plus a summary, not the full content inline.
|
|
178
182
|
- Browser sessions and media transcription cost credits and take longer \u2014 prefer the cheapest tool that
|
|
179
183
|
answers the question (plain search/extract before browser agents).
|
|
180
|
-
- Large results are saved to disk or an artifact and returned as a summary plus a path
|
|
181
|
-
full detail rather than expecting the whole payload inline.
|
|
184
|
+
- Large results are saved to disk or an artifact and returned as a summary plus a path or artifactId;
|
|
185
|
+
read it back for full detail rather than expecting the whole payload inline.
|
|
182
186
|
`.trim();
|
|
187
|
+
}
|
|
188
|
+
var SERVER_INSTRUCTIONS = serverInstructions(true);
|
|
183
189
|
|
|
184
190
|
// src/mcp/paa-mcp-server.ts
|
|
185
191
|
import { McpServer, ResourceTemplate } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
186
192
|
import { readdirSync, readFileSync, statSync } from "fs";
|
|
187
|
-
import { basename, join as
|
|
193
|
+
import { basename, join as join3 } from "path";
|
|
194
|
+
import { createHash } from "crypto";
|
|
188
195
|
|
|
189
196
|
// src/mcp/mcp-response-formatter.ts
|
|
190
|
-
import { mkdirSync, writeFileSync } from "fs";
|
|
191
|
-
import { homedir } from "os";
|
|
192
|
-
import { join } from "path";
|
|
197
|
+
import { mkdirSync as mkdirSync2, writeFileSync as writeFileSync2 } from "fs";
|
|
198
|
+
import { homedir as homedir2 } from "os";
|
|
199
|
+
import { join as join2 } from "path";
|
|
193
200
|
|
|
194
201
|
// src/mcp/workflow-catalog.ts
|
|
195
202
|
var WORKFLOW_RECIPES = [
|
|
@@ -726,7 +733,129 @@ function renderImageSection(audit) {
|
|
|
726
733
|
return lines.join("\n");
|
|
727
734
|
}
|
|
728
735
|
|
|
736
|
+
// src/mcp/report-artifact-offload.ts
|
|
737
|
+
import { randomBytes } from "crypto";
|
|
738
|
+
|
|
739
|
+
// src/api/blob-store.ts
|
|
740
|
+
import { mkdirSync, writeFileSync } from "fs";
|
|
741
|
+
import { homedir } from "os";
|
|
742
|
+
import { join, dirname } from "path";
|
|
743
|
+
function byteLength(data) {
|
|
744
|
+
return Buffer.isBuffer(data) ? data.length : Buffer.byteLength(data);
|
|
745
|
+
}
|
|
746
|
+
var LocalBlobStore = class {
|
|
747
|
+
constructor(baseDir) {
|
|
748
|
+
this.baseDir = baseDir;
|
|
749
|
+
}
|
|
750
|
+
baseDir;
|
|
751
|
+
kind = "local";
|
|
752
|
+
async put(key, data, contentType = "application/octet-stream") {
|
|
753
|
+
const path = join(this.baseDir, "blobs", key);
|
|
754
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
755
|
+
writeFileSync(path, data);
|
|
756
|
+
return { key, url: `file://${path}`, bytes: byteLength(data), contentType };
|
|
757
|
+
}
|
|
758
|
+
async get(key) {
|
|
759
|
+
const path = join(this.baseDir, "blobs", key);
|
|
760
|
+
try {
|
|
761
|
+
const { readFileSync: readFileSync2 } = await import("fs");
|
|
762
|
+
return readFileSync2(path);
|
|
763
|
+
} catch {
|
|
764
|
+
return null;
|
|
765
|
+
}
|
|
766
|
+
}
|
|
767
|
+
};
|
|
768
|
+
function localBaseDir() {
|
|
769
|
+
return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join(homedir(), "Downloads", "mcp-scraper");
|
|
770
|
+
}
|
|
771
|
+
var cached = null;
|
|
772
|
+
function getBlobStore() {
|
|
773
|
+
if (cached) return cached;
|
|
774
|
+
if (process.env.BLOB_READ_WRITE_TOKEN) {
|
|
775
|
+
cached = new VercelBlobStore(process.env.BLOB_READ_WRITE_TOKEN);
|
|
776
|
+
} else {
|
|
777
|
+
cached = new LocalBlobStore(localBaseDir());
|
|
778
|
+
}
|
|
779
|
+
return cached;
|
|
780
|
+
}
|
|
781
|
+
var VercelBlobStore = class {
|
|
782
|
+
constructor(token) {
|
|
783
|
+
this.token = token;
|
|
784
|
+
}
|
|
785
|
+
token;
|
|
786
|
+
kind = "vercel-blob";
|
|
787
|
+
async put(key, data, contentType = "application/octet-stream") {
|
|
788
|
+
const { put } = await import("@vercel/blob");
|
|
789
|
+
const body = Buffer.isBuffer(data) ? data : Buffer.from(data);
|
|
790
|
+
const result = await put(key, body, {
|
|
791
|
+
access: "public",
|
|
792
|
+
token: this.token,
|
|
793
|
+
contentType,
|
|
794
|
+
addRandomSuffix: true
|
|
795
|
+
});
|
|
796
|
+
return { key, url: result.url, bytes: byteLength(data), contentType };
|
|
797
|
+
}
|
|
798
|
+
async get(key) {
|
|
799
|
+
try {
|
|
800
|
+
const { list } = await import("@vercel/blob");
|
|
801
|
+
const res = await list({ prefix: key, token: this.token, limit: 1 });
|
|
802
|
+
const match = res.blobs.find((b) => b.pathname === key || b.pathname.startsWith(key));
|
|
803
|
+
if (!match) return null;
|
|
804
|
+
const resp = await fetch(match.url);
|
|
805
|
+
if (!resp.ok) return null;
|
|
806
|
+
return Buffer.from(await resp.arrayBuffer());
|
|
807
|
+
} catch {
|
|
808
|
+
return null;
|
|
809
|
+
}
|
|
810
|
+
}
|
|
811
|
+
};
|
|
812
|
+
|
|
813
|
+
// src/mcp/report-artifact-offload.ts
|
|
814
|
+
var REPORT_BLOB_TTL_MS = 24 * 60 * 60 * 1e3;
|
|
815
|
+
var REPORT_BLOB_PREFIX = "mcp-reports/";
|
|
816
|
+
var PREVIEW_CHARS = 2e3;
|
|
817
|
+
var ARTIFACT_OFFLOAD_ENABLED = process.env.MCP_SCRAPER_ARTIFACT_OFFLOAD !== "false";
|
|
818
|
+
async function offloadReport(toolName, ownerId, report) {
|
|
819
|
+
const timestamp = Date.now();
|
|
820
|
+
const random = randomBytes(6).toString("hex");
|
|
821
|
+
const key = `${REPORT_BLOB_PREFIX}${ownerId}/${toolName}/${timestamp}-${random}.md`;
|
|
822
|
+
const stored = await getBlobStore().put(key, report, "text/markdown");
|
|
823
|
+
return {
|
|
824
|
+
artifactId: stored.key,
|
|
825
|
+
bytes: stored.bytes,
|
|
826
|
+
expiresAt: new Date(timestamp + REPORT_BLOB_TTL_MS).toISOString(),
|
|
827
|
+
preview: report.slice(0, PREVIEW_CHARS)
|
|
828
|
+
};
|
|
829
|
+
}
|
|
830
|
+
function artifactOwnerId(artifactId) {
|
|
831
|
+
if (!artifactId.startsWith(REPORT_BLOB_PREFIX)) return null;
|
|
832
|
+
const rest = artifactId.slice(REPORT_BLOB_PREFIX.length);
|
|
833
|
+
const segment = rest.split("/")[0];
|
|
834
|
+
return segment || null;
|
|
835
|
+
}
|
|
836
|
+
async function readArtifactWindow(artifactId, offset, maxBytes) {
|
|
837
|
+
const buf = await getBlobStore().get(artifactId);
|
|
838
|
+
if (!buf) return null;
|
|
839
|
+
const totalBytes = buf.length;
|
|
840
|
+
const start = Math.max(0, offset);
|
|
841
|
+
const end = Math.min(totalBytes, start + maxBytes);
|
|
842
|
+
const slice = buf.subarray(start, end);
|
|
843
|
+
const nextOffset = end < totalBytes ? end : null;
|
|
844
|
+
return { text: slice.toString("utf8"), totalBytes, nextOffset };
|
|
845
|
+
}
|
|
846
|
+
function summaryEnvelope(executiveSummary, offloaded) {
|
|
847
|
+
return [
|
|
848
|
+
executiveSummary.trim(),
|
|
849
|
+
"",
|
|
850
|
+
"--- Full report stored as artifact ---",
|
|
851
|
+
`artifactId: ${offloaded.artifactId} \xB7 ${offloaded.bytes} bytes \xB7 expires ${offloaded.expiresAt}`,
|
|
852
|
+
"Read it with report_artifact_read (supports offset/maxBytes windowing)."
|
|
853
|
+
].join("\n");
|
|
854
|
+
}
|
|
855
|
+
|
|
729
856
|
// src/mcp/mcp-response-formatter.ts
|
|
857
|
+
var INLINE_BUDGET_BYTES = Number(process.env.MCP_SCRAPER_INLINE_BUDGET ?? 5e4);
|
|
858
|
+
var STRUCTURED_ARRAY_CAP = 25;
|
|
730
859
|
var reportSavingEnabled = true;
|
|
731
860
|
function configureReportSaving(enabled) {
|
|
732
861
|
reportSavingEnabled = enabled;
|
|
@@ -744,16 +873,16 @@ function reportTitle(full) {
|
|
|
744
873
|
return title?.replace(/^#\s+/, "").trim() || "MCP Scraper Report";
|
|
745
874
|
}
|
|
746
875
|
function outputBaseDir() {
|
|
747
|
-
return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() ||
|
|
876
|
+
return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join2(homedir2(), "Downloads", "mcp-scraper");
|
|
748
877
|
}
|
|
749
878
|
function saveFullReport(full) {
|
|
750
879
|
if (!reportSavingEnabled || process.env.MCP_SCRAPER_SAVE_REPORTS === "false") return null;
|
|
751
880
|
const outDir = outputBaseDir();
|
|
752
881
|
try {
|
|
753
|
-
|
|
882
|
+
mkdirSync2(outDir, { recursive: true });
|
|
754
883
|
const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
755
|
-
const file =
|
|
756
|
-
|
|
884
|
+
const file = join2(outDir, `${stamp}-${slugifyReportName(reportTitle(full))}.md`);
|
|
885
|
+
writeFileSync2(file, full, "utf8");
|
|
757
886
|
return file;
|
|
758
887
|
} catch {
|
|
759
888
|
return null;
|
|
@@ -768,9 +897,9 @@ function saveBulkSite(siteUrl, pages, seo, imageAudit) {
|
|
|
768
897
|
if (!reportSavingActive()) return null;
|
|
769
898
|
try {
|
|
770
899
|
const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
771
|
-
const dir =
|
|
772
|
-
const pagesDir =
|
|
773
|
-
|
|
900
|
+
const dir = join2(outputBaseDir(), `extract-${slugifyReportName(siteUrl)}-${stamp}`);
|
|
901
|
+
const pagesDir = join2(dir, "pages");
|
|
902
|
+
mkdirSync2(pagesDir, { recursive: true });
|
|
774
903
|
const indexRows = pages.map((p, i) => {
|
|
775
904
|
const num = String(i + 1).padStart(4, "0");
|
|
776
905
|
const slug = slugifyReportName(p.url.replace(/^https?:\/\//, "")).slice(0, 60) || "page";
|
|
@@ -784,7 +913,7 @@ function saveBulkSite(siteUrl, pages, seo, imageAudit) {
|
|
|
784
913
|
"",
|
|
785
914
|
body || "_(no content extracted)_"
|
|
786
915
|
].filter(Boolean).join("\n");
|
|
787
|
-
|
|
916
|
+
writeFileSync2(join2(pagesDir, fname), content, "utf8");
|
|
788
917
|
return `| ${i + 1} | ${cell(p.title ?? "Untitled")} | ${p.url} | pages/${fname} |`;
|
|
789
918
|
});
|
|
790
919
|
const dataFilesSection = seo ? [
|
|
@@ -816,40 +945,40 @@ function saveBulkSite(siteUrl, pages, seo, imageAudit) {
|
|
|
816
945
|
|---|-------|-----|------|
|
|
817
946
|
${indexRows.join("\n")}`
|
|
818
947
|
].filter(Boolean).join("\n");
|
|
819
|
-
const indexFile =
|
|
820
|
-
|
|
948
|
+
const indexFile = join2(dir, "index.md");
|
|
949
|
+
writeFileSync2(indexFile, index, "utf8");
|
|
821
950
|
let seoFiles;
|
|
822
951
|
if (seo) {
|
|
823
952
|
const toJsonl = (rows) => rows.map((r) => JSON.stringify(r)).join("\n");
|
|
824
|
-
const pagesJsonl =
|
|
825
|
-
const linksJsonl =
|
|
826
|
-
const metricsJsonl =
|
|
827
|
-
const issuesFile =
|
|
828
|
-
const reportFile =
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
953
|
+
const pagesJsonl = join2(dir, "pages.jsonl");
|
|
954
|
+
const linksJsonl = join2(dir, "links.jsonl");
|
|
955
|
+
const metricsJsonl = join2(dir, "link-metrics.jsonl");
|
|
956
|
+
const issuesFile = join2(dir, "issues.json");
|
|
957
|
+
const reportFile = join2(dir, "report.md");
|
|
958
|
+
writeFileSync2(pagesJsonl, toJsonl(seo.pageRows), "utf8");
|
|
959
|
+
writeFileSync2(linksJsonl, toJsonl(seo.edges), "utf8");
|
|
960
|
+
writeFileSync2(metricsJsonl, toJsonl(seo.metrics), "utf8");
|
|
961
|
+
writeFileSync2(issuesFile, JSON.stringify(seo.issues, null, 2), "utf8");
|
|
962
|
+
writeFileSync2(reportFile, seo.reportMd + (imageAudit ? `
|
|
834
963
|
|
|
835
964
|
${renderImageSection(imageAudit)}` : ""), "utf8");
|
|
836
|
-
const linkReportFile =
|
|
837
|
-
const linksSummaryFile =
|
|
838
|
-
const externalDomainsFile =
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
965
|
+
const linkReportFile = join2(dir, "link-report.md");
|
|
966
|
+
const linksSummaryFile = join2(dir, "links-summary.json");
|
|
967
|
+
const externalDomainsFile = join2(dir, "external-domains.json");
|
|
968
|
+
writeFileSync2(linkReportFile, renderLinkReport(seo.linkReport), "utf8");
|
|
969
|
+
writeFileSync2(linksSummaryFile, JSON.stringify(seo.linkReport.summary, null, 2), "utf8");
|
|
970
|
+
writeFileSync2(externalDomainsFile, JSON.stringify(seo.linkReport.externalDomains, null, 2), "utf8");
|
|
842
971
|
seoFiles = [pagesJsonl, linksJsonl, metricsJsonl, issuesFile, reportFile, linkReportFile, linksSummaryFile, externalDomainsFile];
|
|
843
972
|
if (imageAudit) {
|
|
844
|
-
const imagesJsonl =
|
|
845
|
-
const imagesSummary =
|
|
846
|
-
|
|
847
|
-
|
|
973
|
+
const imagesJsonl = join2(dir, "images.jsonl");
|
|
974
|
+
const imagesSummary = join2(dir, "images-summary.json");
|
|
975
|
+
writeFileSync2(imagesJsonl, imageAudit.rows.map((r) => JSON.stringify(r)).join("\n"), "utf8");
|
|
976
|
+
writeFileSync2(imagesSummary, JSON.stringify(imageAudit.summary, null, 2), "utf8");
|
|
848
977
|
seoFiles.push(imagesJsonl, imagesSummary);
|
|
849
978
|
}
|
|
850
979
|
if (seo.branding) {
|
|
851
|
-
const brandingFile =
|
|
852
|
-
|
|
980
|
+
const brandingFile = join2(dir, "branding.json");
|
|
981
|
+
writeFileSync2(brandingFile, JSON.stringify(seo.branding, null, 2), "utf8");
|
|
853
982
|
seoFiles.push(brandingFile);
|
|
854
983
|
}
|
|
855
984
|
}
|
|
@@ -862,12 +991,12 @@ function saveUrlInventory(siteUrl, urls) {
|
|
|
862
991
|
if (!reportSavingActive()) return null;
|
|
863
992
|
try {
|
|
864
993
|
const outDir = outputBaseDir();
|
|
865
|
-
|
|
994
|
+
mkdirSync2(outDir, { recursive: true });
|
|
866
995
|
const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
867
|
-
const file =
|
|
996
|
+
const file = join2(outDir, `${stamp}-urlmap-${slugifyReportName(siteUrl.replace(/^https?:\/\//, ""))}.csv`);
|
|
868
997
|
const csv = (v) => /[",\n]/.test(v) ? `"${v.replace(/"/g, '""')}"` : v;
|
|
869
998
|
const rows = ["url,status", ...urls.map((u) => `${csv(u.url)},${u.status ?? ""}`)];
|
|
870
|
-
|
|
999
|
+
writeFileSync2(file, rows.join("\n"), "utf8");
|
|
871
1000
|
return file;
|
|
872
1001
|
} catch {
|
|
873
1002
|
return null;
|
|
@@ -876,12 +1005,12 @@ function saveUrlInventory(siteUrl, urls) {
|
|
|
876
1005
|
function persistScreenshotLocally(base64, url) {
|
|
877
1006
|
if (!reportSavingEnabled || process.env.MCP_SCRAPER_SAVE_REPORTS === "false") return null;
|
|
878
1007
|
try {
|
|
879
|
-
const dir =
|
|
880
|
-
|
|
1008
|
+
const dir = join2(outputBaseDir(), "screenshots");
|
|
1009
|
+
mkdirSync2(dir, { recursive: true });
|
|
881
1010
|
const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
882
1011
|
const slug = url.replace(/^https?:\/\//, "").replace(/[^a-z0-9]+/gi, "-").replace(/^-+|-+$/g, "").slice(0, 60);
|
|
883
|
-
const filePath =
|
|
884
|
-
|
|
1012
|
+
const filePath = join2(dir, `${stamp}-${slug}.png`);
|
|
1013
|
+
writeFileSync2(filePath, Buffer.from(base64, "base64"));
|
|
885
1014
|
return filePath;
|
|
886
1015
|
} catch {
|
|
887
1016
|
return null;
|
|
@@ -894,6 +1023,20 @@ function oneBlock(content, diskContent) {
|
|
|
894
1023
|
\u{1F4C4} Saved: \`${filePath}\`` : content;
|
|
895
1024
|
return { content: [{ type: "text", text }] };
|
|
896
1025
|
}
|
|
1026
|
+
async function maybeOffload(toolName, ctx, fullText, summaryText, structuredContent) {
|
|
1027
|
+
if (!ctx?.hosted || !ARTIFACT_OFFLOAD_ENABLED) return null;
|
|
1028
|
+
const bytes = Buffer.byteLength(fullText);
|
|
1029
|
+
if (bytes <= INLINE_BUDGET_BYTES) return null;
|
|
1030
|
+
const offloaded = await offloadReport(toolName, ctx.ownerId, fullText);
|
|
1031
|
+
return {
|
|
1032
|
+
content: [{ type: "text", text: summaryEnvelope(summaryText, offloaded) }],
|
|
1033
|
+
structuredContent: { ...structuredContent, artifact: offloaded }
|
|
1034
|
+
};
|
|
1035
|
+
}
|
|
1036
|
+
function capArray(items, cap) {
|
|
1037
|
+
if (items.length <= cap) return { items };
|
|
1038
|
+
return { items: items.slice(0, cap), truncatedCount: items.length - cap };
|
|
1039
|
+
}
|
|
897
1040
|
function workflowRecipeTable(recipes) {
|
|
898
1041
|
if (!recipes.length) return "";
|
|
899
1042
|
return [
|
|
@@ -1245,7 +1388,7 @@ ${headingSection}${kpoSection}${brandingSection}${bodySectionFull}${memSection}$
|
|
|
1245
1388
|
}
|
|
1246
1389
|
return { ...textResult, structuredContent };
|
|
1247
1390
|
}
|
|
1248
|
-
function formatMapSiteUrls(raw, input) {
|
|
1391
|
+
async function formatMapSiteUrls(raw, input, ctx) {
|
|
1249
1392
|
const parsed = parseData(raw);
|
|
1250
1393
|
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
1251
1394
|
const d = parsed.data;
|
|
@@ -1286,20 +1429,28 @@ ${broken.map((u) => `- ${u.url} (${u.status})`).join("\n")}` : "",
|
|
|
1286
1429
|
- Extract content from all pages: use \`extract_site\`
|
|
1287
1430
|
- Scrape a single page: use \`extract_url\``
|
|
1288
1431
|
].filter(Boolean).join("\n");
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
urls: urls.map((u) => ({ url: u.url, status: u.status ?? null })),
|
|
1300
|
-
durationMs: d.durationMs ?? 0
|
|
1301
|
-
}
|
|
1432
|
+
const structuredContent = {
|
|
1433
|
+
startUrl: d.startUrl ?? input.url,
|
|
1434
|
+
totalFound: d.totalFound ?? urls.length,
|
|
1435
|
+
truncated: d.truncated === true,
|
|
1436
|
+
okCount: ok.length,
|
|
1437
|
+
redirectCount: redirects.length,
|
|
1438
|
+
brokenCount: broken.length,
|
|
1439
|
+
inventoryFile: inventoryFile ?? null,
|
|
1440
|
+
urls: urls.map((u) => ({ url: u.url, status: u.status ?? null })),
|
|
1441
|
+
durationMs: d.durationMs ?? 0
|
|
1302
1442
|
};
|
|
1443
|
+
const summary = `# URL Map: ${input.url}
|
|
1444
|
+
**${d.totalFound} URLs** \xB7 ${(d.durationMs / 1e3).toFixed(1)}s
|
|
1445
|
+
|
|
1446
|
+
## Summary
|
|
1447
|
+
- 2xx: ${ok.length}
|
|
1448
|
+
- 3xx: ${redirects.length}
|
|
1449
|
+
- 4xx+: ${broken.length}`;
|
|
1450
|
+
const capped = capArray(structuredContent.urls, STRUCTURED_ARRAY_CAP);
|
|
1451
|
+
const offloaded = await maybeOffload("map_site_urls", ctx, full, summary, { ...structuredContent, urls: capped.items, truncatedCount: capped.truncatedCount });
|
|
1452
|
+
if (offloaded) return offloaded;
|
|
1453
|
+
return { ...oneBlock(full), structuredContent };
|
|
1303
1454
|
}
|
|
1304
1455
|
function buildSeoExport(siteUrl, pages, branding) {
|
|
1305
1456
|
const { edges, metrics } = buildLinkGraph(pages, siteUrl);
|
|
@@ -1319,7 +1470,7 @@ function buildSeoExport(siteUrl, pages, branding) {
|
|
|
1319
1470
|
branding: branding ?? void 0
|
|
1320
1471
|
};
|
|
1321
1472
|
}
|
|
1322
|
-
function formatExtractSite(raw, input) {
|
|
1473
|
+
async function formatExtractSite(raw, input, ctx) {
|
|
1323
1474
|
const parsed = parseData(raw);
|
|
1324
1475
|
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
1325
1476
|
const d = parsed.data;
|
|
@@ -1354,6 +1505,36 @@ function formatExtractSite(raw, input) {
|
|
|
1354
1505
|
schemaTypes: schemaTypesOf(p)
|
|
1355
1506
|
})));
|
|
1356
1507
|
const preview = pages.slice(0, BULK_PAGE_THRESHOLD).map(pageRow).join("\n");
|
|
1508
|
+
const summaryHeader = [
|
|
1509
|
+
`# Site Extract: ${input.url}`,
|
|
1510
|
+
durationLine,
|
|
1511
|
+
`
|
|
1512
|
+
## Pages (first ${Math.min(BULK_PAGE_THRESHOLD, pages.length)} of ${pages.length})
|
|
1513
|
+
| # | Title | URL | Schema |
|
|
1514
|
+
|---|-------|-----|--------|
|
|
1515
|
+
${preview}`
|
|
1516
|
+
].join("\n");
|
|
1517
|
+
if (!bulk && ctx?.hosted) {
|
|
1518
|
+
const pageDetails2 = pages.map((p, i) => {
|
|
1519
|
+
const body = (p.bodyMarkdown ?? "").trim();
|
|
1520
|
+
return [
|
|
1521
|
+
`
|
|
1522
|
+
## ${i + 1}. ${p.title ?? "Untitled"}`,
|
|
1523
|
+
`- **URL:** ${p.url}`,
|
|
1524
|
+
p.metaDescription ? `- **Description:** ${p.metaDescription}` : "",
|
|
1525
|
+
body ? `
|
|
1526
|
+
${body}` : "_(no content extracted)_"
|
|
1527
|
+
].filter(Boolean).join("\n");
|
|
1528
|
+
}).join("\n");
|
|
1529
|
+
const fullForOffload = `${summaryHeader}
|
|
1530
|
+
|
|
1531
|
+
---
|
|
1532
|
+
# Full Page Content
|
|
1533
|
+
${pageDetails2}${tips}`;
|
|
1534
|
+
const capped = capArray(structuredContent.pages, STRUCTURED_ARRAY_CAP);
|
|
1535
|
+
const offloaded = await maybeOffload("extract_site", ctx, fullForOffload, `${summaryHeader}${tips}`, { ...structuredContent, pages: capped.items, truncatedCount: capped.truncatedCount });
|
|
1536
|
+
if (offloaded) return offloaded;
|
|
1537
|
+
}
|
|
1357
1538
|
const location = bulk ? `
|
|
1358
1539
|
## \u{1F4C1} Bulk scrape saved
|
|
1359
1540
|
- **Folder:** \`${bulk.dir}\` \u2190 all scraped page content is here
|
|
@@ -1406,7 +1587,7 @@ ${body}` : "_(no content extracted)_"
|
|
|
1406
1587
|
${pageDetails}${tips}`;
|
|
1407
1588
|
return { ...oneBlock(full, diskReport), structuredContent };
|
|
1408
1589
|
}
|
|
1409
|
-
async function formatAuditSite(raw, input) {
|
|
1590
|
+
async function formatAuditSite(raw, input, ctx) {
|
|
1410
1591
|
const parsed = parseData(raw);
|
|
1411
1592
|
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
1412
1593
|
const d = parsed.data;
|
|
@@ -1437,6 +1618,24 @@ async function formatAuditSite(raw, input) {
|
|
|
1437
1618
|
images: { unique: img.unique, totalBytes: img.totalBytes, over100kb: img.over100kb, legacyFormat: img.legacyFormat },
|
|
1438
1619
|
links: { internal: lr.internal.totalLinks, external: lr.external.totalLinks, orphans: lr.internal.orphans, brokenInternal: lr.internal.brokenInternal, externalDomains: lr.external.uniqueDomains }
|
|
1439
1620
|
};
|
|
1621
|
+
const summary = [
|
|
1622
|
+
`# Technical SEO Audit: ${input.url}`,
|
|
1623
|
+
durationLine,
|
|
1624
|
+
`
|
|
1625
|
+
## Top issues
|
|
1626
|
+
${topIssues || "_none found_"}`,
|
|
1627
|
+
`
|
|
1628
|
+
## Links
|
|
1629
|
+
${linkLine}`,
|
|
1630
|
+
`
|
|
1631
|
+
## Images
|
|
1632
|
+
${imgLine}`
|
|
1633
|
+
].join("\n");
|
|
1634
|
+
if (!bulk && ctx?.hosted) {
|
|
1635
|
+
const fullForOffload = [summary, "\n---\n# Full Audit Report", seo.reportMd, renderImageSection(imageAudit), renderLinkReport(seo.linkReport)].join("\n");
|
|
1636
|
+
const offloaded = await maybeOffload("audit_site", ctx, fullForOffload, summary, structuredContent);
|
|
1637
|
+
if (offloaded) return offloaded;
|
|
1638
|
+
}
|
|
1440
1639
|
const location = bulk ? `
|
|
1441
1640
|
## \u{1F4C1} Technical audit saved
|
|
1442
1641
|
- **Folder:** \`${bulk.dir}\` \u2190 full audit + page content here
|
|
@@ -2207,7 +2406,7 @@ ${rows}`,
|
|
|
2207
2406
|
}
|
|
2208
2407
|
};
|
|
2209
2408
|
}
|
|
2210
|
-
function formatDirectoryWorkflow(raw, input) {
|
|
2409
|
+
async function formatDirectoryWorkflow(raw, input, ctx) {
|
|
2211
2410
|
const parsed = parseData(raw);
|
|
2212
2411
|
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
2213
2412
|
const d = parsed.data;
|
|
@@ -2259,26 +2458,29 @@ ${businessRows}` : null,
|
|
|
2259
2458
|
durationMs != null ? `
|
|
2260
2459
|
*Completed in ${(durationMs / 1e3).toFixed(1)}s*` : null
|
|
2261
2460
|
].filter(Boolean).join("\n");
|
|
2262
|
-
|
|
2263
|
-
|
|
2264
|
-
|
|
2265
|
-
|
|
2266
|
-
|
|
2267
|
-
|
|
2268
|
-
|
|
2269
|
-
|
|
2270
|
-
|
|
2271
|
-
|
|
2272
|
-
|
|
2273
|
-
|
|
2274
|
-
|
|
2275
|
-
|
|
2276
|
-
|
|
2277
|
-
|
|
2278
|
-
cities,
|
|
2279
|
-
durationMs: durationMs ?? 0
|
|
2280
|
-
}
|
|
2461
|
+
const structuredContent = {
|
|
2462
|
+
query: d.query,
|
|
2463
|
+
state: d.state,
|
|
2464
|
+
minPopulation: d.minPopulation,
|
|
2465
|
+
populationYear: d.populationYear,
|
|
2466
|
+
maxResultsPerCity: d.maxResultsPerCity,
|
|
2467
|
+
concurrency: d.concurrency,
|
|
2468
|
+
censusSourceUrl: d.censusSourceUrl,
|
|
2469
|
+
usZipsSourcePath: d.usZipsSourcePath ?? null,
|
|
2470
|
+
warnings,
|
|
2471
|
+
extractedAt: d.extractedAt,
|
|
2472
|
+
selectedCityCount: d.selectedCityCount,
|
|
2473
|
+
totalResultCount,
|
|
2474
|
+
csvPath,
|
|
2475
|
+
cities,
|
|
2476
|
+
durationMs: durationMs ?? 0
|
|
2281
2477
|
};
|
|
2478
|
+
const summary = `# Directory Workflow: ${input.query}
|
|
2479
|
+
**Markets:** ${cities.length} \xB7 **Maps results:** ${totalResultCount} \xB7 **State:** ${d.state ?? input.state ?? "US"}`;
|
|
2480
|
+
const capped = capArray(cities, STRUCTURED_ARRAY_CAP);
|
|
2481
|
+
const offloaded = await maybeOffload("directory_workflow", ctx, full, summary, { ...structuredContent, cities: capped.items, truncatedCount: capped.truncatedCount });
|
|
2482
|
+
if (offloaded) return offloaded;
|
|
2483
|
+
return { ...oneBlock(full), structuredContent };
|
|
2282
2484
|
}
|
|
2283
2485
|
function formatMapsPlaceIntel(raw, input) {
|
|
2284
2486
|
const parsed = parseData(raw);
|
|
@@ -2799,6 +3001,14 @@ ${rows}` : ""
|
|
|
2799
3001
|
};
|
|
2800
3002
|
}
|
|
2801
3003
|
|
|
3004
|
+
// src/mcp/output-schema-registry.ts
|
|
3005
|
+
var ADVERTISE_OUTPUT_SCHEMAS = process.env.MCP_SCRAPER_ADVERTISE_OUTPUT_SCHEMAS === "true";
|
|
3006
|
+
var OUTPUT_SCHEMAS = {};
|
|
3007
|
+
function recordOutputSchema(name, schema) {
|
|
3008
|
+
OUTPUT_SCHEMAS[name] = schema;
|
|
3009
|
+
return ADVERTISE_OUTPUT_SCHEMAS ? schema : void 0;
|
|
3010
|
+
}
|
|
3011
|
+
|
|
2802
3012
|
// src/mcp/mcp-tool-schemas.ts
|
|
2803
3013
|
import { z } from "zod";
|
|
2804
3014
|
var HarvestPaaInputSchema = {
|
|
@@ -2944,12 +3154,18 @@ var DirectoryWorkflowInputSchema = {
|
|
|
2944
3154
|
maxResultsPerCity: z.number().int().min(1).max(50).default(50).describe("Google Maps candidates to collect per city."),
|
|
2945
3155
|
concurrency: z.number().int().min(1).max(5).default(5).describe("City Maps searches to run in parallel."),
|
|
2946
3156
|
includeZipGroups: z.boolean().default(true).describe("Attach ZIP groups from a configured US ZIPS CSV when available (MCP_SCRAPER_USZIPS_CSV_PATH or usZipsCsvPath)."),
|
|
2947
|
-
usZipsCsvPath: z.string().optional().describe("Local/test-only path to a US ZIPS CSV (state_abbr, zipcode, county, city columns). Deployed APIs should use MCP_SCRAPER_USZIPS_CSV_PATH instead."),
|
|
3157
|
+
usZipsCsvPath: z.string().optional().describe("Local/test-only path to a US ZIPS CSV (state_abbr, zipcode, county, city columns). Deployed APIs should use MCP_SCRAPER_USZIPS_CSV_PATH instead. For ZIP enrichment, set MCP_SCRAPER_USZIPS_CSV_PATH on the server, or pass this in local/test mode."),
|
|
2948
3158
|
saveCsv: z.boolean().default(true).describe("Save a directory-ready CSV of results to the MCP Scraper output directory and return its path."),
|
|
2949
3159
|
proxyMode: z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE).describe("Proxy targeting per city search. Defaults to location (city/ZIP-group residential targeting); configured forces the service proxy; none is local debugging only."),
|
|
2950
3160
|
proxyZip: z.string().regex(/^\d{5}$/).optional().describe("Optional ZIP override for proxy targeting; normally omitted."),
|
|
2951
3161
|
debug: z.boolean().default(false).describe("Include sanitized browser/proxy diagnostics.")
|
|
2952
3162
|
};
|
|
3163
|
+
var ArtifactPointerOutputSchema = z.object({
|
|
3164
|
+
artifactId: z.string(),
|
|
3165
|
+
bytes: z.number().int().min(0),
|
|
3166
|
+
expiresAt: z.string(),
|
|
3167
|
+
preview: z.string()
|
|
3168
|
+
});
|
|
2953
3169
|
var RankTrackerModeSchema = z.enum(["maps", "organic", "ai_overview", "paa"]);
|
|
2954
3170
|
var RankTrackerBlueprintInputSchema = {
|
|
2955
3171
|
projectName: z.string().min(1).optional().describe("Optional name for the rank tracker project, client, or campaign."),
|
|
@@ -3063,7 +3279,9 @@ var DirectoryWorkflowOutputSchema = {
|
|
|
3063
3279
|
attempts: z.array(MapsSearchAttemptOutput),
|
|
3064
3280
|
results: z.array(DirectoryMapsBusinessOutput)
|
|
3065
3281
|
})),
|
|
3066
|
-
durationMs: z.number().int().min(0)
|
|
3282
|
+
durationMs: z.number().int().min(0),
|
|
3283
|
+
truncatedCount: z.number().int().min(0).optional(),
|
|
3284
|
+
artifact: ArtifactPointerOutputSchema.optional()
|
|
3067
3285
|
};
|
|
3068
3286
|
var RankTrackerToolPlanOutput = z.object({
|
|
3069
3287
|
tool: z.string(),
|
|
@@ -3183,7 +3401,9 @@ var ExtractSiteOutputSchema = {
|
|
|
3183
3401
|
title: NullableString,
|
|
3184
3402
|
schemaTypes: z.array(z.string())
|
|
3185
3403
|
})),
|
|
3186
|
-
durationMs: z.number().min(0)
|
|
3404
|
+
durationMs: z.number().min(0),
|
|
3405
|
+
truncatedCount: z.number().int().min(0).optional(),
|
|
3406
|
+
artifact: ArtifactPointerOutputSchema.optional()
|
|
3187
3407
|
};
|
|
3188
3408
|
var AuditSiteOutputSchema = {
|
|
3189
3409
|
url: z.string(),
|
|
@@ -3203,7 +3423,8 @@ var AuditSiteOutputSchema = {
|
|
|
3203
3423
|
orphans: z.number().int().min(0),
|
|
3204
3424
|
brokenInternal: z.number().int().min(0),
|
|
3205
3425
|
externalDomains: z.number().int().min(0)
|
|
3206
|
-
})
|
|
3426
|
+
}),
|
|
3427
|
+
artifact: ArtifactPointerOutputSchema.optional()
|
|
3207
3428
|
};
|
|
3208
3429
|
var MapsPlaceIntelOutputSchema = {
|
|
3209
3430
|
name: z.string(),
|
|
@@ -3275,7 +3496,9 @@ var MapSiteUrlsOutputSchema = {
|
|
|
3275
3496
|
url: z.string(),
|
|
3276
3497
|
status: z.number().int().nullable()
|
|
3277
3498
|
})),
|
|
3278
|
-
durationMs: z.number().min(0)
|
|
3499
|
+
durationMs: z.number().min(0),
|
|
3500
|
+
truncatedCount: z.number().int().min(0).optional(),
|
|
3501
|
+
artifact: ArtifactPointerOutputSchema.optional()
|
|
3279
3502
|
};
|
|
3280
3503
|
var YoutubeHarvestOutputSchema = {
|
|
3281
3504
|
mode: z.string(),
|
|
@@ -3707,6 +3930,17 @@ var CaptureSerpPageSnapshotsInputSchema = {
|
|
|
3707
3930
|
timeoutMs: z.number().int().min(1e3).max(6e4).default(15e3).describe("Per-page capture timeout in milliseconds; timeouts return as structured capture failures."),
|
|
3708
3931
|
debug: z.boolean().default(false).describe("Include sanitized browser/proxy diagnostics.")
|
|
3709
3932
|
};
|
|
3933
|
+
var ReportArtifactReadInputSchema = {
|
|
3934
|
+
artifactId: z.string().min(1).describe("Artifact id returned inline by a tool whose result was too large to inline. Use only a returned artifactId; do not construct one yourself."),
|
|
3935
|
+
offset: z.number().int().min(0).default(0).describe("Byte offset to start reading from. Pass the previous call's nextOffset to continue."),
|
|
3936
|
+
maxBytes: z.number().int().min(1e3).max(1e5).default(2e4).describe("Maximum bytes of artifact text to return in this window.")
|
|
3937
|
+
};
|
|
3938
|
+
var ReportArtifactReadOutputSchema = {
|
|
3939
|
+
artifactId: z.string(),
|
|
3940
|
+
text: z.string(),
|
|
3941
|
+
totalBytes: z.number().int().min(0),
|
|
3942
|
+
nextOffset: z.number().int().min(0).nullable()
|
|
3943
|
+
};
|
|
3710
3944
|
|
|
3711
3945
|
// src/mcp/rank-tracker-blueprint.ts
|
|
3712
3946
|
var DEFAULT_MODES = ["maps", "organic", "ai_overview", "paa"];
|
|
@@ -4032,6 +4266,9 @@ function buildRankTrackerBlueprint(input) {
|
|
|
4032
4266
|
}
|
|
4033
4267
|
|
|
4034
4268
|
// src/mcp/paa-mcp-server.ts
|
|
4269
|
+
function hashOwnerId(callerKey) {
|
|
4270
|
+
return createHash("sha256").update(callerKey).digest("hex").slice(0, 24);
|
|
4271
|
+
}
|
|
4035
4272
|
function liveWebToolAnnotations(title) {
|
|
4036
4273
|
return {
|
|
4037
4274
|
title,
|
|
@@ -4044,16 +4281,16 @@ function liveWebToolAnnotations(title) {
|
|
|
4044
4281
|
function registerSerpIntelligenceCaptureTools(server, executor) {
|
|
4045
4282
|
server.registerTool("capture_serp_snapshot", {
|
|
4046
4283
|
title: "SERP Intelligence Snapshot",
|
|
4047
|
-
description: "Capture a structured SERP Intelligence Google
|
|
4284
|
+
description: "Capture a structured SERP Intelligence snapshot of a Google query \u2014 the persistent evidence format used by rank-tracking and comparison pipelines. Split query from location; leave proxyMode unset.",
|
|
4048
4285
|
inputSchema: CaptureSerpSnapshotInputSchema,
|
|
4049
|
-
outputSchema: CaptureSerpSnapshotOutputSchema,
|
|
4286
|
+
outputSchema: recordOutputSchema("capture_serp_snapshot", CaptureSerpSnapshotOutputSchema),
|
|
4050
4287
|
annotations: liveWebToolAnnotations("SERP Intelligence Snapshot")
|
|
4051
4288
|
}, async (input) => formatCaptureSerpSnapshot(await executor.captureSerpSnapshot(input), input));
|
|
4052
4289
|
server.registerTool("capture_serp_page_snapshots", {
|
|
4053
4290
|
title: "SERP Intelligence Page Snapshots",
|
|
4054
|
-
description: "Capture public ranking
|
|
4291
|
+
description: "Capture public ranking pages as SERP Intelligence page snapshots \u2014 persistent page evidence linked to a captured SERP. Provide urls, or targets to preserve source metadata. Private IPs, localhost, file, and internal URLs are rejected.",
|
|
4055
4292
|
inputSchema: CaptureSerpPageSnapshotsInputSchema,
|
|
4056
|
-
outputSchema: CaptureSerpPageSnapshotsOutputSchema,
|
|
4293
|
+
outputSchema: recordOutputSchema("capture_serp_page_snapshots", CaptureSerpPageSnapshotsOutputSchema),
|
|
4057
4294
|
annotations: liveWebToolAnnotations("SERP Intelligence Page Snapshots")
|
|
4058
4295
|
}, async (input) => formatCaptureSerpPageSnapshots(await executor.captureSerpPageSnapshots(input), input));
|
|
4059
4296
|
}
|
|
@@ -4069,7 +4306,7 @@ function localPlanningToolAnnotations(title) {
|
|
|
4069
4306
|
function listSavedReports() {
|
|
4070
4307
|
try {
|
|
4071
4308
|
const dir = outputBaseDir();
|
|
4072
|
-
return readdirSync(dir).filter((f) => f.endsWith(".md")).map((f) => ({ filename: f, mtimeMs: statSync(
|
|
4309
|
+
return readdirSync(dir).filter((f) => f.endsWith(".md")).map((f) => ({ filename: f, mtimeMs: statSync(join3(dir, f)).mtimeMs })).sort((a, b) => b.mtimeMs - a.mtimeMs).slice(0, 100);
|
|
4073
4310
|
} catch {
|
|
4074
4311
|
return [];
|
|
4075
4312
|
}
|
|
@@ -4095,236 +4332,257 @@ function registerSavedReportResources(server) {
|
|
|
4095
4332
|
const requested = Array.isArray(variables.filename) ? variables.filename[0] : variables.filename;
|
|
4096
4333
|
const filename = basename(decodeURIComponent(String(requested ?? "")));
|
|
4097
4334
|
if (!filename.endsWith(".md")) throw new Error("Only saved .md reports can be read");
|
|
4098
|
-
const text = readFileSync(
|
|
4335
|
+
const text = readFileSync(join3(outputBaseDir(), filename), "utf8");
|
|
4099
4336
|
return { contents: [{ uri: uri.href, mimeType: "text/markdown", text }] };
|
|
4100
4337
|
}
|
|
4101
4338
|
);
|
|
4102
4339
|
}
|
|
4103
4340
|
function buildPaaExtractorMcpServer(executor, options = {}) {
|
|
4104
|
-
const server = new McpServer({ name: "mcp-scraper", version: PACKAGE_VERSION }, { instructions:
|
|
4341
|
+
const server = new McpServer({ name: "mcp-scraper", version: PACKAGE_VERSION }, { instructions: serverInstructions(options.savesReportsLocally !== false) });
|
|
4105
4342
|
registerPaaExtractorMcpTools(server, executor, options);
|
|
4106
4343
|
return server;
|
|
4107
4344
|
}
|
|
4108
4345
|
function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
4109
4346
|
const savesReports = options.savesReportsLocally !== false;
|
|
4110
|
-
const
|
|
4111
|
-
const
|
|
4347
|
+
const fileBehavior = (local, hosted) => savesReports ? local : hosted;
|
|
4348
|
+
const ownerId = options.ownerId ?? "local";
|
|
4349
|
+
const ctx = { hosted: !savesReports, ownerId };
|
|
4112
4350
|
if (savesReports) registerSavedReportResources(server);
|
|
4113
4351
|
server.registerTool("harvest_paa", {
|
|
4114
4352
|
title: "Google PAA + SERP Harvest",
|
|
4115
|
-
description:
|
|
4353
|
+
description: "Best default tool for Google search research: People Also Ask questions with answers/sources, organic SERP, local pack, entity IDs, and AI Overview. Split topic from location; leave proxyMode unset. Warn the user before maxQuestions above 100 \u2014 deep harvests can run several minutes with no interim progress, billed per extracted question.",
|
|
4116
4354
|
inputSchema: HarvestPaaInputSchema,
|
|
4117
|
-
outputSchema: HarvestPaaOutputSchema,
|
|
4355
|
+
outputSchema: recordOutputSchema("harvest_paa", HarvestPaaOutputSchema),
|
|
4118
4356
|
annotations: liveWebToolAnnotations("Google PAA + SERP Harvest")
|
|
4119
4357
|
}, async (input) => formatHarvestPaa(await executor.harvestPaa(input), input));
|
|
4120
4358
|
server.registerTool("search_serp", {
|
|
4121
4359
|
title: "Google SERP Lookup",
|
|
4122
|
-
description:
|
|
4360
|
+
description: "Fast Google SERP lookup without PAA expansion \u2014 rankings, organic results, local pack, positions. Split topic from location; leave proxyMode unset.",
|
|
4123
4361
|
inputSchema: SearchSerpInputSchema,
|
|
4124
|
-
outputSchema: SearchSerpOutputSchema,
|
|
4362
|
+
outputSchema: recordOutputSchema("search_serp", SearchSerpOutputSchema),
|
|
4125
4363
|
annotations: liveWebToolAnnotations("Google SERP Lookup")
|
|
4126
4364
|
}, async (input) => formatSearchSerp(await executor.searchSerp(input), input));
|
|
4127
4365
|
server.registerTool("extract_url", {
|
|
4128
4366
|
title: "Single URL Extract",
|
|
4129
|
-
description:
|
|
4367
|
+
description: "Extract structured data from one public URL: content, schema, headings, metadata, screenshots, branding, or media assets. Set depositToVault:true to save the full page into the user's MCP Memory vault server-side (not returned to chat).",
|
|
4130
4368
|
inputSchema: ExtractUrlInputSchema,
|
|
4131
|
-
outputSchema: ExtractUrlOutputSchema,
|
|
4369
|
+
outputSchema: recordOutputSchema("extract_url", ExtractUrlOutputSchema),
|
|
4132
4370
|
annotations: liveWebToolAnnotations("Single URL Extract")
|
|
4133
4371
|
}, async (input) => formatExtractUrl(await executor.extractUrl(input), input));
|
|
4134
4372
|
server.registerTool("map_site_urls", {
|
|
4135
4373
|
title: "Site URL Map",
|
|
4136
|
-
description:
|
|
4374
|
+
description: `Map/crawl a public website for a sitemap, URL inventory, or broken-link scan. Returns internal URLs with HTTP status; ${fileBehavior("maps over 500 URLs are written to a local CSV file instead of inlined.", "large results are stored as a retrievable artifact \u2014 you get an inline summary plus an artifactId for report_artifact_read.")}`,
|
|
4137
4375
|
inputSchema: MapSiteUrlsInputSchema,
|
|
4138
|
-
outputSchema: MapSiteUrlsOutputSchema,
|
|
4376
|
+
outputSchema: recordOutputSchema("map_site_urls", MapSiteUrlsOutputSchema),
|
|
4139
4377
|
annotations: liveWebToolAnnotations("Site URL Map")
|
|
4140
|
-
}, async (input) => formatMapSiteUrls(await executor.mapSiteUrls(input), input));
|
|
4378
|
+
}, async (input) => formatMapSiteUrls(await executor.mapSiteUrls(input), input, ctx));
|
|
4141
4379
|
server.registerTool("extract_site", {
|
|
4142
4380
|
title: "Multi-Page Site Content Crawl",
|
|
4143
|
-
description:
|
|
4381
|
+
description: `Crawl a public website and return page CONTENT (Markdown) across multiple pages. ${fileBehavior("Bulk crawls over 25 pages are saved as per-page Markdown files in a local folder instead of inlined.", "Large results are stored as a retrievable artifact \u2014 you get an inline summary plus an artifactId for report_artifact_read.")} Content only \u2014 for a technical SEO audit use audit_site instead.`,
|
|
4144
4382
|
inputSchema: ExtractSiteInputSchema,
|
|
4145
|
-
outputSchema: ExtractSiteOutputSchema,
|
|
4383
|
+
outputSchema: recordOutputSchema("extract_site", ExtractSiteOutputSchema),
|
|
4146
4384
|
annotations: liveWebToolAnnotations("Multi-Page Site Content Crawl")
|
|
4147
|
-
}, async (input) => formatExtractSite(await executor.extractSite(input), input));
|
|
4385
|
+
}, async (input) => formatExtractSite(await executor.extractSite(input), input, ctx));
|
|
4148
4386
|
server.registerTool("audit_site", {
|
|
4149
4387
|
title: "Technical SEO Audit",
|
|
4150
|
-
description:
|
|
4388
|
+
description: `Run a full technical SEO audit (Screaming-Frog-style) on a public website: on-page issues, internal link graph, indexability, heading/image analysis. ${fileBehavior("Writes a folder of analysis files plus per-page content, and returns a summary plus the folder path.", "Large results are stored as a retrievable artifact \u2014 you get an inline summary plus an artifactId for report_artifact_read.")} Use extract_site instead for plain page content.`,
|
|
4151
4389
|
inputSchema: AuditSiteInputSchema,
|
|
4152
|
-
outputSchema: AuditSiteOutputSchema,
|
|
4390
|
+
outputSchema: recordOutputSchema("audit_site", AuditSiteOutputSchema),
|
|
4153
4391
|
annotations: liveWebToolAnnotations("Technical SEO Audit")
|
|
4154
|
-
}, async (input) => formatAuditSite(await executor.auditSite(input), input));
|
|
4392
|
+
}, async (input) => formatAuditSite(await executor.auditSite(input), input, ctx));
|
|
4155
4393
|
server.registerTool("youtube_harvest", {
|
|
4156
4394
|
title: "YouTube Video Harvest",
|
|
4157
|
-
description:
|
|
4395
|
+
description: 'Harvest YouTube video metadata by topic search or channel library. Use mode "search" for keyword/topic requests, mode "channel" for @handles/channel IDs/URLs. Returns titles, views, durations, and videoIds.',
|
|
4158
4396
|
inputSchema: YoutubeHarvestInputSchema,
|
|
4159
|
-
outputSchema: YoutubeHarvestOutputSchema,
|
|
4397
|
+
outputSchema: recordOutputSchema("youtube_harvest", YoutubeHarvestOutputSchema),
|
|
4160
4398
|
annotations: liveWebToolAnnotations("YouTube Video Harvest")
|
|
4161
4399
|
}, async (input) => formatYoutubeHarvest(await executor.youtubeHarvest(input), input));
|
|
4162
4400
|
server.registerTool("youtube_transcribe", {
|
|
4163
4401
|
title: "YouTube Transcription",
|
|
4164
|
-
description:
|
|
4402
|
+
description: "Fetch and transcribe captions from a YouTube video. Pass videoId from youtube_harvest, or a url the user pasted. Returns full transcript, timestamped chunks, and word count.",
|
|
4165
4403
|
inputSchema: YoutubeTranscribeInputSchema,
|
|
4166
|
-
outputSchema: YoutubeTranscribeOutputSchema,
|
|
4404
|
+
outputSchema: recordOutputSchema("youtube_transcribe", YoutubeTranscribeOutputSchema),
|
|
4167
4405
|
annotations: liveWebToolAnnotations("YouTube Transcription")
|
|
4168
4406
|
}, async (input) => formatYoutubeTranscribe(await executor.youtubeTranscribe(input), input));
|
|
4169
4407
|
server.registerTool("facebook_page_intel", {
|
|
4170
4408
|
title: "Facebook Advertiser Ad Intel",
|
|
4171
|
-
description:
|
|
4409
|
+
description: "Harvest ads from a Facebook advertiser: copy, creative angles, CTAs, and direct video URLs ready for facebook_ad_transcribe. Accepts pageId, libraryId, or a brand/advertiser name.",
|
|
4172
4410
|
inputSchema: FacebookPageIntelInputSchema,
|
|
4173
|
-
outputSchema: FacebookPageIntelOutputSchema,
|
|
4411
|
+
outputSchema: recordOutputSchema("facebook_page_intel", FacebookPageIntelOutputSchema),
|
|
4174
4412
|
annotations: liveWebToolAnnotations("Facebook Advertiser Ad Intel")
|
|
4175
4413
|
}, async (input) => formatFacebookPageIntel(await executor.facebookPageIntel(input), input));
|
|
4176
4414
|
server.registerTool("facebook_ad_search", {
|
|
4177
4415
|
title: "Facebook Ad Library Search",
|
|
4178
|
-
description:
|
|
4416
|
+
description: "Search Facebook Ad Library to find advertisers by brand, competitor, niche, or keyword. Returns advertisers with ad counts and library IDs.",
|
|
4179
4417
|
inputSchema: FacebookAdSearchInputSchema,
|
|
4180
|
-
outputSchema: FacebookAdSearchOutputSchema,
|
|
4418
|
+
outputSchema: recordOutputSchema("facebook_ad_search", FacebookAdSearchOutputSchema),
|
|
4181
4419
|
annotations: liveWebToolAnnotations("Facebook Ad Library Search")
|
|
4182
4420
|
}, async (input) => formatFacebookAdSearch(await executor.facebookAdSearch(input), input));
|
|
4183
4421
|
server.registerTool("reddit_thread", {
|
|
4184
4422
|
title: "Reddit Thread + Comments",
|
|
4185
|
-
description:
|
|
4423
|
+
description: "Capture a Reddit post and its comment tree from a reddit.com thread URL \u2014 comments, opinions, audience voice. Handles Reddit's bot protection automatically; pass maxComments to cap the list.",
|
|
4186
4424
|
inputSchema: RedditThreadInputSchema,
|
|
4187
|
-
outputSchema: RedditThreadOutputSchema,
|
|
4425
|
+
outputSchema: recordOutputSchema("reddit_thread", RedditThreadOutputSchema),
|
|
4188
4426
|
annotations: liveWebToolAnnotations("Reddit Thread + Comments")
|
|
4189
4427
|
}, async (input) => formatRedditThread(await executor.redditThread(input), input));
|
|
4190
4428
|
server.registerTool("video_frame_analysis", {
|
|
4191
4429
|
title: "Video Breakdown (frame-by-frame + transcript)",
|
|
4192
4430
|
description: "Produce a deep frame-by-frame + transcript breakdown of a video \u2014 pacing, hook, visual style, and how to replicate it. Pass a DIRECT video file URL (.mp4/.webm/.mov), not a page URL. Runs asynchronously and costs a flat $1 (refunded on failure): returns a runId immediately; poll video_frame_analysis_status until done.",
|
|
4193
4431
|
inputSchema: VideoFrameAnalysisInputSchema,
|
|
4194
|
-
outputSchema: VideoFrameAnalysisOutputSchema,
|
|
4432
|
+
outputSchema: recordOutputSchema("video_frame_analysis", VideoFrameAnalysisOutputSchema),
|
|
4195
4433
|
annotations: { title: "Video Breakdown", readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: true }
|
|
4196
4434
|
}, async (input) => executor.videoFrameAnalysis(input));
|
|
4197
4435
|
server.registerTool("video_frame_analysis_status", {
|
|
4198
4436
|
title: "Video Breakdown Status",
|
|
4199
4437
|
description: 'Check progress of a video breakdown started with video_frame_analysis, using its runId. Free to call. When status is "done" it returns the full report and vault path; stop polling on "done" or "failed".',
|
|
4200
4438
|
inputSchema: VideoFrameAnalysisStatusInputSchema,
|
|
4201
|
-
outputSchema: VideoFrameAnalysisStatusOutputSchema,
|
|
4439
|
+
outputSchema: recordOutputSchema("video_frame_analysis_status", VideoFrameAnalysisStatusOutputSchema),
|
|
4202
4440
|
annotations: { title: "Video Breakdown Status", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }
|
|
4203
4441
|
}, async (input) => executor.videoFrameAnalysisStatus(input));
|
|
4204
4442
|
server.registerTool("facebook_ad_transcribe", {
|
|
4205
4443
|
title: "Facebook Ad Transcription",
|
|
4206
4444
|
description: "Transcribe audio from a Facebook ad video CDN URL returned by facebook_page_intel. Use only that direct videoUrl value \u2014 do not pass public Facebook post/reel/share URLs (use facebook_video_transcribe for those).",
|
|
4207
4445
|
inputSchema: FacebookAdTranscribeInputSchema,
|
|
4208
|
-
outputSchema: FacebookAdTranscribeOutputSchema,
|
|
4446
|
+
outputSchema: recordOutputSchema("facebook_ad_transcribe", FacebookAdTranscribeOutputSchema),
|
|
4209
4447
|
annotations: liveWebToolAnnotations("Facebook Ad Transcription")
|
|
4210
4448
|
}, async (input) => formatFacebookAdTranscribe(await executor.facebookAdTranscribe(input), input));
|
|
4211
4449
|
server.registerTool("google_ads_search", {
|
|
4212
4450
|
title: "Google Ads Transparency Search",
|
|
4213
|
-
description:
|
|
4451
|
+
description: "Search the Google Ads Transparency Center to find advertisers by domain or brand name. Returns advertisers with advertiser ID and approximate ad count, to pass to google_ads_page_intel.",
|
|
4214
4452
|
inputSchema: GoogleAdsSearchInputSchema,
|
|
4215
|
-
outputSchema: GoogleAdsSearchOutputSchema,
|
|
4453
|
+
outputSchema: recordOutputSchema("google_ads_search", GoogleAdsSearchOutputSchema),
|
|
4216
4454
|
annotations: liveWebToolAnnotations("Google Ads Transparency Search")
|
|
4217
4455
|
}, async (input) => formatGoogleAdsSearch(await executor.googleAdsSearch(input), input));
|
|
4218
4456
|
server.registerTool("google_ads_page_intel", {
|
|
4219
4457
|
title: "Google Ads Advertiser Intel",
|
|
4220
|
-
description:
|
|
4458
|
+
description: "Harvest an advertiser's ad creatives from the Google Ads Transparency Center: format, image URLs, and \u2014 for video ads \u2014 a YouTube video ID or direct video URL. Accepts advertiserId (from google_ads_search) or a domain.",
|
|
4221
4459
|
inputSchema: GoogleAdsPageIntelInputSchema,
|
|
4222
|
-
outputSchema: GoogleAdsPageIntelOutputSchema,
|
|
4460
|
+
outputSchema: recordOutputSchema("google_ads_page_intel", GoogleAdsPageIntelOutputSchema),
|
|
4223
4461
|
annotations: liveWebToolAnnotations("Google Ads Advertiser Intel")
|
|
4224
4462
|
}, async (input) => formatGoogleAdsPageIntel(await executor.googleAdsPageIntel(input), input));
|
|
4225
4463
|
server.registerTool("google_ads_transcribe", {
|
|
4226
4464
|
title: "Google Ad Video Transcription",
|
|
4227
4465
|
description: "Transcribe audio from a Google video ad's direct videoUrl (a googlevideo.com playback URL) returned by google_ads_page_intel. For YouTube-hosted ads, use youtube_transcribe with the returned youtubeVideoId instead.",
|
|
4228
4466
|
inputSchema: GoogleAdsTranscribeInputSchema,
|
|
4229
|
-
outputSchema: GoogleAdsTranscribeOutputSchema,
|
|
4467
|
+
outputSchema: recordOutputSchema("google_ads_transcribe", GoogleAdsTranscribeOutputSchema),
|
|
4230
4468
|
annotations: liveWebToolAnnotations("Google Ad Video Transcription")
|
|
4231
4469
|
}, async (input) => formatGoogleAdsTranscribe(await executor.googleAdsTranscribe(input), input));
|
|
4232
4470
|
server.registerTool("facebook_video_transcribe", {
|
|
4233
4471
|
title: "Facebook Organic Video Transcription",
|
|
4234
|
-
description:
|
|
4472
|
+
description: "Transcribe audio from an organic Facebook reel/video/post/share URL (including fb.watch). Renders the page, selects the best public CDN MP4, and returns the transcript plus resolved video metadata.",
|
|
4235
4473
|
inputSchema: FacebookVideoTranscribeInputSchema,
|
|
4236
|
-
outputSchema: FacebookVideoTranscribeOutputSchema,
|
|
4474
|
+
outputSchema: recordOutputSchema("facebook_video_transcribe", FacebookVideoTranscribeOutputSchema),
|
|
4237
4475
|
annotations: liveWebToolAnnotations("Facebook Organic Video Transcription")
|
|
4238
4476
|
}, async (input) => formatFacebookVideoTranscribe(await executor.facebookVideoTranscribe(input), input));
|
|
4239
4477
|
server.registerTool("instagram_profile_content", {
|
|
4240
4478
|
title: "Instagram Profile Content Discovery",
|
|
4241
|
-
description:
|
|
4479
|
+
description: "Discover Instagram profile grid content links (posts/reels/tv) for a handle or profile URL, for later selection with instagram_media_download. Returns profile stats and collected URLs.",
|
|
4242
4480
|
inputSchema: InstagramProfileContentInputSchema,
|
|
4243
|
-
outputSchema: InstagramProfileContentOutputSchema,
|
|
4481
|
+
outputSchema: recordOutputSchema("instagram_profile_content", InstagramProfileContentOutputSchema),
|
|
4244
4482
|
annotations: liveWebToolAnnotations("Instagram Profile Content Discovery")
|
|
4245
4483
|
}, async (input) => formatInstagramProfileContent(await executor.instagramProfileContent(input), input));
|
|
4246
4484
|
server.registerTool("instagram_media_download", {
|
|
4247
4485
|
title: "Instagram Post/Reel Media Download",
|
|
4248
|
-
description:
|
|
4486
|
+
description: "Extract and download media from one Instagram post, reel, or tv URL \u2014 image, caption, video/audio tracks, optional muxed MP4, or transcript. Selects the best video/audio track pair and muxes when ffmpeg is available.",
|
|
4249
4487
|
inputSchema: InstagramMediaDownloadInputSchema,
|
|
4250
|
-
outputSchema: InstagramMediaDownloadOutputSchema,
|
|
4488
|
+
outputSchema: recordOutputSchema("instagram_media_download", InstagramMediaDownloadOutputSchema),
|
|
4251
4489
|
annotations: liveWebToolAnnotations("Instagram Post/Reel Media Download")
|
|
4252
4490
|
}, async (input) => formatInstagramMediaDownload(await executor.instagramMediaDownload(input), input));
|
|
4253
4491
|
server.registerTool("maps_place_intel", {
|
|
4254
4492
|
title: "Google Maps Business Profile Details",
|
|
4255
|
-
description:
|
|
4493
|
+
description: "Extract Google Maps business intelligence for one known/named business: rating, reviews, category, address, phone, hours, entity IDs. Not for category searches or multi-business prospect lists \u2014 use maps_search for those. Split business name from location.",
|
|
4256
4494
|
inputSchema: MapsPlaceIntelInputSchema,
|
|
4257
|
-
outputSchema: MapsPlaceIntelOutputSchema,
|
|
4495
|
+
outputSchema: recordOutputSchema("maps_place_intel", MapsPlaceIntelOutputSchema),
|
|
4258
4496
|
annotations: liveWebToolAnnotations("Google Maps Business Profile Details")
|
|
4259
4497
|
}, async (input) => formatMapsPlaceIntel(await executor.mapsPlaceIntel(input), input));
|
|
4260
4498
|
server.registerTool("maps_search", {
|
|
4261
4499
|
title: "Google Maps Business Search",
|
|
4262
|
-
description:
|
|
4500
|
+
description: "Search Google Maps for multiple businesses by category, niche, or local market \u2014 leads, prospects, competitors, or beyond the 3-pack. Leave proxyMode unset. Returns up to 50 candidates (default 10) with names, place URLs, CIDs, ratings.",
|
|
4263
4501
|
inputSchema: MapsSearchInputSchema,
|
|
4264
|
-
outputSchema: MapsSearchOutputSchema,
|
|
4502
|
+
outputSchema: recordOutputSchema("maps_search", MapsSearchOutputSchema),
|
|
4265
4503
|
annotations: liveWebToolAnnotations("Google Maps Business Search")
|
|
4266
4504
|
}, async (input) => formatMapsSearch(await executor.mapsSearch(input), input));
|
|
4267
4505
|
server.registerTool("directory_workflow", {
|
|
4268
4506
|
title: "Directory Workflow: Markets + Maps",
|
|
4269
|
-
description:
|
|
4507
|
+
description: `Build directory/prospecting datasets: selects US city markets from Census population data, optionally joins configured ZIP groups, then runs Google Maps business searches per city in parallel. Use for "all cities over 100k population in a state" or market+Maps workflows. ${fileBehavior("Saves a CSV of results per city.", "Large results are stored as a retrievable artifact \u2014 you get an inline summary plus an artifactId for report_artifact_read.")}`,
|
|
4270
4508
|
inputSchema: DirectoryWorkflowInputSchema,
|
|
4271
|
-
outputSchema: DirectoryWorkflowOutputSchema,
|
|
4509
|
+
outputSchema: recordOutputSchema("directory_workflow", DirectoryWorkflowOutputSchema),
|
|
4272
4510
|
annotations: liveWebToolAnnotations("Directory Workflow: Markets + Maps")
|
|
4273
|
-
}, async (input) => formatDirectoryWorkflow(await executor.directoryWorkflow(input), input));
|
|
4511
|
+
}, async (input) => formatDirectoryWorkflow(await executor.directoryWorkflow(input), input, ctx));
|
|
4274
4512
|
server.registerTool("workflow_list", {
|
|
4275
4513
|
title: "Workflow Catalog",
|
|
4276
4514
|
description: "List MCP Scraper higher-level workflows and recipes \u2014 market analysis, ICP research, CRO audits, competitive positioning, content gap briefs, AI search visibility, and more. Returns runnable workflow ids plus tool-chain guidance.",
|
|
4277
4515
|
inputSchema: WorkflowListInputSchema,
|
|
4278
|
-
outputSchema: WorkflowListOutputSchema,
|
|
4516
|
+
outputSchema: recordOutputSchema("workflow_list", WorkflowListOutputSchema),
|
|
4279
4517
|
annotations: localPlanningToolAnnotations("Workflow Catalog")
|
|
4280
4518
|
}, async (input) => formatWorkflowList(await executor.workflowList(input), input));
|
|
4281
4519
|
server.registerTool("workflow_suggest", {
|
|
4282
4520
|
title: "Workflow Intent Router",
|
|
4283
4521
|
description: "Route a high-level business/research goal (market analysis, ICP research, CRO audit, competitor comparison, content gap brief, AI search visibility, etc) to the right MCP Scraper workflow/tool chain. Free; tells you what to run next.",
|
|
4284
4522
|
inputSchema: WorkflowSuggestInputSchema,
|
|
4285
|
-
outputSchema: WorkflowSuggestOutputSchema,
|
|
4523
|
+
outputSchema: recordOutputSchema("workflow_suggest", WorkflowSuggestOutputSchema),
|
|
4286
4524
|
annotations: localPlanningToolAnnotations("Workflow Intent Router")
|
|
4287
4525
|
}, async (input) => formatWorkflowSuggest(input));
|
|
4288
4526
|
server.registerTool("workflow_run", {
|
|
4289
4527
|
title: "Run Workflow",
|
|
4290
|
-
description:
|
|
4528
|
+
description: "Start a higher-level MCP Scraper workflow (directory, agent-packet, local-competitive-audit, map-comparison, serp-comparison, paa-expansion-brief, ai-overview-language). Use after workflow_suggest or workflow_list. Stepwise workflows return runId + nextStep \u2014 call workflow_step with the runId until done is true.",
|
|
4291
4529
|
inputSchema: WorkflowRunInputSchema,
|
|
4292
|
-
outputSchema: WorkflowRunOutputSchema,
|
|
4530
|
+
outputSchema: recordOutputSchema("workflow_run", WorkflowRunOutputSchema),
|
|
4293
4531
|
annotations: liveWebToolAnnotations("Run Workflow")
|
|
4294
4532
|
}, async (input) => formatWorkflowRun(await executor.workflowRun(input), input));
|
|
4295
4533
|
server.registerTool("workflow_step", {
|
|
4296
4534
|
title: "Advance Workflow Step",
|
|
4297
|
-
description:
|
|
4535
|
+
description: "Run the next leg of a stepwise workflow started with workflow_run. Pass the runId; each call executes one step and returns its output plus nextStep. Keep calling with the same runId until done is true.",
|
|
4298
4536
|
inputSchema: WorkflowStepInputSchema,
|
|
4299
|
-
outputSchema: WorkflowStepOutputSchema,
|
|
4537
|
+
outputSchema: recordOutputSchema("workflow_step", WorkflowStepOutputSchema),
|
|
4300
4538
|
annotations: liveWebToolAnnotations("Advance Workflow Step")
|
|
4301
4539
|
}, async (input) => formatWorkflowStep(await executor.workflowStep(input), input));
|
|
4302
4540
|
server.registerTool("workflow_status", {
|
|
4303
4541
|
title: "Workflow Status",
|
|
4304
4542
|
description: "Fetch a hosted workflow run by id and list its current status and artifacts, to re-open a run or recover artifact ids. Use only a runId returned by workflow_run/workflow_step/workflow_status.",
|
|
4305
4543
|
inputSchema: WorkflowStatusInputSchema,
|
|
4306
|
-
outputSchema: WorkflowStatusOutputSchema,
|
|
4544
|
+
outputSchema: recordOutputSchema("workflow_status", WorkflowStatusOutputSchema),
|
|
4307
4545
|
annotations: liveWebToolAnnotations("Workflow Status")
|
|
4308
4546
|
}, async (input) => formatWorkflowStatus(await executor.workflowStatus(input), input));
|
|
4309
4547
|
server.registerTool("workflow_artifact_read", {
|
|
4310
4548
|
title: "Read Workflow Artifact",
|
|
4311
4549
|
description: "Read a workflow artifact back into context by run id and artifact id, so final deliverables are grounded in generated evidence rather than memory. Use workflow_status first when artifact ids are unknown. Use maxBytes to limit large artifacts.",
|
|
4312
4550
|
inputSchema: WorkflowArtifactReadInputSchema,
|
|
4313
|
-
outputSchema: WorkflowArtifactReadOutputSchema,
|
|
4551
|
+
outputSchema: recordOutputSchema("workflow_artifact_read", WorkflowArtifactReadOutputSchema),
|
|
4314
4552
|
annotations: liveWebToolAnnotations("Read Workflow Artifact")
|
|
4315
4553
|
}, async (input) => formatWorkflowArtifactRead(await executor.workflowArtifactRead(input), input));
|
|
4554
|
+
server.registerTool("report_artifact_read", {
|
|
4555
|
+
title: "Read Report Artifact",
|
|
4556
|
+
description: "Read back a stored report artifact by artifactId (returned by any tool whose result was too large to inline). Windowed: pass offset/maxBytes and keep reading until nextOffset is null.",
|
|
4557
|
+
inputSchema: ReportArtifactReadInputSchema,
|
|
4558
|
+
outputSchema: recordOutputSchema("report_artifact_read", ReportArtifactReadOutputSchema),
|
|
4559
|
+
annotations: liveWebToolAnnotations("Read Report Artifact")
|
|
4560
|
+
}, async (input) => {
|
|
4561
|
+
const owner = artifactOwnerId(input.artifactId);
|
|
4562
|
+
if (!owner || owner !== ownerId) {
|
|
4563
|
+
return { content: [{ type: "text", text: "Artifact not found or not owned by this caller." }], isError: true };
|
|
4564
|
+
}
|
|
4565
|
+
const window = await readArtifactWindow(input.artifactId, input.offset ?? 0, input.maxBytes ?? 2e4);
|
|
4566
|
+
if (!window) {
|
|
4567
|
+
return { content: [{ type: "text", text: "Artifact not found or expired." }], isError: true };
|
|
4568
|
+
}
|
|
4569
|
+
return {
|
|
4570
|
+
content: [{ type: "text", text: window.text }],
|
|
4571
|
+
structuredContent: { artifactId: input.artifactId, text: window.text, totalBytes: window.totalBytes, nextOffset: window.nextOffset }
|
|
4572
|
+
};
|
|
4573
|
+
});
|
|
4316
4574
|
server.registerTool("rank_tracker_workflow", {
|
|
4317
4575
|
title: "Rank Tracker Blueprint Builder",
|
|
4318
4576
|
description: "Generate a build-ready database schema, cron plan, and implementation prompt for a rank tracker powered by MCP Scraper (Maps, organic, AI Overview, or PAA tracking). Local planning only \u2014 does not call the web or spend credits.",
|
|
4319
4577
|
inputSchema: RankTrackerBlueprintInputSchema,
|
|
4320
|
-
outputSchema: RankTrackerBlueprintOutputSchema,
|
|
4578
|
+
outputSchema: recordOutputSchema("rank_tracker_workflow", RankTrackerBlueprintOutputSchema),
|
|
4321
4579
|
annotations: localPlanningToolAnnotations("Rank Tracker Blueprint Builder")
|
|
4322
4580
|
}, async (input) => buildRankTrackerBlueprint(input));
|
|
4323
4581
|
server.registerTool("credits_info", {
|
|
4324
4582
|
title: "MCP Scraper Credits & Costs",
|
|
4325
4583
|
description: "Answer questions about MCP Scraper credits, usage limits, and concurrency upgrades \u2014 balance, tool costs, concurrency limits, billing URL. Does not expose payment methods or card information.",
|
|
4326
4584
|
inputSchema: CreditsInfoInputSchema,
|
|
4327
|
-
outputSchema: CreditsInfoOutputSchema,
|
|
4585
|
+
outputSchema: recordOutputSchema("credits_info", CreditsInfoOutputSchema),
|
|
4328
4586
|
annotations: {
|
|
4329
4587
|
title: "MCP Scraper Credits & Costs",
|
|
4330
4588
|
readOnlyHint: true,
|
|
@@ -4618,30 +4876,30 @@ var HttpMcpToolExecutor = class {
|
|
|
4618
4876
|
|
|
4619
4877
|
// src/mcp/browser-agent-mcp-server.ts
|
|
4620
4878
|
import { McpServer as McpServer2 } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
4621
|
-
import { mkdirSync as
|
|
4622
|
-
import { homedir as
|
|
4623
|
-
import { join as
|
|
4879
|
+
import { mkdirSync as mkdirSync4, writeFileSync as writeFileSync4 } from "fs";
|
|
4880
|
+
import { homedir as homedir4 } from "os";
|
|
4881
|
+
import { join as join6 } from "path";
|
|
4624
4882
|
|
|
4625
4883
|
// src/services/fanout/export.ts
|
|
4626
|
-
import { mkdirSync as
|
|
4627
|
-
import { homedir as
|
|
4628
|
-
import { join as
|
|
4884
|
+
import { mkdirSync as mkdirSync3, writeFileSync as writeFileSync3 } from "fs";
|
|
4885
|
+
import { homedir as homedir3 } from "os";
|
|
4886
|
+
import { join as join4 } from "path";
|
|
4629
4887
|
import Papa from "papaparse";
|
|
4630
4888
|
function outputBaseDir2() {
|
|
4631
|
-
return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() ||
|
|
4889
|
+
return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join4(homedir3(), "Downloads", "mcp-scraper");
|
|
4632
4890
|
}
|
|
4633
4891
|
function safe(value) {
|
|
4634
4892
|
return value.replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80) || "capture";
|
|
4635
4893
|
}
|
|
4636
4894
|
function writeTable(path, rows, delimiter) {
|
|
4637
|
-
|
|
4895
|
+
writeFileSync3(path, Papa.unparse(rows.length ? rows : [{}], { delimiter }));
|
|
4638
4896
|
}
|
|
4639
4897
|
function exportFanout(enriched) {
|
|
4640
4898
|
const stamp = safe(enriched.capturedAt.replace(/[:.]/g, "-"));
|
|
4641
4899
|
const outputDir = outputBaseDir2();
|
|
4642
|
-
const relativeDir =
|
|
4643
|
-
const dir =
|
|
4644
|
-
|
|
4900
|
+
const relativeDir = join4("fanout", `${stamp}-${safe(enriched.platform)}`);
|
|
4901
|
+
const dir = join4(outputDir, relativeDir);
|
|
4902
|
+
mkdirSync3(dir, { recursive: true });
|
|
4645
4903
|
const queryRows = enriched.queries.map((q, i) => ({ index: i + 1, query: q }));
|
|
4646
4904
|
const citationRows = enriched.aggregates.citationOrder.map((c) => {
|
|
4647
4905
|
const src = enriched.citedUrls.find((s) => s.url === c.url);
|
|
@@ -4654,25 +4912,25 @@ function exportFanout(enriched) {
|
|
|
4654
4912
|
const relativePaths = {
|
|
4655
4913
|
relativeTo: "MCP_SCRAPER_OUTPUT_DIR or ~/Downloads/mcp-scraper",
|
|
4656
4914
|
dir: relativeDir,
|
|
4657
|
-
json:
|
|
4658
|
-
queriesCsv:
|
|
4659
|
-
queriesTsv:
|
|
4660
|
-
citationsCsv:
|
|
4661
|
-
sourcesCsv:
|
|
4662
|
-
browsedOnlyCsv:
|
|
4663
|
-
snippetsCsv:
|
|
4664
|
-
domainsCsv:
|
|
4665
|
-
report:
|
|
4915
|
+
json: join4(relativeDir, "fanout.json"),
|
|
4916
|
+
queriesCsv: join4(relativeDir, "queries.csv"),
|
|
4917
|
+
queriesTsv: join4(relativeDir, "queries.tsv"),
|
|
4918
|
+
citationsCsv: join4(relativeDir, "citations.csv"),
|
|
4919
|
+
sourcesCsv: join4(relativeDir, "sources.csv"),
|
|
4920
|
+
browsedOnlyCsv: join4(relativeDir, "browsed-only.csv"),
|
|
4921
|
+
snippetsCsv: join4(relativeDir, "snippets.csv"),
|
|
4922
|
+
domainsCsv: join4(relativeDir, "domains.csv"),
|
|
4923
|
+
report: join4(relativeDir, "report.html")
|
|
4666
4924
|
};
|
|
4667
|
-
|
|
4668
|
-
writeTable(
|
|
4669
|
-
writeTable(
|
|
4670
|
-
writeTable(
|
|
4671
|
-
writeTable(
|
|
4672
|
-
writeTable(
|
|
4673
|
-
writeTable(
|
|
4674
|
-
writeTable(
|
|
4675
|
-
|
|
4925
|
+
writeFileSync3(join4(outputDir, relativePaths.json), JSON.stringify(enriched, null, 2));
|
|
4926
|
+
writeTable(join4(outputDir, relativePaths.queriesCsv), queryRows, ",");
|
|
4927
|
+
writeTable(join4(outputDir, relativePaths.queriesTsv), queryRows, " ");
|
|
4928
|
+
writeTable(join4(outputDir, relativePaths.citationsCsv), citationRows, ",");
|
|
4929
|
+
writeTable(join4(outputDir, relativePaths.sourcesCsv), sourceRows, ",");
|
|
4930
|
+
writeTable(join4(outputDir, relativePaths.browsedOnlyCsv), browsedOnlyRows, ",");
|
|
4931
|
+
writeTable(join4(outputDir, relativePaths.snippetsCsv), snippetRows, ",");
|
|
4932
|
+
writeTable(join4(outputDir, relativePaths.domainsCsv), domainRows, ",");
|
|
4933
|
+
writeFileSync3(join4(outputDir, relativePaths.report), renderReportHtml(enriched));
|
|
4676
4934
|
return relativePaths;
|
|
4677
4935
|
}
|
|
4678
4936
|
function esc(s) {
|
|
@@ -5099,7 +5357,7 @@ var BrowserListSessionsOutputSchema = {
|
|
|
5099
5357
|
import { execFile } from "child_process";
|
|
5100
5358
|
import { mkdtemp, rm, stat, writeFile } from "fs/promises";
|
|
5101
5359
|
import { tmpdir } from "os";
|
|
5102
|
-
import { join as
|
|
5360
|
+
import { join as join5 } from "path";
|
|
5103
5361
|
import { promisify } from "util";
|
|
5104
5362
|
var execFileAsync = promisify(execFile);
|
|
5105
5363
|
function finiteNumber(value) {
|
|
@@ -5322,8 +5580,8 @@ function ffmpegFilterPath(path) {
|
|
|
5322
5580
|
async function annotateReplayVideo(inputFilePath, outputFilePath, options) {
|
|
5323
5581
|
if (!options.annotations.length) throw new Error("annotations must include at least one item");
|
|
5324
5582
|
const size = await videoSize(inputFilePath);
|
|
5325
|
-
const tmp = await mkdtemp(
|
|
5326
|
-
const assPath =
|
|
5583
|
+
const tmp = await mkdtemp(join5(tmpdir(), "mcp-scraper-ass-"));
|
|
5584
|
+
const assPath = join5(tmp, "annotations.ass");
|
|
5327
5585
|
try {
|
|
5328
5586
|
await writeFile(assPath, buildAssSubtitle(options, size), "utf8");
|
|
5329
5587
|
await execFileAsync("ffmpeg", [
|
|
@@ -5393,7 +5651,7 @@ function actionResult(tool, sessionId, ok, data, nextRecommendedTool = "browser_
|
|
|
5393
5651
|
});
|
|
5394
5652
|
}
|
|
5395
5653
|
function outputBaseDir3() {
|
|
5396
|
-
return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() ||
|
|
5654
|
+
return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join6(homedir4(), "Downloads", "mcp-scraper");
|
|
5397
5655
|
}
|
|
5398
5656
|
function safeFilePart(value) {
|
|
5399
5657
|
return value.replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 120) || "replay";
|
|
@@ -5402,7 +5660,7 @@ function replayFilePath(sessionId, replayId, filename) {
|
|
|
5402
5660
|
const requested = filename?.trim();
|
|
5403
5661
|
const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
5404
5662
|
const name = requested ? safeFilePart(requested).replace(/\.mp4$/i, "") : `${stamp}-${safeFilePart(sessionId)}-${safeFilePart(replayId)}`;
|
|
5405
|
-
return
|
|
5663
|
+
return join6(outputBaseDir3(), "browser-replays", `${name}.mp4`);
|
|
5406
5664
|
}
|
|
5407
5665
|
function slugPart(value) {
|
|
5408
5666
|
const trimmed = value?.trim().toLowerCase();
|
|
@@ -5476,8 +5734,8 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
5476
5734
|
}
|
|
5477
5735
|
const bytes = Buffer.from(await res.arrayBuffer());
|
|
5478
5736
|
const filePath = replayFilePath(sessionId, replayId, filename);
|
|
5479
|
-
|
|
5480
|
-
|
|
5737
|
+
mkdirSync4(join6(outputBaseDir3(), "browser-replays"), { recursive: true });
|
|
5738
|
+
writeFileSync4(filePath, bytes);
|
|
5481
5739
|
return {
|
|
5482
5740
|
ok: true,
|
|
5483
5741
|
data: {
|
|
@@ -5520,7 +5778,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
5520
5778
|
title: "Save a Site Login to a Profile",
|
|
5521
5779
|
description: "Save a logged-in browser session so this server can reuse a site the user is signed into (ChatGPT, Claude, Reddit, any account-gated site). Returns a watch_url; the user signs in fresh (existing browser cookies are NOT imported) and cookies save to a named profile. ONE profile holds MANY logins \u2014 call again with the same profile and a different domain to stack another account. NOT for one-off scraping (use extract_url) or driving the browser (use browser_open). After sign-in, poll browser_profile_list until AUTHENTICATED, then browser_open with the profile.",
|
|
5522
5780
|
inputSchema: BrowserProfileConnectInputSchema,
|
|
5523
|
-
outputSchema: BrowserProfileConnectOutputSchema,
|
|
5781
|
+
outputSchema: recordOutputSchema("browser_profile_connect", BrowserProfileConnectOutputSchema),
|
|
5524
5782
|
annotations: annotations("Save a Site Login to a Profile")
|
|
5525
5783
|
},
|
|
5526
5784
|
async (input) => {
|
|
@@ -5578,7 +5836,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
5578
5836
|
title: "List Saved Logins in a Profile",
|
|
5579
5837
|
description: "List every site login saved in a profile with its auth status (NEEDS_AUTH/AUTHENTICATED), email, and note. Use to check what's connected, or to poll a just-saved login until AUTHENTICATED. Read-only, no cost. Pass profile (or email to derive it); narrow with domain or connection_id.",
|
|
5580
5838
|
inputSchema: BrowserProfileListInputSchema,
|
|
5581
|
-
outputSchema: BrowserProfileListOutputSchema,
|
|
5839
|
+
outputSchema: recordOutputSchema("browser_profile_list", BrowserProfileListOutputSchema),
|
|
5582
5840
|
annotations: annotations("List Saved Logins in a Profile", true)
|
|
5583
5841
|
},
|
|
5584
5842
|
async (input) => {
|
|
@@ -5612,7 +5870,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
5612
5870
|
title: "Open Browser Session",
|
|
5613
5871
|
description: "Open a direct no-proxy hosted browser session you can drive. Pass a saved profile name to load a session already logged into that profile's sites (set one up first with browser_profile_connect). Returns a session_id used by all other browser_* tools.",
|
|
5614
5872
|
inputSchema: BrowserOpenInputSchema,
|
|
5615
|
-
outputSchema: BrowserOpenOutputSchema,
|
|
5873
|
+
outputSchema: recordOutputSchema("browser_open", BrowserOpenOutputSchema),
|
|
5616
5874
|
annotations: annotations("Open Browser Session")
|
|
5617
5875
|
},
|
|
5618
5876
|
async (input) => {
|
|
@@ -5646,7 +5904,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
5646
5904
|
title: "See Page (Screenshot + Elements)",
|
|
5647
5905
|
description: "Capture what the browser currently shows: a screenshot plus a text snapshot of interactive elements with x,y coordinates, page url/title, and visible text. Primary way to perceive the page; click elements by their listed x,y. If a Cloudflare/CAPTCHA challenge is visible, wait and screenshot again rather than clicking it.",
|
|
5648
5906
|
inputSchema: BrowserSessionInputSchema,
|
|
5649
|
-
outputSchema: BrowserScreenshotOutputSchema,
|
|
5907
|
+
outputSchema: recordOutputSchema("browser_screenshot", BrowserScreenshotOutputSchema),
|
|
5650
5908
|
annotations: annotations("See Page", true)
|
|
5651
5909
|
},
|
|
5652
5910
|
async (input) => {
|
|
@@ -5678,7 +5936,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
5678
5936
|
title: "Read Page Text + Elements",
|
|
5679
5937
|
description: "Return the page url, title, visible text, and interactive elements (with x,y) without an image. Cheaper than browser_screenshot when you only need to read content or find a click target.",
|
|
5680
5938
|
inputSchema: BrowserSessionInputSchema,
|
|
5681
|
-
outputSchema: BrowserReadOutputSchema,
|
|
5939
|
+
outputSchema: recordOutputSchema("browser_read", BrowserReadOutputSchema),
|
|
5682
5940
|
annotations: annotations("Read Page", true)
|
|
5683
5941
|
},
|
|
5684
5942
|
async (input) => {
|
|
@@ -5702,7 +5960,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
5702
5960
|
title: "Locate DOM Targets",
|
|
5703
5961
|
description: "Locate exact visible DOM elements or text ranges and return left/top/width/height bounds in screenshot pixels. Use before drawing annotations that must circle, box, underline, or point to a real element. Prefer CSS selectors; use text when selector is unknown.",
|
|
5704
5962
|
inputSchema: BrowserLocateInputSchema,
|
|
5705
|
-
outputSchema: BrowserLocateOutputSchema,
|
|
5963
|
+
outputSchema: recordOutputSchema("browser_locate", BrowserLocateOutputSchema),
|
|
5706
5964
|
annotations: annotations("Locate DOM Targets", true)
|
|
5707
5965
|
},
|
|
5708
5966
|
async (input) => {
|
|
@@ -5727,7 +5985,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
5727
5985
|
title: "Navigate To URL",
|
|
5728
5986
|
description: "Navigate an existing browser session to a URL. Use browser_open first if no session exists; follow with browser_screenshot to see the loaded page.",
|
|
5729
5987
|
inputSchema: BrowserGotoInputSchema,
|
|
5730
|
-
outputSchema: BrowserActionOutputSchema,
|
|
5988
|
+
outputSchema: recordOutputSchema("browser_goto", BrowserActionOutputSchema),
|
|
5731
5989
|
annotations: annotations("Navigate To URL")
|
|
5732
5990
|
},
|
|
5733
5991
|
async (input) => {
|
|
@@ -5741,7 +5999,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
5741
5999
|
title: "Click",
|
|
5742
6000
|
description: "Click a visible page target using screenshot pixel coordinates. Use x/y only from the latest browser_screenshot, browser_read, or browser_locate result; do not guess coordinates.",
|
|
5743
6001
|
inputSchema: BrowserClickInputSchema,
|
|
5744
|
-
outputSchema: BrowserActionOutputSchema,
|
|
6002
|
+
outputSchema: recordOutputSchema("browser_click", BrowserActionOutputSchema),
|
|
5745
6003
|
annotations: annotations("Click")
|
|
5746
6004
|
},
|
|
5747
6005
|
async (input) => {
|
|
@@ -5760,7 +6018,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
5760
6018
|
title: "Type Text",
|
|
5761
6019
|
description: 'Type text into the currently focused browser field. Click or Tab to the field first if focus is uncertain. Use browser_press with ["Return"] to submit.',
|
|
5762
6020
|
inputSchema: BrowserTypeInputSchema,
|
|
5763
|
-
outputSchema: BrowserActionOutputSchema,
|
|
6021
|
+
outputSchema: recordOutputSchema("browser_type", BrowserActionOutputSchema),
|
|
5764
6022
|
annotations: annotations("Type Text")
|
|
5765
6023
|
},
|
|
5766
6024
|
async (input) => {
|
|
@@ -5774,7 +6032,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
5774
6032
|
title: "Scroll",
|
|
5775
6033
|
description: "Scroll the page to reveal more content. Positive delta_y scrolls down; negative scrolls up.",
|
|
5776
6034
|
inputSchema: BrowserScrollInputSchema,
|
|
5777
|
-
outputSchema: BrowserActionOutputSchema,
|
|
6035
|
+
outputSchema: recordOutputSchema("browser_scroll", BrowserActionOutputSchema),
|
|
5778
6036
|
annotations: annotations("Scroll")
|
|
5779
6037
|
},
|
|
5780
6038
|
async (input) => {
|
|
@@ -5793,7 +6051,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
5793
6051
|
title: "Press Keys",
|
|
5794
6052
|
description: "Press keyboard keys or combinations in the active browser session \u2014 submit, Escape, Tab navigation, select-all, or shortcuts. Use browser_type for text entry.",
|
|
5795
6053
|
inputSchema: BrowserPressInputSchema,
|
|
5796
|
-
outputSchema: BrowserActionOutputSchema,
|
|
6054
|
+
outputSchema: recordOutputSchema("browser_press", BrowserActionOutputSchema),
|
|
5797
6055
|
annotations: annotations("Press Keys")
|
|
5798
6056
|
},
|
|
5799
6057
|
async (input) => {
|
|
@@ -5807,7 +6065,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
5807
6065
|
title: "Start Recording",
|
|
5808
6066
|
description: "Start recording an MP4 replay of the session. Returns replay_id and a download_url. Stop with browser_replay_stop.",
|
|
5809
6067
|
inputSchema: BrowserSessionInputSchema,
|
|
5810
|
-
outputSchema: BrowserReplayStartOutputSchema,
|
|
6068
|
+
outputSchema: recordOutputSchema("browser_replay_start", BrowserReplayStartOutputSchema),
|
|
5811
6069
|
annotations: annotations("Start Recording")
|
|
5812
6070
|
},
|
|
5813
6071
|
async (input) => {
|
|
@@ -5830,7 +6088,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
5830
6088
|
title: "Stop Recording",
|
|
5831
6089
|
description: "Stop a replay recording and expose its final view_url/download_url. Use browser_replay_download to save the MP4.",
|
|
5832
6090
|
inputSchema: BrowserReplayStopInputSchema,
|
|
5833
|
-
outputSchema: BrowserReplayStopOutputSchema,
|
|
6091
|
+
outputSchema: recordOutputSchema("browser_replay_stop", BrowserReplayStopOutputSchema),
|
|
5834
6092
|
annotations: annotations("Stop Recording")
|
|
5835
6093
|
},
|
|
5836
6094
|
async (input) => {
|
|
@@ -5853,7 +6111,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
5853
6111
|
title: "List Replay Videos",
|
|
5854
6112
|
description: "List replay recordings for a browser session, including view_url and download_url when available.",
|
|
5855
6113
|
inputSchema: BrowserSessionInputSchema,
|
|
5856
|
-
outputSchema: BrowserListReplaysOutputSchema,
|
|
6114
|
+
outputSchema: recordOutputSchema("browser_list_replays", BrowserListReplaysOutputSchema),
|
|
5857
6115
|
annotations: annotations("List Replay Videos", true)
|
|
5858
6116
|
},
|
|
5859
6117
|
async (input) => {
|
|
@@ -5873,9 +6131,9 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
5873
6131
|
"browser_replay_download",
|
|
5874
6132
|
{
|
|
5875
6133
|
title: "Download Replay MP4",
|
|
5876
|
-
description: "Download a replay recording and save the MP4 under MCP_SCRAPER_OUTPUT_DIR/browser-replays. Use after browser_replay_stop or browser_list_replays.",
|
|
6134
|
+
description: opts.savesReportsLocally === false ? "Download a replay recording. Returns the download_url; fetch it directly (nothing is saved on this hosted endpoint). Use after browser_replay_stop or browser_list_replays." : "Download a replay recording and save the MP4 under MCP_SCRAPER_OUTPUT_DIR/browser-replays. Use after browser_replay_stop or browser_list_replays.",
|
|
5877
6135
|
inputSchema: BrowserReplayDownloadInputSchema,
|
|
5878
|
-
outputSchema: BrowserReplayDownloadOutputSchema,
|
|
6136
|
+
outputSchema: recordOutputSchema("browser_replay_download", BrowserReplayDownloadOutputSchema),
|
|
5879
6137
|
annotations: annotations("Download Replay MP4")
|
|
5880
6138
|
},
|
|
5881
6139
|
async (input) => {
|
|
@@ -5899,7 +6157,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
5899
6157
|
title: "Mark Replay Annotation",
|
|
5900
6158
|
description: "While a replay is actively recording, locate one exact DOM target and return a ready-to-use annotation with DOM bounds and replay-relative timing, instead of guessing start_seconds or rectangles. Pass the returned annotations to browser_replay_annotate after stopping the replay.",
|
|
5901
6159
|
inputSchema: BrowserReplayMarkInputSchema,
|
|
5902
|
-
outputSchema: BrowserReplayMarkOutputSchema,
|
|
6160
|
+
outputSchema: recordOutputSchema("browser_replay_mark", BrowserReplayMarkOutputSchema),
|
|
5903
6161
|
annotations: annotations("Mark Replay Annotation", true)
|
|
5904
6162
|
},
|
|
5905
6163
|
async (input) => {
|
|
@@ -5949,7 +6207,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
5949
6207
|
title: "Annotate Replay MP4",
|
|
5950
6208
|
description: "Download a browser replay MP4, render visual annotations (circles/boxes/arrows/labels) over it, and save a new annotated MP4. Prefer annotations from browser_replay_mark for accurate timing; otherwise use exact bounds from browser_locate. Pass source_width/source_height if the replay video size differs from the screenshot coordinate space.",
|
|
5951
6209
|
inputSchema: BrowserReplayAnnotateInputSchema,
|
|
5952
|
-
outputSchema: BrowserReplayAnnotateOutputSchema,
|
|
6210
|
+
outputSchema: recordOutputSchema("browser_replay_annotate", BrowserReplayAnnotateOutputSchema),
|
|
5953
6211
|
annotations: annotations("Annotate Replay MP4")
|
|
5954
6212
|
},
|
|
5955
6213
|
async (input) => {
|
|
@@ -5959,7 +6217,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
5959
6217
|
try {
|
|
5960
6218
|
const sourcePath = String(downloaded.data.file_path);
|
|
5961
6219
|
const outputPath = annotatedReplayFilePath(input.session_id, input.replay_id, input.filename);
|
|
5962
|
-
|
|
6220
|
+
mkdirSync4(join6(outputBaseDir3(), "browser-replays"), { recursive: true });
|
|
5963
6221
|
const result = await annotateReplayVideo(sourcePath, outputPath, {
|
|
5964
6222
|
annotations: input.annotations,
|
|
5965
6223
|
sourceWidth: input.source_width,
|
|
@@ -5991,7 +6249,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
5991
6249
|
title: "Close Browser Session",
|
|
5992
6250
|
description: "Close and release a browser session when the task is done, to end active browser billing. Use browser_list_sessions first to recover a session_id.",
|
|
5993
6251
|
inputSchema: BrowserSessionInputSchema,
|
|
5994
|
-
outputSchema: BrowserCloseOutputSchema,
|
|
6252
|
+
outputSchema: recordOutputSchema("browser_close", BrowserCloseOutputSchema),
|
|
5995
6253
|
annotations: { title: "Close Browser Session", readOnlyHint: false, destructiveHint: true, idempotentHint: true, openWorldHint: false }
|
|
5996
6254
|
},
|
|
5997
6255
|
async (input) => {
|
|
@@ -6012,7 +6270,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
6012
6270
|
title: "List Browser Sessions",
|
|
6013
6271
|
description: "List browser sessions and their status, with a watch_url for each. Use to recover a session_id or decide which session to close.",
|
|
6014
6272
|
inputSchema: BrowserListInputSchema,
|
|
6015
|
-
outputSchema: BrowserListSessionsOutputSchema,
|
|
6273
|
+
outputSchema: recordOutputSchema("browser_list_sessions", BrowserListSessionsOutputSchema),
|
|
6016
6274
|
annotations: annotations("List Browser Sessions", true)
|
|
6017
6275
|
},
|
|
6018
6276
|
async (input) => {
|
|
@@ -6034,7 +6292,7 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
6034
6292
|
title: "Capture AI Search Fan-Out",
|
|
6035
6293
|
description: "Capture the query fan-out behind a ChatGPT or Claude web-search answer for AEO: sub-queries issued, every researched URL split into cited vs browsed-only, and top sourced sites. Returns raw structured data for you to classify and analyze. Set export=true for JSON/CSV/TSV/HTML artifacts. WRITE NOTE: passing prompt submits a real message in the user's logged-in account \u2014 only send when the user wants that; omit it to capture a prompt the user just ran. The session must already be open on chatgpt.com or claude.ai (see browser_profile_connect) while the prompt streams. NOT for Google AI Overview \u2014 use harvest_paa for that.",
|
|
6036
6294
|
inputSchema: BrowserCaptureFanoutInputSchema,
|
|
6037
|
-
outputSchema: BrowserCaptureFanoutOutputSchema,
|
|
6295
|
+
outputSchema: recordOutputSchema("query_fanout_workflow", BrowserCaptureFanoutOutputSchema),
|
|
6038
6296
|
annotations: annotations("Capture AI Search Fan-Out")
|
|
6039
6297
|
},
|
|
6040
6298
|
async (input) => {
|
|
@@ -6097,9 +6355,11 @@ export {
|
|
|
6097
6355
|
renderIssueReport,
|
|
6098
6356
|
auditImages,
|
|
6099
6357
|
renderImageSection,
|
|
6358
|
+
getBlobStore,
|
|
6100
6359
|
configureReportSaving,
|
|
6101
6360
|
outputBaseDir,
|
|
6102
6361
|
SERVER_INSTRUCTIONS,
|
|
6362
|
+
hashOwnerId,
|
|
6103
6363
|
registerSerpIntelligenceCaptureTools,
|
|
6104
6364
|
buildPaaExtractorMcpServer,
|
|
6105
6365
|
registerPaaExtractorMcpTools,
|
|
@@ -6107,4 +6367,4 @@ export {
|
|
|
6107
6367
|
exportFanout,
|
|
6108
6368
|
registerBrowserAgentMcpTools
|
|
6109
6369
|
};
|
|
6110
|
-
//# sourceMappingURL=chunk-
|
|
6370
|
+
//# sourceMappingURL=chunk-SS5YBZVI.js.map
|