mcp-scraper 0.2.16 → 0.2.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -3
- package/dist/bin/api-server.cjs +479 -7
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +1 -1
- package/dist/bin/browser-agent-stdio-server.cjs +1 -1
- package/dist/bin/browser-agent-stdio-server.cjs.map +1 -1
- package/dist/bin/browser-agent-stdio-server.js +2 -2
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs +466 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.js +3 -3
- package/dist/bin/mcp-scraper-install.cjs +6 -5
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +6 -5
- package/dist/bin/mcp-scraper-install.js.map +1 -1
- package/dist/bin/mcp-stdio-server.cjs +466 -1
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +2 -2
- package/dist/{chunk-FMC3V54I.js → chunk-BYF7HTLJ.js} +469 -2
- package/dist/chunk-BYF7HTLJ.js.map +1 -0
- package/dist/chunk-NZG2K6CJ.js +7 -0
- package/dist/chunk-NZG2K6CJ.js.map +1 -0
- package/dist/{chunk-ATJAINML.js → chunk-ZA7IZZKN.js} +2 -2
- package/dist/{server-KSEQLZNP.js → server-AK3UME3Y.js} +3 -3
- package/package.json +1 -1
- package/dist/chunk-FMC3V54I.js.map +0 -1
- package/dist/chunk-KE4JZDLV.js +0 -7
- package/dist/chunk-KE4JZDLV.js.map +0 -1
- /package/dist/{chunk-ATJAINML.js.map → chunk-ZA7IZZKN.js.map} +0 -0
- /package/dist/{server-KSEQLZNP.js.map → server-AK3UME3Y.js.map} +0 -0
|
@@ -76,6 +76,57 @@ var HttpMcpToolExecutor = class {
|
|
|
76
76
|
return { content: [{ type: "text", text: msg }], isError: true };
|
|
77
77
|
}
|
|
78
78
|
}
|
|
79
|
+
async getJson(path, timeoutMs = this.timeoutMs) {
|
|
80
|
+
try {
|
|
81
|
+
const res = await fetch(`${this.baseUrl}${path}`, {
|
|
82
|
+
method: "GET",
|
|
83
|
+
headers: {
|
|
84
|
+
"x-api-key": this.apiKey
|
|
85
|
+
},
|
|
86
|
+
signal: AbortSignal.timeout(timeoutMs)
|
|
87
|
+
});
|
|
88
|
+
const data = await res.json();
|
|
89
|
+
if (!res.ok) {
|
|
90
|
+
return { content: [{ type: "text", text: JSON.stringify(data) }], isError: true };
|
|
91
|
+
}
|
|
92
|
+
return { content: [{ type: "text", text: JSON.stringify(data) }] };
|
|
93
|
+
} catch (err) {
|
|
94
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
95
|
+
return { content: [{ type: "text", text: msg }], isError: true };
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
async getTextArtifact(path, maxBytes, timeoutMs = this.timeoutMs) {
|
|
99
|
+
try {
|
|
100
|
+
const res = await fetch(`${this.baseUrl}${path}`, {
|
|
101
|
+
method: "GET",
|
|
102
|
+
headers: {
|
|
103
|
+
"x-api-key": this.apiKey
|
|
104
|
+
},
|
|
105
|
+
signal: AbortSignal.timeout(timeoutMs)
|
|
106
|
+
});
|
|
107
|
+
if (!res.ok) {
|
|
108
|
+
const data = await res.json().catch(async () => ({ error: await res.text().catch(() => `HTTP ${res.status}`) }));
|
|
109
|
+
return { content: [{ type: "text", text: JSON.stringify(data) }], isError: true };
|
|
110
|
+
}
|
|
111
|
+
const bytes = Buffer.from(await res.arrayBuffer());
|
|
112
|
+
const sliced = bytes.subarray(0, Math.min(maxBytes, bytes.length));
|
|
113
|
+
return {
|
|
114
|
+
content: [{
|
|
115
|
+
type: "text",
|
|
116
|
+
text: JSON.stringify({
|
|
117
|
+
contentType: res.headers.get("content-type") ?? "application/octet-stream",
|
|
118
|
+
bytes: bytes.length,
|
|
119
|
+
truncated: sliced.length < bytes.length,
|
|
120
|
+
maxBytes,
|
|
121
|
+
text: sliced.toString("utf8")
|
|
122
|
+
})
|
|
123
|
+
}]
|
|
124
|
+
};
|
|
125
|
+
} catch (err) {
|
|
126
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
127
|
+
return { content: [{ type: "text", text: msg }], isError: true };
|
|
128
|
+
}
|
|
129
|
+
}
|
|
79
130
|
harvestPaa(input) {
|
|
80
131
|
const timeoutMs = this.httpTimeoutOverrideMs ?? harvestTimeoutBudget(input.maxQuestions ?? 30).clientMs;
|
|
81
132
|
return this.call("/harvest/sync", input, timeoutMs);
|
|
@@ -123,6 +174,26 @@ var HttpMcpToolExecutor = class {
|
|
|
123
174
|
const timeoutMs = this.httpTimeoutOverrideMs ?? Math.min(9e5, Math.max(18e4, Math.ceil(cityCount / concurrency) * 12e4));
|
|
124
175
|
return this.call("/directory/run", input, timeoutMs);
|
|
125
176
|
}
|
|
177
|
+
workflowList(_input) {
|
|
178
|
+
return this.getJson("/workflows/definitions");
|
|
179
|
+
}
|
|
180
|
+
workflowRun(input) {
|
|
181
|
+
const timeoutMs = this.httpTimeoutOverrideMs ?? Number(process.env.MCP_SCRAPER_WORKFLOW_TIMEOUT_MS ?? 9e5);
|
|
182
|
+
return this.call("/workflows/run", {
|
|
183
|
+
workflowId: input.workflowId,
|
|
184
|
+
input: input.input ?? {},
|
|
185
|
+
webhookUrl: input.webhookUrl
|
|
186
|
+
}, Number.isFinite(timeoutMs) && timeoutMs > 0 ? timeoutMs : 9e5);
|
|
187
|
+
}
|
|
188
|
+
workflowStatus(input) {
|
|
189
|
+
return this.getJson(`/workflows/runs/${encodeURIComponent(input.runId)}`);
|
|
190
|
+
}
|
|
191
|
+
workflowArtifactRead(input) {
|
|
192
|
+
return this.getTextArtifact(
|
|
193
|
+
`/workflows/runs/${encodeURIComponent(input.runId)}/artifacts/${encodeURIComponent(input.artifactId)}`,
|
|
194
|
+
input.maxBytes ?? 2e5
|
|
195
|
+
);
|
|
196
|
+
}
|
|
126
197
|
creditsInfo(input) {
|
|
127
198
|
return this.call("/billing/credits", input);
|
|
128
199
|
}
|
|
@@ -141,7 +212,7 @@ var import_node_os2 = require("os");
|
|
|
141
212
|
var import_node_path2 = require("path");
|
|
142
213
|
|
|
143
214
|
// src/version.ts
|
|
144
|
-
var PACKAGE_VERSION = "0.2.
|
|
215
|
+
var PACKAGE_VERSION = "0.2.17";
|
|
145
216
|
|
|
146
217
|
// src/mcp/browser-agent-tool-schemas.ts
|
|
147
218
|
var import_zod = require("zod");
|
|
@@ -931,6 +1002,121 @@ function sanitizeVendorName(message) {
|
|
|
931
1002
|
return message.replace(/kernel\.sh\s+sessions?/gi, "sessions").replace(/kernel\.sh\s+session/gi, "this session").replace(/kernel\.sh/gi, "the service").replace(/kernel\s+sessions?/gi, "sessions").replace(/kernel\s+session/gi, "this session").replace(/\bkernel\b/gi, "the service").replace(/ +/g, " ").trim();
|
|
932
1003
|
}
|
|
933
1004
|
|
|
1005
|
+
// src/mcp/workflow-catalog.ts
|
|
1006
|
+
var WORKFLOW_RECIPES = [
|
|
1007
|
+
{
|
|
1008
|
+
id: "market_analysis",
|
|
1009
|
+
title: "Market Analysis",
|
|
1010
|
+
description: "Compare a niche across a city, state, or market set using Google Maps competitors, SERP evidence, review counts, categories, and opportunity signals.",
|
|
1011
|
+
primaryWorkflowId: "local-competitive-audit",
|
|
1012
|
+
recommendedTools: ["workflow_run", "directory_workflow", "maps_search", "maps_place_intel", "search_serp", "harvest_paa"],
|
|
1013
|
+
requiredInputs: ["query", "state or location"],
|
|
1014
|
+
optionalInputs: ["domain", "minPopulation", "maxCities", "maxResultsPerCity", "hydrateTop", "maxReviews"],
|
|
1015
|
+
produces: ["city summary", "competitor CSV", "review insight CSV", "market difficulty/opportunity notes", "HTML report"],
|
|
1016
|
+
runHint: "Use workflow_run with workflowId local-competitive-audit for state-wide market batches, or map-comparison for one city/location comparison."
|
|
1017
|
+
},
|
|
1018
|
+
{
|
|
1019
|
+
id: "icp_research",
|
|
1020
|
+
title: "ICP Research",
|
|
1021
|
+
description: "Build an evidence packet for ideal-customer-profile research from real search demand, PAA language, competing pages, AI Overview citations, and source domains.",
|
|
1022
|
+
primaryWorkflowId: "agent-packet",
|
|
1023
|
+
recommendedTools: ["workflow_run", "harvest_paa", "search_serp", "extract_url", "youtube_harvest", "facebook_ad_search"],
|
|
1024
|
+
requiredInputs: ["keyword or audience problem"],
|
|
1025
|
+
optionalInputs: ["domain", "location", "maxQuestions"],
|
|
1026
|
+
produces: ["evidence JSON", "sources CSV", "competitor CSV", "brief markdown", "agent task list"],
|
|
1027
|
+
runHint: "Use workflow_run with workflowId agent-packet. Treat keyword as the buyer problem, job-to-be-done, niche, or service category."
|
|
1028
|
+
},
|
|
1029
|
+
{
|
|
1030
|
+
id: "forum_review_research",
|
|
1031
|
+
title: "Forum And Review Research",
|
|
1032
|
+
description: "Acquire customer language from Google PAA, forum/Perspectives SERP surfaces, Google Maps reviews, review topics, Facebook ads, and video transcripts.",
|
|
1033
|
+
primaryWorkflowId: "local-competitive-audit",
|
|
1034
|
+
recommendedTools: ["workflow_run", "harvest_paa", "maps_search", "maps_place_intel", "facebook_page_intel", "facebook_video_transcribe", "youtube_transcribe"],
|
|
1035
|
+
requiredInputs: ["query", "state or location"],
|
|
1036
|
+
optionalInputs: ["maxReviews", "maxQuestions", "competitors", "facebookUrl", "youtubeVideoId"],
|
|
1037
|
+
produces: ["review insight CSV", "PAA/source language", "competitor review topics", "transcripts where supplied", "pain/theme summary"],
|
|
1038
|
+
runHint: "Use local-competitive-audit for review acquisition, then harvest_paa for forum/Perspectives language and transcript tools for supplied videos."
|
|
1039
|
+
},
|
|
1040
|
+
{
|
|
1041
|
+
id: "brand_design_brief",
|
|
1042
|
+
title: "Brand Design Brief",
|
|
1043
|
+
description: "Create a design briefing grounded in a brand site, competitor pages, SERP language, audience pains, visual assets, colors, fonts, screenshots, and positioning evidence.",
|
|
1044
|
+
primaryWorkflowId: "serp-comparison",
|
|
1045
|
+
recommendedTools: ["workflow_run", "extract_url", "extract_site", "capture_serp_page_snapshots", "search_serp", "harvest_paa", "browser_open"],
|
|
1046
|
+
requiredInputs: ["url or domain", "keyword or market category"],
|
|
1047
|
+
optionalInputs: ["competitorUrls", "location", "extractTop"],
|
|
1048
|
+
produces: ["page comparison CSV", "content gaps", "branding/asset evidence", "SERP positioning evidence", "designer-ready brief"],
|
|
1049
|
+
runHint: "Use workflow_run with workflowId serp-comparison for market/page evidence, then extract_url with extractBranding:true for brand assets and colors."
|
|
1050
|
+
},
|
|
1051
|
+
{
|
|
1052
|
+
id: "cro_audit",
|
|
1053
|
+
title: "CRO Audit",
|
|
1054
|
+
description: "Audit conversion clarity using page extraction, screenshots, browser DOM inspection, competitor SERP/page comparisons, forms/buttons, proof, offer, trust, and friction evidence.",
|
|
1055
|
+
primaryWorkflowId: "serp-comparison",
|
|
1056
|
+
recommendedTools: ["workflow_run", "extract_url", "extract_site", "capture_serp_page_snapshots", "browser_open", "browser_screenshot", "browser_locate"],
|
|
1057
|
+
requiredInputs: ["url"],
|
|
1058
|
+
optionalInputs: ["keyword", "domain", "location", "competitorUrls"],
|
|
1059
|
+
produces: ["page extraction", "SERP/page comparison", "screenshot evidence", "friction checklist", "CRO recommendations"],
|
|
1060
|
+
runHint: "Use workflow_run with workflowId serp-comparison when a keyword/domain is known; use extract_url and browser tools for page-level CRO evidence."
|
|
1061
|
+
},
|
|
1062
|
+
{
|
|
1063
|
+
id: "competitive_positioning_brief",
|
|
1064
|
+
title: "Competitive Positioning Brief",
|
|
1065
|
+
description: "Compare competitors across SERP, PAA, AI Overview citations, page headings, Facebook ads, YouTube angles, Maps proof, and website claims.",
|
|
1066
|
+
primaryWorkflowId: "serp-comparison",
|
|
1067
|
+
recommendedTools: ["workflow_run", "facebook_ad_search", "facebook_page_intel", "youtube_harvest", "youtube_transcribe", "maps_search", "extract_url"],
|
|
1068
|
+
requiredInputs: ["keyword or query", "domain or url"],
|
|
1069
|
+
optionalInputs: ["location", "extractTop", "competitorUrls", "brandNames"],
|
|
1070
|
+
produces: ["SERP comparison", "content gap CSV", "AI Overview citation table", "competitor message map", "positioning brief"],
|
|
1071
|
+
runHint: "Use workflow_run with workflowId serp-comparison, then enrich with Facebook, YouTube, and Maps tools when the user asks for campaign or offer positioning."
|
|
1072
|
+
},
|
|
1073
|
+
{
|
|
1074
|
+
id: "content_gap_brief",
|
|
1075
|
+
title: "Content Gap Brief",
|
|
1076
|
+
description: "Turn live SERP, page extraction, PAA questions, AI Overview citations, and source-domain evidence into a writer-ready content brief.",
|
|
1077
|
+
primaryWorkflowId: "serp-comparison",
|
|
1078
|
+
recommendedTools: ["workflow_run", "paa-expansion-brief", "harvest_paa", "extract_url"],
|
|
1079
|
+
requiredInputs: ["keyword"],
|
|
1080
|
+
optionalInputs: ["domain", "url", "location", "maxQuestions", "extractTop"],
|
|
1081
|
+
produces: ["content gaps", "page comparison CSV", "PAA questions CSV", "section map", "writer brief"],
|
|
1082
|
+
runHint: "Use workflow_run with workflowId serp-comparison for competitor gaps or paa-expansion-brief when the user mainly wants a section map from PAA data."
|
|
1083
|
+
},
|
|
1084
|
+
{
|
|
1085
|
+
id: "ai_search_visibility_audit",
|
|
1086
|
+
title: "AI Search Visibility Audit",
|
|
1087
|
+
description: "Audit whether a brand/domain appears in AI Overview citations, PAA sources, organic results, local pack, and source pages, then produce language guidance.",
|
|
1088
|
+
primaryWorkflowId: "ai-overview-language",
|
|
1089
|
+
recommendedTools: ["workflow_run", "search_serp", "harvest_paa", "extract_url", "capture_serp_snapshot"],
|
|
1090
|
+
requiredInputs: ["keyword", "domain or url"],
|
|
1091
|
+
optionalInputs: ["location", "maxQuestions", "extractTop"],
|
|
1092
|
+
produces: ["AI Overview citations CSV", "claim patterns", "language guidance", "citation hooks", "visibility gaps"],
|
|
1093
|
+
runHint: "Use workflow_run with workflowId ai-overview-language. Use serp-comparison when the user also wants ranking-page gaps."
|
|
1094
|
+
}
|
|
1095
|
+
];
|
|
1096
|
+
function normalize(value) {
|
|
1097
|
+
return value.toLowerCase().replace(/[^a-z0-9]+/g, " ").trim();
|
|
1098
|
+
}
|
|
1099
|
+
function suggestWorkflowRecipes(goal, limit = 3) {
|
|
1100
|
+
const normalized = normalize(goal);
|
|
1101
|
+
const scored = WORKFLOW_RECIPES.map((recipe) => {
|
|
1102
|
+
const haystack = normalize([
|
|
1103
|
+
recipe.id,
|
|
1104
|
+
recipe.title,
|
|
1105
|
+
recipe.description,
|
|
1106
|
+
recipe.requiredInputs.join(" "),
|
|
1107
|
+
recipe.optionalInputs.join(" "),
|
|
1108
|
+
recipe.produces.join(" ")
|
|
1109
|
+
].join(" "));
|
|
1110
|
+
let score = 0;
|
|
1111
|
+
for (const token of normalized.split(/\s+/).filter(Boolean)) {
|
|
1112
|
+
if (haystack.includes(token)) score += 1;
|
|
1113
|
+
}
|
|
1114
|
+
if (haystack.includes(normalized)) score += 5;
|
|
1115
|
+
return { recipe, score };
|
|
1116
|
+
});
|
|
1117
|
+
return scored.sort((a, b) => b.score - a.score || a.recipe.title.localeCompare(b.recipe.title)).slice(0, Math.max(1, limit)).map((item) => item.recipe);
|
|
1118
|
+
}
|
|
1119
|
+
|
|
934
1120
|
// src/mcp/mcp-response-formatter.ts
|
|
935
1121
|
var reportSavingEnabled = true;
|
|
936
1122
|
function sanitizeVendorText(text) {
|
|
@@ -982,6 +1168,14 @@ function oneBlock(content) {
|
|
|
982
1168
|
\u{1F4C4} Saved: \`${filePath}\`` : content;
|
|
983
1169
|
return { content: [{ type: "text", text }] };
|
|
984
1170
|
}
|
|
1171
|
+
function workflowRecipeTable(recipes) {
|
|
1172
|
+
if (!recipes.length) return "";
|
|
1173
|
+
return [
|
|
1174
|
+
"| Recipe | Best workflow | What it produces |",
|
|
1175
|
+
"|---|---|---|",
|
|
1176
|
+
...recipes.map((recipe) => `| ${cell(recipe.title)} | ${recipe.primaryWorkflowId ? `\`${recipe.primaryWorkflowId}\`` : "tool chain"} | ${cell(recipe.produces.slice(0, 4).join(", "))} |`)
|
|
1177
|
+
].join("\n");
|
|
1178
|
+
}
|
|
985
1179
|
function formatStructuredError(body, fallback) {
|
|
986
1180
|
if (body.error === "insufficient_balance") {
|
|
987
1181
|
return `Insufficient credits. Balance: ${body.balance_credits} credits. This call requires ${body.required_credits} credits. Top up at ${body.topup_url}`;
|
|
@@ -1554,6 +1748,163 @@ function normalizeMapsAttempts(value) {
|
|
|
1554
1748
|
observedRegion: attempt.observedRegion ?? attempt.observed_region ?? null
|
|
1555
1749
|
}));
|
|
1556
1750
|
}
|
|
1751
|
+
function workflowArtifactsFrom(run) {
|
|
1752
|
+
return Array.isArray(run?.artifacts) ? run.artifacts : [];
|
|
1753
|
+
}
|
|
1754
|
+
function workflowArtifactRows(artifacts) {
|
|
1755
|
+
if (!artifacts.length) return "";
|
|
1756
|
+
return [
|
|
1757
|
+
"| Artifact | Type | ID | Rows/Bytes |",
|
|
1758
|
+
"|---|---|---|---|",
|
|
1759
|
+
...artifacts.map((artifact) => {
|
|
1760
|
+
const rows = artifact.rows_count ?? artifact.rows ?? "";
|
|
1761
|
+
const bytes = artifact.bytes ?? "";
|
|
1762
|
+
const size = [rows ? `${rows} rows` : "", bytes ? `${bytes} bytes` : ""].filter(Boolean).join(" / ");
|
|
1763
|
+
return `| ${cell(String(artifact.label ?? artifact.path ?? "artifact"))} | ${cell(String(artifact.kind ?? artifact.content_type ?? "file"))} | \`${String(artifact.id ?? "")}\` | ${cell(size || "\u2014")} |`;
|
|
1764
|
+
})
|
|
1765
|
+
].join("\n");
|
|
1766
|
+
}
|
|
1767
|
+
function formatWorkflowList(raw, input) {
|
|
1768
|
+
const parsed = parseData(raw);
|
|
1769
|
+
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
1770
|
+
const workflows = Array.isArray(parsed.data.workflows) ? parsed.data.workflows : [];
|
|
1771
|
+
const workflowRows = [
|
|
1772
|
+
"| Workflow ID | Title | Use |",
|
|
1773
|
+
"|---|---|---|",
|
|
1774
|
+
...workflows.map((workflow) => `| \`${String(workflow.id ?? "")}\` | ${cell(String(workflow.title ?? ""))} | ${cell(String(workflow.description ?? ""))} |`)
|
|
1775
|
+
].join("\n");
|
|
1776
|
+
const recipes = input.includeRecipes === false ? [] : WORKFLOW_RECIPES;
|
|
1777
|
+
const full = [
|
|
1778
|
+
"# MCP Scraper Workflows",
|
|
1779
|
+
"Use `workflow_suggest` when the user describes a high-level job. Use `workflow_run` when the workflow id and input are known. Use `workflow_status` and `workflow_artifact_read` to inspect completed runs and pull evidence back into context.",
|
|
1780
|
+
workflows.length ? `
|
|
1781
|
+
## Runnable Workflows
|
|
1782
|
+
${workflowRows}` : "",
|
|
1783
|
+
recipes.length ? `
|
|
1784
|
+
## High-Level Recipes
|
|
1785
|
+
${workflowRecipeTable(recipes)}` : "",
|
|
1786
|
+
recipes.length ? "\nThese recipes cover market analysis, ICP research, forum and review acquisition, brand design briefings, CRO audits, competitive positioning, content gaps, and AI search visibility audits." : ""
|
|
1787
|
+
].filter(Boolean).join("\n");
|
|
1788
|
+
return {
|
|
1789
|
+
...oneBlock(full),
|
|
1790
|
+
structuredContent: {
|
|
1791
|
+
workflows: workflows.map((workflow) => ({
|
|
1792
|
+
id: String(workflow.id ?? ""),
|
|
1793
|
+
title: String(workflow.title ?? ""),
|
|
1794
|
+
description: String(workflow.description ?? "")
|
|
1795
|
+
})),
|
|
1796
|
+
recipes
|
|
1797
|
+
}
|
|
1798
|
+
};
|
|
1799
|
+
}
|
|
1800
|
+
function formatWorkflowSuggest(input) {
|
|
1801
|
+
const suggestions = suggestWorkflowRecipes(input.goal, input.maxSuggestions ?? 3);
|
|
1802
|
+
const full = [
|
|
1803
|
+
`# Workflow Suggestions`,
|
|
1804
|
+
`**Goal:** ${input.goal}`,
|
|
1805
|
+
"",
|
|
1806
|
+
workflowRecipeTable(suggestions),
|
|
1807
|
+
"",
|
|
1808
|
+
"## How To Execute",
|
|
1809
|
+
"1. Pick the closest recipe.",
|
|
1810
|
+
"2. If it has a `primaryWorkflowId`, call `workflow_run` with that id and the required inputs.",
|
|
1811
|
+
"3. After the run completes, use `workflow_artifact_read` on the most relevant CSV/JSON/Markdown artifacts before writing the final answer."
|
|
1812
|
+
].join("\n");
|
|
1813
|
+
return {
|
|
1814
|
+
...oneBlock(full),
|
|
1815
|
+
structuredContent: {
|
|
1816
|
+
goal: input.goal,
|
|
1817
|
+
suggestions
|
|
1818
|
+
}
|
|
1819
|
+
};
|
|
1820
|
+
}
|
|
1821
|
+
function formatWorkflowRun(raw, input) {
|
|
1822
|
+
const parsed = parseData(raw);
|
|
1823
|
+
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
1824
|
+
const run = parsed.data.run;
|
|
1825
|
+
const summary = parsed.data.summary;
|
|
1826
|
+
const artifacts = workflowArtifactsFrom(run);
|
|
1827
|
+
const runId = String(run?.id ?? "");
|
|
1828
|
+
const status = String(run?.status ?? summary?.status ?? "unknown");
|
|
1829
|
+
const full = [
|
|
1830
|
+
`# Workflow Run: ${input.workflowId}`,
|
|
1831
|
+
`**Run ID:** \`${runId || "unknown"}\``,
|
|
1832
|
+
`**Status:** ${status}`,
|
|
1833
|
+
summary?.title ? `**Title:** ${summary.title}` : "",
|
|
1834
|
+
summary?.summary ? `
|
|
1835
|
+
## Summary
|
|
1836
|
+
${summary.summary}` : "",
|
|
1837
|
+
artifacts.length ? `
|
|
1838
|
+
## Artifacts
|
|
1839
|
+
${workflowArtifactRows(artifacts)}` : "",
|
|
1840
|
+
artifacts.length ? "\nUse `workflow_artifact_read` with the run id and artifact id to pull CSV, JSON, Markdown, or report content into context." : ""
|
|
1841
|
+
].filter(Boolean).join("\n");
|
|
1842
|
+
return {
|
|
1843
|
+
...oneBlock(full),
|
|
1844
|
+
structuredContent: {
|
|
1845
|
+
workflowId: input.workflowId,
|
|
1846
|
+
input: input.input ?? {},
|
|
1847
|
+
run,
|
|
1848
|
+
summary,
|
|
1849
|
+
artifacts
|
|
1850
|
+
}
|
|
1851
|
+
};
|
|
1852
|
+
}
|
|
1853
|
+
function formatWorkflowStatus(raw, input) {
|
|
1854
|
+
const parsed = parseData(raw);
|
|
1855
|
+
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
1856
|
+
const run = parsed.data.run;
|
|
1857
|
+
const artifacts = workflowArtifactsFrom(run);
|
|
1858
|
+
const full = [
|
|
1859
|
+
`# Workflow Status`,
|
|
1860
|
+
`**Run ID:** \`${input.runId}\``,
|
|
1861
|
+
`**Workflow:** ${run?.workflow_id ?? "unknown"}`,
|
|
1862
|
+
`**Status:** ${run?.status ?? "unknown"}`,
|
|
1863
|
+
run?.error_message ? `
|
|
1864
|
+
## Error
|
|
1865
|
+
${run.error_message}` : "",
|
|
1866
|
+
artifacts.length ? `
|
|
1867
|
+
## Artifacts
|
|
1868
|
+
${workflowArtifactRows(artifacts)}` : ""
|
|
1869
|
+
].filter(Boolean).join("\n");
|
|
1870
|
+
return {
|
|
1871
|
+
...oneBlock(full),
|
|
1872
|
+
structuredContent: {
|
|
1873
|
+
run,
|
|
1874
|
+
artifacts
|
|
1875
|
+
}
|
|
1876
|
+
};
|
|
1877
|
+
}
|
|
1878
|
+
function formatWorkflowArtifactRead(raw, input) {
|
|
1879
|
+
const parsed = parseData(raw);
|
|
1880
|
+
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
1881
|
+
const text = typeof parsed.data.text === "string" ? parsed.data.text : "";
|
|
1882
|
+
const contentType = String(parsed.data.contentType ?? "text/plain");
|
|
1883
|
+
const bytes = Number(parsed.data.bytes ?? 0);
|
|
1884
|
+
const truncated = parsed.data.truncated === true;
|
|
1885
|
+
const full = [
|
|
1886
|
+
`# Workflow Artifact`,
|
|
1887
|
+
`**Run ID:** \`${input.runId}\``,
|
|
1888
|
+
`**Artifact ID:** \`${input.artifactId}\``,
|
|
1889
|
+
`**Content-Type:** ${contentType}`,
|
|
1890
|
+
`**Bytes:** ${bytes}${truncated ? ` (truncated to ${input.maxBytes ?? 2e5})` : ""}`,
|
|
1891
|
+
"",
|
|
1892
|
+
"```",
|
|
1893
|
+
text,
|
|
1894
|
+
"```"
|
|
1895
|
+
].join("\n");
|
|
1896
|
+
return {
|
|
1897
|
+
...oneBlock(full),
|
|
1898
|
+
structuredContent: {
|
|
1899
|
+
runId: input.runId,
|
|
1900
|
+
artifactId: input.artifactId,
|
|
1901
|
+
contentType,
|
|
1902
|
+
bytes,
|
|
1903
|
+
truncated,
|
|
1904
|
+
text
|
|
1905
|
+
}
|
|
1906
|
+
};
|
|
1907
|
+
}
|
|
1557
1908
|
function formatCreditsInfo(raw, input) {
|
|
1558
1909
|
const parsed = parseData(raw);
|
|
1559
1910
|
if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
|
|
@@ -2424,6 +2775,85 @@ var CreditsInfoInputSchema = {
|
|
|
2424
2775
|
item: import_zod2.z.string().optional().describe('Optional tool, action, or feature to look up, e.g. "maps reviews", "extract_url", "YouTube transcription", or "concurrency"'),
|
|
2425
2776
|
includeLedger: import_zod2.z.boolean().default(false).describe("Whether to include recent credit ledger entries")
|
|
2426
2777
|
};
|
|
2778
|
+
var WorkflowIdSchema = import_zod2.z.enum([
|
|
2779
|
+
"directory",
|
|
2780
|
+
"agent-packet",
|
|
2781
|
+
"local-competitive-audit",
|
|
2782
|
+
"map-comparison",
|
|
2783
|
+
"serp-comparison",
|
|
2784
|
+
"paa-expansion-brief",
|
|
2785
|
+
"ai-overview-language"
|
|
2786
|
+
]);
|
|
2787
|
+
var WorkflowListInputSchema = {
|
|
2788
|
+
includeRecipes: import_zod2.z.boolean().default(true).describe("Include high-level AI-facing recipes such as market analysis, ICP research, forum/review research, brand design briefings, CRO audits, positioning briefs, content gaps, and AI search visibility audits.")
|
|
2789
|
+
};
|
|
2790
|
+
var WorkflowSuggestInputSchema = {
|
|
2791
|
+
goal: import_zod2.z.string().min(1).describe('The user goal or job to route, e.g. "market analysis for roofers in Tennessee", "ICP research for med spas", "CRO audit for this URL", or "brand design briefing".'),
|
|
2792
|
+
query: import_zod2.z.string().optional().describe("Business category, niche, or Maps query when known."),
|
|
2793
|
+
keyword: import_zod2.z.string().optional().describe("Search keyword, audience problem, or content topic when known."),
|
|
2794
|
+
domain: import_zod2.z.string().optional().describe("Target domain or brand domain when known."),
|
|
2795
|
+
url: import_zod2.z.string().url().optional().describe("Target URL when the workflow should inspect a specific page."),
|
|
2796
|
+
location: import_zod2.z.string().optional().describe("City/region/country for localized research, e.g. Denver, CO."),
|
|
2797
|
+
state: import_zod2.z.string().optional().describe("US state abbreviation or name for state-wide market research."),
|
|
2798
|
+
maxSuggestions: import_zod2.z.number().int().min(1).max(8).default(3).describe("Number of matching workflow recipes to return.")
|
|
2799
|
+
};
|
|
2800
|
+
var WorkflowRunInputSchema = {
|
|
2801
|
+
workflowId: WorkflowIdSchema.describe("Workflow to run. Use workflow_list or workflow_suggest first when unsure."),
|
|
2802
|
+
input: import_zod2.z.record(import_zod2.z.unknown()).default({}).describe("Workflow-specific input object. Examples: agent-packet uses {keyword, domain?, location?, maxQuestions?}; local-competitive-audit uses {query, state, minPopulation?, maxCities?, maxResultsPerCity?, hydrateTop?, maxReviews?}; serp-comparison uses {keyword, domain?, url?, location?, extractTop?}."),
|
|
2803
|
+
webhookUrl: import_zod2.z.string().url().optional().describe("Optional HTTPS webhook to receive the completed hosted workflow run event.")
|
|
2804
|
+
};
|
|
2805
|
+
var WorkflowStatusInputSchema = {
|
|
2806
|
+
runId: import_zod2.z.string().min(1).describe("Workflow run id returned by workflow_run.")
|
|
2807
|
+
};
|
|
2808
|
+
var WorkflowArtifactReadInputSchema = {
|
|
2809
|
+
runId: import_zod2.z.string().min(1).describe("Workflow run id returned by workflow_run or workflow_status."),
|
|
2810
|
+
artifactId: import_zod2.z.string().min(1).describe("Artifact id from the run artifact list."),
|
|
2811
|
+
maxBytes: import_zod2.z.number().int().min(1e3).max(1e6).default(2e5).describe("Maximum bytes of artifact text to return inline. Use lower values for large CSV/JSON artifacts; call again with the downloadUrl if needed outside MCP.")
|
|
2812
|
+
};
|
|
2813
|
+
var WorkflowRecipeOutput = import_zod2.z.object({
|
|
2814
|
+
id: import_zod2.z.string(),
|
|
2815
|
+
title: import_zod2.z.string(),
|
|
2816
|
+
description: import_zod2.z.string(),
|
|
2817
|
+
primaryWorkflowId: import_zod2.z.string().nullable(),
|
|
2818
|
+
recommendedTools: import_zod2.z.array(import_zod2.z.string()),
|
|
2819
|
+
requiredInputs: import_zod2.z.array(import_zod2.z.string()),
|
|
2820
|
+
optionalInputs: import_zod2.z.array(import_zod2.z.string()),
|
|
2821
|
+
produces: import_zod2.z.array(import_zod2.z.string()),
|
|
2822
|
+
runHint: import_zod2.z.string()
|
|
2823
|
+
});
|
|
2824
|
+
var WorkflowDefinitionOutput = import_zod2.z.object({
|
|
2825
|
+
id: import_zod2.z.string(),
|
|
2826
|
+
title: import_zod2.z.string(),
|
|
2827
|
+
description: import_zod2.z.string()
|
|
2828
|
+
});
|
|
2829
|
+
var WorkflowArtifactOutput = import_zod2.z.record(import_zod2.z.unknown());
|
|
2830
|
+
var WorkflowListOutputSchema = {
|
|
2831
|
+
workflows: import_zod2.z.array(WorkflowDefinitionOutput),
|
|
2832
|
+
recipes: import_zod2.z.array(WorkflowRecipeOutput)
|
|
2833
|
+
};
|
|
2834
|
+
var WorkflowSuggestOutputSchema = {
|
|
2835
|
+
goal: import_zod2.z.string(),
|
|
2836
|
+
suggestions: import_zod2.z.array(WorkflowRecipeOutput)
|
|
2837
|
+
};
|
|
2838
|
+
var WorkflowRunOutputSchema = {
|
|
2839
|
+
workflowId: import_zod2.z.string(),
|
|
2840
|
+
input: import_zod2.z.record(import_zod2.z.unknown()),
|
|
2841
|
+
run: import_zod2.z.record(import_zod2.z.unknown()).optional(),
|
|
2842
|
+
summary: import_zod2.z.record(import_zod2.z.unknown()).optional(),
|
|
2843
|
+
artifacts: import_zod2.z.array(WorkflowArtifactOutput)
|
|
2844
|
+
};
|
|
2845
|
+
var WorkflowStatusOutputSchema = {
|
|
2846
|
+
run: import_zod2.z.record(import_zod2.z.unknown()).optional(),
|
|
2847
|
+
artifacts: import_zod2.z.array(WorkflowArtifactOutput)
|
|
2848
|
+
};
|
|
2849
|
+
var WorkflowArtifactReadOutputSchema = {
|
|
2850
|
+
runId: import_zod2.z.string(),
|
|
2851
|
+
artifactId: import_zod2.z.string(),
|
|
2852
|
+
contentType: import_zod2.z.string(),
|
|
2853
|
+
bytes: import_zod2.z.number().int().min(0),
|
|
2854
|
+
truncated: import_zod2.z.boolean(),
|
|
2855
|
+
text: import_zod2.z.string()
|
|
2856
|
+
};
|
|
2427
2857
|
var SearchSerpInputSchema = {
|
|
2428
2858
|
query: import_zod2.z.string().min(1).describe('Core search topic only. Separate location when possible. If user says "best dentist in Brooklyn NY serp", use query="best dentist" and location="Brooklyn, NY".'),
|
|
2429
2859
|
location: import_zod2.z.string().optional().describe("City, region, or country for geo-targeted results, inferred from user request when present."),
|
|
@@ -2942,6 +3372,41 @@ function registerPaaExtractorMcpTools(server2, executor, options = {}) {
|
|
|
2942
3372
|
outputSchema: DirectoryWorkflowOutputSchema,
|
|
2943
3373
|
annotations: liveWebToolAnnotations("Directory Workflow: Markets + Maps")
|
|
2944
3374
|
}, async (input) => formatDirectoryWorkflow(await executor.directoryWorkflow(input), input));
|
|
3375
|
+
server2.registerTool("workflow_list", {
|
|
3376
|
+
title: "Workflow Catalog",
|
|
3377
|
+
description: "List MCP Scraper higher-level workflows and AI-facing recipes. Use this when the user asks what MCP Scraper can do beyond single tools, or asks for market analysis, ICP research, forum/review acquisition, full brand design briefings, CRO audits, competitive positioning, content gap briefs, or AI search visibility audits. Returns runnable workflow ids plus recipe guidance for the best tool chain.",
|
|
3378
|
+
inputSchema: WorkflowListInputSchema,
|
|
3379
|
+
outputSchema: WorkflowListOutputSchema,
|
|
3380
|
+
annotations: localPlanningToolAnnotations("Workflow Catalog")
|
|
3381
|
+
}, async (input) => formatWorkflowList(await executor.workflowList(input), input));
|
|
3382
|
+
server2.registerTool("workflow_suggest", {
|
|
3383
|
+
title: "Workflow Intent Router",
|
|
3384
|
+
description: 'Route a high-level business or research goal to the right MCP Scraper workflow/tool chain. Use before running work when the user says things like "do market analysis", "research my ICP", "acquire forum and review language", "build a brand design brief", "run a CRO audit", "compare competitors", "make a content gap brief", or "audit AI search visibility". This tool does not spend credits; it tells the AI exactly which workflow/tool chain to run next.',
|
|
3385
|
+
inputSchema: WorkflowSuggestInputSchema,
|
|
3386
|
+
outputSchema: WorkflowSuggestOutputSchema,
|
|
3387
|
+
annotations: localPlanningToolAnnotations("Workflow Intent Router")
|
|
3388
|
+
}, async (input) => formatWorkflowSuggest(input));
|
|
3389
|
+
server2.registerTool("workflow_run", {
|
|
3390
|
+
title: "Run Workflow",
|
|
3391
|
+
description: withReportNote("Run a higher-level MCP Scraper workflow and return the run id, summary, and artifacts. Use after workflow_suggest or workflow_list. Runnable workflow ids: directory, agent-packet, local-competitive-audit, map-comparison, serp-comparison, paa-expansion-brief, ai-overview-language. This is the main MCP tool for executing market analysis, ICP evidence packets, local competitive audits, Maps/SERP comparisons, content gap briefs, and AI Overview language guidance."),
|
|
3392
|
+
inputSchema: WorkflowRunInputSchema,
|
|
3393
|
+
outputSchema: WorkflowRunOutputSchema,
|
|
3394
|
+
annotations: liveWebToolAnnotations("Run Workflow")
|
|
3395
|
+
}, async (input) => formatWorkflowRun(await executor.workflowRun(input), input));
|
|
3396
|
+
server2.registerTool("workflow_status", {
|
|
3397
|
+
title: "Workflow Status",
|
|
3398
|
+
description: "Fetch a hosted workflow run by id and list its current status and artifacts. Use after workflow_run when the model needs to re-open a run, inspect artifact ids, or recover from a long workflow.",
|
|
3399
|
+
inputSchema: WorkflowStatusInputSchema,
|
|
3400
|
+
outputSchema: WorkflowStatusOutputSchema,
|
|
3401
|
+
annotations: liveWebToolAnnotations("Workflow Status")
|
|
3402
|
+
}, async (input) => formatWorkflowStatus(await executor.workflowStatus(input), input));
|
|
3403
|
+
server2.registerTool("workflow_artifact_read", {
|
|
3404
|
+
title: "Read Workflow Artifact",
|
|
3405
|
+
description: "Read a workflow artifact back into MCP context by run id and artifact id. Use this before writing final deliverables so the answer is grounded in generated evidence.json, CSVs, Markdown briefs, reports, and task files instead of memory. Use maxBytes to limit large CSV/JSON artifacts.",
|
|
3406
|
+
inputSchema: WorkflowArtifactReadInputSchema,
|
|
3407
|
+
outputSchema: WorkflowArtifactReadOutputSchema,
|
|
3408
|
+
annotations: liveWebToolAnnotations("Read Workflow Artifact")
|
|
3409
|
+
}, async (input) => formatWorkflowArtifactRead(await executor.workflowArtifactRead(input), input));
|
|
2945
3410
|
server2.registerTool("rank_tracker_blueprint", {
|
|
2946
3411
|
title: "Rank Tracker Blueprint Builder",
|
|
2947
3412
|
description: "Generate a build-ready database, cron/heartbeat, ingestion, metrics, and AI implementation prompt for a rank tracker powered by MCP Scraper. Supports Maps rankings through directory_workflow/maps_search, organic rankings through search_serp, AI Overview citation tracking, and People Also Ask source presence tracking. This tool is local planning only; it does not call the web or spend credits.",
|