mcp-scraper 0.88.2 → 0.89.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +36 -15
- package/README.md +6 -5
- package/THIRD_PARTY_NOTICES.html +203 -0
- package/dist/analytics-repository-2BMT5JNE.js +1 -0
- package/dist/bin/api-server.js +2 -41
- package/dist/bin/mcp-scraper-cli.js +39 -756
- package/dist/bin/mcp-scraper-core.js +1 -60
- package/dist/bin/mcp-scraper-install.js +2 -25
- package/dist/bin/mcp-stdio-server.js +1 -19
- package/dist/bin/paa-harvest.js +1 -41
- package/dist/chunk-3GP5CYZX.js +1 -0
- package/dist/chunk-4AI7DOS7.js +59 -0
- package/dist/chunk-4FROKQJN.js +1 -0
- package/dist/chunk-7YGVI5J4.js +21710 -0
- package/dist/chunk-CCYWSJNG.js +16 -0
- package/dist/chunk-CFI6CXIV.js +182 -0
- package/dist/chunk-E2WRWV3A.js +1 -0
- package/dist/chunk-GMWKPIYX.js +1172 -0
- package/dist/chunk-HDPYG3XV.js +102 -0
- package/dist/chunk-HE45FFBU.js +1 -0
- package/dist/chunk-HUV2WTRW.js +1 -0
- package/dist/chunk-KJQXUZ4Y.js +4 -0
- package/dist/chunk-L4CGLFPU.js +4 -0
- package/dist/chunk-M22MM4N4.js +84 -0
- package/dist/chunk-MASR22K4.js +73 -0
- package/dist/chunk-MZN4U5BL.js +1 -0
- package/dist/chunk-PUHFVA7P.js +1280 -0
- package/dist/chunk-QPWPR5XG.js +10 -0
- package/dist/chunk-TMB56NCA.js +1 -0
- package/dist/chunk-TXENITMS.js +20 -0
- package/dist/chunk-W2BVJ7S2.js +13 -0
- package/dist/chunk-WO3N5FH2.js +5 -0
- package/dist/chunk-WSCGYRWA.js +2595 -0
- package/dist/chunk-X54CQLK2.js +1 -0
- package/dist/chunk-XLWNEVUZ.js +27 -0
- package/dist/chunk-XPZVJIZ2.js +100 -0
- package/dist/chunk-YQZGZBB4.js +1 -0
- package/dist/chunk-Z2QGQJS2.js +1 -0
- package/dist/db-F2MX63GI.js +1 -0
- package/dist/extract-bundle-SNUIHM3J.js +26 -0
- package/dist/gmail-service-BZ3H75XC.js +1 -0
- package/dist/index.cjs +21750 -6045
- package/dist/index.d.cts +14 -14
- package/dist/index.d.ts +14 -14
- package/dist/index.js +18 -315
- package/dist/lead-list-enrichment-repository-S2H3U7T7.js +1 -0
- package/dist/location-data-repository-OTWHWMV6.js +1 -0
- package/dist/server-RFR2A5UJ.js +7303 -0
- package/dist/site-extract-repository-SE776XDC.js +1 -0
- package/dist/worker-XUDSM3AL.js +1 -0
- package/package.json +17 -124
- package/dist/analytics-repository-GGJJCVVP.js +0 -194
- package/dist/chunk-4QMUF6XM.js +0 -1013
- package/dist/chunk-6HAV7LCE.js +0 -265
- package/dist/chunk-ABF2CGOZ.js +0 -113
- package/dist/chunk-C5Z4OFKW.js +0 -404
- package/dist/chunk-DNM65UCK.js +0 -299
- package/dist/chunk-EQGTEHLZ.js +0 -592
- package/dist/chunk-F5GQJWZU.js +0 -732
- package/dist/chunk-GGZEC22A.js +0 -215
- package/dist/chunk-GXBZXWXB.js +0 -184
- package/dist/chunk-IHXAXYIS.js +0 -843
- package/dist/chunk-K3Z5AQYE.js +0 -683
- package/dist/chunk-K45K75OF.js +0 -6
- package/dist/chunk-LFW2FRPJ.js +0 -224
- package/dist/chunk-MZDNZQWT.js +0 -2078
- package/dist/chunk-NVUKO5NN.js +0 -256
- package/dist/chunk-OM7HVEJ3.js +0 -26
- package/dist/chunk-OPQIGAFB.js +0 -286
- package/dist/chunk-OZJMVCDK.js +0 -16
- package/dist/chunk-P7FWOMU7.js +0 -505
- package/dist/chunk-PGJQDMC2.js +0 -383
- package/dist/chunk-PKZS6SHW.js +0 -33139
- package/dist/chunk-RJ7JVYKU.js +0 -68
- package/dist/chunk-S24LFPL7.js +0 -5262
- package/dist/chunk-T3MZISOF.js +0 -240
- package/dist/chunk-UZPTGUDV.js +0 -1915
- package/dist/chunk-X623GTBV.js +0 -8290
- package/dist/chunk-YXNDOQXN.js +0 -4018
- package/dist/db-Z34LPZNR.js +0 -284
- package/dist/extract-bundle-565SBZCR.js +0 -1003
- package/dist/gmail-service-E6ALS7JG.js +0 -25
- package/dist/lead-list-enrichment-repository-36RPVV6N.js +0 -67
- package/dist/location-data-repository-WPRG62GE.js +0 -34
- package/dist/server-SQZ3A7SY.js +0 -86606
- package/dist/site-extract-repository-VYFZASPU.js +0 -69
- package/dist/worker-LDCAULWL.js +0 -146
|
@@ -1,757 +1,40 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import {
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
}
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
}
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
}
|
|
40
|
-
function
|
|
41
|
-
const env = { MCP_SCRAPER_API_KEY: apiKeyValue(options) };
|
|
42
|
-
const profileName = options.browserProfileName?.trim();
|
|
43
|
-
if (profileName) env.BROWSER_AGENT_PROFILE_NAME = profileName;
|
|
44
|
-
if (options.browserProfileSaveChanges === true) env.BROWSER_AGENT_PROFILE_SAVE_CHANGES = "true";
|
|
45
|
-
return env;
|
|
46
|
-
}
|
|
47
|
-
function claudeMcpRemoveArgs() {
|
|
48
|
-
return ["mcp", "remove", "mcp-scraper", "-s", "user"];
|
|
49
|
-
}
|
|
50
|
-
function claudeMcpGetArgs() {
|
|
51
|
-
return ["mcp", "get", "mcp-scraper"];
|
|
52
|
-
}
|
|
53
|
-
function parseClaudeMcpGet(stdout) {
|
|
54
|
-
const command = stdout.match(/^\s*Command:\s*(.+?)\s*$/m)?.[1];
|
|
55
|
-
if (!command) return null;
|
|
56
|
-
const argLine = stdout.match(/^\s*Args:\s*(.*?)\s*$/m)?.[1] ?? "";
|
|
57
|
-
const args = argLine.length ? argLine.split(/\s+/) : [];
|
|
58
|
-
const env = {};
|
|
59
|
-
const envBlock = stdout.split(/^\s*Environment:\s*$/m)[1];
|
|
60
|
-
if (envBlock) {
|
|
61
|
-
for (const line of envBlock.split("\n")) {
|
|
62
|
-
const pair = line.match(/^\s{2,}([A-Za-z_][A-Za-z0-9_]*)=(.*)$/);
|
|
63
|
-
if (!pair) {
|
|
64
|
-
if (line.trim().length && !/^\s{2,}/.test(line)) break;
|
|
65
|
-
continue;
|
|
66
|
-
}
|
|
67
|
-
env[pair[1]] = pair[2];
|
|
68
|
-
}
|
|
69
|
-
}
|
|
70
|
-
return { command, args, env };
|
|
71
|
-
}
|
|
72
|
-
function claudeMcpRestoreArgs(snapshot) {
|
|
73
|
-
const args = ["mcp", "add", "mcp-scraper", "--scope", "user"];
|
|
74
|
-
for (const [key, value] of Object.entries(snapshot.env)) {
|
|
75
|
-
args.push("--env", `${key}=${value}`);
|
|
76
|
-
}
|
|
77
|
-
args.push("--", snapshot.command, ...snapshot.args);
|
|
78
|
-
return args;
|
|
79
|
-
}
|
|
80
|
-
function claudeMcpAddArgs(options = {}) {
|
|
81
|
-
const args = ["mcp", "add", "mcp-scraper", "--scope", "user"];
|
|
82
|
-
for (const [key, value] of Object.entries(envConfig(options))) {
|
|
83
|
-
args.push("--env", `${key}=${value}`);
|
|
84
|
-
}
|
|
85
|
-
args.push("--", "npx", ...combinedNpxArgs(options));
|
|
86
|
-
return args;
|
|
87
|
-
}
|
|
88
|
-
function normalizeAgentHost(host) {
|
|
89
|
-
if (host === "claude-code") return "claude";
|
|
90
|
-
if (host === "claude" || XRAY_INSTALL_CONTRACT.hosts.some((candidate) => candidate.id === host)) return host;
|
|
91
|
-
throw new Error('Unknown host "' + host + '". Use: codex, claude, claude-code, claude-desktop, cursor, windsurf, cline, or user-action-only');
|
|
92
|
-
}
|
|
93
|
-
function agentHostCapability(host) {
|
|
94
|
-
return xrayHostRecipe(host === "claude" ? "claude-code" : host);
|
|
95
|
-
}
|
|
96
|
-
function renderAgentInstall(host, _options = {}) {
|
|
97
|
-
const normalizedHost = normalizeAgentHost(host);
|
|
98
|
-
const recipe = agentHostCapability(normalizedHost);
|
|
99
|
-
const restart = "Restart the MCP client so it starts a fresh npx process.";
|
|
100
|
-
const applyCommand = 'MCP_SCRAPER_API_KEY="$MCP_SCRAPER_API_KEY" npx -y -p mcp-scraper@latest mcp-scraper-cli agent install claude --apply';
|
|
101
|
-
const protocol = `X-Ray install protocol: ${XRAY_INSTALL_PROTOCOL_VERSION} (${XRAY_INSTALL_PROTOCOL_DIGEST})`;
|
|
102
|
-
if (normalizedHost === "codex") {
|
|
103
|
-
return [
|
|
104
|
-
"# Codex MCP config",
|
|
105
|
-
protocol,
|
|
106
|
-
recipe.exactConfig,
|
|
107
|
-
"",
|
|
108
|
-
`Continuation: ${recipe.continuation}`,
|
|
109
|
-
`Rollback: ${recipe.rollback}`,
|
|
110
|
-
"",
|
|
111
|
-
restart
|
|
112
|
-
].join("\n");
|
|
113
|
-
}
|
|
114
|
-
if (normalizedHost === "claude") {
|
|
115
|
-
return [
|
|
116
|
-
"# Claude Code command",
|
|
117
|
-
protocol,
|
|
118
|
-
recipe.exactConfig,
|
|
119
|
-
"",
|
|
120
|
-
"# One-command Claude Code setup",
|
|
121
|
-
applyCommand,
|
|
122
|
-
"",
|
|
123
|
-
`Continuation: ${recipe.continuation}`,
|
|
124
|
-
`Rollback: ${recipe.rollback}`,
|
|
125
|
-
"",
|
|
126
|
-
restart
|
|
127
|
-
].join("\n");
|
|
128
|
-
}
|
|
129
|
-
if (normalizedHost === "claude-desktop") {
|
|
130
|
-
return [
|
|
131
|
-
"# Claude Desktop config",
|
|
132
|
-
protocol,
|
|
133
|
-
recipe.exactConfig,
|
|
134
|
-
"",
|
|
135
|
-
"Desktop Extension: https://mcpscraper.dev/downloads/mcp-scraper.mcpb",
|
|
136
|
-
`Continuation: ${recipe.continuation}`,
|
|
137
|
-
`Rollback: ${recipe.rollback}`,
|
|
138
|
-
restart
|
|
139
|
-
].join("\n");
|
|
140
|
-
}
|
|
141
|
-
return [
|
|
142
|
-
`# ${recipe.label} MCP config`,
|
|
143
|
-
protocol,
|
|
144
|
-
recipe.exactConfig,
|
|
145
|
-
"",
|
|
146
|
-
`Continuation: ${recipe.continuation}`,
|
|
147
|
-
`Rollback: ${recipe.rollback}`,
|
|
148
|
-
...recipe.restartRequired ? ["", restart] : []
|
|
149
|
-
].join("\n");
|
|
150
|
-
}
|
|
151
|
-
|
|
152
|
-
// src/cli/doctor.ts
|
|
153
|
-
import { access, mkdir, readFile } from "fs/promises";
|
|
154
|
-
import { homedir } from "os";
|
|
155
|
-
import { join } from "path";
|
|
156
|
-
function status(ok, warn = false) {
|
|
157
|
-
if (ok) return "pass";
|
|
158
|
-
return warn ? "warn" : "fail";
|
|
159
|
-
}
|
|
160
|
-
function parseMajor(version) {
|
|
161
|
-
return Number(version.replace(/^v/, "").split(".")[0] ?? 0);
|
|
162
|
-
}
|
|
163
|
-
async function readKeyFile() {
|
|
164
|
-
const path = process.env.MCP_SCRAPER_KEY_PATH?.trim() || join(homedir(), ".mcp-scraper-key");
|
|
165
|
-
try {
|
|
166
|
-
const value = (await readFile(path, "utf8")).trim();
|
|
167
|
-
return value || null;
|
|
168
|
-
} catch {
|
|
169
|
-
return null;
|
|
170
|
-
}
|
|
171
|
-
}
|
|
172
|
-
async function npmLatest(fetchImpl) {
|
|
173
|
-
try {
|
|
174
|
-
const res = await fetchImpl("https://registry.npmjs.org/mcp-scraper/latest", {
|
|
175
|
-
signal: AbortSignal.timeout(5e3)
|
|
176
|
-
});
|
|
177
|
-
if (!res.ok) return null;
|
|
178
|
-
const data = await res.json();
|
|
179
|
-
return data.version ?? null;
|
|
180
|
-
} catch {
|
|
181
|
-
return null;
|
|
182
|
-
}
|
|
183
|
-
}
|
|
184
|
-
async function runDoctor(options = {}) {
|
|
185
|
-
const fetchImpl = options.fetchImpl ?? fetch;
|
|
186
|
-
const apiUrl = (options.apiUrl ?? process.env.MCP_SCRAPER_API_URL ?? "https://mcpscraper.dev").replace(/\/$/, "");
|
|
187
|
-
const outputDir = options.outputDir ?? process.env.MCP_SCRAPER_OUTPUT_DIR ?? join(homedir(), "Downloads", "mcp-scraper");
|
|
188
|
-
const configuredKey = options.apiKey?.trim() || process.env.MCP_SCRAPER_API_KEY?.trim() || await readKeyFile();
|
|
189
|
-
const latest = await npmLatest(fetchImpl);
|
|
190
|
-
const checks = [];
|
|
191
|
-
const nodeMajor = parseMajor(process.version);
|
|
192
|
-
checks.push({
|
|
193
|
-
id: "node",
|
|
194
|
-
label: "Node.js",
|
|
195
|
-
status: status(nodeMajor >= 20),
|
|
196
|
-
detail: process.version,
|
|
197
|
-
fix: nodeMajor >= 20 ? void 0 : "Install Node.js 20 or newer."
|
|
198
|
-
});
|
|
199
|
-
checks.push({
|
|
200
|
-
id: "package_version",
|
|
201
|
-
label: "Local package",
|
|
202
|
-
status: latest && latest !== PACKAGE_VERSION ? "warn" : "pass",
|
|
203
|
-
detail: latest ? `local ${PACKAGE_VERSION}, npm latest ${latest}` : `local ${PACKAGE_VERSION}, npm latest unavailable`,
|
|
204
|
-
fix: latest && latest !== PACKAGE_VERSION ? "Use mcp-scraper@latest and restart the MCP client." : void 0
|
|
205
|
-
});
|
|
206
|
-
checks.push({
|
|
207
|
-
id: "api_key",
|
|
208
|
-
label: "API key",
|
|
209
|
-
status: configuredKey ? "pass" : "warn",
|
|
210
|
-
detail: configuredKey ? "configured" : "not configured",
|
|
211
|
-
fix: configuredKey ? void 0 : "Set MCP_SCRAPER_API_KEY or pass --api-key."
|
|
212
|
-
});
|
|
213
|
-
try {
|
|
214
|
-
await mkdir(outputDir, { recursive: true });
|
|
215
|
-
await access(outputDir);
|
|
216
|
-
checks.push({ id: "output_dir", label: "Output directory", status: "pass", detail: outputDir });
|
|
217
|
-
} catch (err) {
|
|
218
|
-
checks.push({
|
|
219
|
-
id: "output_dir",
|
|
220
|
-
label: "Output directory",
|
|
221
|
-
status: "fail",
|
|
222
|
-
detail: outputDir,
|
|
223
|
-
fix: err instanceof Error ? err.message : "Create a writable output directory."
|
|
224
|
-
});
|
|
225
|
-
}
|
|
226
|
-
if (configuredKey) {
|
|
227
|
-
try {
|
|
228
|
-
const res = await fetchImpl(`${apiUrl}/me`, {
|
|
229
|
-
headers: { "x-api-key": configuredKey },
|
|
230
|
-
signal: AbortSignal.timeout(8e3)
|
|
231
|
-
});
|
|
232
|
-
checks.push({
|
|
233
|
-
id: "api_reachability",
|
|
234
|
-
label: "Hosted API",
|
|
235
|
-
status: res.ok ? "pass" : "fail",
|
|
236
|
-
detail: `${apiUrl}/me returned ${res.status}`,
|
|
237
|
-
fix: res.ok ? void 0 : "Verify the API key and account status."
|
|
238
|
-
});
|
|
239
|
-
} catch (err) {
|
|
240
|
-
checks.push({
|
|
241
|
-
id: "api_reachability",
|
|
242
|
-
label: "Hosted API",
|
|
243
|
-
status: "fail",
|
|
244
|
-
detail: err instanceof Error ? err.message : String(err),
|
|
245
|
-
fix: "Check network access and MCP_SCRAPER_API_URL."
|
|
246
|
-
});
|
|
247
|
-
}
|
|
248
|
-
} else {
|
|
249
|
-
checks.push({
|
|
250
|
-
id: "api_reachability",
|
|
251
|
-
label: "Hosted API",
|
|
252
|
-
status: "skip",
|
|
253
|
-
detail: "skipped because no API key is configured"
|
|
254
|
-
});
|
|
255
|
-
}
|
|
256
|
-
checks.push({
|
|
257
|
-
id: "mcp_config",
|
|
258
|
-
label: "MCP command",
|
|
259
|
-
status: "pass",
|
|
260
|
-
detail: `npx ${combinedNpxArgs().join(" ")}`,
|
|
261
|
-
fix: "Restart the MCP client after package updates."
|
|
262
|
-
});
|
|
263
|
-
return {
|
|
264
|
-
ok: checks.every((check) => check.status === "pass" || check.status === "skip"),
|
|
265
|
-
version: PACKAGE_VERSION,
|
|
266
|
-
npmLatest: latest,
|
|
267
|
-
checks,
|
|
268
|
-
recommendedConfig: {
|
|
269
|
-
command: "npx",
|
|
270
|
-
args: combinedNpxArgs(),
|
|
271
|
-
env: { MCP_SCRAPER_API_KEY: configuredKey ? "$MCP_SCRAPER_API_KEY" : "sk_live_your_key" }
|
|
272
|
-
}
|
|
273
|
-
};
|
|
274
|
-
}
|
|
275
|
-
function renderDoctor(output) {
|
|
276
|
-
const icon = { pass: "PASS", warn: "WARN", fail: "FAIL", skip: "SKIP" };
|
|
277
|
-
const lines = [
|
|
278
|
-
`mcp-scraper doctor v${output.version}`,
|
|
279
|
-
"",
|
|
280
|
-
...output.checks.map((check) => {
|
|
281
|
-
const fix = check.fix ? `
|
|
282
|
-
Fix: ${check.fix}` : "";
|
|
283
|
-
return `[${icon[check.status]}] ${check.label}: ${check.detail}${fix}`;
|
|
284
|
-
}),
|
|
285
|
-
"",
|
|
286
|
-
`Recommended MCP command: npx ${output.recommendedConfig.args.join(" ")}`
|
|
287
|
-
];
|
|
288
|
-
return lines.join("\n");
|
|
289
|
-
}
|
|
290
|
-
|
|
291
|
-
// src/cli/prompts.ts
|
|
292
|
-
var AGENT_PROMPTS = {
|
|
293
|
-
"xray-setup": renderXraySetupPrompt(),
|
|
294
|
-
"agent-packet": [
|
|
295
|
-
"# MCP Scraper Agent Packet Prompt",
|
|
296
|
-
"",
|
|
297
|
-
"Use MCP Scraper as the evidence layer. Run the agent-packet workflow for the target keyword/domain, then treat `evidence.json`, `sources.csv`, and `competitors.csv` as source of truth.",
|
|
298
|
-
"",
|
|
299
|
-
"Do not invent citations. If a recommendation is not supported by the packet, mark it as an assumption. Turn the evidence into a concise SEO brief, an implementation task list, and content recommendations tied to source rows."
|
|
300
|
-
].join("\n"),
|
|
301
|
-
"local-competitive-audit": [
|
|
302
|
-
"# MCP Scraper Local Competitive Audit Prompt",
|
|
303
|
-
"",
|
|
304
|
-
"Run the local-competitive-audit workflow for the niche and markets. Use the city summary, competitor CSV, and review-insight CSV to identify market difficulty, review themes, GBP category patterns, and weak competitors.",
|
|
305
|
-
"",
|
|
306
|
-
"Ground recommendations in Maps rank position, review count, star rating, categories, profile details, and review topics. Do not overstate heuristic opportunity scores."
|
|
307
|
-
].join("\n"),
|
|
308
|
-
"directory-workflow": [
|
|
309
|
-
"# MCP Scraper Directory Workflow Prompt",
|
|
310
|
-
"",
|
|
311
|
-
"Use directory_workflow when the user wants cities selected by population and Google Maps candidates per city. Keep query and location separate. Preserve result_position, source_location, review stars/count, categories, and profile URLs for downstream CSV or directory use."
|
|
312
|
-
].join("\n"),
|
|
313
|
-
"map-comparison": [
|
|
314
|
-
"# MCP Scraper Maps Comparison Prompt",
|
|
315
|
-
"",
|
|
316
|
-
"Run the map-comparison workflow when the user wants to compare local Maps competitors in a city or across selected markets. Use `maps-results.csv`, `map-comparison.csv`, and `profile-insights.csv` as source of truth.",
|
|
317
|
-
"",
|
|
318
|
-
"Ground recommendations in result_position, review_stars, review_count, category, website presence, review topics, and profile attributes. Treat review gaps and missing websites as opportunity signals, not guaranteed ranking factors."
|
|
319
|
-
].join("\n"),
|
|
320
|
-
"serp-comparison": [
|
|
321
|
-
"# MCP Scraper SERP Comparison Prompt",
|
|
322
|
-
"",
|
|
323
|
-
"Run the serp-comparison workflow when the user wants to know why competitors outrank a page or what the SERP rewards. Use organic results, extracted page headings, PAA questions, and AI Overview citations before making recommendations.",
|
|
324
|
-
"",
|
|
325
|
-
"Tie each recommendation to `content-gaps.csv`, `page-comparison.csv`, `paa-questions.csv`, or `ai-overview-citations.csv`. Do not invent missing sections or citations."
|
|
326
|
-
].join("\n"),
|
|
327
|
-
"paa-expansion-brief": [
|
|
328
|
-
"# MCP Scraper PAA Expansion Brief Prompt",
|
|
329
|
-
"",
|
|
330
|
-
"Run the paa-expansion-brief workflow when the user wants to figure out what to write from People Also Ask expansion. Use `section-map.csv` to structure the brief and `paa-questions.csv` for exact customer-language headings.",
|
|
331
|
-
"",
|
|
332
|
-
"Answer the highest-priority questions directly, preserve source URLs when present, and separate evidence-backed sections from assumptions."
|
|
333
|
-
].join("\n"),
|
|
334
|
-
"ai-overview-language": [
|
|
335
|
-
"# MCP Scraper AI Overview Language Prompt",
|
|
336
|
-
"",
|
|
337
|
-
"Run the ai-overview-language workflow when the user wants to know how to phrase content for AI Overview inclusion. Use `claim-patterns.csv`, `language-guidance.csv`, `ai-overview-citations.csv`, and PAA follow-ups as evidence.",
|
|
338
|
-
"",
|
|
339
|
-
"Recommend concise answer blocks, criteria/step language, citation hooks, and follow-up sections. Do not claim the target will be cited; frame the output as evidence-based language guidance."
|
|
340
|
-
].join("\n"),
|
|
341
|
-
"ai-citation-monitor": [
|
|
342
|
-
"# MCP Scraper AI Citation Monitor Prompt",
|
|
343
|
-
"",
|
|
344
|
-
"Use search_serp and harvest_paa to check whether the target brand/domain appears in AI Overview citations, organic results, People Also Ask sources, local pack, forums, and videos. Save evidence before summarizing trends or gaps."
|
|
345
|
-
].join("\n"),
|
|
346
|
-
"serp-brief": [
|
|
347
|
-
"# MCP Scraper SERP Brief Prompt",
|
|
348
|
-
"",
|
|
349
|
-
"Use live SERP, PAA, AI Overview, forum, video, and source evidence to create a writer brief. Tie each recommended section to evidence rows and identify missing proof, entities, comparisons, and customer questions."
|
|
350
|
-
].join("\n")
|
|
351
|
-
};
|
|
352
|
-
function listPrompts() {
|
|
353
|
-
return Object.keys(AGENT_PROMPTS).sort();
|
|
354
|
-
}
|
|
355
|
-
function renderPrompt(name) {
|
|
356
|
-
const prompt = AGENT_PROMPTS[name];
|
|
357
|
-
if (!prompt) throw new Error(`Unknown prompt "${name}". Available: ${listPrompts().join(", ")}`);
|
|
358
|
-
return prompt;
|
|
359
|
-
}
|
|
360
|
-
|
|
361
|
-
// src/cli/human-cli.ts
|
|
362
|
-
function numberOpt(value) {
|
|
363
|
-
if (value === void 0 || value === null || value === "") return void 0;
|
|
364
|
-
const parsed = Number(value);
|
|
365
|
-
return Number.isFinite(parsed) ? parsed : void 0;
|
|
366
|
-
}
|
|
367
|
-
function booleanOpt(value) {
|
|
368
|
-
if (value === void 0) return void 0;
|
|
369
|
-
if (typeof value === "boolean") return value;
|
|
370
|
-
if (value === "true") return true;
|
|
371
|
-
if (value === "false") return false;
|
|
372
|
-
return void 0;
|
|
373
|
-
}
|
|
374
|
-
function compactInput(input) {
|
|
375
|
-
return Object.fromEntries(Object.entries(input).filter(([, value]) => value !== void 0));
|
|
376
|
-
}
|
|
377
|
-
function workflowInput(id, opts) {
|
|
378
|
-
if (id === "agent-packet" || id === "serp-comparison" || id === "ai-overview-language") {
|
|
379
|
-
return compactInput({
|
|
380
|
-
keyword: opts.keyword,
|
|
381
|
-
domain: opts.domain,
|
|
382
|
-
url: opts.url,
|
|
383
|
-
location: opts.location,
|
|
384
|
-
maxResults: numberOpt(opts.maxResults),
|
|
385
|
-
maxQuestions: numberOpt(opts.maxQuestions),
|
|
386
|
-
extractTop: numberOpt(opts.extractTop),
|
|
387
|
-
includeSerp: opts.serp === false ? false : void 0,
|
|
388
|
-
includePaa: opts.paa === false ? false : void 0,
|
|
389
|
-
includeAiOverview: booleanOpt(opts.includeAiOverview),
|
|
390
|
-
returnPartial: opts.returnPartial === false ? false : void 0
|
|
391
|
-
});
|
|
392
|
-
}
|
|
393
|
-
if (id === "paa-expansion-brief") {
|
|
394
|
-
return compactInput({
|
|
395
|
-
keyword: opts.keyword,
|
|
396
|
-
location: opts.location,
|
|
397
|
-
maxQuestions: numberOpt(opts.maxQuestions),
|
|
398
|
-
depth: numberOpt(opts.depth),
|
|
399
|
-
returnPartial: opts.returnPartial === false ? false : void 0
|
|
400
|
-
});
|
|
401
|
-
}
|
|
402
|
-
if (id === "directory" || id === "local-competitive-audit" || id === "map-comparison") {
|
|
403
|
-
return compactInput({
|
|
404
|
-
query: opts.query,
|
|
405
|
-
location: opts.location,
|
|
406
|
-
state: opts.state,
|
|
407
|
-
minPopulation: numberOpt(opts.minPop ?? opts.minPopulation),
|
|
408
|
-
maxCities: numberOpt(opts.maxCities),
|
|
409
|
-
maxResultsPerCity: numberOpt(opts.perCity ?? opts.maxResultsPerCity ?? opts.maxResults),
|
|
410
|
-
concurrency: numberOpt(opts.concurrency),
|
|
411
|
-
proxyMode: opts.proxyMode,
|
|
412
|
-
hydrateTop: id === "local-competitive-audit" || id === "map-comparison" ? numberOpt(opts.hydrateTop) : void 0,
|
|
413
|
-
maxReviews: id === "local-competitive-audit" || id === "map-comparison" ? numberOpt(opts.reviews ?? opts.maxReviews) : void 0,
|
|
414
|
-
returnPartial: opts.returnPartial === false ? false : void 0
|
|
415
|
-
});
|
|
416
|
-
}
|
|
417
|
-
return compactInput(opts);
|
|
418
|
-
}
|
|
419
|
-
function writeOutput(data, json) {
|
|
420
|
-
if (json) {
|
|
421
|
-
process.stdout.write(`${JSON.stringify(data, null, 2)}
|
|
422
|
-
`);
|
|
423
|
-
} else if (typeof data === "string") {
|
|
424
|
-
process.stdout.write(`${data}
|
|
425
|
-
`);
|
|
426
|
-
} else {
|
|
427
|
-
process.stdout.write(`${JSON.stringify(data, null, 2)}
|
|
428
|
-
`);
|
|
429
|
-
}
|
|
430
|
-
}
|
|
431
|
-
function maskSecrets(value) {
|
|
432
|
-
return value.replace(/sk_[A-Za-z0-9_-]+/g, "sk_***");
|
|
433
|
-
}
|
|
434
|
-
function runLocalCommand(command, args) {
|
|
435
|
-
return new Promise((resolve) => {
|
|
436
|
-
const child = spawn(command, args, { stdio: ["ignore", "pipe", "pipe"] });
|
|
437
|
-
const stdout = [];
|
|
438
|
-
const stderr = [];
|
|
439
|
-
let settled = false;
|
|
440
|
-
const done = (result) => {
|
|
441
|
-
if (settled) return;
|
|
442
|
-
settled = true;
|
|
443
|
-
resolve(result);
|
|
444
|
-
};
|
|
445
|
-
child.stdout.on("data", (chunk) => stdout.push(Buffer.from(chunk)));
|
|
446
|
-
child.stderr.on("data", (chunk) => stderr.push(Buffer.from(chunk)));
|
|
447
|
-
child.on("error", (err) => {
|
|
448
|
-
done({ code: 127, stdout: "", stderr: err.message });
|
|
449
|
-
});
|
|
450
|
-
child.on("close", (code) => {
|
|
451
|
-
done({
|
|
452
|
-
code: typeof code === "number" ? code : 1,
|
|
453
|
-
stdout: Buffer.concat(stdout).toString("utf8"),
|
|
454
|
-
stderr: Buffer.concat(stderr).toString("utf8")
|
|
455
|
-
});
|
|
456
|
-
});
|
|
457
|
-
});
|
|
458
|
-
}
|
|
459
|
-
function cadenceOpt(opts) {
|
|
460
|
-
if (opts.daily) return "daily";
|
|
461
|
-
if (opts.monthly) return "monthly";
|
|
462
|
-
return "weekly";
|
|
463
|
-
}
|
|
464
|
-
function apiOptions(opts) {
|
|
465
|
-
const apiKey = String(opts.apiKey ?? process.env.MCP_SCRAPER_API_KEY ?? "").trim();
|
|
466
|
-
if (!apiKey) throw new Error("MCP_SCRAPER_API_KEY is required. Pass --api-key or set the environment variable.");
|
|
467
|
-
return {
|
|
468
|
-
apiUrl: String(opts.apiUrl ?? process.env.MCP_SCRAPER_API_URL ?? "https://mcpscraper.dev").replace(/\/$/, ""),
|
|
469
|
-
apiKey
|
|
470
|
-
};
|
|
471
|
-
}
|
|
472
|
-
function openExternalUrl(url) {
|
|
473
|
-
const command = process.platform === "darwin" ? "open" : process.platform === "win32" ? "cmd" : "xdg-open";
|
|
474
|
-
const args = process.platform === "win32" ? ["/c", "start", "", url] : [url];
|
|
475
|
-
try {
|
|
476
|
-
const child = spawn(command, args, { detached: true, stdio: "ignore" });
|
|
477
|
-
child.unref();
|
|
478
|
-
return true;
|
|
479
|
-
} catch {
|
|
480
|
-
return false;
|
|
481
|
-
}
|
|
482
|
-
}
|
|
483
|
-
async function apiRequest(path, method, opts, body) {
|
|
484
|
-
const { apiUrl, apiKey } = apiOptions(opts);
|
|
485
|
-
const res = await fetch(`${apiUrl}${path}`, {
|
|
486
|
-
method,
|
|
487
|
-
headers: {
|
|
488
|
-
"Content-Type": "application/json",
|
|
489
|
-
"x-api-key": apiKey
|
|
490
|
-
},
|
|
491
|
-
body: body == null ? void 0 : JSON.stringify(body)
|
|
492
|
-
});
|
|
493
|
-
const data = await res.json().catch(() => ({}));
|
|
494
|
-
if (!res.ok) {
|
|
495
|
-
const message = typeof data.error === "string" ? data.error : `API request failed with ${res.status}`;
|
|
496
|
-
throw new Error(message);
|
|
497
|
-
}
|
|
498
|
-
return data;
|
|
499
|
-
}
|
|
500
|
-
function addWorkflowInputOptions(command) {
|
|
501
|
-
return command.option("--keyword <keyword>", "Agent packet keyword").option("--domain <domain>", "Target domain").option("--url <url>", "Target page URL").option("--location <location>", "Target location").option("--max-results <n>", "Maximum SERP or Maps results").option("--max-questions <n>", "Maximum PAA questions").option("--extract-top <n>", "Top result pages or citation pages to extract").option("--depth <n>", "PAA expansion depth").option("--no-serp", "Skip SERP evidence for agent packet").option("--no-paa", "Skip PAA evidence for agent packet").option("--query <query>", "Business category or workflow query").option("--state <state>", "US state").option("--min-pop <n>", "Minimum city population").option("--max-cities <n>", "Maximum selected cities").option("--per-city <n>", "Maps results per city").option("--concurrency <n>", "City search concurrency").option("--proxy-mode <mode>", "Proxy mode: location, configured, or none").option("--hydrate-top <n>", "Profiles to hydrate per city for competitive audits").option("--reviews <n>", "Review cards to collect per hydrated profile").option("--no-return-partial", "Fail instead of writing partial artifacts");
|
|
502
|
-
}
|
|
503
|
-
function buildHumanCli() {
|
|
504
|
-
const program = new Command();
|
|
505
|
-
program.name("mcp-scraper-cli").description("Human CLI for MCP Scraper setup, workflows, reports, and agent-ready SEO artifacts.").version(PACKAGE_VERSION);
|
|
506
|
-
program.command("doctor").description("Check setup, API key, output directory, package version, and recommended MCP config.").option("--api-key <key>", "MCP Scraper API key").option("--api-url <url>", "MCP Scraper API URL", "https://mcpscraper.dev").option("--output-dir <path>", "Output directory to test").option("--json", "Print machine-readable JSON").action(async (opts) => {
|
|
507
|
-
const output = await runDoctor({ apiKey: opts.apiKey, apiUrl: opts.apiUrl, outputDir: opts.outputDir });
|
|
508
|
-
writeOutput(opts.json ? output : renderDoctor(output), opts.json);
|
|
509
|
-
if (!output.ok) process.exitCode = 1;
|
|
510
|
-
});
|
|
511
|
-
const billing = program.command("billing").description("Inspect billing and start checkout flows.");
|
|
512
|
-
const billingConcurrency = billing.command("concurrency").description("Manage MCP Scraper concurrency packs.");
|
|
513
|
-
billingConcurrency.command("info").description("Show current concurrency limit and concurrency-pack price.").option("--api-key <key>", "MCP Scraper API key").option("--api-url <url>", "MCP Scraper API URL", "https://mcpscraper.dev").option("--json", "Print machine-readable JSON").action(async (opts) => {
|
|
514
|
-
const result = await apiRequest("/billing/credits", "POST", opts, {});
|
|
515
|
-
if (opts.json) writeOutput(result.concurrency, true);
|
|
516
|
-
else {
|
|
517
|
-
const upgrade = result.concurrency?.upgrade;
|
|
518
|
-
writeOutput([
|
|
519
|
-
`Current limit: ${result.concurrency?.current_limit ?? "unknown"} concurrent operations`,
|
|
520
|
-
`Extra slots: ${result.concurrency?.current_extra_slots ?? "unknown"}`,
|
|
521
|
-
`Concurrency pack: ${upgrade?.price_label ?? "$5/month for 2 extra browsers"}`,
|
|
522
|
-
`Upgrade command: ${upgrade?.terminal_command ?? "mcp-scraper-cli billing concurrency checkout"}`
|
|
523
|
-
].join("\n"), false);
|
|
524
|
-
}
|
|
525
|
-
});
|
|
526
|
-
billingConcurrency.command("checkout").description("Create or update a hosted Stripe checkout for concurrency packs (two browsers per pack).").argument("[quantity]", "Desired pack quantity", "1").option("--api-key <key>", "MCP Scraper API key").option("--api-url <url>", "MCP Scraper API URL", "https://mcpscraper.dev").option("--json", "Print machine-readable JSON").option("--no-open", "Print the checkout URL without opening a browser").action(async (quantity, opts) => {
|
|
527
|
-
const result = await apiRequest("/billing/concurrency/terminal-checkout", "POST", opts, { quantity: Number(quantity) });
|
|
528
|
-
if (opts.json) {
|
|
529
|
-
writeOutput(result, true);
|
|
530
|
-
return;
|
|
531
|
-
}
|
|
532
|
-
const opened = !!result.checkout_url && opts.open !== false && openExternalUrl(result.checkout_url);
|
|
533
|
-
writeOutput([
|
|
534
|
-
`Concurrency pack: ${result.price?.price_label ?? "$5/month for 2 extra browsers"}`,
|
|
535
|
-
result.updated ? "Concurrency pack quantity updated." : opened ? `Opened checkout: ${result.checkout_url}` : `Checkout URL: ${result.checkout_url}`,
|
|
536
|
-
result.next_step ?? "Complete checkout, then retry the MCP request."
|
|
537
|
-
].join("\n"), false);
|
|
538
|
-
});
|
|
539
|
-
billing.command("subscribe <tier>").description("Subscribe to a plan (starter | growth | scale) via a hosted Stripe checkout link.").option("--api-key <key>", "MCP Scraper API key").option("--api-url <url>", "MCP Scraper API URL", "https://mcpscraper.dev").option("--json", "Print machine-readable JSON").option("--no-open", "Print the checkout URL without opening a browser").action(async (tier, opts) => {
|
|
540
|
-
const result = await apiRequest("/billing/subscribe/terminal-checkout", "POST", opts, { tier: String(tier).toLowerCase() });
|
|
541
|
-
if (opts.json) {
|
|
542
|
-
writeOutput(result, true);
|
|
543
|
-
return;
|
|
544
|
-
}
|
|
545
|
-
if (result.updated) {
|
|
546
|
-
writeOutput(result.message ?? `Switched to ${result.tier}.`, false);
|
|
547
|
-
return;
|
|
548
|
-
}
|
|
549
|
-
const opened = opts.open !== false && !!result.checkout_url && openExternalUrl(result.checkout_url);
|
|
550
|
-
const interval = result.billing_interval === "year" ? "year" : "month";
|
|
551
|
-
const amount = result.amount_usd ?? result.monthly_usd;
|
|
552
|
-
const credits = result.credits_per_interval ?? result.credits_per_month;
|
|
553
|
-
const perInterval = interval === "year" ? "/yr" : "/mo";
|
|
554
|
-
const creditsPer = interval === "year" ? "credits/yr" : "credits/mo";
|
|
555
|
-
const extras = [
|
|
556
|
-
result.credits_never_expire ? "credits never expire" : null,
|
|
557
|
-
result.includes_memory ? "includes Memory Pro" : null,
|
|
558
|
-
result.intro
|
|
559
|
-
].filter(Boolean).join(" \xB7 ");
|
|
560
|
-
writeOutput([
|
|
561
|
-
`Plan: ${result.label ?? tier} \u2014 $${amount}${perInterval} \xB7 ${credits?.toLocaleString()} ${creditsPer} \xB7 ${result.concurrency} concurrency${extras ? ` \xB7 ${extras}` : ""}`,
|
|
562
|
-
opened ? `Opened checkout: ${result.checkout_url}` : `Checkout URL: ${result.checkout_url}`,
|
|
563
|
-
result.next_step ?? "Complete payment in the browser."
|
|
564
|
-
].join("\n"), false);
|
|
565
|
-
});
|
|
566
|
-
const agent = program.command("agent").description("Generate AI-agent install configs and workflow prompts.");
|
|
567
|
-
agent.command("install <host>").description("Print or apply install/config instructions for Codex, Claude, Cursor, Windsurf, Cline, or a handoff-only MCP host.").option("--api-key <key>", "API key to place in generated config").option("--package <spec>", "npm package spec", "mcp-scraper@latest").option("--browser-profile <name>", "Default saved browser profile for browser_open sessions.").option("--save-browser-profile-changes", "Persist cookies and browser storage back to the named browser profile when sessions close").option("--apply", "Apply the config to this client when supported. For Claude Code, this upserts the user-scope mcp-scraper server.").option("--json", "Print machine-readable JSON").action(async (hostInput, opts) => {
|
|
568
|
-
const host = normalizeAgentHost(hostInput);
|
|
569
|
-
const apiKey = opts.apiKey ?? process.env.MCP_SCRAPER_API_KEY;
|
|
570
|
-
if (opts.apply) {
|
|
571
|
-
if (host !== "claude") throw new Error('--apply is currently supported for Claude Code only. Use host "claude" or "claude-code".');
|
|
572
|
-
if (!apiKey?.trim()) {
|
|
573
|
-
throw new Error("MCP_SCRAPER_API_KEY is required for --apply. Set it in the environment or pass --api-key.");
|
|
574
|
-
}
|
|
575
|
-
const configOptions = {
|
|
576
|
-
apiKey,
|
|
577
|
-
packageSpec: opts.package,
|
|
578
|
-
browserProfileName: opts.browserProfile,
|
|
579
|
-
browserProfileSaveChanges: opts.saveBrowserProfileChanges
|
|
580
|
-
};
|
|
581
|
-
const existing = await runLocalCommand("claude", claudeMcpGetArgs());
|
|
582
|
-
const snapshot = existing.code === 0 ? parseClaudeMcpGet(existing.stdout) : null;
|
|
583
|
-
const remove = await runLocalCommand("claude", claudeMcpRemoveArgs());
|
|
584
|
-
const add = await runLocalCommand("claude", claudeMcpAddArgs(configOptions));
|
|
585
|
-
if (add.code !== 0) {
|
|
586
|
-
const reason = maskSecrets([add.stderr, add.stdout].filter(Boolean).join("\n").trim());
|
|
587
|
-
let rollback;
|
|
588
|
-
if (remove.code !== 0) {
|
|
589
|
-
rollback = "No existing entry was removed, so your configuration is unchanged.";
|
|
590
|
-
} else if (!snapshot) {
|
|
591
|
-
rollback = "WARNING: the previous mcp-scraper entry was removed and could not be captured for rollback. Re-add it manually.";
|
|
592
|
-
} else {
|
|
593
|
-
const restore = await runLocalCommand("claude", claudeMcpRestoreArgs(snapshot));
|
|
594
|
-
rollback = restore.code === 0 ? "Your previous mcp-scraper entry was restored; nothing was lost." : "WARNING: the previous mcp-scraper entry was removed and could NOT be restored. Re-add it with:\n claude " + claudeMcpRestoreArgs(snapshot).join(" ");
|
|
595
|
-
}
|
|
596
|
-
throw new Error([
|
|
597
|
-
"Claude Code MCP registration failed.",
|
|
598
|
-
reason || "No error output returned.",
|
|
599
|
-
rollback,
|
|
600
|
-
"Make sure Claude Code is installed and the `claude` command is on PATH."
|
|
601
|
-
].join("\n"));
|
|
602
|
-
}
|
|
603
|
-
const list = await runLocalCommand("claude", ["mcp", "list"]);
|
|
604
|
-
const result = {
|
|
605
|
-
host: "claude",
|
|
606
|
-
applied: true,
|
|
607
|
-
replacedExisting: remove.code === 0,
|
|
608
|
-
command: "npx",
|
|
609
|
-
args: combinedNpxArgs(configOptions),
|
|
610
|
-
nextStep: "Fully exit Claude Code, start a new Claude terminal, then run: claude mcp list",
|
|
611
|
-
list: maskSecrets([list.stdout, list.stderr].filter(Boolean).join("\n").trim())
|
|
612
|
-
};
|
|
613
|
-
if (opts.json) {
|
|
614
|
-
writeOutput(result, true);
|
|
615
|
-
return;
|
|
616
|
-
}
|
|
617
|
-
writeOutput([
|
|
618
|
-
"Applied Claude Code MCP config: mcp-scraper",
|
|
619
|
-
remove.code === 0 ? "Replaced existing mcp-scraper entry." : "No existing mcp-scraper entry found; added a new one.",
|
|
620
|
-
"Command: npx " + combinedNpxArgs(configOptions).join(" "),
|
|
621
|
-
opts.browserProfile ? `Browser profile: ${opts.browserProfile}` : "",
|
|
622
|
-
"",
|
|
623
|
-
"Next step: fully exit Claude Code, start a new Claude terminal, then run:",
|
|
624
|
-
" claude mcp list",
|
|
625
|
-
"",
|
|
626
|
-
result.list ? `Current Claude MCP list:
|
|
627
|
-
${result.list}` : ""
|
|
628
|
-
].filter(Boolean).join("\n"), false);
|
|
629
|
-
return;
|
|
630
|
-
}
|
|
631
|
-
const text = renderAgentInstall(host, {
|
|
632
|
-
apiKey: opts.apiKey,
|
|
633
|
-
packageSpec: opts.package,
|
|
634
|
-
browserProfileName: opts.browserProfile,
|
|
635
|
-
browserProfileSaveChanges: opts.saveBrowserProfileChanges
|
|
636
|
-
});
|
|
637
|
-
writeOutput(opts.json ? {
|
|
638
|
-
host,
|
|
639
|
-
protocolVersion: XRAY_INSTALL_PROTOCOL_VERSION,
|
|
640
|
-
protocolDigest: XRAY_INSTALL_PROTOCOL_DIGEST,
|
|
641
|
-
text
|
|
642
|
-
} : text, opts.json);
|
|
643
|
-
});
|
|
644
|
-
agent.command("prompt [name]").description("Print an agent prompt template.").option("--json", "Print machine-readable JSON").action((name, opts) => {
|
|
645
|
-
if (!name) {
|
|
646
|
-
writeOutput(opts.json ? { prompts: listPrompts() } : listPrompts().join("\n"), opts.json);
|
|
647
|
-
return;
|
|
648
|
-
}
|
|
649
|
-
const text = renderPrompt(name);
|
|
650
|
-
writeOutput(opts.json ? {
|
|
651
|
-
name,
|
|
652
|
-
...name === "xray-setup" ? {
|
|
653
|
-
protocolVersion: XRAY_INSTALL_PROTOCOL_VERSION,
|
|
654
|
-
protocolDigest: XRAY_INSTALL_PROTOCOL_DIGEST
|
|
655
|
-
} : {},
|
|
656
|
-
text
|
|
657
|
-
} : text, opts.json);
|
|
658
|
-
});
|
|
659
|
-
const workflow = program.command("workflow").description("Run named SEO workflows.");
|
|
660
|
-
workflow.command("list").description("List available workflows.").option("--json", "Print machine-readable JSON").action((opts) => {
|
|
661
|
-
const rows = listWorkflowDefinitions();
|
|
662
|
-
if (opts.json) writeOutput({ workflows: rows }, true);
|
|
663
|
-
else writeOutput(rows.map((row) => `${row.id} ${row.title}
|
|
664
|
-
${row.description}`).join("\n"), false);
|
|
665
|
-
});
|
|
666
|
-
workflow.command("run <id>").description("Run a workflow and save local artifacts.").option("--api-key <key>", "MCP Scraper API key").option("--api-url <url>", "MCP Scraper API URL", "https://mcpscraper.dev").option("--output-dir <path>", "Workflow output directory").option("--json", "Print machine-readable JSON").option("--keyword <keyword>", "Agent packet keyword").option("--domain <domain>", "Target domain").option("--url <url>", "Target page URL").option("--location <location>", "Target location").option("--max-results <n>", "Maximum SERP or Maps results").option("--max-questions <n>", "Maximum PAA questions").option("--extract-top <n>", "Top result pages or citation pages to extract").option("--depth <n>", "PAA expansion depth").option("--no-serp", "Skip SERP evidence for agent packet").option("--no-paa", "Skip PAA evidence for agent packet").option("--query <query>", "Business category or workflow query").option("--state <state>", "US state").option("--min-pop <n>", "Minimum city population").option("--max-cities <n>", "Maximum selected cities").option("--per-city <n>", "Maps results per city").option("--concurrency <n>", "City search concurrency").option("--proxy-mode <mode>", "Proxy mode: location, configured, or none").option("--hydrate-top <n>", "Profiles to hydrate per city for competitive audits").option("--reviews <n>", "Review cards to collect per hydrated profile").option("--no-return-partial", "Fail instead of writing partial artifacts").action(async (id, opts) => {
|
|
667
|
-
const summary = await runWorkflow(id, workflowInput(id, opts), {
|
|
668
|
-
apiKey: opts.apiKey,
|
|
669
|
-
apiUrl: opts.apiUrl,
|
|
670
|
-
outputDir: opts.outputDir
|
|
671
|
-
});
|
|
672
|
-
if (opts.json) writeOutput(summary, true);
|
|
673
|
-
else {
|
|
674
|
-
writeOutput([
|
|
675
|
-
`${summary.title}: ${summary.status}`,
|
|
676
|
-
summary.summary,
|
|
677
|
-
summary.reportPath ? `Report: ${summary.reportPath}` : "",
|
|
678
|
-
summary.warnings.length ? `Warnings:
|
|
679
|
-
${summary.warnings.map((w) => `- ${w}`).join("\n")}` : ""
|
|
680
|
-
].filter(Boolean).join("\n"), false);
|
|
681
|
-
}
|
|
682
|
-
});
|
|
683
|
-
const report = program.command("report").description("List and open local workflow reports.");
|
|
684
|
-
report.command("list").description("List recent workflow reports.").option("--output-dir <path>", "Workflow output directory").option("--json", "Print machine-readable JSON").action(async (opts) => {
|
|
685
|
-
const reports = await listWorkflowReports(opts.outputDir);
|
|
686
|
-
writeOutput(opts.json ? { reports } : reports.map((r) => `${r.startedAt} ${r.workflow} ${r.status} ${r.reportPath ?? r.manifestPath}`).join("\n"), opts.json);
|
|
687
|
-
});
|
|
688
|
-
report.command("path [id]").description("Print a report path. Defaults to last.").option("--output-dir <path>", "Workflow output directory").option("--json", "Print machine-readable JSON").action(async (id = "last", opts) => {
|
|
689
|
-
const found = await findWorkflowReport(id, opts.outputDir);
|
|
690
|
-
if (!found?.reportPath) throw new Error(`No report found for "${id}"`);
|
|
691
|
-
writeOutput(opts.json ? { path: found.reportPath, run: found } : found.reportPath, opts.json);
|
|
692
|
-
});
|
|
693
|
-
report.command("open [id]").description("Open a report. Defaults to last.").option("--output-dir <path>", "Workflow output directory").option("--json", "Print machine-readable JSON instead of opening").action(async (id = "last", opts) => {
|
|
694
|
-
if (opts.json) {
|
|
695
|
-
const found = await findWorkflowReport(id, opts.outputDir);
|
|
696
|
-
if (!found?.reportPath) throw new Error(`No report found for "${id}"`);
|
|
697
|
-
writeOutput({ path: found.reportPath, run: found }, true);
|
|
698
|
-
return;
|
|
699
|
-
}
|
|
700
|
-
const path = await openWorkflowReport(id, opts.outputDir);
|
|
701
|
-
writeOutput(`Opened: ${path}`, false);
|
|
702
|
-
});
|
|
703
|
-
const schedule = program.command("schedule").description("Create and manage hosted workflow schedules.");
|
|
704
|
-
addWorkflowInputOptions(schedule.command("create <workflowId>").description("Create a recurring hosted workflow schedule.").option("--api-key <key>", "MCP Scraper API key").option("--api-url <url>", "MCP Scraper API URL", "https://mcpscraper.dev").option("--name <name>", "Schedule name").option("--daily", "Run daily").option("--weekly", "Run weekly").option("--monthly", "Run monthly").option("--timezone <tz>", "Schedule timezone", "UTC").option("--webhook <url>", "HTTPS webhook URL").option("--next-run-at <iso>", "First run time as an ISO timestamp").option("--json", "Print machine-readable JSON")).action(async (workflowId, opts) => {
|
|
705
|
-
const result = await apiRequest("/workflows/schedules", "POST", opts, {
|
|
706
|
-
workflowId,
|
|
707
|
-
name: opts.name,
|
|
708
|
-
input: workflowInput(workflowId, opts),
|
|
709
|
-
cadence: cadenceOpt(opts),
|
|
710
|
-
timezone: opts.timezone,
|
|
711
|
-
webhookUrl: opts.webhook,
|
|
712
|
-
nextRunAt: opts.nextRunAt
|
|
713
|
-
});
|
|
714
|
-
writeOutput(result, opts.json);
|
|
715
|
-
});
|
|
716
|
-
schedule.command("list").description("List hosted workflow schedules.").option("--api-key <key>", "MCP Scraper API key").option("--api-url <url>", "MCP Scraper API URL", "https://mcpscraper.dev").option("--json", "Print machine-readable JSON").action(async (opts) => {
|
|
717
|
-
const result = await apiRequest("/workflows/schedules", "GET", opts);
|
|
718
|
-
if (opts.json) writeOutput(result, true);
|
|
719
|
-
else writeOutput(result.schedules.map((scheduleRow) => `${scheduleRow.id} ${scheduleRow.status} ${scheduleRow.workflow_id} ${scheduleRow.next_run_at ?? ""}`).join("\n"), false);
|
|
720
|
-
});
|
|
721
|
-
schedule.command("pause <id>").description("Pause a hosted workflow schedule.").option("--api-key <key>", "MCP Scraper API key").option("--api-url <url>", "MCP Scraper API URL", "https://mcpscraper.dev").option("--json", "Print machine-readable JSON").action(async (id, opts) => writeOutput(await apiRequest(`/workflows/schedules/${id}`, "PATCH", opts, { status: "paused" }), opts.json));
|
|
722
|
-
schedule.command("resume <id>").description("Resume a hosted workflow schedule.").option("--api-key <key>", "MCP Scraper API key").option("--api-url <url>", "MCP Scraper API URL", "https://mcpscraper.dev").option("--json", "Print machine-readable JSON").action(async (id, opts) => writeOutput(await apiRequest(`/workflows/schedules/${id}`, "PATCH", opts, { status: "active" }), opts.json));
|
|
723
|
-
schedule.command("delete <id>").description("Delete a hosted workflow schedule.").option("--api-key <key>", "MCP Scraper API key").option("--api-url <url>", "MCP Scraper API URL", "https://mcpscraper.dev").option("--json", "Print machine-readable JSON").action(async (id, opts) => writeOutput(await apiRequest(`/workflows/schedules/${id}`, "DELETE", opts), opts.json));
|
|
724
|
-
schedule.command("run <id>").description("Run a hosted workflow schedule now.").option("--api-key <key>", "MCP Scraper API key").option("--api-url <url>", "MCP Scraper API URL", "https://mcpscraper.dev").option("--json", "Print machine-readable JSON").action(async (id, opts) => writeOutput(await apiRequest(`/workflows/schedules/${id}/run`, "POST", opts, {}), opts.json));
|
|
725
|
-
const runs = program.command("runs").description("Inspect and download hosted workflow runs.");
|
|
726
|
-
runs.command("list").description("List hosted workflow runs.").option("--api-key <key>", "MCP Scraper API key").option("--api-url <url>", "MCP Scraper API URL", "https://mcpscraper.dev").option("--json", "Print machine-readable JSON").action(async (opts) => {
|
|
727
|
-
const result = await apiRequest("/workflows/runs", "GET", opts);
|
|
728
|
-
if (opts.json) writeOutput(result, true);
|
|
729
|
-
else writeOutput(result.runs.map((run) => `${run.id} ${run.status} ${run.workflow_id} ${run.queued_at}`).join("\n"), false);
|
|
730
|
-
});
|
|
731
|
-
runs.command("status <id>").description("Show a hosted workflow run.").option("--api-key <key>", "MCP Scraper API key").option("--api-url <url>", "MCP Scraper API URL", "https://mcpscraper.dev").option("--json", "Print machine-readable JSON").action(async (id, opts) => writeOutput(await apiRequest(`/workflows/runs/${id}`, "GET", opts), opts.json));
|
|
732
|
-
runs.command("download <id>").description("Download hosted workflow run artifacts.").option("--api-key <key>", "MCP Scraper API key").option("--api-url <url>", "MCP Scraper API URL", "https://mcpscraper.dev").option("--output-dir <path>", "Download directory").option("--json", "Print machine-readable JSON").action(async (id, opts) => {
|
|
733
|
-
const result = await apiRequest(`/workflows/runs/${id}`, "GET", opts);
|
|
734
|
-
const { apiUrl, apiKey } = apiOptions(opts);
|
|
735
|
-
const outDir = join2(opts.outputDir ?? workflowOutputBaseDir(), "workflow-downloads", id);
|
|
736
|
-
await mkdir2(outDir, { recursive: true });
|
|
737
|
-
const downloaded = [];
|
|
738
|
-
for (const artifact of result.run.artifacts ?? []) {
|
|
739
|
-
const res = await fetch(`${apiUrl}/workflows/runs/${id}/artifacts/${artifact.id}`, { headers: { "x-api-key": apiKey } });
|
|
740
|
-
if (!res.ok) throw new Error(`Failed to download ${artifact.label}: HTTP ${res.status}`);
|
|
741
|
-
const file = join2(outDir, basename(artifact.path));
|
|
742
|
-
await writeFile(file, Buffer.from(await res.arrayBuffer()));
|
|
743
|
-
downloaded.push(file);
|
|
744
|
-
}
|
|
745
|
-
writeOutput(opts.json ? { runId: id, files: downloaded } : downloaded.join("\n"), opts.json);
|
|
746
|
-
});
|
|
747
|
-
return program;
|
|
748
|
-
}
|
|
749
|
-
async function runHumanCli(argv = process.argv) {
|
|
750
|
-
await buildHumanCli().parseAsync(argv);
|
|
751
|
-
}
|
|
752
|
-
|
|
753
|
-
// bin/mcp-scraper-cli.ts
|
|
754
|
-
runHumanCli().catch((err) => {
|
|
755
|
-
console.error(err instanceof Error ? err.message : String(err));
|
|
756
|
-
process.exit(1);
|
|
757
|
-
});
|
|
2
|
+
import{b as E,c as N,d as M,e as L,f as T,j as U}from"../chunk-XLWNEVUZ.js";import"../chunk-KJQXUZ4Y.js";import"../chunk-3GP5CYZX.js";import{g as v,j as b,k as D,l as K,m as H}from"../chunk-WO3N5FH2.js";import"../chunk-HE45FFBU.js";import{a as P}from"../chunk-Z2QGQJS2.js";import{Command as he}from"commander";import{spawn as ne}from"child_process";import{mkdir as ke,writeFile as Pe}from"fs/promises";import{basename as Ce,join as Z}from"path";function se(e){return e.apiKey?.trim()||"sk_live_your_key"}function ce(e){return e.packageSpec?.trim()||"mcp-scraper@latest"}function A(e={}){return["-y","--package",ce(e),"mcp-scraper"]}function pe(e){let n={MCP_SCRAPER_API_KEY:se(e)},c=e.browserProfileName?.trim();return c&&(n.BROWSER_AGENT_PROFILE_NAME=c),e.browserProfileSaveChanges===!0&&(n.BROWSER_AGENT_PROFILE_SAVE_CHANGES="true"),n}function q(){return["mcp","remove","mcp-scraper","-s","user"]}function J(){return["mcp","get","mcp-scraper"]}function B(e){let n=e.match(/^\s*Command:\s*(.+?)\s*$/m)?.[1];if(!n)return null;let c=e.match(/^\s*Args:\s*(.*?)\s*$/m)?.[1]??"",i=c.length?c.split(/\s+/):[],p={},u=e.split(/^\s*Environment:\s*$/m)[1];if(u)for(let a of u.split(`
|
|
3
|
+
`)){let l=a.match(/^\s{2,}([A-Za-z_][A-Za-z0-9_]*)=(.*)$/);if(!l){if(a.trim().length&&!/^\s{2,}/.test(a))break;continue}p[l[1]]=l[2]}return{command:n,args:i,env:p}}function j(e){let n=["mcp","add","mcp-scraper","--scope","user"];for(let[c,i]of Object.entries(e.env))n.push("--env",`${c}=${i}`);return n.push("--",e.command,...e.args),n}function G(e={}){let n=["mcp","add","mcp-scraper","--scope","user"];for(let[c,i]of Object.entries(pe(e)))n.push("--env",`${c}=${i}`);return n.push("--","npx",...A(e)),n}function O(e){if(e==="claude-code")return"claude";if(e==="claude"||D.hosts.some(n=>n.id===e))return e;throw new Error('Unknown host "'+e+'". Use: codex, claude, claude-code, claude-desktop, cursor, windsurf, cline, or user-action-only')}function ue(e){return K(e==="claude"?"claude-code":e)}function W(e,n={}){let c=O(e),i=ue(c),p="Restart the MCP client so it starts a fresh npx process.",u='MCP_SCRAPER_API_KEY="$MCP_SCRAPER_API_KEY" npx -y -p mcp-scraper@latest mcp-scraper-cli agent install claude --apply',a=`X-Ray install protocol: ${v} (${b})`;return c==="codex"?["# Codex MCP config",a,i.exactConfig,"",`Continuation: ${i.continuation}`,`Rollback: ${i.rollback}`,"",p].join(`
|
|
4
|
+
`):c==="claude"?["# Claude Code command",a,i.exactConfig,"","# One-command Claude Code setup",u,"",`Continuation: ${i.continuation}`,`Rollback: ${i.rollback}`,"",p].join(`
|
|
5
|
+
`):c==="claude-desktop"?["# Claude Desktop config",a,i.exactConfig,"","Desktop Extension: https://mcpscraper.dev/downloads/mcp-scraper.mcpb",`Continuation: ${i.continuation}`,`Rollback: ${i.rollback}`,p].join(`
|
|
6
|
+
`):[`# ${i.label} MCP config`,a,i.exactConfig,"",`Continuation: ${i.continuation}`,`Rollback: ${i.rollback}`,...i.restartRequired?["",p]:[]].join(`
|
|
7
|
+
`)}import{access as le,mkdir as de,readFile as me}from"fs/promises";import{homedir as Y}from"os";import{join as F}from"path";function fe(e,n=!1){return e?"pass":n?"warn":"fail"}function ge(e){return Number(e.replace(/^v/,"").split(".")[0]??0)}async function we(){let e=process.env.MCP_SCRAPER_KEY_PATH?.trim()||F(Y(),".mcp-scraper-key");try{return(await me(e,"utf8")).trim()||null}catch{return null}}async function ye(e){try{let n=await e("https://registry.npmjs.org/mcp-scraper/latest",{signal:AbortSignal.timeout(5e3)});return n.ok?(await n.json()).version??null:null}catch{return null}}async function z(e={}){let n=e.fetchImpl??fetch,c=(e.apiUrl??process.env.MCP_SCRAPER_API_URL??"https://mcpscraper.dev").replace(/\/$/,""),i=e.outputDir??process.env.MCP_SCRAPER_OUTPUT_DIR??F(Y(),"Downloads","mcp-scraper"),p=e.apiKey?.trim()||process.env.MCP_SCRAPER_API_KEY?.trim()||await we(),u=await ye(n),a=[],l=ge(process.version);a.push({id:"node",label:"Node.js",status:fe(l>=20),detail:process.version,fix:l>=20?void 0:"Install Node.js 20 or newer."}),a.push({id:"package_version",label:"Local package",status:u&&u!==P?"warn":"pass",detail:u?`local ${P}, npm latest ${u}`:`local ${P}, npm latest unavailable`,fix:u&&u!==P?"Use mcp-scraper@latest and restart the MCP client.":void 0}),a.push({id:"api_key",label:"API key",status:p?"pass":"warn",detail:p?"configured":"not configured",fix:p?void 0:"Set MCP_SCRAPER_API_KEY or pass --api-key."});try{await de(i,{recursive:!0}),await le(i),a.push({id:"output_dir",label:"Output directory",status:"pass",detail:i})}catch(t){a.push({id:"output_dir",label:"Output directory",status:"fail",detail:i,fix:t instanceof Error?t.message:"Create a writable output directory."})}if(p)try{let t=await n(`${c}/me`,{headers:{"x-api-key":p},signal:AbortSignal.timeout(8e3)});a.push({id:"api_reachability",label:"Hosted API",status:t.ok?"pass":"fail",detail:`${c}/me returned ${t.status}`,fix:t.ok?void 0:"Verify the API key and account status."})}catch(t){a.push({id:"api_reachability",label:"Hosted API",status:"fail",detail:t instanceof Error?t.message:String(t),fix:"Check network access and MCP_SCRAPER_API_URL."})}else a.push({id:"api_reachability",label:"Hosted API",status:"skip",detail:"skipped because no API key is configured"});return a.push({id:"mcp_config",label:"MCP command",status:"pass",detail:`npx ${A().join(" ")}`,fix:"Restart the MCP client after package updates."}),{ok:a.every(t=>t.status==="pass"||t.status==="skip"),version:P,npmLatest:u,checks:a,recommendedConfig:{command:"npx",args:A(),env:{MCP_SCRAPER_API_KEY:p?"$MCP_SCRAPER_API_KEY":"sk_live_your_key"}}}}function V(e){let n={pass:"PASS",warn:"WARN",fail:"FAIL",skip:"SKIP"};return[`mcp-scraper doctor v${e.version}`,"",...e.checks.map(i=>{let p=i.fix?`
|
|
8
|
+
Fix: ${i.fix}`:"";return`[${n[i.status]}] ${i.label}: ${i.detail}${p}`}),"",`Recommended MCP command: npx ${e.recommendedConfig.args.join(" ")}`].join(`
|
|
9
|
+
`)}var X={"xray-setup":H(),"agent-packet":["# MCP Scraper Agent Packet Prompt","","Use MCP Scraper as the evidence layer. Run the agent-packet workflow for the target keyword/domain, then treat `evidence.json`, `sources.csv`, and `competitors.csv` as source of truth.","","Do not invent citations. If a recommendation is not supported by the packet, mark it as an assumption. Turn the evidence into a concise SEO brief, an implementation task list, and content recommendations tied to source rows."].join(`
|
|
10
|
+
`),"local-competitive-audit":["# MCP Scraper Local Competitive Audit Prompt","","Run the local-competitive-audit workflow for the niche and markets. Use the city summary, competitor CSV, and review-insight CSV to identify market difficulty, review themes, GBP category patterns, and weak competitors.","","Ground recommendations in Maps rank position, review count, star rating, categories, profile details, and review topics. Do not overstate heuristic opportunity scores."].join(`
|
|
11
|
+
`),"directory-workflow":["# MCP Scraper Directory Workflow Prompt","","Use directory_workflow when the user wants cities selected by population and Google Maps candidates per city. Keep query and location separate. Preserve result_position, source_location, review stars/count, categories, and profile URLs for downstream CSV or directory use."].join(`
|
|
12
|
+
`),"map-comparison":["# MCP Scraper Maps Comparison Prompt","","Run the map-comparison workflow when the user wants to compare local Maps competitors in a city or across selected markets. Use `maps-results.csv`, `map-comparison.csv`, and `profile-insights.csv` as source of truth.","","Ground recommendations in result_position, review_stars, review_count, category, website presence, review topics, and profile attributes. Treat review gaps and missing websites as opportunity signals, not guaranteed ranking factors."].join(`
|
|
13
|
+
`),"serp-comparison":["# MCP Scraper SERP Comparison Prompt","","Run the serp-comparison workflow when the user wants to know why competitors outrank a page or what the SERP rewards. Use organic results, extracted page headings, PAA questions, and AI Overview citations before making recommendations.","","Tie each recommendation to `content-gaps.csv`, `page-comparison.csv`, `paa-questions.csv`, or `ai-overview-citations.csv`. Do not invent missing sections or citations."].join(`
|
|
14
|
+
`),"paa-expansion-brief":["# MCP Scraper PAA Expansion Brief Prompt","","Run the paa-expansion-brief workflow when the user wants to figure out what to write from People Also Ask expansion. Use `section-map.csv` to structure the brief and `paa-questions.csv` for exact customer-language headings.","","Answer the highest-priority questions directly, preserve source URLs when present, and separate evidence-backed sections from assumptions."].join(`
|
|
15
|
+
`),"ai-overview-language":["# MCP Scraper AI Overview Language Prompt","","Run the ai-overview-language workflow when the user wants to know how to phrase content for AI Overview inclusion. Use `claim-patterns.csv`, `language-guidance.csv`, `ai-overview-citations.csv`, and PAA follow-ups as evidence.","","Recommend concise answer blocks, criteria/step language, citation hooks, and follow-up sections. Do not claim the target will be cited; frame the output as evidence-based language guidance."].join(`
|
|
16
|
+
`),"ai-citation-monitor":["# MCP Scraper AI Citation Monitor Prompt","","Use search_serp and harvest_paa to check whether the target brand/domain appears in AI Overview citations, organic results, People Also Ask sources, local pack, forums, and videos. Save evidence before summarizing trends or gaps."].join(`
|
|
17
|
+
`),"serp-brief":["# MCP Scraper SERP Brief Prompt","","Use live SERP, PAA, AI Overview, forum, video, and source evidence to create a writer brief. Tie each recommended section to evidence rows and identify missing proof, entities, comparisons, and customer questions."].join(`
|
|
18
|
+
`)};function _(){return Object.keys(X).sort()}function Q(e){let n=X[e];if(!n)throw new Error(`Unknown prompt "${e}". Available: ${_().join(", ")}`);return n}function f(e){if(e==null||e==="")return;let n=Number(e);return Number.isFinite(n)?n:void 0}function Ae(e){if(e!==void 0){if(typeof e=="boolean")return e;if(e==="true")return!0;if(e==="false")return!1}}function x(e){return Object.fromEntries(Object.entries(e).filter(([,n])=>n!==void 0))}function ee(e,n){return x(e==="agent-packet"||e==="serp-comparison"||e==="ai-overview-language"?{keyword:n.keyword,domain:n.domain,url:n.url,location:n.location,maxResults:f(n.maxResults),maxQuestions:f(n.maxQuestions),extractTop:f(n.extractTop),includeSerp:n.serp===!1?!1:void 0,includePaa:n.paa===!1?!1:void 0,includeAiOverview:Ae(n.includeAiOverview),returnPartial:n.returnPartial===!1?!1:void 0}:e==="paa-expansion-brief"?{keyword:n.keyword,location:n.location,maxQuestions:f(n.maxQuestions),depth:f(n.depth),returnPartial:n.returnPartial===!1?!1:void 0}:e==="directory"||e==="local-competitive-audit"||e==="map-comparison"?{query:n.query,location:n.location,state:n.state,minPopulation:f(n.minPop??n.minPopulation),maxCities:f(n.maxCities),maxResultsPerCity:f(n.perCity??n.maxResultsPerCity??n.maxResults),concurrency:f(n.concurrency),proxyMode:n.proxyMode,hydrateTop:e==="local-competitive-audit"||e==="map-comparison"?f(n.hydrateTop):void 0,maxReviews:e==="local-competitive-audit"||e==="map-comparison"?f(n.reviews??n.maxReviews):void 0,returnPartial:n.returnPartial===!1?!1:void 0}:n)}function s(e,n){n?process.stdout.write(`${JSON.stringify(e,null,2)}
|
|
19
|
+
`):typeof e=="string"?process.stdout.write(`${e}
|
|
20
|
+
`):process.stdout.write(`${JSON.stringify(e,null,2)}
|
|
21
|
+
`)}function re(e){return e.replace(/sk_[A-Za-z0-9_-]+/g,"sk_***")}function S(e,n){return new Promise(c=>{let i=ne(e,n,{stdio:["ignore","pipe","pipe"]}),p=[],u=[],a=!1,l=t=>{a||(a=!0,c(t))};i.stdout.on("data",t=>p.push(Buffer.from(t))),i.stderr.on("data",t=>u.push(Buffer.from(t))),i.on("error",t=>{l({code:127,stdout:"",stderr:t.message})}),i.on("close",t=>{l({code:typeof t=="number"?t:1,stdout:Buffer.concat(p).toString("utf8"),stderr:Buffer.concat(u).toString("utf8")})})})}function ve(e){return e.daily?"daily":e.monthly?"monthly":"weekly"}function oe(e){let n=String(e.apiKey??process.env.MCP_SCRAPER_API_KEY??"").trim();if(!n)throw new Error("MCP_SCRAPER_API_KEY is required. Pass --api-key or set the environment variable.");return{apiUrl:String(e.apiUrl??process.env.MCP_SCRAPER_API_URL??"https://mcpscraper.dev").replace(/\/$/,""),apiKey:n}}function te(e){let n=process.platform==="darwin"?"open":process.platform==="win32"?"cmd":"xdg-open",c=process.platform==="win32"?["/c","start","",e]:[e];try{return ne(n,c,{detached:!0,stdio:"ignore"}).unref(),!0}catch{return!1}}async function m(e,n,c,i){let{apiUrl:p,apiKey:u}=oe(c),a=await fetch(`${p}${e}`,{method:n,headers:{"Content-Type":"application/json","x-api-key":u},body:i==null?void 0:JSON.stringify(i)}),l=await a.json().catch(()=>({}));if(!a.ok){let t=typeof l.error=="string"?l.error:`API request failed with ${a.status}`;throw new Error(t)}return l}function be(e){return e.option("--keyword <keyword>","Agent packet keyword").option("--domain <domain>","Target domain").option("--url <url>","Target page URL").option("--location <location>","Target location").option("--max-results <n>","Maximum SERP or Maps results").option("--max-questions <n>","Maximum PAA questions").option("--extract-top <n>","Top result pages or citation pages to extract").option("--depth <n>","PAA expansion depth").option("--no-serp","Skip SERP evidence for agent packet").option("--no-paa","Skip PAA evidence for agent packet").option("--query <query>","Business category or workflow query").option("--state <state>","US state").option("--min-pop <n>","Minimum city population").option("--max-cities <n>","Maximum selected cities").option("--per-city <n>","Maps results per city").option("--concurrency <n>","City search concurrency").option("--proxy-mode <mode>","Proxy mode: location, configured, or none").option("--hydrate-top <n>","Profiles to hydrate per city for competitive audits").option("--reviews <n>","Review cards to collect per hydrated profile").option("--no-return-partial","Fail instead of writing partial artifacts")}function Se(){let e=new he;e.name("mcp-scraper-cli").description("Human CLI for MCP Scraper setup, workflows, reports, and agent-ready SEO artifacts.").version(P),e.command("doctor").description("Check setup, API key, output directory, package version, and recommended MCP config.").option("--api-key <key>","MCP Scraper API key").option("--api-url <url>","MCP Scraper API URL","https://mcpscraper.dev").option("--output-dir <path>","Output directory to test").option("--json","Print machine-readable JSON").action(async t=>{let r=await z({apiKey:t.apiKey,apiUrl:t.apiUrl,outputDir:t.outputDir});s(t.json?r:V(r),t.json),r.ok||(process.exitCode=1)});let n=e.command("billing").description("Inspect billing and start checkout flows."),c=n.command("concurrency").description("Manage MCP Scraper concurrency packs.");c.command("info").description("Show current concurrency limit and concurrency-pack price.").option("--api-key <key>","MCP Scraper API key").option("--api-url <url>","MCP Scraper API URL","https://mcpscraper.dev").option("--json","Print machine-readable JSON").action(async t=>{let r=await m("/billing/credits","POST",t,{});if(t.json)s(r.concurrency,!0);else{let o=r.concurrency?.upgrade;s([`Current limit: ${r.concurrency?.current_limit??"unknown"} concurrent operations`,`Extra slots: ${r.concurrency?.current_extra_slots??"unknown"}`,`Concurrency pack: ${o?.price_label??"$5/month for 2 extra browsers"}`,`Upgrade command: ${o?.terminal_command??"mcp-scraper-cli billing concurrency checkout"}`].join(`
|
|
22
|
+
`),!1)}}),c.command("checkout").description("Create or update a hosted Stripe checkout for concurrency packs (two browsers per pack).").argument("[quantity]","Desired pack quantity","1").option("--api-key <key>","MCP Scraper API key").option("--api-url <url>","MCP Scraper API URL","https://mcpscraper.dev").option("--json","Print machine-readable JSON").option("--no-open","Print the checkout URL without opening a browser").action(async(t,r)=>{let o=await m("/billing/concurrency/terminal-checkout","POST",r,{quantity:Number(t)});if(r.json){s(o,!0);return}let d=!!o.checkout_url&&r.open!==!1&&te(o.checkout_url);s([`Concurrency pack: ${o.price?.price_label??"$5/month for 2 extra browsers"}`,o.updated?"Concurrency pack quantity updated.":d?`Opened checkout: ${o.checkout_url}`:`Checkout URL: ${o.checkout_url}`,o.next_step??"Complete checkout, then retry the MCP request."].join(`
|
|
23
|
+
`),!1)}),n.command("subscribe <tier>").description("Subscribe to a plan (starter | growth | scale) via a hosted Stripe checkout link.").option("--api-key <key>","MCP Scraper API key").option("--api-url <url>","MCP Scraper API URL","https://mcpscraper.dev").option("--json","Print machine-readable JSON").option("--no-open","Print the checkout URL without opening a browser").action(async(t,r)=>{let o=await m("/billing/subscribe/terminal-checkout","POST",r,{tier:String(t).toLowerCase()});if(r.json){s(o,!0);return}if(o.updated){s(o.message??`Switched to ${o.tier}.`,!1);return}let d=r.open!==!1&&!!o.checkout_url&&te(o.checkout_url),C=o.billing_interval==="year"?"year":"month",h=o.amount_usd??o.monthly_usd,k=o.credits_per_interval??o.credits_per_month,g=C==="year"?"/yr":"/mo",w=C==="year"?"credits/yr":"credits/mo",y=[o.credits_never_expire?"credits never expire":null,o.includes_memory?"includes Memory Pro":null,o.intro].filter(Boolean).join(" \xB7 ");s([`Plan: ${o.label??t} \u2014 $${h}${g} \xB7 ${k?.toLocaleString()} ${w} \xB7 ${o.concurrency} concurrency${y?` \xB7 ${y}`:""}`,d?`Opened checkout: ${o.checkout_url}`:`Checkout URL: ${o.checkout_url}`,o.next_step??"Complete payment in the browser."].join(`
|
|
24
|
+
`),!1)});let i=e.command("agent").description("Generate AI-agent install configs and workflow prompts.");i.command("install <host>").description("Print or apply install/config instructions for Codex, Claude, Cursor, Windsurf, Cline, or a handoff-only MCP host.").option("--api-key <key>","API key to place in generated config").option("--package <spec>","npm package spec","mcp-scraper@latest").option("--browser-profile <name>","Default saved browser profile for browser_open sessions.").option("--save-browser-profile-changes","Persist cookies and browser storage back to the named browser profile when sessions close").option("--apply","Apply the config to this client when supported. For Claude Code, this upserts the user-scope mcp-scraper server.").option("--json","Print machine-readable JSON").action(async(t,r)=>{let o=O(t),d=r.apiKey??process.env.MCP_SCRAPER_API_KEY;if(r.apply){if(o!=="claude")throw new Error('--apply is currently supported for Claude Code only. Use host "claude" or "claude-code".');if(!d?.trim())throw new Error("MCP_SCRAPER_API_KEY is required for --apply. Set it in the environment or pass --api-key.");let h={apiKey:d,packageSpec:r.package,browserProfileName:r.browserProfile,browserProfileSaveChanges:r.saveBrowserProfileChanges},k=await S("claude",J()),g=k.code===0?B(k.stdout):null,w=await S("claude",q()),y=await S("claude",G(h));if(y.code!==0){let ae=re([y.stderr,y.stdout].filter(Boolean).join(`
|
|
25
|
+
`).trim()),R;throw w.code!==0?R="No existing entry was removed, so your configuration is unchanged.":g?R=(await S("claude",j(g))).code===0?"Your previous mcp-scraper entry was restored; nothing was lost.":`WARNING: the previous mcp-scraper entry was removed and could NOT be restored. Re-add it with:
|
|
26
|
+
claude `+j(g).join(" "):R="WARNING: the previous mcp-scraper entry was removed and could not be captured for rollback. Re-add it manually.",new Error(["Claude Code MCP registration failed.",ae||"No error output returned.",R,"Make sure Claude Code is installed and the `claude` command is on PATH."].join(`
|
|
27
|
+
`))}let I=await S("claude",["mcp","list"]),$={host:"claude",applied:!0,replacedExisting:w.code===0,command:"npx",args:A(h),nextStep:"Fully exit Claude Code, start a new Claude terminal, then run: claude mcp list",list:re([I.stdout,I.stderr].filter(Boolean).join(`
|
|
28
|
+
`).trim())};if(r.json){s($,!0);return}s(["Applied Claude Code MCP config: mcp-scraper",w.code===0?"Replaced existing mcp-scraper entry.":"No existing mcp-scraper entry found; added a new one.","Command: npx "+A(h).join(" "),r.browserProfile?`Browser profile: ${r.browserProfile}`:"","","Next step: fully exit Claude Code, start a new Claude terminal, then run:"," claude mcp list","",$.list?`Current Claude MCP list:
|
|
29
|
+
${$.list}`:""].filter(Boolean).join(`
|
|
30
|
+
`),!1);return}let C=W(o,{apiKey:r.apiKey,packageSpec:r.package,browserProfileName:r.browserProfile,browserProfileSaveChanges:r.saveBrowserProfileChanges});s(r.json?{host:o,protocolVersion:v,protocolDigest:b,text:C}:C,r.json)}),i.command("prompt [name]").description("Print an agent prompt template.").option("--json","Print machine-readable JSON").action((t,r)=>{if(!t){s(r.json?{prompts:_()}:_().join(`
|
|
31
|
+
`),r.json);return}let o=Q(t);s(r.json?{name:t,...t==="xray-setup"?{protocolVersion:v,protocolDigest:b}:{},text:o}:o,r.json)});let p=e.command("workflow").description("Run named SEO workflows.");p.command("list").description("List available workflows.").option("--json","Print machine-readable JSON").action(t=>{let r=T();t.json?s({workflows:r},!0):s(r.map(o=>`${o.id} ${o.title}
|
|
32
|
+
${o.description}`).join(`
|
|
33
|
+
`),!1)}),p.command("run <id>").description("Run a workflow and save local artifacts.").option("--api-key <key>","MCP Scraper API key").option("--api-url <url>","MCP Scraper API URL","https://mcpscraper.dev").option("--output-dir <path>","Workflow output directory").option("--json","Print machine-readable JSON").option("--keyword <keyword>","Agent packet keyword").option("--domain <domain>","Target domain").option("--url <url>","Target page URL").option("--location <location>","Target location").option("--max-results <n>","Maximum SERP or Maps results").option("--max-questions <n>","Maximum PAA questions").option("--extract-top <n>","Top result pages or citation pages to extract").option("--depth <n>","PAA expansion depth").option("--no-serp","Skip SERP evidence for agent packet").option("--no-paa","Skip PAA evidence for agent packet").option("--query <query>","Business category or workflow query").option("--state <state>","US state").option("--min-pop <n>","Minimum city population").option("--max-cities <n>","Maximum selected cities").option("--per-city <n>","Maps results per city").option("--concurrency <n>","City search concurrency").option("--proxy-mode <mode>","Proxy mode: location, configured, or none").option("--hydrate-top <n>","Profiles to hydrate per city for competitive audits").option("--reviews <n>","Review cards to collect per hydrated profile").option("--no-return-partial","Fail instead of writing partial artifacts").action(async(t,r)=>{let o=await U(t,ee(t,r),{apiKey:r.apiKey,apiUrl:r.apiUrl,outputDir:r.outputDir});r.json?s(o,!0):s([`${o.title}: ${o.status}`,o.summary,o.reportPath?`Report: ${o.reportPath}`:"",o.warnings.length?`Warnings:
|
|
34
|
+
${o.warnings.map(d=>`- ${d}`).join(`
|
|
35
|
+
`)}`:""].filter(Boolean).join(`
|
|
36
|
+
`),!1)});let u=e.command("report").description("List and open local workflow reports.");u.command("list").description("List recent workflow reports.").option("--output-dir <path>","Workflow output directory").option("--json","Print machine-readable JSON").action(async t=>{let r=await N(t.outputDir);s(t.json?{reports:r}:r.map(o=>`${o.startedAt} ${o.workflow} ${o.status} ${o.reportPath??o.manifestPath}`).join(`
|
|
37
|
+
`),t.json)}),u.command("path [id]").description("Print a report path. Defaults to last.").option("--output-dir <path>","Workflow output directory").option("--json","Print machine-readable JSON").action(async(t="last",r)=>{let o=await M(t,r.outputDir);if(!o?.reportPath)throw new Error(`No report found for "${t}"`);s(r.json?{path:o.reportPath,run:o}:o.reportPath,r.json)}),u.command("open [id]").description("Open a report. Defaults to last.").option("--output-dir <path>","Workflow output directory").option("--json","Print machine-readable JSON instead of opening").action(async(t="last",r)=>{if(r.json){let d=await M(t,r.outputDir);if(!d?.reportPath)throw new Error(`No report found for "${t}"`);s({path:d.reportPath,run:d},!0);return}let o=await L(t,r.outputDir);s(`Opened: ${o}`,!1)});let a=e.command("schedule").description("Create and manage hosted workflow schedules.");be(a.command("create <workflowId>").description("Create a recurring hosted workflow schedule.").option("--api-key <key>","MCP Scraper API key").option("--api-url <url>","MCP Scraper API URL","https://mcpscraper.dev").option("--name <name>","Schedule name").option("--daily","Run daily").option("--weekly","Run weekly").option("--monthly","Run monthly").option("--timezone <tz>","Schedule timezone","UTC").option("--webhook <url>","HTTPS webhook URL").option("--next-run-at <iso>","First run time as an ISO timestamp").option("--json","Print machine-readable JSON")).action(async(t,r)=>{let o=await m("/workflows/schedules","POST",r,{workflowId:t,name:r.name,input:ee(t,r),cadence:ve(r),timezone:r.timezone,webhookUrl:r.webhook,nextRunAt:r.nextRunAt});s(o,r.json)}),a.command("list").description("List hosted workflow schedules.").option("--api-key <key>","MCP Scraper API key").option("--api-url <url>","MCP Scraper API URL","https://mcpscraper.dev").option("--json","Print machine-readable JSON").action(async t=>{let r=await m("/workflows/schedules","GET",t);t.json?s(r,!0):s(r.schedules.map(o=>`${o.id} ${o.status} ${o.workflow_id} ${o.next_run_at??""}`).join(`
|
|
38
|
+
`),!1)}),a.command("pause <id>").description("Pause a hosted workflow schedule.").option("--api-key <key>","MCP Scraper API key").option("--api-url <url>","MCP Scraper API URL","https://mcpscraper.dev").option("--json","Print machine-readable JSON").action(async(t,r)=>s(await m(`/workflows/schedules/${t}`,"PATCH",r,{status:"paused"}),r.json)),a.command("resume <id>").description("Resume a hosted workflow schedule.").option("--api-key <key>","MCP Scraper API key").option("--api-url <url>","MCP Scraper API URL","https://mcpscraper.dev").option("--json","Print machine-readable JSON").action(async(t,r)=>s(await m(`/workflows/schedules/${t}`,"PATCH",r,{status:"active"}),r.json)),a.command("delete <id>").description("Delete a hosted workflow schedule.").option("--api-key <key>","MCP Scraper API key").option("--api-url <url>","MCP Scraper API URL","https://mcpscraper.dev").option("--json","Print machine-readable JSON").action(async(t,r)=>s(await m(`/workflows/schedules/${t}`,"DELETE",r),r.json)),a.command("run <id>").description("Run a hosted workflow schedule now.").option("--api-key <key>","MCP Scraper API key").option("--api-url <url>","MCP Scraper API URL","https://mcpscraper.dev").option("--json","Print machine-readable JSON").action(async(t,r)=>s(await m(`/workflows/schedules/${t}/run`,"POST",r,{}),r.json));let l=e.command("runs").description("Inspect and download hosted workflow runs.");return l.command("list").description("List hosted workflow runs.").option("--api-key <key>","MCP Scraper API key").option("--api-url <url>","MCP Scraper API URL","https://mcpscraper.dev").option("--json","Print machine-readable JSON").action(async t=>{let r=await m("/workflows/runs","GET",t);t.json?s(r,!0):s(r.runs.map(o=>`${o.id} ${o.status} ${o.workflow_id} ${o.queued_at}`).join(`
|
|
39
|
+
`),!1)}),l.command("status <id>").description("Show a hosted workflow run.").option("--api-key <key>","MCP Scraper API key").option("--api-url <url>","MCP Scraper API URL","https://mcpscraper.dev").option("--json","Print machine-readable JSON").action(async(t,r)=>s(await m(`/workflows/runs/${t}`,"GET",r),r.json)),l.command("download <id>").description("Download hosted workflow run artifacts.").option("--api-key <key>","MCP Scraper API key").option("--api-url <url>","MCP Scraper API URL","https://mcpscraper.dev").option("--output-dir <path>","Download directory").option("--json","Print machine-readable JSON").action(async(t,r)=>{let o=await m(`/workflows/runs/${t}`,"GET",r),{apiUrl:d,apiKey:C}=oe(r),h=Z(r.outputDir??E(),"workflow-downloads",t);await ke(h,{recursive:!0});let k=[];for(let g of o.run.artifacts??[]){let w=await fetch(`${d}/workflows/runs/${t}/artifacts/${g.id}`,{headers:{"x-api-key":C}});if(!w.ok)throw new Error(`Failed to download ${g.label}: HTTP ${w.status}`);let y=Z(h,Ce(g.path));await Pe(y,Buffer.from(await w.arrayBuffer())),k.push(y)}s(r.json?{runId:t,files:k}:k.join(`
|
|
40
|
+
`),r.json)}),e}async function ie(e=process.argv){await Se().parseAsync(e)}ie().catch(e=>{console.error(e instanceof Error?e.message:String(e)),process.exit(1)});
|