mcp-scraper 0.2.19 → 0.2.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +36 -5
- package/dist/bin/api-server.cjs +1072 -349
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +3 -3
- package/dist/bin/browser-agent-stdio-server.cjs +1274 -75
- package/dist/bin/browser-agent-stdio-server.cjs.map +1 -1
- package/dist/bin/browser-agent-stdio-server.js +4 -2
- package/dist/bin/browser-agent-stdio-server.js.map +1 -1
- package/dist/bin/mcp-scraper-cli.cjs +659 -210
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +108 -9
- package/dist/bin/mcp-scraper-cli.js.map +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs +2104 -566
- package/dist/bin/mcp-scraper-combined-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-scraper-combined-stdio-server.js +7 -4
- package/dist/bin/mcp-scraper-combined-stdio-server.js.map +1 -1
- package/dist/bin/mcp-scraper-install.cjs +3 -3
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +2 -2
- package/dist/bin/mcp-stdio-server.cjs +726 -387
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +3 -2
- package/dist/bin/mcp-stdio-server.js.map +1 -1
- package/dist/bin/paa-harvest.cjs +5 -3
- package/dist/bin/paa-harvest.cjs.map +1 -1
- package/dist/bin/paa-harvest.js +5 -2
- package/dist/bin/paa-harvest.js.map +1 -1
- package/dist/{chunk-TL7YTFLH.js → chunk-DBQDG7EH.js} +99 -27
- package/dist/chunk-DBQDG7EH.js.map +1 -0
- package/dist/chunk-DUEW4EOO.js +214 -0
- package/dist/chunk-DUEW4EOO.js.map +1 -0
- package/dist/{chunk-F44RBOJ5.js → chunk-H2R232HK.js} +216 -108
- package/dist/chunk-H2R232HK.js.map +1 -0
- package/dist/chunk-HS6OKELW.js +1786 -0
- package/dist/chunk-HS6OKELW.js.map +1 -0
- package/dist/{chunk-CQTAKXBN.js → chunk-IPW4LFOT.js} +11 -116
- package/dist/chunk-IPW4LFOT.js.map +1 -0
- package/dist/chunk-LFATOGDF.js +111 -0
- package/dist/chunk-LFATOGDF.js.map +1 -0
- package/dist/{chunk-MCBOZ2JF.js → chunk-MLNCKQ2B.js} +3 -3
- package/dist/chunk-MLNCKQ2B.js.map +1 -0
- package/dist/{chunk-4UEVYYDW.js → chunk-O5J7E4VX.js} +394 -57
- package/dist/chunk-O5J7E4VX.js.map +1 -0
- package/dist/{chunk-BSYPATSM.js → chunk-SXTXMFEQ.js} +2 -2
- package/dist/chunk-WN7PBKMV.js +28 -0
- package/dist/chunk-WN7PBKMV.js.map +1 -0
- package/dist/chunk-XKUDVN2E.js +7 -0
- package/dist/chunk-XKUDVN2E.js.map +1 -0
- package/dist/{db-P5X6UQ3E.js → db-BE4JVB3V.js} +8 -2
- package/dist/index.cjs +5 -3
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +3 -1
- package/dist/index.js.map +1 -1
- package/dist/{server-CRUSZA2Q.js → server-24XIQ6W6.js} +248 -40
- package/dist/server-24XIQ6W6.js.map +1 -0
- package/dist/{worker-OZSWIS3F.js → worker-SLQ375UG.js} +8 -5
- package/dist/{worker-OZSWIS3F.js.map → worker-SLQ375UG.js.map} +1 -1
- package/docs/adr/0001-in-page-graphql-interception-for-anti-bot-scraping.md +2 -2
- package/docs/mcp-tool-craft-lint.generated.md +51 -0
- package/docs/mcp-tool-manifest.generated.json +1534 -0
- package/docs/specs/local-competitive-audit-spec.md +1 -2
- package/docs/specs/mcp-tool-definition-quality-audit-spec.md +1602 -0
- package/package.json +4 -1
- package/dist/chunk-4UEVYYDW.js.map +0 -1
- package/dist/chunk-76P4DKWR.js +0 -793
- package/dist/chunk-76P4DKWR.js.map +0 -1
- package/dist/chunk-CQTAKXBN.js.map +0 -1
- package/dist/chunk-F44RBOJ5.js.map +0 -1
- package/dist/chunk-MCBOZ2JF.js.map +0 -1
- package/dist/chunk-RU2HZD55.js +0 -7
- package/dist/chunk-RU2HZD55.js.map +0 -1
- package/dist/chunk-TL7YTFLH.js.map +0 -1
- package/dist/server-CRUSZA2Q.js.map +0 -1
- /package/dist/{chunk-BSYPATSM.js.map → chunk-SXTXMFEQ.js.map} +0 -0
- /package/dist/{db-P5X6UQ3E.js.map → db-BE4JVB3V.js.map} +0 -0
|
@@ -4,11 +4,213 @@
|
|
|
4
4
|
// src/cli/human-cli.ts
|
|
5
5
|
var import_commander = require("commander");
|
|
6
6
|
var import_node_child_process2 = require("child_process");
|
|
7
|
-
var
|
|
8
|
-
var
|
|
7
|
+
var import_promises6 = require("fs/promises");
|
|
8
|
+
var import_node_path5 = require("path");
|
|
9
9
|
|
|
10
10
|
// src/version.ts
|
|
11
|
-
var PACKAGE_VERSION = "0.2.
|
|
11
|
+
var PACKAGE_VERSION = "0.2.20";
|
|
12
|
+
|
|
13
|
+
// src/lib/chrome-profiles.ts
|
|
14
|
+
var import_promises = require("fs/promises");
|
|
15
|
+
var import_node_os = require("os");
|
|
16
|
+
var import_node_path = require("path");
|
|
17
|
+
function defaultChromeUserDataDir() {
|
|
18
|
+
if (process.platform === "darwin") return (0, import_node_path.join)((0, import_node_os.homedir)(), "Library", "Application Support", "Google", "Chrome");
|
|
19
|
+
if (process.platform === "win32") {
|
|
20
|
+
return (0, import_node_path.join)(process.env.LOCALAPPDATA ?? (0, import_node_path.join)((0, import_node_os.homedir)(), "AppData", "Local"), "Google", "Chrome", "User Data");
|
|
21
|
+
}
|
|
22
|
+
return (0, import_node_path.join)(process.env.XDG_CONFIG_HOME ?? (0, import_node_path.join)((0, import_node_os.homedir)(), ".config"), "google-chrome");
|
|
23
|
+
}
|
|
24
|
+
function recommendedKernelProfileName(value) {
|
|
25
|
+
return value.trim().toLowerCase().replace(/@/g, "-").replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80) || "browser-profile";
|
|
26
|
+
}
|
|
27
|
+
async function listLocalChromeProfiles(userDataDir = defaultChromeUserDataDir(), emailFilter) {
|
|
28
|
+
const raw = await (0, import_promises.readFile)((0, import_node_path.join)(userDataDir, "Local State"), "utf8");
|
|
29
|
+
const localState = JSON.parse(raw);
|
|
30
|
+
const wanted = emailFilter?.trim().toLowerCase();
|
|
31
|
+
return Object.entries(localState.profile?.info_cache ?? {}).map(([directory, info]) => {
|
|
32
|
+
const email = (info.user_name ?? "").trim();
|
|
33
|
+
const displayName = (info.name ?? info.shortcut_name ?? "").trim();
|
|
34
|
+
const labelSource = email || displayName || directory;
|
|
35
|
+
return {
|
|
36
|
+
email,
|
|
37
|
+
displayName,
|
|
38
|
+
chromeProfileDirectory: directory,
|
|
39
|
+
chromeProfilePath: (0, import_node_path.join)(userDataDir, directory),
|
|
40
|
+
recommendedKernelProfileName: recommendedKernelProfileName(labelSource)
|
|
41
|
+
};
|
|
42
|
+
}).filter((row) => !wanted || row.email.toLowerCase() === wanted || row.displayName.toLowerCase().includes(wanted));
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
// src/lib/local-browser-profiles.ts
|
|
46
|
+
var import_node_fs = require("fs");
|
|
47
|
+
var import_promises2 = require("fs/promises");
|
|
48
|
+
var import_node_os2 = require("os");
|
|
49
|
+
var import_node_path2 = require("path");
|
|
50
|
+
var EXCLUDED_PROFILE_NAMES = [
|
|
51
|
+
"Cache",
|
|
52
|
+
"Code Cache",
|
|
53
|
+
"Crashpad",
|
|
54
|
+
"DawnCache",
|
|
55
|
+
"DawnGraphiteCache",
|
|
56
|
+
"DawnWebGPUCache",
|
|
57
|
+
"GPUCache",
|
|
58
|
+
"GrShaderCache",
|
|
59
|
+
"ShaderCache",
|
|
60
|
+
"Shared Dictionary"
|
|
61
|
+
];
|
|
62
|
+
function defaultImportedBrowserProfilesDir() {
|
|
63
|
+
return process.env.MCP_SCRAPER_BROWSER_PROFILE_BASE_DIR?.trim() || (0, import_node_path2.join)((0, import_node_os2.homedir)(), ".mcp-scraper", "browser-profiles");
|
|
64
|
+
}
|
|
65
|
+
function defaultChromeExecutablePath() {
|
|
66
|
+
const envPath = process.env.MCP_SCRAPER_BROWSER_EXECUTABLE?.trim() || process.env.CHROME_PATH?.trim();
|
|
67
|
+
if (envPath) return envPath;
|
|
68
|
+
const candidates = process.platform === "darwin" ? ["/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"] : process.platform === "win32" ? [
|
|
69
|
+
(0, import_node_path2.join)(process.env.PROGRAMFILES ?? "C:\\Program Files", "Google", "Chrome", "Application", "chrome.exe"),
|
|
70
|
+
(0, import_node_path2.join)(process.env["PROGRAMFILES(X86)"] ?? "C:\\Program Files (x86)", "Google", "Chrome", "Application", "chrome.exe")
|
|
71
|
+
] : ["/usr/bin/google-chrome", "/usr/bin/google-chrome-stable", "/usr/bin/chromium-browser", "/usr/bin/chromium"];
|
|
72
|
+
return candidates.find((candidate) => (0, import_node_fs.existsSync)(candidate));
|
|
73
|
+
}
|
|
74
|
+
function importedProfileName(value) {
|
|
75
|
+
return recommendedKernelProfileName(value);
|
|
76
|
+
}
|
|
77
|
+
function importedProfileRoot(name, outputDir = defaultImportedBrowserProfilesDir()) {
|
|
78
|
+
return (0, import_node_path2.join)(outputDir, importedProfileName(name));
|
|
79
|
+
}
|
|
80
|
+
function importedProfileManifestPath(name, outputDir = defaultImportedBrowserProfilesDir()) {
|
|
81
|
+
return (0, import_node_path2.join)(importedProfileRoot(name, outputDir), "manifest.json");
|
|
82
|
+
}
|
|
83
|
+
function importedProfileUserDataDir(name, outputDir = defaultImportedBrowserProfilesDir()) {
|
|
84
|
+
return (0, import_node_path2.join)(importedProfileRoot(name, outputDir), "user-data");
|
|
85
|
+
}
|
|
86
|
+
async function readExistingManifest(path) {
|
|
87
|
+
try {
|
|
88
|
+
return JSON.parse(await (0, import_promises2.readFile)(path, "utf8"));
|
|
89
|
+
} catch {
|
|
90
|
+
return null;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
function isExcludedProfilePath(sourceRoot, sourcePath) {
|
|
94
|
+
const rel = (0, import_node_path2.relative)(sourceRoot, sourcePath);
|
|
95
|
+
if (!rel) return false;
|
|
96
|
+
const parts = rel.split(import_node_path2.sep).filter(Boolean);
|
|
97
|
+
const last = parts[parts.length - 1] ?? "";
|
|
98
|
+
if (last.startsWith("Singleton")) return true;
|
|
99
|
+
if (last === "LOCK" || last.endsWith(".lock")) return true;
|
|
100
|
+
if (parts.some((part) => EXCLUDED_PROFILE_NAMES.includes(part))) return true;
|
|
101
|
+
if (parts.includes("Service Worker") && parts.includes("CacheStorage")) return true;
|
|
102
|
+
return false;
|
|
103
|
+
}
|
|
104
|
+
async function copyProfileTree(sourceProfilePath, destinationProfilePath) {
|
|
105
|
+
await (0, import_promises2.cp)(sourceProfilePath, destinationProfilePath, {
|
|
106
|
+
recursive: true,
|
|
107
|
+
force: true,
|
|
108
|
+
errorOnExist: false,
|
|
109
|
+
dereference: false,
|
|
110
|
+
filter: (source) => !isExcludedProfilePath(sourceProfilePath, source)
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
function selectProfile(profiles, options) {
|
|
114
|
+
const requestedDirectory = options.chromeProfileDirectory?.trim();
|
|
115
|
+
const requestedEmail = options.email?.trim().toLowerCase();
|
|
116
|
+
const exactDirectory = requestedDirectory ? profiles.find((profile) => profile.chromeProfileDirectory === requestedDirectory) : void 0;
|
|
117
|
+
if (exactDirectory) return exactDirectory;
|
|
118
|
+
const exactEmail = requestedEmail ? profiles.find((profile) => profile.email.toLowerCase() === requestedEmail) : void 0;
|
|
119
|
+
if (exactEmail) return exactEmail;
|
|
120
|
+
const first = profiles[0];
|
|
121
|
+
if (first) return first;
|
|
122
|
+
const hint = requestedDirectory ? `Chrome profile directory "${requestedDirectory}"` : requestedEmail ? `Chrome account "${requestedEmail}"` : "Chrome profile";
|
|
123
|
+
throw new Error(`${hint} was not found in ${options.sourceUserDataDir ?? defaultChromeUserDataDir()}`);
|
|
124
|
+
}
|
|
125
|
+
async function ensureProfilePathExists(profile) {
|
|
126
|
+
const details = await (0, import_promises2.stat)(profile.chromeProfilePath).catch(() => null);
|
|
127
|
+
if (!details?.isDirectory()) {
|
|
128
|
+
throw new Error(`Chrome profile path does not exist or is not a directory: ${profile.chromeProfilePath}`);
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
async function writeNormalizedLocalState(sourceLocalStatePath, destinationLocalStatePath, sourceProfileDirectory) {
|
|
132
|
+
const raw = await (0, import_promises2.readFile)(sourceLocalStatePath, "utf8");
|
|
133
|
+
const localState = JSON.parse(raw);
|
|
134
|
+
const selectedInfo = localState.profile?.info_cache?.[sourceProfileDirectory] ?? localState.profile?.info_cache?.Default ?? {};
|
|
135
|
+
localState.profile = {
|
|
136
|
+
...localState.profile ?? {},
|
|
137
|
+
info_cache: { Default: selectedInfo },
|
|
138
|
+
last_used: "Default",
|
|
139
|
+
last_active_profiles: ["Default"]
|
|
140
|
+
};
|
|
141
|
+
await (0, import_promises2.writeFile)(destinationLocalStatePath, `${JSON.stringify(localState, null, 2)}
|
|
142
|
+
`);
|
|
143
|
+
}
|
|
144
|
+
async function importChromeProfile(options = {}) {
|
|
145
|
+
const sourceUserDataDir = options.sourceUserDataDir?.trim() || defaultChromeUserDataDir();
|
|
146
|
+
const profiles = await listLocalChromeProfiles(sourceUserDataDir, options.email);
|
|
147
|
+
const sourceProfile = selectProfile(profiles, { ...options, sourceUserDataDir });
|
|
148
|
+
await ensureProfilePathExists(sourceProfile);
|
|
149
|
+
const name = importedProfileName(options.name?.trim() || sourceProfile.recommendedKernelProfileName);
|
|
150
|
+
const outputDir = options.outputDir?.trim() || defaultImportedBrowserProfilesDir();
|
|
151
|
+
const profileRoot = importedProfileRoot(name, outputDir);
|
|
152
|
+
const manifestPath = importedProfileManifestPath(name, outputDir);
|
|
153
|
+
const userDataDir = importedProfileUserDataDir(name, outputDir);
|
|
154
|
+
const existingManifest = await readExistingManifest(manifestPath);
|
|
155
|
+
const existing = (0, import_node_fs.existsSync)(profileRoot);
|
|
156
|
+
if (existing && !options.overwrite) {
|
|
157
|
+
throw new Error(`Imported browser profile "${name}" already exists at ${profileRoot}. Re-run with --overwrite to refresh it.`);
|
|
158
|
+
}
|
|
159
|
+
if (existing) await (0, import_promises2.rm)(profileRoot, { recursive: true, force: true });
|
|
160
|
+
await (0, import_promises2.mkdir)((0, import_node_path2.join)(userDataDir, "Default"), { recursive: true });
|
|
161
|
+
await writeNormalizedLocalState(
|
|
162
|
+
(0, import_node_path2.join)(sourceUserDataDir, "Local State"),
|
|
163
|
+
(0, import_node_path2.join)(userDataDir, "Local State"),
|
|
164
|
+
sourceProfile.chromeProfileDirectory
|
|
165
|
+
);
|
|
166
|
+
await copyProfileTree(sourceProfile.chromeProfilePath, (0, import_node_path2.join)(userDataDir, "Default"));
|
|
167
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
168
|
+
const manifest = {
|
|
169
|
+
layoutVersion: 1,
|
|
170
|
+
name,
|
|
171
|
+
profileRoot,
|
|
172
|
+
userDataDir,
|
|
173
|
+
browserExecutablePath: options.browserExecutablePath?.trim() || defaultChromeExecutablePath() || null,
|
|
174
|
+
sourceUserDataDir,
|
|
175
|
+
sourceProfileDirectory: sourceProfile.chromeProfileDirectory,
|
|
176
|
+
sourceProfilePath: sourceProfile.chromeProfilePath,
|
|
177
|
+
email: sourceProfile.email,
|
|
178
|
+
displayName: sourceProfile.displayName,
|
|
179
|
+
createdAt: existingManifest?.createdAt ?? now,
|
|
180
|
+
updatedAt: now,
|
|
181
|
+
copyMode: "chrome-profile-clone"
|
|
182
|
+
};
|
|
183
|
+
await (0, import_promises2.writeFile)(manifestPath, `${JSON.stringify(manifest, null, 2)}
|
|
184
|
+
`);
|
|
185
|
+
return {
|
|
186
|
+
profile: manifest,
|
|
187
|
+
sourceProfile,
|
|
188
|
+
excludedNames: EXCLUDED_PROFILE_NAMES
|
|
189
|
+
};
|
|
190
|
+
}
|
|
191
|
+
async function loadImportedBrowserProfile(name = process.env.MCP_SCRAPER_BROWSER_PROFILE?.trim() || "default", outputDir = defaultImportedBrowserProfilesDir()) {
|
|
192
|
+
const profileName = importedProfileName(name);
|
|
193
|
+
const manifestPath = importedProfileManifestPath(profileName, outputDir);
|
|
194
|
+
const manifest = await readExistingManifest(manifestPath);
|
|
195
|
+
if (!manifest) {
|
|
196
|
+
throw new Error(
|
|
197
|
+
`Imported browser profile "${profileName}" was not found. Run: mcp-scraper-cli browser import-chrome --name ${profileName}`
|
|
198
|
+
);
|
|
199
|
+
}
|
|
200
|
+
return manifest;
|
|
201
|
+
}
|
|
202
|
+
async function syncImportedChromeProfile(name, options = {}) {
|
|
203
|
+
const outputDir = options.outputDir?.trim() || defaultImportedBrowserProfilesDir();
|
|
204
|
+
const manifest = await loadImportedBrowserProfile(name, outputDir);
|
|
205
|
+
return importChromeProfile({
|
|
206
|
+
name: manifest.name,
|
|
207
|
+
sourceUserDataDir: manifest.sourceUserDataDir,
|
|
208
|
+
chromeProfileDirectory: manifest.sourceProfileDirectory,
|
|
209
|
+
outputDir,
|
|
210
|
+
overwrite: true,
|
|
211
|
+
browserExecutablePath: options.browserExecutablePath ?? manifest.browserExecutablePath ?? void 0
|
|
212
|
+
});
|
|
213
|
+
}
|
|
12
214
|
|
|
13
215
|
// src/cli/agent-config.ts
|
|
14
216
|
function apiKeyValue(options) {
|
|
@@ -20,18 +222,37 @@ function packageSpec(options) {
|
|
|
20
222
|
function combinedNpxArgs(options = {}) {
|
|
21
223
|
return ["-y", "-p", packageSpec(options), "mcp-scraper-combined"];
|
|
22
224
|
}
|
|
225
|
+
function envConfig(options) {
|
|
226
|
+
const env = { MCP_SCRAPER_API_KEY: apiKeyValue(options) };
|
|
227
|
+
const profileName = options.browserProfileName?.trim();
|
|
228
|
+
if (options.browserMode === "local") {
|
|
229
|
+
env.MCP_SCRAPER_BROWSER_MODE = "local";
|
|
230
|
+
if (profileName) env.MCP_SCRAPER_BROWSER_PROFILE = profileName;
|
|
231
|
+
const executablePath = options.browserExecutablePath?.trim();
|
|
232
|
+
if (executablePath) env.MCP_SCRAPER_BROWSER_EXECUTABLE = executablePath;
|
|
233
|
+
} else {
|
|
234
|
+
if (profileName) env.BROWSER_AGENT_PROFILE_NAME = profileName;
|
|
235
|
+
if (options.browserProfileSaveChanges === true) env.BROWSER_AGENT_PROFILE_SAVE_CHANGES = "true";
|
|
236
|
+
}
|
|
237
|
+
return env;
|
|
238
|
+
}
|
|
239
|
+
function tomlInlineTable(value) {
|
|
240
|
+
const entries = Object.entries(value).map(([key, item]) => `${key} = ${JSON.stringify(item)}`);
|
|
241
|
+
return `{ ${entries.join(", ")} }`;
|
|
242
|
+
}
|
|
23
243
|
function renderCodexConfig(options = {}) {
|
|
24
244
|
return [
|
|
25
245
|
"[mcp_servers.mcp-scraper]",
|
|
26
246
|
'command = "npx"',
|
|
27
247
|
`args = ${JSON.stringify(combinedNpxArgs(options))}`,
|
|
28
|
-
`env =
|
|
248
|
+
`env = ${tomlInlineTable(envConfig(options))}`
|
|
29
249
|
].join("\n");
|
|
30
250
|
}
|
|
31
251
|
function renderClaudeCommand(options = {}) {
|
|
252
|
+
const env = Object.entries(envConfig(options)).map(([key, value]) => ` --env ${key}=${value}`);
|
|
32
253
|
return [
|
|
33
254
|
"claude mcp add mcp-scraper --scope user",
|
|
34
|
-
|
|
255
|
+
...env,
|
|
35
256
|
` -- npx ${combinedNpxArgs(options).join(" ")}`
|
|
36
257
|
].join(" \\\n");
|
|
37
258
|
}
|
|
@@ -41,9 +262,7 @@ function renderClaudeDesktopConfig(options = {}) {
|
|
|
41
262
|
"mcp-scraper": {
|
|
42
263
|
command: "npx",
|
|
43
264
|
args: combinedNpxArgs(options),
|
|
44
|
-
env:
|
|
45
|
-
MCP_SCRAPER_API_KEY: apiKeyValue(options)
|
|
46
|
-
}
|
|
265
|
+
env: envConfig(options)
|
|
47
266
|
}
|
|
48
267
|
}
|
|
49
268
|
}, null, 2);
|
|
@@ -76,9 +295,9 @@ function renderAgentInstall(host, options = {}) {
|
|
|
76
295
|
}
|
|
77
296
|
|
|
78
297
|
// src/cli/doctor.ts
|
|
79
|
-
var
|
|
80
|
-
var
|
|
81
|
-
var
|
|
298
|
+
var import_promises3 = require("fs/promises");
|
|
299
|
+
var import_node_os3 = require("os");
|
|
300
|
+
var import_node_path3 = require("path");
|
|
82
301
|
function status(ok, warn = false) {
|
|
83
302
|
if (ok) return "pass";
|
|
84
303
|
return warn ? "warn" : "fail";
|
|
@@ -87,9 +306,9 @@ function parseMajor(version) {
|
|
|
87
306
|
return Number(version.replace(/^v/, "").split(".")[0] ?? 0);
|
|
88
307
|
}
|
|
89
308
|
async function readKeyFile() {
|
|
90
|
-
const path = process.env.MCP_SCRAPER_KEY_PATH?.trim() || (0,
|
|
309
|
+
const path = process.env.MCP_SCRAPER_KEY_PATH?.trim() || (0, import_node_path3.join)((0, import_node_os3.homedir)(), ".mcp-scraper-key");
|
|
91
310
|
try {
|
|
92
|
-
const value = (await (0,
|
|
311
|
+
const value = (await (0, import_promises3.readFile)(path, "utf8")).trim();
|
|
93
312
|
return value || null;
|
|
94
313
|
} catch {
|
|
95
314
|
return null;
|
|
@@ -110,7 +329,7 @@ async function npmLatest(fetchImpl) {
|
|
|
110
329
|
async function runDoctor(options = {}) {
|
|
111
330
|
const fetchImpl = options.fetchImpl ?? fetch;
|
|
112
331
|
const apiUrl = (options.apiUrl ?? process.env.MCP_SCRAPER_API_URL ?? "https://mcpscraper.dev").replace(/\/$/, "");
|
|
113
|
-
const outputDir = options.outputDir ?? process.env.MCP_SCRAPER_OUTPUT_DIR ?? (0,
|
|
332
|
+
const outputDir = options.outputDir ?? process.env.MCP_SCRAPER_OUTPUT_DIR ?? (0, import_node_path3.join)((0, import_node_os3.homedir)(), "Downloads", "mcp-scraper");
|
|
114
333
|
const configuredKey = options.apiKey?.trim() || process.env.MCP_SCRAPER_API_KEY?.trim() || await readKeyFile();
|
|
115
334
|
const latest = await npmLatest(fetchImpl);
|
|
116
335
|
const checks = [];
|
|
@@ -137,8 +356,8 @@ async function runDoctor(options = {}) {
|
|
|
137
356
|
fix: configuredKey ? void 0 : "Set MCP_SCRAPER_API_KEY or pass --api-key."
|
|
138
357
|
});
|
|
139
358
|
try {
|
|
140
|
-
await (0,
|
|
141
|
-
await (0,
|
|
359
|
+
await (0, import_promises3.mkdir)(outputDir, { recursive: true });
|
|
360
|
+
await (0, import_promises3.access)(outputDir);
|
|
142
361
|
checks.push({ id: "output_dir", label: "Output directory", status: "pass", detail: outputDir });
|
|
143
362
|
} catch (err) {
|
|
144
363
|
checks.push({
|
|
@@ -284,10 +503,10 @@ function renderPrompt(name) {
|
|
|
284
503
|
}
|
|
285
504
|
|
|
286
505
|
// src/workflows/artifact-writer.ts
|
|
287
|
-
var
|
|
288
|
-
var
|
|
289
|
-
var
|
|
290
|
-
var
|
|
506
|
+
var import_promises4 = require("fs/promises");
|
|
507
|
+
var import_node_fs2 = require("fs");
|
|
508
|
+
var import_node_os4 = require("os");
|
|
509
|
+
var import_node_path4 = require("path");
|
|
291
510
|
var import_node_child_process = require("child_process");
|
|
292
511
|
|
|
293
512
|
// src/directory/csv.ts
|
|
@@ -310,7 +529,7 @@ function slugify(s) {
|
|
|
310
529
|
|
|
311
530
|
// src/workflows/artifact-writer.ts
|
|
312
531
|
function workflowOutputBaseDir(outputDir) {
|
|
313
|
-
return outputDir?.trim() || process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0,
|
|
532
|
+
return outputDir?.trim() || process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path4.join)((0, import_node_os4.homedir)(), "Downloads", "mcp-scraper");
|
|
314
533
|
}
|
|
315
534
|
function timestamp() {
|
|
316
535
|
return (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
@@ -341,31 +560,31 @@ var ArtifactWriter = class _ArtifactWriter {
|
|
|
341
560
|
const runId = forcedRunId?.trim() || `${timestamp()}-${safeSlug(workflowId)}-${Math.random().toString(36).slice(2, 8)}`;
|
|
342
561
|
const nameSource = String(input.keyword ?? input.query ?? input.domain ?? input.state ?? workflowId);
|
|
343
562
|
const baseDir = workflowOutputBaseDir(outputDir);
|
|
344
|
-
const runDir = (0,
|
|
345
|
-
await (0,
|
|
563
|
+
const runDir = (0, import_node_path4.join)(baseDir, "workflows", workflowId, `${timestamp()}-${safeSlug(nameSource)}-${safeSlug(runId).slice(0, 20)}`);
|
|
564
|
+
await (0, import_promises4.mkdir)(runDir, { recursive: true });
|
|
346
565
|
return new _ArtifactWriter(workflowId, title, runId, baseDir, runDir, startedAt, input);
|
|
347
566
|
}
|
|
348
567
|
async remember(kind, label, path, rows) {
|
|
349
|
-
const size = await (0,
|
|
568
|
+
const size = await (0, import_promises4.stat)(path).then((s) => s.size).catch(() => void 0);
|
|
350
569
|
this.artifacts.push({ kind, label, path, bytes: size, rows });
|
|
351
570
|
return path;
|
|
352
571
|
}
|
|
353
572
|
async writeJson(label, relativePath, data) {
|
|
354
|
-
const path = (0,
|
|
355
|
-
await (0,
|
|
356
|
-
await (0,
|
|
573
|
+
const path = (0, import_node_path4.join)(this.runDir, relativePath);
|
|
574
|
+
await (0, import_promises4.mkdir)((0, import_node_path4.dirname)(path), { recursive: true });
|
|
575
|
+
await (0, import_promises4.writeFile)(path, JSON.stringify(data, null, 2), "utf8");
|
|
357
576
|
return this.remember("json", label, path);
|
|
358
577
|
}
|
|
359
578
|
async writeText(label, relativePath, text, kind = "markdown") {
|
|
360
|
-
const path = (0,
|
|
361
|
-
await (0,
|
|
362
|
-
await (0,
|
|
579
|
+
const path = (0, import_node_path4.join)(this.runDir, relativePath);
|
|
580
|
+
await (0, import_promises4.mkdir)((0, import_node_path4.dirname)(path), { recursive: true });
|
|
581
|
+
await (0, import_promises4.writeFile)(path, text, "utf8");
|
|
363
582
|
return this.remember(kind, label, path);
|
|
364
583
|
}
|
|
365
584
|
async writeCsv(label, relativePath, headers, rows) {
|
|
366
|
-
const path = (0,
|
|
367
|
-
await (0,
|
|
368
|
-
await (0,
|
|
585
|
+
const path = (0, import_node_path4.join)(this.runDir, relativePath);
|
|
586
|
+
await (0, import_promises4.mkdir)((0, import_node_path4.dirname)(path), { recursive: true });
|
|
587
|
+
await (0, import_promises4.writeFile)(path, rowsToCsv(headers, rows), "utf8");
|
|
369
588
|
return this.remember("csv", label, path, rows.length);
|
|
370
589
|
}
|
|
371
590
|
async writeHtml(label, relativePath, html) {
|
|
@@ -385,20 +604,20 @@ var ArtifactWriter = class _ArtifactWriter {
|
|
|
385
604
|
errors,
|
|
386
605
|
counts
|
|
387
606
|
};
|
|
388
|
-
const path = (0,
|
|
389
|
-
await (0,
|
|
607
|
+
const path = (0, import_node_path4.join)(this.runDir, "manifest.json");
|
|
608
|
+
await (0, import_promises4.writeFile)(path, JSON.stringify(manifest, null, 2), "utf8");
|
|
390
609
|
await updateWorkflowIndex(manifest, path, this.baseDir);
|
|
391
610
|
return path;
|
|
392
611
|
}
|
|
393
612
|
};
|
|
394
613
|
function indexPath(baseDir = workflowOutputBaseDir()) {
|
|
395
|
-
return (0,
|
|
614
|
+
return (0, import_node_path4.join)(baseDir, "workflows", "index.json");
|
|
396
615
|
}
|
|
397
616
|
async function readIndex(baseDir) {
|
|
398
617
|
const path = indexPath(baseDir);
|
|
399
|
-
if (!(0,
|
|
618
|
+
if (!(0, import_node_fs2.existsSync)(path)) return { runs: [] };
|
|
400
619
|
try {
|
|
401
|
-
return JSON.parse(await (0,
|
|
620
|
+
return JSON.parse(await (0, import_promises4.readFile)(path, "utf8"));
|
|
402
621
|
} catch {
|
|
403
622
|
return { runs: [] };
|
|
404
623
|
}
|
|
@@ -418,8 +637,8 @@ async function updateWorkflowIndex(manifest, manifestPath, baseDir) {
|
|
|
418
637
|
summary: `${manifest.title} (${manifest.status})`
|
|
419
638
|
};
|
|
420
639
|
index.runs = [entry, ...index.runs.filter((r) => r.runId !== manifest.runId)].slice(0, 200);
|
|
421
|
-
await (0,
|
|
422
|
-
await (0,
|
|
640
|
+
await (0, import_promises4.mkdir)((0, import_node_path4.dirname)(path), { recursive: true });
|
|
641
|
+
await (0, import_promises4.writeFile)(path, JSON.stringify(index, null, 2), "utf8");
|
|
423
642
|
}
|
|
424
643
|
async function listWorkflowReports(outputDir) {
|
|
425
644
|
return (await readIndex(workflowOutputBaseDir(outputDir))).runs;
|
|
@@ -432,7 +651,7 @@ async function findWorkflowReport(id, outputDir) {
|
|
|
432
651
|
async function openWorkflowReport(id, outputDir) {
|
|
433
652
|
const run = await findWorkflowReport(id, outputDir);
|
|
434
653
|
if (!run?.reportPath) throw new Error(`No report found for "${id}"`);
|
|
435
|
-
if ((0,
|
|
654
|
+
if ((0, import_node_os4.platform)() === "darwin") {
|
|
436
655
|
await new Promise((resolve, reject) => {
|
|
437
656
|
(0, import_node_child_process.execFile)("open", [run.reportPath], (err) => err ? reject(err) : resolve());
|
|
438
657
|
});
|
|
@@ -441,7 +660,8 @@ async function openWorkflowReport(id, outputDir) {
|
|
|
441
660
|
}
|
|
442
661
|
|
|
443
662
|
// src/workflows/registry.ts
|
|
444
|
-
var
|
|
663
|
+
var import_promises5 = require("fs/promises");
|
|
664
|
+
var import_zod6 = require("zod");
|
|
445
665
|
|
|
446
666
|
// src/workflows/http-client.ts
|
|
447
667
|
var WorkflowHttpClient = class {
|
|
@@ -622,115 +842,254 @@ var agentPacketWorkflowDefinition = {
|
|
|
622
842
|
title: "Agent-Ready SEO Packet",
|
|
623
843
|
description: "Create an evidence folder for AI agents from live SERP/PAA/AI search surfaces.",
|
|
624
844
|
inputSchema: AgentPacketInputSchema,
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
845
|
+
createState: () => ({ serp: null, paa: null, warnings: [] }),
|
|
846
|
+
steps: [
|
|
847
|
+
{
|
|
848
|
+
id: "harvest-serp",
|
|
849
|
+
title: "Harvest organic SERP + AI Overview",
|
|
850
|
+
async run({ input, state, ctx }) {
|
|
851
|
+
if (!input.includeSerp) {
|
|
852
|
+
return { state, output: { skipped: true, reason: "includeSerp is false" } };
|
|
853
|
+
}
|
|
854
|
+
try {
|
|
855
|
+
const serp = await ctx.client.post("/harvest/sync", {
|
|
856
|
+
query: input.keyword,
|
|
857
|
+
location: input.location,
|
|
858
|
+
serpOnly: true,
|
|
859
|
+
maxQuestions: 1,
|
|
860
|
+
format: "json"
|
|
861
|
+
}, 18e4);
|
|
862
|
+
await ctx.artifacts.writeJson("SERP raw JSON", "raw/serp.json", serp);
|
|
863
|
+
return {
|
|
864
|
+
state: { ...state, serp },
|
|
865
|
+
output: {
|
|
866
|
+
organicResults: serp.organicResults?.length ?? 0,
|
|
867
|
+
localPack: serp.localPack?.length ?? 0,
|
|
868
|
+
aiOverviewDetected: Boolean(serp.aiOverview?.detected),
|
|
869
|
+
aiOverviewCitations: serp.aiOverview?.citations?.length ?? 0
|
|
870
|
+
}
|
|
871
|
+
};
|
|
872
|
+
} catch (err) {
|
|
873
|
+
const message = `SERP evidence unavailable: ${err instanceof Error ? err.message : String(err)}`;
|
|
874
|
+
return { state: { ...state, warnings: [...state.warnings, message] }, output: { error: message }, warnings: [message] };
|
|
875
|
+
}
|
|
642
876
|
}
|
|
643
|
-
}
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
877
|
+
},
|
|
878
|
+
{
|
|
879
|
+
id: "harvest-paa",
|
|
880
|
+
title: "Harvest People Also Ask",
|
|
881
|
+
async run({ input, state, ctx }) {
|
|
882
|
+
if (!input.includePaa) {
|
|
883
|
+
return { state, output: { skipped: true, reason: "includePaa is false" } };
|
|
884
|
+
}
|
|
885
|
+
try {
|
|
886
|
+
const paa = await ctx.client.post("/harvest/sync", {
|
|
887
|
+
query: input.keyword,
|
|
888
|
+
location: input.location,
|
|
889
|
+
maxQuestions: input.maxQuestions,
|
|
890
|
+
format: "json"
|
|
891
|
+
}, 28e4);
|
|
892
|
+
await ctx.artifacts.writeJson("PAA raw JSON", "raw/paa.json", paa);
|
|
893
|
+
return {
|
|
894
|
+
state: { ...state, paa },
|
|
895
|
+
output: { paaQuestions: paa.flat?.length ?? 0 }
|
|
896
|
+
};
|
|
897
|
+
} catch (err) {
|
|
898
|
+
const message = `PAA evidence unavailable: ${err instanceof Error ? err.message : String(err)}`;
|
|
899
|
+
return { state: { ...state, warnings: [...state.warnings, message] }, output: { error: message }, warnings: [message] };
|
|
900
|
+
}
|
|
901
|
+
}
|
|
902
|
+
},
|
|
903
|
+
{
|
|
904
|
+
id: "assemble",
|
|
905
|
+
title: "Assemble evidence packet",
|
|
906
|
+
async run({ input, state, ctx }) {
|
|
907
|
+
const { serp, paa } = state;
|
|
908
|
+
const warnings = state.warnings;
|
|
909
|
+
if (!serp && !paa && !input.returnPartial) throw new Error("No SEO evidence was collected");
|
|
910
|
+
const rows = sourceRows(input, serp, paa);
|
|
911
|
+
const target = normalizeDomain(input.domain);
|
|
912
|
+
const competitors = competitorRows(rows, target);
|
|
913
|
+
const evidence = {
|
|
914
|
+
input,
|
|
915
|
+
serp: serp ?? { status: "skipped" },
|
|
916
|
+
paa: paa ?? { status: "skipped" },
|
|
917
|
+
target: {
|
|
918
|
+
domain: target,
|
|
919
|
+
organicPositions: rows.filter((r) => r.surface === "organic" && r.is_target).map((r) => Number(r.position)),
|
|
920
|
+
citedInAiOverview: rows.some((r) => r.surface === "ai_overview" && r.is_target),
|
|
921
|
+
citedInPaa: rows.some((r) => r.surface === "paa" && r.is_target)
|
|
922
|
+
},
|
|
923
|
+
competitors,
|
|
924
|
+
warnings
|
|
925
|
+
};
|
|
926
|
+
await ctx.artifacts.writeJson("Evidence JSON", "evidence.json", evidence);
|
|
927
|
+
await ctx.artifacts.writeCsv("Sources CSV", "sources.csv", ["surface", "position", "title", "url", "domain", "question", "answer_excerpt", "is_target", "is_competitor"], rows);
|
|
928
|
+
await ctx.artifacts.writeCsv("Competitors CSV", "competitors.csv", ["domain", "organic_best_position", "organic_count", "paa_mentions", "ai_overview_citations", "source_url_count"], competitors);
|
|
929
|
+
const brief = [
|
|
930
|
+
`# SEO Evidence Brief: ${input.keyword}`,
|
|
931
|
+
"",
|
|
932
|
+
`Location: ${input.location ?? "not specified"}`,
|
|
933
|
+
`Target domain: ${target ?? "not specified"}`,
|
|
934
|
+
"",
|
|
935
|
+
"## Evidence Summary",
|
|
936
|
+
`- Organic rows: ${rows.filter((r) => r.surface === "organic").length}`,
|
|
937
|
+
`- PAA rows: ${rows.filter((r) => r.surface === "paa").length}`,
|
|
938
|
+
`- AI Overview citations: ${rows.filter((r) => r.surface === "ai_overview").length}`,
|
|
939
|
+
`- Competitor domains: ${competitors.length}`,
|
|
940
|
+
"",
|
|
941
|
+
"## Recommended Use",
|
|
942
|
+
"Use the CSV files as source of truth. Tie content recommendations to evidence rows and mark unsupported ideas as assumptions."
|
|
943
|
+
].join("\n");
|
|
944
|
+
await ctx.artifacts.writeText("Brief", "brief.md", brief);
|
|
945
|
+
await ctx.artifacts.writeText("Agent tasks", "tasks.md", [
|
|
946
|
+
"# Agent Tasks",
|
|
947
|
+
"",
|
|
948
|
+
"- [ ] Read `evidence.json` before writing recommendations.",
|
|
949
|
+
"- [ ] Compare the target domain against `competitors.csv`.",
|
|
950
|
+
"- [ ] Use `sources.csv` for citations and source-grounded page sections.",
|
|
951
|
+
"- [ ] Mark unsupported recommendations as assumptions."
|
|
952
|
+
].join("\n"));
|
|
953
|
+
await ctx.artifacts.writeText("Agent instructions", "agent-instructions.md", [
|
|
954
|
+
"# Agent Instructions",
|
|
955
|
+
"",
|
|
956
|
+
"You are working from an MCP Scraper SEO evidence packet. Use `evidence.json` and CSV files as source of truth. Do not invent citations. If a recommendation is not supported by evidence, mark it as an assumption."
|
|
957
|
+
].join("\n"));
|
|
958
|
+
const summary = `${rows.length} evidence rows and ${competitors.length} competitor domains collected.`;
|
|
959
|
+
const reportPath = await ctx.artifacts.writeHtml("HTML report", "report.html", renderWorkflowReport({
|
|
960
|
+
title: "Agent-Ready SEO Packet",
|
|
961
|
+
subtitle: `${input.keyword}${input.location ? ` \xB7 ${input.location}` : ""}`,
|
|
962
|
+
summary,
|
|
963
|
+
warnings,
|
|
964
|
+
tables: [
|
|
965
|
+
{ title: "Competitor Domains", columns: ["domain", "organic_best_position", "organic_count", "paa_mentions", "ai_overview_citations", "source_url_count"], rows: competitors.slice(0, 50) },
|
|
966
|
+
{ title: "Evidence Sources", columns: ["surface", "position", "title", "domain", "question", "is_target"], rows: rows.slice(0, 100) }
|
|
967
|
+
]
|
|
968
|
+
}));
|
|
969
|
+
const status2 = warnings.length ? "partial" : "succeeded";
|
|
970
|
+
const counts = { sources: rows.length, competitors: competitors.length };
|
|
971
|
+
await ctx.artifacts.writeManifest(status2, counts, warnings, []);
|
|
972
|
+
return {
|
|
973
|
+
state,
|
|
974
|
+
output: { sources: rows.length, competitors: competitors.length, status: status2 },
|
|
975
|
+
summary: { title: "Agent-Ready SEO Packet", summary, status: status2, counts, warnings, errors: [], reportPath }
|
|
976
|
+
};
|
|
654
977
|
}
|
|
655
978
|
}
|
|
656
|
-
|
|
657
|
-
const rows = sourceRows(input, serp, paa);
|
|
658
|
-
const target = normalizeDomain(input.domain);
|
|
659
|
-
const competitors = competitorRows(rows, target);
|
|
660
|
-
const evidence = {
|
|
661
|
-
input,
|
|
662
|
-
serp: serp ?? { status: "skipped" },
|
|
663
|
-
paa: paa ?? { status: "skipped" },
|
|
664
|
-
target: {
|
|
665
|
-
domain: target,
|
|
666
|
-
organicPositions: rows.filter((r) => r.surface === "organic" && r.is_target).map((r) => Number(r.position)),
|
|
667
|
-
citedInAiOverview: rows.some((r) => r.surface === "ai_overview" && r.is_target),
|
|
668
|
-
citedInPaa: rows.some((r) => r.surface === "paa" && r.is_target)
|
|
669
|
-
},
|
|
670
|
-
competitors,
|
|
671
|
-
warnings
|
|
672
|
-
};
|
|
673
|
-
await ctx.artifacts.writeJson("Evidence JSON", "evidence.json", evidence);
|
|
674
|
-
await ctx.artifacts.writeCsv("Sources CSV", "sources.csv", ["surface", "position", "title", "url", "domain", "question", "answer_excerpt", "is_target", "is_competitor"], rows);
|
|
675
|
-
await ctx.artifacts.writeCsv("Competitors CSV", "competitors.csv", ["domain", "organic_best_position", "organic_count", "paa_mentions", "ai_overview_citations", "source_url_count"], competitors);
|
|
676
|
-
const brief = [
|
|
677
|
-
`# SEO Evidence Brief: ${input.keyword}`,
|
|
678
|
-
"",
|
|
679
|
-
`Location: ${input.location ?? "not specified"}`,
|
|
680
|
-
`Target domain: ${target ?? "not specified"}`,
|
|
681
|
-
"",
|
|
682
|
-
"## Evidence Summary",
|
|
683
|
-
`- Organic rows: ${rows.filter((r) => r.surface === "organic").length}`,
|
|
684
|
-
`- PAA rows: ${rows.filter((r) => r.surface === "paa").length}`,
|
|
685
|
-
`- AI Overview citations: ${rows.filter((r) => r.surface === "ai_overview").length}`,
|
|
686
|
-
`- Competitor domains: ${competitors.length}`,
|
|
687
|
-
"",
|
|
688
|
-
"## Recommended Use",
|
|
689
|
-
"Use the CSV files as source of truth. Tie content recommendations to evidence rows and mark unsupported ideas as assumptions."
|
|
690
|
-
].join("\n");
|
|
691
|
-
await ctx.artifacts.writeText("Brief", "brief.md", brief);
|
|
692
|
-
await ctx.artifacts.writeText("Agent tasks", "tasks.md", [
|
|
693
|
-
"# Agent Tasks",
|
|
694
|
-
"",
|
|
695
|
-
"- [ ] Read `evidence.json` before writing recommendations.",
|
|
696
|
-
"- [ ] Compare the target domain against `competitors.csv`.",
|
|
697
|
-
"- [ ] Use `sources.csv` for citations and source-grounded page sections.",
|
|
698
|
-
"- [ ] Mark unsupported recommendations as assumptions."
|
|
699
|
-
].join("\n"));
|
|
700
|
-
await ctx.artifacts.writeText("Agent instructions", "agent-instructions.md", [
|
|
701
|
-
"# Agent Instructions",
|
|
702
|
-
"",
|
|
703
|
-
"You are working from an MCP Scraper SEO evidence packet. Use `evidence.json` and CSV files as source of truth. Do not invent citations. If a recommendation is not supported by evidence, mark it as an assumption."
|
|
704
|
-
].join("\n"));
|
|
705
|
-
const summary = `${rows.length} evidence rows and ${competitors.length} competitor domains collected.`;
|
|
706
|
-
const reportPath = await ctx.artifacts.writeHtml("HTML report", "report.html", renderWorkflowReport({
|
|
707
|
-
title: "Agent-Ready SEO Packet",
|
|
708
|
-
subtitle: `${input.keyword}${input.location ? ` \xB7 ${input.location}` : ""}`,
|
|
709
|
-
summary,
|
|
710
|
-
warnings,
|
|
711
|
-
tables: [
|
|
712
|
-
{ title: "Competitor Domains", columns: ["domain", "organic_best_position", "organic_count", "paa_mentions", "ai_overview_citations", "source_url_count"], rows: competitors.slice(0, 50) },
|
|
713
|
-
{ title: "Evidence Sources", columns: ["surface", "position", "title", "domain", "question", "is_target"], rows: rows.slice(0, 100) }
|
|
714
|
-
]
|
|
715
|
-
}));
|
|
716
|
-
const status2 = warnings.length ? "partial" : "succeeded";
|
|
717
|
-
const counts = { sources: rows.length, competitors: competitors.length };
|
|
718
|
-
await ctx.artifacts.writeManifest(status2, counts, warnings, errors);
|
|
719
|
-
return { title: "Agent-Ready SEO Packet", summary, status: status2, counts, warnings, errors, reportPath };
|
|
720
|
-
}
|
|
979
|
+
]
|
|
721
980
|
};
|
|
722
981
|
|
|
723
982
|
// src/workflows/workflows/directory.ts
|
|
983
|
+
var import_zod3 = require("zod");
|
|
984
|
+
|
|
985
|
+
// src/schemas.ts
|
|
724
986
|
var import_zod2 = require("zod");
|
|
725
|
-
var
|
|
987
|
+
var DEFAULT_PROXY_MODE = "configured";
|
|
988
|
+
var DEFAULT_MAPS_PROXY_MODE = "location";
|
|
989
|
+
var HarvestOptionsSchema = import_zod2.z.object({
|
|
990
|
+
query: import_zod2.z.string().min(1),
|
|
991
|
+
location: import_zod2.z.string().optional(),
|
|
992
|
+
gl: import_zod2.z.string().length(2).default("us"),
|
|
993
|
+
hl: import_zod2.z.string().length(2).default("en"),
|
|
994
|
+
device: import_zod2.z.enum(["desktop", "mobile"]).default("desktop"),
|
|
995
|
+
proxyMode: import_zod2.z.enum(["location", "configured", "none"]).default(DEFAULT_PROXY_MODE),
|
|
996
|
+
proxyZip: import_zod2.z.string().regex(/^\d{5}$/).optional(),
|
|
997
|
+
debug: import_zod2.z.boolean().default(false),
|
|
998
|
+
depth: import_zod2.z.number().int().min(1).max(30).default(3),
|
|
999
|
+
maxQuestions: import_zod2.z.number().int().min(1).max(1e3).default(100),
|
|
1000
|
+
headless: import_zod2.z.boolean().default(false),
|
|
1001
|
+
profileDir: import_zod2.z.string().optional(),
|
|
1002
|
+
proxy: import_zod2.z.string().url().optional(),
|
|
1003
|
+
kernelApiKey: import_zod2.z.string().optional(),
|
|
1004
|
+
kernelProxyId: import_zod2.z.string().optional(),
|
|
1005
|
+
kernelProxyResolution: import_zod2.z.unknown().optional(),
|
|
1006
|
+
outputDir: import_zod2.z.string().default("./paa-output"),
|
|
1007
|
+
format: import_zod2.z.enum(["json", "csv", "both"]).default("both"),
|
|
1008
|
+
serpOnly: import_zod2.z.boolean().default(false),
|
|
1009
|
+
pages: import_zod2.z.number().int().min(1).max(2).default(1)
|
|
1010
|
+
});
|
|
1011
|
+
var MapsPlaceOptionsSchema = import_zod2.z.object({
|
|
1012
|
+
businessName: import_zod2.z.string().min(1),
|
|
1013
|
+
location: import_zod2.z.string().min(1),
|
|
1014
|
+
gl: import_zod2.z.string().length(2).default("us"),
|
|
1015
|
+
hl: import_zod2.z.string().length(2).default("en"),
|
|
1016
|
+
includeReviews: import_zod2.z.boolean().default(false),
|
|
1017
|
+
maxReviews: import_zod2.z.number().int().min(1).max(500).default(50),
|
|
1018
|
+
kernelApiKey: import_zod2.z.string().optional(),
|
|
1019
|
+
kernelProxyId: import_zod2.z.string().optional(),
|
|
1020
|
+
headless: import_zod2.z.boolean().default(true)
|
|
1021
|
+
});
|
|
1022
|
+
var MapsSearchOptionsSchema = import_zod2.z.object({
|
|
726
1023
|
query: import_zod2.z.string().min(1),
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
1024
|
+
location: import_zod2.z.string().optional(),
|
|
1025
|
+
gl: import_zod2.z.string().length(2).default("us"),
|
|
1026
|
+
hl: import_zod2.z.string().length(2).default("en"),
|
|
1027
|
+
maxResults: import_zod2.z.number().int().min(1).max(50).default(10),
|
|
1028
|
+
proxyMode: import_zod2.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
|
|
1029
|
+
proxyZip: import_zod2.z.string().regex(/^\d{5}$/).optional(),
|
|
1030
|
+
debug: import_zod2.z.boolean().default(false),
|
|
1031
|
+
kernelApiKey: import_zod2.z.string().optional(),
|
|
1032
|
+
kernelProxyId: import_zod2.z.string().optional(),
|
|
1033
|
+
kernelProxyResolution: import_zod2.z.unknown().optional(),
|
|
1034
|
+
headless: import_zod2.z.boolean().default(true)
|
|
1035
|
+
});
|
|
1036
|
+
var RawPAAItemSchema = import_zod2.z.object({
|
|
1037
|
+
question: import_zod2.z.string().min(1),
|
|
1038
|
+
answer: import_zod2.z.string().optional(),
|
|
1039
|
+
sourceTitle: import_zod2.z.string().optional(),
|
|
1040
|
+
sourceSite: import_zod2.z.string().optional(),
|
|
1041
|
+
sourceCite: import_zod2.z.string().optional()
|
|
1042
|
+
});
|
|
1043
|
+
var RawMapsOverviewSchema = import_zod2.z.object({
|
|
1044
|
+
name: import_zod2.z.string().nullable(),
|
|
1045
|
+
rating: import_zod2.z.string().nullable(),
|
|
1046
|
+
reviewCount: import_zod2.z.string().nullable(),
|
|
1047
|
+
category: import_zod2.z.string().nullable(),
|
|
1048
|
+
address: import_zod2.z.string().nullable(),
|
|
1049
|
+
hoursSummary: import_zod2.z.string().nullable(),
|
|
1050
|
+
phone: import_zod2.z.string().nullable(),
|
|
1051
|
+
phoneDisplay: import_zod2.z.string().nullable(),
|
|
1052
|
+
website: import_zod2.z.string().nullable(),
|
|
1053
|
+
plusCode: import_zod2.z.string().nullable(),
|
|
1054
|
+
bookingUrl: import_zod2.z.string().nullable()
|
|
1055
|
+
});
|
|
1056
|
+
var RawMapsHoursRowSchema = import_zod2.z.object({
|
|
1057
|
+
day: import_zod2.z.string(),
|
|
1058
|
+
hours: import_zod2.z.string()
|
|
1059
|
+
});
|
|
1060
|
+
var RawMapsReviewStatsSchema = import_zod2.z.object({
|
|
1061
|
+
reviewHistogram: import_zod2.z.array(import_zod2.z.object({
|
|
1062
|
+
stars: import_zod2.z.number(),
|
|
1063
|
+
count: import_zod2.z.string()
|
|
1064
|
+
})),
|
|
1065
|
+
reviewTopics: import_zod2.z.array(import_zod2.z.object({
|
|
1066
|
+
label: import_zod2.z.string(),
|
|
1067
|
+
count: import_zod2.z.string()
|
|
1068
|
+
}))
|
|
1069
|
+
});
|
|
1070
|
+
var RawMapsReviewCardSchema = import_zod2.z.object({
|
|
1071
|
+
reviewId: import_zod2.z.string(),
|
|
1072
|
+
author: import_zod2.z.string().nullable(),
|
|
1073
|
+
stars: import_zod2.z.string().nullable(),
|
|
1074
|
+
date: import_zod2.z.string().nullable(),
|
|
1075
|
+
text: import_zod2.z.string().nullable(),
|
|
1076
|
+
ownerResponse: import_zod2.z.string().nullable()
|
|
1077
|
+
});
|
|
1078
|
+
var RawMapsAboutAttributeSchema = import_zod2.z.object({
|
|
1079
|
+
section: import_zod2.z.string(),
|
|
1080
|
+
attribute: import_zod2.z.string()
|
|
1081
|
+
});
|
|
1082
|
+
|
|
1083
|
+
// src/workflows/workflows/directory.ts
|
|
1084
|
+
var DirectoryWorkflowCliInputSchema = import_zod3.z.object({
|
|
1085
|
+
query: import_zod3.z.string().min(1),
|
|
1086
|
+
state: import_zod3.z.string().min(2).default("TN"),
|
|
1087
|
+
minPopulation: import_zod3.z.number().int().min(0).default(1e5),
|
|
1088
|
+
maxCities: import_zod3.z.number().int().min(1).max(100).default(25),
|
|
1089
|
+
maxResultsPerCity: import_zod3.z.number().int().min(1).max(50).default(20),
|
|
1090
|
+
concurrency: import_zod3.z.number().int().min(1).max(5).default(5),
|
|
1091
|
+
proxyMode: import_zod3.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
|
|
1092
|
+
saveCsv: import_zod3.z.boolean().default(true)
|
|
734
1093
|
});
|
|
735
1094
|
function directoryRows(result) {
|
|
736
1095
|
const rows = [];
|
|
@@ -854,18 +1213,18 @@ ${summary}
|
|
|
854
1213
|
};
|
|
855
1214
|
|
|
856
1215
|
// src/workflows/workflows/local-competitive-audit.ts
|
|
857
|
-
var
|
|
858
|
-
var LocalCompetitiveAuditInputSchema =
|
|
859
|
-
query:
|
|
860
|
-
state:
|
|
861
|
-
minPopulation:
|
|
862
|
-
maxCities:
|
|
863
|
-
maxResultsPerCity:
|
|
864
|
-
hydrateTop:
|
|
865
|
-
maxReviews:
|
|
866
|
-
concurrency:
|
|
867
|
-
proxyMode:
|
|
868
|
-
returnPartial:
|
|
1216
|
+
var import_zod4 = require("zod");
|
|
1217
|
+
var LocalCompetitiveAuditInputSchema = import_zod4.z.object({
|
|
1218
|
+
query: import_zod4.z.string().min(1),
|
|
1219
|
+
state: import_zod4.z.string().min(2).default("TN"),
|
|
1220
|
+
minPopulation: import_zod4.z.number().int().min(0).default(1e5),
|
|
1221
|
+
maxCities: import_zod4.z.number().int().min(1).max(100).default(25),
|
|
1222
|
+
maxResultsPerCity: import_zod4.z.number().int().min(1).max(50).default(20),
|
|
1223
|
+
hydrateTop: import_zod4.z.number().int().min(0).max(10).default(5),
|
|
1224
|
+
maxReviews: import_zod4.z.number().int().min(0).max(500).default(50),
|
|
1225
|
+
concurrency: import_zod4.z.number().int().min(1).max(5).default(5),
|
|
1226
|
+
proxyMode: import_zod4.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
|
|
1227
|
+
returnPartial: import_zod4.z.boolean().default(true)
|
|
869
1228
|
});
|
|
870
1229
|
async function mapLimit(items, limit, fn) {
|
|
871
1230
|
const out = new Array(items.length);
|
|
@@ -1053,7 +1412,7 @@ ${summary}
|
|
|
1053
1412
|
};
|
|
1054
1413
|
|
|
1055
1414
|
// src/workflows/workflows/comparison-briefs.ts
|
|
1056
|
-
var
|
|
1415
|
+
var import_zod5 = require("zod");
|
|
1057
1416
|
|
|
1058
1417
|
// src/workflows/workflows/seo-workflow-utils.ts
|
|
1059
1418
|
var STOP_WORDS = /* @__PURE__ */ new Set([
|
|
@@ -1211,49 +1570,49 @@ async function mapLimit2(items, limit, fn) {
|
|
|
1211
1570
|
}
|
|
1212
1571
|
|
|
1213
1572
|
// src/workflows/workflows/comparison-briefs.ts
|
|
1214
|
-
var ProxyModeSchema =
|
|
1215
|
-
var MapComparisonInputSchema =
|
|
1216
|
-
query:
|
|
1217
|
-
location:
|
|
1218
|
-
state:
|
|
1219
|
-
minPopulation:
|
|
1220
|
-
maxCities:
|
|
1221
|
-
maxResultsPerCity:
|
|
1222
|
-
hydrateTop:
|
|
1223
|
-
maxReviews:
|
|
1224
|
-
concurrency:
|
|
1225
|
-
proxyMode: ProxyModeSchema.default(
|
|
1226
|
-
returnPartial:
|
|
1573
|
+
var ProxyModeSchema = import_zod5.z.enum(["location", "configured", "none"]);
|
|
1574
|
+
var MapComparisonInputSchema = import_zod5.z.object({
|
|
1575
|
+
query: import_zod5.z.string().min(1),
|
|
1576
|
+
location: import_zod5.z.string().optional(),
|
|
1577
|
+
state: import_zod5.z.string().optional(),
|
|
1578
|
+
minPopulation: import_zod5.z.number().int().min(0).default(1e5),
|
|
1579
|
+
maxCities: import_zod5.z.number().int().min(1).max(100).default(5),
|
|
1580
|
+
maxResultsPerCity: import_zod5.z.number().int().min(1).max(50).default(20),
|
|
1581
|
+
hydrateTop: import_zod5.z.number().int().min(0).max(10).default(5),
|
|
1582
|
+
maxReviews: import_zod5.z.number().int().min(0).max(500).default(25),
|
|
1583
|
+
concurrency: import_zod5.z.number().int().min(1).max(5).default(5),
|
|
1584
|
+
proxyMode: ProxyModeSchema.default(DEFAULT_MAPS_PROXY_MODE),
|
|
1585
|
+
returnPartial: import_zod5.z.boolean().default(true)
|
|
1227
1586
|
}).refine((input) => input.location || input.state, {
|
|
1228
1587
|
message: "Either location or state is required for map-comparison"
|
|
1229
1588
|
});
|
|
1230
|
-
var SerpComparisonInputSchema =
|
|
1231
|
-
keyword:
|
|
1232
|
-
domain:
|
|
1233
|
-
url:
|
|
1234
|
-
location:
|
|
1235
|
-
maxResults:
|
|
1236
|
-
maxQuestions:
|
|
1237
|
-
extractTop:
|
|
1238
|
-
includePaa:
|
|
1239
|
-
includeAiOverview:
|
|
1240
|
-
returnPartial:
|
|
1589
|
+
var SerpComparisonInputSchema = import_zod5.z.object({
|
|
1590
|
+
keyword: import_zod5.z.string().min(1),
|
|
1591
|
+
domain: import_zod5.z.string().optional(),
|
|
1592
|
+
url: import_zod5.z.string().url().optional(),
|
|
1593
|
+
location: import_zod5.z.string().optional(),
|
|
1594
|
+
maxResults: import_zod5.z.number().int().min(1).max(20).default(10),
|
|
1595
|
+
maxQuestions: import_zod5.z.number().int().min(1).max(200).default(40),
|
|
1596
|
+
extractTop: import_zod5.z.number().int().min(0).max(10).default(5),
|
|
1597
|
+
includePaa: import_zod5.z.boolean().default(true),
|
|
1598
|
+
includeAiOverview: import_zod5.z.boolean().default(true),
|
|
1599
|
+
returnPartial: import_zod5.z.boolean().default(true)
|
|
1241
1600
|
});
|
|
1242
|
-
var PaaExpansionBriefInputSchema =
|
|
1243
|
-
keyword:
|
|
1244
|
-
location:
|
|
1245
|
-
maxQuestions:
|
|
1246
|
-
depth:
|
|
1247
|
-
returnPartial:
|
|
1601
|
+
var PaaExpansionBriefInputSchema = import_zod5.z.object({
|
|
1602
|
+
keyword: import_zod5.z.string().min(1),
|
|
1603
|
+
location: import_zod5.z.string().optional(),
|
|
1604
|
+
maxQuestions: import_zod5.z.number().int().min(1).max(300).default(80),
|
|
1605
|
+
depth: import_zod5.z.number().int().min(1).max(6).default(3),
|
|
1606
|
+
returnPartial: import_zod5.z.boolean().default(true)
|
|
1248
1607
|
});
|
|
1249
|
-
var AiOverviewLanguageInputSchema =
|
|
1250
|
-
keyword:
|
|
1251
|
-
domain:
|
|
1252
|
-
url:
|
|
1253
|
-
location:
|
|
1254
|
-
maxQuestions:
|
|
1255
|
-
extractTop:
|
|
1256
|
-
returnPartial:
|
|
1608
|
+
var AiOverviewLanguageInputSchema = import_zod5.z.object({
|
|
1609
|
+
keyword: import_zod5.z.string().min(1),
|
|
1610
|
+
domain: import_zod5.z.string().optional(),
|
|
1611
|
+
url: import_zod5.z.string().url().optional(),
|
|
1612
|
+
location: import_zod5.z.string().optional(),
|
|
1613
|
+
maxQuestions: import_zod5.z.number().int().min(1).max(200).default(40),
|
|
1614
|
+
extractTop: import_zod5.z.number().int().min(0).max(8).default(3),
|
|
1615
|
+
returnPartial: import_zod5.z.boolean().default(true)
|
|
1257
1616
|
});
|
|
1258
1617
|
function businessRowsFromMaps(location, query, results) {
|
|
1259
1618
|
return results.map((result) => ({
|
|
@@ -1769,22 +2128,36 @@ function workflowDefinition(id) {
|
|
|
1769
2128
|
if (!definition) throw new Error(`Unknown workflow "${id}". Available: ${DEFINITIONS.map((def) => def.id).join(", ")}`);
|
|
1770
2129
|
return definition;
|
|
1771
2130
|
}
|
|
2131
|
+
function resolveApiKey(options) {
|
|
2132
|
+
const apiKey = options.apiKey?.trim() || process.env.MCP_SCRAPER_API_KEY?.trim();
|
|
2133
|
+
if (!apiKey) throw new Error("MCP_SCRAPER_API_KEY is required for workflow runs. Pass --api-key or set the environment variable.");
|
|
2134
|
+
return apiKey;
|
|
2135
|
+
}
|
|
2136
|
+
function resolveApiUrl(options) {
|
|
2137
|
+
return options.apiUrl?.trim() || process.env.MCP_SCRAPER_API_URL?.trim() || "https://mcpscraper.dev";
|
|
2138
|
+
}
|
|
1772
2139
|
async function runWorkflow(id, rawInput, options = {}) {
|
|
1773
2140
|
const definition = workflowDefinition(id);
|
|
1774
2141
|
const input = definition.inputSchema.parse(rawInput);
|
|
1775
|
-
const apiKey = options
|
|
1776
|
-
|
|
1777
|
-
const apiUrl = options.apiUrl?.trim() || process.env.MCP_SCRAPER_API_URL?.trim() || "https://mcpscraper.dev";
|
|
2142
|
+
const apiKey = resolveApiKey(options);
|
|
2143
|
+
const apiUrl = resolveApiUrl(options);
|
|
1778
2144
|
const artifacts = await ArtifactWriter.create(definition.id, definition.title, input, options.outputDir, options.runId);
|
|
1779
2145
|
const client = new WorkflowHttpClient(apiUrl, apiKey, options.fetchImpl, options.headers);
|
|
2146
|
+
const ctx = { runId: artifacts.runId, startedAt: artifacts.startedAt, client, artifacts, signal: options.signal };
|
|
1780
2147
|
try {
|
|
1781
|
-
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
|
|
2148
|
+
if (definition.steps && definition.steps.length > 0) {
|
|
2149
|
+
let state = definition.createState ? definition.createState(input) : {};
|
|
2150
|
+
let summary = null;
|
|
2151
|
+
for (const step of definition.steps) {
|
|
2152
|
+
const outcome = await step.run({ input, state, ctx });
|
|
2153
|
+
state = outcome.state;
|
|
2154
|
+
if (outcome.summary) summary = outcome.summary;
|
|
2155
|
+
}
|
|
2156
|
+
if (!summary) throw new Error(`Workflow "${id}" produced no terminal step summary`);
|
|
2157
|
+
return summary;
|
|
2158
|
+
}
|
|
2159
|
+
if (definition.run) return await definition.run(input, ctx);
|
|
2160
|
+
throw new Error(`Workflow "${id}" has neither steps nor a run() implementation`);
|
|
1788
2161
|
} catch (err) {
|
|
1789
2162
|
const message = err instanceof Error ? err.message : String(err);
|
|
1790
2163
|
await artifacts.writeText("Failure", "summary.md", `# ${definition.title}
|
|
@@ -1946,11 +2319,87 @@ function buildHumanCli() {
|
|
|
1946
2319
|
result.next_step ?? "Complete checkout, then retry the MCP request."
|
|
1947
2320
|
].join("\n"), false);
|
|
1948
2321
|
});
|
|
2322
|
+
const browser = program.command("browser").description("Inspect and manage browser-agent profile setup helpers.");
|
|
2323
|
+
browser.command("profiles").description("List local Chrome profiles and suggested managed browser profile names.").option("--email <email>", "Filter by Chrome account email or displayed profile name").option("--user-data-dir <path>", "Chrome user data directory").option("--json", "Print machine-readable JSON").action(async (opts) => {
|
|
2324
|
+
const profiles = await listLocalChromeProfiles(opts.userDataDir, opts.email);
|
|
2325
|
+
if (opts.json) {
|
|
2326
|
+
writeOutput({ profiles }, true);
|
|
2327
|
+
return;
|
|
2328
|
+
}
|
|
2329
|
+
if (!profiles.length) {
|
|
2330
|
+
writeOutput("No matching Chrome profiles found.", false);
|
|
2331
|
+
return;
|
|
2332
|
+
}
|
|
2333
|
+
const lines = [
|
|
2334
|
+
"Local Chrome profiles:",
|
|
2335
|
+
...profiles.map((profile) => [
|
|
2336
|
+
`${profile.email || "(no email)"} ${profile.chromeProfileDirectory} ${profile.displayName}`,
|
|
2337
|
+
` suggested profile name: ${profile.recommendedKernelProfileName}`,
|
|
2338
|
+
` local Chrome path: ${profile.chromeProfilePath}`
|
|
2339
|
+
].join("\n")),
|
|
2340
|
+
"",
|
|
2341
|
+
"To clone a real local Chrome profile for MCP local browser mode:",
|
|
2342
|
+
" mcp-scraper-cli browser import-chrome --email you@example.com --name <suggested profile name>",
|
|
2343
|
+
"Then generate config with:",
|
|
2344
|
+
" mcp-scraper-cli agent install claude-desktop --browser-mode local --browser-profile <suggested profile name>"
|
|
2345
|
+
];
|
|
2346
|
+
writeOutput(lines.join("\n"), false);
|
|
2347
|
+
});
|
|
2348
|
+
browser.command("import-chrome").description("Clone a local Chrome profile into MCP Scraper managed local browser storage.").option("--email <email>", "Chrome account email or displayed profile name to import").option("--name <name>", "Managed MCP Scraper browser profile name").option("--chrome-profile <directory>", "Chrome profile directory, for example Default or Profile 1").option("--user-data-dir <path>", "Source Chrome user data directory").option("--output-dir <path>", "Managed profile base directory").option("--browser-executable <path>", "Chrome executable path for local launches").option("--overwrite", "Overwrite an existing managed profile clone").option("--json", "Print machine-readable JSON").action(async (opts) => {
|
|
2349
|
+
const result = await importChromeProfile({
|
|
2350
|
+
email: opts.email,
|
|
2351
|
+
name: opts.name,
|
|
2352
|
+
chromeProfileDirectory: opts.chromeProfile,
|
|
2353
|
+
sourceUserDataDir: opts.userDataDir,
|
|
2354
|
+
outputDir: opts.outputDir,
|
|
2355
|
+
browserExecutablePath: opts.browserExecutable,
|
|
2356
|
+
overwrite: opts.overwrite
|
|
2357
|
+
});
|
|
2358
|
+
if (opts.json) {
|
|
2359
|
+
writeOutput(result, true);
|
|
2360
|
+
return;
|
|
2361
|
+
}
|
|
2362
|
+
writeOutput([
|
|
2363
|
+
`Imported Chrome profile: ${result.profile.name}`,
|
|
2364
|
+
`Source: ${result.sourceProfile.email || result.sourceProfile.displayName || result.sourceProfile.chromeProfileDirectory} (${result.sourceProfile.chromeProfileDirectory})`,
|
|
2365
|
+
`Managed user data dir: ${result.profile.userDataDir}`,
|
|
2366
|
+
"",
|
|
2367
|
+
"Install/config command:",
|
|
2368
|
+
` mcp-scraper-cli agent install codex --browser-mode local --browser-profile ${result.profile.name}`,
|
|
2369
|
+
"",
|
|
2370
|
+
"Restart the MCP client after updating config. Re-run sync-profile after signing into new sites in normal Chrome."
|
|
2371
|
+
].join("\n"), false);
|
|
2372
|
+
});
|
|
2373
|
+
browser.command("sync-profile <name>").description("Refresh a managed local browser profile from its recorded source Chrome profile.").option("--output-dir <path>", "Managed profile base directory").option("--browser-executable <path>", "Chrome executable path for local launches").option("--json", "Print machine-readable JSON").action(async (name, opts) => {
|
|
2374
|
+
const result = await syncImportedChromeProfile(name, {
|
|
2375
|
+
outputDir: opts.outputDir,
|
|
2376
|
+
browserExecutablePath: opts.browserExecutable
|
|
2377
|
+
});
|
|
2378
|
+
if (opts.json) {
|
|
2379
|
+
writeOutput(result, true);
|
|
2380
|
+
return;
|
|
2381
|
+
}
|
|
2382
|
+
writeOutput([
|
|
2383
|
+
`Synced Chrome profile: ${result.profile.name}`,
|
|
2384
|
+
`Source: ${result.sourceProfile.email || result.sourceProfile.displayName || result.sourceProfile.chromeProfileDirectory} (${result.sourceProfile.chromeProfileDirectory})`,
|
|
2385
|
+
`Managed user data dir: ${result.profile.userDataDir}`,
|
|
2386
|
+
"Restart any local browser sessions that were using this profile."
|
|
2387
|
+
].join("\n"), false);
|
|
2388
|
+
});
|
|
1949
2389
|
const agent = program.command("agent").description("Generate AI-agent install configs and workflow prompts.");
|
|
1950
|
-
agent.command("install <host>").description("Print install/config instructions for codex, claude, or claude-desktop.").option("--api-key <key>", "API key to place in generated config").option("--package <spec>", "npm package spec", "mcp-scraper@latest").option("--json", "Print machine-readable JSON").action((host, opts) => {
|
|
2390
|
+
agent.command("install <host>").description("Print install/config instructions for codex, claude, or claude-desktop.").option("--api-key <key>", "API key to place in generated config").option("--package <spec>", "npm package spec", "mcp-scraper@latest").option("--browser-mode <mode>", "Browser mode: hosted or local", "hosted").option("--browser-profile <name>", "Default browser profile for browser_open sessions. In local mode this is a managed MCP Scraper profile name; in hosted mode it is a Kernel profile name.").option("--browser-executable <path>", "Chrome executable path for local browser mode").option("--save-browser-profile-changes", "Persist cookies/local storage back to the named browser profile when sessions close").option("--json", "Print machine-readable JSON").action((host, opts) => {
|
|
1951
2391
|
const valid = ["codex", "claude", "claude-desktop"];
|
|
1952
2392
|
if (!valid.includes(host)) throw new Error(`Unknown host "${host}". Use: ${valid.join(", ")}`);
|
|
1953
|
-
const
|
|
2393
|
+
const browserMode = String(opts.browserMode ?? "hosted");
|
|
2394
|
+
if (!["hosted", "local"].includes(browserMode)) throw new Error("Unknown browser mode. Use: hosted or local");
|
|
2395
|
+
const text = renderAgentInstall(host, {
|
|
2396
|
+
apiKey: opts.apiKey,
|
|
2397
|
+
packageSpec: opts.package,
|
|
2398
|
+
browserMode,
|
|
2399
|
+
browserProfileName: opts.browserProfile,
|
|
2400
|
+
browserProfileSaveChanges: opts.saveBrowserProfileChanges,
|
|
2401
|
+
browserExecutablePath: opts.browserExecutable
|
|
2402
|
+
});
|
|
1954
2403
|
writeOutput(opts.json ? { host, text } : text, opts.json);
|
|
1955
2404
|
});
|
|
1956
2405
|
agent.command("prompt [name]").description("Print an agent prompt template.").option("--json", "Print machine-readable JSON").action((name, opts) => {
|
|
@@ -2037,14 +2486,14 @@ ${summary.warnings.map((w) => `- ${w}`).join("\n")}` : ""
|
|
|
2037
2486
|
runs.command("download <id>").description("Download hosted workflow run artifacts.").option("--api-key <key>", "MCP Scraper API key").option("--api-url <url>", "MCP Scraper API URL", "https://mcpscraper.dev").option("--output-dir <path>", "Download directory").option("--json", "Print machine-readable JSON").action(async (id, opts) => {
|
|
2038
2487
|
const result = await apiRequest(`/workflows/runs/${id}`, "GET", opts);
|
|
2039
2488
|
const { apiUrl, apiKey } = apiOptions(opts);
|
|
2040
|
-
const outDir = (0,
|
|
2041
|
-
await (0,
|
|
2489
|
+
const outDir = (0, import_node_path5.join)(opts.outputDir ?? workflowOutputBaseDir(), "workflow-downloads", id);
|
|
2490
|
+
await (0, import_promises6.mkdir)(outDir, { recursive: true });
|
|
2042
2491
|
const downloaded = [];
|
|
2043
2492
|
for (const artifact of result.run.artifacts ?? []) {
|
|
2044
2493
|
const res = await fetch(`${apiUrl}/workflows/runs/${id}/artifacts/${artifact.id}`, { headers: { "x-api-key": apiKey } });
|
|
2045
2494
|
if (!res.ok) throw new Error(`Failed to download ${artifact.label}: HTTP ${res.status}`);
|
|
2046
|
-
const file = (0,
|
|
2047
|
-
await (0,
|
|
2495
|
+
const file = (0, import_node_path5.join)(outDir, (0, import_node_path5.basename)(artifact.path));
|
|
2496
|
+
await (0, import_promises6.writeFile)(file, Buffer.from(await res.arrayBuffer()));
|
|
2048
2497
|
downloaded.push(file);
|
|
2049
2498
|
}
|
|
2050
2499
|
writeOutput(opts.json ? { runId: id, files: downloaded } : downloaded.join("\n"), opts.json);
|