mcp-scraper 0.2.20 → 0.2.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/README.md +34 -3
  2. package/dist/bin/api-server.cjs +52 -1
  3. package/dist/bin/api-server.cjs.map +1 -1
  4. package/dist/bin/api-server.js +2 -2
  5. package/dist/bin/browser-agent-stdio-server.cjs +960 -20
  6. package/dist/bin/browser-agent-stdio-server.cjs.map +1 -1
  7. package/dist/bin/browser-agent-stdio-server.js +3 -1
  8. package/dist/bin/browser-agent-stdio-server.js.map +1 -1
  9. package/dist/bin/mcp-scraper-cli.cjs +342 -47
  10. package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
  11. package/dist/bin/mcp-scraper-cli.js +105 -7
  12. package/dist/bin/mcp-scraper-cli.js.map +1 -1
  13. package/dist/bin/mcp-scraper-combined-stdio-server.cjs +976 -36
  14. package/dist/bin/mcp-scraper-combined-stdio-server.cjs.map +1 -1
  15. package/dist/bin/mcp-scraper-combined-stdio-server.js +3 -1
  16. package/dist/bin/mcp-scraper-combined-stdio-server.js.map +1 -1
  17. package/dist/bin/paa-harvest.cjs.map +1 -1
  18. package/dist/bin/paa-harvest.js +4 -2
  19. package/dist/bin/paa-harvest.js.map +1 -1
  20. package/dist/chunk-DUEW4EOO.js +214 -0
  21. package/dist/chunk-DUEW4EOO.js.map +1 -0
  22. package/dist/{chunk-L4OWOUGR.js → chunk-HS6OKELW.js} +737 -3
  23. package/dist/chunk-HS6OKELW.js.map +1 -0
  24. package/dist/{chunk-XGUDTDZ2.js → chunk-IPW4LFOT.js} +5 -13
  25. package/dist/chunk-IPW4LFOT.js.map +1 -0
  26. package/dist/chunk-WN7PBKMV.js +28 -0
  27. package/dist/chunk-WN7PBKMV.js.map +1 -0
  28. package/dist/index.cjs.map +1 -1
  29. package/dist/index.js +2 -1
  30. package/dist/index.js.map +1 -1
  31. package/dist/{server-7NE45K2A.js → server-24XIQ6W6.js} +40 -6
  32. package/dist/server-24XIQ6W6.js.map +1 -0
  33. package/dist/{worker-MIYG2B2I.js → worker-SLQ375UG.js} +5 -3
  34. package/dist/{worker-MIYG2B2I.js.map → worker-SLQ375UG.js.map} +1 -1
  35. package/docs/mcp-tool-manifest.generated.json +132 -4
  36. package/package.json +1 -1
  37. package/dist/chunk-L4OWOUGR.js.map +0 -1
  38. package/dist/chunk-XGUDTDZ2.js.map +0 -1
  39. package/dist/server-7NE45K2A.js.map +0 -1
@@ -1,7 +1,9 @@
1
1
  #!/usr/bin/env node
2
2
  import {
3
3
  buildBrowserAgentMcpServer
4
- } from "../chunk-L4OWOUGR.js";
4
+ } from "../chunk-HS6OKELW.js";
5
+ import "../chunk-WN7PBKMV.js";
6
+ import "../chunk-DUEW4EOO.js";
5
7
  import "../chunk-XKUDVN2E.js";
6
8
 
7
9
  // bin/browser-agent-stdio-server.ts
@@ -1 +1 @@
1
- {"version":3,"sources":["../../bin/browser-agent-stdio-server.ts"],"sourcesContent":["#!/usr/bin/env node\nimport { readFileSync } from 'node:fs'\nimport { homedir } from 'node:os'\nimport { join } from 'node:path'\nimport { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'\nimport { buildBrowserAgentMcpServer } from '../src/mcp/browser-agent-mcp-server.js'\n\nfunction readApiKeyFile(): string | undefined {\n const explicitPath = process.env.MCP_SCRAPER_KEY_PATH?.trim()\n const paths = [explicitPath, join(homedir(), '.mcp-scraper-key')].filter(Boolean) as string[]\n for (const path of paths) {\n try {\n const value = readFileSync(path, 'utf8').trim()\n if (value) return value\n } catch {\n void 0\n }\n }\n return undefined\n}\n\nconst apiKey = (\n process.env.MCP_SCRAPER_API_KEY ??\n process.env.MCP_SCRAPER_KEY ??\n process.env.MCP_API_KEY ??\n readApiKeyFile()\n)?.trim()\nif (!apiKey) {\n process.stderr.write('MCP_SCRAPER_API_KEY env var or ~/.mcp-scraper-key is required\\n')\n process.exit(1)\n}\n\nconst baseUrl = process.env.MCP_SCRAPER_BASE_URL?.trim() || process.env.MCP_BASE_URL?.trim() || 'https://mcpscraper.dev'\nconst consoleBaseUrl = process.env.BROWSER_AGENT_CONSOLE_URL?.trim() || baseUrl\nconst server = buildBrowserAgentMcpServer({ baseUrl, apiKey, consoleBaseUrl })\nconst transport = new StdioServerTransport()\n\nasync function main() {\n await server.connect(transport)\n}\n\nmain().catch(err => {\n process.stderr.write(`${err instanceof Error ? err.message : String(err)}\\n`)\n process.exit(1)\n})\n"],"mappings":";;;;;;;AACA,SAAS,oBAAoB;AAC7B,SAAS,eAAe;AACxB,SAAS,YAAY;AACrB,SAAS,4BAA4B;AAGrC,SAAS,iBAAqC;AAC5C,QAAM,eAAe,QAAQ,IAAI,sBAAsB,KAAK;AAC5D,QAAM,QAAQ,CAAC,cAAc,KAAK,QAAQ,GAAG,kBAAkB,CAAC,EAAE,OAAO,OAAO;AAChF,aAAW,QAAQ,OAAO;AACxB,QAAI;AACF,YAAM,QAAQ,aAAa,MAAM,MAAM,EAAE,KAAK;AAC9C,UAAI,MAAO,QAAO;AAAA,IACpB,QAAQ;AAAA,IAER;AAAA,EACF;AACA,SAAO;AACT;AAEA,IAAM,UACJ,QAAQ,IAAI,uBACZ,QAAQ,IAAI,mBACZ,QAAQ,IAAI,eACZ,eAAe,IACd,KAAK;AACR,IAAI,CAAC,QAAQ;AACX,UAAQ,OAAO,MAAM,iEAAiE;AACtF,UAAQ,KAAK,CAAC;AAChB;AAEA,IAAM,UAAU,QAAQ,IAAI,sBAAsB,KAAK,KAAK,QAAQ,IAAI,cAAc,KAAK,KAAK;AAChG,IAAM,iBAAiB,QAAQ,IAAI,2BAA2B,KAAK,KAAK;AACxE,IAAM,SAAS,2BAA2B,EAAE,SAAS,QAAQ,eAAe,CAAC;AAC7E,IAAM,YAAY,IAAI,qBAAqB;AAE3C,eAAe,OAAO;AACpB,QAAM,OAAO,QAAQ,SAAS;AAChC;AAEA,KAAK,EAAE,MAAM,SAAO;AAClB,UAAQ,OAAO,MAAM,GAAG,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,CAAI;AAC5E,UAAQ,KAAK,CAAC;AAChB,CAAC;","names":[]}
1
+ {"version":3,"sources":["../../bin/browser-agent-stdio-server.ts"],"sourcesContent":["#!/usr/bin/env node\nimport { readFileSync } from 'node:fs'\nimport { homedir } from 'node:os'\nimport { join } from 'node:path'\nimport { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'\nimport { buildBrowserAgentMcpServer } from '../src/mcp/browser-agent-mcp-server.js'\n\nfunction readApiKeyFile(): string | undefined {\n const explicitPath = process.env.MCP_SCRAPER_KEY_PATH?.trim()\n const paths = [explicitPath, join(homedir(), '.mcp-scraper-key')].filter(Boolean) as string[]\n for (const path of paths) {\n try {\n const value = readFileSync(path, 'utf8').trim()\n if (value) return value\n } catch {\n void 0\n }\n }\n return undefined\n}\n\nconst apiKey = (\n process.env.MCP_SCRAPER_API_KEY ??\n process.env.MCP_SCRAPER_KEY ??\n process.env.MCP_API_KEY ??\n readApiKeyFile()\n)?.trim()\nif (!apiKey) {\n process.stderr.write('MCP_SCRAPER_API_KEY env var or ~/.mcp-scraper-key is required\\n')\n process.exit(1)\n}\n\nconst baseUrl = process.env.MCP_SCRAPER_BASE_URL?.trim() || process.env.MCP_BASE_URL?.trim() || 'https://mcpscraper.dev'\nconst consoleBaseUrl = process.env.BROWSER_AGENT_CONSOLE_URL?.trim() || baseUrl\nconst server = buildBrowserAgentMcpServer({ baseUrl, apiKey, consoleBaseUrl })\nconst transport = new StdioServerTransport()\n\nasync function main() {\n await server.connect(transport)\n}\n\nmain().catch(err => {\n process.stderr.write(`${err instanceof Error ? err.message : String(err)}\\n`)\n process.exit(1)\n})\n"],"mappings":";;;;;;;;;AACA,SAAS,oBAAoB;AAC7B,SAAS,eAAe;AACxB,SAAS,YAAY;AACrB,SAAS,4BAA4B;AAGrC,SAAS,iBAAqC;AAC5C,QAAM,eAAe,QAAQ,IAAI,sBAAsB,KAAK;AAC5D,QAAM,QAAQ,CAAC,cAAc,KAAK,QAAQ,GAAG,kBAAkB,CAAC,EAAE,OAAO,OAAO;AAChF,aAAW,QAAQ,OAAO;AACxB,QAAI;AACF,YAAM,QAAQ,aAAa,MAAM,MAAM,EAAE,KAAK;AAC9C,UAAI,MAAO,QAAO;AAAA,IACpB,QAAQ;AAAA,IAER;AAAA,EACF;AACA,SAAO;AACT;AAEA,IAAM,UACJ,QAAQ,IAAI,uBACZ,QAAQ,IAAI,mBACZ,QAAQ,IAAI,eACZ,eAAe,IACd,KAAK;AACR,IAAI,CAAC,QAAQ;AACX,UAAQ,OAAO,MAAM,iEAAiE;AACtF,UAAQ,KAAK,CAAC;AAChB;AAEA,IAAM,UAAU,QAAQ,IAAI,sBAAsB,KAAK,KAAK,QAAQ,IAAI,cAAc,KAAK,KAAK;AAChG,IAAM,iBAAiB,QAAQ,IAAI,2BAA2B,KAAK,KAAK;AACxE,IAAM,SAAS,2BAA2B,EAAE,SAAS,QAAQ,eAAe,CAAC;AAC7E,IAAM,YAAY,IAAI,qBAAqB;AAE3C,eAAe,OAAO;AACpB,QAAM,OAAO,QAAQ,SAAS;AAChC;AAEA,KAAK,EAAE,MAAM,SAAO;AAClB,UAAQ,OAAO,MAAM,GAAG,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,CAAI;AAC5E,UAAQ,KAAK,CAAC;AAChB,CAAC;","names":[]}
@@ -4,12 +4,214 @@
4
4
  // src/cli/human-cli.ts
5
5
  var import_commander = require("commander");
6
6
  var import_node_child_process2 = require("child_process");
7
- var import_promises4 = require("fs/promises");
8
- var import_node_path3 = require("path");
7
+ var import_promises6 = require("fs/promises");
8
+ var import_node_path5 = require("path");
9
9
 
10
10
  // src/version.ts
11
11
  var PACKAGE_VERSION = "0.2.20";
12
12
 
13
+ // src/lib/chrome-profiles.ts
14
+ var import_promises = require("fs/promises");
15
+ var import_node_os = require("os");
16
+ var import_node_path = require("path");
17
+ function defaultChromeUserDataDir() {
18
+ if (process.platform === "darwin") return (0, import_node_path.join)((0, import_node_os.homedir)(), "Library", "Application Support", "Google", "Chrome");
19
+ if (process.platform === "win32") {
20
+ return (0, import_node_path.join)(process.env.LOCALAPPDATA ?? (0, import_node_path.join)((0, import_node_os.homedir)(), "AppData", "Local"), "Google", "Chrome", "User Data");
21
+ }
22
+ return (0, import_node_path.join)(process.env.XDG_CONFIG_HOME ?? (0, import_node_path.join)((0, import_node_os.homedir)(), ".config"), "google-chrome");
23
+ }
24
+ function recommendedKernelProfileName(value) {
25
+ return value.trim().toLowerCase().replace(/@/g, "-").replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80) || "browser-profile";
26
+ }
27
+ async function listLocalChromeProfiles(userDataDir = defaultChromeUserDataDir(), emailFilter) {
28
+ const raw = await (0, import_promises.readFile)((0, import_node_path.join)(userDataDir, "Local State"), "utf8");
29
+ const localState = JSON.parse(raw);
30
+ const wanted = emailFilter?.trim().toLowerCase();
31
+ return Object.entries(localState.profile?.info_cache ?? {}).map(([directory, info]) => {
32
+ const email = (info.user_name ?? "").trim();
33
+ const displayName = (info.name ?? info.shortcut_name ?? "").trim();
34
+ const labelSource = email || displayName || directory;
35
+ return {
36
+ email,
37
+ displayName,
38
+ chromeProfileDirectory: directory,
39
+ chromeProfilePath: (0, import_node_path.join)(userDataDir, directory),
40
+ recommendedKernelProfileName: recommendedKernelProfileName(labelSource)
41
+ };
42
+ }).filter((row) => !wanted || row.email.toLowerCase() === wanted || row.displayName.toLowerCase().includes(wanted));
43
+ }
44
+
45
+ // src/lib/local-browser-profiles.ts
46
+ var import_node_fs = require("fs");
47
+ var import_promises2 = require("fs/promises");
48
+ var import_node_os2 = require("os");
49
+ var import_node_path2 = require("path");
50
+ var EXCLUDED_PROFILE_NAMES = [
51
+ "Cache",
52
+ "Code Cache",
53
+ "Crashpad",
54
+ "DawnCache",
55
+ "DawnGraphiteCache",
56
+ "DawnWebGPUCache",
57
+ "GPUCache",
58
+ "GrShaderCache",
59
+ "ShaderCache",
60
+ "Shared Dictionary"
61
+ ];
62
+ function defaultImportedBrowserProfilesDir() {
63
+ return process.env.MCP_SCRAPER_BROWSER_PROFILE_BASE_DIR?.trim() || (0, import_node_path2.join)((0, import_node_os2.homedir)(), ".mcp-scraper", "browser-profiles");
64
+ }
65
+ function defaultChromeExecutablePath() {
66
+ const envPath = process.env.MCP_SCRAPER_BROWSER_EXECUTABLE?.trim() || process.env.CHROME_PATH?.trim();
67
+ if (envPath) return envPath;
68
+ const candidates = process.platform === "darwin" ? ["/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"] : process.platform === "win32" ? [
69
+ (0, import_node_path2.join)(process.env.PROGRAMFILES ?? "C:\\Program Files", "Google", "Chrome", "Application", "chrome.exe"),
70
+ (0, import_node_path2.join)(process.env["PROGRAMFILES(X86)"] ?? "C:\\Program Files (x86)", "Google", "Chrome", "Application", "chrome.exe")
71
+ ] : ["/usr/bin/google-chrome", "/usr/bin/google-chrome-stable", "/usr/bin/chromium-browser", "/usr/bin/chromium"];
72
+ return candidates.find((candidate) => (0, import_node_fs.existsSync)(candidate));
73
+ }
74
+ function importedProfileName(value) {
75
+ return recommendedKernelProfileName(value);
76
+ }
77
+ function importedProfileRoot(name, outputDir = defaultImportedBrowserProfilesDir()) {
78
+ return (0, import_node_path2.join)(outputDir, importedProfileName(name));
79
+ }
80
+ function importedProfileManifestPath(name, outputDir = defaultImportedBrowserProfilesDir()) {
81
+ return (0, import_node_path2.join)(importedProfileRoot(name, outputDir), "manifest.json");
82
+ }
83
+ function importedProfileUserDataDir(name, outputDir = defaultImportedBrowserProfilesDir()) {
84
+ return (0, import_node_path2.join)(importedProfileRoot(name, outputDir), "user-data");
85
+ }
86
+ async function readExistingManifest(path) {
87
+ try {
88
+ return JSON.parse(await (0, import_promises2.readFile)(path, "utf8"));
89
+ } catch {
90
+ return null;
91
+ }
92
+ }
93
+ function isExcludedProfilePath(sourceRoot, sourcePath) {
94
+ const rel = (0, import_node_path2.relative)(sourceRoot, sourcePath);
95
+ if (!rel) return false;
96
+ const parts = rel.split(import_node_path2.sep).filter(Boolean);
97
+ const last = parts[parts.length - 1] ?? "";
98
+ if (last.startsWith("Singleton")) return true;
99
+ if (last === "LOCK" || last.endsWith(".lock")) return true;
100
+ if (parts.some((part) => EXCLUDED_PROFILE_NAMES.includes(part))) return true;
101
+ if (parts.includes("Service Worker") && parts.includes("CacheStorage")) return true;
102
+ return false;
103
+ }
104
+ async function copyProfileTree(sourceProfilePath, destinationProfilePath) {
105
+ await (0, import_promises2.cp)(sourceProfilePath, destinationProfilePath, {
106
+ recursive: true,
107
+ force: true,
108
+ errorOnExist: false,
109
+ dereference: false,
110
+ filter: (source) => !isExcludedProfilePath(sourceProfilePath, source)
111
+ });
112
+ }
113
+ function selectProfile(profiles, options) {
114
+ const requestedDirectory = options.chromeProfileDirectory?.trim();
115
+ const requestedEmail = options.email?.trim().toLowerCase();
116
+ const exactDirectory = requestedDirectory ? profiles.find((profile) => profile.chromeProfileDirectory === requestedDirectory) : void 0;
117
+ if (exactDirectory) return exactDirectory;
118
+ const exactEmail = requestedEmail ? profiles.find((profile) => profile.email.toLowerCase() === requestedEmail) : void 0;
119
+ if (exactEmail) return exactEmail;
120
+ const first = profiles[0];
121
+ if (first) return first;
122
+ const hint = requestedDirectory ? `Chrome profile directory "${requestedDirectory}"` : requestedEmail ? `Chrome account "${requestedEmail}"` : "Chrome profile";
123
+ throw new Error(`${hint} was not found in ${options.sourceUserDataDir ?? defaultChromeUserDataDir()}`);
124
+ }
125
+ async function ensureProfilePathExists(profile) {
126
+ const details = await (0, import_promises2.stat)(profile.chromeProfilePath).catch(() => null);
127
+ if (!details?.isDirectory()) {
128
+ throw new Error(`Chrome profile path does not exist or is not a directory: ${profile.chromeProfilePath}`);
129
+ }
130
+ }
131
+ async function writeNormalizedLocalState(sourceLocalStatePath, destinationLocalStatePath, sourceProfileDirectory) {
132
+ const raw = await (0, import_promises2.readFile)(sourceLocalStatePath, "utf8");
133
+ const localState = JSON.parse(raw);
134
+ const selectedInfo = localState.profile?.info_cache?.[sourceProfileDirectory] ?? localState.profile?.info_cache?.Default ?? {};
135
+ localState.profile = {
136
+ ...localState.profile ?? {},
137
+ info_cache: { Default: selectedInfo },
138
+ last_used: "Default",
139
+ last_active_profiles: ["Default"]
140
+ };
141
+ await (0, import_promises2.writeFile)(destinationLocalStatePath, `${JSON.stringify(localState, null, 2)}
142
+ `);
143
+ }
144
+ async function importChromeProfile(options = {}) {
145
+ const sourceUserDataDir = options.sourceUserDataDir?.trim() || defaultChromeUserDataDir();
146
+ const profiles = await listLocalChromeProfiles(sourceUserDataDir, options.email);
147
+ const sourceProfile = selectProfile(profiles, { ...options, sourceUserDataDir });
148
+ await ensureProfilePathExists(sourceProfile);
149
+ const name = importedProfileName(options.name?.trim() || sourceProfile.recommendedKernelProfileName);
150
+ const outputDir = options.outputDir?.trim() || defaultImportedBrowserProfilesDir();
151
+ const profileRoot = importedProfileRoot(name, outputDir);
152
+ const manifestPath = importedProfileManifestPath(name, outputDir);
153
+ const userDataDir = importedProfileUserDataDir(name, outputDir);
154
+ const existingManifest = await readExistingManifest(manifestPath);
155
+ const existing = (0, import_node_fs.existsSync)(profileRoot);
156
+ if (existing && !options.overwrite) {
157
+ throw new Error(`Imported browser profile "${name}" already exists at ${profileRoot}. Re-run with --overwrite to refresh it.`);
158
+ }
159
+ if (existing) await (0, import_promises2.rm)(profileRoot, { recursive: true, force: true });
160
+ await (0, import_promises2.mkdir)((0, import_node_path2.join)(userDataDir, "Default"), { recursive: true });
161
+ await writeNormalizedLocalState(
162
+ (0, import_node_path2.join)(sourceUserDataDir, "Local State"),
163
+ (0, import_node_path2.join)(userDataDir, "Local State"),
164
+ sourceProfile.chromeProfileDirectory
165
+ );
166
+ await copyProfileTree(sourceProfile.chromeProfilePath, (0, import_node_path2.join)(userDataDir, "Default"));
167
+ const now = (/* @__PURE__ */ new Date()).toISOString();
168
+ const manifest = {
169
+ layoutVersion: 1,
170
+ name,
171
+ profileRoot,
172
+ userDataDir,
173
+ browserExecutablePath: options.browserExecutablePath?.trim() || defaultChromeExecutablePath() || null,
174
+ sourceUserDataDir,
175
+ sourceProfileDirectory: sourceProfile.chromeProfileDirectory,
176
+ sourceProfilePath: sourceProfile.chromeProfilePath,
177
+ email: sourceProfile.email,
178
+ displayName: sourceProfile.displayName,
179
+ createdAt: existingManifest?.createdAt ?? now,
180
+ updatedAt: now,
181
+ copyMode: "chrome-profile-clone"
182
+ };
183
+ await (0, import_promises2.writeFile)(manifestPath, `${JSON.stringify(manifest, null, 2)}
184
+ `);
185
+ return {
186
+ profile: manifest,
187
+ sourceProfile,
188
+ excludedNames: EXCLUDED_PROFILE_NAMES
189
+ };
190
+ }
191
+ async function loadImportedBrowserProfile(name = process.env.MCP_SCRAPER_BROWSER_PROFILE?.trim() || "default", outputDir = defaultImportedBrowserProfilesDir()) {
192
+ const profileName = importedProfileName(name);
193
+ const manifestPath = importedProfileManifestPath(profileName, outputDir);
194
+ const manifest = await readExistingManifest(manifestPath);
195
+ if (!manifest) {
196
+ throw new Error(
197
+ `Imported browser profile "${profileName}" was not found. Run: mcp-scraper-cli browser import-chrome --name ${profileName}`
198
+ );
199
+ }
200
+ return manifest;
201
+ }
202
+ async function syncImportedChromeProfile(name, options = {}) {
203
+ const outputDir = options.outputDir?.trim() || defaultImportedBrowserProfilesDir();
204
+ const manifest = await loadImportedBrowserProfile(name, outputDir);
205
+ return importChromeProfile({
206
+ name: manifest.name,
207
+ sourceUserDataDir: manifest.sourceUserDataDir,
208
+ chromeProfileDirectory: manifest.sourceProfileDirectory,
209
+ outputDir,
210
+ overwrite: true,
211
+ browserExecutablePath: options.browserExecutablePath ?? manifest.browserExecutablePath ?? void 0
212
+ });
213
+ }
214
+
13
215
  // src/cli/agent-config.ts
14
216
  function apiKeyValue(options) {
15
217
  return options.apiKey?.trim() || "sk_live_your_key";
@@ -20,18 +222,37 @@ function packageSpec(options) {
20
222
  function combinedNpxArgs(options = {}) {
21
223
  return ["-y", "-p", packageSpec(options), "mcp-scraper-combined"];
22
224
  }
225
+ function envConfig(options) {
226
+ const env = { MCP_SCRAPER_API_KEY: apiKeyValue(options) };
227
+ const profileName = options.browserProfileName?.trim();
228
+ if (options.browserMode === "local") {
229
+ env.MCP_SCRAPER_BROWSER_MODE = "local";
230
+ if (profileName) env.MCP_SCRAPER_BROWSER_PROFILE = profileName;
231
+ const executablePath = options.browserExecutablePath?.trim();
232
+ if (executablePath) env.MCP_SCRAPER_BROWSER_EXECUTABLE = executablePath;
233
+ } else {
234
+ if (profileName) env.BROWSER_AGENT_PROFILE_NAME = profileName;
235
+ if (options.browserProfileSaveChanges === true) env.BROWSER_AGENT_PROFILE_SAVE_CHANGES = "true";
236
+ }
237
+ return env;
238
+ }
239
+ function tomlInlineTable(value) {
240
+ const entries = Object.entries(value).map(([key, item]) => `${key} = ${JSON.stringify(item)}`);
241
+ return `{ ${entries.join(", ")} }`;
242
+ }
23
243
  function renderCodexConfig(options = {}) {
24
244
  return [
25
245
  "[mcp_servers.mcp-scraper]",
26
246
  'command = "npx"',
27
247
  `args = ${JSON.stringify(combinedNpxArgs(options))}`,
28
- `env = { MCP_SCRAPER_API_KEY = "${apiKeyValue(options)}" }`
248
+ `env = ${tomlInlineTable(envConfig(options))}`
29
249
  ].join("\n");
30
250
  }
31
251
  function renderClaudeCommand(options = {}) {
252
+ const env = Object.entries(envConfig(options)).map(([key, value]) => ` --env ${key}=${value}`);
32
253
  return [
33
254
  "claude mcp add mcp-scraper --scope user",
34
- ` --env MCP_SCRAPER_API_KEY=${apiKeyValue(options)}`,
255
+ ...env,
35
256
  ` -- npx ${combinedNpxArgs(options).join(" ")}`
36
257
  ].join(" \\\n");
37
258
  }
@@ -41,9 +262,7 @@ function renderClaudeDesktopConfig(options = {}) {
41
262
  "mcp-scraper": {
42
263
  command: "npx",
43
264
  args: combinedNpxArgs(options),
44
- env: {
45
- MCP_SCRAPER_API_KEY: apiKeyValue(options)
46
- }
265
+ env: envConfig(options)
47
266
  }
48
267
  }
49
268
  }, null, 2);
@@ -76,9 +295,9 @@ function renderAgentInstall(host, options = {}) {
76
295
  }
77
296
 
78
297
  // src/cli/doctor.ts
79
- var import_promises = require("fs/promises");
80
- var import_node_os = require("os");
81
- var import_node_path = require("path");
298
+ var import_promises3 = require("fs/promises");
299
+ var import_node_os3 = require("os");
300
+ var import_node_path3 = require("path");
82
301
  function status(ok, warn = false) {
83
302
  if (ok) return "pass";
84
303
  return warn ? "warn" : "fail";
@@ -87,9 +306,9 @@ function parseMajor(version) {
87
306
  return Number(version.replace(/^v/, "").split(".")[0] ?? 0);
88
307
  }
89
308
  async function readKeyFile() {
90
- const path = process.env.MCP_SCRAPER_KEY_PATH?.trim() || (0, import_node_path.join)((0, import_node_os.homedir)(), ".mcp-scraper-key");
309
+ const path = process.env.MCP_SCRAPER_KEY_PATH?.trim() || (0, import_node_path3.join)((0, import_node_os3.homedir)(), ".mcp-scraper-key");
91
310
  try {
92
- const value = (await (0, import_promises.readFile)(path, "utf8")).trim();
311
+ const value = (await (0, import_promises3.readFile)(path, "utf8")).trim();
93
312
  return value || null;
94
313
  } catch {
95
314
  return null;
@@ -110,7 +329,7 @@ async function npmLatest(fetchImpl) {
110
329
  async function runDoctor(options = {}) {
111
330
  const fetchImpl = options.fetchImpl ?? fetch;
112
331
  const apiUrl = (options.apiUrl ?? process.env.MCP_SCRAPER_API_URL ?? "https://mcpscraper.dev").replace(/\/$/, "");
113
- const outputDir = options.outputDir ?? process.env.MCP_SCRAPER_OUTPUT_DIR ?? (0, import_node_path.join)((0, import_node_os.homedir)(), "Downloads", "mcp-scraper");
332
+ const outputDir = options.outputDir ?? process.env.MCP_SCRAPER_OUTPUT_DIR ?? (0, import_node_path3.join)((0, import_node_os3.homedir)(), "Downloads", "mcp-scraper");
114
333
  const configuredKey = options.apiKey?.trim() || process.env.MCP_SCRAPER_API_KEY?.trim() || await readKeyFile();
115
334
  const latest = await npmLatest(fetchImpl);
116
335
  const checks = [];
@@ -137,8 +356,8 @@ async function runDoctor(options = {}) {
137
356
  fix: configuredKey ? void 0 : "Set MCP_SCRAPER_API_KEY or pass --api-key."
138
357
  });
139
358
  try {
140
- await (0, import_promises.mkdir)(outputDir, { recursive: true });
141
- await (0, import_promises.access)(outputDir);
359
+ await (0, import_promises3.mkdir)(outputDir, { recursive: true });
360
+ await (0, import_promises3.access)(outputDir);
142
361
  checks.push({ id: "output_dir", label: "Output directory", status: "pass", detail: outputDir });
143
362
  } catch (err) {
144
363
  checks.push({
@@ -284,10 +503,10 @@ function renderPrompt(name) {
284
503
  }
285
504
 
286
505
  // src/workflows/artifact-writer.ts
287
- var import_promises2 = require("fs/promises");
288
- var import_node_fs = require("fs");
289
- var import_node_os2 = require("os");
290
- var import_node_path2 = require("path");
506
+ var import_promises4 = require("fs/promises");
507
+ var import_node_fs2 = require("fs");
508
+ var import_node_os4 = require("os");
509
+ var import_node_path4 = require("path");
291
510
  var import_node_child_process = require("child_process");
292
511
 
293
512
  // src/directory/csv.ts
@@ -310,7 +529,7 @@ function slugify(s) {
310
529
 
311
530
  // src/workflows/artifact-writer.ts
312
531
  function workflowOutputBaseDir(outputDir) {
313
- return outputDir?.trim() || process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path2.join)((0, import_node_os2.homedir)(), "Downloads", "mcp-scraper");
532
+ return outputDir?.trim() || process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path4.join)((0, import_node_os4.homedir)(), "Downloads", "mcp-scraper");
314
533
  }
315
534
  function timestamp() {
316
535
  return (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
@@ -341,31 +560,31 @@ var ArtifactWriter = class _ArtifactWriter {
341
560
  const runId = forcedRunId?.trim() || `${timestamp()}-${safeSlug(workflowId)}-${Math.random().toString(36).slice(2, 8)}`;
342
561
  const nameSource = String(input.keyword ?? input.query ?? input.domain ?? input.state ?? workflowId);
343
562
  const baseDir = workflowOutputBaseDir(outputDir);
344
- const runDir = (0, import_node_path2.join)(baseDir, "workflows", workflowId, `${timestamp()}-${safeSlug(nameSource)}-${safeSlug(runId).slice(0, 20)}`);
345
- await (0, import_promises2.mkdir)(runDir, { recursive: true });
563
+ const runDir = (0, import_node_path4.join)(baseDir, "workflows", workflowId, `${timestamp()}-${safeSlug(nameSource)}-${safeSlug(runId).slice(0, 20)}`);
564
+ await (0, import_promises4.mkdir)(runDir, { recursive: true });
346
565
  return new _ArtifactWriter(workflowId, title, runId, baseDir, runDir, startedAt, input);
347
566
  }
348
567
  async remember(kind, label, path, rows) {
349
- const size = await (0, import_promises2.stat)(path).then((s) => s.size).catch(() => void 0);
568
+ const size = await (0, import_promises4.stat)(path).then((s) => s.size).catch(() => void 0);
350
569
  this.artifacts.push({ kind, label, path, bytes: size, rows });
351
570
  return path;
352
571
  }
353
572
  async writeJson(label, relativePath, data) {
354
- const path = (0, import_node_path2.join)(this.runDir, relativePath);
355
- await (0, import_promises2.mkdir)((0, import_node_path2.dirname)(path), { recursive: true });
356
- await (0, import_promises2.writeFile)(path, JSON.stringify(data, null, 2), "utf8");
573
+ const path = (0, import_node_path4.join)(this.runDir, relativePath);
574
+ await (0, import_promises4.mkdir)((0, import_node_path4.dirname)(path), { recursive: true });
575
+ await (0, import_promises4.writeFile)(path, JSON.stringify(data, null, 2), "utf8");
357
576
  return this.remember("json", label, path);
358
577
  }
359
578
  async writeText(label, relativePath, text, kind = "markdown") {
360
- const path = (0, import_node_path2.join)(this.runDir, relativePath);
361
- await (0, import_promises2.mkdir)((0, import_node_path2.dirname)(path), { recursive: true });
362
- await (0, import_promises2.writeFile)(path, text, "utf8");
579
+ const path = (0, import_node_path4.join)(this.runDir, relativePath);
580
+ await (0, import_promises4.mkdir)((0, import_node_path4.dirname)(path), { recursive: true });
581
+ await (0, import_promises4.writeFile)(path, text, "utf8");
363
582
  return this.remember(kind, label, path);
364
583
  }
365
584
  async writeCsv(label, relativePath, headers, rows) {
366
- const path = (0, import_node_path2.join)(this.runDir, relativePath);
367
- await (0, import_promises2.mkdir)((0, import_node_path2.dirname)(path), { recursive: true });
368
- await (0, import_promises2.writeFile)(path, rowsToCsv(headers, rows), "utf8");
585
+ const path = (0, import_node_path4.join)(this.runDir, relativePath);
586
+ await (0, import_promises4.mkdir)((0, import_node_path4.dirname)(path), { recursive: true });
587
+ await (0, import_promises4.writeFile)(path, rowsToCsv(headers, rows), "utf8");
369
588
  return this.remember("csv", label, path, rows.length);
370
589
  }
371
590
  async writeHtml(label, relativePath, html) {
@@ -385,20 +604,20 @@ var ArtifactWriter = class _ArtifactWriter {
385
604
  errors,
386
605
  counts
387
606
  };
388
- const path = (0, import_node_path2.join)(this.runDir, "manifest.json");
389
- await (0, import_promises2.writeFile)(path, JSON.stringify(manifest, null, 2), "utf8");
607
+ const path = (0, import_node_path4.join)(this.runDir, "manifest.json");
608
+ await (0, import_promises4.writeFile)(path, JSON.stringify(manifest, null, 2), "utf8");
390
609
  await updateWorkflowIndex(manifest, path, this.baseDir);
391
610
  return path;
392
611
  }
393
612
  };
394
613
  function indexPath(baseDir = workflowOutputBaseDir()) {
395
- return (0, import_node_path2.join)(baseDir, "workflows", "index.json");
614
+ return (0, import_node_path4.join)(baseDir, "workflows", "index.json");
396
615
  }
397
616
  async function readIndex(baseDir) {
398
617
  const path = indexPath(baseDir);
399
- if (!(0, import_node_fs.existsSync)(path)) return { runs: [] };
618
+ if (!(0, import_node_fs2.existsSync)(path)) return { runs: [] };
400
619
  try {
401
- return JSON.parse(await (0, import_promises2.readFile)(path, "utf8"));
620
+ return JSON.parse(await (0, import_promises4.readFile)(path, "utf8"));
402
621
  } catch {
403
622
  return { runs: [] };
404
623
  }
@@ -418,8 +637,8 @@ async function updateWorkflowIndex(manifest, manifestPath, baseDir) {
418
637
  summary: `${manifest.title} (${manifest.status})`
419
638
  };
420
639
  index.runs = [entry, ...index.runs.filter((r) => r.runId !== manifest.runId)].slice(0, 200);
421
- await (0, import_promises2.mkdir)((0, import_node_path2.dirname)(path), { recursive: true });
422
- await (0, import_promises2.writeFile)(path, JSON.stringify(index, null, 2), "utf8");
640
+ await (0, import_promises4.mkdir)((0, import_node_path4.dirname)(path), { recursive: true });
641
+ await (0, import_promises4.writeFile)(path, JSON.stringify(index, null, 2), "utf8");
423
642
  }
424
643
  async function listWorkflowReports(outputDir) {
425
644
  return (await readIndex(workflowOutputBaseDir(outputDir))).runs;
@@ -432,7 +651,7 @@ async function findWorkflowReport(id, outputDir) {
432
651
  async function openWorkflowReport(id, outputDir) {
433
652
  const run = await findWorkflowReport(id, outputDir);
434
653
  if (!run?.reportPath) throw new Error(`No report found for "${id}"`);
435
- if ((0, import_node_os2.platform)() === "darwin") {
654
+ if ((0, import_node_os4.platform)() === "darwin") {
436
655
  await new Promise((resolve, reject) => {
437
656
  (0, import_node_child_process.execFile)("open", [run.reportPath], (err) => err ? reject(err) : resolve());
438
657
  });
@@ -441,7 +660,7 @@ async function openWorkflowReport(id, outputDir) {
441
660
  }
442
661
 
443
662
  // src/workflows/registry.ts
444
- var import_promises3 = require("fs/promises");
663
+ var import_promises5 = require("fs/promises");
445
664
  var import_zod6 = require("zod");
446
665
 
447
666
  // src/workflows/http-client.ts
@@ -2100,11 +2319,87 @@ function buildHumanCli() {
2100
2319
  result.next_step ?? "Complete checkout, then retry the MCP request."
2101
2320
  ].join("\n"), false);
2102
2321
  });
2322
+ const browser = program.command("browser").description("Inspect and manage browser-agent profile setup helpers.");
2323
+ browser.command("profiles").description("List local Chrome profiles and suggested managed browser profile names.").option("--email <email>", "Filter by Chrome account email or displayed profile name").option("--user-data-dir <path>", "Chrome user data directory").option("--json", "Print machine-readable JSON").action(async (opts) => {
2324
+ const profiles = await listLocalChromeProfiles(opts.userDataDir, opts.email);
2325
+ if (opts.json) {
2326
+ writeOutput({ profiles }, true);
2327
+ return;
2328
+ }
2329
+ if (!profiles.length) {
2330
+ writeOutput("No matching Chrome profiles found.", false);
2331
+ return;
2332
+ }
2333
+ const lines = [
2334
+ "Local Chrome profiles:",
2335
+ ...profiles.map((profile) => [
2336
+ `${profile.email || "(no email)"} ${profile.chromeProfileDirectory} ${profile.displayName}`,
2337
+ ` suggested profile name: ${profile.recommendedKernelProfileName}`,
2338
+ ` local Chrome path: ${profile.chromeProfilePath}`
2339
+ ].join("\n")),
2340
+ "",
2341
+ "To clone a real local Chrome profile for MCP local browser mode:",
2342
+ " mcp-scraper-cli browser import-chrome --email you@example.com --name <suggested profile name>",
2343
+ "Then generate config with:",
2344
+ " mcp-scraper-cli agent install claude-desktop --browser-mode local --browser-profile <suggested profile name>"
2345
+ ];
2346
+ writeOutput(lines.join("\n"), false);
2347
+ });
2348
+ browser.command("import-chrome").description("Clone a local Chrome profile into MCP Scraper managed local browser storage.").option("--email <email>", "Chrome account email or displayed profile name to import").option("--name <name>", "Managed MCP Scraper browser profile name").option("--chrome-profile <directory>", "Chrome profile directory, for example Default or Profile 1").option("--user-data-dir <path>", "Source Chrome user data directory").option("--output-dir <path>", "Managed profile base directory").option("--browser-executable <path>", "Chrome executable path for local launches").option("--overwrite", "Overwrite an existing managed profile clone").option("--json", "Print machine-readable JSON").action(async (opts) => {
2349
+ const result = await importChromeProfile({
2350
+ email: opts.email,
2351
+ name: opts.name,
2352
+ chromeProfileDirectory: opts.chromeProfile,
2353
+ sourceUserDataDir: opts.userDataDir,
2354
+ outputDir: opts.outputDir,
2355
+ browserExecutablePath: opts.browserExecutable,
2356
+ overwrite: opts.overwrite
2357
+ });
2358
+ if (opts.json) {
2359
+ writeOutput(result, true);
2360
+ return;
2361
+ }
2362
+ writeOutput([
2363
+ `Imported Chrome profile: ${result.profile.name}`,
2364
+ `Source: ${result.sourceProfile.email || result.sourceProfile.displayName || result.sourceProfile.chromeProfileDirectory} (${result.sourceProfile.chromeProfileDirectory})`,
2365
+ `Managed user data dir: ${result.profile.userDataDir}`,
2366
+ "",
2367
+ "Install/config command:",
2368
+ ` mcp-scraper-cli agent install codex --browser-mode local --browser-profile ${result.profile.name}`,
2369
+ "",
2370
+ "Restart the MCP client after updating config. Re-run sync-profile after signing into new sites in normal Chrome."
2371
+ ].join("\n"), false);
2372
+ });
2373
+ browser.command("sync-profile <name>").description("Refresh a managed local browser profile from its recorded source Chrome profile.").option("--output-dir <path>", "Managed profile base directory").option("--browser-executable <path>", "Chrome executable path for local launches").option("--json", "Print machine-readable JSON").action(async (name, opts) => {
2374
+ const result = await syncImportedChromeProfile(name, {
2375
+ outputDir: opts.outputDir,
2376
+ browserExecutablePath: opts.browserExecutable
2377
+ });
2378
+ if (opts.json) {
2379
+ writeOutput(result, true);
2380
+ return;
2381
+ }
2382
+ writeOutput([
2383
+ `Synced Chrome profile: ${result.profile.name}`,
2384
+ `Source: ${result.sourceProfile.email || result.sourceProfile.displayName || result.sourceProfile.chromeProfileDirectory} (${result.sourceProfile.chromeProfileDirectory})`,
2385
+ `Managed user data dir: ${result.profile.userDataDir}`,
2386
+ "Restart any local browser sessions that were using this profile."
2387
+ ].join("\n"), false);
2388
+ });
2103
2389
  const agent = program.command("agent").description("Generate AI-agent install configs and workflow prompts.");
2104
- agent.command("install <host>").description("Print install/config instructions for codex, claude, or claude-desktop.").option("--api-key <key>", "API key to place in generated config").option("--package <spec>", "npm package spec", "mcp-scraper@latest").option("--json", "Print machine-readable JSON").action((host, opts) => {
2390
+ agent.command("install <host>").description("Print install/config instructions for codex, claude, or claude-desktop.").option("--api-key <key>", "API key to place in generated config").option("--package <spec>", "npm package spec", "mcp-scraper@latest").option("--browser-mode <mode>", "Browser mode: hosted or local", "hosted").option("--browser-profile <name>", "Default browser profile for browser_open sessions. In local mode this is a managed MCP Scraper profile name; in hosted mode it is a Kernel profile name.").option("--browser-executable <path>", "Chrome executable path for local browser mode").option("--save-browser-profile-changes", "Persist cookies/local storage back to the named browser profile when sessions close").option("--json", "Print machine-readable JSON").action((host, opts) => {
2105
2391
  const valid = ["codex", "claude", "claude-desktop"];
2106
2392
  if (!valid.includes(host)) throw new Error(`Unknown host "${host}". Use: ${valid.join(", ")}`);
2107
- const text = renderAgentInstall(host, { apiKey: opts.apiKey, packageSpec: opts.package });
2393
+ const browserMode = String(opts.browserMode ?? "hosted");
2394
+ if (!["hosted", "local"].includes(browserMode)) throw new Error("Unknown browser mode. Use: hosted or local");
2395
+ const text = renderAgentInstall(host, {
2396
+ apiKey: opts.apiKey,
2397
+ packageSpec: opts.package,
2398
+ browserMode,
2399
+ browserProfileName: opts.browserProfile,
2400
+ browserProfileSaveChanges: opts.saveBrowserProfileChanges,
2401
+ browserExecutablePath: opts.browserExecutable
2402
+ });
2108
2403
  writeOutput(opts.json ? { host, text } : text, opts.json);
2109
2404
  });
2110
2405
  agent.command("prompt [name]").description("Print an agent prompt template.").option("--json", "Print machine-readable JSON").action((name, opts) => {
@@ -2191,14 +2486,14 @@ ${summary.warnings.map((w) => `- ${w}`).join("\n")}` : ""
2191
2486
  runs.command("download <id>").description("Download hosted workflow run artifacts.").option("--api-key <key>", "MCP Scraper API key").option("--api-url <url>", "MCP Scraper API URL", "https://mcpscraper.dev").option("--output-dir <path>", "Download directory").option("--json", "Print machine-readable JSON").action(async (id, opts) => {
2192
2487
  const result = await apiRequest(`/workflows/runs/${id}`, "GET", opts);
2193
2488
  const { apiUrl, apiKey } = apiOptions(opts);
2194
- const outDir = (0, import_node_path3.join)(opts.outputDir ?? workflowOutputBaseDir(), "workflow-downloads", id);
2195
- await (0, import_promises4.mkdir)(outDir, { recursive: true });
2489
+ const outDir = (0, import_node_path5.join)(opts.outputDir ?? workflowOutputBaseDir(), "workflow-downloads", id);
2490
+ await (0, import_promises6.mkdir)(outDir, { recursive: true });
2196
2491
  const downloaded = [];
2197
2492
  for (const artifact of result.run.artifacts ?? []) {
2198
2493
  const res = await fetch(`${apiUrl}/workflows/runs/${id}/artifacts/${artifact.id}`, { headers: { "x-api-key": apiKey } });
2199
2494
  if (!res.ok) throw new Error(`Failed to download ${artifact.label}: HTTP ${res.status}`);
2200
- const file = (0, import_node_path3.join)(outDir, (0, import_node_path3.basename)(artifact.path));
2201
- await (0, import_promises4.writeFile)(file, Buffer.from(await res.arrayBuffer()));
2495
+ const file = (0, import_node_path5.join)(outDir, (0, import_node_path5.basename)(artifact.path));
2496
+ await (0, import_promises6.writeFile)(file, Buffer.from(await res.arrayBuffer()));
2202
2497
  downloaded.push(file);
2203
2498
  }
2204
2499
  writeOutput(opts.json ? { runId: id, files: downloaded } : downloaded.join("\n"), opts.json);