mcp-scraper 0.3.5 → 0.3.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/README.md +18 -26
  2. package/dist/bin/api-server.cjs +164 -329
  3. package/dist/bin/api-server.cjs.map +1 -1
  4. package/dist/bin/api-server.js +1 -1
  5. package/dist/bin/browser-agent-stdio-server.cjs +39 -1750
  6. package/dist/bin/browser-agent-stdio-server.cjs.map +1 -1
  7. package/dist/bin/browser-agent-stdio-server.js +2 -4
  8. package/dist/bin/browser-agent-stdio-server.js.map +1 -1
  9. package/dist/bin/mcp-scraper-cli.cjs +47 -328
  10. package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
  11. package/dist/bin/mcp-scraper-cli.js +7 -91
  12. package/dist/bin/mcp-scraper-cli.js.map +1 -1
  13. package/dist/bin/mcp-scraper-combined-stdio-server.cjs +78 -1796
  14. package/dist/bin/mcp-scraper-combined-stdio-server.cjs.map +1 -1
  15. package/dist/bin/mcp-scraper-combined-stdio-server.js +4 -6
  16. package/dist/bin/mcp-scraper-combined-stdio-server.js.map +1 -1
  17. package/dist/bin/mcp-scraper-install.cjs +4 -4
  18. package/dist/bin/mcp-scraper-install.cjs.map +1 -1
  19. package/dist/bin/mcp-scraper-install.js +2 -2
  20. package/dist/bin/mcp-stdio-server.cjs +18 -25
  21. package/dist/bin/mcp-stdio-server.cjs.map +1 -1
  22. package/dist/bin/mcp-stdio-server.js +2 -2
  23. package/dist/chunk-2NEVRPK3.js +7 -0
  24. package/dist/chunk-2NEVRPK3.js.map +1 -0
  25. package/dist/{chunk-WZT3XKES.js → chunk-2O7RAK6E.js} +19 -26
  26. package/dist/chunk-2O7RAK6E.js.map +1 -0
  27. package/dist/{chunk-RVYHP7QN.js → chunk-ELI6NDEG.js} +4 -4
  28. package/dist/{chunk-RVYHP7QN.js.map → chunk-ELI6NDEG.js.map} +1 -1
  29. package/dist/{chunk-CMK5MMEE.js → chunk-IWNLHCKA.js} +13 -694
  30. package/dist/chunk-IWNLHCKA.js.map +1 -0
  31. package/dist/{server-BC4Z2L4P.js → server-HRVD52CX.js} +842 -122
  32. package/dist/server-HRVD52CX.js.map +1 -0
  33. package/docs/mcp-tool-craft-lint.generated.md +2 -5
  34. package/docs/mcp-tool-manifest.generated.json +9 -101
  35. package/package.json +1 -1
  36. package/dist/chunk-CMK5MMEE.js.map +0 -1
  37. package/dist/chunk-DUEW4EOO.js +0 -214
  38. package/dist/chunk-DUEW4EOO.js.map +0 -1
  39. package/dist/chunk-GHBU5SGQ.js +0 -7
  40. package/dist/chunk-GHBU5SGQ.js.map +0 -1
  41. package/dist/chunk-NXRWFOEZ.js +0 -816
  42. package/dist/chunk-NXRWFOEZ.js.map +0 -1
  43. package/dist/chunk-WZT3XKES.js.map +0 -1
  44. package/dist/server-BC4Z2L4P.js.map +0 -1
@@ -1,11 +1,9 @@
1
1
  #!/usr/bin/env node
2
2
  import {
3
3
  buildBrowserAgentMcpServer
4
- } from "../chunk-CMK5MMEE.js";
5
- import "../chunk-NXRWFOEZ.js";
4
+ } from "../chunk-IWNLHCKA.js";
6
5
  import "../chunk-WN7PBKMV.js";
7
- import "../chunk-DUEW4EOO.js";
8
- import "../chunk-GHBU5SGQ.js";
6
+ import "../chunk-2NEVRPK3.js";
9
7
 
10
8
  // bin/browser-agent-stdio-server.ts
11
9
  import { readFileSync } from "fs";
@@ -1 +1 @@
1
- {"version":3,"sources":["../../bin/browser-agent-stdio-server.ts"],"sourcesContent":["#!/usr/bin/env node\nimport { readFileSync } from 'node:fs'\nimport { homedir } from 'node:os'\nimport { join } from 'node:path'\nimport { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'\nimport { buildBrowserAgentMcpServer } from '../src/mcp/browser-agent-mcp-server.js'\n\nfunction readApiKeyFile(): string | undefined {\n const explicitPath = process.env.MCP_SCRAPER_KEY_PATH?.trim()\n const paths = [explicitPath, join(homedir(), '.mcp-scraper-key')].filter(Boolean) as string[]\n for (const path of paths) {\n try {\n const value = readFileSync(path, 'utf8').trim()\n if (value) return value\n } catch {\n void 0\n }\n }\n return undefined\n}\n\nconst apiKey = (\n process.env.MCP_SCRAPER_API_KEY ??\n process.env.MCP_SCRAPER_KEY ??\n process.env.MCP_API_KEY ??\n readApiKeyFile()\n)?.trim()\nif (!apiKey) {\n process.stderr.write('MCP_SCRAPER_API_KEY env var or ~/.mcp-scraper-key is required\\n')\n process.exit(1)\n}\n\nconst baseUrl = process.env.MCP_SCRAPER_BASE_URL?.trim() || process.env.MCP_BASE_URL?.trim() || 'https://mcpscraper.dev'\nconst consoleBaseUrl = process.env.BROWSER_AGENT_CONSOLE_URL?.trim() || baseUrl\nconst server = buildBrowserAgentMcpServer({ baseUrl, apiKey, consoleBaseUrl })\nconst transport = new StdioServerTransport()\n\nasync function main() {\n await server.connect(transport)\n}\n\nmain().catch(err => {\n process.stderr.write(`${err instanceof Error ? err.message : String(err)}\\n`)\n process.exit(1)\n})\n"],"mappings":";;;;;;;;;;AACA,SAAS,oBAAoB;AAC7B,SAAS,eAAe;AACxB,SAAS,YAAY;AACrB,SAAS,4BAA4B;AAGrC,SAAS,iBAAqC;AAC5C,QAAM,eAAe,QAAQ,IAAI,sBAAsB,KAAK;AAC5D,QAAM,QAAQ,CAAC,cAAc,KAAK,QAAQ,GAAG,kBAAkB,CAAC,EAAE,OAAO,OAAO;AAChF,aAAW,QAAQ,OAAO;AACxB,QAAI;AACF,YAAM,QAAQ,aAAa,MAAM,MAAM,EAAE,KAAK;AAC9C,UAAI,MAAO,QAAO;AAAA,IACpB,QAAQ;AAAA,IAER;AAAA,EACF;AACA,SAAO;AACT;AAEA,IAAM,UACJ,QAAQ,IAAI,uBACZ,QAAQ,IAAI,mBACZ,QAAQ,IAAI,eACZ,eAAe,IACd,KAAK;AACR,IAAI,CAAC,QAAQ;AACX,UAAQ,OAAO,MAAM,iEAAiE;AACtF,UAAQ,KAAK,CAAC;AAChB;AAEA,IAAM,UAAU,QAAQ,IAAI,sBAAsB,KAAK,KAAK,QAAQ,IAAI,cAAc,KAAK,KAAK;AAChG,IAAM,iBAAiB,QAAQ,IAAI,2BAA2B,KAAK,KAAK;AACxE,IAAM,SAAS,2BAA2B,EAAE,SAAS,QAAQ,eAAe,CAAC;AAC7E,IAAM,YAAY,IAAI,qBAAqB;AAE3C,eAAe,OAAO;AACpB,QAAM,OAAO,QAAQ,SAAS;AAChC;AAEA,KAAK,EAAE,MAAM,SAAO;AAClB,UAAQ,OAAO,MAAM,GAAG,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,CAAI;AAC5E,UAAQ,KAAK,CAAC;AAChB,CAAC;","names":[]}
1
+ {"version":3,"sources":["../../bin/browser-agent-stdio-server.ts"],"sourcesContent":["#!/usr/bin/env node\nimport { readFileSync } from 'node:fs'\nimport { homedir } from 'node:os'\nimport { join } from 'node:path'\nimport { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'\nimport { buildBrowserAgentMcpServer } from '../src/mcp/browser-agent-mcp-server.js'\n\nfunction readApiKeyFile(): string | undefined {\n const explicitPath = process.env.MCP_SCRAPER_KEY_PATH?.trim()\n const paths = [explicitPath, join(homedir(), '.mcp-scraper-key')].filter(Boolean) as string[]\n for (const path of paths) {\n try {\n const value = readFileSync(path, 'utf8').trim()\n if (value) return value\n } catch {\n void 0\n }\n }\n return undefined\n}\n\nconst apiKey = (\n process.env.MCP_SCRAPER_API_KEY ??\n process.env.MCP_SCRAPER_KEY ??\n process.env.MCP_API_KEY ??\n readApiKeyFile()\n)?.trim()\nif (!apiKey) {\n process.stderr.write('MCP_SCRAPER_API_KEY env var or ~/.mcp-scraper-key is required\\n')\n process.exit(1)\n}\n\nconst baseUrl = process.env.MCP_SCRAPER_BASE_URL?.trim() || process.env.MCP_BASE_URL?.trim() || 'https://mcpscraper.dev'\nconst consoleBaseUrl = process.env.BROWSER_AGENT_CONSOLE_URL?.trim() || baseUrl\nconst server = buildBrowserAgentMcpServer({ baseUrl, apiKey, consoleBaseUrl })\nconst transport = new StdioServerTransport()\n\nasync function main() {\n await server.connect(transport)\n}\n\nmain().catch(err => {\n process.stderr.write(`${err instanceof Error ? err.message : String(err)}\\n`)\n process.exit(1)\n})\n"],"mappings":";;;;;;;;AACA,SAAS,oBAAoB;AAC7B,SAAS,eAAe;AACxB,SAAS,YAAY;AACrB,SAAS,4BAA4B;AAGrC,SAAS,iBAAqC;AAC5C,QAAM,eAAe,QAAQ,IAAI,sBAAsB,KAAK;AAC5D,QAAM,QAAQ,CAAC,cAAc,KAAK,QAAQ,GAAG,kBAAkB,CAAC,EAAE,OAAO,OAAO;AAChF,aAAW,QAAQ,OAAO;AACxB,QAAI;AACF,YAAM,QAAQ,aAAa,MAAM,MAAM,EAAE,KAAK;AAC9C,UAAI,MAAO,QAAO;AAAA,IACpB,QAAQ;AAAA,IAER;AAAA,EACF;AACA,SAAO;AACT;AAEA,IAAM,UACJ,QAAQ,IAAI,uBACZ,QAAQ,IAAI,mBACZ,QAAQ,IAAI,eACZ,eAAe,IACd,KAAK;AACR,IAAI,CAAC,QAAQ;AACX,UAAQ,OAAO,MAAM,iEAAiE;AACtF,UAAQ,KAAK,CAAC;AAChB;AAEA,IAAM,UAAU,QAAQ,IAAI,sBAAsB,KAAK,KAAK,QAAQ,IAAI,cAAc,KAAK,KAAK;AAChG,IAAM,iBAAiB,QAAQ,IAAI,2BAA2B,KAAK,KAAK;AACxE,IAAM,SAAS,2BAA2B,EAAE,SAAS,QAAQ,eAAe,CAAC;AAC7E,IAAM,YAAY,IAAI,qBAAqB;AAE3C,eAAe,OAAO;AACpB,QAAM,OAAO,QAAQ,SAAS;AAChC;AAEA,KAAK,EAAE,MAAM,SAAO;AAClB,UAAQ,OAAO,MAAM,GAAG,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,CAAI;AAC5E,UAAQ,KAAK,CAAC;AAChB,CAAC;","names":[]}
@@ -4,213 +4,11 @@
4
4
  // src/cli/human-cli.ts
5
5
  var import_commander = require("commander");
6
6
  var import_node_child_process2 = require("child_process");
7
- var import_promises6 = require("fs/promises");
8
- var import_node_path5 = require("path");
7
+ var import_promises4 = require("fs/promises");
8
+ var import_node_path3 = require("path");
9
9
 
10
10
  // src/version.ts
11
- var PACKAGE_VERSION = "0.3.5";
12
-
13
- // src/lib/chrome-profiles.ts
14
- var import_promises = require("fs/promises");
15
- var import_node_os = require("os");
16
- var import_node_path = require("path");
17
- function defaultChromeUserDataDir() {
18
- if (process.platform === "darwin") return (0, import_node_path.join)((0, import_node_os.homedir)(), "Library", "Application Support", "Google", "Chrome");
19
- if (process.platform === "win32") {
20
- return (0, import_node_path.join)(process.env.LOCALAPPDATA ?? (0, import_node_path.join)((0, import_node_os.homedir)(), "AppData", "Local"), "Google", "Chrome", "User Data");
21
- }
22
- return (0, import_node_path.join)(process.env.XDG_CONFIG_HOME ?? (0, import_node_path.join)((0, import_node_os.homedir)(), ".config"), "google-chrome");
23
- }
24
- function recommendedKernelProfileName(value) {
25
- return value.trim().toLowerCase().replace(/@/g, "-").replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80) || "browser-profile";
26
- }
27
- async function listLocalChromeProfiles(userDataDir = defaultChromeUserDataDir(), emailFilter) {
28
- const raw = await (0, import_promises.readFile)((0, import_node_path.join)(userDataDir, "Local State"), "utf8");
29
- const localState = JSON.parse(raw);
30
- const wanted = emailFilter?.trim().toLowerCase();
31
- return Object.entries(localState.profile?.info_cache ?? {}).map(([directory, info]) => {
32
- const email = (info.user_name ?? "").trim();
33
- const displayName = (info.name ?? info.shortcut_name ?? "").trim();
34
- const labelSource = email || displayName || directory;
35
- return {
36
- email,
37
- displayName,
38
- chromeProfileDirectory: directory,
39
- chromeProfilePath: (0, import_node_path.join)(userDataDir, directory),
40
- recommendedKernelProfileName: recommendedKernelProfileName(labelSource)
41
- };
42
- }).filter((row) => !wanted || row.email.toLowerCase() === wanted || row.displayName.toLowerCase().includes(wanted));
43
- }
44
-
45
- // src/lib/local-browser-profiles.ts
46
- var import_node_fs = require("fs");
47
- var import_promises2 = require("fs/promises");
48
- var import_node_os2 = require("os");
49
- var import_node_path2 = require("path");
50
- var EXCLUDED_PROFILE_NAMES = [
51
- "Cache",
52
- "Code Cache",
53
- "Crashpad",
54
- "DawnCache",
55
- "DawnGraphiteCache",
56
- "DawnWebGPUCache",
57
- "GPUCache",
58
- "GrShaderCache",
59
- "ShaderCache",
60
- "Shared Dictionary"
61
- ];
62
- function defaultImportedBrowserProfilesDir() {
63
- return process.env.MCP_SCRAPER_BROWSER_PROFILE_BASE_DIR?.trim() || (0, import_node_path2.join)((0, import_node_os2.homedir)(), ".mcp-scraper", "browser-profiles");
64
- }
65
- function defaultChromeExecutablePath() {
66
- const envPath = process.env.MCP_SCRAPER_BROWSER_EXECUTABLE?.trim() || process.env.CHROME_PATH?.trim();
67
- if (envPath) return envPath;
68
- const candidates = process.platform === "darwin" ? ["/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"] : process.platform === "win32" ? [
69
- (0, import_node_path2.join)(process.env.PROGRAMFILES ?? "C:\\Program Files", "Google", "Chrome", "Application", "chrome.exe"),
70
- (0, import_node_path2.join)(process.env["PROGRAMFILES(X86)"] ?? "C:\\Program Files (x86)", "Google", "Chrome", "Application", "chrome.exe")
71
- ] : ["/usr/bin/google-chrome", "/usr/bin/google-chrome-stable", "/usr/bin/chromium-browser", "/usr/bin/chromium"];
72
- return candidates.find((candidate) => (0, import_node_fs.existsSync)(candidate));
73
- }
74
- function importedProfileName(value) {
75
- return recommendedKernelProfileName(value);
76
- }
77
- function importedProfileRoot(name, outputDir = defaultImportedBrowserProfilesDir()) {
78
- return (0, import_node_path2.join)(outputDir, importedProfileName(name));
79
- }
80
- function importedProfileManifestPath(name, outputDir = defaultImportedBrowserProfilesDir()) {
81
- return (0, import_node_path2.join)(importedProfileRoot(name, outputDir), "manifest.json");
82
- }
83
- function importedProfileUserDataDir(name, outputDir = defaultImportedBrowserProfilesDir()) {
84
- return (0, import_node_path2.join)(importedProfileRoot(name, outputDir), "user-data");
85
- }
86
- async function readExistingManifest(path) {
87
- try {
88
- return JSON.parse(await (0, import_promises2.readFile)(path, "utf8"));
89
- } catch {
90
- return null;
91
- }
92
- }
93
- function isExcludedProfilePath(sourceRoot, sourcePath) {
94
- const rel = (0, import_node_path2.relative)(sourceRoot, sourcePath);
95
- if (!rel) return false;
96
- const parts = rel.split(import_node_path2.sep).filter(Boolean);
97
- const last = parts[parts.length - 1] ?? "";
98
- if (last.startsWith("Singleton")) return true;
99
- if (last === "LOCK" || last.endsWith(".lock")) return true;
100
- if (parts.some((part) => EXCLUDED_PROFILE_NAMES.includes(part))) return true;
101
- if (parts.includes("Service Worker") && parts.includes("CacheStorage")) return true;
102
- return false;
103
- }
104
- async function copyProfileTree(sourceProfilePath, destinationProfilePath) {
105
- await (0, import_promises2.cp)(sourceProfilePath, destinationProfilePath, {
106
- recursive: true,
107
- force: true,
108
- errorOnExist: false,
109
- dereference: false,
110
- filter: (source) => !isExcludedProfilePath(sourceProfilePath, source)
111
- });
112
- }
113
- function selectProfile(profiles, options) {
114
- const requestedDirectory = options.chromeProfileDirectory?.trim();
115
- const requestedEmail = options.email?.trim().toLowerCase();
116
- const exactDirectory = requestedDirectory ? profiles.find((profile) => profile.chromeProfileDirectory === requestedDirectory) : void 0;
117
- if (exactDirectory) return exactDirectory;
118
- const exactEmail = requestedEmail ? profiles.find((profile) => profile.email.toLowerCase() === requestedEmail) : void 0;
119
- if (exactEmail) return exactEmail;
120
- const first = profiles[0];
121
- if (first) return first;
122
- const hint = requestedDirectory ? `Chrome profile directory "${requestedDirectory}"` : requestedEmail ? `Chrome account "${requestedEmail}"` : "Chrome profile";
123
- throw new Error(`${hint} was not found in ${options.sourceUserDataDir ?? defaultChromeUserDataDir()}`);
124
- }
125
- async function ensureProfilePathExists(profile) {
126
- const details = await (0, import_promises2.stat)(profile.chromeProfilePath).catch(() => null);
127
- if (!details?.isDirectory()) {
128
- throw new Error(`Chrome profile path does not exist or is not a directory: ${profile.chromeProfilePath}`);
129
- }
130
- }
131
- async function writeNormalizedLocalState(sourceLocalStatePath, destinationLocalStatePath, sourceProfileDirectory) {
132
- const raw = await (0, import_promises2.readFile)(sourceLocalStatePath, "utf8");
133
- const localState = JSON.parse(raw);
134
- const selectedInfo = localState.profile?.info_cache?.[sourceProfileDirectory] ?? localState.profile?.info_cache?.Default ?? {};
135
- localState.profile = {
136
- ...localState.profile ?? {},
137
- info_cache: { Default: selectedInfo },
138
- last_used: "Default",
139
- last_active_profiles: ["Default"]
140
- };
141
- await (0, import_promises2.writeFile)(destinationLocalStatePath, `${JSON.stringify(localState, null, 2)}
142
- `);
143
- }
144
- async function importChromeProfile(options = {}) {
145
- const sourceUserDataDir = options.sourceUserDataDir?.trim() || defaultChromeUserDataDir();
146
- const profiles = await listLocalChromeProfiles(sourceUserDataDir, options.email);
147
- const sourceProfile = selectProfile(profiles, { ...options, sourceUserDataDir });
148
- await ensureProfilePathExists(sourceProfile);
149
- const name = importedProfileName(options.name?.trim() || sourceProfile.recommendedKernelProfileName);
150
- const outputDir = options.outputDir?.trim() || defaultImportedBrowserProfilesDir();
151
- const profileRoot = importedProfileRoot(name, outputDir);
152
- const manifestPath = importedProfileManifestPath(name, outputDir);
153
- const userDataDir = importedProfileUserDataDir(name, outputDir);
154
- const existingManifest = await readExistingManifest(manifestPath);
155
- const existing = (0, import_node_fs.existsSync)(profileRoot);
156
- if (existing && !options.overwrite) {
157
- throw new Error(`Imported browser profile "${name}" already exists at ${profileRoot}. Re-run with --overwrite to refresh it.`);
158
- }
159
- if (existing) await (0, import_promises2.rm)(profileRoot, { recursive: true, force: true });
160
- await (0, import_promises2.mkdir)((0, import_node_path2.join)(userDataDir, "Default"), { recursive: true });
161
- await writeNormalizedLocalState(
162
- (0, import_node_path2.join)(sourceUserDataDir, "Local State"),
163
- (0, import_node_path2.join)(userDataDir, "Local State"),
164
- sourceProfile.chromeProfileDirectory
165
- );
166
- await copyProfileTree(sourceProfile.chromeProfilePath, (0, import_node_path2.join)(userDataDir, "Default"));
167
- const now = (/* @__PURE__ */ new Date()).toISOString();
168
- const manifest = {
169
- layoutVersion: 1,
170
- name,
171
- profileRoot,
172
- userDataDir,
173
- browserExecutablePath: options.browserExecutablePath?.trim() || defaultChromeExecutablePath() || null,
174
- sourceUserDataDir,
175
- sourceProfileDirectory: sourceProfile.chromeProfileDirectory,
176
- sourceProfilePath: sourceProfile.chromeProfilePath,
177
- email: sourceProfile.email,
178
- displayName: sourceProfile.displayName,
179
- createdAt: existingManifest?.createdAt ?? now,
180
- updatedAt: now,
181
- copyMode: "chrome-profile-clone"
182
- };
183
- await (0, import_promises2.writeFile)(manifestPath, `${JSON.stringify(manifest, null, 2)}
184
- `);
185
- return {
186
- profile: manifest,
187
- sourceProfile,
188
- excludedNames: EXCLUDED_PROFILE_NAMES
189
- };
190
- }
191
- async function loadImportedBrowserProfile(name = process.env.MCP_SCRAPER_BROWSER_PROFILE?.trim() || "default", outputDir = defaultImportedBrowserProfilesDir()) {
192
- const profileName = importedProfileName(name);
193
- const manifestPath = importedProfileManifestPath(profileName, outputDir);
194
- const manifest = await readExistingManifest(manifestPath);
195
- if (!manifest) {
196
- throw new Error(
197
- `Imported browser profile "${profileName}" was not found. Run: mcp-scraper-cli browser import-chrome --name ${profileName}`
198
- );
199
- }
200
- return manifest;
201
- }
202
- async function syncImportedChromeProfile(name, options = {}) {
203
- const outputDir = options.outputDir?.trim() || defaultImportedBrowserProfilesDir();
204
- const manifest = await loadImportedBrowserProfile(name, outputDir);
205
- return importChromeProfile({
206
- name: manifest.name,
207
- sourceUserDataDir: manifest.sourceUserDataDir,
208
- chromeProfileDirectory: manifest.sourceProfileDirectory,
209
- outputDir,
210
- overwrite: true,
211
- browserExecutablePath: options.browserExecutablePath ?? manifest.browserExecutablePath ?? void 0
212
- });
213
- }
11
+ var PACKAGE_VERSION = "0.3.7";
214
12
 
215
13
  // src/cli/agent-config.ts
216
14
  function apiKeyValue(options) {
@@ -225,15 +23,8 @@ function combinedNpxArgs(options = {}) {
225
23
  function envConfig(options) {
226
24
  const env = { MCP_SCRAPER_API_KEY: apiKeyValue(options) };
227
25
  const profileName = options.browserProfileName?.trim();
228
- if (options.browserMode === "local") {
229
- env.MCP_SCRAPER_BROWSER_MODE = "local";
230
- if (profileName) env.MCP_SCRAPER_BROWSER_PROFILE = profileName;
231
- const executablePath = options.browserExecutablePath?.trim();
232
- if (executablePath) env.MCP_SCRAPER_BROWSER_EXECUTABLE = executablePath;
233
- } else {
234
- if (profileName) env.BROWSER_AGENT_PROFILE_NAME = profileName;
235
- if (options.browserProfileSaveChanges === true) env.BROWSER_AGENT_PROFILE_SAVE_CHANGES = "true";
236
- }
26
+ if (profileName) env.BROWSER_AGENT_PROFILE_NAME = profileName;
27
+ if (options.browserProfileSaveChanges === true) env.BROWSER_AGENT_PROFILE_SAVE_CHANGES = "true";
237
28
  return env;
238
29
  }
239
30
  function tomlInlineTable(value) {
@@ -316,9 +107,9 @@ function renderAgentInstall(host, options = {}) {
316
107
  }
317
108
 
318
109
  // src/cli/doctor.ts
319
- var import_promises3 = require("fs/promises");
320
- var import_node_os3 = require("os");
321
- var import_node_path3 = require("path");
110
+ var import_promises = require("fs/promises");
111
+ var import_node_os = require("os");
112
+ var import_node_path = require("path");
322
113
  function status(ok, warn = false) {
323
114
  if (ok) return "pass";
324
115
  return warn ? "warn" : "fail";
@@ -327,9 +118,9 @@ function parseMajor(version) {
327
118
  return Number(version.replace(/^v/, "").split(".")[0] ?? 0);
328
119
  }
329
120
  async function readKeyFile() {
330
- const path = process.env.MCP_SCRAPER_KEY_PATH?.trim() || (0, import_node_path3.join)((0, import_node_os3.homedir)(), ".mcp-scraper-key");
121
+ const path = process.env.MCP_SCRAPER_KEY_PATH?.trim() || (0, import_node_path.join)((0, import_node_os.homedir)(), ".mcp-scraper-key");
331
122
  try {
332
- const value = (await (0, import_promises3.readFile)(path, "utf8")).trim();
123
+ const value = (await (0, import_promises.readFile)(path, "utf8")).trim();
333
124
  return value || null;
334
125
  } catch {
335
126
  return null;
@@ -350,7 +141,7 @@ async function npmLatest(fetchImpl) {
350
141
  async function runDoctor(options = {}) {
351
142
  const fetchImpl = options.fetchImpl ?? fetch;
352
143
  const apiUrl = (options.apiUrl ?? process.env.MCP_SCRAPER_API_URL ?? "https://mcpscraper.dev").replace(/\/$/, "");
353
- const outputDir = options.outputDir ?? process.env.MCP_SCRAPER_OUTPUT_DIR ?? (0, import_node_path3.join)((0, import_node_os3.homedir)(), "Downloads", "mcp-scraper");
144
+ const outputDir = options.outputDir ?? process.env.MCP_SCRAPER_OUTPUT_DIR ?? (0, import_node_path.join)((0, import_node_os.homedir)(), "Downloads", "mcp-scraper");
354
145
  const configuredKey = options.apiKey?.trim() || process.env.MCP_SCRAPER_API_KEY?.trim() || await readKeyFile();
355
146
  const latest = await npmLatest(fetchImpl);
356
147
  const checks = [];
@@ -377,8 +168,8 @@ async function runDoctor(options = {}) {
377
168
  fix: configuredKey ? void 0 : "Set MCP_SCRAPER_API_KEY or pass --api-key."
378
169
  });
379
170
  try {
380
- await (0, import_promises3.mkdir)(outputDir, { recursive: true });
381
- await (0, import_promises3.access)(outputDir);
171
+ await (0, import_promises.mkdir)(outputDir, { recursive: true });
172
+ await (0, import_promises.access)(outputDir);
382
173
  checks.push({ id: "output_dir", label: "Output directory", status: "pass", detail: outputDir });
383
174
  } catch (err) {
384
175
  checks.push({
@@ -524,10 +315,10 @@ function renderPrompt(name) {
524
315
  }
525
316
 
526
317
  // src/workflows/artifact-writer.ts
527
- var import_promises4 = require("fs/promises");
528
- var import_node_fs2 = require("fs");
529
- var import_node_os4 = require("os");
530
- var import_node_path4 = require("path");
318
+ var import_promises2 = require("fs/promises");
319
+ var import_node_fs = require("fs");
320
+ var import_node_os2 = require("os");
321
+ var import_node_path2 = require("path");
531
322
  var import_node_child_process = require("child_process");
532
323
 
533
324
  // src/directory/csv.ts
@@ -550,7 +341,7 @@ function slugify(s) {
550
341
 
551
342
  // src/workflows/artifact-writer.ts
552
343
  function workflowOutputBaseDir(outputDir) {
553
- return outputDir?.trim() || process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path4.join)((0, import_node_os4.homedir)(), "Downloads", "mcp-scraper");
344
+ return outputDir?.trim() || process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path2.join)((0, import_node_os2.homedir)(), "Downloads", "mcp-scraper");
554
345
  }
555
346
  function timestamp() {
556
347
  return (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
@@ -581,31 +372,31 @@ var ArtifactWriter = class _ArtifactWriter {
581
372
  const runId = forcedRunId?.trim() || `${timestamp()}-${safeSlug(workflowId)}-${Math.random().toString(36).slice(2, 8)}`;
582
373
  const nameSource = String(input.keyword ?? input.query ?? input.domain ?? input.state ?? workflowId);
583
374
  const baseDir = workflowOutputBaseDir(outputDir);
584
- const runDir = (0, import_node_path4.join)(baseDir, "workflows", workflowId, `${timestamp()}-${safeSlug(nameSource)}-${safeSlug(runId).slice(0, 20)}`);
585
- await (0, import_promises4.mkdir)(runDir, { recursive: true });
375
+ const runDir = (0, import_node_path2.join)(baseDir, "workflows", workflowId, `${timestamp()}-${safeSlug(nameSource)}-${safeSlug(runId).slice(0, 20)}`);
376
+ await (0, import_promises2.mkdir)(runDir, { recursive: true });
586
377
  return new _ArtifactWriter(workflowId, title, runId, baseDir, runDir, startedAt, input);
587
378
  }
588
379
  async remember(kind, label, path, rows) {
589
- const size = await (0, import_promises4.stat)(path).then((s) => s.size).catch(() => void 0);
380
+ const size = await (0, import_promises2.stat)(path).then((s) => s.size).catch(() => void 0);
590
381
  this.artifacts.push({ kind, label, path, bytes: size, rows });
591
382
  return path;
592
383
  }
593
384
  async writeJson(label, relativePath, data) {
594
- const path = (0, import_node_path4.join)(this.runDir, relativePath);
595
- await (0, import_promises4.mkdir)((0, import_node_path4.dirname)(path), { recursive: true });
596
- await (0, import_promises4.writeFile)(path, JSON.stringify(data, null, 2), "utf8");
385
+ const path = (0, import_node_path2.join)(this.runDir, relativePath);
386
+ await (0, import_promises2.mkdir)((0, import_node_path2.dirname)(path), { recursive: true });
387
+ await (0, import_promises2.writeFile)(path, JSON.stringify(data, null, 2), "utf8");
597
388
  return this.remember("json", label, path);
598
389
  }
599
390
  async writeText(label, relativePath, text, kind = "markdown") {
600
- const path = (0, import_node_path4.join)(this.runDir, relativePath);
601
- await (0, import_promises4.mkdir)((0, import_node_path4.dirname)(path), { recursive: true });
602
- await (0, import_promises4.writeFile)(path, text, "utf8");
391
+ const path = (0, import_node_path2.join)(this.runDir, relativePath);
392
+ await (0, import_promises2.mkdir)((0, import_node_path2.dirname)(path), { recursive: true });
393
+ await (0, import_promises2.writeFile)(path, text, "utf8");
603
394
  return this.remember(kind, label, path);
604
395
  }
605
396
  async writeCsv(label, relativePath, headers, rows) {
606
- const path = (0, import_node_path4.join)(this.runDir, relativePath);
607
- await (0, import_promises4.mkdir)((0, import_node_path4.dirname)(path), { recursive: true });
608
- await (0, import_promises4.writeFile)(path, rowsToCsv(headers, rows), "utf8");
397
+ const path = (0, import_node_path2.join)(this.runDir, relativePath);
398
+ await (0, import_promises2.mkdir)((0, import_node_path2.dirname)(path), { recursive: true });
399
+ await (0, import_promises2.writeFile)(path, rowsToCsv(headers, rows), "utf8");
609
400
  return this.remember("csv", label, path, rows.length);
610
401
  }
611
402
  async writeHtml(label, relativePath, html) {
@@ -625,20 +416,20 @@ var ArtifactWriter = class _ArtifactWriter {
625
416
  errors,
626
417
  counts
627
418
  };
628
- const path = (0, import_node_path4.join)(this.runDir, "manifest.json");
629
- await (0, import_promises4.writeFile)(path, JSON.stringify(manifest, null, 2), "utf8");
419
+ const path = (0, import_node_path2.join)(this.runDir, "manifest.json");
420
+ await (0, import_promises2.writeFile)(path, JSON.stringify(manifest, null, 2), "utf8");
630
421
  await updateWorkflowIndex(manifest, path, this.baseDir);
631
422
  return path;
632
423
  }
633
424
  };
634
425
  function indexPath(baseDir = workflowOutputBaseDir()) {
635
- return (0, import_node_path4.join)(baseDir, "workflows", "index.json");
426
+ return (0, import_node_path2.join)(baseDir, "workflows", "index.json");
636
427
  }
637
428
  async function readIndex(baseDir) {
638
429
  const path = indexPath(baseDir);
639
- if (!(0, import_node_fs2.existsSync)(path)) return { runs: [] };
430
+ if (!(0, import_node_fs.existsSync)(path)) return { runs: [] };
640
431
  try {
641
- return JSON.parse(await (0, import_promises4.readFile)(path, "utf8"));
432
+ return JSON.parse(await (0, import_promises2.readFile)(path, "utf8"));
642
433
  } catch {
643
434
  return { runs: [] };
644
435
  }
@@ -658,8 +449,8 @@ async function updateWorkflowIndex(manifest, manifestPath, baseDir) {
658
449
  summary: `${manifest.title} (${manifest.status})`
659
450
  };
660
451
  index.runs = [entry, ...index.runs.filter((r) => r.runId !== manifest.runId)].slice(0, 200);
661
- await (0, import_promises4.mkdir)((0, import_node_path4.dirname)(path), { recursive: true });
662
- await (0, import_promises4.writeFile)(path, JSON.stringify(index, null, 2), "utf8");
452
+ await (0, import_promises2.mkdir)((0, import_node_path2.dirname)(path), { recursive: true });
453
+ await (0, import_promises2.writeFile)(path, JSON.stringify(index, null, 2), "utf8");
663
454
  }
664
455
  async function listWorkflowReports(outputDir) {
665
456
  return (await readIndex(workflowOutputBaseDir(outputDir))).runs;
@@ -672,7 +463,7 @@ async function findWorkflowReport(id, outputDir) {
672
463
  async function openWorkflowReport(id, outputDir) {
673
464
  const run = await findWorkflowReport(id, outputDir);
674
465
  if (!run?.reportPath) throw new Error(`No report found for "${id}"`);
675
- if ((0, import_node_os4.platform)() === "darwin") {
466
+ if ((0, import_node_os2.platform)() === "darwin") {
676
467
  await new Promise((resolve, reject) => {
677
468
  (0, import_node_child_process.execFile)("open", [run.reportPath], (err) => err ? reject(err) : resolve());
678
469
  });
@@ -681,7 +472,7 @@ async function openWorkflowReport(id, outputDir) {
681
472
  }
682
473
 
683
474
  // src/workflows/registry.ts
684
- var import_promises5 = require("fs/promises");
475
+ var import_promises3 = require("fs/promises");
685
476
  var import_zod6 = require("zod");
686
477
 
687
478
  // src/workflows/http-client.ts
@@ -2368,77 +2159,9 @@ function buildHumanCli() {
2368
2159
  result.next_step ?? "Complete checkout, then retry the MCP request."
2369
2160
  ].join("\n"), false);
2370
2161
  });
2371
- const browser = program.command("browser").description("Inspect developer-only local browser helpers. Customer auth uses hosted browser_profile_onboard.");
2372
- browser.command("profiles").description("Developer-only: list local Chrome profiles and suggested managed browser profile names.").option("--email <email>", "Filter by Chrome account email or displayed profile name").option("--user-data-dir <path>", "Chrome user data directory").option("--json", "Print machine-readable JSON").action(async (opts) => {
2373
- const profiles = await listLocalChromeProfiles(opts.userDataDir, opts.email);
2374
- if (opts.json) {
2375
- writeOutput({ profiles }, true);
2376
- return;
2377
- }
2378
- if (!profiles.length) {
2379
- writeOutput("No matching Chrome profiles found.", false);
2380
- return;
2381
- }
2382
- const lines = [
2383
- "Developer-only local Chrome profiles:",
2384
- ...profiles.map((profile) => [
2385
- `${profile.email || "(no email)"} ${profile.chromeProfileDirectory} ${profile.displayName}`,
2386
- ` suggested profile name: ${profile.recommendedKernelProfileName}`,
2387
- ` local Chrome path: ${profile.chromeProfilePath}`
2388
- ].join("\n")),
2389
- "",
2390
- "Customer auth setup does not use local Chrome import.",
2391
- 'In the MCP client, call browser_profile_onboard with url="https://google.com/", give the user the watch_url, then call browser_close after login to save the hosted profile.'
2392
- ];
2393
- writeOutput(lines.join("\n"), false);
2394
- });
2395
- browser.command("import-chrome").description("Developer-only: clone a local Chrome profile into MCP Scraper managed local browser storage.").option("--email <email>", "Chrome account email or displayed profile name to import").option("--name <name>", "Managed MCP Scraper browser profile name").option("--chrome-profile <directory>", "Chrome profile directory, for example Default or Profile 1").option("--user-data-dir <path>", "Source Chrome user data directory").option("--output-dir <path>", "Managed profile base directory").option("--browser-executable <path>", "Chrome executable path for local launches").option("--overwrite", "Overwrite an existing managed profile clone").option("--json", "Print machine-readable JSON").action(async (opts) => {
2396
- const result = await importChromeProfile({
2397
- email: opts.email,
2398
- name: opts.name,
2399
- chromeProfileDirectory: opts.chromeProfile,
2400
- sourceUserDataDir: opts.userDataDir,
2401
- outputDir: opts.outputDir,
2402
- browserExecutablePath: opts.browserExecutable,
2403
- overwrite: opts.overwrite
2404
- });
2405
- if (opts.json) {
2406
- writeOutput(result, true);
2407
- return;
2408
- }
2409
- writeOutput([
2410
- `Imported Chrome profile: ${result.profile.name}`,
2411
- `Source: ${result.sourceProfile.email || result.sourceProfile.displayName || result.sourceProfile.chromeProfileDirectory} (${result.sourceProfile.chromeProfileDirectory})`,
2412
- `Managed user data dir: ${result.profile.userDataDir}`,
2413
- "",
2414
- "Developer-only local install/config command:",
2415
- ` MCP_SCRAPER_API_KEY=sk_live_your_key mcp-scraper-cli agent install claude --apply --browser-mode local --browser-profile ${result.profile.name}`,
2416
- "",
2417
- "Customer auth setup should use browser_profile_onboard at Google, the hosted watch_url, and browser_close to save the hosted profile.",
2418
- "Fully exit Claude Code and start a new Claude terminal after applying developer local config. Re-run sync-profile after signing into new sites in normal Chrome."
2419
- ].join("\n"), false);
2420
- });
2421
- browser.command("sync-profile <name>").description("Refresh a managed local browser profile from its recorded source Chrome profile.").option("--output-dir <path>", "Managed profile base directory").option("--browser-executable <path>", "Chrome executable path for local launches").option("--json", "Print machine-readable JSON").action(async (name, opts) => {
2422
- const result = await syncImportedChromeProfile(name, {
2423
- outputDir: opts.outputDir,
2424
- browserExecutablePath: opts.browserExecutable
2425
- });
2426
- if (opts.json) {
2427
- writeOutput(result, true);
2428
- return;
2429
- }
2430
- writeOutput([
2431
- `Synced Chrome profile: ${result.profile.name}`,
2432
- `Source: ${result.sourceProfile.email || result.sourceProfile.displayName || result.sourceProfile.chromeProfileDirectory} (${result.sourceProfile.chromeProfileDirectory})`,
2433
- `Managed user data dir: ${result.profile.userDataDir}`,
2434
- "Restart any local browser sessions that were using this profile."
2435
- ].join("\n"), false);
2436
- });
2437
2162
  const agent = program.command("agent").description("Generate AI-agent install configs and workflow prompts.");
2438
- agent.command("install <host>").description("Print or apply install/config instructions for codex, claude/claude-code, or claude-desktop.").option("--api-key <key>", "API key to place in generated config").option("--package <spec>", "npm package spec", "mcp-scraper@latest").option("--browser-mode <mode>", "Browser mode: hosted or local", "hosted").option("--browser-profile <name>", "Default browser profile for browser_open sessions. In local mode this is a managed MCP Scraper profile name; in hosted mode it is a Kernel profile name.").option("--browser-executable <path>", "Chrome executable path for local browser mode").option("--save-browser-profile-changes", "Persist cookies/local storage back to the named browser profile when sessions close").option("--apply", "Apply the config to the local client when supported. For Claude Code, this upserts the user-scope mcp-scraper server.").option("--json", "Print machine-readable JSON").action(async (hostInput, opts) => {
2163
+ agent.command("install <host>").description("Print or apply install/config instructions for codex, claude/claude-code, or claude-desktop.").option("--api-key <key>", "API key to place in generated config").option("--package <spec>", "npm package spec", "mcp-scraper@latest").option("--browser-profile <name>", "Default saved browser profile for browser_open sessions.").option("--save-browser-profile-changes", "Persist cookies/local storage back to the named browser profile when sessions close").option("--apply", "Apply the config to the local client when supported. For Claude Code, this upserts the user-scope mcp-scraper server.").option("--json", "Print machine-readable JSON").action(async (hostInput, opts) => {
2439
2164
  const host = normalizeAgentHost(hostInput);
2440
- const browserMode = String(opts.browserMode ?? "hosted");
2441
- if (!["hosted", "local"].includes(browserMode)) throw new Error("Unknown browser mode. Use: hosted or local");
2442
2165
  const apiKey = opts.apiKey ?? process.env.MCP_SCRAPER_API_KEY;
2443
2166
  if (opts.apply) {
2444
2167
  if (host !== "claude") throw new Error('--apply is currently supported for Claude Code only. Use host "claude" or "claude-code".');
@@ -2448,10 +2171,8 @@ function buildHumanCli() {
2448
2171
  const configOptions = {
2449
2172
  apiKey,
2450
2173
  packageSpec: opts.package,
2451
- browserMode,
2452
2174
  browserProfileName: opts.browserProfile,
2453
- browserProfileSaveChanges: opts.saveBrowserProfileChanges,
2454
- browserExecutablePath: opts.browserExecutable
2175
+ browserProfileSaveChanges: opts.saveBrowserProfileChanges
2455
2176
  };
2456
2177
  const remove = await runLocalCommand("claude", claudeMcpRemoveArgs());
2457
2178
  const add = await runLocalCommand("claude", claudeMcpAddArgs(configOptions));
@@ -2481,7 +2202,7 @@ function buildHumanCli() {
2481
2202
  "Applied Claude Code MCP config: mcp-scraper",
2482
2203
  remove.code === 0 ? "Replaced existing mcp-scraper entry." : "No existing mcp-scraper entry found; added a new one.",
2483
2204
  "Command: npx -y -p " + opts.package + " mcp-scraper-combined",
2484
- browserMode === "local" && opts.browserProfile ? `Local browser profile: ${opts.browserProfile}` : "",
2205
+ opts.browserProfile ? `Browser profile: ${opts.browserProfile}` : "",
2485
2206
  "",
2486
2207
  "Next step: fully exit Claude Code, start a new Claude terminal, then run:",
2487
2208
  " claude mcp list",
@@ -2494,10 +2215,8 @@ ${result.list}` : ""
2494
2215
  const text = renderAgentInstall(host, {
2495
2216
  apiKey: opts.apiKey,
2496
2217
  packageSpec: opts.package,
2497
- browserMode,
2498
2218
  browserProfileName: opts.browserProfile,
2499
- browserProfileSaveChanges: opts.saveBrowserProfileChanges,
2500
- browserExecutablePath: opts.browserExecutable
2219
+ browserProfileSaveChanges: opts.saveBrowserProfileChanges
2501
2220
  });
2502
2221
  writeOutput(opts.json ? { host, text } : text, opts.json);
2503
2222
  });
@@ -2585,14 +2304,14 @@ ${summary.warnings.map((w) => `- ${w}`).join("\n")}` : ""
2585
2304
  runs.command("download <id>").description("Download hosted workflow run artifacts.").option("--api-key <key>", "MCP Scraper API key").option("--api-url <url>", "MCP Scraper API URL", "https://mcpscraper.dev").option("--output-dir <path>", "Download directory").option("--json", "Print machine-readable JSON").action(async (id, opts) => {
2586
2305
  const result = await apiRequest(`/workflows/runs/${id}`, "GET", opts);
2587
2306
  const { apiUrl, apiKey } = apiOptions(opts);
2588
- const outDir = (0, import_node_path5.join)(opts.outputDir ?? workflowOutputBaseDir(), "workflow-downloads", id);
2589
- await (0, import_promises6.mkdir)(outDir, { recursive: true });
2307
+ const outDir = (0, import_node_path3.join)(opts.outputDir ?? workflowOutputBaseDir(), "workflow-downloads", id);
2308
+ await (0, import_promises4.mkdir)(outDir, { recursive: true });
2590
2309
  const downloaded = [];
2591
2310
  for (const artifact of result.run.artifacts ?? []) {
2592
2311
  const res = await fetch(`${apiUrl}/workflows/runs/${id}/artifacts/${artifact.id}`, { headers: { "x-api-key": apiKey } });
2593
2312
  if (!res.ok) throw new Error(`Failed to download ${artifact.label}: HTTP ${res.status}`);
2594
- const file = (0, import_node_path5.join)(outDir, (0, import_node_path5.basename)(artifact.path));
2595
- await (0, import_promises6.writeFile)(file, Buffer.from(await res.arrayBuffer()));
2313
+ const file = (0, import_node_path3.join)(outDir, (0, import_node_path3.basename)(artifact.path));
2314
+ await (0, import_promises4.writeFile)(file, Buffer.from(await res.arrayBuffer()));
2596
2315
  downloaded.push(file);
2597
2316
  }
2598
2317
  writeOutput(opts.json ? { runId: id, files: downloaded } : downloaded.join("\n"), opts.json);