mcp-scraper 0.3.5 → 0.3.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/README.md +18 -26
  2. package/dist/bin/api-server.cjs +164 -329
  3. package/dist/bin/api-server.cjs.map +1 -1
  4. package/dist/bin/api-server.js +1 -1
  5. package/dist/bin/browser-agent-stdio-server.cjs +39 -1750
  6. package/dist/bin/browser-agent-stdio-server.cjs.map +1 -1
  7. package/dist/bin/browser-agent-stdio-server.js +2 -4
  8. package/dist/bin/browser-agent-stdio-server.js.map +1 -1
  9. package/dist/bin/mcp-scraper-cli.cjs +47 -328
  10. package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
  11. package/dist/bin/mcp-scraper-cli.js +7 -91
  12. package/dist/bin/mcp-scraper-cli.js.map +1 -1
  13. package/dist/bin/mcp-scraper-combined-stdio-server.cjs +78 -1796
  14. package/dist/bin/mcp-scraper-combined-stdio-server.cjs.map +1 -1
  15. package/dist/bin/mcp-scraper-combined-stdio-server.js +4 -6
  16. package/dist/bin/mcp-scraper-combined-stdio-server.js.map +1 -1
  17. package/dist/bin/mcp-scraper-install.cjs +4 -4
  18. package/dist/bin/mcp-scraper-install.cjs.map +1 -1
  19. package/dist/bin/mcp-scraper-install.js +2 -2
  20. package/dist/bin/mcp-stdio-server.cjs +18 -25
  21. package/dist/bin/mcp-stdio-server.cjs.map +1 -1
  22. package/dist/bin/mcp-stdio-server.js +2 -2
  23. package/dist/chunk-2NEVRPK3.js +7 -0
  24. package/dist/chunk-2NEVRPK3.js.map +1 -0
  25. package/dist/{chunk-WZT3XKES.js → chunk-2O7RAK6E.js} +19 -26
  26. package/dist/chunk-2O7RAK6E.js.map +1 -0
  27. package/dist/{chunk-RVYHP7QN.js → chunk-ELI6NDEG.js} +4 -4
  28. package/dist/{chunk-RVYHP7QN.js.map → chunk-ELI6NDEG.js.map} +1 -1
  29. package/dist/{chunk-CMK5MMEE.js → chunk-IWNLHCKA.js} +13 -694
  30. package/dist/chunk-IWNLHCKA.js.map +1 -0
  31. package/dist/{server-BC4Z2L4P.js → server-HRVD52CX.js} +842 -122
  32. package/dist/server-HRVD52CX.js.map +1 -0
  33. package/docs/mcp-tool-craft-lint.generated.md +2 -5
  34. package/docs/mcp-tool-manifest.generated.json +9 -101
  35. package/package.json +1 -1
  36. package/dist/chunk-CMK5MMEE.js.map +0 -1
  37. package/dist/chunk-DUEW4EOO.js +0 -214
  38. package/dist/chunk-DUEW4EOO.js.map +0 -1
  39. package/dist/chunk-GHBU5SGQ.js +0 -7
  40. package/dist/chunk-GHBU5SGQ.js.map +0 -1
  41. package/dist/chunk-NXRWFOEZ.js +0 -816
  42. package/dist/chunk-NXRWFOEZ.js.map +0 -1
  43. package/dist/chunk-WZT3XKES.js.map +0 -1
  44. package/dist/server-BC4Z2L4P.js.map +0 -1
@@ -1,71 +1,17 @@
1
1
  #!/usr/bin/env node
2
2
  "use strict";
3
- var __create = Object.create;
4
- var __defProp = Object.defineProperty;
5
- var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
6
- var __getOwnPropNames = Object.getOwnPropertyNames;
7
- var __getProtoOf = Object.getPrototypeOf;
8
- var __hasOwnProp = Object.prototype.hasOwnProperty;
9
- var __copyProps = (to, from, except, desc) => {
10
- if (from && typeof from === "object" || typeof from === "function") {
11
- for (let key of __getOwnPropNames(from))
12
- if (!__hasOwnProp.call(to, key) && key !== except)
13
- __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
14
- }
15
- return to;
16
- };
17
- var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
18
- // If the importer is in node compatibility mode or this is not an ESM
19
- // file that has been converted to a CommonJS file using a Babel-
20
- // compatible transform (i.e. "__esModule" has not been set), then set
21
- // "default" to the CommonJS "module.exports" for node compatibility.
22
- isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
23
- mod
24
- ));
25
3
 
26
4
  // bin/browser-agent-stdio-server.ts
27
- var import_node_fs5 = require("fs");
28
- var import_node_os6 = require("os");
29
- var import_node_path7 = require("path");
5
+ var import_node_fs2 = require("fs");
6
+ var import_node_os3 = require("os");
7
+ var import_node_path3 = require("path");
30
8
  var import_stdio = require("@modelcontextprotocol/sdk/server/stdio.js");
31
9
 
32
10
  // src/mcp/browser-agent-mcp-server.ts
33
11
  var import_mcp = require("@modelcontextprotocol/sdk/server/mcp.js");
34
- var import_node_fs4 = require("fs");
35
- var import_node_os5 = require("os");
36
- var import_node_path6 = require("path");
37
-
38
- // src/lib/chrome-profiles.ts
39
- var import_promises = require("fs/promises");
40
- var import_node_os = require("os");
41
- var import_node_path = require("path");
42
- function defaultChromeUserDataDir() {
43
- if (process.platform === "darwin") return (0, import_node_path.join)((0, import_node_os.homedir)(), "Library", "Application Support", "Google", "Chrome");
44
- if (process.platform === "win32") {
45
- return (0, import_node_path.join)(process.env.LOCALAPPDATA ?? (0, import_node_path.join)((0, import_node_os.homedir)(), "AppData", "Local"), "Google", "Chrome", "User Data");
46
- }
47
- return (0, import_node_path.join)(process.env.XDG_CONFIG_HOME ?? (0, import_node_path.join)((0, import_node_os.homedir)(), ".config"), "google-chrome");
48
- }
49
- function recommendedKernelProfileName(value) {
50
- return value.trim().toLowerCase().replace(/@/g, "-").replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80) || "browser-profile";
51
- }
52
- async function listLocalChromeProfiles(userDataDir = defaultChromeUserDataDir(), emailFilter) {
53
- const raw = await (0, import_promises.readFile)((0, import_node_path.join)(userDataDir, "Local State"), "utf8");
54
- const localState = JSON.parse(raw);
55
- const wanted = emailFilter?.trim().toLowerCase();
56
- return Object.entries(localState.profile?.info_cache ?? {}).map(([directory, info]) => {
57
- const email = (info.user_name ?? "").trim();
58
- const displayName = (info.name ?? info.shortcut_name ?? "").trim();
59
- const labelSource = email || displayName || directory;
60
- return {
61
- email,
62
- displayName,
63
- chromeProfileDirectory: directory,
64
- chromeProfilePath: (0, import_node_path.join)(userDataDir, directory),
65
- recommendedKernelProfileName: recommendedKernelProfileName(labelSource)
66
- };
67
- }).filter((row) => !wanted || row.email.toLowerCase() === wanted || row.displayName.toLowerCase().includes(wanted));
68
- }
12
+ var import_node_fs = require("fs");
13
+ var import_node_os2 = require("os");
14
+ var import_node_path2 = require("path");
69
15
 
70
16
  // src/lib/browser-service-env.ts
71
17
  function browserServiceProfileName() {
@@ -80,1363 +26,8 @@ function browserServiceProfileSaveChanges() {
80
26
  return void 0;
81
27
  }
82
28
 
83
- // src/lib/local-browser-profiles.ts
84
- var import_node_fs = require("fs");
85
- var import_promises2 = require("fs/promises");
86
- var import_node_os2 = require("os");
87
- var import_node_path2 = require("path");
88
- var EXCLUDED_PROFILE_NAMES = [
89
- "Cache",
90
- "Code Cache",
91
- "Crashpad",
92
- "DawnCache",
93
- "DawnGraphiteCache",
94
- "DawnWebGPUCache",
95
- "GPUCache",
96
- "GrShaderCache",
97
- "ShaderCache",
98
- "Shared Dictionary"
99
- ];
100
- function defaultImportedBrowserProfilesDir() {
101
- return process.env.MCP_SCRAPER_BROWSER_PROFILE_BASE_DIR?.trim() || (0, import_node_path2.join)((0, import_node_os2.homedir)(), ".mcp-scraper", "browser-profiles");
102
- }
103
- function defaultChromeExecutablePath() {
104
- const envPath = process.env.MCP_SCRAPER_BROWSER_EXECUTABLE?.trim() || process.env.CHROME_PATH?.trim();
105
- if (envPath) return envPath;
106
- const candidates = process.platform === "darwin" ? ["/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"] : process.platform === "win32" ? [
107
- (0, import_node_path2.join)(process.env.PROGRAMFILES ?? "C:\\Program Files", "Google", "Chrome", "Application", "chrome.exe"),
108
- (0, import_node_path2.join)(process.env["PROGRAMFILES(X86)"] ?? "C:\\Program Files (x86)", "Google", "Chrome", "Application", "chrome.exe")
109
- ] : ["/usr/bin/google-chrome", "/usr/bin/google-chrome-stable", "/usr/bin/chromium-browser", "/usr/bin/chromium"];
110
- return candidates.find((candidate) => (0, import_node_fs.existsSync)(candidate));
111
- }
112
- function localBrowserModeEnabled() {
113
- return process.env.MCP_SCRAPER_BROWSER_MODE?.trim().toLowerCase() === "local";
114
- }
115
- function importedProfileName(value) {
116
- return recommendedKernelProfileName(value);
117
- }
118
- function importedProfileRoot(name, outputDir = defaultImportedBrowserProfilesDir()) {
119
- return (0, import_node_path2.join)(outputDir, importedProfileName(name));
120
- }
121
- function importedProfileManifestPath(name, outputDir = defaultImportedBrowserProfilesDir()) {
122
- return (0, import_node_path2.join)(importedProfileRoot(name, outputDir), "manifest.json");
123
- }
124
- function importedProfileUserDataDir(name, outputDir = defaultImportedBrowserProfilesDir()) {
125
- return (0, import_node_path2.join)(importedProfileRoot(name, outputDir), "user-data");
126
- }
127
- async function readExistingManifest(path) {
128
- try {
129
- return JSON.parse(await (0, import_promises2.readFile)(path, "utf8"));
130
- } catch {
131
- return null;
132
- }
133
- }
134
- function isExcludedProfilePath(sourceRoot, sourcePath) {
135
- const rel = (0, import_node_path2.relative)(sourceRoot, sourcePath);
136
- if (!rel) return false;
137
- const parts = rel.split(import_node_path2.sep).filter(Boolean);
138
- const last = parts[parts.length - 1] ?? "";
139
- if (last.startsWith("Singleton")) return true;
140
- if (last === "LOCK" || last.endsWith(".lock")) return true;
141
- if (parts.some((part) => EXCLUDED_PROFILE_NAMES.includes(part))) return true;
142
- if (parts.includes("Service Worker") && parts.includes("CacheStorage")) return true;
143
- return false;
144
- }
145
- async function copyProfileTree(sourceProfilePath, destinationProfilePath) {
146
- await (0, import_promises2.cp)(sourceProfilePath, destinationProfilePath, {
147
- recursive: true,
148
- force: true,
149
- errorOnExist: false,
150
- dereference: false,
151
- filter: (source) => !isExcludedProfilePath(sourceProfilePath, source)
152
- });
153
- }
154
- function selectProfile(profiles, options) {
155
- const requestedDirectory = options.chromeProfileDirectory?.trim();
156
- const requestedEmail = options.email?.trim().toLowerCase();
157
- const exactDirectory = requestedDirectory ? profiles.find((profile) => profile.chromeProfileDirectory === requestedDirectory) : void 0;
158
- if (exactDirectory) return exactDirectory;
159
- const exactEmail = requestedEmail ? profiles.find((profile) => profile.email.toLowerCase() === requestedEmail) : void 0;
160
- if (exactEmail) return exactEmail;
161
- const first = profiles[0];
162
- if (first) return first;
163
- const hint = requestedDirectory ? `Chrome profile directory "${requestedDirectory}"` : requestedEmail ? `Chrome account "${requestedEmail}"` : "Chrome profile";
164
- throw new Error(`${hint} was not found in ${options.sourceUserDataDir ?? defaultChromeUserDataDir()}`);
165
- }
166
- async function ensureProfilePathExists(profile) {
167
- const details = await (0, import_promises2.stat)(profile.chromeProfilePath).catch(() => null);
168
- if (!details?.isDirectory()) {
169
- throw new Error(`Chrome profile path does not exist or is not a directory: ${profile.chromeProfilePath}`);
170
- }
171
- }
172
- async function writeNormalizedLocalState(sourceLocalStatePath, destinationLocalStatePath, sourceProfileDirectory) {
173
- const raw = await (0, import_promises2.readFile)(sourceLocalStatePath, "utf8");
174
- const localState = JSON.parse(raw);
175
- const selectedInfo = localState.profile?.info_cache?.[sourceProfileDirectory] ?? localState.profile?.info_cache?.Default ?? {};
176
- localState.profile = {
177
- ...localState.profile ?? {},
178
- info_cache: { Default: selectedInfo },
179
- last_used: "Default",
180
- last_active_profiles: ["Default"]
181
- };
182
- await (0, import_promises2.writeFile)(destinationLocalStatePath, `${JSON.stringify(localState, null, 2)}
183
- `);
184
- }
185
- async function importChromeProfile(options = {}) {
186
- const sourceUserDataDir = options.sourceUserDataDir?.trim() || defaultChromeUserDataDir();
187
- const profiles = await listLocalChromeProfiles(sourceUserDataDir, options.email);
188
- const sourceProfile = selectProfile(profiles, { ...options, sourceUserDataDir });
189
- await ensureProfilePathExists(sourceProfile);
190
- const name = importedProfileName(options.name?.trim() || sourceProfile.recommendedKernelProfileName);
191
- const outputDir = options.outputDir?.trim() || defaultImportedBrowserProfilesDir();
192
- const profileRoot = importedProfileRoot(name, outputDir);
193
- const manifestPath = importedProfileManifestPath(name, outputDir);
194
- const userDataDir = importedProfileUserDataDir(name, outputDir);
195
- const existingManifest = await readExistingManifest(manifestPath);
196
- const existing = (0, import_node_fs.existsSync)(profileRoot);
197
- if (existing && !options.overwrite) {
198
- throw new Error(`Imported browser profile "${name}" already exists at ${profileRoot}. Re-run with --overwrite to refresh it.`);
199
- }
200
- if (existing) await (0, import_promises2.rm)(profileRoot, { recursive: true, force: true });
201
- await (0, import_promises2.mkdir)((0, import_node_path2.join)(userDataDir, "Default"), { recursive: true });
202
- await writeNormalizedLocalState(
203
- (0, import_node_path2.join)(sourceUserDataDir, "Local State"),
204
- (0, import_node_path2.join)(userDataDir, "Local State"),
205
- sourceProfile.chromeProfileDirectory
206
- );
207
- await copyProfileTree(sourceProfile.chromeProfilePath, (0, import_node_path2.join)(userDataDir, "Default"));
208
- const now = (/* @__PURE__ */ new Date()).toISOString();
209
- const manifest = {
210
- layoutVersion: 1,
211
- name,
212
- profileRoot,
213
- userDataDir,
214
- browserExecutablePath: options.browserExecutablePath?.trim() || defaultChromeExecutablePath() || null,
215
- sourceUserDataDir,
216
- sourceProfileDirectory: sourceProfile.chromeProfileDirectory,
217
- sourceProfilePath: sourceProfile.chromeProfilePath,
218
- email: sourceProfile.email,
219
- displayName: sourceProfile.displayName,
220
- createdAt: existingManifest?.createdAt ?? now,
221
- updatedAt: now,
222
- copyMode: "chrome-profile-clone"
223
- };
224
- await (0, import_promises2.writeFile)(manifestPath, `${JSON.stringify(manifest, null, 2)}
225
- `);
226
- return {
227
- profile: manifest,
228
- sourceProfile,
229
- excludedNames: EXCLUDED_PROFILE_NAMES
230
- };
231
- }
232
- async function loadImportedBrowserProfile(name = process.env.MCP_SCRAPER_BROWSER_PROFILE?.trim() || "default", outputDir = defaultImportedBrowserProfilesDir()) {
233
- const profileName = importedProfileName(name);
234
- const manifestPath = importedProfileManifestPath(profileName, outputDir);
235
- const manifest = await readExistingManifest(manifestPath);
236
- if (!manifest) {
237
- throw new Error(
238
- `Imported browser profile "${profileName}" was not found. Run: mcp-scraper-cli browser import-chrome --name ${profileName}`
239
- );
240
- }
241
- return manifest;
242
- }
243
- async function syncImportedChromeProfile(name, options = {}) {
244
- const outputDir = options.outputDir?.trim() || defaultImportedBrowserProfilesDir();
245
- const manifest = await loadImportedBrowserProfile(name, outputDir);
246
- return importChromeProfile({
247
- name: manifest.name,
248
- sourceUserDataDir: manifest.sourceUserDataDir,
249
- chromeProfileDirectory: manifest.sourceProfileDirectory,
250
- outputDir,
251
- overwrite: true,
252
- browserExecutablePath: options.browserExecutablePath ?? manifest.browserExecutablePath ?? void 0
253
- });
254
- }
255
-
256
- // src/services/browser-agent/local-browser-agent-service.ts
257
- var import_node_crypto = require("crypto");
258
- var import_node_fs3 = require("fs");
259
- var import_node_path4 = require("path");
260
- var import_playwright_extra = require("playwright-extra");
261
- var import_puppeteer_extra_plugin_stealth = __toESM(require("puppeteer-extra-plugin-stealth"), 1);
262
-
263
- // src/services/fanout/cdp-capture.ts
264
- var ANSWER_RE = /\/backend-api\/(f\/)?conversation$|\/chat_conversations\/[^/]+\/(completion|retry_completion)$|\/completion$/i;
265
- var FanoutCdpCapture = class {
266
- client = null;
267
- reqUrl = /* @__PURE__ */ new Map();
268
- ctById = /* @__PURE__ */ new Map();
269
- answerIds = /* @__PURE__ */ new Set();
270
- bodies = [];
271
- pending = [];
272
- answerStreams = 0;
273
- async attach(page) {
274
- if (this.client) return;
275
- const client = await page.context().newCDPSession(page);
276
- this.client = client;
277
- await client.send("Network.enable", { maxTotalBufferSize: 2e8, maxResourceBufferSize: 2e8 });
278
- client.on("Network.requestWillBeSent", (e) => {
279
- const id = e.requestId;
280
- const req = e.request;
281
- if (id) this.reqUrl.set(id, req?.url || "");
282
- });
283
- client.on("Network.responseReceived", (e) => {
284
- const id = e.requestId;
285
- const res = e.response;
286
- const url = res?.url || this.reqUrl.get(id) || "";
287
- const headers = res?.headers || {};
288
- const ct = String(headers["content-type"] || headers["Content-Type"] || "");
289
- if (id) this.ctById.set(id, ct);
290
- if (id && (this.isAnswerUrl(url) || /event-stream/i.test(ct))) this.answerIds.add(id);
291
- });
292
- client.on("Network.loadingFinished", (e) => this.grab(e.requestId));
293
- }
294
- reset() {
295
- this.bodies = [];
296
- this.answerStreams = 0;
297
- this.answerIds.clear();
298
- this.pending = [];
299
- }
300
- answerStreamCount() {
301
- return this.answerStreams;
302
- }
303
- async drain() {
304
- await Promise.all(this.pending);
305
- }
306
- bestAnswerText() {
307
- const eventStreams = this.bodies.filter((b) => /event-stream/i.test(b.ct));
308
- const answerStreams = eventStreams.filter((b) => this.isAnswerUrl(b.url)).sort((a, b) => b.text.length - a.text.length);
309
- if (answerStreams.length) return answerStreams.map((b) => b.text).join("\n");
310
- const answerJson = this.bodies.filter((b) => this.isAnswerUrl(b.url)).sort((a, b) => b.text.length - a.text.length);
311
- if (answerJson.length) return answerJson[0].text;
312
- if (eventStreams.length) return eventStreams.sort((a, b) => b.text.length - a.text.length)[0].text;
313
- return "";
314
- }
315
- async detach() {
316
- try {
317
- await this.client?.detach();
318
- } catch {
319
- this.client = null;
320
- }
321
- this.client = null;
322
- }
323
- isAnswerUrl(url) {
324
- let path = "";
325
- try {
326
- path = new URL(url).pathname;
327
- } catch {
328
- path = String(url || "").split("?")[0];
329
- }
330
- return ANSWER_RE.test(path);
331
- }
332
- grab(requestId) {
333
- if (!requestId || !this.client || !this.answerIds.has(requestId)) return;
334
- const url = this.reqUrl.get(requestId) || "";
335
- const ct = this.ctById.get(requestId) || "";
336
- const client = this.client;
337
- const p = client.send("Network.getResponseBody", { requestId }).then((r) => {
338
- const text = r.base64Encoded ? Buffer.from(String(r.body), "base64").toString("utf8") : String(r.body || "");
339
- if (text) {
340
- this.bodies.push({ url, ct, text });
341
- if (this.isAnswerUrl(url)) this.answerStreams++;
342
- }
343
- }).catch(() => void 0);
344
- this.pending.push(p);
345
- }
346
- };
347
-
348
- // src/services/fanout/parse-helpers.ts
349
- var INTERNAL = /chatgpt\.com|openai\.com|oaiusercontent|oaistatic|claude\.ai|anthropic\.com/i;
350
- var TRACK = /^(utm_|oai|fbclid|gclid|ref$|ref_|sa$|usg$|ved$|sca_esv$|sources$)/i;
351
- function domainOf(u) {
352
- try {
353
- return new URL(u).hostname.replace(/^www\./, "");
354
- } catch {
355
- return "";
356
- }
357
- }
358
- function normUrl(u) {
359
- try {
360
- const x = new URL(u);
361
- x.hash = "";
362
- for (const k of [...x.searchParams.keys()]) {
363
- if (TRACK.test(k)) x.searchParams.delete(k);
364
- }
365
- return x.toString().replace(/\/$/, "").toLowerCase();
366
- } catch {
367
- return String(u || "").toLowerCase();
368
- }
369
- }
370
- function isExternal(u) {
371
- const h = domainOf(u);
372
- return Boolean(h) && !INTERNAL.test(h);
373
- }
374
- function unesc(s) {
375
- try {
376
- return JSON.parse('"' + s + '"');
377
- } catch {
378
- return String(s).replace(/\\(.)/g, "$1");
379
- }
380
- }
381
- function stripCitationMarkers(s) {
382
- return String(s || "").replace(/[-]/g, "").replace(/cite(?:turn\d+\w+?\d+)+/g, "").replace(/\s+/g, " ").trim();
383
- }
384
- function sliceBalanced(text, openIdx) {
385
- const open = text[openIdx];
386
- const close = open === "[" ? "]" : "}";
387
- let depth = 0;
388
- let inStr = false;
389
- let esc2 = false;
390
- for (let i = openIdx; i < text.length; i++) {
391
- const ch = text[i];
392
- if (inStr) {
393
- if (esc2) esc2 = false;
394
- else if (ch === "\\") esc2 = true;
395
- else if (ch === '"') inStr = false;
396
- continue;
397
- }
398
- if (ch === '"') {
399
- inStr = true;
400
- continue;
401
- }
402
- if (ch === open) depth++;
403
- else if (ch === close) {
404
- depth--;
405
- if (depth === 0) return text.slice(openIdx, i + 1);
406
- }
407
- }
408
- return null;
409
- }
410
- function findJsonValues(text, field) {
411
- const out = [];
412
- const re = new RegExp('"' + field + '"\\s*:\\s*(\\[|\\{)', "g");
413
- let m;
414
- while (m = re.exec(text)) {
415
- const openIdx = m.index + m[0].length - 1;
416
- const slice = sliceBalanced(text, openIdx);
417
- if (slice) {
418
- try {
419
- out.push(JSON.parse(slice));
420
- } catch {
421
- continue;
422
- }
423
- }
424
- }
425
- return out;
426
- }
427
- function stringsInArrayBody(arrBody) {
428
- const out = [];
429
- const re = /"((?:[^"\\]|\\.){2,300}?)"/g;
430
- let m;
431
- while (m = re.exec(arrBody)) out.push(unesc(m[1]));
432
- return out;
433
- }
434
- function cleanQuery(q) {
435
- const t = (q || "").trim();
436
- if (t.length < 2 || t.length > 300) return null;
437
- return t;
438
- }
439
-
440
- // src/services/fanout/adapters/chatgpt.ts
441
- function roundOf(refId) {
442
- const m = /^turn(\d+)/.exec(refId || "");
443
- return m ? Number(m[1]) : null;
444
- }
445
- function normRef(r) {
446
- if (typeof r === "string") {
447
- const m = /turn(\d+)(search|news|ref|video|image|product)(\d+)/.exec(r);
448
- return m ? `turn${m[1]}${m[2]}${m[3]}` : null;
449
- }
450
- if (r && typeof r === "object") {
451
- const o = r;
452
- if (o.turn_index != null) {
453
- return `turn${o.turn_index}${o.ref_type || "search"}${o.ref_index != null ? o.ref_index : ""}`;
454
- }
455
- }
456
- return null;
457
- }
458
- function refsInText(s) {
459
- return (String(s || "").match(/turn\d+(?:search|news|ref|video|image|product)\d+/g) || []).map(normRef).filter((x) => Boolean(x));
460
- }
461
- function collectInto(arr, v) {
462
- if (Array.isArray(v)) arr.push(...v);
463
- else if (v && typeof v === "object") arr.push(v);
464
- }
465
- function reconstructFromSsePatches(raw) {
466
- const contentReferences = [];
467
- const searchResultGroups = [];
468
- const citations = [];
469
- const citedUrls = /* @__PURE__ */ new Set();
470
- const state = { found: false };
471
- const handleOp = (op) => {
472
- if (!op || typeof op !== "object") return;
473
- const o = op;
474
- const v = o.v;
475
- if (o.o === "patch" && Array.isArray(v)) {
476
- state.found = true;
477
- v.forEach(handleOp);
478
- return;
479
- }
480
- const path = typeof o.p === "string" ? o.p : "";
481
- if ((path === "" || path === "/message") && v && typeof v === "object") {
482
- const root = v;
483
- const msg = root.message || root;
484
- const md = msg && msg.metadata;
485
- if (md) {
486
- collectInto(contentReferences, md.content_references);
487
- collectInto(searchResultGroups, md.search_result_groups);
488
- collectInto(citations, md.citations);
489
- state.found = true;
490
- }
491
- }
492
- if (!path) return;
493
- if (/\/safe_urls$/.test(path) && Array.isArray(v)) {
494
- for (const u of v) if (typeof u === "string") citedUrls.add(u);
495
- state.found = true;
496
- } else if (path.includes("content_references")) {
497
- collectInto(contentReferences, v);
498
- state.found = true;
499
- } else if (path.includes("search_result_groups")) {
500
- collectInto(searchResultGroups, v);
501
- state.found = true;
502
- } else if (path.includes("citations")) {
503
- collectInto(citations, v);
504
- state.found = true;
505
- }
506
- };
507
- for (const line of raw.split("\n")) {
508
- const s = (line.startsWith("data:") ? line.slice(5) : line).trim();
509
- if (!s || s === "[DONE]" || s[0] !== "{" && s[0] !== "[") continue;
510
- let parsed;
511
- try {
512
- parsed = JSON.parse(s);
513
- } catch {
514
- continue;
515
- }
516
- const ops = Array.isArray(parsed) ? parsed : [parsed];
517
- ops.forEach(handleOp);
518
- }
519
- if (!state.found) return raw;
520
- const citedRefs = [...citedUrls].map((url) => ({ type: "grouped_webpages", items: [{ url }] }));
521
- const synthetic = JSON.stringify({
522
- content_references: [...contentReferences, ...citedRefs],
523
- search_result_groups: searchResultGroups,
524
- citations
525
- });
526
- return raw + "\n" + synthetic;
527
- }
528
- var chatgptAdapter = {
529
- platform: "ChatGPT",
530
- matchesHost(hostname) {
531
- return /(^|\.)chatgpt\.com$|(^|\.)openai\.com$/i.test(hostname);
532
- },
533
- parse(rawInput, prompt) {
534
- const raw = reconstructFromSsePatches(rawInput);
535
- const sources = /* @__PURE__ */ new Map();
536
- const snippets = /* @__PURE__ */ new Map();
537
- const queries = /* @__PURE__ */ new Set();
538
- const unmapped = /* @__PURE__ */ new Set();
539
- const meta = { model: "", finishType: "", title: "", rounds: 0 };
540
- const ensure = (url) => {
541
- if (!isExternal(url)) return null;
542
- const k = normUrl(url);
543
- let s = sources.get(k);
544
- if (!s) {
545
- s = { url, domain: domainOf(url), title: "", cited: false, timesCited: 0, firstOrder: Infinity, snippet: "", round: null };
546
- sources.set(k, s);
547
- }
548
- return s;
549
- };
550
- const addSnippet = (url, title, text) => {
551
- const clean = stripCitationMarkers(text);
552
- if (!isExternal(url) || clean.length < 2) return;
553
- snippets.set(normUrl(url), { url, domain: domainOf(url), title: title || domainOf(url), text: clean.slice(0, 400) });
554
- };
555
- for (const m of raw.matchAll(/"([a-z_]*(?:quer|search)[a-z_]*)"\s*:/gi)) unmapped.add(m[1]);
556
- for (const m of raw.matchAll(/"search_model_queries"\s*:\s*\[([^\]]*)\]/g)) {
557
- for (const q of stringsInArrayBody(m[1])) {
558
- const c = cleanQuery(q);
559
- if (c) queries.add(c);
560
- }
561
- }
562
- for (const m of raw.matchAll(/"(?:search_queries|queries|search_query)"\s*:\s*\[([^\]]*)\]/g)) {
563
- for (const q of stringsInArrayBody(m[1])) {
564
- const c = cleanQuery(q);
565
- if (c) queries.add(c);
566
- }
567
- }
568
- for (const v of findJsonValues(raw, "search_result_groups")) {
569
- if (!Array.isArray(v)) continue;
570
- for (const g of v) {
571
- const entries = g && typeof g === "object" && Array.isArray(g.entries) ? g.entries : [];
572
- for (const e of entries) {
573
- const url = typeof e.url === "string" ? e.url : "";
574
- const s = ensure(url);
575
- if (!s) continue;
576
- if (!s.title && typeof e.title === "string") s.title = e.title.slice(0, 200);
577
- const ref = normRef(e.ref_id);
578
- if (ref && s.round == null) s.round = roundOf(ref);
579
- if (typeof e.snippet === "string" && e.snippet) addSnippet(url, s.title, e.snippet);
580
- }
581
- }
582
- }
583
- for (const m of raw.matchAll(/"url"\s*:\s*"(https?:\/\/[^"\\]+)"(?:[\s\S]{0,120}?"(?:title|name)"\s*:\s*"((?:[^"\\]|\\.){0,200})")?/g)) {
584
- const s = ensure(m[1]);
585
- if (s && !s.title && m[2]) s.title = unesc(m[2]).slice(0, 200);
586
- }
587
- let spanTotal = 0;
588
- for (const cr of findJsonValues(raw, "content_references")) {
589
- if (!Array.isArray(cr)) continue;
590
- for (const ref of cr) {
591
- const type = typeof ref.type === "string" ? ref.type : "";
592
- const isBundle = /sources_footnote|nav_list|alt_text/.test(type);
593
- const refs = refsInText(typeof ref.matched_text === "string" ? ref.matched_text : "");
594
- if (!isBundle) spanTotal++;
595
- const ord = typeof ref.start_idx === "number" ? ref.start_idx : isBundle ? Infinity : spanTotal;
596
- const items = Array.isArray(ref.items) ? ref.items : [];
597
- items.forEach((it, idx) => {
598
- const refForItem = refs.length === items.length ? refs[idx] : refs[0];
599
- const urls = [it.url].concat((Array.isArray(it.supporting_websites) ? it.supporting_websites : []).map((s) => s?.url)).filter((u) => typeof u === "string");
600
- for (const u of urls) {
601
- const s = ensure(u);
602
- if (!s) continue;
603
- if (!isBundle) {
604
- s.cited = true;
605
- s.timesCited++;
606
- if (ord < s.firstOrder) s.firstOrder = ord;
607
- } else if (!s.cited) {
608
- s.cited = true;
609
- }
610
- if (refForItem && s.round == null) s.round = roundOf(refForItem);
611
- }
612
- if (typeof it.url === "string" && typeof it.snippet === "string") addSnippet(it.url, typeof it.title === "string" ? it.title : "", it.snippet);
613
- });
614
- }
615
- }
616
- for (const v of findJsonValues(raw, "citations")) {
617
- if (!Array.isArray(v)) continue;
618
- v.forEach((c, i) => {
619
- const md = c && typeof c === "object" ? c.metadata : null;
620
- if (!md || typeof md.url !== "string") return;
621
- const s = ensure(md.url);
622
- if (!s) return;
623
- s.cited = true;
624
- s.timesCited++;
625
- const ord = typeof c.start_ix === "number" ? c.start_ix : i;
626
- if (ord < s.firstOrder) s.firstOrder = ord;
627
- if (typeof md.text === "string") addSnippet(md.url, typeof md.title === "string" ? md.title : "", md.text);
628
- });
629
- }
630
- for (const m of raw.matchAll(/"content_type"\s*:\s*"tether_quote"([\s\S]{0,4000}?)"text"\s*:\s*"((?:[^"\\]|\\.)*)"/g)) {
631
- const url = (m[1].match(/"url"\s*:\s*"([^"]+)"/) || [])[1];
632
- const title = (m[1].match(/"title"\s*:\s*"((?:[^"\\]|\\.)*)"/) || [])[1];
633
- if (url) {
634
- ensure(url);
635
- addSnippet(url, title ? unesc(title) : "", unesc(m[2]));
636
- }
637
- }
638
- const modelMatches = [...raw.matchAll(/"resolved_model_slug"\s*:\s*"([^"]+)"/g)];
639
- const slugMatches = modelMatches.length ? modelMatches : [...raw.matchAll(/"model_slug"\s*:\s*"([^"]+)"/g)];
640
- if (slugMatches.length) meta.model = slugMatches[slugMatches.length - 1][1];
641
- const finish = raw.match(/"finish_details"\s*:\s*\{[^}]*"type"\s*:\s*"([^"]+)"/);
642
- if (finish) meta.finishType = finish[1];
643
- const searchRounds = (raw.match(/"recipient"\s*:\s*"(?:web|browser)"[\s\S]{0,400}?"content_type"\s*:\s*"code"[\s\S]{0,200}?search\(/g) || []).length;
644
- const sourceRounds = /* @__PURE__ */ new Set();
645
- for (const s of sources.values()) if (s.round != null) sourceRounds.add(s.round);
646
- meta.rounds = Math.max(searchRounds, sourceRounds.size);
647
- const ordered = [...sources.values()].sort((a, b) => a.cited === b.cited ? b.timesCited - a.timesCited || a.firstOrder - b.firstOrder : a.cited ? -1 : 1).map((s) => ({
648
- url: s.url,
649
- domain: s.domain,
650
- title: s.title || s.domain,
651
- cited: s.cited,
652
- timesCited: s.timesCited,
653
- snippet: snippets.get(normUrl(s.url))?.text || "",
654
- round: s.round
655
- }));
656
- return {
657
- subQueries: [...queries],
658
- sources: ordered,
659
- snippets: [...snippets.values()],
660
- meta,
661
- unmappedKeys: [...unmapped].slice(0, 50)
662
- };
663
- }
664
- };
665
-
666
- // src/services/fanout/adapters/claude.ts
667
- function isFavicon(u) {
668
- return /\/s2\/favicons/i.test(u) || /favicon/i.test(u);
669
- }
670
- var claudeAdapter = {
671
- platform: "Claude",
672
- matchesHost(hostname) {
673
- return /(^|\.)claude\.ai$/i.test(hostname);
674
- },
675
- parse(raw, prompt) {
676
- const sources = /* @__PURE__ */ new Map();
677
- const snippets = /* @__PURE__ */ new Map();
678
- const queries = /* @__PURE__ */ new Set();
679
- const unmapped = /* @__PURE__ */ new Set();
680
- const meta = { model: "", finishType: "", title: "", rounds: 0 };
681
- const ensure = (url) => {
682
- if (!url || !isExternal(url) || isFavicon(url)) return null;
683
- const k = normUrl(url);
684
- let s = sources.get(k);
685
- if (!s) {
686
- s = { url, domain: domainOf(url), title: "", cited: false, timesCited: 0, snippet: "", round: null };
687
- sources.set(k, s);
688
- }
689
- return s;
690
- };
691
- const toolInputs = /* @__PURE__ */ new Map();
692
- const searchBlocks = /* @__PURE__ */ new Set();
693
- const finalizeTool = (index) => {
694
- const t = toolInputs.get(index);
695
- if (!t || !/web_search|web_fetch|search|browse/i.test(t.name)) return;
696
- try {
697
- const input = JSON.parse(t.buf);
698
- if (typeof input.query === "string") {
699
- const q = cleanQuery(input.query);
700
- if (q) queries.add(q);
701
- }
702
- } catch {
703
- const m = t.buf.match(/"query"\s*:\s*"((?:[^"\\]|\\.)*)"/);
704
- if (m) {
705
- const q = cleanQuery(unesc(m[1]));
706
- if (q) queries.add(q);
707
- }
708
- }
709
- };
710
- for (const line of raw.split("\n")) {
711
- const s = (line.startsWith("data:") ? line.slice(5) : line).trim();
712
- if (!s || s[0] !== "{" && s[0] !== "[") continue;
713
- let ev;
714
- try {
715
- ev = JSON.parse(s);
716
- } catch {
717
- continue;
718
- }
719
- const type = ev.type;
720
- if (type === "content_block_start") {
721
- const cb = ev.content_block || {};
722
- const index = typeof ev.index === "number" ? ev.index : -1;
723
- if (cb.type === "tool_use" && index >= 0) {
724
- const name = typeof cb.name === "string" ? cb.name : "";
725
- toolInputs.set(index, { name, buf: "" });
726
- if (/web_search|web_fetch|search|browse/i.test(name)) searchBlocks.add(index);
727
- }
728
- } else if (type === "content_block_delta") {
729
- const d = ev.delta || {};
730
- const index = typeof ev.index === "number" ? ev.index : -1;
731
- if (d.type === "input_json_delta" && typeof d.partial_json === "string") {
732
- const t = toolInputs.get(index);
733
- if (t) t.buf += d.partial_json;
734
- } else if (d.type === "citation_start_delta") {
735
- const c = d.citation || {};
736
- if (typeof c.url === "string") {
737
- const src = ensure(c.url);
738
- if (src) {
739
- src.cited = true;
740
- src.timesCited++;
741
- if (!src.title && typeof c.title === "string") src.title = c.title.slice(0, 200);
742
- if (typeof c.cited_text === "string" && c.cited_text) {
743
- snippets.set(normUrl(c.url), { url: c.url, domain: domainOf(c.url), title: src.title, text: String(c.cited_text).slice(0, 400) });
744
- }
745
- }
746
- }
747
- }
748
- } else if (type === "content_block_stop") {
749
- const index = typeof ev.index === "number" ? ev.index : -1;
750
- if (index >= 0) finalizeTool(index);
751
- } else if (type === "message_delta" || type === "message_stop" || type === "message_start") {
752
- const d = ev.delta || ev.message || {};
753
- if (typeof d.stop_reason === "string" && !meta.finishType) meta.finishType = d.stop_reason;
754
- if (typeof d.model === "string" && !meta.model) meta.model = d.model;
755
- }
756
- }
757
- for (const index of searchBlocks) finalizeTool(index);
758
- for (const m of raw.matchAll(/"title"\s*:\s*"((?:[^"\\]|\\.){1,200})"\s*,\s*"url"\s*:\s*"(https?:\/\/[^"\\]+)"/g)) {
759
- const src = ensure(m[2]);
760
- if (src && !src.title) src.title = unesc(m[1]).slice(0, 200);
761
- }
762
- for (const m of raw.matchAll(/"url"\s*:\s*"(https?:\/\/[^"\\]+)"/g)) ensure(m[1]);
763
- for (const m of raw.matchAll(/\\"title\\"\s*:\s*\\"((?:[^"\\]|\\.){1,160}?)\\"[\s\S]{0,160}?\\"url\\"\s*:\s*\\"(https?:[^"\\]+)\\"/g)) {
764
- const src = ensure(m[2]);
765
- if (src && !src.title) src.title = unesc(unesc(m[1])).slice(0, 200);
766
- }
767
- for (const m of raw.matchAll(/\\"url\\"\s*:\s*\\"(https?:[^"\\]+)\\"/g)) ensure(m[1]);
768
- if (!meta.model) {
769
- const mm = raw.match(/"model"\s*:\s*"(claude[^"]+)"/);
770
- if (mm) meta.model = mm[1];
771
- }
772
- meta.rounds = queries.size > 0 ? searchBlocks.size : 0;
773
- for (const m of raw.matchAll(/"([a-z_]*(?:quer|search)[a-z_]*)"\s*:/gi)) unmapped.add(m[1]);
774
- const ordered = [...sources.values()].sort((a, b) => a.cited === b.cited ? b.timesCited - a.timesCited : a.cited ? -1 : 1).map((s) => ({
775
- url: s.url,
776
- domain: s.domain,
777
- title: s.title || s.domain,
778
- cited: s.cited,
779
- timesCited: s.timesCited,
780
- snippet: snippets.get(normUrl(s.url))?.text || "",
781
- round: s.round
782
- }));
783
- return {
784
- subQueries: [...queries],
785
- sources: ordered,
786
- snippets: [...snippets.values()],
787
- meta,
788
- unmappedKeys: [...unmapped].slice(0, 50)
789
- };
790
- }
791
- };
792
-
793
- // src/services/fanout/adapters/index.ts
794
- var FANOUT_ADAPTERS = [chatgptAdapter, claudeAdapter];
795
- function adapterForHost(hostname) {
796
- return FANOUT_ADAPTERS.find((a) => a.matchesHost(hostname)) ?? null;
797
- }
798
-
799
- // src/services/fanout/build.ts
800
- function buildFanoutResult(ctx) {
801
- const { platform, parsed, ready, rawBytes } = ctx;
802
- const browsedUrls = parsed.sources;
803
- const citedUrls = browsedUrls.filter((s) => s.cited);
804
- const browsedOnly = browsedUrls.filter((s) => !s.cited);
805
- const prompt = ctx.prompt || parsed.meta.title || "";
806
- const result = {
807
- platform,
808
- capturedAt: (/* @__PURE__ */ new Date()).toISOString(),
809
- prompt,
810
- meta: parsed.meta,
811
- subQueries: parsed.subQueries,
812
- browsedUrls,
813
- citedUrls,
814
- browsedOnly,
815
- snippets: parsed.snippets,
816
- counts: {
817
- subQueries: parsed.subQueries.length,
818
- browsed: browsedUrls.length,
819
- cited: citedUrls.length,
820
- browsedOnly: browsedOnly.length
821
- }
822
- };
823
- if (parsed.subQueries.length === 0 && browsedUrls.length === 0) {
824
- result.debug = {
825
- interceptorReady: ready,
826
- rawBytes,
827
- unmappedKeys: parsed.unmappedKeys,
828
- note: ready ? "Capture hook is live but no fan-out was seen yet. Run a NEW prompt that triggers web search in this session, then call browser_capture_fanout again. Fan-out is only captured as it streams; past answers from before the session opened cannot be recovered." : "Capture hook not detected on this page. The session must be opened with MCP_SCRAPER_BROWSER_MODE=local and navigated to chatgpt.com or claude.ai after opening, so the hook installs before the site scripts run."
829
- };
830
- }
831
- return result;
832
- }
833
-
834
- // src/services/fanout/classify.ts
835
- var SOCIAL = /(^|\.)(youtube\.com|youtu\.be|instagram\.com|tiktok\.com|facebook\.com|fb\.com|x\.com|twitter\.com|linkedin\.com|pinterest\.com|threads\.net|vimeo\.com)$/i;
836
- var ENCYCLOPEDIA = /(^|\.)(wikipedia\.org|wikimedia\.org|britannica\.com|merriam-webster\.com|dictionary\.com|fandom\.com|investopedia\.com)$/i;
837
- var REVIEW = /(^|\.)(yelp\.com|trustpilot\.com|g2\.com|capterra\.com|angi\.com|angieslist\.com|bbb\.org|consumeraffairs\.com|bestcompany\.com|sitejabber\.com|clutch\.co|bestpickreports\.com|homeguide\.com|porch\.com|ontoplist\.com|thumbtack\.com)$/i;
838
- var NEWS = /(^|\.)(reuters\.com|apnews\.com|nytimes\.com|wsj\.com|washingtonpost\.com|bloomberg\.com|cnbc\.com|forbes\.com|bbc\.com|theguardian\.com|cnn\.com|npr\.org|usatoday\.com|expressnews\.com|mysanantonio\.com|beaumontenterprise\.com)$/i;
839
- function classifyDomain(domain, firstPartyDomain) {
840
- const d = domain.toLowerCase().replace(/^www\./, "");
841
- if (firstPartyDomain) {
842
- const fp = firstPartyDomain.toLowerCase().replace(/^www\./, "");
843
- if (fp && (d === fp || d.endsWith("." + fp))) return "First-party/vendor";
844
- }
845
- if (/(^|\.)reddit\.com$/i.test(d)) return "Reddit";
846
- if (SOCIAL.test(d)) return "Social/video";
847
- if (ENCYCLOPEDIA.test(d)) return "Encyclopedia";
848
- if (REVIEW.test(d)) return "Review site";
849
- if (NEWS.test(d) || /(^|\.)news\./i.test(d)) return "News/media";
850
- if (/^docs?\.|^developer\.|^developers\.|readthedocs|\.dev$/i.test(d)) return "Docs";
851
- return "Blog";
852
- }
853
- function emptyCategoryCounts() {
854
- return { "First-party/vendor": 0, "News/media": 0, Reddit: 0, "Social/video": 0, Encyclopedia: 0, "Review site": 0, Docs: 0, Blog: 0 };
855
- }
856
- function enrichFanout(result, opts = {}) {
857
- const addSiteType = (s) => ({ ...s, siteType: classifyDomain(s.domain, opts.firstPartyDomain) });
858
- const browsedUrls = result.browsedUrls.map(addSiteType);
859
- const citedUrls = result.citedUrls.map(addSiteType);
860
- const browsedOnly = result.browsedOnly.map(addSiteType);
861
- const byCategory = emptyCategoryCounts();
862
- const domainAgg = /* @__PURE__ */ new Map();
863
- for (const s of browsedUrls) {
864
- byCategory[s.siteType]++;
865
- const cur = domainAgg.get(s.domain);
866
- if (!cur) domainAgg.set(s.domain, { domain: s.domain, count: 1, cited: s.cited, timesCited: s.timesCited, siteType: s.siteType });
867
- else {
868
- cur.count++;
869
- cur.cited = cur.cited || s.cited;
870
- cur.timesCited += s.timesCited;
871
- }
872
- }
873
- const topSites = [...domainAgg.values()].sort((a, b) => b.count - a.count || b.timesCited - a.timesCited);
874
- const citationOrder = citedUrls.slice().sort((a, b) => b.timesCited - a.timesCited).map((s, i) => ({ rank: i + 1, domain: s.domain, url: s.url, timesCited: s.timesCited }));
875
- return {
876
- platform: result.platform,
877
- capturedAt: result.capturedAt,
878
- prompt: result.prompt,
879
- meta: result.meta,
880
- queries: result.subQueries,
881
- browsedUrls,
882
- citedUrls,
883
- browsedOnly,
884
- snippets: result.snippets,
885
- counts: result.counts,
886
- aggregates: { topSites, citationOrder, byCategory },
887
- firstPartyDomain: opts.firstPartyDomain ? opts.firstPartyDomain.toLowerCase().replace(/^www\./, "") : null,
888
- ...result.debug ? { debug: result.debug } : {}
889
- };
890
- }
891
-
892
- // src/services/fanout/export.ts
893
- var import_node_fs2 = require("fs");
894
- var import_node_os3 = require("os");
895
- var import_node_path3 = require("path");
896
- var import_papaparse = __toESM(require("papaparse"), 1);
897
- function outputBaseDir() {
898
- return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path3.join)((0, import_node_os3.homedir)(), "Downloads", "mcp-scraper");
899
- }
900
- function safe(value) {
901
- return value.replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80) || "capture";
902
- }
903
- function writeTable(path, rows, delimiter) {
904
- (0, import_node_fs2.writeFileSync)(path, import_papaparse.default.unparse(rows.length ? rows : [{}], { delimiter }));
905
- }
906
- function exportFanout(enriched) {
907
- const stamp = safe(enriched.capturedAt.replace(/[:.]/g, "-"));
908
- const dir = (0, import_node_path3.join)(outputBaseDir(), "fanout", `${stamp}-${safe(enriched.platform)}`);
909
- (0, import_node_fs2.mkdirSync)(dir, { recursive: true });
910
- const queryRows = enriched.queries.map((q, i) => ({ index: i + 1, query: q }));
911
- const citationRows = enriched.aggregates.citationOrder.map((c) => {
912
- const src = enriched.citedUrls.find((s) => s.url === c.url);
913
- return { rank: c.rank, domain: c.domain, url: c.url, timesCited: c.timesCited, siteType: src?.siteType ?? "", title: src?.title ?? "" };
914
- });
915
- const sourceRows = enriched.browsedUrls.map((s) => ({ domain: s.domain, url: s.url, cited: s.cited, timesCited: s.timesCited, siteType: s.siteType, title: s.title, round: s.round ?? "" }));
916
- const domainRows = enriched.aggregates.topSites.map((d) => ({ domain: d.domain, count: d.count, cited: d.cited, timesCited: d.timesCited, siteType: d.siteType }));
917
- const paths = {
918
- dir,
919
- json: (0, import_node_path3.join)(dir, "fanout.json"),
920
- queriesCsv: (0, import_node_path3.join)(dir, "queries.csv"),
921
- queriesTsv: (0, import_node_path3.join)(dir, "queries.tsv"),
922
- citationsCsv: (0, import_node_path3.join)(dir, "citations.csv"),
923
- sourcesCsv: (0, import_node_path3.join)(dir, "sources.csv"),
924
- domainsCsv: (0, import_node_path3.join)(dir, "domains.csv"),
925
- report: (0, import_node_path3.join)(dir, "report.html")
926
- };
927
- (0, import_node_fs2.writeFileSync)(paths.json, JSON.stringify(enriched, null, 2));
928
- writeTable(paths.queriesCsv, queryRows, ",");
929
- writeTable(paths.queriesTsv, queryRows, " ");
930
- writeTable(paths.citationsCsv, citationRows, ",");
931
- writeTable(paths.sourcesCsv, sourceRows, ",");
932
- writeTable(paths.domainsCsv, domainRows, ",");
933
- (0, import_node_fs2.writeFileSync)(paths.report, renderReportHtml(enriched));
934
- return paths;
935
- }
936
- function esc(s) {
937
- return String(s).replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;");
938
- }
939
- function renderReportHtml(enriched) {
940
- const data = JSON.stringify(enriched).replace(/</g, "\\u003c");
941
- return `<!doctype html>
942
- <html lang="en">
943
- <head>
944
- <meta charset="utf-8" />
945
- <meta name="viewport" content="width=device-width, initial-scale=1" />
946
- <title>AI Search Fan-Out \u2014 ${esc(enriched.platform)}</title>
947
- <style>
948
- :root{--bg:#0b0d12;--panel:#151922;--line:#252b38;--ink:#e8ecf3;--mut:#8b95a7;--acc:#5b9dff;--cite:#39d98a;--browse:#5a6677}
949
- *{box-sizing:border-box}body{margin:0;background:var(--bg);color:var(--ink);font:14px/1.5 -apple-system,BlinkMacSystemFont,Segoe UI,Roboto,sans-serif}
950
- .wrap{max-width:980px;margin:0 auto;padding:28px 20px 80px}
951
- h1{font-size:20px;margin:0 0 2px}.sub{color:var(--mut);margin:0 0 20px;font-size:13px}
952
- .stat{font-size:30px;font-weight:700}.stat small{font-size:13px;font-weight:400;color:var(--mut)}
953
- .cards{display:grid;grid-template-columns:repeat(4,1fr);gap:10px;margin:18px 0}
954
- .card{background:var(--panel);border:1px solid var(--line);border-radius:10px;padding:12px 14px}
955
- .card b{font-size:22px;display:block}.card span{color:var(--mut);font-size:12px}
956
- .bars{margin:8px 0 22px}.bar{display:flex;align-items:center;gap:8px;margin:4px 0}
957
- .bar .lab{width:150px;color:var(--mut);font-size:12px;text-align:right}.bar .track{flex:1;background:#10141c;border-radius:5px;overflow:hidden}
958
- .bar .fill{background:var(--acc);height:16px}.bar .n{width:34px;font-size:12px;color:var(--mut)}
959
- .tabs{display:flex;gap:8px;margin:22px 0 12px}.tab{background:var(--panel);border:1px solid var(--line);color:var(--ink);padding:7px 14px;border-radius:8px;cursor:pointer}
960
- .tab.on{background:var(--acc);border-color:var(--acc);color:#04101f;font-weight:600}
961
- .chips{display:flex;flex-wrap:wrap;gap:6px;margin-bottom:12px}
962
- .chip{background:var(--panel);border:1px solid var(--line);color:var(--mut);padding:4px 10px;border-radius:20px;cursor:pointer;font-size:12px}
963
- .chip.on{border-color:var(--acc);color:var(--ink)}.chip b{color:var(--ink)}
964
- .row{background:var(--panel);border:1px solid var(--line);border-radius:8px;padding:10px 12px;margin:6px 0;display:flex;gap:10px;align-items:flex-start}
965
- .row .idx{color:var(--mut);min-width:22px}.row .main{flex:1}
966
- .tag{font-size:11px;padding:2px 7px;border-radius:6px;border:1px solid var(--line);color:var(--mut);margin-right:5px}
967
- .dom{font-weight:600}.url{color:var(--mut);font-size:12px;word-break:break-all}
968
- .pill{font-size:11px;padding:2px 7px;border-radius:6px}.pill.cite{background:rgba(57,217,138,.15);color:var(--cite)}.pill.brow{background:rgba(90,102,119,.2);color:#aeb8c8}
969
- .copy{background:none;border:1px solid var(--line);color:var(--mut);border-radius:6px;cursor:pointer;font-size:11px;padding:2px 8px}
970
- .hide{display:none}
971
- </style>
972
- </head>
973
- <body><div class="wrap">
974
- <h1>AI Search Fan-Out</h1>
975
- <p class="sub" id="sub"></p>
976
- <div class="stat" id="stat"></div>
977
- <div class="cards" id="cards"></div>
978
- <div id="charts"></div>
979
- <div class="tabs"><button class="tab on" data-tab="q">Queries</button><button class="tab" data-tab="u">URLs</button></div>
980
- <div class="chips" id="chips"></div>
981
- <div id="list"></div>
982
- </div>
983
- <script>
984
- const D=${data};
985
- const el=(t,c,h)=>{const e=document.createElement(t);if(c)e.className=c;if(h!=null)e.innerHTML=h;return e};
986
- document.getElementById('sub').textContent=D.platform+' \xB7 '+(D.meta.model||'model n/a')+' \xB7 '+D.meta.rounds+' search round(s) \xB7 prompt: '+(D.prompt||'');
987
- document.getElementById('stat').innerHTML='Total Fan-Out: '+D.counts.subQueries+' <small>sub-queries \xB7 '+D.counts.browsed+' URLs researched \xB7 '+D.counts.cited+' cited</small>';
988
- const cards=[['Sub-queries',D.counts.subQueries],['Researched',D.counts.browsed],['Cited',D.counts.cited],['Browsed-only',D.counts.browsedOnly]];
989
- const cc=document.getElementById('cards');cards.forEach(([k,v])=>{const c=el('div','card');c.appendChild(el('b',null,v));c.appendChild(el('span',null,k));cc.appendChild(c)});
990
- function barChart(title,obj){const max=Math.max(1,...Object.values(obj));const box=el('div','bars');box.appendChild(el('div',null,'<b>'+title+'</b>'));Object.entries(obj).forEach(([k,v])=>{const r=el('div','bar');r.appendChild(el('div','lab',k));const t=el('div','track');const f=el('div','fill');f.style.width=(v/max*100)+'%';t.appendChild(f);r.appendChild(t);r.appendChild(el('div','n',v));box.appendChild(r)});return box}
991
- const charts=document.getElementById('charts');
992
- charts.appendChild(barChart('URL Categories',D.aggregates.byCategory));
993
- let tab='q',filter=null;
994
- const chips=document.getElementById('chips'),list=document.getElementById('list');
995
- function render(){
996
- chips.innerHTML='';list.innerHTML='';
997
- if(tab==='q'){
998
- D.queries.forEach((q,i)=>{const r=el('div','row');r.appendChild(el('div','idx',i+1));const m=el('div','main');m.appendChild(el('div',null,esc(q)));r.appendChild(m);const b=el('button','copy','copy');b.onclick=()=>navigator.clipboard.writeText(q);r.appendChild(b);list.appendChild(r)});
999
- } else {
1000
- const counts={};D.browsedUrls.forEach(s=>{counts[s.siteType]=(counts[s.siteType]||0)+1});
1001
- Object.entries(counts).forEach(([k,v])=>{const c=el('button','chip'+(filter===k?' on':''),k+' <b>'+v+'</b>');c.onclick=()=>{filter=filter===k?null:k;render()};chips.appendChild(c)});
1002
- D.browsedUrls.forEach(s=>{if(filter&&s.siteType!==filter)return;const r=el('div','row');const m=el('div','main');m.appendChild(el('div',null,'<span class="dom">'+esc(s.domain)+'</span> <span class="pill '+(s.cited?'cite':'brow')+'">'+(s.cited?'cited '+s.timesCited+'\xD7':'browsed')+'</span> <span class="tag">'+s.siteType+'</span>'));m.appendChild(el('div','url',esc(s.url)));r.appendChild(m);list.appendChild(r)});
1003
- }
1004
- }
1005
- function esc(s){return String(s).replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;')}
1006
- document.querySelectorAll('.tab').forEach(t=>t.onclick=()=>{document.querySelectorAll('.tab').forEach(x=>x.classList.remove('on'));t.classList.add('on');tab=t.dataset.tab;filter=null;render()});
1007
- render();
1008
- </script>
1009
- </body></html>`;
1010
- }
1011
-
1012
- // src/services/fanout/run-capture.ts
1013
- var COMPOSER_SELECTORS = [
1014
- "#prompt-textarea",
1015
- "div#prompt-textarea",
1016
- 'textarea[data-testid="prompt-textarea"]',
1017
- 'div.ProseMirror[contenteditable="true"]',
1018
- 'div[contenteditable="true"]',
1019
- "textarea"
1020
- ];
1021
- async function sendPrompt(page, text) {
1022
- for (const selector of COMPOSER_SELECTORS) {
1023
- const locator = page.locator(selector).first();
1024
- try {
1025
- await locator.waitFor({ state: "visible", timeout: 12e3 });
1026
- } catch {
1027
- continue;
1028
- }
1029
- await locator.click({ timeout: 4e3 }).catch(() => void 0);
1030
- await locator.focus().catch(() => void 0);
1031
- await page.keyboard.type(text, { delay: 12 });
1032
- await page.waitForTimeout(300).catch(() => void 0);
1033
- await page.keyboard.press("Enter");
1034
- return true;
1035
- }
1036
- return false;
1037
- }
1038
- async function waitForAnswer(page, cap, baseline, waitMs) {
1039
- const start = Date.now();
1040
- while (Date.now() - start < waitMs && cap.answerStreamCount() <= baseline) {
1041
- await page.waitForTimeout(400).catch(() => void 0);
1042
- }
1043
- await page.waitForTimeout(1e3).catch(() => void 0);
1044
- }
1045
- async function runFanoutCapture(page, input) {
1046
- const cap = new FanoutCdpCapture();
1047
- await cap.attach(page);
1048
- try {
1049
- const baseline = cap.answerStreamCount();
1050
- if (input.prompt) await sendPrompt(page, input.prompt).catch(() => false);
1051
- const waitMs = input.wait_ms ?? (input.prompt ? 9e4 : 8e3);
1052
- await waitForAnswer(page, cap, baseline, waitMs);
1053
- await cap.drain();
1054
- let hostname = "";
1055
- try {
1056
- hostname = new URL(page.url()).hostname;
1057
- } catch {
1058
- hostname = "";
1059
- }
1060
- const adapter = adapterForHost(hostname);
1061
- if (!adapter) {
1062
- throw new Error(`browser_capture_fanout supports chatgpt.com and claude.ai only; the session is on ${hostname || "an unknown page"}. Navigate there first with browser_goto.`);
1063
- }
1064
- const text = cap.bestAnswerText();
1065
- const parsed = adapter.parse(text, input.prompt || "");
1066
- const result = buildFanoutResult({ platform: adapter.platform, parsed, prompt: input.prompt || "", ready: true, rawBytes: text.length });
1067
- const enriched = enrichFanout(result, { firstPartyDomain: input.first_party_domain });
1068
- const exports2 = input.export ? exportFanout(enriched) : null;
1069
- return { result: enriched, exports: exports2 };
1070
- } finally {
1071
- await cap.detach().catch(() => void 0);
1072
- }
1073
- }
1074
-
1075
- // src/services/browser-agent/local-browser-agent-service.ts
1076
- import_playwright_extra.chromium.use((0, import_puppeteer_extra_plugin_stealth.default)());
1077
- function directProfileDir() {
1078
- return process.env.MCP_SCRAPER_BROWSER_PROFILE_DIR?.trim() || void 0;
1079
- }
1080
- function configuredProfileName(profile) {
1081
- return profile?.trim() || process.env.MCP_SCRAPER_BROWSER_PROFILE?.trim() || (directProfileDir() ? (0, import_node_path4.basename)(directProfileDir()) : "default");
1082
- }
1083
- function launchExecutablePath(profile) {
1084
- const configured = process.env.MCP_SCRAPER_BROWSER_EXECUTABLE?.trim() || profile.browserExecutablePath || defaultChromeExecutablePath();
1085
- return configured && (0, import_node_fs3.existsSync)(configured) ? configured : void 0;
1086
- }
1087
- function ignoreDefaultArgsFor(executablePath) {
1088
- if (!executablePath) return void 0;
1089
- if (process.platform !== "darwin") return void 0;
1090
- if (!/Google Chrome\.app\/Contents\/MacOS\/Google Chrome$/.test(executablePath)) return void 0;
1091
- return ["--password-store=basic", "--use-mock-keychain"];
1092
- }
1093
- function normalizeKeyCombo(value) {
1094
- return value.split("+").map((part) => {
1095
- const trimmed = part.trim();
1096
- const lower = trimmed.toLowerCase();
1097
- if (lower === "ctrl" || lower === "control") return "Control";
1098
- if (lower === "cmd" || lower === "command" || lower === "meta") return "Meta";
1099
- if (lower === "option" || lower === "alt") return "Alt";
1100
- if (lower === "return") return "Enter";
1101
- if (lower === "esc") return "Escape";
1102
- if (trimmed.length === 1) return trimmed.toUpperCase();
1103
- return trimmed;
1104
- }).join("+");
1105
- }
1106
- function sessionSummary(session) {
1107
- return {
1108
- session_id: session.sessionId,
1109
- status: session.status,
1110
- label: session.label,
1111
- profile: session.profileName,
1112
- user_data_dir: session.userDataDir,
1113
- created_at: session.createdAt,
1114
- last_action_at: session.lastActionAt,
1115
- closed_at: session.closedAt,
1116
- watch_url: `local://browser-session/${session.sessionId}`,
1117
- url: session.status === "open" ? session.page.url() : null
1118
- };
1119
- }
1120
- var LocalBrowserAgentService = class {
1121
- contexts = /* @__PURE__ */ new Map();
1122
- sessions = /* @__PURE__ */ new Map();
1123
- async open(input) {
1124
- const profile = await this.resolveProfile(input.profile);
1125
- const contextState = await this.ensureContext(profile);
1126
- const pages = contextState.context.pages();
1127
- const page = pages.length === 1 && !this.hasOpenSessionForContext(contextState.key) ? pages[0] : await contextState.context.newPage();
1128
- const now = (/* @__PURE__ */ new Date()).toISOString();
1129
- const sessionId = `lbs_${(0, import_node_crypto.randomUUID)().replace(/-/g, "").slice(0, 20)}`;
1130
- const session = {
1131
- sessionId,
1132
- label: input.label ?? null,
1133
- page,
1134
- contextKey: contextState.key,
1135
- profileName: profile.name,
1136
- userDataDir: profile.userDataDir,
1137
- createdAt: now,
1138
- lastActionAt: now,
1139
- closedAt: null,
1140
- status: "open"
1141
- };
1142
- const timeoutSeconds = input.timeout_seconds ?? 600;
1143
- if (timeoutSeconds > 0) {
1144
- session.timeout = setTimeout(() => {
1145
- void this.close(sessionId);
1146
- }, timeoutSeconds * 1e3);
1147
- session.timeout.unref();
1148
- }
1149
- this.sessions.set(sessionId, session);
1150
- if (input.url) await this.goto(sessionId, input.url);
1151
- return {
1152
- session_id: sessionId,
1153
- watch_url: `local://browser-session/${sessionId}`,
1154
- live_view_url: null,
1155
- profile: profile.name,
1156
- user_data_dir: profile.userDataDir,
1157
- url: input.url ?? null,
1158
- mode: "local"
1159
- };
1160
- }
1161
- async screenshot(sessionId) {
1162
- const session = this.requireOpenSession(sessionId);
1163
- await this.waitForPage(session.page);
1164
- const image = await session.page.screenshot({ type: "png", fullPage: false });
1165
- const page = await this.read(sessionId);
1166
- return {
1167
- ...page,
1168
- image_base64: image.toString("base64"),
1169
- mime_type: "image/png"
1170
- };
1171
- }
1172
- async read(sessionId) {
1173
- const session = this.requireOpenSession(sessionId);
1174
- await this.waitForPage(session.page);
1175
- this.touch(session);
1176
- const title = await session.page.title().catch(() => "");
1177
- const snapshot = await session.page.evaluate(() => {
1178
- const cssEscape = (value) => {
1179
- const escapeFn = globalThis.CSS?.escape;
1180
- return escapeFn ? escapeFn(value) : value.replace(/["\\#.:>+~[\]()]/g, "\\$&");
1181
- };
1182
- const textFor = (element) => {
1183
- const html = element;
1184
- return (element.getAttribute("aria-label") || element.getAttribute("title") || html.placeholder || html.value || html.innerText || element.textContent || "").replace(/\s+/g, " ").trim();
1185
- };
1186
- const selectorFor = (element) => {
1187
- const id = element.id;
1188
- if (id) return `#${cssEscape(id)}`;
1189
- const name = element.getAttribute("name");
1190
- const tag = element.tagName.toLowerCase();
1191
- if (name) return `${tag}[name="${name.replace(/"/g, '\\"')}"]`;
1192
- const parts = [];
1193
- let current = element;
1194
- while (current && current !== document.body && parts.length < 4) {
1195
- const currentTag = current.tagName.toLowerCase();
1196
- const parent = current.parentElement;
1197
- if (!parent) {
1198
- parts.unshift(currentTag);
1199
- break;
1200
- }
1201
- const children = Array.from(parent.children);
1202
- const siblings = children.filter((child) => child.tagName === current.tagName);
1203
- const index = siblings.indexOf(current) + 1;
1204
- parts.unshift(siblings.length > 1 ? `${currentTag}:nth-of-type(${index})` : currentTag);
1205
- current = parent;
1206
- }
1207
- return parts.join(" > ") || tag;
1208
- };
1209
- const candidates = Array.from(document.querySelectorAll(
1210
- 'a, button, input, textarea, select, summary, [role], [contenteditable="true"], [tabindex]:not([tabindex="-1"])'
1211
- ));
1212
- const elements = candidates.map((element, index) => {
1213
- const rect = element.getBoundingClientRect();
1214
- const style = window.getComputedStyle(element);
1215
- const visible = rect.width > 0 && rect.height > 0 && style.display !== "none" && style.visibility !== "hidden" && Number(style.opacity || "1") > 0;
1216
- if (!visible) return null;
1217
- const label = textFor(element);
1218
- return {
1219
- index,
1220
- tag: element.tagName.toLowerCase(),
1221
- role: element.getAttribute("role") || null,
1222
- text: label.slice(0, 180),
1223
- selector: selectorFor(element),
1224
- x: Math.round(rect.left + rect.width / 2),
1225
- y: Math.round(rect.top + rect.height / 2),
1226
- left: Math.round(rect.left),
1227
- top: Math.round(rect.top),
1228
- width: Math.round(rect.width),
1229
- height: Math.round(rect.height)
1230
- };
1231
- }).filter(Boolean).slice(0, 120);
1232
- return {
1233
- text: (document.body?.innerText || "").replace(/\n{3,}/g, "\n\n").slice(0, 3e4),
1234
- elements,
1235
- viewport: {
1236
- width: window.innerWidth,
1237
- height: window.innerHeight,
1238
- devicePixelRatio: window.devicePixelRatio,
1239
- scrollX: window.scrollX,
1240
- scrollY: window.scrollY
1241
- }
1242
- };
1243
- });
1244
- return {
1245
- url: session.page.url(),
1246
- title,
1247
- text: snapshot.text,
1248
- elements: snapshot.elements,
1249
- viewport: snapshot.viewport
1250
- };
1251
- }
1252
- async locate(sessionId, targets) {
1253
- const session = this.requireOpenSession(sessionId);
1254
- this.touch(session);
1255
- const located = [];
1256
- for (const target of targets) {
1257
- try {
1258
- const locator = target.selector ? session.page.locator(target.selector) : session.page.getByText(target.text ?? "", { exact: target.match === "exact" });
1259
- const selected = locator.nth(target.index ?? 0);
1260
- await selected.waitFor({ state: "visible", timeout: 3e3 }).catch(() => void 0);
1261
- const box = await selected.boundingBox();
1262
- const text = await selected.textContent({ timeout: 1e3 }).catch(() => null);
1263
- located.push({
1264
- input: target,
1265
- found: Boolean(box),
1266
- element: box ? {
1267
- left: Math.round(box.x),
1268
- top: Math.round(box.y),
1269
- width: Math.round(box.width),
1270
- height: Math.round(box.height),
1271
- x: Math.round(box.x + box.width / 2),
1272
- y: Math.round(box.y + box.height / 2),
1273
- text: (text ?? "").replace(/\s+/g, " ").trim().slice(0, 220)
1274
- } : null
1275
- });
1276
- } catch (err) {
1277
- located.push({
1278
- input: target,
1279
- found: false,
1280
- element: null,
1281
- error: err instanceof Error ? err.message : String(err)
1282
- });
1283
- }
1284
- }
1285
- const read = await this.read(sessionId);
1286
- return {
1287
- url: read.url,
1288
- title: read.title,
1289
- viewport: read.viewport,
1290
- replay: null,
1291
- targets: located
1292
- };
1293
- }
1294
- async goto(sessionId, url) {
1295
- const session = this.requireOpenSession(sessionId);
1296
- this.touch(session);
1297
- const response = await session.page.goto(url, { waitUntil: "domcontentloaded", timeout: 45e3 }).catch((err) => {
1298
- throw new Error(err instanceof Error ? err.message : String(err));
1299
- });
1300
- await this.waitForPage(session.page);
1301
- return {
1302
- ok: true,
1303
- url: session.page.url(),
1304
- status: response?.status() ?? null,
1305
- mode: "local"
1306
- };
1307
- }
1308
- async click(sessionId, input) {
1309
- const session = this.requireOpenSession(sessionId);
1310
- this.touch(session);
1311
- await session.page.mouse.click(input.x, input.y, {
1312
- button: input.button ?? "left",
1313
- clickCount: input.num_clicks ?? 1
1314
- });
1315
- await this.waitForPage(session.page);
1316
- return { ok: true, mode: "local", url: session.page.url() };
1317
- }
1318
- async type(sessionId, input) {
1319
- const session = this.requireOpenSession(sessionId);
1320
- this.touch(session);
1321
- await session.page.keyboard.type(input.text, { delay: input.delay });
1322
- return { ok: true, mode: "local", url: session.page.url() };
1323
- }
1324
- async scroll(sessionId, input) {
1325
- const session = this.requireOpenSession(sessionId);
1326
- this.touch(session);
1327
- if (typeof input.x === "number" && typeof input.y === "number") {
1328
- await session.page.mouse.move(input.x, input.y);
1329
- }
1330
- await session.page.mouse.wheel((input.delta_x ?? 0) * 120, (input.delta_y ?? 5) * 120);
1331
- await session.page.waitForTimeout(150).catch(() => void 0);
1332
- return { ok: true, mode: "local", url: session.page.url() };
1333
- }
1334
- async press(sessionId, input) {
1335
- const session = this.requireOpenSession(sessionId);
1336
- this.touch(session);
1337
- for (const key of input.keys) {
1338
- await session.page.keyboard.press(normalizeKeyCombo(key));
1339
- }
1340
- await this.waitForPage(session.page);
1341
- return { ok: true, mode: "local", url: session.page.url() };
1342
- }
1343
- async close(sessionId) {
1344
- const session = this.sessions.get(sessionId);
1345
- if (!session || session.status === "closed") {
1346
- return { ok: true, closed: true, already_closed: true, mode: "local" };
1347
- }
1348
- if (session.timeout) clearTimeout(session.timeout);
1349
- session.status = "closed";
1350
- session.closedAt = (/* @__PURE__ */ new Date()).toISOString();
1351
- await session.page.close().catch(() => void 0);
1352
- await this.closeContextIfUnused(session.contextKey);
1353
- return { ok: true, closed: true, mode: "local" };
1354
- }
1355
- list(includeClosed = false) {
1356
- return Array.from(this.sessions.values()).filter((session) => includeClosed || session.status === "open").map(sessionSummary);
1357
- }
1358
- async captureFanout(sessionId, input) {
1359
- const session = this.requireOpenSession(sessionId);
1360
- this.touch(session);
1361
- return runFanoutCapture(session.page, input);
1362
- }
1363
- async screenshotOnly(sessionId) {
1364
- const session = this.requireOpenSession(sessionId);
1365
- await this.waitForPage(session.page);
1366
- const img = await session.page.screenshot({ type: "png", fullPage: false });
1367
- return img.toString("base64");
1368
- }
1369
- async resolveProfile(profile) {
1370
- const directDir = directProfileDir();
1371
- const requestedName = configuredProfileName(profile);
1372
- if (directDir && (!profile || requestedName === process.env.MCP_SCRAPER_BROWSER_PROFILE?.trim())) {
1373
- return {
1374
- name: requestedName,
1375
- userDataDir: directDir,
1376
- browserExecutablePath: process.env.MCP_SCRAPER_BROWSER_EXECUTABLE?.trim() || defaultChromeExecutablePath() || null,
1377
- manifest: null
1378
- };
1379
- }
1380
- const manifest = await loadImportedBrowserProfile(requestedName);
1381
- return {
1382
- name: manifest.name,
1383
- userDataDir: manifest.userDataDir,
1384
- browserExecutablePath: manifest.browserExecutablePath,
1385
- manifest
1386
- };
1387
- }
1388
- async ensureContext(profile) {
1389
- const existing = this.contexts.get(profile.userDataDir);
1390
- if (existing) return existing;
1391
- const executablePath = launchExecutablePath(profile);
1392
- const context = await import_playwright_extra.chromium.launchPersistentContext(profile.userDataDir, {
1393
- headless: false,
1394
- viewport: { width: 1440, height: 900 },
1395
- acceptDownloads: true,
1396
- ...executablePath ? { executablePath } : {},
1397
- ...ignoreDefaultArgsFor(executablePath) ? { ignoreDefaultArgs: ignoreDefaultArgsFor(executablePath) } : {}
1398
- });
1399
- const state = { key: profile.userDataDir, context, profile };
1400
- context.on("close", () => {
1401
- this.contexts.delete(profile.userDataDir);
1402
- const closedAt = (/* @__PURE__ */ new Date()).toISOString();
1403
- for (const session of this.sessions.values()) {
1404
- if (session.contextKey === profile.userDataDir && session.status === "open") {
1405
- session.status = "closed";
1406
- session.closedAt = closedAt;
1407
- if (session.timeout) clearTimeout(session.timeout);
1408
- }
1409
- }
1410
- });
1411
- this.contexts.set(profile.userDataDir, state);
1412
- return state;
1413
- }
1414
- requireOpenSession(sessionId) {
1415
- const session = this.sessions.get(sessionId);
1416
- if (!session || session.status !== "open") throw new Error(`Local browser session not found or closed: ${sessionId}`);
1417
- return session;
1418
- }
1419
- touch(session) {
1420
- session.lastActionAt = (/* @__PURE__ */ new Date()).toISOString();
1421
- }
1422
- hasOpenSessionForContext(contextKey) {
1423
- return Array.from(this.sessions.values()).some((session) => session.contextKey === contextKey && session.status === "open");
1424
- }
1425
- async closeContextIfUnused(contextKey) {
1426
- if (this.hasOpenSessionForContext(contextKey)) return;
1427
- const state = this.contexts.get(contextKey);
1428
- if (!state) return;
1429
- this.contexts.delete(contextKey);
1430
- await state.context.close().catch(() => void 0);
1431
- }
1432
- async waitForPage(page) {
1433
- await page.waitForLoadState("domcontentloaded", { timeout: 5e3 }).catch(() => void 0);
1434
- await page.waitForTimeout(200).catch(() => void 0);
1435
- }
1436
- };
1437
-
1438
29
  // src/version.ts
1439
- var PACKAGE_VERSION = "0.3.5";
30
+ var PACKAGE_VERSION = "0.3.7";
1440
31
 
1441
32
  // src/mcp/browser-agent-tool-schemas.ts
1442
33
  var import_zod = require("zod");
@@ -1459,32 +50,13 @@ var BrowserOpenInputSchema = {
1459
50
  save_profile_changes: import_zod.z.boolean().optional().describe("Persist cookies and local storage back to the named profile when the session is closed. Use this for profile setup or intentional auth refreshes; avoid parallel sessions writing to the same profile."),
1460
51
  timeout_seconds: import_zod.z.number().int().min(60).max(259200).optional().describe("How long the direct no-proxy browser session may live before auto-termination. Defaults to 600. The browser idles into a zero-cost standby between actions, so a longer timeout is cheap.")
1461
52
  };
1462
- var BrowserProfileListInputSchema = {
1463
- email: import_zod.z.string().optional().describe("Optional Chrome account email or displayed profile name to filter for, for example seo@example.com."),
1464
- user_data_dir: import_zod.z.string().optional().describe("Optional Chrome user data directory. Defaults to the current OS Chrome profile root.")
1465
- };
1466
53
  var BrowserProfileOnboardInputSchema = {
1467
54
  email: import_zod.z.string().optional().describe("Optional account email used to derive a stable MCP Scraper profile name. This does not import cookies or browser state."),
1468
55
  profile: import_zod.z.string().optional().describe("Optional saved MCP Scraper hosted browser profile name to create or load. If omitted, email is used to derive one."),
1469
- user_data_dir: import_zod.z.string().optional().describe("Optional Chrome metadata directory for deriving a profile name only. This is not a local browser setup path."),
1470
56
  url: import_zod.z.string().url().optional().describe("Setup URL to open after creating the direct no-proxy browser. Defaults to https://google.com/ so the user can click Sign in."),
1471
57
  label: import_zod.z.string().optional().describe("Optional human label for this setup session."),
1472
58
  timeout_seconds: import_zod.z.number().int().min(60).max(259200).optional().describe("How long the direct no-proxy setup session may live before auto-termination. Defaults to 600.")
1473
59
  };
1474
- var BrowserProfileImportInputSchema = {
1475
- email: import_zod.z.string().optional().describe("Optional Chrome account email or displayed profile name to select from Chrome Local State."),
1476
- name: import_zod.z.string().optional().describe("Name for the managed MCP Scraper browser profile. Defaults to a slug derived from the selected Chrome account."),
1477
- chrome_profile_directory: import_zod.z.string().optional().describe("Optional Chrome profile directory to import, for example Default or Profile 1."),
1478
- user_data_dir: import_zod.z.string().optional().describe("Optional source Chrome user data directory. Defaults to the current OS Chrome profile root."),
1479
- output_dir: import_zod.z.string().optional().describe("Optional managed profile base directory. Defaults to ~/.mcp-scraper/browser-profiles."),
1480
- browser_executable_path: import_zod.z.string().optional().describe("Optional Chrome executable path for local browser launches. Defaults to the installed Google Chrome path when found."),
1481
- overwrite: import_zod.z.boolean().default(false).describe("Overwrite an existing managed profile clone with the same name.")
1482
- };
1483
- var BrowserProfileSyncInputSchema = {
1484
- name: import_zod.z.string().describe("Managed MCP Scraper browser profile name to refresh from its recorded source Chrome profile."),
1485
- output_dir: import_zod.z.string().optional().describe("Optional managed profile base directory. Defaults to ~/.mcp-scraper/browser-profiles."),
1486
- browser_executable_path: import_zod.z.string().optional().describe("Optional Chrome executable path to store in the refreshed profile manifest.")
1487
- };
1488
60
  var BrowserSessionInputSchema = {
1489
61
  session_id: import_zod.z.string().describe("The session id returned by browser_open or browser_list_sessions. Use only a returned session_id; do not construct one yourself.")
1490
62
  };
@@ -1653,14 +225,6 @@ var BrowserOpenOutputSchema = {
1653
225
  hint: import_zod.z.string(),
1654
226
  raw: BrowserRawObject.optional()
1655
227
  };
1656
- var BrowserProfileListOutputSchema = {
1657
- ok: import_zod.z.boolean(),
1658
- tool: import_zod.z.literal("browser_profile_list"),
1659
- session_id: import_zod.z.null(),
1660
- profiles: import_zod.z.array(BrowserRawObject),
1661
- count: import_zod.z.number().int().min(0),
1662
- hint: import_zod.z.string()
1663
- };
1664
228
  var BrowserProfileOnboardOutputSchema = {
1665
229
  ok: import_zod.z.boolean(),
1666
230
  tool: import_zod.z.literal("browser_profile_onboard"),
@@ -1669,27 +233,10 @@ var BrowserProfileOnboardOutputSchema = {
1669
233
  live_view_url: NullableString,
1670
234
  profile: import_zod.z.string(),
1671
235
  setup_url: import_zod.z.string(),
1672
- chrome_profile: BrowserRawObject.nullable(),
236
+ account_email: NullableString,
1673
237
  next_steps: import_zod.z.array(import_zod.z.string()),
1674
238
  raw: BrowserRawObject.optional()
1675
239
  };
1676
- var BrowserProfileImportOutputSchema = {
1677
- ok: import_zod.z.boolean(),
1678
- tool: import_zod.z.literal("browser_profile_import"),
1679
- session_id: import_zod.z.null(),
1680
- profile: BrowserRawObject,
1681
- source_profile: BrowserRawObject,
1682
- next_steps: import_zod.z.array(import_zod.z.string()),
1683
- hint: import_zod.z.string()
1684
- };
1685
- var BrowserProfileSyncOutputSchema = {
1686
- ok: import_zod.z.boolean(),
1687
- tool: import_zod.z.literal("browser_profile_sync"),
1688
- session_id: import_zod.z.null(),
1689
- profile: BrowserRawObject,
1690
- source_profile: BrowserRawObject,
1691
- hint: import_zod.z.string()
1692
- };
1693
240
  var BrowserScreenshotOutputSchema = {
1694
241
  ...BrowserBaseOutput,
1695
242
  tool: import_zod.z.literal("browser_screenshot"),
@@ -1790,9 +337,9 @@ var BrowserListSessionsOutputSchema = {
1790
337
 
1791
338
  // src/mcp/replay-annotator.ts
1792
339
  var import_node_child_process = require("child_process");
1793
- var import_promises3 = require("fs/promises");
1794
- var import_node_os4 = require("os");
1795
- var import_node_path5 = require("path");
340
+ var import_promises = require("fs/promises");
341
+ var import_node_os = require("os");
342
+ var import_node_path = require("path");
1796
343
  var import_node_util = require("util");
1797
344
  var execFileAsync = (0, import_node_util.promisify)(import_node_child_process.execFile);
1798
345
  function finiteNumber(value) {
@@ -1802,11 +349,11 @@ function clamp(value, min, max) {
1802
349
  return Math.min(max, Math.max(min, value));
1803
350
  }
1804
351
  function formatAssTime(seconds) {
1805
- const safe2 = Math.max(0, seconds);
1806
- const hours = Math.floor(safe2 / 3600);
1807
- const minutes = Math.floor(safe2 % 3600 / 60);
1808
- const wholeSeconds = Math.floor(safe2 % 60);
1809
- const centiseconds = Math.floor((safe2 - Math.floor(safe2)) * 100);
352
+ const safe = Math.max(0, seconds);
353
+ const hours = Math.floor(safe / 3600);
354
+ const minutes = Math.floor(safe % 3600 / 60);
355
+ const wholeSeconds = Math.floor(safe % 60);
356
+ const centiseconds = Math.floor((safe - Math.floor(safe)) * 100);
1810
357
  return `${hours}:${String(minutes).padStart(2, "0")}:${String(wholeSeconds).padStart(2, "0")}.${String(centiseconds).padStart(2, "0")}`;
1811
358
  }
1812
359
  function cssHexToAssColor(color) {
@@ -2015,10 +562,10 @@ function ffmpegFilterPath(path) {
2015
562
  async function annotateReplayVideo(inputFilePath, outputFilePath, options) {
2016
563
  if (!options.annotations.length) throw new Error("annotations must include at least one item");
2017
564
  const size = await videoSize(inputFilePath);
2018
- const tmp = await (0, import_promises3.mkdtemp)((0, import_node_path5.join)((0, import_node_os4.tmpdir)(), "mcp-scraper-ass-"));
2019
- const assPath = (0, import_node_path5.join)(tmp, "annotations.ass");
565
+ const tmp = await (0, import_promises.mkdtemp)((0, import_node_path.join)((0, import_node_os.tmpdir)(), "mcp-scraper-ass-"));
566
+ const assPath = (0, import_node_path.join)(tmp, "annotations.ass");
2020
567
  try {
2021
- await (0, import_promises3.writeFile)(assPath, buildAssSubtitle(options, size), "utf8");
568
+ await (0, import_promises.writeFile)(assPath, buildAssSubtitle(options, size), "utf8");
2022
569
  await execFileAsync("ffmpeg", [
2023
570
  "-y",
2024
571
  "-i",
@@ -2035,7 +582,7 @@ async function annotateReplayVideo(inputFilePath, outputFilePath, options) {
2035
582
  "copy",
2036
583
  outputFilePath
2037
584
  ], { maxBuffer: 1024 * 1024 * 20 });
2038
- const out = await (0, import_promises3.stat)(outputFilePath);
585
+ const out = await (0, import_promises.stat)(outputFilePath);
2039
586
  return {
2040
587
  filePath: outputFilePath,
2041
588
  bytes: out.size,
@@ -2044,7 +591,7 @@ async function annotateReplayVideo(inputFilePath, outputFilePath, options) {
2044
591
  annotationCount: options.annotations.length
2045
592
  };
2046
593
  } finally {
2047
- await (0, import_promises3.rm)(tmp, { recursive: true, force: true });
594
+ await (0, import_promises.rm)(tmp, { recursive: true, force: true });
2048
595
  }
2049
596
  }
2050
597
 
@@ -2084,8 +631,8 @@ function actionResult(tool, sessionId, ok, data, nextRecommendedTool = "browser_
2084
631
  nextRecommendedTool
2085
632
  });
2086
633
  }
2087
- function outputBaseDir2() {
2088
- return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path6.join)((0, import_node_os5.homedir)(), "Downloads", "mcp-scraper");
634
+ function outputBaseDir() {
635
+ return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path2.join)((0, import_node_os2.homedir)(), "Downloads", "mcp-scraper");
2089
636
  }
2090
637
  function safeFilePart(value) {
2091
638
  return value.replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 120) || "replay";
@@ -2094,7 +641,12 @@ function replayFilePath(sessionId, replayId, filename) {
2094
641
  const requested = filename?.trim();
2095
642
  const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
2096
643
  const name = requested ? safeFilePart(requested).replace(/\.mp4$/i, "") : `${stamp}-${safeFilePart(sessionId)}-${safeFilePart(replayId)}`;
2097
- return (0, import_node_path6.join)(outputBaseDir2(), "browser-replays", `${name}.mp4`);
644
+ return (0, import_node_path2.join)(outputBaseDir(), "browser-replays", `${name}.mp4`);
645
+ }
646
+ function savedProfileNameFromEmail(email) {
647
+ const value = email?.trim().toLowerCase();
648
+ if (!value) return null;
649
+ return value.replace(/[^a-z0-9._-]+/g, "-").replace(/[._-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80) || null;
2098
650
  }
2099
651
  function annotatedReplayFilePath(sessionId, replayId, filename) {
2100
652
  const requested = filename?.trim();
@@ -2130,7 +682,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
2130
682
  const baseUrl2 = opts.baseUrl.replace(/\/$/, "");
2131
683
  const consoleBase = (opts.consoleBaseUrl ?? opts.baseUrl).replace(/\/$/, "");
2132
684
  const timeoutMs = opts.timeoutMs ?? 9e4;
2133
- const localBrowser = new LocalBrowserAgentService();
2134
685
  async function req(method, path, body) {
2135
686
  try {
2136
687
  const res = await fetch(`${baseUrl2}${path}`, {
@@ -2159,8 +710,8 @@ function registerBrowserAgentMcpTools(server2, opts) {
2159
710
  }
2160
711
  const bytes = Buffer.from(await res.arrayBuffer());
2161
712
  const filePath = replayFilePath(sessionId, replayId, filename);
2162
- (0, import_node_fs4.mkdirSync)((0, import_node_path6.join)(outputBaseDir2(), "browser-replays"), { recursive: true });
2163
- (0, import_node_fs4.writeFileSync)(filePath, bytes);
713
+ (0, import_node_fs.mkdirSync)((0, import_node_path2.join)(outputBaseDir(), "browser-replays"), { recursive: true });
714
+ (0, import_node_fs.writeFileSync)(filePath, bytes);
2164
715
  return {
2165
716
  ok: true,
2166
717
  data: {
@@ -2182,131 +733,20 @@ function registerBrowserAgentMcpTools(server2, opts) {
2182
733
  idempotentHint: false,
2183
734
  openWorldHint: true
2184
735
  });
2185
- function selectChromeProfile(matches, email) {
2186
- const wanted = email?.trim().toLowerCase();
2187
- if (!wanted) return matches[0] ?? null;
2188
- return matches.find((profile) => profile.email.toLowerCase() === wanted) ?? matches[0] ?? null;
2189
- }
2190
- function localReplayUnsupported(tool, sessionId = null, replayId = null) {
2191
- return errorResult(tool, {
2192
- error: "Local browser mode does not support replay recording yet. Use hosted mode for browser_replay_* tools.",
2193
- mode: "local"
2194
- }, sessionId, replayId);
2195
- }
2196
- server2.registerTool(
2197
- "browser_profile_list",
2198
- {
2199
- title: "Developer: List Local Chrome Profiles",
2200
- description: "Developer/local-machine helper only. List local Chrome profile metadata visible to this MCP server: account email, Chrome profile directory, local path, and a suggested managed browser profile name. This reads Chrome Local State only; it does not read cookies, passwords, browsing history, or copy local browser state. Do not use this for customer AI visibility setup or reconnects; use browser_profile_onboard at Google and provide the hosted watch_url instead.",
2201
- inputSchema: BrowserProfileListInputSchema,
2202
- outputSchema: BrowserProfileListOutputSchema,
2203
- annotations: annotations("Developer: List Local Chrome Profiles", true)
2204
- },
2205
- async (input) => {
2206
- try {
2207
- const profiles = await listLocalChromeProfiles(input.user_data_dir, input.email);
2208
- return structuredResult({
2209
- ok: true,
2210
- tool: "browser_profile_list",
2211
- session_id: null,
2212
- profiles,
2213
- count: profiles.length,
2214
- hint: "For customer auth setup, use browser_profile_onboard so the user signs in through the hosted watch_url. Local import is only for developer/local-machine mode."
2215
- });
2216
- } catch (err) {
2217
- return errorResult("browser_profile_list", { error: err instanceof Error ? err.message : String(err) });
2218
- }
2219
- }
2220
- );
2221
- server2.registerTool(
2222
- "browser_profile_import",
2223
- {
2224
- title: "Developer: Import Local Chrome Profile",
2225
- description: "Developer/local-machine helper only. Clone one local Chrome profile into MCP Scraper managed storage for local browser mode. This copies browser state files such as cookies, history, local storage, session storage, and password database metadata into ~/.mcp-scraper/browser-profiles by default; it skips cache and lock files and never prints cookie or password values. Do not use this for customer AI visibility setup or reconnects; use browser_profile_onboard at Google and provide the hosted watch_url instead.",
2226
- inputSchema: BrowserProfileImportInputSchema,
2227
- outputSchema: BrowserProfileImportOutputSchema,
2228
- annotations: { title: "Developer: Import Local Chrome Profile", readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: false }
2229
- },
2230
- async (input) => {
2231
- try {
2232
- const result = await importChromeProfile({
2233
- email: input.email,
2234
- name: input.name,
2235
- chromeProfileDirectory: input.chrome_profile_directory,
2236
- sourceUserDataDir: input.user_data_dir,
2237
- outputDir: input.output_dir,
2238
- overwrite: input.overwrite,
2239
- browserExecutablePath: input.browser_executable_path
2240
- });
2241
- return structuredResult({
2242
- ok: true,
2243
- tool: "browser_profile_import",
2244
- session_id: null,
2245
- profile: result.profile,
2246
- source_profile: result.sourceProfile,
2247
- next_steps: [
2248
- `Set MCP_SCRAPER_BROWSER_MODE=local in the MCP server config.`,
2249
- `Set MCP_SCRAPER_BROWSER_PROFILE=${result.profile.name}.`,
2250
- "Restart the MCP client so it starts the local browser mode process.",
2251
- "Call browser_open with the same MCP server; it will open a local Chrome window using the cloned profile."
2252
- ],
2253
- hint: "Developer-only local clone. For customer auth setup, use browser_profile_onboard at Google, provide the hosted watch_url, then browser_close to save the hosted profile."
2254
- });
2255
- } catch (err) {
2256
- return errorResult("browser_profile_import", { error: err instanceof Error ? err.message : String(err) });
2257
- }
2258
- }
2259
- );
2260
- server2.registerTool(
2261
- "browser_profile_sync",
2262
- {
2263
- title: "Developer: Sync Local Chrome Profile",
2264
- description: "Developer/local-machine helper only. Refresh an existing MCP Scraper managed local browser profile from the source Chrome profile recorded in its manifest. This overwrites the managed clone, copies browser state files again, skips cache and lock files, and never prints cookie or password values. Do not use this for customer AI visibility setup or reconnects; use browser_profile_onboard at Google and provide the hosted watch_url instead.",
2265
- inputSchema: BrowserProfileSyncInputSchema,
2266
- outputSchema: BrowserProfileSyncOutputSchema,
2267
- annotations: { title: "Developer: Sync Local Chrome Profile", readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: false }
2268
- },
2269
- async (input) => {
2270
- try {
2271
- const result = await syncImportedChromeProfile(input.name, {
2272
- outputDir: input.output_dir,
2273
- browserExecutablePath: input.browser_executable_path
2274
- });
2275
- return structuredResult({
2276
- ok: true,
2277
- tool: "browser_profile_sync",
2278
- session_id: null,
2279
- profile: result.profile,
2280
- source_profile: result.sourceProfile,
2281
- hint: `Developer-only local clone refreshed for ${result.profile.name}. Customer reconnects should use browser_profile_onboard with the same hosted profile name.`
2282
- });
2283
- } catch (err) {
2284
- return errorResult("browser_profile_sync", { error: err instanceof Error ? err.message : String(err) });
2285
- }
2286
- }
2287
- );
2288
736
  server2.registerTool(
2289
737
  "browser_profile_onboard",
2290
738
  {
2291
739
  title: "Connect Hosted Browser Profile",
2292
- description: "Create or load a saved MCP Scraper hosted browser profile and open a direct no-proxy setup browser with profile saving enabled. This is the customer auth setup and reconnect flow: open Google, give the user the watch_url, have them click Sign in and complete login, then call browser_close to save cookies and local storage. If a saved profile later shows login controls, run this same setup flow again. Do not use local Chrome import for this customer setup.",
740
+ description: "Create or load a saved MCP Scraper hosted browser profile and open a direct no-proxy setup browser with profile saving enabled. This is the customer auth setup and reconnect flow: open Google, give the user the watch_url, have them click Sign in and complete login, then call browser_close to save cookies and storage. If a saved profile later shows login controls, run this same setup flow again.",
2293
741
  inputSchema: BrowserProfileOnboardInputSchema,
2294
742
  outputSchema: BrowserProfileOnboardOutputSchema,
2295
743
  annotations: annotations("Connect Hosted Browser Profile")
2296
744
  },
2297
745
  async (input) => {
2298
- let chromeProfile = null;
2299
- if (input.email) {
2300
- try {
2301
- chromeProfile = selectChromeProfile(await listLocalChromeProfiles(input.user_data_dir, input.email), input.email);
2302
- } catch (err) {
2303
- return errorResult("browser_profile_onboard", { error: err instanceof Error ? err.message : String(err) });
2304
- }
2305
- }
2306
- const profile = input.profile?.trim() || chromeProfile?.recommendedKernelProfileName || browserServiceProfileName();
746
+ const profile = input.profile?.trim() || savedProfileNameFromEmail(input.email) || browserServiceProfileName();
2307
747
  if (!profile) {
2308
748
  return errorResult("browser_profile_onboard", {
2309
- error: "profile is required when no email match or BROWSER_AGENT_PROFILE_NAME is available"
749
+ error: "profile or email is required when BROWSER_AGENT_PROFILE_NAME is not available"
2310
750
  });
2311
751
  }
2312
752
  const setupUrl = input.url ?? "https://google.com/";
@@ -2329,10 +769,10 @@ function registerBrowserAgentMcpTools(server2, opts) {
2329
769
  live_view_url: session.live_view_url ?? null,
2330
770
  profile,
2331
771
  setup_url: setupUrl,
2332
- chrome_profile: chromeProfile,
772
+ account_email: input.email?.trim() || null,
2333
773
  next_steps: [
2334
774
  "Open the watch_url and let the user click Sign in on Google.",
2335
- "After the user confirms login is complete, call browser_close with this session_id to persist cookies and local storage to the saved profile.",
775
+ "After the user confirms login is complete, call browser_close with this session_id to persist cookies and storage to the saved profile.",
2336
776
  "If the saved profile later shows login controls or disconnects, repeat browser_profile_onboard and close the setup session again.",
2337
777
  "Use BROWSER_AGENT_PROFILE_NAME or browser_open.profile with this profile name for future authenticated sessions."
2338
778
  ],
@@ -2350,23 +790,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
2350
790
  annotations: annotations("Open Browser Session")
2351
791
  },
2352
792
  async (input) => {
2353
- if (localBrowserModeEnabled()) {
2354
- try {
2355
- const session2 = await localBrowser.open(input);
2356
- return structuredResult({
2357
- ok: true,
2358
- tool: "browser_open",
2359
- session_id: session2.session_id,
2360
- watch_url: session2.watch_url,
2361
- live_view_url: null,
2362
- url: input.url ?? null,
2363
- hint: "Local Chrome is open on this machine. Call browser_screenshot to see the page and click by x,y coordinates.",
2364
- raw: session2
2365
- });
2366
- } catch (err) {
2367
- return errorResult("browser_open", { error: err instanceof Error ? err.message : String(err), mode: "local" });
2368
- }
2369
- }
2370
793
  const profile = input.profile ?? browserServiceProfileName();
2371
794
  const saveProfileChanges = input.save_profile_changes ?? browserServiceProfileSaveChanges();
2372
795
  const open = await req("POST", "/agent/sessions", {
@@ -2403,27 +826,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
2403
826
  annotations: annotations("See Page", true)
2404
827
  },
2405
828
  async (input) => {
2406
- if (localBrowserModeEnabled()) {
2407
- try {
2408
- const res2 = await localBrowser.screenshot(input.session_id);
2409
- const content2 = [];
2410
- if (res2.image_base64) content2.push({ type: "image", data: res2.image_base64, mimeType: res2.mime_type });
2411
- const structured2 = {
2412
- ok: true,
2413
- tool: "browser_screenshot",
2414
- session_id: input.session_id,
2415
- url: res2.url ?? null,
2416
- title: res2.title ?? null,
2417
- text: res2.text,
2418
- elements: res2.elements,
2419
- screenshot: { mime_type: res2.mime_type, inline: true }
2420
- };
2421
- content2.push({ type: "text", text: JSON.stringify(structured2) });
2422
- return { content: content2, structuredContent: structured2 };
2423
- } catch (err) {
2424
- return errorResult("browser_screenshot", { error: err instanceof Error ? err.message : String(err), mode: "local" }, input.session_id);
2425
- }
2426
- }
2427
829
  const res = await req("POST", `/agent/sessions/${input.session_id}/screenshot`);
2428
830
  if (!res.ok) return errorResult("browser_screenshot", res.data, input.session_id);
2429
831
  const { image_base64, mime_type, url, title, elements, text } = res.data;
@@ -2456,23 +858,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
2456
858
  annotations: annotations("Read Page", true)
2457
859
  },
2458
860
  async (input) => {
2459
- if (localBrowserModeEnabled()) {
2460
- try {
2461
- const res2 = await localBrowser.read(input.session_id);
2462
- return structuredResult({
2463
- ok: true,
2464
- tool: "browser_read",
2465
- session_id: input.session_id,
2466
- url: res2.url ?? null,
2467
- title: res2.title ?? null,
2468
- text: res2.text,
2469
- elements: res2.elements,
2470
- raw: res2
2471
- });
2472
- } catch (err) {
2473
- return errorResult("browser_read", { error: err instanceof Error ? err.message : String(err), mode: "local" }, input.session_id);
2474
- }
2475
- }
2476
861
  const res = await req("POST", `/agent/sessions/${input.session_id}/read`);
2477
862
  if (!res.ok) return errorResult("browser_read", res.data, input.session_id);
2478
863
  return structuredResult({
@@ -2497,24 +882,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
2497
882
  annotations: annotations("Locate DOM Targets", true)
2498
883
  },
2499
884
  async (input) => {
2500
- if (localBrowserModeEnabled()) {
2501
- try {
2502
- const res2 = await localBrowser.locate(input.session_id, input.targets);
2503
- return structuredResult({
2504
- ok: true,
2505
- tool: "browser_locate",
2506
- session_id: input.session_id,
2507
- url: res2.url ?? null,
2508
- title: res2.title ?? null,
2509
- viewport: res2.viewport ?? null,
2510
- replay: null,
2511
- targets: Array.isArray(res2.targets) ? res2.targets : [],
2512
- raw: res2
2513
- });
2514
- } catch (err) {
2515
- return errorResult("browser_locate", { error: err instanceof Error ? err.message : String(err), mode: "local" }, input.session_id);
2516
- }
2517
- }
2518
885
  const res = await req("POST", `/agent/sessions/${input.session_id}/locate`, { targets: input.targets });
2519
886
  if (!res.ok) return errorResult("browser_locate", res.data, input.session_id);
2520
887
  return structuredResult({
@@ -2540,14 +907,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
2540
907
  annotations: annotations("Navigate To URL")
2541
908
  },
2542
909
  async (input) => {
2543
- if (localBrowserModeEnabled()) {
2544
- try {
2545
- const res2 = await localBrowser.goto(input.session_id, input.url);
2546
- return actionResult("browser_goto", input.session_id, true, res2, "browser_screenshot");
2547
- } catch (err) {
2548
- return errorResult("browser_goto", { error: err instanceof Error ? err.message : String(err), mode: "local" }, input.session_id);
2549
- }
2550
- }
2551
910
  const res = await req("POST", `/agent/sessions/${input.session_id}/goto`, { url: input.url });
2552
911
  return actionResult("browser_goto", input.session_id, res.ok, res.data, "browser_screenshot");
2553
912
  }
@@ -2562,14 +921,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
2562
921
  annotations: annotations("Click")
2563
922
  },
2564
923
  async (input) => {
2565
- if (localBrowserModeEnabled()) {
2566
- try {
2567
- const res2 = await localBrowser.click(input.session_id, input);
2568
- return actionResult("browser_click", input.session_id, true, res2, "browser_screenshot");
2569
- } catch (err) {
2570
- return errorResult("browser_click", { error: err instanceof Error ? err.message : String(err), mode: "local" }, input.session_id);
2571
- }
2572
- }
2573
924
  const res = await req("POST", `/agent/sessions/${input.session_id}/click`, {
2574
925
  x: input.x,
2575
926
  y: input.y,
@@ -2589,14 +940,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
2589
940
  annotations: annotations("Type Text")
2590
941
  },
2591
942
  async (input) => {
2592
- if (localBrowserModeEnabled()) {
2593
- try {
2594
- const res2 = await localBrowser.type(input.session_id, input);
2595
- return actionResult("browser_type", input.session_id, true, res2, "browser_screenshot");
2596
- } catch (err) {
2597
- return errorResult("browser_type", { error: err instanceof Error ? err.message : String(err), mode: "local" }, input.session_id);
2598
- }
2599
- }
2600
943
  const res = await req("POST", `/agent/sessions/${input.session_id}/type`, { text: input.text, delay: input.delay });
2601
944
  return actionResult("browser_type", input.session_id, res.ok, res.data, "browser_screenshot");
2602
945
  }
@@ -2611,14 +954,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
2611
954
  annotations: annotations("Scroll")
2612
955
  },
2613
956
  async (input) => {
2614
- if (localBrowserModeEnabled()) {
2615
- try {
2616
- const res2 = await localBrowser.scroll(input.session_id, input);
2617
- return actionResult("browser_scroll", input.session_id, true, res2, "browser_screenshot");
2618
- } catch (err) {
2619
- return errorResult("browser_scroll", { error: err instanceof Error ? err.message : String(err), mode: "local" }, input.session_id);
2620
- }
2621
- }
2622
957
  const res = await req("POST", `/agent/sessions/${input.session_id}/scroll`, {
2623
958
  delta_y: input.delta_y,
2624
959
  delta_x: input.delta_x,
@@ -2638,14 +973,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
2638
973
  annotations: annotations("Press Keys")
2639
974
  },
2640
975
  async (input) => {
2641
- if (localBrowserModeEnabled()) {
2642
- try {
2643
- const res2 = await localBrowser.press(input.session_id, input);
2644
- return actionResult("browser_press", input.session_id, true, res2, "browser_screenshot");
2645
- } catch (err) {
2646
- return errorResult("browser_press", { error: err instanceof Error ? err.message : String(err), mode: "local" }, input.session_id);
2647
- }
2648
- }
2649
976
  const res = await req("POST", `/agent/sessions/${input.session_id}/press`, { keys: input.keys });
2650
977
  return actionResult("browser_press", input.session_id, res.ok, res.data, "browser_screenshot");
2651
978
  }
@@ -2660,7 +987,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
2660
987
  annotations: annotations("Start Recording")
2661
988
  },
2662
989
  async (input) => {
2663
- if (localBrowserModeEnabled()) return localReplayUnsupported("browser_replay_start", input.session_id);
2664
990
  const res = await req("POST", `/agent/sessions/${input.session_id}/replay/start`);
2665
991
  if (!res.ok) return errorResult("browser_replay_start", res.data, input.session_id);
2666
992
  return structuredResult({
@@ -2684,7 +1010,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
2684
1010
  annotations: annotations("Stop Recording")
2685
1011
  },
2686
1012
  async (input) => {
2687
- if (localBrowserModeEnabled()) return localReplayUnsupported("browser_replay_stop", input.session_id, input.replay_id);
2688
1013
  const res = await req("POST", `/agent/sessions/${input.session_id}/replay/stop`, { replay_id: input.replay_id });
2689
1014
  if (!res.ok) return errorResult("browser_replay_stop", res.data, input.session_id, input.replay_id);
2690
1015
  return structuredResult({
@@ -2708,7 +1033,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
2708
1033
  annotations: annotations("List Replay Videos", true)
2709
1034
  },
2710
1035
  async (input) => {
2711
- if (localBrowserModeEnabled()) return localReplayUnsupported("browser_list_replays", input.session_id);
2712
1036
  const res = await req("GET", `/agent/sessions/${input.session_id}/replays`);
2713
1037
  if (!res.ok) return errorResult("browser_list_replays", res.data, input.session_id);
2714
1038
  const replays = Array.isArray(res.data?.replays) ? res.data.replays : [];
@@ -2731,7 +1055,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
2731
1055
  annotations: annotations("Download Replay MP4")
2732
1056
  },
2733
1057
  async (input) => {
2734
- if (localBrowserModeEnabled()) return localReplayUnsupported("browser_replay_download", input.session_id, input.replay_id);
2735
1058
  const res = await downloadReplay(input.session_id, input.replay_id, input.filename);
2736
1059
  if (!res.ok) return errorResult("browser_replay_download", res.data, input.session_id, input.replay_id);
2737
1060
  return structuredResult({
@@ -2756,7 +1079,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
2756
1079
  annotations: annotations("Mark Replay Annotation", true)
2757
1080
  },
2758
1081
  async (input) => {
2759
- if (localBrowserModeEnabled()) return localReplayUnsupported("browser_replay_mark", input.session_id);
2760
1082
  const res = await req("POST", `/agent/sessions/${input.session_id}/locate`, { targets: [input.target] });
2761
1083
  if (!res.ok) return errorResult("browser_replay_mark", res.data, input.session_id);
2762
1084
  const target = res.data?.targets?.[0];
@@ -2807,14 +1129,13 @@ function registerBrowserAgentMcpTools(server2, opts) {
2807
1129
  annotations: annotations("Annotate Replay MP4")
2808
1130
  },
2809
1131
  async (input) => {
2810
- if (localBrowserModeEnabled()) return localReplayUnsupported("browser_replay_annotate", input.session_id, input.replay_id);
2811
1132
  const sourceName = input.filename ? `${input.filename}-source` : void 0;
2812
1133
  const downloaded = await downloadReplay(input.session_id, input.replay_id, sourceName);
2813
1134
  if (!downloaded.ok) return errorResult("browser_replay_annotate", downloaded.data, input.session_id, input.replay_id);
2814
1135
  try {
2815
1136
  const sourcePath = String(downloaded.data.file_path);
2816
1137
  const outputPath = annotatedReplayFilePath(input.session_id, input.replay_id, input.filename);
2817
- (0, import_node_fs4.mkdirSync)((0, import_node_path6.join)(outputBaseDir2(), "browser-replays"), { recursive: true });
1138
+ (0, import_node_fs.mkdirSync)((0, import_node_path2.join)(outputBaseDir(), "browser-replays"), { recursive: true });
2818
1139
  const result = await annotateReplayVideo(sourcePath, outputPath, {
2819
1140
  annotations: input.annotations,
2820
1141
  sourceWidth: input.source_width,
@@ -2850,20 +1171,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
2850
1171
  annotations: { title: "Close Browser Session", readOnlyHint: false, destructiveHint: true, idempotentHint: true, openWorldHint: false }
2851
1172
  },
2852
1173
  async (input) => {
2853
- if (localBrowserModeEnabled()) {
2854
- try {
2855
- const res2 = await localBrowser.close(input.session_id);
2856
- return structuredResult({
2857
- ok: true,
2858
- tool: "browser_close",
2859
- session_id: input.session_id,
2860
- closed: true,
2861
- raw: res2
2862
- });
2863
- } catch (err) {
2864
- return errorResult("browser_close", { error: err instanceof Error ? err.message : String(err), mode: "local" }, input.session_id);
2865
- }
2866
- }
2867
1174
  const res = await req("DELETE", `/agent/sessions/${input.session_id}`);
2868
1175
  if (!res.ok) return errorResult("browser_close", res.data, input.session_id);
2869
1176
  return structuredResult({
@@ -2885,16 +1192,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
2885
1192
  annotations: annotations("List Browser Sessions", true)
2886
1193
  },
2887
1194
  async (input) => {
2888
- if (localBrowserModeEnabled()) {
2889
- const sessions2 = localBrowser.list(input.include_closed);
2890
- return structuredResult({
2891
- ok: true,
2892
- tool: "browser_list_sessions",
2893
- session_id: null,
2894
- sessions: sessions2,
2895
- count: sessions2.length
2896
- });
2897
- }
2898
1195
  const res = await req("GET", `/agent/sessions${input.include_closed ? "?all=1" : ""}`);
2899
1196
  if (!res.ok) return errorResult("browser_list_sessions", res.data);
2900
1197
  const sessions = (res.data.sessions ?? []).map((s) => ({ ...s, watch_url: `${consoleBase}/console/${s.session_id}` }));
@@ -2936,14 +1233,6 @@ function registerBrowserAgentMcpTools(server2, opts) {
2936
1233
  exports: exports2 ?? null,
2937
1234
  ...result.debug ? { debug: result.debug } : {}
2938
1235
  });
2939
- if (localBrowserModeEnabled()) {
2940
- try {
2941
- const { result, exports: exports2 } = await localBrowser.captureFanout(input.session_id, input);
2942
- return emit(result, exports2);
2943
- } catch (err) {
2944
- return errorResult("browser_capture_fanout", { error: err instanceof Error ? err.message : String(err), mode: "local" }, input.session_id);
2945
- }
2946
- }
2947
1236
  const res = await req("POST", `/agent/sessions/${input.session_id}/capture-fanout`, {
2948
1237
  prompt: input.prompt,
2949
1238
  wait_ms: input.wait_ms,
@@ -2960,10 +1249,10 @@ function registerBrowserAgentMcpTools(server2, opts) {
2960
1249
  // bin/browser-agent-stdio-server.ts
2961
1250
  function readApiKeyFile() {
2962
1251
  const explicitPath = process.env.MCP_SCRAPER_KEY_PATH?.trim();
2963
- const paths = [explicitPath, (0, import_node_path7.join)((0, import_node_os6.homedir)(), ".mcp-scraper-key")].filter(Boolean);
1252
+ const paths = [explicitPath, (0, import_node_path3.join)((0, import_node_os3.homedir)(), ".mcp-scraper-key")].filter(Boolean);
2964
1253
  for (const path of paths) {
2965
1254
  try {
2966
- const value = (0, import_node_fs5.readFileSync)(path, "utf8").trim();
1255
+ const value = (0, import_node_fs2.readFileSync)(path, "utf8").trim();
2967
1256
  if (value) return value;
2968
1257
  } catch {
2969
1258
  }